{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"},{"sourceId":8236972,"sourceType":"datasetVersion","datasetId":4885707},{"sourceId":13213620,"sourceType":"datasetVersion","datasetId":8127880}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Installing required dependencies","metadata":{}},{"cell_type":"code","source":"!pip install duckdb --no-index --find-links=/kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/duck_pkg\n!pip install python-gdcm\n!pip install pylibjpeg\n!pip install pylibjpeg-libjpeg==2.2.0\n!pip install pylibjpeg-openjpeg==2.3.0\n!pip install matplotlib==3.10.3\n!pip install scikit-learn==1.7.0\n!pip install polars --no-index --find-links=/kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/polars_pkg\n!pip install pydicom","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:34:20.883678Z","iopub.execute_input":"2025-10-04T16:34:20.88454Z","iopub.status.idle":"2025-10-04T16:35:19.853898Z","shell.execute_reply.started":"2025-10-04T16:34:20.884507Z","shell.execute_reply":"2025-10-04T16:35:19.852832Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Importing required libraries","metadata":{}},{"cell_type":"code","source":"from pydicom import dcmread\nfrom pydicom.dataset import FileDataset, FileMetaDataset\nfrom pydicom.uid import generate_uid, ImplicitVRLittleEndian\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport polars as pl\nimport duckdb as dd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport cv2\nimport pickle\nimport gc\nimport ctypes\nfrom pathlib import Path\nimport logging\nimport json\nimport multiprocessing as mp\nfrom concurrent.futures import ProcessPoolExecutor\nimport datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\nimport tensorflow as tf\nimport tensorflow_io as tfio\nfrom tensorflow import keras","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:35:24.721577Z","iopub.execute_input":"2025-10-04T16:35:24.721929Z","iopub.status.idle":"2025-10-04T16:35:43.467842Z","shell.execute_reply.started":"2025-10-04T16:35:24.721898Z","shell.execute_reply":"2025-10-04T16:35:43.467103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mp.cpu_count()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:47:28.054367Z","iopub.execute_input":"2025-09-30T04:47:28.054798Z","iopub.status.idle":"2025-09-30T04:47:28.067078Z","shell.execute_reply.started":"2025-09-30T04:47:28.054778Z","shell.execute_reply":"2025-09-30T04:47:28.063688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(tf.__version__)\nprint(tfio.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:47:29.085986Z","iopub.execute_input":"2025-09-30T04:47:29.086216Z","iopub.status.idle":"2025-09-30T04:47:29.094694Z","shell.execute_reply.started":"2025-09-30T04:47:29.086198Z","shell.execute_reply":"2025-09-30T04:47:29.090022Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Initializing the TPU","metadata":{}},{"cell_type":"code","source":"tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='local')\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.TPUStrategy(tpu)\n\nprint(\"Number of accelerators: \", tpu_strategy.num_replicas_in_sync)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:47:30.854979Z","iopub.execute_input":"2025-09-30T04:47:30.855164Z","iopub.status.idle":"2025-09-30T04:47:46.80495Z","shell.execute_reply.started":"2025-09-30T04:47:30.855148Z","shell.execute_reply":"2025-09-30T04:47:46.800036Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Setting polars configs to view the dataframes better","metadata":{}},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=1000)\npl.Config.set_tbl_rows(1000)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:39.352462Z","iopub.execute_input":"2025-09-30T04:50:39.352884Z","iopub.status.idle":"2025-09-30T04:50:39.363222Z","shell.execute_reply.started":"2025-09-30T04:50:39.35286Z","shell.execute_reply":"2025-09-30T04:50:39.361311Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load the metadata of the training images\n## Also separate out the localizer coordinates into individual columns","metadata":{}},{"cell_type":"code","source":"train_meta_data = pl.read_csv('/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv'\\\n                              , low_memory=True)\n\ntrain_locale_meta_data = pl.read_csv('/kaggle/input/rsna-intracranial-aneurysm-detection/train_localizers.csv'\\\n                              , low_memory=True)\n\ndef parse_coordinates(coord_str):\n    if coord_str is None:\n        return None, None\n    try:\n        coord_dict = json.loads(coord_str.replace(\"'\", '\"'))\n        return float(coord_dict.get('x', 0.0)), float(coord_dict.get('y', 0.0)), int(coord_dict.get('f', 0.0))\n    except (json.JSONDecodeError, KeyError, ValueError, AttributeError):\n        return None, None\n\ntrain_locale_meta_data = train_locale_meta_data.with_columns([\n    pl.col(\"coordinates\")\n    .map_elements(lambda x: parse_coordinates(x)[0], return_dtype=pl.Float64)\n    .cast(pl.Float64)\n    .alias(\"coordinates_x\"),\n    \n    pl.col(\"coordinates\")\n    .map_elements(lambda x: parse_coordinates(x)[1], return_dtype=pl.Float64)\n    .cast(pl.Float64)\n    .alias(\"coordinates_y\"),\n    \n    pl.col(\"coordinates\")\n    .map_elements(lambda x: parse_coordinates(x)[2], return_dtype=pl.Int32)\n    .cast(pl.Int32)\n    .alias(\"coordinates_f\")\n])\n\nprint(\"Train CSV shape : \", train_meta_data.shape)\nprint(\"Train Localizers CSV shape : \", train_locale_meta_data.shape)\n# Show the first few rows\nprint(train_locale_meta_data.filter(pl.col('coordinates_f') != 0.0)\\\n      .select([\"coordinates\", \"coordinates_x\", \"coordinates_y\", \"coordinates_f\"]).head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:41.223587Z","iopub.execute_input":"2025-09-30T04:50:41.223766Z","iopub.status.idle":"2025-09-30T04:50:41.392747Z","shell.execute_reply.started":"2025-09-30T04:50:41.22375Z","shell.execute_reply":"2025-09-30T04:50:41.387935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Glancing at the datasets so far","metadata":{}},{"cell_type":"code","source":"train_meta_data.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:45.755579Z","iopub.execute_input":"2025-09-30T04:50:45.755814Z","iopub.status.idle":"2025-09-30T04:50:45.770122Z","shell.execute_reply.started":"2025-09-30T04:50:45.755798Z","shell.execute_reply":"2025-09-30T04:50:45.765265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_locale_meta_data.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:48.701367Z","iopub.execute_input":"2025-09-30T04:50:48.701641Z","iopub.status.idle":"2025-09-30T04:50:48.715101Z","shell.execute_reply.started":"2025-09-30T04:50:48.701622Z","shell.execute_reply":"2025-09-30T04:50:48.710248Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Get summary statistics of the new columns","metadata":{}},{"cell_type":"code","source":"print(train_locale_meta_data.select([\"coordinates_x\", \"coordinates_y\", \"coordinates_f\"]).describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:20:34.592448Z","iopub.execute_input":"2025-09-30T04:20:34.592713Z","iopub.status.idle":"2025-09-30T04:20:34.607031Z","shell.execute_reply.started":"2025-09-30T04:20:34.592696Z","shell.execute_reply":"2025-09-30T04:20:34.60292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Get the image metadata from each training series and create a dataframe out of them","metadata":{}},{"cell_type":"code","source":"allowed_tags = ['BitsAllocated', 'BitsStored', 'Rows', 'Columns', 'FrameOfReferenceUID', 'HighBit', 'ImageOrientationPatient'\n                , 'ImagePositionPatient', 'InstanceNumber', 'Modality', 'PhotometricInterpretation'\n                , 'PixelRepresentation', 'PixelSpacing', 'PlanarConfiguration', 'RescaleIntercept', 'RescaleSlope'\n                , 'RescaleType', 'SamplesPerPixel', 'SliceThickness', 'SpacingBetweenSlices']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:11:59.30872Z","iopub.execute_input":"2025-10-04T17:11:59.309023Z","iopub.status.idle":"2025-10-04T17:11:59.313859Z","shell.execute_reply.started":"2025-10-04T17:11:59.309002Z","shell.execute_reply":"2025-10-04T17:11:59.312793Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Functions to collect metadata without the image arrays\n### We use python slots to reduce the memory foorprint\n### Also use multithreading to speed up processing","metadata":{}},{"cell_type":"code","source":"class DicomRecord:\n    \"\"\"\n    Memory-efficient class for storing DICOM metadata using __slots__\n    \"\"\"\n    __slots__ = ['folder_name', 'file_name', 'file_path', 'image_shape', 'min_max_diff'] + [\n        'BitsAllocated', 'BitsStored', 'Rows', 'Columns', 'FrameOfReferenceUID',\n        'HighBit', 'ImageOrientationPatient', 'ImagePositionPatient', 'InstanceNumber',\n        'Modality', 'PhotometricInterpretation', 'PixelRepresentation', 'PixelSpacing',\n        'PlanarConfiguration', 'RescaleIntercept', 'RescaleSlope', 'RescaleType',\n        'SamplesPerPixel', 'SliceThickness', 'SpacingBetweenSlices'\n    ]\n    \n    def __init__(self, folder_name, file_name, file_path, image_shape, min_max_diff):\n        self.folder_name = folder_name\n        self.file_name = file_name\n        self.file_path = file_path\n        self.image_shape = image_shape\n        self.min_max_diff = min_max_diff\n        for tag in self.__slots__[5:]:  \n            setattr(self, tag, None)\n    \n    def to_dict(self):\n        return {slot: getattr(self, slot) for slot in self.__slots__}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:55.593542Z","iopub.execute_input":"2025-09-30T04:50:55.593783Z","iopub.status.idle":"2025-09-30T04:50:55.604361Z","shell.execute_reply.started":"2025-09-30T04:50:55.593765Z","shell.execute_reply":"2025-09-30T04:50:55.600003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_single_folder(folder_path, allowed_tags):\n    \"\"\"\n    Process a single folder of DICOM files and save image arrays\n    \"\"\"\n    try:\n        data = []\n        dcm_files = list(Path(folder_path).glob(\"*.dcm\"))\n        folder_name = Path(folder_path).name\n        \n        for dcm_file in dcm_files:\n            try:\n                # Read DICOM file\n                ds = dcmread(str(dcm_file))\n                original_shape = str(ds.pixel_array.shape)\n                min_max_diff = round(np.max(ds.pixel_array) - np.min(ds.pixel_array),5)\n                \n                # Create record\n                record = DicomRecord(folder_name, dcm_file.name, str(dcm_file), original_shape, min_max_diff)\n                \n                # Fill in tags\n                for tag in allowed_tags:\n                    try:\n                        value = getattr(ds, tag)\n                        if hasattr(value, '__iter__') and not isinstance(value, str):\n                            value = str(list(map(str, value)))\n                        else:\n                            value = str(value)\n                        setattr(record, tag, value)\n                    except (AttributeError, TypeError):\n                        continue\n                \n                data.append(record.to_dict())\n                \n            except Exception as e:\n                print(f\"Error processing file {dcm_file}: {e}\")\n                continue\n                \n        return data\n        \n    except Exception as e:\n        print(f\"Error processing folder {folder_path}: {e}\")\n        return []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:50:57.506622Z","iopub.execute_input":"2025-09-30T04:50:57.506852Z","iopub.status.idle":"2025-09-30T04:50:57.517525Z","shell.execute_reply.started":"2025-09-30T04:50:57.506835Z","shell.execute_reply":"2025-09-30T04:50:57.513961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_dicom_dataset(root_folder, allowed_tags, num_processes=None, chunk_size=100):\n    \"\"\"\n    Create dataset with metadata DataFrame and memory-mapped image arrays\n    \"\"\"\n    root_path = Path(root_folder)\n    folders = [f for f in root_path.iterdir() if f.is_dir()]\n    \n    if not num_processes:\n        num_processes = mp.cpu_count()\n    \n    # Create directories for temporary and array storage\n    temp_dir = Path(\"temp_chunks\")\n    temp_dir.mkdir(exist_ok=True)\n    \n    # Create schema\n    schema = {\n        'folder_name': pl.Utf8,\n        'file_name': pl.Utf8,\n        'file_path': pl.String,\n        'image_shape': pl.String,\n        'min_max_diff': pl.Float32\n    }\n    schema.update({tag: pl.Utf8 for tag in allowed_tags})\n    \n    # Process folders in parallel\n    with ProcessPoolExecutor(max_workers=num_processes) as executor:\n        for i in range(0, len(folders), chunk_size):\n            chunk_folders = folders[i:i+chunk_size]\n            chunk_data = []\n            \n            futures = [\n                executor.submit(\n                    process_single_folder, \n                    str(folder), \n                    allowed_tags\n                    #arrays_dir\n                )\n                for folder in chunk_folders\n            ]\n            \n            for future in tqdm(futures, \n                             desc=f\"Processing chunk {i//chunk_size + 1}/{(len(folders)-1)//chunk_size + 1}\"):\n                chunk_data.extend(future.result())\n            \n            if chunk_data:\n                chunk_df = pl.DataFrame(\n                    chunk_data,\n                    schema=schema,\n                    infer_schema_length=None\n                )\n                \n                chunk_df.write_parquet(\n                    temp_dir / f\"dicom_metadata_chunk_{i//chunk_size}.parquet\",\n                    compression=\"snappy\"\n                )\n                \n                del chunk_data\n                del chunk_df\n    \n    # Combine chunks\n    print(\"\\nCombining chunks...\")\n    chunk_files = list(temp_dir.glob(\"dicom_metadata_chunk_*.parquet\"))\n    final_df = pl.concat([\n        pl.scan_parquet(str(chunk_file))\n        for chunk_file in chunk_files\n    ]).collect()\n    \n    # Clean up temporary files\n    for f in chunk_files:\n        f.unlink()\n    temp_dir.rmdir()\n    \n    return final_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:51:00.526127Z","iopub.execute_input":"2025-09-30T04:51:00.526374Z","iopub.status.idle":"2025-09-30T04:51:00.53908Z","shell.execute_reply.started":"2025-09-30T04:51:00.526356Z","shell.execute_reply":"2025-09-30T04:51:00.535606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Starting the collection of metadata","metadata":{}},{"cell_type":"code","source":"#with tpu_strategy.scope():\nroot_folder = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series\"\n\ntry:\n    metadata_df = create_dicom_dataset(\n        root_folder, \n        allowed_tags, \n        num_processes=mp.cpu_count(),\n        chunk_size=192\n    )\nexcept Exception as e:\n    print(f\"Error: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:51:04.747858Z","iopub.execute_input":"2025-09-30T04:51:04.748087Z","iopub.status.idle":"2025-09-30T05:01:30.172108Z","shell.execute_reply.started":"2025-09-30T04:51:04.74807Z","shell.execute_reply":"2025-09-30T05:01:30.166841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_df.write_parquet('metadata_df.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:04:04.489681Z","iopub.execute_input":"2025-09-30T05:04:04.489973Z","iopub.status.idle":"2025-09-30T05:04:05.928369Z","shell.execute_reply.started":"2025-09-30T05:04:04.489954Z","shell.execute_reply":"2025-09-30T05:04:05.922953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:40:41.764788Z","iopub.execute_input":"2025-09-30T04:40:41.765162Z","iopub.status.idle":"2025-09-30T04:40:41.7754Z","shell.execute_reply.started":"2025-09-30T04:40:41.765139Z","shell.execute_reply":"2025-09-30T04:40:41.772671Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Images where the min and max of the numpy array is zero","metadata":{}},{"cell_type":"code","source":"metadata_df.filter(pl.col('min_max_diff') == 0).select(['file_path'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T04:42:54.085421Z","iopub.execute_input":"2025-09-30T04:42:54.085684Z","iopub.status.idle":"2025-09-30T04:42:54.110339Z","shell.execute_reply.started":"2025-09-30T04:42:54.085666Z","shell.execute_reply":"2025-09-30T04:42:54.106751Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create the full training data\n## Remove the images with zero min max\n* Bring in the localizer coordinates for the series where aneurysm is present\n* Create another column to signify whether aneurysm is shown in a specific image within a series\n* There can be cases where some images of a series cannot catch aneurysm presence\n* Bringing all the rows at the image file granularity, i.e. if a file has coordinates then it has aneurysm else not","metadata":{}},{"cell_type":"code","source":"df_all_coordinates = dd.sql( \\\n    \"select t2.coordinates_x, t2.coordinates_y, coalesce(t2.coordinates_f,0) as coordinates_f, t1.* \\\n    from (select * from metadata_df where min_max_diff != 0)t1 \\\n    left join train_locale_meta_data t2 \\\n    on t1.folder_name = t2.SeriesInstanceUID \\\n    and replace(t1.file_name, '.dcm','') = t2.SOPInstanceUID \"\\\n).pl()\n\nprint(df_all_coordinates.shape)\nprint(df_all_coordinates.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:07:44.167107Z","iopub.execute_input":"2025-09-30T05:07:44.167374Z","iopub.status.idle":"2025-09-30T05:07:46.036405Z","shell.execute_reply.started":"2025-09-30T05:07:44.167355Z","shell.execute_reply":"2025-09-30T05:07:46.033412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_columns = [col.lower().replace(\" \", \"_\") for col in train_meta_data.columns]\ntrain_meta_data.columns = new_columns\nprint(train_meta_data.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:07:52.782615Z","iopub.execute_input":"2025-09-30T05:07:52.782911Z","iopub.status.idle":"2025-09-30T05:07:52.789855Z","shell.execute_reply.started":"2025-09-30T05:07:52.782892Z","shell.execute_reply":"2025-09-30T05:07:52.7877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data = dd.sql( \\\n    \"select t2.file_name, t2.file_path, t2.image_shape, t2.coordinates_x, t2.coordinates_y, t2.coordinates_f, t2.min_max_diff \\\n    , t1.aneurysm_present as aneurysm_present_in_series \\\n    , case when t2.coordinates_x is not null then 1 else 0 end as aneurysm_present_in_image \\\n    , t1.seriesinstanceuid, t1.patientage, t1.patientsex, t1.modality \\\n    , case when t2.coordinates_x is not null then t1.left_infraclinoid_internal_carotid_artery \\\n    else 0 end as left_infraclinoid_internal_carotid_artery \\\n    , case when t2.coordinates_x is not null then t1.right_infraclinoid_internal_carotid_artery \\\n    else 0 end as right_infraclinoid_internal_carotid_artery \\\n    , case when t2.coordinates_x is not null then t1.left_supraclinoid_internal_carotid_artery \\\n    else 0 end as left_supraclinoid_internal_carotid_artery \\\n    , case when t2.coordinates_x is not null then t1.right_supraclinoid_internal_carotid_artery \\\n    else 0 end as right_supraclinoid_internal_carotid_artery \\\n    , case when t2.coordinates_x is not null then t1.left_middle_cerebral_artery \\\n    else 0 end as left_middle_cerebral_artery \\\n    , case when t2.coordinates_x is not null then t1.right_middle_cerebral_artery \\\n    else 0 end as right_middle_cerebral_artery \\\n    , case when t2.coordinates_x is not null then t1.anterior_communicating_artery \\\n    else 0 end as anterior_communicating_artery \\\n    , case when t2.coordinates_x is not null then t1.left_anterior_cerebral_artery \\\n    else 0 end as left_anterior_cerebral_artery \\\n    , case when t2.coordinates_x is not null then t1.right_anterior_cerebral_artery \\\n    else 0 end as right_anterior_cerebral_artery \\\n    , case when t2.coordinates_x is not null then t1.left_posterior_communicating_artery \\\n    else 0 end as left_posterior_communicating_artery \\\n    , case when t2.coordinates_x is not null then t1.right_posterior_communicating_artery \\\n    else 0 end as right_posterior_communicating_artery \\\n    , case when t2.coordinates_x is not null then t1.basilar_tip \\\n    else 0 end as basilar_tip \\\n    , case when t2.coordinates_x is not null then t1.other_posterior_circulation \\\n    else 0 end as other_posterior_circulation \\\n    from train_meta_data t1 \\\n    join df_all_coordinates t2 \\\n    on t1.SeriesInstanceUID = t2.folder_name\" \\\n).pl()\n\nprint(\"Full training data: \", df_all_data.shape)\nprint(\"Full training data columns: \", df_all_data.columns)\nprint(\"Aneurysm not present in {0} series\".format(df_all_data.filter(pl.col(\"coordinates_x\").is_null()).shape[0]))\n\nprint(\"Aneurysm present in {0} series\".format(df_all_data.filter(pl.col(\"coordinates_x\").is_not_null()).shape[0]))\n\nprint(\"Aneurysm not shown in {0} images\".format(df_all_data.filter(pl.col(\"aneurysm_present_in_image\")==0).shape[0]))\n\nprint(\"Aneurysm shown in {0} images\".format(df_all_data.filter(pl.col(\"aneurysm_present_in_image\")==1).shape[0]))\n\nprint(df_all_data.select([\"coordinates_x\", \"coordinates_y\", \"coordinates_f\"]).describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:08:21.848577Z","iopub.execute_input":"2025-09-30T05:08:21.848857Z","iopub.status.idle":"2025-09-30T05:08:23.64009Z","shell.execute_reply.started":"2025-09-30T05:08:21.848836Z","shell.execute_reply":"2025-09-30T05:08:23.636165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:09:02.292505Z","iopub.execute_input":"2025-09-30T05:09:02.292784Z","iopub.status.idle":"2025-09-30T05:09:02.305748Z","shell.execute_reply.started":"2025-09-30T05:09:02.292766Z","shell.execute_reply":"2025-09-30T05:09:02.302205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\" \\\nselect t1.modality, t1.aneurysm_present_in_image, t1.per_mod_count, \\\nround(t1.per_mod_count/t2.total_count ,3) as modality_pct \\\nfrom \\\n( \\\nselect modality, aneurysm_present_in_image, cast(count(1) as float) as per_mod_count from df_all_data \\\ngroup by modality, aneurysm_present_in_image \\\n)t1 \\\njoin \\\n(select cast(count(1) as float) as total_count from df_all_data)t2 \\\non 1=1 \\\norder by 1 \\\n\").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:38:22.573597Z","iopub.execute_input":"2025-10-04T16:38:22.573907Z","iopub.status.idle":"2025-10-04T16:38:23.834912Z","shell.execute_reply.started":"2025-10-04T16:38:22.573883Z","shell.execute_reply":"2025-10-04T16:38:23.834074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\" \\\nselect t1.image_shape, t1.aneurysm_present_in_image, t1.per_mod_count, \\\nround(t1.per_mod_count/t2.total_count ,3) as modality_pct \\\nfrom \\\n( \\\nselect image_shape, aneurysm_present_in_image, cast(count(1) as float) as per_mod_count from df_all_data \\\ngroup by image_shape, aneurysm_present_in_image \\\n)t1 \\\njoin \\\n(select cast(count(1) as float) as total_count from df_all_data)t2 \\\non 1=1 \\\norder by 3 desc \\\n\").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:42:30.016367Z","iopub.execute_input":"2025-10-04T16:42:30.016702Z","iopub.status.idle":"2025-10-04T16:42:30.682042Z","shell.execute_reply.started":"2025-10-04T16:42:30.016682Z","shell.execute_reply":"2025-10-04T16:42:30.681342Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data = dd.sql(\" select \\\ncase when left_infraclinoid_internal_carotid_artery = 1 then \\'left_infraclinoid_internal_carotid_artery\\' \\\nwhen right_infraclinoid_internal_carotid_artery = 1 then \\'right_infraclinoid_internal_carotid_artery\\' \\\nwhen left_supraclinoid_internal_carotid_artery = 1 then \\'left_supraclinoid_internal_carotid_artery\\' \\\nwhen right_supraclinoid_internal_carotid_artery = 1 then \\'right_supraclinoid_internal_carotid_artery\\' \\\nwhen left_middle_cerebral_artery = 1 then \\'left_middle_cerebral_artery\\' \\\nwhen right_middle_cerebral_artery = 1 then \\'right_middle_cerebral_artery\\' \\\nwhen anterior_communicating_artery = 1 then \\'anterior_communicating_artery\\' \\\nwhen left_anterior_cerebral_artery = 1 then \\'left_anterior_cerebral_artery\\' \\\nwhen right_anterior_cerebral_artery = 1 then \\'right_anterior_cerebral_artery\\' \\\nwhen left_posterior_communicating_artery = 1 then \\'left_posterior_communicating_artery\\' \\\nwhen right_posterior_communicating_artery = 1 then \\'right_posterior_communicating_artery\\' \\\nwhen basilar_tip = 1 then \\'basilar_tip\\' \\\nwhen other_posterior_circulation = 1 then \\'other_posterior_circulation\\' \\\nelse \\'no_aneurysm\\' end as aneurysm_position \\\n, * \\\nfrom df_all_data\" \\\n).pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:09:33.72094Z","iopub.execute_input":"2025-09-30T05:09:33.721262Z","iopub.status.idle":"2025-09-30T05:09:34.637141Z","shell.execute_reply.started":"2025-09-30T05:09:33.72119Z","shell.execute_reply":"2025-09-30T05:09:34.633255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:09:37.493104Z","iopub.execute_input":"2025-09-30T05:09:37.493374Z","iopub.status.idle":"2025-09-30T05:09:37.505118Z","shell.execute_reply.started":"2025-09-30T05:09:37.493356Z","shell.execute_reply":"2025-09-30T05:09:37.502969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\n\ndf_all_data = df_all_data.with_columns(\n        pl.Series(\n            \"aneurysm_position_encoded\",\n            le.fit_transform(df_all_data[\"aneurysm_position\"].to_numpy())\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:09:52.189796Z","iopub.execute_input":"2025-09-30T05:09:52.190071Z","iopub.status.idle":"2025-09-30T05:09:52.362056Z","shell.execute_reply.started":"2025-09-30T05:09:52.190051Z","shell.execute_reply":"2025-09-30T05:09:52.359666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.write_parquet('full_training_data.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-30T05:10:42.530125Z","iopub.execute_input":"2025-09-30T05:10:42.530472Z","iopub.status.idle":"2025-09-30T05:10:43.837348Z","shell.execute_reply.started":"2025-09-30T05:10:42.530436Z","shell.execute_reply":"2025-09-30T05:10:43.833493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data = pl.read_parquet('/kaggle/input/rsna-aneurysm-train-metadata-suman/full_training_data_v2.parquet')\nprint(\"Shape of training metadata\", df_all_data.shape)\ndf_all_data.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:37:51.190733Z","iopub.execute_input":"2025-10-04T16:37:51.191474Z","iopub.status.idle":"2025-10-04T16:37:52.818609Z","shell.execute_reply.started":"2025-10-04T16:37:51.191445Z","shell.execute_reply":"2025-10-04T16:37:52.817693Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Looking at specific images\n## Taking an example each from CTA, MRA, MRI T2 and MRI T1 post","metadata":{}},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=1000)\npl.Config.set_tbl_rows(1000)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:40:12.905191Z","iopub.execute_input":"2025-10-04T16:40:12.905531Z","iopub.status.idle":"2025-10-04T16:40:12.91293Z","shell.execute_reply.started":"2025-10-04T16:40:12.905509Z","shell.execute_reply":"2025-10-04T16:40:12.912138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('file_path')).item(0, 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:55:50.902231Z","iopub.execute_input":"2025-10-04T16:55:50.902589Z","iopub.status.idle":"2025-10-04T16:55:50.989956Z","shell.execute_reply.started":"2025-10-04T16:55:50.902568Z","shell.execute_reply":"2025-10-04T16:55:50.989311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_x')).item(0, 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:57:29.974621Z","iopub.execute_input":"2025-10-04T16:57:29.97495Z","iopub.status.idle":"2025-10-04T16:57:29.984264Z","shell.execute_reply.started":"2025-10-04T16:57:29.974927Z","shell.execute_reply":"2025-10-04T16:57:29.983353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_y')).item(0, 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:57:50.313993Z","iopub.execute_input":"2025-10-04T16:57:50.314329Z","iopub.status.idle":"2025-10-04T16:57:50.323462Z","shell.execute_reply.started":"2025-10-04T16:57:50.314307Z","shell.execute_reply":"2025-10-04T16:57:50.322758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\"select * from df_all_data where modality = 'MRI T1post' and aneurysm_present_in_image = 1 limit 5 \").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:40:17.431157Z","iopub.execute_input":"2025-10-04T16:40:17.43153Z","iopub.status.idle":"2025-10-04T16:40:17.754796Z","shell.execute_reply.started":"2025-10-04T16:40:17.431504Z","shell.execute_reply":"2025-10-04T16:40:17.753966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\"select * from df_all_data where modality = 'MRI T2' and aneurysm_present_in_image = 1 limit 5 \").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:48:39.262033Z","iopub.execute_input":"2025-10-04T16:48:39.262408Z","iopub.status.idle":"2025-10-04T16:48:39.597745Z","shell.execute_reply.started":"2025-10-04T16:48:39.262385Z","shell.execute_reply":"2025-10-04T16:48:39.596934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\"select * from df_all_data where modality = 'MRA' and aneurysm_present_in_image = 1 limit 5 \").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:48:48.271831Z","iopub.execute_input":"2025-10-04T16:48:48.272905Z","iopub.status.idle":"2025-10-04T16:48:48.602975Z","shell.execute_reply.started":"2025-10-04T16:48:48.272873Z","shell.execute_reply":"2025-10-04T16:48:48.602105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dd.sql(\"select * from df_all_data where modality = 'CTA' and aneurysm_present_in_image = 1 limit 5 \").pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:49:04.778179Z","iopub.execute_input":"2025-10-04T16:49:04.778505Z","iopub.status.idle":"2025-10-04T16:49:05.102648Z","shell.execute_reply.started":"2025-10-04T16:49:04.778483Z","shell.execute_reply":"2025-10-04T16:49:05.101809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_single_frame(multiframe_path, slice_number, output_path=None):\n    \"\"\"\n    Extract a single frame from a multi-frame DICOM\n    \n    Args:\n        multiframe_path: Path to multi-frame DICOM file\n        slice_number: The slice number to extract (0-based index)\n        output_path: Path to save the single-frame DICOM. If None, returns the dataset\n    \"\"\"\n    try:\n        # Read the multi-frame DICOM with force=True to handle potentially corrupted files\n        multi_ds = dcmread(multiframe_path, force=True)\n        \n        # Verify it's a multi-frame image\n        if not hasattr(multi_ds, 'NumberOfFrames'):\n            raise ValueError(\"Input DICOM is not a multi-frame image\")\n        \n        # Check if slice number is valid\n        if slice_number >= multi_ds.NumberOfFrames:\n            raise ValueError(f\"Slice number {slice_number} is out of range. \"\n                           f\"Image has {multi_ds.NumberOfFrames} frames\")\n        \n        # Create new dataset for single frame\n        single_ds = FileDataset(output_path or \"temp.dcm\", {}, \n                              file_meta=FileMetaDataset(), \n                              preamble=b\"\\0\" * 128)\n        \n        # Copy attributes from multi-frame dataset\n        attrs_to_copy = allowed_tags\n        \n        for attr in attrs_to_copy:\n            if hasattr(multi_ds, attr):\n                setattr(single_ds, attr, getattr(multi_ds, attr))\n        \n        # Generate new UIDs\n        single_ds.SOPInstanceUID = generate_uid()\n        single_ds.file_meta.MediaStorageSOPInstanceUID = single_ds.SOPInstanceUID\n        \n        # Set transfer syntax to uncompressed little endian\n        single_ds.file_meta.TransferSyntaxUID = ImplicitVRLittleEndian\n        single_ds.file_meta.MediaStorageSOPClassUID = multi_ds.file_meta.MediaStorageSOPClassUID\n        if hasattr(multi_ds.file_meta, 'ImplementationClassUID'):\n            single_ds.file_meta.ImplementationClassUID = multi_ds.file_meta.ImplementationClassUID\n        \n        # Set instance-specific attributes\n        single_ds.InstanceNumber = slice_number + 1\n        \n        try:\n            # Try to get pixel array directly\n            pixel_array = multi_ds.pixel_array[slice_number]\n        except Exception as e:\n            #print(f\"Warning: Could not directly access pixel_array: {e}\")\n            # Alternative approach: decompress and get pixels\n            if hasattr(multi_ds, 'decompress'):\n                multi_ds.decompress()\n            pixel_array = multi_ds.pixel_array[slice_number]\n        \n        # Set pixel data\n        single_ds.PixelData = pixel_array.tobytes()\n        \n        # Update image-specific attributes\n        single_ds.NumberOfFrames = 1\n        \n        # Try to copy position and orientation\n        try:\n            if hasattr(multi_ds, 'PerFrameFunctionalGroupsSequence'):\n                frame_content = multi_ds.PerFrameFunctionalGroupsSequence[slice_number]\n                \n                if hasattr(frame_content, 'PlanePositionSequence'):\n                    position = frame_content.PlanePositionSequence[0].ImagePositionPatient\n                    single_ds.ImagePositionPatient = position\n                \n                if hasattr(frame_content, 'PlaneOrientationSequence'):\n                    orientation = frame_content.PlaneOrientationSequence[0].ImageOrientationPatient\n                    single_ds.ImageOrientationPatient = orientation\n        except Exception as e:\n            #print(f\"Warning: Could not copy position/orientation: {e}\")\n            raise\n        \n        # Add creation timestamp\n        dt = datetime.datetime.now()\n        single_ds.ContentDate = dt.strftime('%Y%m%d')\n        single_ds.ContentTime = dt.strftime('%H%M%S.%f')\n        \n        # Save or return the dataset\n        if output_path:\n            single_ds.save_as(output_path, write_like_original=False)\n            return None\n        return single_ds\n    \n    except Exception as e:\n        #print(f\"Error extracting frame: {e}\")\n        raise\n\n# Alternative version using different approach for compressed files\ndef extract_single_frame_alternative(multiframe_path, slice_number, output_path=None):\n    \"\"\"\n    Alternative version for handling problematic files\n    \"\"\"\n    try:\n        # Read with force and stop before pixels\n        multi_ds = dcmread(multiframe_path, force=True, stop_before_pixels=True)\n        \n        # Read pixel data separately\n        with open(multiframe_path, 'rb') as f:\n            multi_ds.PixelData = f.read()\n        \n        # Decompress if needed\n        if hasattr(multi_ds, 'decompress'):\n            multi_ds.decompress()\n        \n        # Get pixel array\n        pixel_array = multi_ds.pixel_array[slice_number]\n        \n        # Create new dataset\n        single_ds = FileDataset(output_path or \"temp.dcm\", {}, \n                              file_meta=FileMetaDataset(), \n                              preamble=b\"\\0\" * 128)\n        \n        # Copy attributes (same as before)\n        attrs_to_copy = allowed_tags\n        \n        for attr in attrs_to_copy:\n            if hasattr(multi_ds, attr):\n                setattr(single_ds, attr, getattr(multi_ds, attr))\n        \n        # Generate new UIDs\n        single_ds.SOPInstanceUID = generate_uid()\n        single_ds.file_meta.MediaStorageSOPInstanceUID = single_ds.SOPInstanceUID\n        \n        # Set transfer syntax to uncompressed little endian\n        single_ds.file_meta.TransferSyntaxUID = ImplicitVRLittleEndian\n        single_ds.file_meta.MediaStorageSOPClassUID = multi_ds.file_meta.MediaStorageSOPClassUID\n        if hasattr(multi_ds.file_meta, 'ImplementationClassUID'):\n            single_ds.file_meta.ImplementationClassUID = multi_ds.file_meta.ImplementationClassUID\n        \n        # Set instance-specific attributes\n        single_ds.InstanceNumber = slice_number + 1\n        \n        try:\n            # Try to get pixel array directly\n            pixel_array = multi_ds.pixel_array[slice_number]\n        except Exception as e:\n            #print(f\"Warning: Could not directly access pixel_array: {e}\")\n            # Alternative approach: decompress and get pixels\n            if hasattr(multi_ds, 'decompress'):\n                multi_ds.decompress()\n            pixel_array = multi_ds.pixel_array[slice_number]\n        \n        # Set pixel data\n        single_ds.PixelData = pixel_array.tobytes()\n        \n        # Update image-specific attributes\n        single_ds.NumberOfFrames = 1\n        \n        # Try to copy position and orientation\n        try:\n            if hasattr(multi_ds, 'PerFrameFunctionalGroupsSequence'):\n                frame_content = multi_ds.PerFrameFunctionalGroupsSequence[slice_number]\n                \n                if hasattr(frame_content, 'PlanePositionSequence'):\n                    position = frame_content.PlanePositionSequence[0].ImagePositionPatient\n                    single_ds.ImagePositionPatient = position\n                \n                if hasattr(frame_content, 'PlaneOrientationSequence'):\n                    orientation = frame_content.PlaneOrientationSequence[0].ImageOrientationPatient\n                    single_ds.ImageOrientationPatient = orientation\n        except Exception as e:\n            #print(f\"Warning: Could not copy position/orientation: {e}\")\n            raise\n        \n        # Add creation timestamp\n        dt = datetime.datetime.now()\n        single_ds.ContentDate = dt.strftime('%Y%m%d')\n        single_ds.ContentTime = dt.strftime('%H%M%S.%f')\n        \n        # Save or return the dataset\n        if output_path:\n            single_ds.save_as(output_path, write_like_original=False)\n            return None\n        return single_ds\n        \n    except Exception as e:\n        #print(f\"Error in alternative extraction: {e}\")\n        raise\n\n# Function to try both methods\ndef safe_extract_single_frame(multiframe_path, slice_number, output_path=None):\n    \"\"\"\n    Try both extraction methods\n    \"\"\"\n    try:\n        return extract_single_frame(multiframe_path, slice_number, output_path)\n    except Exception as e:\n        #print(f\"Primary method failed: {e}\")\n        #print(\"Trying alternative method...\")\n        try:\n            return extract_single_frame_alternative(multiframe_path, slice_number, output_path)\n        except Exception as e2:\n            #print(f\"Alternative method also failed: {e2}\")\n            raise\n\n# Version with zoom functionality\ndef load_and_view_single_slice_with_zoom(dcm_path, x_coord, y_coord, f_coord=None, zoom_size=100):\n    \"\"\"\n    Load and display a single DICOM slice with crosshair and zoomed inset\n    \n    Args:\n        dcm_path: Path to the DICOM file\n        x_coord: x coordinate for the crosshair\n        y_coord: y coordinate for the crosshair\n        zoom_size: Size of the zoom window in pixels\n    \"\"\"\n    # Read DICOM file\n    if f_coord:\n        ds = safe_extract_single_frame(dcm_path, f_coord)\n    else:\n        ds = dcmread(dcm_path)\n    img = ds.pixel_array\n    \n    # Create figure and axes\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\n    \n    # Main image with crosshair\n    ax1.imshow(img, cmap='gray')\n    ax1.axvline(x=x_coord, color='red', alpha=0.5)\n    ax1.axhline(y=y_coord, color='red', alpha=0.5)\n    ax1.plot(x_coord, y_coord, 'r+', markersize=10, markeredgewidth=2)\n    \n    # Zoomed region\n    x_start = int(max(0, x_coord - zoom_size/2))\n    x_end = int(min(img.shape[1], x_coord + zoom_size/2))\n    y_start = int(max(0, y_coord - zoom_size/2))\n    y_end = int(min(img.shape[0], y_coord + zoom_size/2))\n    \n    zoomed = img[y_start:y_end, x_start:x_end]\n    ax2.imshow(zoomed, cmap='gray')\n    \n    # Add crosshair to zoomed region\n    center_x = x_coord - x_start\n    center_y = y_coord - y_start\n    ax2.axvline(x=center_x, color='red', alpha=0.5)\n    ax2.axhline(y=center_y, color='red', alpha=0.5)\n    ax2.plot(center_x, center_y, 'r+', markersize=10, markeredgewidth=2)\n    \n    ax1.axis('off')\n    ax2.axis('off')\n    ax1.set_title('Full Image')\n    ax2.set_title('Zoomed Region')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T16:49:45.649071Z","iopub.execute_input":"2025-10-04T16:49:45.65016Z","iopub.status.idle":"2025-10-04T16:49:45.673729Z","shell.execute_reply.started":"2025-10-04T16:49:45.650124Z","shell.execute_reply":"2025-10-04T16:49:45.672704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dcm_path = df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('file_path')).item(0, 0)\n\nx_coord = df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_x')).item(0, 0)\n\ny_coord = df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_y')).item(0, 0)\n\nf_coord = df_all_data.filter((pl.col('modality')=='MRI T1post') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_f')).item(0, 0)\n\nload_and_view_single_slice_with_zoom(dcm_path, x_coord, y_coord, f_coord)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:14:53.386907Z","iopub.execute_input":"2025-10-04T17:14:53.387243Z","iopub.status.idle":"2025-10-04T17:14:53.816365Z","shell.execute_reply.started":"2025-10-04T17:14:53.38722Z","shell.execute_reply":"2025-10-04T17:14:53.815503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if f_coord:\n    ds = safe_extract_single_frame(dcm_path, f_coord)\nelse:\n    ds = dcmread(dcm_path)\n\nfor tag in allowed_tags:\n    print(\"{0} = {1}\".format(tag, getattr(ds, tag, None)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:15:00.207828Z","iopub.execute_input":"2025-10-04T17:15:00.208226Z","iopub.status.idle":"2025-10-04T17:15:00.217586Z","shell.execute_reply.started":"2025-10-04T17:15:00.20816Z","shell.execute_reply":"2025-10-04T17:15:00.216584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dcm_path = df_all_data.filter((pl.col('modality')=='MRI T2') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('file_path')).item(0, 0)\n\nx_coord = df_all_data.filter((pl.col('modality')=='MRI T2') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_x')).item(0, 0)\n\ny_coord = df_all_data.filter((pl.col('modality')=='MRI T2') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_y')).item(0, 0)\n\nf_coord = df_all_data.filter((pl.col('modality')=='MRI T2') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_f')).item(0, 0)\n\nload_and_view_single_slice_with_zoom(dcm_path, x_coord, y_coord, f_coord)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:14:24.600639Z","iopub.execute_input":"2025-10-04T17:14:24.600939Z","iopub.status.idle":"2025-10-04T17:14:25.040477Z","shell.execute_reply.started":"2025-10-04T17:14:24.600918Z","shell.execute_reply":"2025-10-04T17:14:25.039589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if f_coord:\n    ds = safe_extract_single_frame(dcm_path, f_coord)\nelse:\n    ds = dcmread(dcm_path)\n\nfor tag in allowed_tags:\n    print(\"{0} = {1}\".format(tag, getattr(ds, tag, None)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:14:30.114033Z","iopub.execute_input":"2025-10-04T17:14:30.114447Z","iopub.status.idle":"2025-10-04T17:14:30.123853Z","shell.execute_reply.started":"2025-10-04T17:14:30.11442Z","shell.execute_reply":"2025-10-04T17:14:30.122963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dcm_path = df_all_data.filter((pl.col('modality')=='MRA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('file_path')).item(0, 0)\n\nx_coord = df_all_data.filter((pl.col('modality')=='MRA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_x')).item(0, 0)\n\ny_coord = df_all_data.filter((pl.col('modality')=='MRA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_y')).item(0, 0)\n\nf_coord = df_all_data.filter((pl.col('modality')=='MRA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_f')).item(0, 0)\n\nload_and_view_single_slice_with_zoom(dcm_path, x_coord, y_coord, f_coord)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:13:39.596698Z","iopub.execute_input":"2025-10-04T17:13:39.597011Z","iopub.status.idle":"2025-10-04T17:13:40.038583Z","shell.execute_reply.started":"2025-10-04T17:13:39.596988Z","shell.execute_reply":"2025-10-04T17:13:40.037543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if f_coord:\n    ds = safe_extract_single_frame(dcm_path, f_coord)\nelse:\n    ds = dcmread(dcm_path)\n\nfor tag in allowed_tags:\n    print(\"{0} = {1}\".format(tag, getattr(ds, tag, None)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:13:44.664503Z","iopub.execute_input":"2025-10-04T17:13:44.665225Z","iopub.status.idle":"2025-10-04T17:13:44.673245Z","shell.execute_reply.started":"2025-10-04T17:13:44.665195Z","shell.execute_reply":"2025-10-04T17:13:44.672393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dcm_path = df_all_data.filter((pl.col('modality')=='CTA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('file_path')).item(0, 0)\n\nx_coord = df_all_data.filter((pl.col('modality')=='CTA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_x')).item(0, 0)\n\ny_coord = df_all_data.filter((pl.col('modality')=='CTA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_y')).item(0, 0)\n\nf_coord = df_all_data.filter((pl.col('modality')=='CTA') & (pl.col('aneurysm_present_in_image')==1))\\\n.select(pl.col('coordinates_f')).item(0, 0)\n\nload_and_view_single_slice_with_zoom(dcm_path, x_coord, y_coord, f_coord)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:00:28.869449Z","iopub.execute_input":"2025-10-04T17:00:28.870002Z","iopub.status.idle":"2025-10-04T17:00:29.301581Z","shell.execute_reply.started":"2025-10-04T17:00:28.869975Z","shell.execute_reply":"2025-10-04T17:00:29.300467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if f_coord:\n    ds = safe_extract_single_frame(dcm_path, f_coord)\nelse:\n    ds = dcmread(dcm_path)\n\nfor tag in allowed_tags:\n    print(\"{0} = {1}\".format(tag, getattr(ds, tag, None)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T17:12:56.987303Z","iopub.execute_input":"2025-10-04T17:12:56.987636Z","iopub.status.idle":"2025-10-04T17:12:56.999104Z","shell.execute_reply.started":"2025-10-04T17:12:56.987613Z","shell.execute_reply":"2025-10-04T17:12:56.998021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if f_coord:\n    ds = safe_extract_single_frame(dcm_path, f_coord)\nelse:\n    ds = dcmread(dcm_path)\n\narr = ds.pixel_array.astype(np.float32)\n\n# Apply rescale if present\nslope = float(getattr(ds, \"RescaleSlope\", 1) or 1)\nintercept = float(getattr(ds, \"RescaleIntercept\", 0) or 0)\narr = arr * slope + intercept\n\n# Handle MONOCHROME1 (invert)\nif getattr(ds, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n    arr = arr.max() - arr\n\nimage = tf.convert_to_tensor(arr)\n\nexpanded_image = tf.expand_dims(image, -1)\nm, M=tf.math.reduce_min(expanded_image), tf.math.reduce_max(expanded_image)\nexpanded_image = (tf.image.grayscale_to_rgb(expanded_image)-m)/(M-m)\nexpanded_image = tf.image.resize(expanded_image, (128,128))\nsqzd_image = tf.squeeze(expanded_image)\n\ntrain_img = tf.reshape(sqzd_image, shape=(128, 128, 3))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_np = train_img.numpy()\nplt.imshow(image_np)\nplt.title(\"TensorFlow Image Visualization\")\nplt.axis('off') # Hide axes for cleaner image display\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### This is a multi-label classification problem where each instance (i.e. each DICOM image) can be labelled with at-most 13 labels (i.e. brain locations).\n#### The resulting predictions will then need to be aggregated up at the level of each scan series\n#### Finally the main target variable has to be calculated as the max of all the 13 labels, i.e. if at least one of the 13 labels is 1 then the final target variable is 1","metadata":{}}]}