{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13258161,"sourceType":"datasetVersion","datasetId":8401368},{"sourceId":13266765,"sourceType":"datasetVersion","datasetId":8407116}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## I’ve preprocessed the original 3D DICOM images into 2D .npy files, which massively SPEEDS UP training\n## You can find the 2d 3channels images in npy file format here:- https://www.kaggle.com/datasets/khanramshaayub/rsna-preprocessed-images \n### It's recommended to read the data description for usage.","metadata":{}},{"cell_type":"code","source":"##---------------------- IMPORTING ALL LIBRARIES -------------------------\nimport pandas as pd\nimport pydicom #for dicom files\nimport warnings\nwarnings.filterwarnings('ignore')\nimport numpy as np\nfrom pathlib import Path\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom torchmetrics.classification import AUROC\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.nn import BCEWithLogitsLoss\nimport polars as pl\nfrom torch.cuda.amp import autocast\nimport gc\nimport shutil\nfrom IPython.display import display","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T05:52:13.136369Z","iopub.execute_input":"2025-10-04T05:52:13.13699Z","iopub.status.idle":"2025-10-04T05:52:37.209527Z","shell.execute_reply.started":"2025-10-04T05:52:13.136957Z","shell.execute_reply":"2025-10-04T05:52:37.208924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from dataclasses import dataclass\n\n@dataclass\nclass Config:\n    image_size = 512\n    num_slices = 32\n    \ncfg = Config()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T05:52:37.21086Z","iopub.execute_input":"2025-10-04T05:52:37.21155Z","iopub.status.idle":"2025-10-04T05:52:37.216043Z","shell.execute_reply.started":"2025-10-04T05:52:37.211529Z","shell.execute_reply":"2025-10-04T05:52:37.215353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#---------------------------------- IMAGE PROCESSING ---------------------------------\n\ndef adaptive_windowing(image, modality):\n    percentile_range = (5, 95)\n\n    img_flat = image.flatten()\n    img_flat = img_flat[img_flat > 0]\n    if len(img_flat) == 0:\n        return np.zeros_like(image, dtype=np.uint8)\n    low_val = np.percentile(img_flat, percentile_range[0])\n    \n    high_val = np.percentile(img_flat, percentile_range[1])\n\n    if modality in ['CTA']:\n        window_width = (high_val - low_val) * 1.5\n        window_center = (high_val + low_val) / 2\n    elif modality in ['MRA']:\n        window_width = (high_val - low_val) * 1.2\n        window_center = high_val * 0.7\n    else:\n        window_width = high_val - low_val\n        window_center = (high_val + low_val) / 2    \n\n    img_min = window_center - window_width / 2\n    img_max = window_center + window_width / 2\n    img_windowed = np.clip(image, img_min, img_max)\n\n## normalize 0-255\n    if img_max > img_min:\n        img_normalized = ((img_windowed - img_min) / (img_max - img_min) * 255).astype(np.uint8)\n    else: \n        img_normalized = np.zeros_like(image, dtype = np.uint8)\n    return img_normalized\n\n\ndef process_dicom_series(series_path):\n\n    all_filepaths = []\n    for root, _, files in os.walk(series_path):\n        for file in files:\n            if file.endswith('.dcm'):\n                all_filepaths.append(os.path.join(root, file))\n    all_filepaths.sort()\n    \n    if len(all_filepaths) == 0: \n        #no files extracted\n        print(\"Ops! NO files extracted\")\n        volume = np.zeros((cfg.num_slices, cfg.image_size, cfg.image_size), dtype = np.uint8)\n        metadeta = {'age': 40, 'sex': 0, 'modality': 'CT'}\n        return volume, metadata\n\n    metadata = {}\n    dicom_data = []\n    for i, filepath in enumerate(all_filepaths):\n        ds = pydicom.dcmread(filepath, force = True)\n        img = ds.pixel_array \n        if img.ndim == 3:\n            if img.shape[-1] == 3:\n                img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2GRAY).astype(np.float32)\n            else:\n                img = img[:, :, 0]\n        instance_num = getattr(ds, 'InstanceNumber', i) # get instance number for proper slice sorting\n        if i == 0: # extract the metadata\n            metadata[\"modality\"] = getattr(ds, 'Modality', 'CT')\n            try: \n                age_str = getattr(ds, 'PatientAge', '50Y')\n                age = int(''.join(filter(str.isdigit, age_str[:3])) or '50')\n                metadata['age'] = min(age, 100)\n            except:\n                metadata['age'] = 50\n            try:\n                sex = getattr(ds, 'PatientSex', 'M')\n                metadata['sex'] = 1 if sex == 'M' else 0\n            except:\n                metadata['sex'] = 0\n        \n        #### ------ APPLY RESCALING------------\n        if hasattr(ds, 'RescaleSlope') and hasattr(ds, 'RescaleIntercept'):\n            img = img * ds.RescaleSlope + ds.RescaleIntercept\n            \n        #input arrays shape not matching so resizing image should be done before stacking\n        img_resized = cv2.resize(img, (cfg.image_size, cfg.image_size))\n        dicom_data.append((instance_num, img_resized))\n\n    if len(dicom_data) == 0:\n        volume = np.zeros((cfg.num_slices, cfg.image_size, cfg.image_size), dtype = np.uint8)\n        return volume, metadata\n            \n    dicom_data.sort(key = lambda x: x[0])\n    raw_slices = [d[1] for d in dicom_data]\n    volume_3d = np.stack(raw_slices, axis = 0) #stacking on top of each other (row-wise)\n\n    ## apply modality specific intensity windowing\n    volume_windowed = adaptive_windowing(volume_3d, metadata['modality'])\n\n    ## Resize slices \n    processed_slices = []\n    for img in volume_windowed:\n        resized = cv2.resize(img, (cfg.image_size, cfg.image_size))\n        processed_slices.append(resized)\n    volume = np.array(processed_slices)\n\n    if len(processed_slices) > cfg.num_slices:\n        indices = np.linspace(0, len(processed_slices) - 1, cfg.num_slices).astype(int)\n        volume = volume[indices]\n    elif len(processed_slices) < cfg.num_slices:\n        pad_size = cfg.num_slices - len(processed_slices)\n        volume = np.pad(volume, ((0,pad_size), (0,0) , (0,0)), mode = 'edge')\n    return volume, metadata\n    \n#####----------------- CREATE RICH MULTI-CHANNEL REPRESENTATON OF 3D IMAGE------------------\n\"\"\"\nSince \"volumes\" obtained are 3D, in order to pass it in 2D network we need a smart way fro converting 3D to 2D without losing finer details \n\n\n\"\"\"\ndef create_multichannel_img(volume):\n    depth, height, width = volume.shape\n    #--------Channel 1: Adaptive maximum intensity projection---------\n    start = int(depth * 0.15)\n    end = int(depth * 0.85)\n    core_vol = volume[start: end]\n    mip = np.max(core_vol, axis = 0)\n\n    #------ Channel 2: weighted avg of high intensity slices------\n    #------------ Focusing on brightness-----------\n    slices_means = np.mean(volume, axis = (1,2))\n    top_percentile = np.percentile(slices_means, 75)\n    high_intensity = slices_means >= top_percentile\n    if np.any(high_intensity):\n        weighted_avg = np.mean(volume[high_intensity], axis = 0)\n    else:\n        weighted_avg = np.mean(volume, axis = 0)\n\n    ##-------- Channel 3: Std projection ----------\n    std_proj = np.zeros_like(volume[0])\n    window_size = min(5, depth // 4)\n    for i in range(depth - window_size + 1):\n        window_std = np.std(volume[i : i + window_size], axis = 0)\n        std_proj = np.maximum(std_proj, window_std)\n    #------- normalize all channels 0-255-----------\n    channels = []\n    for channel in [mip, weighted_avg, std_proj]:\n        if channel.max() > channel.min():\n            channel_norm = ((channel - channel.min()) / (channel.max() - channel.min()) * 255).astype(np.uint8)\n        else:  \n            channel_norm = np.zeros_like(channel, dtype = np.uint8)\n        channels.append(channel_norm)\n    return np.stack(channels, axis = -1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T05:52:37.217015Z","iopub.execute_input":"2025-10-04T05:52:37.217343Z","iopub.status.idle":"2025-10-04T05:52:37.244581Z","shell.execute_reply.started":"2025-10-04T05:52:37.217318Z","shell.execute_reply":"2025-10-04T05:52:37.243889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nimages_path = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series\"\npreprocessed_path = \"/kaggle/working/preprocessed_npy\"\nos.makedirs(preprocessed_path, exist_ok=True)\n\ndef preprocess_single_npy(row):\n    \"\"\"\n    Save as float32 numpy array (preserves full precision)\n    \"\"\"\n    instance_id = row['SeriesInstanceUID']\n    series_dir = os.path.join(images_path, instance_id)\n    \n    if not os.path.exists(series_dir):\n        return None\n    \n    volume, metadata = process_dicom_series(series_dir)\n    img = create_multichannel_img(volume)  # (H, W, 3) uint8\n    \n    # Convert back to float32 for better precision\n    img_float = img.astype(np.float32) / 255.0  # Normalize to [0, 1]\n    \n    # Save as numpy array (much better for medical data!)\n    output_path = os.path.join(preprocessed_path, f\"{instance_id}.npy\")\n    np.save(output_path, img_float)\n    \n    return instance_id\n\n# Precompute\nsubset_df = pd.read_csv(\"/kaggle/input/subset3000/train_subset_3000.csv\")\n\nwith Pool(processes=os.cpu_count()) as pool:\n    results = list(tqdm(\n        pool.imap(preprocess_single_npy, [row for _, row in subset_df.iterrows()]),\n        total=len(subset_df),\n        desc=\"Precomputing .npy images\"\n    ))\n\nprint(f\"Done! {len([r for r in results if r])} images saved\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-04T05:52:37.245841Z","iopub.execute_input":"2025-10-04T05:52:37.246072Z","iopub.status.idle":"2025-10-04T07:59:45.508147Z","shell.execute_reply.started":"2025-10-04T05:52:37.246045Z","shell.execute_reply":"2025-10-04T07:59:45.501785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install kaggle --quiet\n!mkdir -p ~/.kaggle\n!echo '{\"username\":\"putusername\",\"key\":\"putyourkey\"}' > ~/.kaggle/kaggle.json\n!chmod 600 ~/.kaggle/kaggle.json","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T09:12:26.597163Z","iopub.execute_input":"2025-10-04T09:12:26.597337Z","iopub.status.idle":"2025-10-04T09:12:33.122317Z","shell.execute_reply.started":"2025-10-04T09:12:26.597321Z","shell.execute_reply":"2025-10-04T09:12:33.121277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle datasets init -p /kaggle/working/preprocessed_npy","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle datasets create -p /kaggle/working/preprocessed_npy --dir-mode zip\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T09:15:44.527233Z","iopub.execute_input":"2025-10-04T09:15:44.527523Z","iopub.status.idle":"2025-10-04T09:15:44.531473Z","shell.execute_reply.started":"2025-10-04T09:15:44.527502Z","shell.execute_reply":"2025-10-04T09:15:44.530772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nmetadata['title'] = \"RSNA Aneurysm Preprocessed NPY 3000\"\nmetadata['id'] = \"khanramshaayub/rsna-aneurysm-preprocessed-3000\"  \nmetadata['licenses'] = [{\"name\": \"CC0-1.0\"}]\n\nmetadata['subtitle'] = \"Preprocessed multichannel images for RSNA Intracranial Aneurysm Detection\"\nmetadata['description'] = \"Preprocessed .npy files (512x512x3, float32) from DICOM series. Ready for fast training.\"\n\nwith open(metadata_path, 'w') as f:\n    json.dump(metadata, f, indent=2)\n\nprint(\"\\n✓ Updated metadata:\")\nprint(json.dumps(metadata, indent=2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T09:15:56.176727Z","iopub.execute_input":"2025-10-04T09:15:56.177041Z","iopub.status.idle":"2025-10-04T09:15:56.180822Z","shell.execute_reply.started":"2025-10-04T09:15:56.177016Z","shell.execute_reply":"2025-10-04T09:15:56.180141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle datasets create -p /kaggle/working/preprocessed_npy --dir-mode zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T08:08:36.624171Z","iopub.execute_input":"2025-10-04T08:08:36.624795Z","iopub.status.idle":"2025-10-04T08:37:17.401167Z","shell.execute_reply.started":"2025-10-04T08:08:36.624767Z","shell.execute_reply":"2025-10-04T08:37:17.400339Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\nshutil.make_archive(\"/kaggle/working/rsna_preprocessed_npy\", 'zip', \"/kaggle/working/preprocessed_npy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-04T08:48:16.703765Z","iopub.execute_input":"2025-10-04T08:48:16.704274Z","iopub.status.idle":"2025-10-04T08:54:25.090079Z","shell.execute_reply.started":"2025-10-04T08:48:16.704248Z","shell.execute_reply":"2025-10-04T08:54:25.089273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport os\n\npreprocessed_path = \"/kaggle/input/rsna-preprocessed-images\"\n\ndef load_precomputed_npy(instance_id):\n    npy_path = os.path.join(preprocessed_path, f\"{instance_id}.npy\")\n    if os.path.exists(npy_path):\n        img = np.load(npy_path)  # shape (H, W, C), dtype float32, normalized [0,1]\n        return img\n    else:\n        return None\nimg = load_precomputed_npy(\"1.2.826.0.1.3680043.8.498.10005158603912009425635473100344077317\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T09:22:44.49884Z","iopub.execute_input":"2025-10-11T09:22:44.499047Z","iopub.status.idle":"2025-10-11T09:22:44.562216Z","shell.execute_reply.started":"2025-10-11T09:22:44.499025Z","shell.execute_reply":"2025-10-11T09:22:44.561637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T09:22:44.56358Z","iopub.execute_input":"2025-10-11T09:22:44.563777Z","iopub.status.idle":"2025-10-11T09:22:44.569701Z","shell.execute_reply.started":"2025-10-11T09:22:44.56376Z","shell.execute_reply":"2025-10-11T09:22:44.568935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_uint8 = (img * 255).astype(np.uint8)\n\nplt.figure(figsize=(15, 4))\nplt.subplot(141)\nplt.imshow(img_uint8[:,:,0], cmap='gray')\nplt.title('Channel 0: MIP')\nplt.axis('off')\n\nplt.subplot(142)\nplt.imshow(img_uint8[:,:,1], cmap='gray')\nplt.title('Channel 1: Weighted Avg')\nplt.axis('off')\n\nplt.subplot(143)\nplt.imshow(img_uint8[:,:,2], cmap='gray')\nplt.title('Channel 2: Std Projection')\nplt.axis('off')\n\nplt.subplot(144)\nplt.imshow(img_uint8)\nplt.title('Combined RGB')\nplt.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-11T09:22:44.570242Z","iopub.execute_input":"2025-10-11T09:22:44.570468Z","iopub.status.idle":"2025-10-11T09:22:45.221273Z","shell.execute_reply.started":"2025-10-11T09:22:44.570451Z","shell.execute_reply":"2025-10-11T09:22:45.220528Z"}},"outputs":[],"execution_count":null}]}