{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":13451,"datasetId":654585,"databundleVersionId":1188070},{"sourceType":"datasetVersion","sourceId":13272281,"datasetId":8411032,"databundleVersionId":13971596}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Checking if data is loaded","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:50:33.29372Z","iopub.execute_input":"2025-10-09T23:50:33.293965Z","iopub.status.idle":"2025-10-09T23:50:33.448985Z","shell.execute_reply.started":"2025-10-09T23:50:33.293944Z","shell.execute_reply":"2025-10-09T23:50:33.447437Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Importing the modules and Supressing the warnings","metadata":{}},{"cell_type":"code","source":"import os\nimport pydicom\nfrom glob import glob\nfrom collections import defaultdict\nfrom tqdm import tqdm\nfrom concurrent.futures import ThreadPoolExecutor\nimport pickle\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nimport random\nrandom.seed(2708)\nimport json\nimport shutil\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"pydicom\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:50:37.463718Z","iopub.execute_input":"2025-10-09T23:50:37.464047Z","iopub.status.idle":"2025-10-09T23:50:38.889423Z","shell.execute_reply.started":"2025-10-09T23:50:37.464017Z","shell.execute_reply":"2025-10-09T23:50:38.88825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Defining and loading the paths of the files and the labels","metadata":{}},{"cell_type":"code","source":"base = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\"\ndata_dir = os.path.join(base, \"stage_2_train\")\nlabel_csv = os.path.join(base, \"stage_2_train.csv\")\nlabels_df = pd.read_csv(label_csv)\ndicom_paths = glob(os.path.join(data_dir, \"*.dcm\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:50:45.133209Z","iopub.execute_input":"2025-10-09T23:50:45.133859Z","iopub.status.idle":"2025-10-09T23:51:02.063052Z","shell.execute_reply.started":"2025-10-09T23:50:45.13382Z","shell.execute_reply":"2025-10-09T23:51:02.062016Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Creating the volume dictionaries (Dont run if the .pkl files are available)","metadata":{}},{"cell_type":"code","source":"# Initializing the dictionaries\nvolumes_counts = defaultdict(int)\nvolumes_slices = defaultdict(list)\n\n# Creating a function to read DICOM metadata and return (series_id, dicom_id) tuple\ndef process_dicom(path):\n    # Reading each data file\n    dcm = pydicom.dcmread(path, stop_before_pixels=True)\n    # Extracting the series instance ID which allows us to group the 2D data to 3D volumes\n    series_id = dcm.SeriesInstanceUID\n    # Extracting the file ID for easy lookup while merging the 2D data to 3D volumes\n    dicom_id = os.path.basename(path).replace(\".dcm\", \"\")\n    # Returning the extracted tuple\n    return series_id, dicom_id\n\nprint(f\"Processing {len(dicom_paths)} DICOM files in parallel.\")\n\n# Using ThreadPoolExecutor for parallel processing\nwith ThreadPoolExecutor(max_workers=8) as executor:\n    # Scheduling all files and returns results in order using executor.map\n    # Iterating over the tasks and visualizing the progress using tqdm\n    for series_id, dicom_id in tqdm(\n        executor.map(process_dicom, dicom_paths),\n        total=len(dicom_paths),\n        desc=\"Building the dictionaries\"\n    ):\n        # Updating both the dictionaries appropriately\n        volume_counts[series_id] += 1\n        volume_slices[series_id].append(dicom_id)\n\nprint(f\"\\nFound {len(volume_counts)} unique SeriesInstanceUIDs.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Storing the volume dictionaries (Dont run if the .pkl files are available)","metadata":{}},{"cell_type":"code","source":"# Saving the volume_counts dictionary\nwith open(\"volume_counts.pkl\", \"wb\") as f:\n    pickle.dump(volume_counts, f)\n# Saving the volume_slices dictionary\nwith open(\"volume_slices.pkl\", \"wb\") as f:\n    pickle.dump(volume_slices, f)\nprint(\"Dictionaries saved successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Creating the label dictionary (Dont run if the .pkl files are available)","metadata":{}},{"cell_type":"code","source":"# Defining the subtypes\nsubtypes = ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']\n\n# Preprocessing the labels into a dictionary of sets for fast lookup\nlabels_dict = defaultdict(set)\nfor idx, row in tqdm(labels_df.iterrows(), total=len(labels_df), desc=\"Processing all volume labels\"):\n    # Extracting the ID and label value for that row\n    dicom_id, label = row['ID'], row['Label']\n    # If the label is positive\n    if label == 1:\n        # Extracting the dicom ID\n        dicom_base = dicom_id.rsplit('_', 1)[0]\n        # Extracting the subtype\n        subtype = dicom_id.rsplit('_', 1)[-1]\n        # Adding the subtype to the set of that particular dicom ID\n        labels_dict[dicom_base].add(subtype)\n\n# Creating a function to get 6 tuple binary labels\ndef get_binary_labels(dicom_id):\n    present = labels_dict.get(dicom_id, set())\n    return tuple(1 if st in present else 0 for st in subtypes)\n\n# Building the dictionary for the volumes\nvolume_binary_labels = {}\nfor volume_id, dicom_ids in tqdm(volume_slices.items(), desc=\"Converting to binary labels\"):\n    for dicom_id in dicom_ids:\n        volume_binary_labels[dicom_id] = get_binary_labels(dicom_id)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Storing the label dictionaries (Dont run if the .pkl files are available)","metadata":{}},{"cell_type":"code","source":"# Saving the volume_binary_labels dictionary\nwith open(\"volume_binary_labels.pkl\", \"wb\") as f:\n    pickle.dump(volume_binary_labels, f)\nprint(\"Dictionary saved successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Loading all the dictionaries","metadata":{}},{"cell_type":"code","source":"# Loading the volume_counts dictionary\nwith open(\"/kaggle/input/loaded-data/volume_counts.pkl\", \"rb\") as f:\n    volume_counts = pickle.load(f)\n# Loading the volume_slices dictionary\nwith open(\"/kaggle/input/loaded-data/volume_slices.pkl\", \"rb\") as f:\n    volume_slices = pickle.load(f)\n# Loading the volume_binary_labels dictionary\nwith open(\"/kaggle/input/loaded-data/volume_binary_labels.pkl\", \"rb\") as f:\n    volume_binary_labels = pickle.load(f)\nprint(\"Dictionaries loaded successfully!\")\nprint(f\"Number of volumes: {len(volume_counts)}\")\nprint(f\"Number of slices: {sum(volume_counts.values())}\")\nprint(f\"Number of labels: {len(volume_binary_labels)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:51:04.05972Z","iopub.execute_input":"2025-10-09T23:51:04.060009Z","iopub.status.idle":"2025-10-09T23:51:06.12213Z","shell.execute_reply.started":"2025-10-09T23:51:04.059988Z","shell.execute_reply":"2025-10-09T23:51:06.121055Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Visualizing the histogram of the results","metadata":{}},{"cell_type":"code","source":"# Extracting the counts (number of slices per volumes)\ncounts = list(volume_counts.values())\n\n# Plotting the histogram\nplt.figure(figsize=(10, 5))\nplt.hist(counts, bins=50, color='skyblue', edgecolor='black')\nplt.title(\"Histogram of Number of Slices per Volume\")\nplt.xlabel(\"Number of slices in volume\")\nplt.ylabel(\"Number of volumes\")\nplt.grid(axis='y', alpha=0.75)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:51:09.789551Z","iopub.execute_input":"2025-10-09T23:51:09.79062Z","iopub.status.idle":"2025-10-09T23:51:10.363063Z","shell.execute_reply.started":"2025-10-09T23:51:09.790586Z","shell.execute_reply":"2025-10-09T23:51:10.361711Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Verifying how many slices are present in most of the volumes","metadata":{}},{"cell_type":"code","source":"# Count how many volumes have each slice count\nslice_count_freq = Counter(volume_counts.values())\n\n# Setting the threshold\nthreshold = 2000\n\n# Filter slice counts that occur in more than threshold number of volumes\nthreshold_counts = {count: freq for count, freq in slice_count_freq.items() if freq > threshold}\n\nprint(f\"Slice counts occurring in more than {threshold} volumes:\")\nfor count, freq in sorted(threshold_counts.items()):\n    print(f\"{count} slices : {freq} volumes\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:51:12.879943Z","iopub.execute_input":"2025-10-09T23:51:12.880258Z","iopub.status.idle":"2025-10-09T23:51:12.889122Z","shell.execute_reply.started":"2025-10-09T23:51:12.880234Z","shell.execute_reply":"2025-10-09T23:51:12.887917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Viewing the slices and labels for a particular randomly chosen volume","metadata":{}},{"cell_type":"code","source":"# Pick a random volume ID from volume_slices\ntarget_volume_id = random.choice(list(volume_slices.keys()))\n\nprint(f\"Randomly selected volume: {target_volume_id}\")\nprint(f\"Number of slices in this volume: {len(volume_slices[target_volume_id])}\")\n\n# Get DICOM files for the target volume\ndicom_ids = volume_slices[target_volume_id]\ndicom_files = [os.path.join(data_dir, f\"{dicom_id}.dcm\") for dicom_id in dicom_ids]\n\n# Creating a function to get the aggregated label\ndef aggregate_labels_max(dicom_ids):\n    # Taking the max value per suptype as if a hemorrage exists in one slice, it exists in the entire volume too\n    return tuple(np.max([volume_binary_labels[id_] for id_ in dicom_ids],axis=0))\n\n# Creating a function to get the z-position from the ImagePositionPatient parameter from the metadata\ndef get_z_position(fpath):\n    # Reading each data file\n    dcm = pydicom.dcmread(fpath, stop_before_pixels=True)\n    # Extrcating the image position patient data which provides the z-axis (depth / sequencial) position\n    pos = dcm.ImagePositionPatient\n    if pos is not None:\n        # Returning the depth coordinate\n        return float(pos[2])\n    else:\n        # If the depth coordinate is not present then appending it to the end\n        return float('inf')\n\n# Sorting the files / slices by z-position\ndicom_files_sorted = sorted(dicom_files, key=get_z_position)\n\nprint(f\"Aggregated label: {aggregate_labels_max(dicom_ids)}\")\n\n# Plotting all slices in a grid\nn_slices = len(dicom_files_sorted)\nn_cols = 8\nn_rows = (n_slices + n_cols - 1) // n_cols\nplt.figure(figsize=(20, n_rows*2.5))\nfor i, fpath in enumerate(dicom_files_sorted):\n    dcm = pydicom.dcmread(fpath)\n    id_ = os.path.basename(fpath).replace(\".dcm\", \"\")\n    img = dcm.pixel_array\n    plt.subplot(n_rows, n_cols, i+1)\n    plt.imshow(img, cmap='gray')\n    plt.axis('off')\n    plt.title(f\"Slice {i+1} \\n {volume_binary_labels[id_]}\", fontsize=18)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:51:43.080206Z","iopub.execute_input":"2025-10-09T23:51:43.080529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Counting the frequency of the triplet labels","metadata":{}},{"cell_type":"code","source":"# Initializing the dictionary to store the triplets in an aggregated (grouped) manner with respect to their labels\nvolume_triplets = defaultdict(list)\n\nfor series_id, series in tqdm(volume_slices.items(), desc=\"Extracting triplets\"):\n    # Skipping if the volume has less than 3 slices\n    if len(series) < 3:\n        continue\n    i = 0\n    # Scanning all the slices within the volume\n    while i < len(series) - 2:\n        # Checking if three consecutive slices share identical labels\n        l1 = volume_binary_labels[series[i]][:-1]\n        l2 = volume_binary_labels[series[i + 1]][:-1]\n        l3 = volume_binary_labels[series[i + 2]][:-1]\n        # If they do share the same labels then we consider them\n        if l1 == l2 == l3:\n            volume_triplets[tuple(l1)].append([series[i], series[i + 1], series[i + 2]])\n            # Jumping ahead by 3 to avoid overlap\n            i += 3\n        else:\n            # If they do not share the same labels then we keep sweeping\n            i += 1\n\nprint(f\"Found {sum(len(v) for v in volume_triplets.values())} volumes with valid triplets.\")\nprint(f\"Unique label configurations: {len(volume_triplets)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:51:59.832477Z","iopub.execute_input":"2025-10-09T23:51:59.832854Z","iopub.status.idle":"2025-10-09T23:52:00.85829Z","shell.execute_reply.started":"2025-10-09T23:51:59.83283Z","shell.execute_reply":"2025-10-09T23:52:00.857064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(volume_triplets.keys())\nprint([len(v) for v in volume_triplets.values()])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:52:05.374463Z","iopub.execute_input":"2025-10-09T23:52:05.374811Z","iopub.status.idle":"2025-10-09T23:52:05.381135Z","shell.execute_reply.started":"2025-10-09T23:52:05.374786Z","shell.execute_reply":"2025-10-09T23:52:05.379707Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sampling triplets in a stratified manner","metadata":{}},{"cell_type":"code","source":"# Defining the desired target sample sizes per category\ntargets = {\n    (1,0,0,0,0): 85,  # epidural\n    (0,1,0,0,0): 300, # intraparenchymal\n    (0,0,1,0,0): 300, # intraventricular\n    (0,0,0,1,0): 300, # subarachnoid\n    (0,0,0,0,1): 300, # subdural\n    'multiple': 500,  # multiple hemorrhage types\n    (0,0,0,0,0): 500  # no hemorrhage\n}\n\n# Initializing the subset dictionary\nbalanced_triplets = defaultdict(list)\n\n# Iterating over the target configurations\nfor label, target_count in tqdm(targets.items(), desc=\"Selecting balanced triplets\"):\n    # Selecting candidate triplets based on label type\n    if label == 'multiple':\n        # Labels having more than one positive subtype\n        candidates = [t for k, v in volume_triplets.items() if sum(k) > 1 for t in v]\n        key = 'multiple'\n    else:\n        # Directly looking up for single-label or none cases\n        candidates = volume_triplets.get(label, [])\n        key = label\n    # Randomly choosing the triplets\n    sampled = random.sample(candidates, target_count)\n    # Storing the selected triplets under the correct key\n    balanced_triplets[key].extend(sampled)\n\n# Print a quick summary\nprint(\"\\n Balanced subset created successfully!\")\nfor k, v in balanced_triplets.items():\n    label_name = k if k == 'multiple' else str(k)\n    print(f\"Label {label_name}: {len(v)} triplets\")\n\ntotal_triplets = sum(len(v) for v in balanced_triplets.values())\nprint(f\"Total selected triplets: {total_triplets}\")\nprint(f\"Total selected slices: {total_triplets*3}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:52:09.650426Z","iopub.execute_input":"2025-10-09T23:52:09.65083Z","iopub.status.idle":"2025-10-09T23:52:09.666961Z","shell.execute_reply.started":"2025-10-09T23:52:09.650804Z","shell.execute_reply":"2025-10-09T23:52:09.665842Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Saving the sampled (subset) data","metadata":{}},{"cell_type":"code","source":"# Defining the root output directory\noutput_root = \"balanced_triplets\"\nos.makedirs(output_root, exist_ok=True)\n\n# Mapping the triplet to its label\ntriplet_label = {}\n\n# Initializing the index for the triplets\nidx = 0\n\n# Iterating through each label category\nfor label, triplets in tqdm(balanced_triplets.items(), desc=\"Saving triplets\"):\n    # Making a label-specific subfolder, if it does not exist\n    label_dir = os.path.join(output_root, str(label))\n    os.makedirs(label_dir, exist_ok=True)\n    # Iterating through all triplets in this label\n    for triplet in triplets:\n        triplet_id = f\"triplet_{idx:03d}\"\n        # Making a triplet-specific sub-subfolder, if it does not exist\n        triplet_dir = os.path.join(label_dir, triplet_id)\n        os.makedirs(triplet_dir, exist_ok=True)\n        # Copying the DICOM files for this triplet\n        for slice_id in triplet:\n            src = os.path.join(data_dir, f\"{slice_id}.dcm\")\n            dst = os.path.join(triplet_dir, f\"{slice_id}.dcm\")\n            shutil.copy2(src, dst)\n        # Storing the label for the triplet\n        triplet_label[triplet_id] = label\n        # Incrementing the triplet index\n        idx+=1\n\n# Saving the mapping as JSON\njson_path = os.path.join(output_root, \"labels.json\")\nwith open(json_path, \"w\") as f:\n    json.dump(triplet_label, f, indent=2)\n\n# Zipping everything allowing us to export it\nshutil.make_archive(\"triplets\", 'zip', output_root)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T23:52:22.024725Z","iopub.execute_input":"2025-10-09T23:52:22.02503Z","iopub.status.idle":"2025-10-10T00:00:22.167954Z","shell.execute_reply.started":"2025-10-09T23:52:22.02501Z","shell.execute_reply":"2025-10-10T00:00:22.166777Z"}},"outputs":[],"execution_count":null}]}