{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":99552,"databundleVersionId":13190393,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CTA Precaching for RSNA 2025 Intracranial Aneurysm Detection\n\nThis notebook will precahe the CTA dicoms to tensor for quicker import to GPUs.\n\n📦 **Dataset available here:**  \n🔗 [RSNA 2025 IA CTA 224 Tensors – on Kaggle](https://www.kaggle.com/datasets/dennisfong/rsna-2025-ia-cta-224-tensors)\n\n---\n\nWhole Process from precache, training to inference:\n\n📦 **Precaching Notebook available here:**  \n🔗 [2.5D EfficientNet - RSNA CTA - Precache Tensor](https://www.kaggle.com/code/dennisfong/2-5d-efficientnet-rsna-cta-precache-tensor)\n\n📦 **Training Notebook available here:**  \n🔗 [2.5D EfficientNet - RSNA CTA - Training (You are reading)](https://www.kaggle.com/code/dennisfong/2-5d-efficientnet-rsna-cta-training)\n\n📦 **Inference Notebook available here:**  \n🔗 [2.5D EfficientNet - RSNA CTA - Inference](https://www.kaggle.com/code/dennisfong/2-5d-efficientnet-rsna-cta-inference)\n\n---\n\n# 🚀 Help Others Discover This Work!\n\n## 👍 If the notebooks or dataset were helpful, **please give it an upvote**!\n\nYour support is appreciated and keeps the community growing. 🙌\n\n## 🌟 Please **UPVOTE** if you found it helpful!\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"# Config and Import Libraries","metadata":{}},{"cell_type":"code","source":"# Parallel precache RSNA series to .npy (image CHW float16 + coords float32)\n# - Train coords from train_localizers.csv (mid-slice SOPInstanceUID). Missing → zeros\n# - Test coords zeros\n# - Image: 2.5D (5 slices), per-series normalized, resized, CHW float16\n# - Uses all CPUs via ProcessPoolExecutor (caps per-worker threads)\n# - Cache dir: /kaggle/working/cache\n\nimport os\nimport ast\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom tqdm.auto import tqdm\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nfrom typing import Sequence\n\n# ========= Config =========\nIMG_SIZE = 224 #456 for EfficientNetB5 , 224 for EfficientNetB0\nSERIES_ROOT_TRAIN = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series\"\nSERIES_ROOT_TEST  = \"/kaggle/input/rsna-intracranial-aneurysm-detection/test/series\"\nTRAIN_CSV         = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\nLOCALIZER_CSV     = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train_localizers.csv\"\nCACHE_DIR_5c      = \"/kaggle/working/cache\"\nos.makedirs(CACHE_DIR_5c, exist_ok=True)\n\nENABLE_ZIP_IMG = True\nALL_SLICES = True #True for all slices in a series, otherwise just 5 slices\n\nDEBUG = True #True for precaching 5 series","metadata":{"execution":{"iopub.status.busy":"2025-08-14T06:57:53.386008Z","iopub.execute_input":"2025-08-14T06:57:53.386366Z","iopub.status.idle":"2025-08-14T06:57:53.393084Z","shell.execute_reply.started":"2025-08-14T06:57:53.386336Z","shell.execute_reply":"2025-08-14T06:57:53.392269Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load CSV","metadata":{}},{"cell_type":"code","source":"\n\n# Load the CSV file\ntrain_df = pd.read_csv(TRAIN_CSV)\n\n# Check the value counts for the 'Modality' column\ntrain_df['Modality'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2025-08-14T06:57:59.838385Z","iopub.execute_input":"2025-08-14T06:57:59.83873Z","iopub.status.idle":"2025-08-14T06:57:59.89123Z","shell.execute_reply.started":"2025-08-14T06:57:59.838701Z","shell.execute_reply":"2025-08-14T06:57:59.890485Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Set the number of CPUs for the precaching job","metadata":{}},{"cell_type":"code","source":"\n# Cap intra-op threads inside workers to avoid oversubscription\nos.environ[\"OMP_NUM_THREADS\"] = \"1\"\nos.environ[\"MKL_NUM_THREADS\"] = \"1\"\ntry:\n    cv2.setNumThreads(0)\nexcept Exception:\n    pass\n\nNUM_CPUS = os.cpu_count() or 2\nMAX_WORKERS = max(1, NUM_CPUS )  # NUM_CPUS - 1 to keep 1 core free","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T06:57:59.892486Z","iopub.execute_input":"2025-08-14T06:57:59.892694Z","iopub.status.idle":"2025-08-14T06:57:59.901318Z","shell.execute_reply.started":"2025-08-14T06:57:59.892676Z","shell.execute_reply":"2025-08-14T06:57:59.900593Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define some helper functions","metadata":{}},{"cell_type":"code","source":"\n# ========= Helpers =========\ndef cache_paths(sid: str):\n    img_ext = \"npz\" if ENABLE_ZIP_IMG else \"npy\"\n    return (\n        os.path.join(CACHE_DIR_5c, f\"{sid}_img.{img_ext}\"),\n        os.path.join(CACHE_DIR_5c, f\"{sid}_coords.npy\"),\n    )\n\ndef cache_paths_5c(sid: str):\n    img_ext = \"npz\" if ENABLE_ZIP_IMG else \"npy\"\n    return (\n        os.path.join(CACHE_DIR_5c, f\"{sid}_img.{img_ext}\"),\n        os.path.join(CACHE_DIR_5c, f\"{sid}_coords.npy\"),\n    )\n\ndef sort_dicom_slices(filepaths):\n    dicoms = [pydicom.dcmread(fp, force=True) for fp in filepaths]\n    try:\n        dicoms.sort(key=lambda d: float(d.ImagePositionPatient[2]))\n    except Exception:\n        dicoms.sort(key=lambda d: int(getattr(d, \"InstanceNumber\", 0)))\n    return dicoms\n\ndef parse_coordinates(coord_str: str):\n    d = ast.literal_eval(coord_str)\n    return float(d[\"x\"]), float(d[\"y\"])\n\ndef normalize_and_save_coords(dicoms, loc_row, coord_path):\n    import numpy as np\n    if len(loc_row) > 0:\n        coords = loc_row['coordinates'].values[0].astype(np.float32)\n    else:\n        coords = np.array([0.0, 0.0], dtype=np.float32)\n\n    # Use middle slice geometry\n    H0, W0 = dicoms[len(dicoms)//2].pixel_array.shape\n    # If coords look like pixels (>1), convert to normalized [0,1]\n    if np.max(coords) > 1.0:\n        coords = np.array([coords[0] / W0, coords[1] / H0], dtype=np.float32)\n\n    # Clamp to [0,1]\n    coords = np.clip(coords, 0.0, 1.0)\n    np.save(coord_path, coords)\n\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-08-14T06:57:59.902176Z","iopub.execute_input":"2025-08-14T06:57:59.902424Z","iopub.status.idle":"2025-08-14T06:57:59.916863Z","shell.execute_reply.started":"2025-08-14T06:57:59.902404Z","shell.execute_reply":"2025-08-14T06:57:59.916064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ========= Metadata =========\n#train_df = pd.read_csv(TRAIN_CSV)\ntrain_df = train_df[train_df[\"Modality\"] == \"CTA\"].reset_index(drop=True)\ntotal_count = len(train_df[\"SeriesInstanceUID\"].astype(str).unique().tolist())\nprint(total_count)\n\ntrain_sids = sorted(train_df[\"SeriesInstanceUID\"].astype(str).unique().tolist())\nif DEBUG:\n    train_sids = train_sids[:10]\n\n\ntest_sids = []\nif os.path.isdir(SERIES_ROOT_TEST):\n    test_sids = sorted([d for d in os.listdir(SERIES_ROOT_TEST) if os.path.isdir(os.path.join(SERIES_ROOT_TEST, d))])\n\n# SOPInstanceUID -> (x, y) mapping for fast lookup in workers\ndef to_xy_array(val):\n    # handle NaN\n    if pd.isna(val):\n        return np.array([0.0, 0.0], dtype=np.float32)\n\n    obj = val\n    if isinstance(val, str):\n        s = val.strip()\n        if s == \"\":\n            return np.array([0.0, 0.0], dtype=np.float32)\n        try:\n            obj = ast.literal_eval(s)\n        except Exception:\n            return np.array([0.0, 0.0], dtype=np.float32)\n\n    if isinstance(obj, dict):\n        x = obj.get(\"x\") if \"x\" in obj else obj.get(\"X\")\n        y = obj.get(\"y\") if \"y\" in obj else obj.get(\"Y\")\n        if x is None or y is None:\n            return np.array([0.0, 0.0], dtype=np.float32)\n        return np.array([float(x), float(y)], dtype=np.float32)\n\n    if isinstance(obj, (list, tuple, np.ndarray)) and len(obj) >= 2:\n        return np.array([float(obj[0]), float(obj[1])], dtype=np.float32)\n\n    return np.array([0.0, 0.0], dtype=np.float32)\n\nlocalizer_df = pd.read_csv(LOCALIZER_CSV)\nlocalizer_df[\"coordinates\"] = localizer_df[\"coordinates\"].map(to_xy_array)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T06:57:59.918531Z","iopub.execute_input":"2025-08-14T06:57:59.918808Z","iopub.status.idle":"2025-08-14T06:58:00.024589Z","shell.execute_reply.started":"2025-08-14T06:57:59.918785Z","shell.execute_reply":"2025-08-14T06:58:00.02367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Define precaching worker functions","metadata":{}},{"cell_type":"code","source":"# ========= Worker =========\ndef precache_one(sid: str, is_train: bool, offsets: Sequence[int]) -> tuple[str, bool, str | None]:\n    try:\n        # choose cache path fn by number of slices requested\n        use_5c = len(offsets) == 5\n        cache_paths_fn = cache_paths_5c if use_5c else cache_paths\n\n        img_path, coord_path = cache_paths_fn(sid)\n        \n        if os.path.exists(img_path) and os.path.exists(coord_path):\n            return sid, True, None\n\n        series_root = SERIES_ROOT_TRAIN if is_train else SERIES_ROOT_TEST\n        series_path = os.path.join(series_root, sid)\n        if not os.path.isdir(series_path):\n            return sid, False, f\"missing series folder\"\n\n        dcm_files = [os.path.join(series_path, f) for f in os.listdir(series_path) if f.endswith(\".dcm\")]\n        if not dcm_files:\n            return sid, False, \"no dicoms\"\n\n        dicoms = sort_dicom_slices(dcm_files)\n        n = len(dicoms)\n        mid = n // 2\n\n        # Map SOP -> slice index\n        sop_to_idx = {str(getattr(d, \"SOPInstanceUID\", \"\")): i for i, d in enumerate(dicoms)}\n\n        # Default center: series mid\n        center_idx = mid\n        picked_coords = None\n\n        if is_train and os.path.exists(LOCALIZER_CSV):\n            # localizers for this series and present in this series' SOPs\n            cands = localizer_df[\n                (localizer_df[\"SeriesInstanceUID\"].astype(str) == sid) &\n                (localizer_df[\"SOPInstanceUID\"].astype(str).isin(sop_to_idx.keys()))\n            ].copy()\n\n            if len(cands) > 0:\n                # If multiple, pick the one nearest series mid\n                cands[\"idx\"] = cands[\"SOPInstanceUID\"].astype(str).map(sop_to_idx)\n                pick = cands.iloc[(cands[\"idx\"] - mid).abs().argmin()]\n                center_idx = int(pick[\"idx\"])\n\n                coords = pick[\"coordinates\"].astype(np.float32)  # already parsed to [x,y]\n                H0, W0 = dicoms[center_idx].pixel_array.shape\n                if np.max(coords) > 1.0:  # pixel -> normalized\n                    coords = np.array([coords[0] / W0, coords[1] / H0], dtype=np.float32)\n                picked_coords = np.clip(coords, 0.0, 1.0)\n\n        # Build 5-slice indices around the chosen center\n        if offsets is None:\n            offsets = (-2, -1, 0, 1, 2)\n        idxs = [min(max(0, center_idx + o), n - 1) for o in offsets]\n            \n        #idxs = [max(0, mid - 1), mid, min(len(dicoms) - 1, mid + 1)]\n        \n        # Build resized slice list to ensure consistent HxW before stacking\n        slices_resized = []\n        indices = range(len(dicoms)) if ALL_SLICES else idxs\n        for i in indices:\n            arr = dicoms[i].pixel_array\n            if arr is None or arr.size == 0:\n                return sid, False, \"empty pixel_array\"\n            arr = arr.astype(np.float32)\n            arr_resized = cv2.resize(arr, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_AREA)\n            slices_resized.append(arr_resized)\n\n        if len(slices_resized) == 0:\n            return sid, False, \"no slices\"\n\n        img = np.stack(slices_resized, axis=-1)  # H,W,C (C = num slices used)\n        img = (img - img.mean()) / (img.std() + 1e-6)\n\n        img_chw = np.transpose(img, (2, 0, 1)).astype(np.float16)  # CHW float16\n        if ENABLE_ZIP_IMG:\n            np.savez_compressed(img_path, img_chw)\n        else:\n            np.save(img_path, img_chw)\n\n\n        # Save coords: prefer picked_coords (centered on localizer slice); else fallback to mid-slice localizer; else zeros\n        if picked_coords is not None:\n            np.save(coord_path, picked_coords.astype(np.float32))\n        else:\n            mid_sop = str(getattr(dicoms[mid], \"SOPInstanceUID\", \"\"))\n            if is_train and os.path.exists(LOCALIZER_CSV):\n                loc_row = localizer_df[localizer_df[\"SOPInstanceUID\"].astype(str) == mid_sop]\n                normalize_and_save_coords(dicoms, loc_row, coord_path)\n            else:\n                np.save(coord_path, np.array([0.0, 0.0], dtype=np.float32))\n\n        \"\"\"\n        if is_train and sop_to_coords:\n            sop = getattr(dicoms[mid], \"SOPInstanceUID\", None)\n            if sop is not None and str(sop) in sop_to_coords:\n                cx, cy = sop_to_coords[str(sop)]\n                coords = np.array([cx, cy], dtype=np.float32)\n            else:\n                coords = np.array([0.0, 0.0], dtype=np.float32)\n        else:\n            coords = np.array([0.0, 0.0], dtype=np.float32)\n        \"\"\"\n\n        \n        #np.save(coord_path, coords)\n        return sid, True, None\n    except Exception as e:\n        return sid, False, str(e)\n\ndef parallel_precache(sids, is_train: bool, offsets, desc: str):\n    failures = []\n    with ProcessPoolExecutor(max_workers=MAX_WORKERS) as ex:\n        futures = {ex.submit(precache_one, sid, is_train, offsets): sid for sid in sids}\n        for fut in tqdm(as_completed(futures), total=len(futures), desc=desc):\n            sid, ok, err = fut.result()\n            if not ok:\n                failures.append((sid, err))\n    return failures\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T06:58:00.025867Z","iopub.execute_input":"2025-08-14T06:58:00.026159Z","iopub.status.idle":"2025-08-14T06:58:00.044111Z","shell.execute_reply.started":"2025-08-14T06:58:00.026111Z","shell.execute_reply":"2025-08-14T06:58:00.04333Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Precache Running","metadata":{}},{"cell_type":"code","source":"# ========= Run =========\nprint(f\"CPUs: {NUM_CPUS}, workers: {MAX_WORKERS}\")\nprint(f\"Caching to: {CACHE_DIR_5c}\")\n\ntrain_fail = parallel_precache(train_sids, is_train=True, offsets = (-2, -1, 0, 1, 2), desc=\"train cache\")\nif test_sids:\n    test_fail  = parallel_precache(test_sids,  is_train=False, offsets = (-2, -1, 0, 1, 2), desc=\"test cache\")\nelse:\n    test_fail = []\n\nprint(f\"Done. Train failures: {len(train_fail)} | Test failures: {len(test_fail)}\")\nif train_fail[:5]:\n    print(\"Sample train failures:\", train_fail[:5])\nif test_fail[:5]:\n    print(\"Sample test failures:\", test_fail[:5])\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T06:58:00.044916Z","iopub.execute_input":"2025-08-14T06:58:00.045164Z","iopub.status.idle":"2025-08-14T06:58:54.335221Z","shell.execute_reply.started":"2025-08-14T06:58:00.045109Z","shell.execute_reply":"2025-08-14T06:58:54.334053Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Quick sanity check","metadata":{}},{"cell_type":"code","source":"\nif train_sids:\n    s = train_sids[0]\n    ip, cp = cache_paths(s)\n    if os.path.exists(ip) and os.path.exists(cp):\n        if ENABLE_ZIP_IMG and ip.endswith('.npz'):\n            z = np.load(ip)\n            a = z[list(z.files)[0]]  # stored array in .npz\n        else:\n            a = np.load(ip, mmap_mode=\"r\")\n        c = np.load(cp)\n        print(f\"Sample {s}: image {a.shape} {a.dtype}, coords {c.shape} {c.dtype}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-14T06:58:54.337434Z","iopub.execute_input":"2025-08-14T06:58:54.3378Z","iopub.status.idle":"2025-08-14T06:58:54.564761Z","shell.execute_reply.started":"2025-08-14T06:58:54.337773Z","shell.execute_reply":"2025-08-14T06:58:54.563501Z"}},"outputs":[],"execution_count":null}]}