{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:47.614867Z","iopub.execute_input":"2026-05-16T12:19:47.617365Z","iopub.status.idle":"2026-05-16T12:19:54.942024Z","shell.execute_reply.started":"2026-05-16T12:19:47.617292Z","shell.execute_reply":"2026-05-16T12:19:54.940559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport ast\nimport time\nimport json\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\nimport os\nimport shutil\n\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:54.944439Z","iopub.execute_input":"2026-05-16T12:19:54.944747Z","iopub.status.idle":"2026-05-16T12:19:54.954631Z","shell.execute_reply.started":"2026-05-16T12:19:54.944715Z","shell.execute_reply":"2026-05-16T12:19:54.953755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RESET_CACHE = True\n\nif RESET_CACHE:\n    for folder in [\"cache\", \"xai_cases\"]:\n        if os.path.exists(folder):\n            shutil.rmtree(folder)\n\nos.makedirs(\"cache/train\", exist_ok=True)\nos.makedirs(\"cache/val\", exist_ok=True)\nos.makedirs(\"cache/test\", exist_ok=True)\nos.makedirs(\"xai_cases\", exist_ok=True)\n\nprint(\"CACHE READY\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\nRESET_CACHE = True\n\nif RESET_CACHE:\n    for folder in [\"cache\", \"xai_cases\"]:\n        if os.path.exists(folder):\n            shutil.rmtree(folder)\n\nos.makedirs(\"cache/train\", exist_ok=True)\nos.makedirs(\"cache/val\", exist_ok=True)\nos.makedirs(\"cache/test\", exist_ok=True)\nos.makedirs(\"xai_cases\", exist_ok=True)\n\nprint(\"CACHE READY\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\"\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\n\nIMG_SIZE = 384\nNUM_CLASSES = 15\nRANDOM_STATE = 42\n\nCACHE_DIR = \"cache\"\nTRAIN_CACHE_DIR = os.path.join(CACHE_DIR, \"train\")\nVAL_CACHE_DIR = os.path.join(CACHE_DIR, \"val\")\nTEST_CACHE_DIR = os.path.join(CACHE_DIR, \"test\")\n\nXAI_DIR = \"xai_cases\"\n\nos.makedirs(TRAIN_CACHE_DIR, exist_ok=True)\nos.makedirs(VAL_CACHE_DIR, exist_ok=True)\nos.makedirs(TEST_CACHE_DIR, exist_ok=True)\nos.makedirs(XAI_DIR, exist_ok=True)\n\nprint(\"READY\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:54.955938Z","iopub.execute_input":"2026-05-16T12:19:54.956486Z","iopub.status.idle":"2026-05-16T12:19:54.982634Z","shell.execute_reply.started":"2026-05-16T12:19:54.956453Z","shell.execute_reply":"2026-05-16T12:19:54.981791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASS_NAMES = [\n    \"Aortic enlargement\",\n    \"Atelectasis\",\n    \"Calcification\",\n    \"Cardiomegaly\",\n    \"Consolidation\",\n    \"ILD\",\n    \"Infiltration\",\n    \"Lung Opacity\",\n    \"Nodule/Mass\",\n    \"Other lesion\",\n    \"Pleural effusion\",\n    \"Pleural thickening\",\n    \"Pneumothorax\",\n    \"Pulmonary fibrosis\",\n    \"No finding\"\n]\n\nCLASS_TO_ID = {name: idx for idx, name in enumerate(CLASS_NAMES)}\nID_TO_CLASS = {idx: name for idx, name in enumerate(CLASS_NAMES)}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:54.984867Z","iopub.execute_input":"2026-05-16T12:19:54.985149Z","iopub.status.idle":"2026-05-16T12:19:55.001437Z","shell.execute_reply.started":"2026-05-16T12:19:54.985123Z","shell.execute_reply":"2026-05-16T12:19:55.000324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\n\ndf = df[df[\"class_id\"].between(0, 14)].reset_index(drop=True)\n\nprint(df.shape)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:55.002833Z","iopub.execute_input":"2026-05-16T12:19:55.003112Z","iopub.status.idle":"2026-05-16T12:19:55.246342Z","shell.execute_reply.started":"2026-05-16T12:19:55.003085Z","shell.execute_reply":"2026-05-16T12:19:55.245482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"grouped = df.groupby(\"image_id\")[\"class_id\"].apply(\n    lambda x: sorted(list(set(x)))\n).reset_index(name=\"class_id\")\n\ndef fix_no_finding(labels):\n    labels = set(labels)\n    if 14 in labels and len(labels) > 1:\n        labels.remove(14)\n    return sorted(list(labels))\n\ngrouped[\"class_id\"] = grouped[\"class_id\"].apply(fix_no_finding)\n\nprint(\"TOTAL IMAGES:\", len(grouped))\ngrouped.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:55.247604Z","iopub.execute_input":"2026-05-16T12:19:55.247964Z","iopub.status.idle":"2026-05-16T12:19:56.198006Z","shell.execute_reply.started":"2026-05-16T12:19:55.247928Z","shell.execute_reply":"2026-05-16T12:19:56.197078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def multi_hot(labels):\n    target = np.zeros(NUM_CLASSES, dtype=np.float32)\n    for label in labels:\n        target[int(label)] = 1.0\n    return target\n\ngrouped[\"target\"] = grouped[\"class_id\"].apply(\n    lambda x: multi_hot(x).tolist()\n)\n\nbad_no_finding = grouped[\n    grouped[\"class_id\"].apply(lambda x: 14 in x and len(x) > 1)\n]\n\nprint(\"NO FINDING CONFLICT:\", len(bad_no_finding))\ngrouped.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:56.198927Z","iopub.execute_input":"2026-05-16T12:19:56.199178Z","iopub.status.idle":"2026-05-16T12:19:56.254634Z","shell.execute_reply.started":"2026-05-16T12:19:56.199153Z","shell.execute_reply":"2026-05-16T12:19:56.253673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y = np.array(grouped[\"target\"].tolist())\n\ntry:\n    from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\n\n    splitter_1 = MultilabelStratifiedShuffleSplit(\n        n_splits=1,\n        test_size=0.30,\n        random_state=RANDOM_STATE\n    )\n\n    train_idx, temp_idx = next(splitter_1.split(grouped, Y))\n\n    train_df = grouped.iloc[train_idx].reset_index(drop=True)\n    temp_df = grouped.iloc[temp_idx].reset_index(drop=True)\n\n    Y_temp = np.array(temp_df[\"target\"].tolist())\n\n    splitter_2 = MultilabelStratifiedShuffleSplit(\n        n_splits=1,\n        test_size=0.50,\n        random_state=RANDOM_STATE\n    )\n\n    val_idx, test_idx = next(splitter_2.split(temp_df, Y_temp))\n\n    val_df = temp_df.iloc[val_idx].reset_index(drop=True)\n    test_df = temp_df.iloc[test_idx].reset_index(drop=True)\n\n    split_method = \"multilabel_stratified\"\n\nexcept Exception as e:\n    train_df, temp_df = train_test_split(\n        grouped,\n        test_size=0.30,\n        random_state=RANDOM_STATE\n    )\n\n    val_df, test_df = train_test_split(\n        temp_df,\n        test_size=0.50,\n        random_state=RANDOM_STATE\n    )\n\n    train_df = train_df.reset_index(drop=True)\n    val_df = val_df.reset_index(drop=True)\n    test_df = test_df.reset_index(drop=True)\n\n    split_method = \"random\"\n\nprint(\"SPLIT METHOD:\", split_method)\nprint(\"TRAIN:\", len(train_df))\nprint(\"VAL:\", len(val_df))\nprint(\"TEST:\", len(test_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:56.256057Z","iopub.execute_input":"2026-05-16T12:19:56.256445Z","iopub.status.idle":"2026-05-16T12:19:56.70726Z","shell.execute_reply.started":"2026-05-16T12:19:56.256413Z","shell.execute_reply":"2026-05-16T12:19:56.706353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_label_distribution(dataframe):\n    targets = np.array(dataframe[\"target\"].tolist())\n    counts = targets.sum(axis=0).astype(int)\n\n    result = pd.DataFrame({\n        \"class_id\": list(range(NUM_CLASSES)),\n        \"class_name\": CLASS_NAMES,\n        \"count\": counts\n    })\n\n    result[\"ratio\"] = result[\"count\"] / len(dataframe)\n\n    return result\n\ntrain_dist = get_label_distribution(train_df)\nval_dist = get_label_distribution(val_df)\ntest_dist = get_label_distribution(test_df)\n\ntrain_dist.to_csv(\"train_label_distribution.csv\", index=False)\nval_dist.to_csv(\"val_label_distribution.csv\", index=False)\ntest_dist.to_csv(\"test_label_distribution.csv\", index=False)\n\nprint(\"TRAIN DISTRIBUTION\")\ndisplay(train_dist)\n\nprint(\"VAL DISTRIBUTION\")\ndisplay(val_dist)\n\nprint(\"TEST DISTRIBUTION\")\ndisplay(test_dist)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:56.70833Z","iopub.execute_input":"2026-05-16T12:19:56.70879Z","iopub.status.idle":"2026-05-16T12:19:56.777754Z","shell.execute_reply.started":"2026-05-16T12:19:56.708733Z","shell.execute_reply":"2026-05-16T12:19:56.776952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def expand_targets(dataframe):\n    output = dataframe[[\"image_id\", \"class_id\", \"target\"]].copy()\n    targets = np.array(output[\"target\"].tolist())\n\n    for idx, class_name in enumerate(CLASS_NAMES):\n        output[class_name] = targets[:, idx].astype(int)\n\n    return output\n\ntrain_split_expanded = expand_targets(train_df)\nval_split_expanded = expand_targets(val_df)\ntest_split_expanded = expand_targets(test_df)\n\ntrain_df.to_csv(\"train_split.csv\", index=False)\nval_df.to_csv(\"val_split.csv\", index=False)\ntest_df.to_csv(\"test_split.csv\", index=False)\n\ntrain_split_expanded.to_csv(\"train_split_expanded.csv\", index=False)\nval_split_expanded.to_csv(\"val_split_expanded.csv\", index=False)\ntest_split_expanded.to_csv(\"test_split_expanded.csv\", index=False)\n\nsplit_info = {\n    \"split_method\": split_method,\n    \"random_state\": RANDOM_STATE,\n    \"img_size\": IMG_SIZE,\n    \"num_classes\": NUM_CLASSES,\n    \"total_images\": len(grouped),\n    \"train_images\": len(train_df),\n    \"val_images\": len(val_df),\n    \"test_images\": len(test_df)\n}\n\nwith open(\"split_info.json\", \"w\") as f:\n    json.dump(split_info, f, indent=4)\n\nprint(\"SPLIT FILES SAVED\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:56.781118Z","iopub.execute_input":"2026-05-16T12:19:56.781539Z","iopub.status.idle":"2026-05-16T12:19:57.121974Z","shell.execute_reply.started":"2026-05-16T12:19:56.781511Z","shell.execute_reply":"2026-05-16T12:19:57.120995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def auto_crop_thorax(img):\n    original_h, original_w = img.shape[:2]\n\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    blur = cv2.GaussianBlur(gray, (5, 5), 0)\n\n    _, thresh = cv2.threshold(\n        blur,\n        0,\n        255,\n        cv2.THRESH_BINARY + cv2.THRESH_OTSU\n    )\n\n    contours, _ = cv2.findContours(\n        thresh,\n        cv2.RETR_EXTERNAL,\n        cv2.CHAIN_APPROX_SIMPLE\n    )\n\n    if len(contours) == 0:\n        meta = {\n            \"x1\": 0,\n            \"y1\": 0,\n            \"x2\": original_w,\n            \"y2\": original_h,\n            \"orig_w\": original_w,\n            \"orig_h\": original_h,\n            \"crop_w\": original_w,\n            \"crop_h\": original_h\n        }\n        return img, meta\n\n    cnt = max(contours, key=cv2.contourArea)\n    x, y, w, h = cv2.boundingRect(cnt)\n\n    pad_x = int(w * 0.05)\n    pad_y = int(h * 0.05)\n\n    x1 = max(0, x + pad_x)\n    y1 = max(0, y + pad_y)\n    x2 = min(original_w, x + w - pad_x)\n    y2 = min(original_h, y + h - pad_y)\n\n    if x2 <= x1 or y2 <= y1:\n        x1, y1, x2, y2 = 0, 0, original_w, original_h\n\n    crop = img[y1:y2, x1:x2]\n\n    meta = {\n        \"x1\": int(x1),\n        \"y1\": int(y1),\n        \"x2\": int(x2),\n        \"y2\": int(y2),\n        \"orig_w\": int(original_w),\n        \"orig_h\": int(original_h),\n        \"crop_w\": int(x2 - x1),\n        \"crop_h\": int(y2 - y1)\n    }\n\n    return crop, meta","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:57.123042Z","iopub.execute_input":"2026-05-16T12:19:57.123328Z","iopub.status.idle":"2026-05-16T12:19:57.134963Z","shell.execute_reply.started":"2026-05-16T12:19:57.123303Z","shell.execute_reply":"2026-05-16T12:19:57.133805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_dicom(path):\n    dicom = pydicom.dcmread(path)\n\n    img = apply_voi_lut(dicom.pixel_array, dicom)\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.max(img) - img\n\n    img = img.astype(np.float32)\n\n    lower = np.percentile(img, 1)\n    upper = np.percentile(img, 99)\n\n    img = np.clip(img, lower, upper)\n\n    img = (img - img.min()) / (img.max() - img.min() + 1e-6)\n    img = (img * 255).astype(np.uint8)\n    img = np.stack([img] * 3, axis=-1)\n\n    img, crop_meta = auto_crop_thorax(img)\n\n    crop_h, crop_w = img.shape[:2]\n\n    img = cv2.resize(img, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_AREA)\n\n    resize_meta = {\n        \"resize_w\": IMG_SIZE,\n        \"resize_h\": IMG_SIZE,\n        \"scale_x\": IMG_SIZE / crop_w,\n        \"scale_y\": IMG_SIZE / crop_h\n    }\n\n    meta = {}\n    meta.update(crop_meta)\n    meta.update(resize_meta)\n\n    return img, meta","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:57.136162Z","iopub.execute_input":"2026-05-16T12:19:57.136526Z","iopub.status.idle":"2026-05-16T12:19:57.165905Z","shell.execute_reply.started":"2026-05-16T12:19:57.136493Z","shell.execute_reply":"2026-05-16T12:19:57.164629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_path = os.path.join(\n    TRAIN_DIR,\n    train_df.iloc[0][\"image_id\"] + \".dicom\"\n)\n\nsample_img, sample_meta = read_dicom(sample_path)\n\nprint(sample_img.shape)\nprint(sample_img.min(), sample_img.max())\nprint(sample_meta)\n\nplt.figure(figsize=(6, 6))\nplt.imshow(sample_img)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:57.167454Z","iopub.execute_input":"2026-05-16T12:19:57.169036Z","iopub.status.idle":"2026-05-16T12:19:58.057562Z","shell.execute_reply.started":"2026-05-16T12:19:57.168983Z","shell.execute_reply":"2026-05-16T12:19:58.056363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"samples = train_df.sample(min(16, len(train_df)), random_state=RANDOM_STATE)\n\nplt.figure(figsize=(12, 12))\n\nfor idx, (_, row) in enumerate(samples.iterrows()):\n    path = os.path.join(TRAIN_DIR, row[\"image_id\"] + \".dicom\")\n    img, _ = read_dicom(path)\n\n    plt.subplot(4, 4, idx + 1)\n    plt.imshow(img)\n    plt.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:19:58.058783Z","iopub.execute_input":"2026-05-16T12:19:58.059044Z","iopub.status.idle":"2026-05-16T12:20:16.065038Z","shell.execute_reply.started":"2026-05-16T12:19:58.059019Z","shell.execute_reply":"2026-05-16T12:20:16.063742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"failed_images = []\npreprocess_metadata = []\n\ndef save_sample(row, save_dir, split_name):\n    image_id = row[\"image_id\"]\n\n    dicom_path = os.path.join(\n        TRAIN_DIR,\n        image_id + \".dicom\"\n    )\n\n    save_path = os.path.join(\n        save_dir,\n        image_id + \".npy\"\n    )\n\n    if os.path.exists(save_path):\n        return\n\n    try:\n        img, meta = read_dicom(dicom_path)\n        np.save(save_path, img)\n\n        record = {\n            \"image_id\": image_id,\n            \"split\": split_name,\n            \"save_path\": save_path\n        }\n\n        record.update(meta)\n        preprocess_metadata.append(record)\n\n    except Exception as e:\n        failed_images.append({\n            \"image_id\": image_id,\n            \"split\": split_name,\n            \"error\": str(e)\n        })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:20:16.066577Z","iopub.execute_input":"2026-05-16T12:20:16.066984Z","iopub.status.idle":"2026-05-16T12:20:16.07773Z","shell.execute_reply.started":"2026-05-16T12:20:16.066944Z","shell.execute_reply":"2026-05-16T12:20:16.076701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"start_total = time.time()\n\nprint(\"CACHING TRAIN\")\n\nstart_train = time.time()\n\nfor _, row in tqdm(train_df.iterrows(), total=len(train_df)):\n    save_sample(row, TRAIN_CACHE_DIR, \"train\")\n\ntrain_time = time.time() - start_train\n\nprint(f\"TRAIN DONE | Time: {train_time / 60:.2f} mins\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T12:20:16.078943Z","iopub.execute_input":"2026-05-16T12:20:16.079408Z","iopub.status.idle":"2026-05-16T14:58:18.303069Z","shell.execute_reply.started":"2026-05-16T12:20:16.079373Z","shell.execute_reply":"2026-05-16T14:58:18.299746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"CACHING VAL\")\n\nstart_val = time.time()\n\nfor _, row in tqdm(val_df.iterrows(), total=len(val_df)):\n    save_sample(row, VAL_CACHE_DIR, \"val\")\n\nval_time = time.time() - start_val\n\nprint(f\"VAL DONE | Time: {val_time / 60:.2f} mins\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T14:58:18.310592Z","iopub.execute_input":"2026-05-16T14:58:18.311395Z","iopub.status.idle":"2026-05-16T15:52:59.860469Z","shell.execute_reply.started":"2026-05-16T14:58:18.311348Z","shell.execute_reply":"2026-05-16T15:52:59.857199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"CACHING TEST\")\n\nstart_test = time.time()\n\nfor _, row in tqdm(test_df.iterrows(), total=len(test_df)):\n    save_sample(row, TEST_CACHE_DIR, \"test\")\n\ntest_time = time.time() - start_test\n\nprint(f\"TEST DONE | Time: {test_time / 60:.2f} mins\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T15:52:59.867687Z","iopub.execute_input":"2026-05-16T15:52:59.868482Z","iopub.status.idle":"2026-05-16T16:47:47.316646Z","shell.execute_reply.started":"2026-05-16T15:52:59.868417Z","shell.execute_reply":"2026-05-16T16:47:47.313286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocess_metadata_df = pd.DataFrame(preprocess_metadata)\nfailed_images_df = pd.DataFrame(failed_images)\n\nif len(preprocess_metadata_df) > 0:\n    preprocess_metadata_df.to_csv(\"preprocess_metadata.csv\", index=False)\n\nif len(failed_images_df) > 0:\n    failed_images_df.to_csv(\"failed_preprocessing.csv\", index=False)\nelse:\n    pd.DataFrame(columns=[\"image_id\", \"split\", \"error\"]).to_csv(\n        \"failed_preprocessing.csv\",\n        index=False\n    )\n\nprint(\"FAILED:\", len(failed_images_df))\nprint(\"METADATA:\", len(preprocess_metadata_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:47.324337Z","iopub.execute_input":"2026-05-16T16:47:47.325135Z","iopub.status.idle":"2026-05-16T16:47:47.603376Z","shell.execute_reply.started":"2026-05-16T16:47:47.325069Z","shell.execute_reply":"2026-05-16T16:47:47.60236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_cached_files(cache_dir):\n    return len([name for name in os.listdir(cache_dir) if name.endswith(\".npy\")])\n\ncache_report = pd.DataFrame([\n    {\n        \"split\": \"train\",\n        \"expected\": len(train_df),\n        \"cached\": count_cached_files(TRAIN_CACHE_DIR)\n    },\n    {\n        \"split\": \"val\",\n        \"expected\": len(val_df),\n        \"cached\": count_cached_files(VAL_CACHE_DIR)\n    },\n    {\n        \"split\": \"test\",\n        \"expected\": len(test_df),\n        \"cached\": count_cached_files(TEST_CACHE_DIR)\n    }\n])\n\ncache_report[\"missing\"] = cache_report[\"expected\"] - cache_report[\"cached\"]\n\ncache_report.to_csv(\"cache_report.csv\", index=False)\n\ndisplay(cache_report)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:47.604793Z","iopub.execute_input":"2026-05-16T16:47:47.60528Z","iopub.status.idle":"2026-05-16T16:47:47.669847Z","shell.execute_reply.started":"2026-05-16T16:47:47.605227Z","shell.execute_reply":"2026-05-16T16:47:47.668705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"XAI_CLASSES = [\n    \"Cardiomegaly\",\n    \"Pneumothorax\",\n    \"Pleural effusion\",\n    \"No finding\"\n]\n\nXAI_PER_CLASS = 20\n\nxai_records = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:47.671613Z","iopub.execute_input":"2026-05-16T16:47:47.672324Z","iopub.status.idle":"2026-05-16T16:47:47.677497Z","shell.execute_reply.started":"2026-05-16T16:47:47.672287Z","shell.execute_reply":"2026-05-16T16:47:47.676584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for cls in XAI_CLASSES:\n    cls_idx = CLASS_TO_ID[cls]\n\n    cls_dir = os.path.join(\n        XAI_DIR,\n        cls.replace(\" \", \"_\")\n    )\n\n    os.makedirs(cls_dir, exist_ok=True)\n\n    if cls_idx == 14:\n        subset = test_df[\n            test_df[\"target\"].apply(lambda x: x[cls_idx] == 1)\n        ].copy()\n    else:\n        single_subset = test_df[\n            test_df[\"target\"].apply(\n                lambda x: x[cls_idx] == 1 and np.sum(x) == 1\n            )\n        ].copy()\n\n        multi_subset = test_df[\n            test_df[\"target\"].apply(\n                lambda x: x[cls_idx] == 1 and np.sum(x) > 1\n            )\n        ].copy()\n\n        if len(single_subset) >= XAI_PER_CLASS:\n            subset = single_subset\n        else:\n            need = XAI_PER_CLASS - len(single_subset)\n            extra = multi_subset.sample(\n                min(need, len(multi_subset)),\n                random_state=RANDOM_STATE\n            )\n            subset = pd.concat([single_subset, extra], axis=0)\n\n    subset = subset.sample(\n        min(XAI_PER_CLASS, len(subset)),\n        random_state=RANDOM_STATE\n    ).reset_index(drop=True)\n\n    print(cls, len(subset))\n\n    for _, row in subset.iterrows():\n        image_id = row[\"image_id\"]\n\n        src = os.path.join(\n            TEST_CACHE_DIR,\n            image_id + \".npy\"\n        )\n\n        dst = os.path.join(\n            cls_dir,\n            image_id + \".npy\"\n        )\n\n        if os.path.exists(src):\n            shutil.copy2(src, dst)\n\n            xai_records.append({\n                \"image_id\": image_id,\n                \"xai_class\": cls,\n                \"xai_class_id\": cls_idx,\n                \"target\": row[\"target\"],\n                \"class_id\": row[\"class_id\"],\n                \"num_labels\": int(np.sum(row[\"target\"])),\n                \"source_path\": src,\n                \"xai_path\": dst\n            })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:47.678934Z","iopub.execute_input":"2026-05-16T16:47:47.67936Z","iopub.status.idle":"2026-05-16T16:47:48.226826Z","shell.execute_reply.started":"2026-05-16T16:47:47.679315Z","shell.execute_reply":"2026-05-16T16:47:48.225116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xai_cases_df = pd.DataFrame(xai_records)\n\nxai_cases_df.to_csv(\"xai_cases.csv\", index=False)\n\ndisplay(xai_cases_df.head())\nprint(\"XAI CASES:\", len(xai_cases_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:48.228468Z","iopub.execute_input":"2026-05-16T16:47:48.229101Z","iopub.status.idle":"2026-05-16T16:47:48.257124Z","shell.execute_reply.started":"2026-05-16T16:47:48.229036Z","shell.execute_reply":"2026-05-16T16:47:48.255914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xai_distribution = xai_cases_df.groupby(\n    [\"xai_class\", \"xai_class_id\"]\n).size().reset_index(name=\"count\")\n\nxai_distribution.to_csv(\"xai_distribution.csv\", index=False)\n\ndisplay(xai_distribution)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:48.258806Z","iopub.execute_input":"2026-05-16T16:47:48.259963Z","iopub.status.idle":"2026-05-16T16:47:48.299566Z","shell.execute_reply.started":"2026-05-16T16:47:48.259912Z","shell.execute_reply":"2026-05-16T16:47:48.298572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_time = time.time() - start_total\n\nruntime_report = {\n    \"train_cache_time_minutes\": train_time / 60,\n    \"val_cache_time_minutes\": val_time / 60,\n    \"test_cache_time_minutes\": test_time / 60,\n    \"total_cache_time_hours\": total_time / 3600,\n    \"failed_images\": len(failed_images_df),\n    \"xai_cases\": len(xai_cases_df)\n}\n\nwith open(\"runtime_report.json\", \"w\") as f:\n    json.dump(runtime_report, f, indent=4)\n\nprint(f\"TOTAL CACHE TIME: {total_time / 3600:.2f} hours\")\nprint(\"ALL DONE\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:48.301029Z","iopub.execute_input":"2026-05-16T16:47:48.301371Z","iopub.status.idle":"2026-05-16T16:47:48.313901Z","shell.execute_reply.started":"2026-05-16T16:47:48.301342Z","shell.execute_reply":"2026-05-16T16:47:48.312812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_files = [\n    \"train_split.csv\",\n    \"val_split.csv\",\n    \"test_split.csv\",\n    \"train_split_expanded.csv\",\n    \"val_split_expanded.csv\",\n    \"test_split_expanded.csv\",\n    \"train_label_distribution.csv\",\n    \"val_label_distribution.csv\",\n    \"test_label_distribution.csv\",\n    \"preprocess_metadata.csv\",\n    \"failed_preprocessing.csv\",\n    \"cache_report.csv\",\n    \"xai_cases.csv\",\n    \"xai_distribution.csv\",\n    \"split_info.json\",\n    \"runtime_report.json\"\n]\n\nfor file in output_files:\n    if os.path.exists(file):\n        print(file)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-16T16:47:48.315384Z","iopub.execute_input":"2026-05-16T16:47:48.315704Z","iopub.status.idle":"2026-05-16T16:47:48.336551Z","shell.execute_reply.started":"2026-05-16T16:47:48.315675Z","shell.execute_reply":"2026-05-16T16:47:48.335502Z"}},"outputs":[],"execution_count":null}]}