{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport ast\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\n\nVINBIGDATA_DIR = \"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\"\nVIN_TRAIN_CSV = os.path.join(VINBIGDATA_DIR, \"train.csv\")\n\nOUTPUT_DIR = \"/kaggle/working/box_resized_outputs\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\nIMG_SIZE = 384\nNUM_CLASSES = 15\nNO_FINDING_ID = 14\nRANDOM_STATE = 42\n\nCLASS_NAMES = [\n    \"Aortic enlargement\",\n    \"Atelectasis\",\n    \"Calcification\",\n    \"Cardiomegaly\",\n    \"Consolidation\",\n    \"ILD\",\n    \"Infiltration\",\n    \"Lung Opacity\",\n    \"Nodule/Mass\",\n    \"Other lesion\",\n    \"Pleural effusion\",\n    \"Pleural thickening\",\n    \"Pneumothorax\",\n    \"Pulmonary fibrosis\",\n    \"No finding\"\n]\n\ndef find_file(filename, search_root=\"/kaggle/input\"):\n    matches = glob.glob(os.path.join(search_root, \"**\", filename), recursive=True)\n    if len(matches) == 0:\n        raise FileNotFoundError(f\"Không tìm thấy file: {filename}\")\n    matches = sorted(matches, key=lambda x: len(x))\n    return matches[0]\n\nTRAIN_SPLIT_CSV = find_file(\"train_split.csv\")\nVAL_SPLIT_CSV = find_file(\"val_split.csv\")\nTEST_SPLIT_CSV = find_file(\"test_split.csv\")\nPREPROCESS_META_CSV = find_file(\"preprocess_metadata.csv\")\n\nCACHE_ROOT = os.path.dirname(PREPROCESS_META_CSV)\n\nprint(\"VIN_TRAIN_CSV:\", VIN_TRAIN_CSV)\nprint(\"TRAIN_SPLIT_CSV:\", TRAIN_SPLIT_CSV)\nprint(\"VAL_SPLIT_CSV:\", VAL_SPLIT_CSV)\nprint(\"TEST_SPLIT_CSV:\", TEST_SPLIT_CSV)\nprint(\"PREPROCESS_META_CSV:\", PREPROCESS_META_CSV)\nprint(\"CACHE_ROOT:\", CACHE_ROOT)\n\nif not os.path.exists(VIN_TRAIN_CSV):\n    raise FileNotFoundError(\"Không tìm thấy train.csv gốc của VinBigData. Kiểm tra lại input competition.\")\n\nvin_df = pd.read_csv(VIN_TRAIN_CSV)\ntrain_split = pd.read_csv(TRAIN_SPLIT_CSV)\nval_split = pd.read_csv(VAL_SPLIT_CSV)\ntest_split = pd.read_csv(TEST_SPLIT_CSV)\nmeta_df = pd.read_csv(PREPROCESS_META_CSV)\n\nrequired_vin_cols = [\"image_id\", \"class_name\", \"class_id\", \"rad_id\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\nrequired_meta_cols = [\"image_id\", \"split\", \"x1\", \"y1\", \"x2\", \"y2\", \"orig_w\", \"orig_h\", \"crop_w\", \"crop_h\", \"scale_x\", \"scale_y\"]\n\nfor col in required_vin_cols:\n    if col not in vin_df.columns:\n        raise ValueError(f\"Thiếu cột {col} trong train.csv gốc\")\n\nfor col in required_meta_cols:\n    if col not in meta_df.columns:\n        raise ValueError(f\"Thiếu cột {col} trong preprocess_metadata.csv\")\n\nsplit_map = {}\n\nfor image_id in train_split[\"image_id\"].astype(str).tolist():\n    split_map[image_id] = \"train\"\n\nfor image_id in val_split[\"image_id\"].astype(str).tolist():\n    split_map[image_id] = \"val\"\n\nfor image_id in test_split[\"image_id\"].astype(str).tolist():\n    split_map[image_id] = \"test\"\n\nsplit_df = pd.DataFrame({\n    \"image_id\": list(split_map.keys()),\n    \"split_from_csv\": list(split_map.values())\n})\n\nbox_df = vin_df.copy()\nbox_df[\"image_id\"] = box_df[\"image_id\"].astype(str)\n\nbox_df = box_df[\n    (box_df[\"class_id\"] != NO_FINDING_ID) &\n    (box_df[\"x_min\"].notna()) &\n    (box_df[\"y_min\"].notna()) &\n    (box_df[\"x_max\"].notna()) &\n    (box_df[\"y_max\"].notna())\n].copy()\n\nbox_df[\"class_id\"] = box_df[\"class_id\"].astype(int)\nbox_df[\"class_name\"] = box_df[\"class_id\"].map(lambda x: CLASS_NAMES[x])\n\nbox_df = box_df.merge(split_df, on=\"image_id\", how=\"inner\")\nbox_df = box_df.merge(meta_df, on=\"image_id\", how=\"inner\", suffixes=(\"\", \"_meta\"))\n\nbox_df[\"x_min_384\"] = (box_df[\"x_min\"] - box_df[\"x1\"]) * box_df[\"scale_x\"]\nbox_df[\"y_min_384\"] = (box_df[\"y_min\"] - box_df[\"y1\"]) * box_df[\"scale_y\"]\nbox_df[\"x_max_384\"] = (box_df[\"x_max\"] - box_df[\"x1\"]) * box_df[\"scale_x\"]\nbox_df[\"y_max_384\"] = (box_df[\"y_max\"] - box_df[\"y1\"]) * box_df[\"scale_y\"]\n\nfor col in [\"x_min_384\", \"y_min_384\", \"x_max_384\", \"y_max_384\"]:\n    box_df[col] = box_df[col].clip(0, IMG_SIZE - 1)\n\nbox_df[\"box_w_384\"] = box_df[\"x_max_384\"] - box_df[\"x_min_384\"]\nbox_df[\"box_h_384\"] = box_df[\"y_max_384\"] - box_df[\"y_min_384\"]\nbox_df[\"box_area_384\"] = box_df[\"box_w_384\"] * box_df[\"box_h_384\"]\n\nbefore_filter = len(box_df)\n\nbox_df = box_df[\n    (box_df[\"box_w_384\"] >= 2) &\n    (box_df[\"box_h_384\"] >= 2) &\n    (box_df[\"box_area_384\"] >= 4)\n].copy().reset_index(drop=True)\n\nafter_filter = len(box_df)\n\nkeep_cols = [\n    \"image_id\",\n    \"split_from_csv\",\n    \"class_id\",\n    \"class_name\",\n    \"rad_id\",\n    \"x_min\",\n    \"y_min\",\n    \"x_max\",\n    \"y_max\",\n    \"x_min_384\",\n    \"y_min_384\",\n    \"x_max_384\",\n    \"y_max_384\",\n    \"box_w_384\",\n    \"box_h_384\",\n    \"box_area_384\",\n    \"x1\",\n    \"y1\",\n    \"x2\",\n    \"y2\",\n    \"orig_w\",\n    \"orig_h\",\n    \"crop_w\",\n    \"crop_h\",\n    \"scale_x\",\n    \"scale_y\"\n]\n\nbox_df = box_df[keep_cols].copy()\n\ntrain_boxes = box_df[box_df[\"split_from_csv\"] == \"train\"].copy().reset_index(drop=True)\nval_boxes = box_df[box_df[\"split_from_csv\"] == \"val\"].copy().reset_index(drop=True)\ntest_boxes = box_df[box_df[\"split_from_csv\"] == \"test\"].copy().reset_index(drop=True)\n\nall_boxes_path = os.path.join(OUTPUT_DIR, \"all_boxes_resized_384.csv\")\ntrain_boxes_path = os.path.join(OUTPUT_DIR, \"train_boxes_resized.csv\")\nval_boxes_path = os.path.join(OUTPUT_DIR, \"val_boxes_resized.csv\")\ntest_boxes_path = os.path.join(OUTPUT_DIR, \"test_boxes_resized.csv\")\n\nbox_df.to_csv(all_boxes_path, index=False)\ntrain_boxes.to_csv(train_boxes_path, index=False)\nval_boxes.to_csv(val_boxes_path, index=False)\ntest_boxes.to_csv(test_boxes_path, index=False)\n\nsummary = {\n    \"img_size\": IMG_SIZE,\n    \"total_original_box_rows_without_no_finding\": int(len(vin_df[\n        (vin_df[\"class_id\"] != NO_FINDING_ID) &\n        (vin_df[\"x_min\"].notna()) &\n        (vin_df[\"y_min\"].notna()) &\n        (vin_df[\"x_max\"].notna()) &\n        (vin_df[\"y_max\"].notna())\n    ])),\n    \"matched_box_rows_before_filter\": int(before_filter),\n    \"valid_box_rows_after_filter\": int(after_filter),\n    \"removed_invalid_after_crop_resize\": int(before_filter - after_filter),\n    \"train_box_rows\": int(len(train_boxes)),\n    \"val_box_rows\": int(len(val_boxes)),\n    \"test_box_rows\": int(len(test_boxes)),\n    \"train_images_with_box\": int(train_boxes[\"image_id\"].nunique()),\n    \"val_images_with_box\": int(val_boxes[\"image_id\"].nunique()),\n    \"test_images_with_box\": int(test_boxes[\"image_id\"].nunique()),\n    \"output_dir\": OUTPUT_DIR\n}\n\nwith open(os.path.join(OUTPUT_DIR, \"box_resize_summary.json\"), \"w\") as f:\n    json.dump(summary, f, indent=4)\n\nprint(\"=\" * 80)\nprint(\"DONE\")\nprint(\"=\" * 80)\nfor k, v in summary.items():\n    print(f\"{k}: {v}\")\n\nprint(\"\\nSaved files:\")\nprint(all_boxes_path)\nprint(train_boxes_path)\nprint(val_boxes_path)\nprint(test_boxes_path)\n\ndisplay(box_df.head())\ndisplay(\n    box_df.groupby([\"split_from_csv\", \"class_id\", \"class_name\"])\n    .size()\n    .reset_index(name=\"box_count\")\n)\n\ndef find_cache_image_path(image_id, split_name):\n    candidates = [\n        os.path.join(CACHE_ROOT, \"cache\", split_name, image_id + \".npy\"),\n        os.path.join(CACHE_ROOT, split_name, image_id + \".npy\"),\n        os.path.join(\"/kaggle/input\", \"**\", \"cache\", split_name, image_id + \".npy\"),\n        os.path.join(\"/kaggle/input\", \"**\", split_name, image_id + \".npy\")\n    ]\n\n    for pattern in candidates:\n        matches = glob.glob(pattern, recursive=True)\n        if len(matches) > 0:\n            return matches[0]\n\n    return None\n\ndef draw_box_sample(boxes_df, split_name, n=6):\n    sample_ids = boxes_df[\"image_id\"].drop_duplicates().sample(\n        min(n, boxes_df[\"image_id\"].nunique()),\n        random_state=RANDOM_STATE\n    ).tolist()\n\n    if len(sample_ids) == 0:\n        print(f\"Không có ảnh có box trong split {split_name}\")\n        return\n\n    cols = 3\n    rows = int(np.ceil(len(sample_ids) / cols))\n\n    plt.figure(figsize=(cols * 5, rows * 5))\n\n    for idx, image_id in enumerate(sample_ids):\n        img_path = find_cache_image_path(image_id, split_name)\n        if img_path is None:\n            print(\"Không tìm thấy ảnh cache:\", image_id, split_name)\n            continue\n\n        img = np.load(img_path)\n\n        if img.dtype != np.uint8:\n            img_show = img.astype(np.float32)\n            img_show = (img_show - img_show.min()) / (img_show.max() - img_show.min() + 1e-6)\n        else:\n            img_show = img\n\n        img_boxes = boxes_df[boxes_df[\"image_id\"] == image_id]\n\n        ax = plt.subplot(rows, cols, idx + 1)\n        ax.imshow(img_show, cmap=\"gray\" if img_show.ndim == 2 else None)\n\n        title_classes = []\n\n        for _, row in img_boxes.iterrows():\n            x1 = float(row[\"x_min_384\"])\n            y1 = float(row[\"y_min_384\"])\n            x2 = float(row[\"x_max_384\"])\n            y2 = float(row[\"y_max_384\"])\n            w = x2 - x1\n            h = y2 - y1\n\n            rect = patches.Rectangle(\n                (x1, y1),\n                w,\n                h,\n                linewidth=2,\n                edgecolor=\"red\",\n                facecolor=\"none\"\n            )\n            ax.add_patch(rect)\n            title_classes.append(str(row[\"class_name\"]))\n\n        title_classes = sorted(list(set(title_classes)))\n        ax.set_title(f\"{split_name} | {image_id[:8]}\\n\" + \", \".join(title_classes), fontsize=9)\n        ax.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()\n\nprint(\"\\nCHECK TRAIN BOXES\")\ndraw_box_sample(train_boxes, \"train\", n=6)\n\nprint(\"\\nCHECK VAL BOXES\")\ndraw_box_sample(val_boxes, \"val\", n=6)\n\nprint(\"\\nCHECK TEST BOXES\")\ndraw_box_sample(test_boxes, \"test\", n=6)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-05T10:16:51.081179Z","iopub.execute_input":"2026-06-05T10:16:51.081401Z","iopub.status.idle":"2026-06-05T10:18:47.780934Z","shell.execute_reply.started":"2026-06-05T10:16:51.081374Z","shell.execute_reply":"2026-06-05T10:18:47.780228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nVINBIGDATA_DIR = \"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\"\nDICOM_DIR = os.path.join(VINBIGDATA_DIR, \"train\")\n\nBOX_OUTPUT_DIR = \"/kaggle/working/box_resized_outputs\"\nALL_BOXES_PATH = os.path.join(BOX_OUTPUT_DIR, \"all_boxes_resized_384.csv\")\n\nboxes_df = pd.read_csv(ALL_BOXES_PATH)\n\ndef find_cache_image_path(image_id, split_name):\n    patterns = [\n        f\"/kaggle/input/**/cache/{split_name}/{image_id}.npy\",\n        f\"/kaggle/input/**/{split_name}/{image_id}.npy\",\n        f\"/kaggle/working/**/cache/{split_name}/{image_id}.npy\",\n        f\"/kaggle/working/**/{split_name}/{image_id}.npy\",\n    ]\n\n    for pattern in patterns:\n        matches = glob.glob(pattern, recursive=True)\n        if len(matches) > 0:\n            return matches[0]\n\n    return None\n\ndef read_dicom_original(path):\n    dicom = pydicom.dcmread(path)\n    img = apply_voi_lut(dicom.pixel_array, dicom)\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.max(img) - img\n\n    img = img.astype(np.float32)\n    lower = np.percentile(img, 1)\n    upper = np.percentile(img, 99)\n    img = np.clip(img, lower, upper)\n    img = (img - img.min()) / (img.max() - img.min() + 1e-6)\n\n    return img\n\ndef show_original_vs_cache(sample_ids=None, split_name=\"train\", n=6):\n    if sample_ids is None:\n        split_boxes = boxes_df[boxes_df[\"split_from_csv\"] == split_name].copy()\n        sample_ids = split_boxes[\"image_id\"].drop_duplicates().sample(\n            min(n, split_boxes[\"image_id\"].nunique()),\n            random_state=42\n        ).tolist()\n\n    rows = len(sample_ids)\n    fig, axes = plt.subplots(rows, 2, figsize=(12, rows * 5))\n\n    if rows == 1:\n        axes = np.expand_dims(axes, axis=0)\n\n    for i, image_id in enumerate(sample_ids):\n        img_boxes = boxes_df[\n            (boxes_df[\"image_id\"] == image_id) &\n            (boxes_df[\"split_from_csv\"] == split_name)\n        ].copy()\n\n        dicom_path = os.path.join(DICOM_DIR, image_id + \".dicom\")\n        cache_path = find_cache_image_path(image_id, split_name)\n\n        if not os.path.exists(dicom_path):\n            print(\"Không tìm thấy DICOM:\", dicom_path)\n            continue\n\n        if cache_path is None:\n            print(\"Không tìm thấy cache:\", image_id)\n            continue\n\n        img_original = read_dicom_original(dicom_path)\n        img_cache = np.load(cache_path)\n\n        ax1 = axes[i, 0]\n        ax1.imshow(img_original, cmap=\"gray\")\n\n        for _, row in img_boxes.iterrows():\n            x1 = float(row[\"x_min\"])\n            y1 = float(row[\"y_min\"])\n            x2 = float(row[\"x_max\"])\n            y2 = float(row[\"y_max\"])\n\n            rect = patches.Rectangle(\n                (x1, y1),\n                x2 - x1,\n                y2 - y1,\n                linewidth=2,\n                edgecolor=\"red\",\n                facecolor=\"none\"\n            )\n            ax1.add_patch(rect)\n\n        ax1.set_title(f\"Original DICOM | {image_id[:8]}\\nBox gốc\", fontsize=10)\n        ax1.axis(\"off\")\n\n        ax2 = axes[i, 1]\n        ax2.imshow(img_cache)\n\n        for _, row in img_boxes.iterrows():\n            x1 = float(row[\"x_min_384\"])\n            y1 = float(row[\"y_min_384\"])\n            x2 = float(row[\"x_max_384\"])\n            y2 = float(row[\"y_max_384\"])\n\n            rect = patches.Rectangle(\n                (x1, y1),\n                x2 - x1,\n                y2 - y1,\n                linewidth=2,\n                edgecolor=\"red\",\n                facecolor=\"none\"\n            )\n            ax2.add_patch(rect)\n\n        cls_names = \", \".join(sorted(img_boxes[\"class_name\"].unique()))\n        ax2.set_title(f\"Cache 384x384 | {split_name}\\n{cls_names}\", fontsize=10)\n        ax2.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()\n\nshow_original_vs_cache(split_name=\"train\", n=6)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-05T10:23:15.409273Z","iopub.execute_input":"2026-06-05T10:23:15.410101Z","iopub.status.idle":"2026-06-05T10:23:56.486496Z","shell.execute_reply.started":"2026-06-05T10:23:15.410074Z","shell.execute_reply":"2026-06-05T10:23:56.484162Z"}},"outputs":[],"execution_count":null}]}