{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":15892005,"datasetId":10190211,"databundleVersionId":16846413},{"sourceType":"datasetVersion","sourceId":15094918,"datasetId":9664561,"databundleVersionId":15979687},{"sourceType":"datasetVersion","sourceId":15588375,"datasetId":9973820,"databundleVersionId":16520657}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install pydicom pylibjpeg pylibjpeg-libjpeg pylibjpeg-openjpeg ultralytics albumentations\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2026-04-25T05:31:56.515717Z","iopub.execute_input":"2026-04-25T05:31:56.516298Z","iopub.status.idle":"2026-04-25T05:32:03.195954Z","shell.execute_reply.started":"2026-04-25T05:31:56.516269Z","shell.execute_reply":"2026-04-25T05:32:03.194958Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport yaml\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport albumentations as A\n\nfrom collections import defaultdict\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:32:03.197641Z","iopub.execute_input":"2026-04-25T05:32:03.198216Z","iopub.status.idle":"2026-04-25T05:32:09.602401Z","shell.execute_reply.started":"2026-04-25T05:32:03.198187Z","shell.execute_reply":"2026-04-25T05:32:09.601809Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# =========================\n# PATH CONFIG\n# =========================\nfrom pathlib import Path\n\nROOT = Path(\"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\")\n\nTRAIN_DIR = ROOT / \"train\"\nTEST_DIR = ROOT / \"test\"\nTRAIN_CSV = Path('/kaggle/input/datasets/benxelua/clean-dataset/annotations_train_merged_other_500nf.csv')\nTEST_CSV = Path('/kaggle/input/datasets/benxelua/clean-dataset/annotations_test_merged_other_500nf.csv')\n\n# output YOLO dataset\nOUT_ROOT = Path(\"/kaggle/working/yolo_vindr_multiclass\")\n\nPNG_DIR = OUT_ROOT / \"images\"\nLBL_DIR = OUT_ROOT / \"labels\"\n\nIMG_TRAIN_DIR = PNG_DIR / \"train\"\nIMG_VAL_DIR   = PNG_DIR / \"val\"\nIMG_TEST_DIR  = PNG_DIR / \"test\"\nLBL_TRAIN_DIR = LBL_DIR / \"train\"\nLBL_VAL_DIR   = LBL_DIR / \"val\"\nLBL_TEST_DIR  = LBL_DIR / \"test\"\n\nfor d in [IMG_TRAIN_DIR, IMG_VAL_DIR, IMG_TEST_DIR, LBL_TRAIN_DIR, LBL_VAL_DIR, LBL_TEST_DIR]:\n    d.mkdir(parents=True, exist_ok=True)\n\n# =========================\n# SUBSET CONFIG\n# =========================\nUSE_SMALL_SUBSET = False     # True = subset nhỏ, False = full dataset\nIMAGES_PER_CLASS = 2         # mỗi class lấy khoảng N ảnh trong small subset\nNO_FINDING_IMAGES = 1000      # giữ khoảng 500-1000 ảnh No finding/background; đổi 500/1000 nếu muốn\nSEED = 42\n\n# Sau khi gộp train + test gốc, chia lại theo image-level multilabel stratification.\nSPLIT_RATIOS = {\n    \"train\": 0.70,\n    \"val\": 0.15,\n    \"test\": 0.15,\n}\n\n# =========================\n# IMAGE CONFIG\n# =========================\nIMG_SIZE = 512              # có thể đổi 640 nếu muốn\nSAVE_AS_JPG = False         # True nhanh hơn PNG\nJPG_QUALITY = 95","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:32:09.603611Z","iopub.execute_input":"2026-04-25T05:32:09.604068Z","iopub.status.idle":"2026-04-25T05:32:09.611051Z","shell.execute_reply.started":"2026-04-25T05:32:09.604042Z","shell.execute_reply":"2026-04-25T05:32:09.61044Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n\nseed_everything(SEED)\n\n\ndef find_dicom_files_for_image_ids(folder: Path, image_ids):\n    requested_ids = sorted({str(x).strip() for x in image_ids if pd.notna(x) and str(x).strip()})\n    matched = {}\n\n    for image_id in requested_ids:\n        direct_candidates = [\n            folder / f\"{image_id}.dicom\",\n            folder / f\"{image_id}.dcm\",\n            folder / image_id,\n        ]\n        for candidate in direct_candidates:\n            if candidate.is_file():\n                matched[image_id] = candidate\n                break\n\n    remaining = [image_id for image_id in requested_ids if image_id not in matched]\n    if remaining:\n        remaining_set = set(remaining)\n        for p in sorted(folder.rglob(\"*\")):\n            if not remaining_set:\n                break\n            if p.is_file() and p.suffix.lower() in {\".dicom\", \".dcm\"} and p.stem in remaining_set:\n                matched[p.stem] = p\n                remaining_set.remove(p.stem)\n\n    missing = [image_id for image_id in requested_ids if image_id not in matched]\n    matched_files = [matched[image_id] for image_id in requested_ids if image_id in matched]\n    return matched_files, matched, missing\n\n\nfrom pathlib import Path\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_modality_lut\n\ndef read_dicom_to_uint8(path: Path):\n    ds = pydicom.dcmread(str(path))\n\n    # 1. Rescale / Modality LUT\n    img = apply_modality_lut(ds.pixel_array, ds).astype(np.float32)\n\n    # 2. Handle MONOCHROME1\n    if getattr(ds, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n        img = img.max() - img\n\n    # 3. Min-max normalization\n    img_min = img.min()\n    img_max = img.max()\n    if img_max > img_min:\n        img = (img - img_min) / (img_max - img_min)\n    else:\n        img = np.zeros_like(img)\n\n    # 4. Convert to uint8\n    return (img * 255.0).clip(0, 255).astype(np.uint8)\n\nfrom pathlib import Path\nimport numpy as np\nimport pydicom\n\n# def read_dicom_to_uint8(path: Path):\n#     ds = pydicom.dcmread(str(path))\n#     \n#     # 1. Read raw pixel array only\n#     img = ds.pixel_array.astype(np.float32)\n\n#     # 2. Handle Photometric Interpretation\n#     if getattr(ds, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n#         img = np.max(img) - img\n\n#     # 3. Raw min-max normalization\n#     img = img - np.min(img)\n#     max_val = np.max(img)\n#     if max_val > 0:\n#         img = img / max_val\n\n#     # 4. Convert to uint8\n#     img = (img * 255.0).clip(0, 255).astype(np.uint8)\n#     return img\n\ndef resize_image_keep_shape(img, size=1024):\n    h, w = img.shape[:2]\n    resized = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)\n    return resized, w, h\n","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:32:09.612676Z","iopub.execute_input":"2026-04-25T05:32:09.612887Z","iopub.status.idle":"2026-04-25T05:32:09.63109Z","shell.execute_reply.started":"2026-04-25T05:32:09.612868Z","shell.execute_reply":"2026-04-25T05:32:09.630501Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Đọc CSV và chuẩn hóa annotation","metadata":{}},{"cell_type":"code","source":"# Label fusion helpers for VinDr-CXR training annotations\nfrom dataclasses import dataclass\n\n@dataclass(frozen=True)\nclass FusionConfig:\n    iou_threshold: float = 0.5\n    ios_threshold: float = 0.7\n    merge_mode: str = \"median\"  # options: median, mean, enclosing\n\n\ndef _box_area(box):\n    width = max(0.0, float(box[2]) - float(box[0]))\n    height = max(0.0, float(box[3]) - float(box[1]))\n    return width * height\n\n\ndef _intersection_area(box_a, box_b):\n    x1 = max(float(box_a[0]), float(box_b[0]))\n    y1 = max(float(box_a[1]), float(box_b[1]))\n    x2 = min(float(box_a[2]), float(box_b[2]))\n    y2 = min(float(box_a[3]), float(box_b[3]))\n    width = max(0.0, x2 - x1)\n    height = max(0.0, y2 - y1)\n    return width * height\n\n\ndef compute_iou(box_a, box_b):\n    inter = _intersection_area(box_a, box_b)\n    if inter <= 0.0:\n        return 0.0\n    union = _box_area(box_a) + _box_area(box_b) - inter\n    if union <= 0.0:\n        return 0.0\n    return inter / union\n\n\ndef compute_ios(box_a, box_b):\n    inter = _intersection_area(box_a, box_b)\n    if inter <= 0.0:\n        return 0.0\n    smaller = min(_box_area(box_a), _box_area(box_b))\n    if smaller <= 0.0:\n        return 0.0\n    return inter / smaller\n\n\ndef should_link_boxes(box_a, box_b, config):\n    return (\n        compute_iou(box_a, box_b) >= config.iou_threshold\n        or compute_ios(box_a, box_b) >= config.ios_threshold\n    )\n\n\ndef merge_cluster_boxes(boxes, merge_mode=\"median\"):\n    boxes = np.asarray(boxes, dtype=float)\n    if len(boxes) == 1:\n        return boxes[0].astype(float)\n\n    if merge_mode == \"median\":\n        merged = np.median(boxes, axis=0)\n    elif merge_mode == \"mean\":\n        merged = np.mean(boxes, axis=0)\n    elif merge_mode == \"enclosing\":\n        merged = np.array([\n            np.min(boxes[:, 0]),\n            np.min(boxes[:, 1]),\n            np.max(boxes[:, 2]),\n            np.max(boxes[:, 3]),\n        ], dtype=float)\n    else:\n        raise ValueError(f\"Unsupported merge_mode: {merge_mode}\")\n\n    if merged[2] <= merged[0]:\n        merged[2] = merged[0] + 1e-6\n    if merged[3] <= merged[1]:\n        merged[3] = merged[1] + 1e-6\n    return merged.astype(float)\n\n\ndef _connected_components(indices, adjacency):\n    components = []\n    visited = set()\n\n    for start in indices:\n        if start in visited:\n            continue\n        stack = [start]\n        component = []\n        visited.add(start)\n\n        while stack:\n            node = stack.pop()\n            component.append(node)\n            for neighbor in adjacency[node]:\n                if neighbor not in visited:\n                    visited.add(neighbor)\n                    stack.append(neighbor)\n\n        components.append(sorted(component))\n\n    return components\n\n\ndef fuse_same_class_boxes(class_df, bbox_cols=(\"x_min\", \"y_min\", \"x_max\", \"y_max\"), config=None):\n    config = config or FusionConfig()\n    if len(class_df) <= 1:\n        out = class_df.copy()\n        out[\"fusion_group_size\"] = 1\n        out[\"was_fused\"] = False\n        return out\n\n    coords = class_df.loc[:, list(bbox_cols)].to_numpy(dtype=float)\n    indices = list(range(len(class_df)))\n    adjacency = {idx: set() for idx in indices}\n\n    for i in range(len(coords)):\n        for j in range(i + 1, len(coords)):\n            if should_link_boxes(coords[i], coords[j], config):\n                adjacency[i].add(j)\n                adjacency[j].add(i)\n\n    components = _connected_components(indices, adjacency)\n    fused_rows = []\n\n    for component in components:\n        component_df = class_df.iloc[component].copy()\n        merged_box = merge_cluster_boxes(\n            component_df.loc[:, list(bbox_cols)].to_numpy(dtype=float),\n            merge_mode=config.merge_mode,\n        )\n\n        row = component_df.iloc[0].copy()\n        row[bbox_cols[0]] = float(merged_box[0])\n        row[bbox_cols[1]] = float(merged_box[1])\n        row[bbox_cols[2]] = float(merged_box[2])\n        row[bbox_cols[3]] = float(merged_box[3])\n        row[\"fusion_group_size\"] = int(len(component))\n        row[\"was_fused\"] = bool(len(component) > 1)\n        fused_rows.append(row)\n\n    return pd.DataFrame(fused_rows).reset_index(drop=True)\n\n\ndef fuse_annotation_dataframe(\n    df,\n    image_col=\"image_id\",\n    class_col=\"class_name\",\n    bbox_cols=(\"x_min\", \"y_min\", \"x_max\", \"y_max\"),\n    config=None,\n):\n    config = config or FusionConfig()\n    fused_groups = []\n    grouped = df.groupby([image_col, class_col], sort=False, group_keys=False)\n    for _, group_df in grouped:\n        fused_groups.append(\n            fuse_same_class_boxes(\n                class_df=group_df,\n                bbox_cols=bbox_cols,\n                config=config,\n            )\n        )\n\n    if not fused_groups:\n        out = df.copy()\n        out[\"fusion_group_size\"] = 1\n        out[\"was_fused\"] = False\n        return out\n\n    return pd.concat(fused_groups, ignore_index=True).reset_index(drop=True)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-25T05:32:09.632078Z","iopub.execute_input":"2026-04-25T05:32:09.632451Z","iopub.status.idle":"2026-04-25T05:32:09.652713Z","shell.execute_reply.started":"2026-04-25T05:32:09.632429Z","shell.execute_reply":"2026-04-25T05:32:09.651976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_annotation_csv(csv_path: Path, source_name: str):\n    csv_path = Path(csv_path)\n    if not csv_path.exists():\n        raise FileNotFoundError(f\"Không thấy annotation CSV cho {source_name}: {csv_path}\")\n\n    out = pd.read_csv(csv_path)\n    print(f\"{source_name} CSV shape:\", out.shape)\n    print(f\"{source_name} CSV columns:\", out.columns.tolist())\n    display(out.head())\n\n    required_cols = [\"image_id\", \"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n    missing_cols = [c for c in required_cols if c not in out.columns]\n    if missing_cols:\n        raise ValueError(f\"Thiếu cột trong {source_name} CSV: {missing_cols}\")\n\n    out = out.copy()\n    out[\"source\"] = source_name\n    out[\"image_id\"] = out[\"image_id\"].astype(\"string\").str.strip()\n    out[\"class_name\"] = out[\"class_name\"].astype(\"string\").str.strip()\n    out = out.dropna(subset=[\"image_id\", \"class_name\"]).copy()\n\n    for c in [\"x_min\", \"y_min\", \"x_max\", \"y_max\"]:\n        out[c] = pd.to_numeric(out[c], errors=\"coerce\")\n\n    # sample_id là khóa duy nhất sau khi trộn train/test, tránh trùng image_id giữa 2 nguồn.\n    out[\"sample_id\"] = out[\"source\"].astype(str) + \"__\" + out[\"image_id\"].astype(str)\n    return out.reset_index(drop=True)\n\n\ntrain_csv_all = normalize_annotation_csv(TRAIN_CSV, \"train\")\ntest_csv_all = normalize_annotation_csv(TEST_CSV, \"test\")\ndf_csv_all = pd.concat([train_csv_all, test_csv_all], ignore_index=True)\n\nall_sample_ids = set(df_csv_all[\"sample_id\"].dropna().astype(str).str.strip().unique().tolist())\nall_csv_ids = all_sample_ids  # giữ tên cũ để các cell sau vẫn dễ đọc\nno_finding_sample_ids = set(\n    df_csv_all.loc[\n        df_csv_all[\"class_name\"].fillna(\"\").str.lower() == \"no finding\",\n        \"sample_id\",\n    ].astype(str)\n)\n\nprint(\"Merged CSV shape:\", df_csv_all.shape)\nprint(\"Merged unique sample_id:\", len(all_sample_ids))\nprint(\"No finding/background sample_id:\", len(no_finding_sample_ids))\nprint(\"Merged source distribution:\")\ndisplay(df_csv_all.drop_duplicates(\"sample_id\")[\"source\"].value_counts().rename_axis(\"source\").reset_index(name=\"images\"))\n\n# Không export \"No finding\" thành class bbox cho YOLO detection.\n# Ảnh No finding vẫn được giữ trong all_sample_ids, được split/export như background với file label rỗng.\ndf_full = df_csv_all.copy()\ndf_full = df_full[df_full[\"class_name\"].fillna(\"\").str.lower() != \"no finding\"].copy()\ndf_full = df_full.dropna(subset=[\"image_id\", \"sample_id\", \"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]).copy()\ndf_full = df_full[(df_full[\"x_max\"] > df_full[\"x_min\"]) & (df_full[\"y_max\"] > df_full[\"y_min\"])].copy()\n\n# =========================\n# OPTIONAL LABEL FUSION\n# =========================\n# VinDr-CXR có thể có nhiều bbox chồng nhau cho cùng 1 lesion vì được annotate bởi nhiều radiologists.\n# Bước này chỉ clean annotations trước khi export label.\n# Nó KHÔNG phải post-processing ở inference time.\nBOX_FUSION_ENABLED = True\nBOX_FUSION_IOU_THRESHOLD = 0.5\nBOX_FUSION_IOS_THRESHOLD = 0.7\nBOX_FUSION_MERGE_MODE = \"median\"   # options: median, mean, enclosing\n\ndf_before_fusion = df_full.copy()\n\nif BOX_FUSION_ENABLED:\n    before_fusion_boxes = len(df_full)\n    before_fusion_images = df_full[\"sample_id\"].nunique()\n    fusion_config = FusionConfig(\n        iou_threshold=BOX_FUSION_IOU_THRESHOLD,\n        ios_threshold=BOX_FUSION_IOS_THRESHOLD,\n        merge_mode=BOX_FUSION_MERGE_MODE,\n    )\n    df_full = fuse_annotation_dataframe(\n        df_full,\n        image_col=\"sample_id\",\n        class_col=\"class_name\",\n        config=fusion_config,\n    )\n    print(\"Label fusion enabled\")\n    print(\"Fusion config:\", fusion_config)\n    print(\"Boxes before fusion:\", before_fusion_boxes)\n    print(\"Boxes after fusion:\", len(df_full))\n    print(\"Images with abnormal boxes before fusion:\", before_fusion_images)\n    print(\"Images with abnormal boxes after fusion:\", df_full[\"sample_id\"].nunique())\n    print(\"Rows created from merged clusters:\", int(df_full[\"was_fused\"].sum()))\n    display(\n        df_full.loc[df_full[\"was_fused\"], [\"source\", \"image_id\", \"sample_id\", \"class_name\", \"fusion_group_size\"]]\n        .head(10)\n        .reset_index(drop=True)\n    )\n\n# Build class list từ train + test đã gộp.\nCLASS_NAMES = sorted(df_full[\"class_name\"].unique().tolist())\nCLASS2ID = {name: i for i, name in enumerate(CLASS_NAMES)}\nID2CLASS = {i: name for name, i in CLASS2ID.items()}\n\nabnormal_sample_ids = set(df_full[\"sample_id\"].astype(str).unique())\nbackground_sample_ids = sorted(all_sample_ids - abnormal_sample_ids)\nno_finding_only_sample_ids = sorted(set(no_finding_sample_ids) & set(background_sample_ids))\n\nprint(\"Num classes:\", len(CLASS_NAMES))\nprint(\"CLASS_NAMES:\", CLASS_NAMES)\nprint(\"CLASS2ID:\", CLASS2ID)\nprint(\"Total unique sample_id in merged CSV:\", len(all_sample_ids))\nprint(\"Images with >=1 valid abnormal bbox:\", len(abnormal_sample_ids))\nprint(\"Background-only sample_id:\", len(background_sample_ids))\nprint(\"No finding only sample_id:\", len(no_finding_only_sample_ids))","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:32:09.653676Z","iopub.execute_input":"2026-04-25T05:32:09.654047Z","iopub.status.idle":"2026-04-25T05:32:55.564949Z","shell.execute_reply.started":"2026-04-25T05:32:09.653991Z","shell.execute_reply":"2026-04-25T05:32:55.564224Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Debug visualize label fusion before vs after for one training image\n# import matplotlib.pyplot as plt\n# from matplotlib.patches import Rectangle\n\n# DEBUG_FUSION_IMAGE_ID = \"\"  # ví dụ: \"0005e8e3701dfb1dd93d53e2ff537b6e\"\n# SHOW_CLASS_TEXT = True\n# FIGSIZE = (18, 9)\n\n# if not DEBUG_FUSION_IMAGE_ID:\n#     raise ValueError(\"Điền DEBUG_FUSION_IMAGE_ID trước khi chạy cell này\")\n# if 'df_before_fusion' not in globals():\n#     raise ValueError(\"Chạy cell fusion trước để tạo df_before_fusion và df_full\")\n\n\n# def find_train_dicom_path(train_dir: Path, image_id: str):\n#     direct_candidates = [\n#         train_dir / f\"{image_id}.dicom\",\n#         train_dir / f\"{image_id}.dcm\",\n#         train_dir / image_id,\n#     ]\n#     for candidate in direct_candidates:\n#         if candidate.is_file():\n#             return candidate\n\n#     for p in sorted(train_dir.rglob(\"*\")):\n#         if p.is_file() and p.suffix.lower() in {\".dicom\", \".dcm\"} and p.stem == image_id:\n#             return p\n#     return None\n\n\n# def draw_boxes(ax, boxes_df, color, title_prefix=\"\"):\n#     for _, row in boxes_df.iterrows():\n#         x1, y1, x2, y2 = float(row['x_min']), float(row['y_min']), float(row['x_max']), float(row['y_max'])\n#         rect = Rectangle((x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor=color, linewidth=2)\n#         ax.add_patch(rect)\n#         if SHOW_CLASS_TEXT:\n#             label = str(row['class_name'])\n#             if 'fusion_group_size' in row and pd.notna(row.get('fusion_group_size', np.nan)):\n#                 if int(row.get('fusion_group_size', 1)) > 1:\n#                     label = f\"{label} (n={int(row['fusion_group_size'])})\"\n#             ax.text(\n#                 x1,\n#                 max(5.0, y1 - 4.0),\n#                 f\"{title_prefix}{label}\",\n#                 color=color,\n#                 fontsize=9,\n#                 bbox=dict(facecolor='black', alpha=0.5, pad=1),\n#             )\n\n\n# before_df = df_before_fusion[df_before_fusion['image_id'].astype(str) == str(DEBUG_FUSION_IMAGE_ID)].copy()\n# after_df = df_full[df_full['image_id'].astype(str) == str(DEBUG_FUSION_IMAGE_ID)].copy()\n\n# print(f\"image_id = {DEBUG_FUSION_IMAGE_ID}\")\n# print(f\"Boxes before fusion: {len(before_df)}\")\n# print(f\"Boxes after fusion: {len(after_df)}\")\n\n# if len(before_df) == 0 and len(after_df) == 0:\n#     raise ValueError(f\"Không tìm thấy annotation cho image_id={DEBUG_FUSION_IMAGE_ID}\")\n\n# display(before_df[['image_id', 'class_name', 'x_min', 'y_min', 'x_max', 'y_max']].reset_index(drop=True))\n# display(after_df[['image_id', 'class_name', 'x_min', 'y_min', 'x_max', 'y_max', 'fusion_group_size', 'was_fused']].reset_index(drop=True))\n\n# dicom_path = find_train_dicom_path(TRAIN_DIR, str(DEBUG_FUSION_IMAGE_ID))\n# if dicom_path is None:\n#     raise FileNotFoundError(f\"Không tìm thấy file DICOM cho image_id={DEBUG_FUSION_IMAGE_ID} trong {TRAIN_DIR}\")\n\n# img = read_dicom_to_uint8(dicom_path)\n\n# fig, axes = plt.subplots(1, 2, figsize=FIGSIZE)\n# for ax in axes:\n#     if img.ndim == 2:\n#         ax.imshow(img, cmap='gray')\n#     else:\n#         ax.imshow(img)\n#     ax.axis('off')\n\n# axes[0].set_title(f\"Before fusion: {DEBUG_FUSION_IMAGE_ID}\")\n# draw_boxes(axes[0], before_df, color='red', title_prefix='before: ')\n\n# axes[1].set_title(f\"After fusion: {DEBUG_FUSION_IMAGE_ID}\")\n# draw_boxes(axes[1], after_df, color='lime', title_prefix='after: ')\n\n# plt.tight_layout()\n# plt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-25T05:32:55.56615Z","iopub.execute_input":"2026-04-25T05:32:55.566407Z","iopub.status.idle":"2026-04-25T05:32:55.571182Z","shell.execute_reply.started":"2026-04-25T05:32:55.566386Z","shell.execute_reply":"2026-04-25T05:32:55.570424Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Chọn subset nhỏ theo từng class từ train + test đã gộp","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nprint(\"Building subset from merged train + test CSV...\")\n\n# subset theo annotation bbox hợp lệ, nhưng full mode vẫn lấy toàn bộ sample_id từ CSV.\ndf_work = df_full.copy()\n\nclass_to_ids = (\n    df_work.groupby(\"class_name\")[\"sample_id\"]\n    .unique()\n    .to_dict()\n)\n\n\ndef build_small_subset_by_class_fast(class_to_ids, images_per_class=12, seed=42, background_ids=None):\n    rng = np.random.default_rng(seed)\n\n    selected_ids = set()\n    per_class_summary = []\n\n    for cls in sorted(class_to_ids.keys()):\n        cls_ids = list(class_to_ids[cls])\n        n_take = min(images_per_class, len(cls_ids))\n\n        if n_take > 0:\n            chosen = rng.choice(cls_ids, size=n_take, replace=False).tolist()\n        else:\n            chosen = []\n\n        selected_ids.update(chosen)\n\n        per_class_summary.append({\n            \"class_name\": cls,\n            \"available_images\": len(cls_ids),\n            \"selected_images\": len(chosen)\n        })\n\n    background_ids = sorted(set(background_ids or []))\n    n_take_bg = min(images_per_class, len(background_ids))\n    if n_take_bg > 0:\n        chosen_bg = rng.choice(background_ids, size=n_take_bg, replace=False).tolist()\n    else:\n        chosen_bg = []\n    selected_ids.update(chosen_bg)\n    per_class_summary.append({\n        \"class_name\": \"No finding\",\n        \"available_images\": len(background_ids),\n        \"selected_images\": len(chosen_bg),\n    })\n\n    return selected_ids, pd.DataFrame(per_class_summary)\n\n\nrng_subset = np.random.default_rng(SEED)\nbackground_pool = sorted(set(background_sample_ids))\nn_no_finding_take = min(int(NO_FINDING_IMAGES), len(background_pool))\nselected_background_ids = set(\n    rng_subset.choice(background_pool, size=n_no_finding_take, replace=False).tolist()\n) if n_no_finding_take > 0 else set()\n\nif USE_SMALL_SUBSET:\n    chosen_ids, subset_summary_df = build_small_subset_by_class_fast(\n        class_to_ids,\n        images_per_class=IMAGES_PER_CLASS,\n        seed=SEED,\n        background_ids=background_pool,\n    )\n    chosen_ids = set(map(str, chosen_ids)) & all_sample_ids\n    print(\"Using SMALL subset mode\")\n    print(\"Total selected unique samples from merged CSV:\", len(chosen_ids))\n    display(subset_summary_df)\nelse:\n    # Full mode: giữ toàn bộ ảnh có bbox bất thường, nhưng chỉ lấy một lượng No finding/background vừa phải.\n    chosen_ids = set(abnormal_sample_ids) | selected_background_ids\n    print(\"Using FULL dataset mode with capped No finding/background\")\n    print(\"Abnormal samples selected:\", len(abnormal_sample_ids))\n    print(\"Background/No finding samples selected:\", len(selected_background_ids), \"of\", len(background_pool))\n    print(\"NO_FINDING_IMAGES:\", NO_FINDING_IMAGES)\n    print(\"Total selected unique samples from merged CSV:\", len(chosen_ids))\n\nprint(\"Matching requested DICOM files from both source folders...\")\n\nSOURCE_DIRS = {\n    \"train\": TRAIN_DIR,\n    \"test\": TEST_DIR,\n}\n\nsample_lookup_df = (\n    df_csv_all[[\"sample_id\", \"source\", \"image_id\"]]\n    .drop_duplicates(\"sample_id\")\n    .reset_index(drop=True)\n)\n\n\ndef find_dicom_records_for_sample_ids(sample_ids, sample_lookup_df, source_dirs):\n    requested = set(map(str, sample_ids))\n    lookup = sample_lookup_df[sample_lookup_df[\"sample_id\"].isin(requested)].copy()\n\n    records = []\n    path_map = {}\n    missing_sample_ids = []\n\n    for source, group in lookup.groupby(\"source\", sort=False):\n        source_dir = Path(source_dirs[source])\n        if not source_dir.exists():\n            missing_sample_ids.extend(group[\"sample_id\"].astype(str).tolist())\n            continue\n\n        image_ids = group[\"image_id\"].astype(str).tolist()\n        _, image_path_map, missing_image_ids = find_dicom_files_for_image_ids(source_dir, image_ids)\n        missing_image_ids = set(missing_image_ids)\n\n        for row in group.itertuples(index=False):\n            sample_id = str(row.sample_id)\n            image_id = str(row.image_id)\n            if image_id in image_path_map:\n                record = {\n                    \"sample_id\": sample_id,\n                    \"source\": str(row.source),\n                    \"image_id\": image_id,\n                    \"path\": image_path_map[image_id],\n                }\n                records.append(record)\n                path_map[sample_id] = image_path_map[image_id]\n            elif image_id in missing_image_ids:\n                missing_sample_ids.append(sample_id)\n\n    records = sorted(records, key=lambda x: x[\"sample_id\"])\n    missing_sample_ids = sorted(set(missing_sample_ids) | (requested - set(path_map.keys())))\n    return records, path_map, missing_sample_ids\n\n\ndicom_records_selected, dicom_path_map, missing_dicom_ids = find_dicom_records_for_sample_ids(\n    chosen_ids,\n    sample_lookup_df=sample_lookup_df,\n    source_dirs=SOURCE_DIRS,\n)\ndicom_files_selected = [record[\"path\"] for record in dicom_records_selected]\nused_sample_ids = sorted(dicom_path_map.keys())\nused_image_ids = used_sample_ids  # alias cũ, giờ là sample_id\n\nprint(\"Requested merged sample_ids:\", len(chosen_ids))\nprint(\"Matched DICOM records:\", len(dicom_records_selected))\nprint(\"Missing requested DICOM samples:\", len(missing_dicom_ids))\nif missing_dicom_ids:\n    print(\"First missing sample_ids:\", missing_dicom_ids[:10])\n\nsource_summary = pd.DataFrame(dicom_records_selected)[\"source\"].value_counts().rename_axis(\"source\").reset_index(name=\"matched_images\")\ndisplay(source_summary)","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:32:55.572143Z","iopub.execute_input":"2026-04-25T05:32:55.572444Z","iopub.status.idle":"2026-04-25T05:33:19.78104Z","shell.execute_reply.started":"2026-04-25T05:32:55.572412Z","shell.execute_reply":"2026-04-25T05:33:19.780429Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"used_sample_ids = sorted(dicom_path_map.keys())\nused_image_ids = used_sample_ids  # alias cũ, giờ là sample_id\n\ndf = df_full[df_full[\"sample_id\"].astype(str).str.strip().isin(used_sample_ids)].copy()\nused_negative_only_ids = sorted(set(used_sample_ids) - set(df[\"sample_id\"].astype(str).unique()))\n\nprint(\"Requested merged sample_ids:\", len(chosen_ids))\nprint(\"Used DICOM records:\", len(dicom_records_selected))\nprint(\"Selected annotation rows:\", len(df))\nprint(\"Used annotated sample_id:\", df[\"sample_id\"].nunique())\nprint(\"Used negative-only sample_id:\", len(used_negative_only_ids))\nprint(\"Missing requested sample_ids:\", len(missing_dicom_ids))","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:19.782091Z","iopub.execute_input":"2026-04-25T05:33:19.782429Z","iopub.status.idle":"2026-04-25T05:33:19.807069Z","shell.execute_reply.started":"2026-04-25T05:33:19.782406Z","shell.execute_reply":"2026-04-25T05:33:19.806286Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Stratified split train/val/test","metadata":{}},{"cell_type":"code","source":"from collections import Counter, defaultdict\nimport numpy as np\nimport pandas as pd\n\nBACKGROUND_CLASS = \"No finding\"\n\n\ndef normalize_split_ratios(split_ratios):\n    split_ratios = dict(split_ratios)\n    total = sum(float(v) for v in split_ratios.values())\n    if total <= 0:\n        raise ValueError(\"SPLIT_RATIOS phải có tổng > 0\")\n    return {k: float(v) / total for k, v in split_ratios.items()}\n\n\ndef compute_target_sizes(n_items, split_ratios):\n    raw = {split: n_items * ratio for split, ratio in split_ratios.items()}\n    sizes = {split: int(np.floor(value)) for split, value in raw.items()}\n    remaining = n_items - sum(sizes.values())\n\n    order = sorted(split_ratios.keys(), key=lambda split: (raw[split] - sizes[split], split), reverse=True)\n    for split in order[:remaining]:\n        sizes[split] += 1\n\n    nonzero_splits = [split for split, ratio in split_ratios.items() if ratio > 0]\n    if n_items >= len(nonzero_splits):\n        for split in nonzero_splits:\n            if sizes[split] == 0:\n                donor = max(sizes, key=sizes.get)\n                sizes[donor] -= 1\n                sizes[split] += 1\n\n    return sizes\n\n\ndef build_image_class_sets(df, image_ids, image_col=\"sample_id\", class_col=\"class_name\"):\n    image_ids = sorted(set(map(str, image_ids)))\n    grouped = (\n        df.groupby(image_col)[class_col]\n        .apply(lambda x: sorted(set(map(str, x))))\n        .to_dict()\n    )\n    return {\n        image_id: grouped.get(image_id, [BACKGROUND_CLASS])\n        for image_id in image_ids\n    }\n\n\ndef build_multilabel_stratified_split(df, image_ids, split_ratios=None, seed=42):\n    split_ratios = normalize_split_ratios(split_ratios or {\"train\": 0.7, \"val\": 0.15, \"test\": 0.15})\n    rng = np.random.default_rng(seed)\n\n    image_ids = sorted(set(map(str, image_ids)))\n    image_class_sets = build_image_class_sets(df, image_ids)\n    split_names = list(split_ratios.keys())\n\n    target_sizes = compute_target_sizes(len(image_ids), split_ratios)\n    class_total = Counter(cls for classes in image_class_sets.values() for cls in classes)\n    target_class_counts = {\n        split: {cls: class_total[cls] * split_ratios[split] for cls in class_total}\n        for split in split_names\n    }\n\n    current_sizes = {split: 0 for split in split_names}\n    current_class_counts = {split: Counter() for split in split_names}\n    split_sets = {split: set() for split in split_names}\n\n    # Ưu tiên gán ảnh chứa class hiếm/multilabel trước để giữ phân bố class tốt hơn.\n    ordered_ids = sorted(\n        image_ids,\n        key=lambda image_id: (\n            -sum(1.0 / max(class_total[cls], 1) for cls in image_class_sets[image_id]),\n            -len(image_class_sets[image_id]),\n            rng.random(),\n        ),\n    )\n\n    for image_id in ordered_ids:\n        classes = image_class_sets[image_id]\n        candidates = [split for split in split_names if current_sizes[split] < target_sizes[split]]\n        if not candidates:\n            candidates = split_names\n\n        best_split = None\n        best_score = None\n\n        for split in candidates:\n            size_deficit = (target_sizes[split] - current_sizes[split]) / max(target_sizes[split], 1)\n            class_deficit = 0.0\n            over_penalty = 0.0\n\n            for cls in classes:\n                target = max(target_class_counts[split][cls], 1e-9)\n                before = current_class_counts[split][cls]\n                after = before + 1\n                class_deficit += max(target - before, 0.0) / target\n                over_penalty += max(after - target, 0.0) / target\n\n            score = size_deficit + class_deficit - over_penalty\n            tie_breaker = rng.random() * 1e-6\n            score += tie_breaker\n\n            if best_score is None or score > best_score:\n                best_score = score\n                best_split = split\n\n        split_sets[best_split].add(image_id)\n        current_sizes[best_split] += 1\n        for cls in classes:\n            current_class_counts[best_split][cls] += 1\n\n    return split_sets, image_class_sets\n\n\ndef summarize_split_distribution(split_sets, image_class_sets):\n    class_names = sorted({cls for classes in image_class_sets.values() for cls in classes})\n    rows = []\n    for cls in class_names:\n        total = sum(1 for classes in image_class_sets.values() if cls in classes)\n        row = {\"class_name\": cls, \"total_images\": total}\n        for split, ids in split_sets.items():\n            count = sum(1 for image_id in ids if cls in image_class_sets[image_id])\n            row[f\"{split}_images\"] = count\n            row[f\"{split}_pct_of_class\"] = round(100.0 * count / total, 2) if total else 0.0\n        rows.append(row)\n    return pd.DataFrame(rows).sort_values(\"class_name\").reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:19.8096Z","iopub.execute_input":"2026-04-25T05:33:19.809913Z","iopub.status.idle":"2026-04-25T05:33:19.82786Z","shell.execute_reply.started":"2026-04-25T05:33:19.809892Z","shell.execute_reply":"2026-04-25T05:33:19.827225Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_ids_selected = used_sample_ids.copy()\n\nsplit_sets, image_class_sets = build_multilabel_stratified_split(\n    df=df,\n    image_ids=image_ids_selected,\n    split_ratios=SPLIT_RATIOS,\n    seed=SEED,\n)\n\ntrain_ids = split_sets[\"train\"]\nval_ids = split_sets[\"val\"]\ntest_ids = split_sets[\"test\"]\nsplit_distribution_df = summarize_split_distribution(split_sets, image_class_sets)\n\nprint(\"Used images for split:\", len(image_ids_selected))\nprint(\"Split ratios:\", normalize_split_ratios(SPLIT_RATIOS))\nprint(\"Train images:\", len(train_ids))\nprint(\"Val images:\", len(val_ids))\nprint(\"Test images:\", len(test_ids))\nprint(\"Overlap train/val:\", len(train_ids & val_ids))\nprint(\"Overlap train/test:\", len(train_ids & test_ids))\nprint(\"Overlap val/test:\", len(val_ids & test_ids))\n\ndisplay(split_distribution_df)","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:19.828847Z","iopub.execute_input":"2026-04-25T05:33:19.829587Z","iopub.status.idle":"2026-04-25T05:33:20.040839Z","shell.execute_reply.started":"2026-04-25T05:33:19.829565Z","shell.execute_reply":"2026-04-25T05:33:20.039952Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Lưu metadata để lần sau dùng lại","metadata":{}},{"cell_type":"code","source":"import json\n\nmeta = {\n    \"use_small_subset\": USE_SMALL_SUBSET,\n    \"images_per_class\": IMAGES_PER_CLASS,\n    \"no_finding_images\": NO_FINDING_IMAGES,\n    \"selected_background_ids\": sorted(list(selected_background_ids)),\n    \"seed\": SEED,\n    \"split_ratios\": normalize_split_ratios(SPLIT_RATIOS),\n    \"img_size\": IMG_SIZE,\n    \"save_as_jpg\": SAVE_AS_JPG,\n    \"jpg_quality\": JPG_QUALITY,\n    \"num_classes\": len(CLASS_NAMES),\n    \"class_names\": CLASS_NAMES,\n    \"chosen_sample_ids\": sorted(list(chosen_ids)),\n    \"used_sample_ids\": sorted(list(used_sample_ids)),\n    \"missing_dicom_sample_ids\": sorted(list(missing_dicom_ids)),\n    \"train_ids\": sorted(list(train_ids)),\n    \"val_ids\": sorted(list(val_ids)),\n    \"test_ids\": sorted(list(test_ids)),\n}\n\nwith open(OUT_ROOT / \"meta.json\", \"w\") as f:\n    json.dump(meta, f, indent=2)\n\nsplit_distribution_df.to_csv(OUT_ROOT / \"split_distribution.csv\", index=False)\n\nprint(\"Saved:\", OUT_ROOT / \"meta.json\")\nprint(\"Saved:\", OUT_ROOT / \"split_distribution.csv\")","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:20.041771Z","iopub.execute_input":"2026-04-25T05:33:20.041975Z","iopub.status.idle":"2026-04-25T05:33:20.069121Z","shell.execute_reply.started":"2026-04-25T05:33:20.041955Z","shell.execute_reply":"2026-04-25T05:33:20.068482Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build annotation map nhanh","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\n\nann_map = defaultdict(list)\n\nfor r in df.itertuples(index=False):\n    class_id = CLASS2ID[r.class_name]\n    ann_map[str(r.sample_id)].append(\n        (r.x_min, r.y_min, r.x_max, r.y_max, class_id)\n    )\n\nsplit_map = {}\nfor x in train_ids:\n    split_map[str(x)] = \"train\"\nfor x in val_ids:\n    split_map[str(x)] = \"val\"\nfor x in test_ids:\n    split_map[str(x)] = \"test\"\n\nprint(\"Images with annotations:\", len(ann_map))\nprint(\"Images without annotations:\", len(set(split_map) - set(ann_map)))\nprint(\"Images in split map:\", len(split_map))\nprint(\"Split map counts:\", pd.Series(split_map).value_counts().to_dict())","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:20.06999Z","iopub.execute_input":"2026-04-25T05:33:20.07054Z","iopub.status.idle":"2026-04-25T05:33:20.134206Z","shell.execute_reply.started":"2026-04-25T05:33:20.070512Z","shell.execute_reply":"2026-04-25T05:33:20.133558Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Export ảnh + label nhanh, có track time","metadata":{}},{"cell_type":"code","source":"import albumentations as A\n\nDICOM_TRAIN_AUG = A.Compose(\n    [\n        # Random Horizontal Flip\n        A.HorizontalFlip(p=0.5),\n\n        # Sharpness Enhancement (gần với mô tả dùng Gaussian để làm rõ ảnh)\n        A.UnsharpMask(\n            blur_limit=(3, 5),\n            sigma_limit=(0.1, 1.5),\n            alpha=(0.1, 0.3),\n            threshold=10,\n            p=0.20\n        ),\n\n        # Gaussian Blur\n        A.GaussianBlur(\n            blur_limit=(3, 5),\n            sigma_limit=(0.1, 1.5),\n            p=0.20\n        ),\n\n        # Image Histogram Equalization\n        A.Equalize(\n            mode=\"cv\",\n            by_channels=False,\n            p=0.20\n        ),\n\n        # Random Brightness only\n        A.RandomBrightnessContrast(\n            brightness_limit=0.10,\n            contrast_limit=0.0,\n            p=0.20\n        ),\n    ],\n    bbox_params=A.BboxParams(\n        format=\"pascal_voc\",\n        label_fields=[\"class_labels\"],\n        min_visibility=0.0,\n    ),\n)\n\n\ndef sanitize_pascal_voc_boxes(boxes, width, height):\n    sanitized = []\n    for (x1, y1, x2, y2, class_id) in boxes:\n        x1 = float(np.clip(x1, 0, width))\n        y1 = float(np.clip(y1, 0, height))\n        x2 = float(np.clip(x2, 0, width))\n        y2 = float(np.clip(y2, 0, height))\n\n        if x2 <= x1 or y2 <= y1:\n            continue\n\n        sanitized.append((x1, y1, x2, y2, class_id))\n\n    return sanitized\n\n\ndef apply_dicom_augmentation(img, boxes, split):\n    height, width = img.shape[:2]\n    boxes = sanitize_pascal_voc_boxes(boxes, width=width, height=height)\n\n    if split != \"train\" or len(boxes) == 0:\n        return img, boxes\n\n    bboxes = [(x1, y1, x2, y2) for (x1, y1, x2, y2, _) in boxes]\n    class_labels = [class_id for (_, _, _, _, class_id) in boxes]\n\n    transformed = DICOM_TRAIN_AUG(\n        image=img,\n        bboxes=bboxes,\n        class_labels=class_labels,\n    )\n\n    aug_boxes = [\n        (x1, y1, x2, y2, class_id)\n        for (x1, y1, x2, y2), class_id in zip(\n            transformed[\"bboxes\"],\n            transformed[\"class_labels\"],\n        )\n    ]\n    aug_boxes = sanitize_pascal_voc_boxes(aug_boxes, width=width, height=height)\n    return transformed[\"image\"], aug_boxes\n\n\ndef build_yolo_lines(boxes, orig_w, orig_h, img_size):\n    yolo_lines = []\n\n    if len(boxes) == 0:\n        return yolo_lines\n\n    sx = img_size / orig_w\n    sy = img_size / orig_h\n\n    for (x1, y1, x2, y2, class_id) in boxes:\n        x1 *= sx\n        x2 *= sx\n        y1 *= sy\n        y2 *= sy\n\n        xc = ((x1 + x2) / 2.0) / img_size\n        yc = ((y1 + y2) / 2.0) / img_size\n        bw = (x2 - x1) / img_size\n        bh = (y2 - y1) / img_size\n\n        xc = min(max(xc, 0.0), 1.0)\n        yc = min(max(yc, 0.0), 1.0)\n        bw = min(max(bw, 0.0), 1.0)\n        bh = min(max(bh, 0.0), 1.0)\n\n        if bw > 0 and bh > 0:\n            yolo_lines.append(f\"{class_id} {xc:.6f} {yc:.6f} {bw:.6f} {bh:.6f}\")\n\n    return yolo_lines\n\n\ndef save_image_and_label(img, boxes, image_id, split):\n    img_resized, orig_w, orig_h = resize_image_keep_shape(img, size=IMG_SIZE)\n\n    ext = \".jpg\" if SAVE_AS_JPG else \".png\"\n\n    if split == \"train\":\n        img_out_path = IMG_TRAIN_DIR / f\"{image_id}{ext}\"\n        lbl_out_path = LBL_TRAIN_DIR / f\"{image_id}.txt\"\n    elif split == \"val\":\n        img_out_path = IMG_VAL_DIR / f\"{image_id}{ext}\"\n        lbl_out_path = LBL_VAL_DIR / f\"{image_id}.txt\"\n    elif split == \"test\":\n        img_out_path = IMG_TEST_DIR / f\"{image_id}{ext}\"\n        lbl_out_path = LBL_TEST_DIR / f\"{image_id}.txt\"\n    else:\n        raise ValueError(f\"Unsupported split: {split}\")\n\n    if SAVE_AS_JPG:\n        cv2.imwrite(\n            str(img_out_path),\n            img_resized,\n            [cv2.IMWRITE_JPEG_QUALITY, JPG_QUALITY]\n        )\n    else:\n        cv2.imwrite(\n            str(img_out_path),\n            img_resized,\n            [cv2.IMWRITE_PNG_COMPRESSION, 0]\n        )\n\n    yolo_lines = build_yolo_lines(\n        boxes=boxes,\n        orig_w=orig_w,\n        orig_h=orig_h,\n        img_size=IMG_SIZE,\n    )\n\n    with open(lbl_out_path, \"w\") as f:\n        f.write(\"\\n\".join(yolo_lines))\n\n\ndef process_one_dicom(record):\n    t0 = time.perf_counter()\n\n    sample_id = str(record[\"sample_id\"])\n    dicom_path = Path(record[\"path\"])\n    split = split_map.get(sample_id)\n    if split is None:\n        return {\n            \"image_id\": sample_id,\n            \"ok\": False,\n            \"reason\": \"not_in_split\",\n            \"elapsed\": 0.0,\n            \"num_saved\": 0,\n        }\n\n    boxes = ann_map.get(sample_id, [])\n\n    # đọc ảnh gốc\n    img_raw = read_dicom_to_uint8(dicom_path)\n\n    try:\n        # 1) luôn lưu ảnh gốc\n        save_image_and_label(\n            img=img_raw,\n            boxes=boxes,\n            image_id=sample_id,\n            split=split,\n        )\n\n        num_saved = 1\n\n        # 2) nếu là train thì lưu thêm ảnh augment\n        if split == \"train\" and len(boxes) > 0:\n            img_aug, boxes_aug = apply_dicom_augmentation(\n                img_raw.copy(),\n                boxes,\n                split,\n            )\n\n            save_image_and_label(\n                img=img_aug,\n                boxes=boxes_aug,\n                image_id=f\"{sample_id}_aug\",\n                split=split,\n            )\n            num_saved += 1\n\n        elapsed = time.perf_counter() - t0\n        return {\n            \"image_id\": sample_id,\n            \"ok\": True,\n            \"reason\": \"done\",\n            \"elapsed\": elapsed,\n            \"num_saved\": num_saved,\n        }\n\n    except Exception as e:\n        elapsed = time.perf_counter() - t0\n        return {\n            \"image_id\": sample_id,\n            \"ok\": False,\n            \"reason\": str(e),\n            \"elapsed\": elapsed,\n            \"num_saved\": 0,\n        }","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:20.135088Z","iopub.execute_input":"2026-04-25T05:33:20.135316Z","iopub.status.idle":"2026-04-25T05:33:20.159411Z","shell.execute_reply.started":"2026-04-25T05:33:20.135294Z","shell.execute_reply":"2026-04-25T05:33:20.158803Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Chạy export song song + ETA","metadata":{}},{"cell_type":"code","source":"max_workers = min(8, os.cpu_count() or 4)\nprint(\"max_workers =\", max_workers)\n\ntotal_files = len(dicom_records_selected)\nstart_all = time.perf_counter()\n\ndone = 0\nok_count = 0\ntimes = []\n\nlog_every = 20\n\nwith ThreadPoolExecutor(max_workers=max_workers) as executor:\n    futures = [executor.submit(process_one_dicom, record) for record in dicom_records_selected]\n\n    for future in as_completed(futures):\n        result = future.result()\n        done += 1\n\n        if result[\"ok\"]:\n            ok_count += 1\n            times.append(result[\"elapsed\"])\n\n        if done % log_every == 0 or done == total_files:\n            elapsed_all = time.perf_counter() - start_all\n            avg_per_file = elapsed_all / done\n            speed = done / elapsed_all if elapsed_all > 0 else 0.0\n            remaining = total_files - done\n            eta_sec = remaining * avg_per_file\n\n            avg_worker = np.mean(times) if len(times) > 0 else 0.0\n            p50_worker = np.median(times) if len(times) > 0 else 0.0\n\n            print(\n                f\"[{done}/{total_files}] \"\n                f\"ok={ok_count} | \"\n                f\"wall={elapsed_all:.1f}s | \"\n                f\"speed={speed:.2f} img/s | \"\n                f\"avg_wall/file={avg_per_file:.3f}s | \"\n                f\"avg_worker={avg_worker:.3f}s | \"\n                f\"p50_worker={p50_worker:.3f}s | \"\n                f\"ETA={eta_sec:.1f}s\"\n            )\n\ntotal_elapsed = time.perf_counter() - start_all\nprint(\"\\nDONE\")\nprint(f\"Processed: {done}/{total_files}\")\nprint(f\"Success:   {ok_count}\")\nprint(f\"Total time: {total_elapsed:.2f}s\")\nprint(f\"Overall speed: {done / total_elapsed:.2f} img/s\")","metadata":{"execution":{"iopub.status.busy":"2026-04-25T05:33:20.160266Z","iopub.execute_input":"2026-04-25T05:33:20.160634Z","iopub.status.idle":"2026-04-25T06:22:07.2702Z","shell.execute_reply.started":"2026-04-25T05:33:20.160599Z","shell.execute_reply":"2026-04-25T06:22:07.269226Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_ext = \"*.jpg\" if SAVE_AS_JPG else \"*.png\"\n\nprint(\"Used source train images:\", len(train_ids))\nprint(\"Used source val images:\", len(val_ids))\nprint(\"Used source test images:\", len(test_ids))\nprint(\"Train images:\", len(list(IMG_TRAIN_DIR.glob(img_ext))))\nprint(\"Val images:\", len(list(IMG_VAL_DIR.glob(img_ext))))\nprint(\"Test images:\", len(list(IMG_TEST_DIR.glob(img_ext))))\nprint(\"Train labels:\", len(list(LBL_TRAIN_DIR.glob(\"*.txt\"))))\nprint(\"Val labels:\", len(list(LBL_VAL_DIR.glob(\"*.txt\"))))\nprint(\"Test labels:\", len(list(LBL_TEST_DIR.glob(\"*.txt\"))))","metadata":{"execution":{"iopub.status.busy":"2026-04-25T06:22:07.271669Z","iopub.execute_input":"2026-04-25T06:22:07.271893Z","iopub.status.idle":"2026-04-25T06:22:07.340588Z","shell.execute_reply.started":"2026-04-25T06:22:07.271872Z","shell.execute_reply":"2026-04-25T06:22:07.340001Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tạo data.yaml","metadata":{}},{"cell_type":"code","source":"data_yaml = {\n    \"path\": str(OUT_ROOT),\n    \"train\": \"images/train\",\n    \"val\": \"images/val\",\n    \"test\": \"images/test\",\n    \"names\": {i: name for i, name in enumerate(CLASS_NAMES)}\n}\n\nyaml_path = OUT_ROOT / \"data.yaml\"\nwith open(yaml_path, \"w\") as f:\n    yaml.dump(data_yaml, f, sort_keys=False)\n\nprint(\"Saved:\", yaml_path)\nprint(yaml_path.read_text())","metadata":{"execution":{"iopub.status.busy":"2026-04-25T06:22:07.341544Z","iopub.execute_input":"2026-04-25T06:22:07.341838Z","iopub.status.idle":"2026-04-25T06:22:07.348417Z","shell.execute_reply.started":"2026-04-25T06:22:07.341814Z","shell.execute_reply":"2026-04-25T06:22:07.347768Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor i in range(10):\n    img_ext = \".jpg\" if SAVE_AS_JPG else \".png\"\n    sample_image_path = list(IMG_TRAIN_DIR.glob(f\"*{img_ext}\"))[i]\n    sample_label_path = LBL_TRAIN_DIR / f\"{sample_image_path.stem}.txt\"\n    \n    img = cv2.imread(str(sample_image_path))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    \n    h, w = img.shape[:2]\n    \n    if sample_label_path.exists():\n        lines = sample_label_path.read_text().strip().splitlines()\n        for line in lines:\n            parts = line.strip().split()\n            if len(parts) != 5:\n                continue\n    \n            cls_id = int(float(parts[0]))\n            xc, yc, bw, bh = map(float, parts[1:])\n    \n            x1 = int((xc - bw/2) * w)\n            y1 = int((yc - bh/2) * h)\n            x2 = int((xc + bw/2) * w)\n            y2 = int((yc + bh/2) * h)\n    \n            cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2)\n            cv2.putText(\n                img,\n                ID2CLASS[cls_id],\n                (x1, max(20, y1 - 5)),\n                cv2.FONT_HERSHEY_SIMPLEX,\n                0.6,\n                (255, 0, 0),\n                2\n            )\n    \n    plt.figure(figsize=(10, 10))\n    plt.imshow(img)\n    plt.title(sample_image_path.name)\n    plt.axis(\"off\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2026-04-25T06:22:07.349385Z","iopub.execute_input":"2026-04-25T06:22:07.349656Z","iopub.status.idle":"2026-04-25T06:22:10.383061Z","shell.execute_reply.started":"2026-04-25T06:22:07.349635Z","shell.execute_reply":"2026-04-25T06:22:10.382185Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom ultralytics import YOLO\n\ndevice = 0 if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", device)\n\n# model nhỏ để test nhanh\nmodel = YOLO(\"yolov8n.pt\")\n\nmodel.train(\n    data=str(yaml_path),\n    epochs=50,               # test nhanh, có thể đổi 1 / 3 / 10\n    imgsz=IMG_SIZE,\n    batch=8,                # nếu thiếu VRAM thì giảm xuống 4\n    device=device,\n    project=str(OUT_ROOT / \"runs\"),\n    name=\"yolov8n_multiclass_test\",\n    exist_ok=True\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Eval trên test split đã được tạo từ bước gộp train + test rồi stratified split.\n# Không preprocess lại test CSV ở đây nữa, tránh ghi đè test split mới.\n\nimport torch\nfrom ultralytics import YOLO\n\nEVAL_BATCH = 8\nTRAIN_RUN_NAME = \"yolov8n_multiclass_test\"  # sửa nếu run name khác\nBEST_WEIGHT_PATH = OUT_ROOT / \"runs\" / TRAIN_RUN_NAME / \"weights\" / \"best.pt\"\n\nif not BEST_WEIGHT_PATH.exists():\n    raise FileNotFoundError(f\"Không thấy best weight: {BEST_WEIGHT_PATH}\")\n\nbest_model = YOLO(str(BEST_WEIGHT_PATH))\ndevice = 0 if torch.cuda.is_available() else \"cpu\"\nmetrics = best_model.val(\n    data=str(yaml_path),\n    split=\"test\",\n    imgsz=IMG_SIZE,\n    batch=EVAL_BATCH,\n    device=device,\n    verbose=True,\n)\n\nprint(\"Test metrics:\")\nprint({\n    \"precision\": float(metrics.box.mp),\n    \"recall\": float(metrics.box.mr),\n    \"map50\": float(metrics.box.map50),\n    \"map50_95\": float(metrics.box.map),\n})","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}