{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# STEP 1 - VINBIGDATA DATA PREP FOR YOLO\n# DICOM -> PNG CLAHE -> WBF LABELS -> YOLO FORMAT -> 5-FOLD -> data.yaml\n# =========================================================\n\n!pip install -q pydicom ensemble-boxes iterative-stratification","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T19:09:29.060173Z","iopub.execute_input":"2026-05-12T19:09:29.060514Z","iopub.status.idle":"2026-05-12T19:09:34.876976Z","shell.execute_reply.started":"2026-05-12T19:09:29.060473Z","shell.execute_reply":"2026-05-12T19:09:34.875579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport shutil\nimport pydicom\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom ensemble_boxes import weighted_boxes_fusion\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.pyplot as plt\nimport random\n\n# =========================================================\n# CONFIG\n# =========================================================\n\nINPUT_DIR = Path(\"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\")\nTRAIN_CSV = INPUT_DIR / \"train.csv\"\nTRAIN_DICOM_DIR = INPUT_DIR / \"train\"\n\nOUTPUT_DIR = Path(\"/kaggle/working/vinbigdata_yolo\")\nIMAGE_OUT_DIR = OUTPUT_DIR / \"images\"\nLABEL_OUT_DIR = OUTPUT_DIR / \"labels\"\n\nIMAGE_OUT_DIR.mkdir(parents=True, exist_ok=True)\nLABEL_OUT_DIR.mkdir(parents=True, exist_ok=True)\n\nMAX_SIZE = 1536\nN_FOLDS = 5\nFOLD = 0\n\nWBF_IOU_THR = 0.45\nWBF_SKIP_BOX_THR = 0.0001\n\nUSE_CLAHE = True\nUSE_SYMLINK = True\nSEED = 42\n\nnp.random.seed(SEED)\nrandom.seed(SEED)\n\n# =========================================================\n# READ CSV\n# =========================================================\n\ndf = pd.read_csv(TRAIN_CSV)\n\nprint(\"Train CSV shape:\", df.shape)\nprint(df.head())\nprint(\"\\nClass distribution:\")\nprint(df[\"class_id\"].value_counts().sort_index())\n\n# =========================================================\n# CLASS NAMES\n# YOLO chỉ dùng class 0 -> 13, bỏ class 14 No finding\n# =========================================================\n\nclass_df = (\n    df[df[\"class_id\"] != 14][[\"class_id\", \"class_name\"]]\n    .drop_duplicates()\n    .sort_values(\"class_id\")\n)\n\nCLASS_NAMES = class_df[\"class_name\"].tolist()\n\nprint(\"\\nDetection classes:\")\nfor i, name in enumerate(CLASS_NAMES):\n    print(i, name)\n\n# =========================================================\n# DICOM -> PNG WITH CLAHE\n# =========================================================\n\ndef read_xray_dicom(path):\n    \"\"\"\n    Đọc DICOM X-ray và trả về ảnh uint8 grayscale.\n    Có xử lý VOI LUT và MONOCHROME1.\n    \"\"\"\n    dicom = pydicom.dcmread(str(path))\n\n    try:\n        img = apply_voi_lut(dicom.pixel_array, dicom)\n    except Exception:\n        img = dicom.pixel_array\n\n    img = img.astype(np.float32)\n\n    if getattr(dicom, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n        img = np.max(img) - img\n\n    img -= np.min(img)\n    max_val = np.max(img)\n\n    if max_val > 0:\n        img /= max_val\n\n    img = (img * 255).astype(np.uint8)\n    return img\n\n\ndef apply_clahe(img, clip_limit=2.0, tile_grid_size=(8, 8)):\n    \"\"\"\n    CLAHE giúp tăng tương phản cục bộ cho ảnh X-ray.\n    \"\"\"\n    clahe = cv2.createCLAHE(\n        clipLimit=clip_limit,\n        tileGridSize=tile_grid_size\n    )\n    return clahe.apply(img)\n\n\ndef resize_keep_aspect(img, max_size=1536):\n    \"\"\"\n    Resize giữ nguyên tỉ lệ.\n    Nếu ảnh nhỏ hơn max_size thì giữ nguyên.\n    \"\"\"\n    h, w = img.shape[:2]\n    scale = max_size / max(h, w)\n\n    if scale >= 1:\n        return img, 1.0\n\n    new_w = int(w * scale)\n    new_h = int(h * scale)\n\n    resized = cv2.resize(img, (new_w, new_h), interpolation=cv2.INTER_AREA)\n    return resized, scale\n\n\ndef convert_dicom_to_png(image_id, save_dir, max_size=1536, use_clahe=True):\n    \"\"\"\n    Convert một ảnh DICOM sang PNG.\n    Trả về width, height sau khi resize.\n    \"\"\"\n    dicom_path = TRAIN_DICOM_DIR / f\"{image_id}.dicom\"\n    save_path = save_dir / f\"{image_id}.png\"\n\n    if save_path.exists():\n        img = cv2.imread(str(save_path), cv2.IMREAD_GRAYSCALE)\n        h, w = img.shape[:2]\n        return w, h\n\n    img = read_xray_dicom(dicom_path)\n\n    if use_clahe:\n        img = apply_clahe(img)\n\n    img, _ = resize_keep_aspect(img, max_size=max_size)\n\n    cv2.imwrite(str(save_path), img)\n\n    h, w = img.shape[:2]\n    return w, h\n\n\nimage_ids = sorted(df[\"image_id\"].unique())\n\nimage_meta = []\n\nfor image_id in tqdm(image_ids, desc=\"Converting DICOM to PNG\"):\n    w, h = convert_dicom_to_png(\n        image_id=image_id,\n        save_dir=IMAGE_OUT_DIR,\n        max_size=MAX_SIZE,\n        use_clahe=USE_CLAHE\n    )\n\n    image_meta.append({\n        \"image_id\": image_id,\n        \"width\": w,\n        \"height\": h\n    })\n\nmeta_df = pd.DataFrame(image_meta)\nmeta_df.to_csv(OUTPUT_DIR / \"image_meta.csv\", index=False)\n\nprint(\"\\nImage meta saved:\", OUTPUT_DIR / \"image_meta.csv\")\nprint(meta_df.head())\n\n# =========================================================\n# WBF LABEL MERGING\n# =========================================================\n\nmeta_map = meta_df.set_index(\"image_id\")[[\"width\", \"height\"]].to_dict(\"index\")\n\n\ndef clip_box(box):\n    \"\"\"\n    Giới hạn bbox trong [0, 1].\n    \"\"\"\n    x1, y1, x2, y2 = box\n\n    x1 = min(max(float(x1), 0.0), 1.0)\n    y1 = min(max(float(y1), 0.0), 1.0)\n    x2 = min(max(float(x2), 0.0), 1.0)\n    y2 = min(max(float(y2), 0.0), 1.0)\n\n    if x2 < x1:\n        x1, x2 = x2, x1\n\n    if y2 < y1:\n        y1, y2 = y2, y1\n\n    return [x1, y1, x2, y2]\n\n\ndef run_wbf_for_image(img_df, width, height):\n    \"\"\"\n    WBF cho một image_id.\n    Mỗi rad_id được xem như một annotator/model.\n    \"\"\"\n    pos_df = img_df[img_df[\"class_id\"] != 14].copy()\n\n    if len(pos_df) == 0:\n        return []\n\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n\n    for rad_id, rad_df in pos_df.groupby(\"rad_id\"):\n        boxes = []\n        scores = []\n        labels = []\n\n        for _, row in rad_df.iterrows():\n            x1 = row[\"x_min\"] / width\n            y1 = row[\"y_min\"] / height\n            x2 = row[\"x_max\"] / width\n            y2 = row[\"y_max\"] / height\n\n            box = clip_box([x1, y1, x2, y2])\n\n            if box[2] - box[0] <= 0 or box[3] - box[1] <= 0:\n                continue\n\n            boxes.append(box)\n            scores.append(1.0)\n            labels.append(int(row[\"class_id\"]))\n\n        if len(boxes) > 0:\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n\n    if len(boxes_list) == 0:\n        return []\n\n    boxes, scores, labels = weighted_boxes_fusion(\n        boxes_list=boxes_list,\n        scores_list=scores_list,\n        labels_list=labels_list,\n        weights=None,\n        iou_thr=WBF_IOU_THR,\n        skip_box_thr=WBF_SKIP_BOX_THR\n    )\n\n    results = []\n\n    for box, score, label in zip(boxes, scores, labels):\n        x1, y1, x2, y2 = clip_box(box.tolist())\n\n        results.append({\n            \"class_id\": int(label),\n            \"x_min\": x1,\n            \"y_min\": y1,\n            \"x_max\": x2,\n            \"y_max\": y2,\n            \"confidence\": float(score)\n        })\n\n    return results\n\n\nwbf_records = []\n\nfor image_id, img_df in tqdm(df.groupby(\"image_id\"), desc=\"Running WBF labels\"):\n    width = meta_map[image_id][\"width\"]\n    height = meta_map[image_id][\"height\"]\n\n    fused_boxes = run_wbf_for_image(img_df, width, height)\n\n    if len(fused_boxes) == 0:\n        wbf_records.append({\n            \"image_id\": image_id,\n            \"class_id\": 14,\n            \"x_min\": np.nan,\n            \"y_min\": np.nan,\n            \"x_max\": np.nan,\n            \"y_max\": np.nan,\n            \"confidence\": 1.0\n        })\n    else:\n        for b in fused_boxes:\n            wbf_records.append({\n                \"image_id\": image_id,\n                **b\n            })\n\nwbf_df = pd.DataFrame(wbf_records)\nwbf_df.to_csv(OUTPUT_DIR / \"train_wbf.csv\", index=False)\n\nprint(\"\\nWBF labels saved:\", OUTPUT_DIR / \"train_wbf.csv\")\nprint(wbf_df.head())\n\n# =========================================================\n# YOLO LABEL FORMAT\n# class_id x_center y_center width height\n# =========================================================\n\ndef bbox_xyxy_to_yolo(x1, y1, x2, y2):\n    \"\"\"\n    Input normalized xyxy.\n    Output normalized YOLO xywh.\n    \"\"\"\n    x_center = (x1 + x2) / 2\n    y_center = (y1 + y2) / 2\n    bw = x2 - x1\n    bh = y2 - y1\n\n    return x_center, y_center, bw, bh\n\n\ndef write_yolo_label_file(image_id, boxes_df, label_dir):\n    \"\"\"\n    Ghi file YOLO label.\n    Nếu ảnh No finding thì file txt rỗng.\n    \"\"\"\n    label_path = label_dir / f\"{image_id}.txt\"\n    lines = []\n\n    boxes_df = boxes_df[boxes_df[\"class_id\"] != 14]\n\n    for _, row in boxes_df.iterrows():\n        cls = int(row[\"class_id\"])\n\n        x1 = float(row[\"x_min\"])\n        y1 = float(row[\"y_min\"])\n        x2 = float(row[\"x_max\"])\n        y2 = float(row[\"y_max\"])\n\n        x_center, y_center, bw, bh = bbox_xyxy_to_yolo(x1, y1, x2, y2)\n\n        if bw <= 0 or bh <= 0:\n            continue\n\n        line = f\"{cls} {x_center:.6f} {y_center:.6f} {bw:.6f} {bh:.6f}\"\n        lines.append(line)\n\n    with open(label_path, \"w\") as f:\n        f.write(\"\\n\".join(lines))\n\n\nfor image_id, boxes_df in tqdm(wbf_df.groupby(\"image_id\"), desc=\"Writing YOLO labels\"):\n    write_yolo_label_file(\n        image_id=image_id,\n        boxes_df=boxes_df,\n        label_dir=LABEL_OUT_DIR\n    )\n\nfor image_id in tqdm(image_ids, desc=\"Checking empty labels\"):\n    label_path = LABEL_OUT_DIR / f\"{image_id}.txt\"\n\n    if not label_path.exists():\n        open(label_path, \"w\").close()\n\nprint(\"\\nYOLO labels saved:\", LABEL_OUT_DIR)\n\n# =========================================================\n# 5-FOLD SPLIT\n# =========================================================\n\n# =========================================================\n# 5-FOLD SPLIT (NÂNG CẤP: MULTILABEL STRATIFIED K-FOLD)\n# =========================================================\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nfold_df = pd.DataFrame({\"image_id\": image_ids})\nfold_df[\"fold\"] = -1\n\n# Tạo ma trận multi-hot cho 14 class (từ 0 đến 13)\n# Lưu ý: Các ảnh \"No finding\" sẽ có mảng toàn số 0, thuật toán vẫn chia đều được.\nmulti_hot = np.zeros((len(image_ids), 14), dtype=int)\n\nfor idx, image_id in enumerate(image_ids):\n    # Lấy các loại bệnh (khác 14) có trong ảnh này từ kết quả WBF\n    classes = wbf_df[(wbf_df[\"image_id\"] == image_id) & (wbf_df[\"class_id\"] != 14)][\"class_id\"].unique()\n    for c in classes:\n        multi_hot[idx, int(c)] = 1\n\nmskf = MultilabelStratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, val_idx) in enumerate(mskf.split(image_ids, multi_hot)):\n    fold_df.loc[val_idx, \"fold\"] = fold\n\nfold_df.to_csv(OUTPUT_DIR / \"folds.csv\", index=False)\n\nprint(\"\\nFolds saved:\", OUTPUT_DIR / \"folds.csv\")\nprint(fold_df[\"fold\"].value_counts().sort_index())\n# =========================================================\n# CREATE YOLO FOLD DATASET STRUCTURE\n# =========================================================\n\nYOLO_DATASET_DIR = OUTPUT_DIR / f\"yolo_fold{FOLD}\"\n\nfor split in [\"train\", \"val\"]:\n    (YOLO_DATASET_DIR / \"images\" / split).mkdir(parents=True, exist_ok=True)\n    (YOLO_DATASET_DIR / \"labels\" / split).mkdir(parents=True, exist_ok=True)\n\n\ndef safe_link_or_copy(src, dst, use_symlink=True):\n    \"\"\"\n    Tạo symlink để tiết kiệm dung lượng.\n    Nếu lỗi thì tự động copy.\n    \"\"\"\n    if dst.exists():\n        return\n\n    if use_symlink:\n        try:\n            os.symlink(src, dst)\n        except Exception:\n            shutil.copy(src, dst)\n    else:\n        shutil.copy(src, dst)\n\n\ntrain_ids = fold_df[fold_df[\"fold\"] != FOLD][\"image_id\"].tolist()\nval_ids = fold_df[fold_df[\"fold\"] == FOLD][\"image_id\"].tolist()\n\nprint(\"\\nTrain images:\", len(train_ids))\nprint(\"Val images:\", len(val_ids))\n\nfor split, ids in [(\"train\", train_ids), (\"val\", val_ids)]:\n    for image_id in tqdm(ids, desc=f\"Preparing {split} split\"):\n        src_img = IMAGE_OUT_DIR / f\"{image_id}.png\"\n        dst_img = YOLO_DATASET_DIR / \"images\" / split / f\"{image_id}.png\"\n\n        src_lbl = LABEL_OUT_DIR / f\"{image_id}.txt\"\n        dst_lbl = YOLO_DATASET_DIR / \"labels\" / split / f\"{image_id}.txt\"\n\n        safe_link_or_copy(src_img, dst_img, use_symlink=USE_SYMLINK)\n        safe_link_or_copy(src_lbl, dst_lbl, use_symlink=USE_SYMLINK)\n\n# =========================================================\n# CREATE data.yaml\n# =========================================================\n\ndata_yaml = f\"\"\"\npath: {YOLO_DATASET_DIR}\ntrain: images/train\nval: images/val\n\nnc: {len(CLASS_NAMES)}\nnames:\n\"\"\"\n\nfor i, name in enumerate(CLASS_NAMES):\n    safe_name = str(name).replace(\"'\", \"\")\n    data_yaml += f\"  {i}: '{safe_name}'\\n\"\n\nyaml_path = YOLO_DATASET_DIR / \"data.yaml\"\n\nwith open(yaml_path, \"w\") as f:\n    f.write(data_yaml)\n\nprint(\"\\ndata.yaml saved:\", yaml_path)\nprint(data_yaml)\n\n# =========================================================\n# QUICK LABEL STATS\n# =========================================================\n\nlabel_counts = []\n\nfor txt_file in LABEL_OUT_DIR.glob(\"*.txt\"):\n    with open(txt_file, \"r\") as f:\n        lines = [line for line in f.read().splitlines() if line.strip()]\n    label_counts.append(len(lines))\n\nlabel_counts = np.array(label_counts)\n\nprint(\"\\nLabel statistics:\")\nprint(\"Total images:\", len(label_counts))\nprint(\"Images with boxes:\", int(np.sum(label_counts > 0)))\nprint(\"Images without boxes:\", int(np.sum(label_counts == 0)))\nprint(\"Average boxes/image:\", float(label_counts.mean()))\nprint(\"Max boxes/image:\", int(label_counts.max()))\n\nprint(\"\\nClass distribution after WBF:\")\nprint(wbf_df[wbf_df[\"class_id\"] != 14][\"class_id\"].value_counts().sort_index())\n\n# =========================================================\n# VISUALIZE SAMPLE\n# =========================================================\n\ndef visualize_yolo_sample(image_id, dataset_dir, split=\"train\"):\n    img_path = dataset_dir / \"images\" / split / f\"{image_id}.png\"\n    lbl_path = dataset_dir / \"labels\" / split / f\"{image_id}.txt\"\n\n    img = cv2.imread(str(img_path), cv2.IMREAD_GRAYSCALE)\n\n    if img is None:\n        print(\"Cannot read image:\", img_path)\n        return\n\n    img_rgb = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n    h, w = img.shape[:2]\n\n    if lbl_path.exists():\n        with open(lbl_path, \"r\") as f:\n            lines = f.read().strip().splitlines()\n\n        for line in lines:\n            if not line.strip():\n                continue\n\n            cls, xc, yc, bw, bh = line.split()\n            cls = int(cls)\n            xc, yc, bw, bh = map(float, [xc, yc, bw, bh])\n\n            x1 = int((xc - bw / 2) * w)\n            y1 = int((yc - bh / 2) * h)\n            x2 = int((xc + bw / 2) * w)\n            y2 = int((yc + bh / 2) * h)\n\n            cv2.rectangle(img_rgb, (x1, y1), (x2, y2), (255, 0, 0), 2)\n\n            cv2.putText(\n                img_rgb,\n                CLASS_NAMES[cls],\n                (x1, max(y1 - 5, 10)),\n                cv2.FONT_HERSHEY_SIMPLEX,\n                0.6,\n                (255, 0, 0),\n                2\n            )\n\n    plt.figure(figsize=(10, 10))\n    plt.imshow(img_rgb)\n    plt.axis(\"off\")\n    plt.title(f\"{split}: {image_id}\")\n    plt.show()\n\n\nsample_id = random.choice(train_ids)\nvisualize_yolo_sample(sample_id, YOLO_DATASET_DIR, split=\"train\")\n\n# =========================================================\n# FINAL CHECK\n# =========================================================\n\nprint(\"\\nSTEP 1 COMPLETED\")\nprint(\"OUTPUT_DIR:\", OUTPUT_DIR)\nprint(\"YOLO_DATASET_DIR:\", YOLO_DATASET_DIR)\nprint(\"DATA YAML:\", yaml_path)\nprint(\"Train images:\", len(list((YOLO_DATASET_DIR / 'images/train').glob('*.png'))))\nprint(\"Val images:\", len(list((YOLO_DATASET_DIR / 'images/val').glob('*.png'))))\nprint(\"Train labels:\", len(list((YOLO_DATASET_DIR / 'labels/train').glob('*.txt'))))\nprint(\"Val labels:\", len(list((YOLO_DATASET_DIR / 'labels/val').glob('*.txt'))))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T19:10:16.777319Z","iopub.execute_input":"2026-05-12T19:10:16.778106Z"}},"outputs":[],"execution_count":null}]}