{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":16226735,"datasetId":10404334,"databundleVersionId":17208131}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"    working/\n    ├── vinbig_yolo/\n    │\n    ├── images/\n    │   ├── train/\n    │   ├── val/\n    │\n    ├── labels/\n    │   ├── train/\n    │   ├── val/\n    │\n    ├── data.yaml","metadata":{}},{"cell_type":"code","source":"pip install dicomsdl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:21:11.815256Z","iopub.execute_input":"2026-08-07T08:21:11.815776Z","iopub.status.idle":"2026-08-07T08:21:16.557825Z","shell.execute_reply.started":"2026-08-07T08:21:11.815747Z","shell.execute_reply":"2026-08-07T08:21:16.5568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport dicomsdl\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom concurrent.futures import ProcessPoolExecutor\nfrom sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:21:20.592593Z","iopub.execute_input":"2026-08-07T08:21:20.593153Z","iopub.status.idle":"2026-08-07T08:21:22.04584Z","shell.execute_reply.started":"2026-08-07T08:21:20.593116Z","shell.execute_reply":"2026-08-07T08:21:22.045236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================================================\n# CONFIG\n# =====================================================\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport dicomsdl\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom concurrent.futures import ProcessPoolExecutor\n\nROOT = \"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\"\nTRAIN_DIR = f\"{ROOT}/train\"\nCSV_PATH = f\"{ROOT}/train.csv\"\nOUT_ROOT = \"/kaggle/working/vinbig_yolo\"\n\nIMG_SIZE = 768\nJPEG_QUALITY = 85\nNUM_WORKERS = 4\n\n# =====================================================\n# OUTPUT FOLDER\n# =====================================================\nfor split in [\"train\", \"val\"]:\n    os.makedirs(f\"{OUT_ROOT}/images/{split}\", exist_ok=True)\n    os.makedirs(f\"{OUT_ROOT}/labels/{split}\", exist_ok=True)\n\n# =====================================================\n# LOAD CSV\n# =====================================================\ndf = pd.read_csv(CSV_PATH)\ngrouped = df.groupby(\"image_id\")\nimage_ids = list(grouped.groups.keys())\n\n# =====================================================\n# SPLIT train val\n# =====================================================\ntrain_ids, val_ids = train_test_split(\n    image_ids,\n    test_size=0.2,\n    random_state=42\n)\ntrain_set = set(train_ids)\n\n# =====================================================\n# CLAHE\n# =====================================================\nclahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(4,4))\n\n# =====================================================\n# NORMALIZE\n# =====================================================\ndef normalize(img):\n    img = img.astype(np.float32)\n    img -= img.min()\n    img /= (img.max() + 1e-6)\n    img *= 255\n    return img.astype(np.uint8)\n\n# =====================================================\n# PROCESS ONE SAMPLE\n# =====================================================\ndef process_one(image_id):\n    try:\n        dicom_path = f\"{TRAIN_DIR}/{image_id}.dicom\"\n        ds = dicomsdl.open(dicom_path)\n        img = ds.pixelData()\n        h0, w0 = img.shape\n\n        img = normalize(img)\n        img = clahe.apply(img)\n        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE), interpolation=cv2.INTER_AREA)\n\n        split = \"train\" if image_id in train_set else \"val\"\n\n        # save jpg image\n        img_out = f\"{OUT_ROOT}/images/{split}/{image_id}.jpg\"\n        cv2.imwrite(img_out, img, [cv2.IMWRITE_JPEG_QUALITY, JPEG_QUALITY])\n\n        # generate yolo label txt\n        rows = grouped.get_group(image_id)\n        lines = []\n        for _, row in rows.iterrows():\n            cls = int(row[\"class_id\"])\n            if cls == 14:  # No finding 跳过\n                continue\n            x1 = row[\"x_min\"]\n            y1 = row[\"y_min\"]\n            x2 = row[\"x_max\"]\n            y2 = row[\"y_max\"]\n\n            # yolo normalized cx cy w h (基于原始dicom尺寸，不是resize后)\n            xc = ((x1 + x2) / 2) / w0\n            yc = ((y1 + y2) / 2) / h0\n            bw = (x2 - x1) / w0\n            bh = (y2 - y1) / h0\n            lines.append(f\"{cls} {xc:.6f} {yc:.6f} {bw:.6f} {bh:.6f}\")\n\n        label_out = f\"{OUT_ROOT}/labels/{split}/{image_id}.txt\"\n        with open(label_out, \"w\", encoding=\"utf8\") as f:\n            f.write(\"\\n\".join(lines))\n\n    except Exception as e:\n        print(f\"ERROR {image_id}: {str(e)}\")\n\n# =====================================================\n# MULTIPROCESS RUN\n# =====================================================\nif __name__ == \"__main__\":\n    with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor:\n        list(tqdm(\n            executor.map(process_one, image_ids, chunksize=32),\n            total=len(image_ids)\n        ))\n    print(\"✅ DONE convert vinbigdata to YOLO format\")\n\n    # 生成yolo dataset yaml\n    yaml_text = f\"\"\"path: {OUT_ROOT}\ntrain: images/train\nval: images/val\nnames:\n  0: Aortic enlargement\n  1: Atelectasis\n  2: Calcification\n  3: Cardiomegaly\n  4: Consolidation\n  5: ILD\n  6: Infiltration\n  7: Lung Opacity\n  8: Nodule/Mass\n  9: Pleural effusion\n  10: Pleural thickening\n  11: Pneumothorax\n  12: Pulmonary fibrosis\n  13: Other lesion\n\"\"\"\n    with open(os.path.join(OUT_ROOT, \"vinbig.yaml\"),\"w\",encoding=\"utf8\") as f:\n        f.write(yaml_text)\n    print(f\"✅ yaml saved at {OUT_ROOT}/vinbig.yaml\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:21:30.92376Z","iopub.execute_input":"2026-08-07T08:21:30.92449Z","iopub.status.idle":"2026-08-07T09:05:18.249604Z","shell.execute_reply.started":"2026-08-07T08:21:30.92446Z","shell.execute_reply":"2026-08-07T09:05:18.248664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # preprocess xong\n!tar -czf /kaggle/working/vinbig_yolo.tar.gz \\\n     -C /kaggle/working vinbig_yolo\n\n# # xóa folder gốc\n!rm -rf /kaggle/working/vinbig_yolo\nprint(\"DONE\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T09:06:12.327815Z","iopub.execute_input":"2026-08-07T09:06:12.328454Z","iopub.status.idle":"2026-08-07T09:07:07.718046Z","shell.execute_reply.started":"2026-08-07T09:06:12.328411Z","shell.execute_reply":"2026-08-07T09:07:07.717218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_yaml = \"\"\"\npath: /kaggle/input/datasets/keyshiftf/vinbig-yolo/vinbig_yolo\n\ntrain: images/train\nval: images/val\n\nnames:\n  0: Aortic enlargement\n  1: Atelectasis\n  2: Calcification\n  3: Cardiomegaly\n  4: Consolidation\n  5: ILD\n  6: Infiltration\n  7: Lung Opacity\n  8: Nodule/Mass\n  9: Other lesion\n  10: Pleural effusion\n  11: Pleural thickening\n  12: Pneumothorax\n  13: Pulmonary fibrosis\n\"\"\"\n\nwith open(\n    \"/kaggle/working/data.yaml\",\n    \"w\"\n) as f:\n\n    f.write(data_yaml)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T09:08:02.729699Z","iopub.execute_input":"2026-08-07T09:08:02.73059Z","iopub.status.idle":"2026-08-07T09:08:02.735717Z","shell.execute_reply.started":"2026-08-07T09:08:02.730533Z","shell.execute_reply":"2026-08-07T09:08:02.734951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T09:08:07.489382Z","iopub.execute_input":"2026-08-07T09:08:07.489987Z","iopub.status.idle":"2026-08-07T09:08:13.079765Z","shell.execute_reply.started":"2026-08-07T09:08:07.489957Z","shell.execute_reply":"2026-08-07T09:08:13.078832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nOptimized YOLOv8 Training Script for VinDr‑CXR Chest X‑ray (Kaggle T4x2)\nTarget: Improve mAP50‑95 for medical lesion detection\nFix kernel cache warning, DDP dual‑gpu ready, medical‑friendly augmentation\n\"\"\"\n# ========== 必须放在所有torch/ultralytics导入之前！修复kernel缓存警告 ==========\nimport os\nos.environ[\"TORCH_KERNEL_CACHE_PATH\"] = \"/tmp/torch_kernels\"\nos.makedirs(\"/tmp/torch_kernels\", exist_ok=True)\n\nfrom ultralytics import YOLO\nfrom pathlib import Path\nimport torch\nimport yaml\n\n\ndef detect_kaggle_gpu():\n    \"\"\"Detect available GPUs on Kaggle\"\"\"\n    if torch.cuda.is_available():\n        gpu_count = torch.cuda.device_count()\n        gpu_name = torch.cuda.get_device_name(0)\n        total_mem = torch.cuda.get_device_properties(0).total_memory / 1e9\n        print(f\"GPUs detected: {gpu_count}\")\n        print(f\"GPU Model: {gpu_name}\")\n        print(f\"Total Memory: {total_mem:.1f} GB\")\n        return gpu_count\n    else:\n        print(\"No GPU detected, using CPU (SLOW)\")\n        return 0\n\n\ndef analyze_dataset(data_yaml_path):\n    \"\"\"Analyze dataset class distribution\"\"\"\n    if not Path(data_yaml_path).exists():\n        print(f\"{data_yaml_path} not found, skipping analysis\")\n        return\n\n    with open(data_yaml_path, \"r\", encoding=\"utf-8\") as f:\n        data = yaml.safe_load(f)\n\n    print(\"\\nDataset Configuration:\")\n    print(f\"  Classes(nc): {data.get('nc', '?')}\")\n    print(f\"  Class names: {list(data.get('names', {}).values())}\")\n\n\ndef get_optimal_config(gpu_count):\n    \"\"\"Get optimal training config based on GPU resource\"\"\"\n    configs = {\n        \"t4_single\": {\n            \"batch\": 8,\n            \"workers\": 4,\n            \"imgsz\": 640,\n            \"nbs\": 16,       # 等效基准batch\n            \"device\": 0,\n            \"note\": \"T4 (16GB) - Single GPU\"\n        },\n        \"t4_dual\": {\n            \"batch\": 16,\n            \"workers\": 8,\n            \"imgsz\": 768,\n            \"nbs\": 32,       # batch16 + accumulate2 → 等效32\n            \"device\": [0, 1],\n            \"note\": \"T4x2 (32GB) - Dual GPU\"\n        },\n        \"p100\": {\n            \"batch\": 32,\n            \"workers\": 8,\n            \"imgsz\": 1024,\n            \"nbs\": 32,\n            \"device\": 0,\n            \"note\": \"P100 (16GB)\"\n        }\n    }\n\n    if gpu_count >= 2:\n        return configs[\"t4_dual\"]\n    elif gpu_count == 1:\n        mem_gb = torch.cuda.get_device_properties(0).total_memory / 1e9\n        if mem_gb > 15:\n            return configs[\"p100\"]\n    return configs[\"t4_single\"]\n\n\ndef main():\n    print(\"=\" * 70)\n    print(\"YOLOv8 Medical X‑ray Detection Training (VinDr‑CXR Optimized)\")\n    print(\"=\" * 70)\n\n    gpu_count = detect_kaggle_gpu()\n    config = get_optimal_config(gpu_count)\n    print(f\"\\n🔧 Running Config: {config['note']}\")\n\n    # ========= 修改这里为你的data.yaml真实路径 =========\n    data_yaml = \"/kaggle/working/data.yaml\"\n    analyze_dataset(data_yaml)\n\n    # 模型选择：双T4可以直接切换 yolov8l.pt 获取更高精度\n    model_name = \"yolov8m.pt\"\n    if gpu_count >= 2:\n        print(\"\\n💡 Tip: Dual T4 available, try model_name='yolov8l.pt' for better lesion feature\")\n\n    model = YOLO(model_name)\n\n    print(\"\\n\" + \"=\" * 70)\n    print(\"Start Training\")\n    print(\"=\" * 70)\n\n    train_config = {\n        # -------- Dataset --------\n        \"data\": data_yaml,\n\n        # -------- Basic Training --------\n        \"epochs\": 50,\n        \"imgsz\": config[\"imgsz\"],\n        \"batch\": config[\"batch\"],\n        \"device\": config[\"device\"],\n        \"workers\": config[\"workers\"],\n\n        # -------- LR Schedule (Cosine for medical task) --------\n        \"lr0\": 8e-5,\n        \"lrf\": 0.005,\n        \"cos_lr\": True,\n        \"warmup_epochs\": 4.0,\n        \"warmup_bias_lr\": 0.1,\n        \"warmup_momentum\": 0.8,\n\n        # -------- Optimizer --------\n        \"optimizer\": \"AdamW\",\n        \"weight_decay\": 5e-4,\n        \"momentum\": 0.937,\n\n        # -------- Loss Weights | Key for improving mAP50‑95 --------\n        \"iou\": 0.7,\n        \"box\": 8.0,\n        \"cls\": 0.5,\n        \"dfl\": 1.5,\n        \"label_smoothing\": 0.1,\n\n        # -------- Early Stopping --------\n        \"patience\": 15,\n\n        # -------- Performance --------\n        \"amp\": True,\n        \"cache\": \"disk\",   # Kaggle input readonly, use disk cache under /kaggle/working\n        \"rect\": True,\n        \"nbs\": config[\"nbs\"],\n\n        # -------- Augmentation: Medical X‑ray friendly (reduce heavy aug) --------\n        \"fliplr\": 0.5,\n        \"flipud\": 0.0,    # Chest X‑ray do NOT flip vertically\n        \"degrees\": 5,\n        \"translate\": 0.08,\n        \"scale\": 0.15,\n        \"mosaic\": 0.8,\n        \"mixup\": 0.1,\n        \"copy_paste\": 0.3,\n        \"hsv_h\": 0.01,\n        \"hsv_s\": 0.5,\n        \"hsv_v\": 0.3,\n        \"erasing\": 0.2,\n        \"dropout\": 0.0,\n\n        # -------- Save & Validation --------\n        \"val\": True,\n        \"save\": True,\n        \"plots\": True,\n        \"save_period\": 5,\n        \"exist_ok\": True,\n\n        # -------- Reproducibility --------\n        \"seed\": 42,\n        \"deterministic\": True,\n        \"verbose\": True\n    }\n\n    print(\"\\n👉 Training Params Summary:\")\n    print(f\"  Model         : {model_name}\")\n    print(f\"  Batch         : {train_config['batch']}\")\n    print(f\"  Imgsz         : {train_config['imgsz']}\")\n    print(f\"  NBS(effective batch ref): {train_config['nbs']}\")\n    print(f\"  LR            : {train_config['lr0']} → {train_config['lrf']*train_config['lr0']}\")\n    print(f\"  box/cls/dfl   : {train_config['box']}/{train_config['cls']}/{train_config['dfl']}\")\n    print(f\"  Device        : {train_config['device']}\")\n    print(f\"  patience(early‑stop): {train_config['patience']}\")\n\n    results = model.train(**train_config)\n\n    print(\"\\n\" + \"=\" * 70)\n    print(\"✅ Training Finished\")\n    print(f\"Best weights: runs/detect/train/weights/best.pt\")\n    print(f\"Metrics file: runs/detect/train/results.csv\")\n    print(\"=\" * 70)\n    return results\n\n\nif __name__ == \"__main__\":\n    res = main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:07:05.437819Z","iopub.execute_input":"2026-08-07T10:07:05.438462Z","iopub.status.idle":"2026-08-07T14:38:10.674273Z","shell.execute_reply.started":"2026-08-07T10:07:05.438425Z","shell.execute_reply":"2026-08-07T14:38:10.673564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\n\nmodel = YOLO(\n    \"/kaggle/working/runs/detect/train/weights/best.pt\"\n)\n\nmetrics = model.val()\n\nprint(metrics)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T14:39:01.945511Z","iopub.execute_input":"2026-08-07T14:39:01.946258Z","iopub.status.idle":"2026-08-07T14:41:01.865935Z","shell.execute_reply.started":"2026-08-07T14:39:01.946225Z","shell.execute_reply":"2026-08-07T14:41:01.865008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\n\nmodel = YOLO(\n    \"/kaggle/working/runs/detect/train/weights/best.pt\"\n)\n\nmetrics = model.val(\n    plots=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T14:41:26.533769Z","iopub.execute_input":"2026-08-07T14:41:26.534947Z","iopub.status.idle":"2026-08-07T14:43:20.461089Z","shell.execute_reply.started":"2026-08-07T14:41:26.534889Z","shell.execute_reply":"2026-08-07T14:43:20.459913Z"}},"outputs":[],"execution_count":null}]}