{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":null,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"5dded8ee-1f34-4fc5-abdc-aad4145a1090","cell_type":"markdown","source":"# 第2步：VinDr-CXR数据清洗与YOLO标签生成（Kaggle）\n\n运行本Notebook前，请在`Add Input`中同时加入：\n\n1. 原始VinBigData/VinDr-CXR数据集（包含`train.csv`）；\n2. 第1步Notebook保存的数据集（包含`image_manifest.csv`和`images/`）。\n\n本Notebook完成：14类映射、无效框清理、等权多医生框融合、YOLO标签、\n训练/验证划分、论文框数量对照和最终审计。默认会把第1步PNG复制进最终输出，\n从而生成一个可独立挂载训练的完整YOLO数据集。\n\n","metadata":{}},{"id":"725e8559-24f6-40f5-8c58-5ae49d63bc8f","cell_type":"code","source":"import hashlib\nimport json\nimport random\nimport shutil\nfrom collections import Counter\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.auto import tqdm\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"12c73b55-fb33-4aa7-a031-bf0417490d7d","cell_type":"markdown","source":"## 1. 配置\n\n","metadata":{}},{"id":"d5bbee0c-e452-4529-b3b3-285f027e053a","cell_type":"code","source":"SEED = 2026\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\nINPUT_ROOT = Path(\"/kaggle/input\")\nOUTPUT_ROOT = Path(\"/kaggle/working/vindr_yolo_clean_v1\")\n\n# 自动发现失败或候选不唯一时，手工填写这两个路径。\n# RAW_DATA_ROOT示例：Path(\"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection\")\n# PREPROCESSED_ROOT示例：Path(\"/kaggle/input/vindr-preprocessed-v1\")\nRAW_DATA_ROOT = None\nPREPROCESSED_ROOT = None\n\n# True：复制PNG到最终输出，生成自包含YOLO数据集（推荐）。\n# False：标签与图片位于不同Kaggle输入中，不适合直接另开Notebook训练。\nCOPY_IMAGES_TO_FINAL_DATASET = True\n\n# 以下是本实验设置；YOLO-CXR论文未公开具体WBF阈值和验证集比例。\nWBF_IOU_THRESHOLD = 0.50\nMIN_RADIOLOGISTS = 1\nENFORCE_ONE_BOX_PER_RADIOLOGIST_PER_CLUSTER = True\nMIN_BOX_SIDE_PIXELS = 2.0\nVAL_FRACTION = 0.20\n\nCLASS_NAMES = [\n    \"Aortic enlargement\",\n    \"Atelectasis\",\n    \"Calcification\",\n    \"Cardiomegaly\",\n    \"Consolidation\",\n    \"ILD\",\n    \"Infiltration\",\n    \"Lung Opacity\",\n    \"Nodule/Mass\",\n    \"Other lesion\",\n    \"Pleural effusion\",\n    \"Pleural thickening\",\n    \"Pneumothorax\",\n    \"Pulmonary fibrosis\",\n]\nCLASS_TO_ID = {name: index for index, name in enumerate(CLASS_NAMES)}\nNO_FINDING_NAME = \"No finding\"\nCLASS_ALIASES = {name.casefold(): name for name in CLASS_NAMES}\nCLASS_ALIASES.update({\n    \"interstitial lung disease\": \"ILD\",\n    \"other lesions\": \"Other lesion\",\n    \"pneumo-thorax\": \"Pneumothorax\",\n    \"no finding\": NO_FINDING_NAME,\n})\n\nfor directory in [\n    OUTPUT_ROOT / \"images\" / \"trainval\",\n    OUTPUT_ROOT / \"images\" / \"test\",\n    OUTPUT_ROOT / \"labels\" / \"trainval\",\n    OUTPUT_ROOT / \"labels\" / \"test\",\n    OUTPUT_ROOT / \"reports\",\n    OUTPUT_ROOT / \"splits\",\n]:\n    directory.mkdir(parents=True, exist_ok=True)\n\nconfig = {\n    \"step\": \"02_data_cleaning_and_yolo_labels\",\n    \"seed\": SEED,\n    \"copy_images_to_final_dataset\": COPY_IMAGES_TO_FINAL_DATASET,\n    \"wbf_iou_threshold\": WBF_IOU_THRESHOLD,\n    \"minimum_radiologists\": MIN_RADIOLOGISTS,\n    \"enforce_one_box_per_radiologist_per_cluster\": ENFORCE_ONE_BOX_PER_RADIOLOGIST_PER_CLUSTER,\n    \"minimum_box_side_pixels\": MIN_BOX_SIDE_PIXELS,\n    \"validation_fraction\": VAL_FRACTION,\n    \"training_input_size\": 640,\n    \"paper_note\": (\n        \"The paper reports 14 classes, WBF, and 640x640 input, but does not disclose \"\n        \"the WBF IoU threshold or validation split ratio.\"\n    ),\n}\n(OUTPUT_ROOT / \"cleaning_config.json\").write_text(\n    json.dumps(config, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n)\nprint(json.dumps(config, ensure_ascii=False, indent=2))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"61d5fccf-9c25-4566-b308-d75ffa2d0821","cell_type":"markdown","source":"## 2. 自动发现原始标注与第1步输出\n\n","metadata":{}},{"id":"5fc6a455-4238-4730-9f8e-e548e3e2fcea","cell_type":"code","source":"def find_unique(candidates, description):\n    candidates = sorted(set(Path(path) for path in candidates))\n    if len(candidates) == 1:\n        return candidates[0]\n    if not candidates:\n        raise FileNotFoundError(f\"未找到{description}，请检查Kaggle的Add Input。\")\n    raise RuntimeError(\n        f\"发现多个{description}候选，请手工配置路径：\\n\"\n        + \"\\n\".join(str(path) for path in candidates)\n    )\n\n\nif RAW_DATA_ROOT is None:\n    TRAIN_CSV = find_unique(INPUT_ROOT.rglob(\"train.csv\"), \"原始train.csv\")\n    RAW_DATA_ROOT = TRAIN_CSV.parent\nelse:\n    RAW_DATA_ROOT = Path(RAW_DATA_ROOT)\n    TRAIN_CSV = RAW_DATA_ROOT / \"train.csv\"\n    if not TRAIN_CSV.exists():\n        TRAIN_CSV = find_unique(RAW_DATA_ROOT.rglob(\"train.csv\"), \"原始train.csv\")\n\nif PREPROCESSED_ROOT is None:\n    MANIFEST_CSV = find_unique(INPUT_ROOT.rglob(\"image_manifest.csv\"), \"第1步image_manifest.csv\")\n    PREPROCESSED_ROOT = MANIFEST_CSV.parent\nelse:\n    PREPROCESSED_ROOT = Path(PREPROCESSED_ROOT)\n    MANIFEST_CSV = PREPROCESSED_ROOT / \"image_manifest.csv\"\n    if not MANIFEST_CSV.exists():\n        MANIFEST_CSV = find_unique(PREPROCESSED_ROOT.rglob(\"image_manifest.csv\"), \"image_manifest.csv\")\n        PREPROCESSED_ROOT = MANIFEST_CSV.parent\n\nTEST_ANNOTATION_CANDIDATES = list(RAW_DATA_ROOT.rglob(\"annotations_test.csv\"))\nTEST_ANNOTATIONS = TEST_ANNOTATION_CANDIDATES[0] if len(TEST_ANNOTATION_CANDIDATES) == 1 else None\n\nmanifest = pd.read_csv(MANIFEST_CSV, dtype={\"image_id\": str})\nrequired_manifest_columns = {\n    \"image_id\", \"official_split\", \"relative_png_path\", \"original_width\",\n    \"original_height\", \"output_width\", \"output_height\", \"pixel_sha256\",\n}\nmissing_manifest_columns = sorted(required_manifest_columns - set(manifest.columns))\nif missing_manifest_columns:\n    raise ValueError(f\"第1步manifest缺少字段：{missing_manifest_columns}\")\n\nmanifest[\"source_png_path\"] = manifest[\"relative_png_path\"].map(\n    lambda value: str(PREPROCESSED_ROOT / str(value))\n)\nmissing_pngs = manifest[~manifest[\"source_png_path\"].map(lambda value: Path(value).exists())]\nif not missing_pngs.empty:\n    missing_pngs.to_csv(OUTPUT_ROOT / \"reports\" / \"missing_preprocessed_pngs.csv\", index=False)\n    raise FileNotFoundError(\"第1步部分PNG不存在，请检查missing_preprocessed_pngs.csv。\")\n\ntrain_manifest = manifest[manifest[\"official_split\"] == \"official_train\"].copy()\ntest_manifest = manifest[manifest[\"official_split\"] == \"official_test\"].copy()\n\nprint(\"原始训练标注：\", TRAIN_CSV)\nprint(\"第1步输出目录：\", PREPROCESSED_ROOT)\nprint(\"训练PNG：\", len(train_manifest))\nprint(\"测试PNG：\", len(test_manifest))\nprint(\"测试标注：\", TEST_ANNOTATIONS)\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"c2681026-a1c4-4f13-bc29-0faa0510f819","cell_type":"markdown","source":"## 3. 将第1步PNG复制到最终YOLO数据集\n\n这样最终输出中的`images/`和`labels/`位于同一数据集，Ultralytics可以自动匹配。\n\n","metadata":{}},{"id":"7062bc2e-3e4c-499c-90a2-c39763dff295","cell_type":"code","source":"def copy_images(split_manifest, output_subdir):\n    destination_dir = OUTPUT_ROOT / \"images\" / output_subdir\n    records = []\n    for row in tqdm(\n        split_manifest.itertuples(index=False),\n        total=len(split_manifest),\n        desc=f\"准备{output_subdir}图片\",\n    ):\n        source = Path(row.source_png_path)\n        destination = destination_dir / f\"{row.image_id}.png\"\n        if COPY_IMAGES_TO_FINAL_DATASET:\n            if not destination.exists() or destination.stat().st_size != source.stat().st_size:\n                shutil.copy2(source, destination)\n            final_path = destination\n        else:\n            final_path = source\n        records.append({\"image_id\": row.image_id, \"png_path\": str(final_path.resolve())})\n    return pd.DataFrame(records)\n\n\ntrain_png_paths = copy_images(train_manifest, \"trainval\")\ntest_png_paths = copy_images(test_manifest, \"test\")\ntrain_manifest = train_manifest.drop(columns=[\"source_png_path\"]).merge(\n    train_png_paths, on=\"image_id\", how=\"left\", validate=\"one_to_one\"\n)\ntest_manifest = test_manifest.drop(columns=[\"source_png_path\"]).merge(\n    test_png_paths, on=\"image_id\", how=\"left\", validate=\"one_to_one\"\n)\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"9730bf60-6a9f-4704-8513-f0214bcb83aa","cell_type":"markdown","source":"## 4. 标注清理\n\n","metadata":{}},{"id":"f74f38b8-18c6-4939-8b50-b2fa73196517","cell_type":"code","source":"def canonicalize_class_name(value):\n    return CLASS_ALIASES.get(str(value).strip().casefold())\n\n\ndef clean_annotation_csv(csv_path, split_manifest, report_prefix, has_radiologist=True):\n    raw = pd.read_csv(csv_path)\n    required = {\"image_id\", \"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"}\n    if has_radiologist:\n        required.add(\"rad_id\")\n    missing = sorted(required - set(raw.columns))\n    if missing:\n        raise ValueError(f\"标注CSV缺少字段：{missing}\")\n\n    report = Counter(source_rows=len(raw))\n    raw = raw.copy()\n    raw[\"image_id\"] = raw[\"image_id\"].astype(str)\n    raw[\"canonical_class_name\"] = raw[\"class_name\"].map(canonicalize_class_name)\n\n    unknown = raw[raw[\"canonical_class_name\"].isna()][\"class_name\"].value_counts()\n    if not unknown.empty:\n        unknown.rename_axis(\"source_class_name\").reset_index(name=\"rows\").to_csv(\n            OUTPUT_ROOT / \"reports\" / f\"{report_prefix}_unknown_classes.csv\", index=False\n        )\n        raise ValueError(f\"发现未配置类别，请查看{report_prefix}_unknown_classes.csv。\")\n\n    manifest_ids = set(split_manifest[\"image_id\"].astype(str))\n    outside = raw[~raw[\"image_id\"].isin(manifest_ids)].copy()\n    outside.to_csv(\n        OUTPUT_ROOT / \"reports\" / f\"{report_prefix}_annotations_outside_manifest.csv\",\n        index=False,\n    )\n    report[\"outside_manifest_rows\"] = len(outside)\n    raw = raw[raw[\"image_id\"].isin(manifest_ids)].copy()\n\n    no_finding = raw[raw[\"canonical_class_name\"] == NO_FINDING_NAME].copy()\n    no_finding.to_csv(\n        OUTPUT_ROOT / \"reports\" / f\"{report_prefix}_no_finding_rows.csv\", index=False\n    )\n    report[\"no_finding_rows\"] = len(no_finding)\n    boxes = raw[raw[\"canonical_class_name\"] != NO_FINDING_NAME].copy()\n\n    coordinate_columns = [\"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n    for column in coordinate_columns:\n        boxes[column] = pd.to_numeric(boxes[column], errors=\"coerce\")\n\n    finite = np.isfinite(boxes[coordinate_columns]).all(axis=1)\n    report[\"non_finite_boxes_removed\"] = int((~finite).sum())\n    boxes = boxes[finite].copy()\n\n    positive_area = (boxes[\"x_max\"] > boxes[\"x_min\"]) & (boxes[\"y_max\"] > boxes[\"y_min\"])\n    report[\"non_positive_area_boxes_removed\"] = int((~positive_area).sum())\n    boxes = boxes[positive_area].copy()\n\n    dimensions = split_manifest.set_index(\"image_id\")[[\"original_width\", \"original_height\"]]\n    boxes = boxes.join(dimensions, on=\"image_id\", how=\"left\", validate=\"many_to_one\")\n    if boxes[[\"original_width\", \"original_height\"]].isna().any().any():\n        raise RuntimeError(\"部分标注无法连接到原始DICOM尺寸。\")\n\n    before_clip = boxes[coordinate_columns].copy()\n    boxes[\"x_min\"] = boxes[\"x_min\"].clip(lower=0)\n    boxes[\"y_min\"] = boxes[\"y_min\"].clip(lower=0)\n    boxes[\"x_max\"] = np.minimum(boxes[\"x_max\"], boxes[\"original_width\"])\n    boxes[\"y_max\"] = np.minimum(boxes[\"y_max\"], boxes[\"original_height\"])\n    clipped = (before_clip != boxes[coordinate_columns]).any(axis=1)\n    boxes.loc[clipped].to_csv(\n        OUTPUT_ROOT / \"reports\" / f\"{report_prefix}_clipped_boxes.csv\", index=False\n    )\n    report[\"boxes_clipped_to_image\"] = int(clipped.sum())\n\n    large_enough = (\n        (boxes[\"x_max\"] - boxes[\"x_min\"] >= MIN_BOX_SIDE_PIXELS)\n        & (boxes[\"y_max\"] - boxes[\"y_min\"] >= MIN_BOX_SIDE_PIXELS)\n    )\n    report[\"too_small_boxes_removed\"] = int((~large_enough).sum())\n    boxes = boxes[large_enough].copy()\n\n    boxes[\"class_name\"] = boxes[\"canonical_class_name\"]\n    boxes[\"class_id\"] = boxes[\"class_name\"].map(CLASS_TO_ID).astype(int)\n\n    dedupe_columns = [\"image_id\", \"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n    if has_radiologist:\n        boxes[\"rad_id\"] = boxes[\"rad_id\"].astype(str)\n        dedupe_columns.insert(2, \"rad_id\")\n    else:\n        boxes[\"rad_id\"] = \"consensus\"\n\n    before_dedup = len(boxes)\n    boxes = boxes.drop_duplicates(dedupe_columns, keep=\"first\").copy()\n    report[\"exact_duplicate_boxes_removed\"] = before_dedup - len(boxes)\n    report[\"clean_box_rows\"] = len(boxes)\n    return boxes, no_finding, dict(report)\n\n\nclean_boxes, no_finding_rows, cleaning_report = clean_annotation_csv(\n    TRAIN_CSV, train_manifest, \"train\", has_radiologist=True\n)\nclean_boxes.to_csv(OUTPUT_ROOT / \"clean_boxes_before_fusion.csv\", index=False)\n(OUTPUT_ROOT / \"reports\" / \"annotation_cleaning_summary.json\").write_text(\n    json.dumps(cleaning_report, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n)\nprint(json.dumps(cleaning_report, ensure_ascii=False, indent=2))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"2b965156-880c-4417-bb40-5584ff1281f2","cell_type":"markdown","source":"## 5. 等权多医生框融合\n\n原始标注没有医生置信度，因此各医生权重设为1。此设置基于WBF思想，\n不能表述为与作者未公开的WBF实现完全相同。\n\n","metadata":{}},{"id":"b5a1735c-0140-46f1-ab7b-b7bc77e5975a","cell_type":"code","source":"def box_iou(box_a, box_b):\n    x1, y1 = max(box_a[0], box_b[0]), max(box_a[1], box_b[1])\n    x2, y2 = min(box_a[2], box_b[2]), min(box_a[3], box_b[3])\n    intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1)\n    area_a = max(0.0, box_a[2] - box_a[0]) * max(0.0, box_a[3] - box_a[1])\n    area_b = max(0.0, box_b[2] - box_b[0]) * max(0.0, box_b[3] - box_b[1])\n    union = area_a + area_b - intersection\n    return intersection / union if union > 0 else 0.0\n\n\ndef cluster_coordinate(cluster):\n    return np.average(\n        np.asarray(cluster[\"boxes\"], dtype=float),\n        axis=0,\n        weights=np.asarray(cluster[\"weights\"], dtype=float),\n    )\n\n\ndef fuse_one_image_class(group):\n    ordered = group.sort_values(\n        [\"rad_id\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"], kind=\"mergesort\"\n    )\n    clusters = []\n    for row in ordered.itertuples(index=False):\n        candidate = np.array([row.x_min, row.y_min, row.x_max, row.y_max], dtype=float)\n        best_index, best_iou = None, -1.0\n        for index, cluster in enumerate(clusters):\n            if (\n                ENFORCE_ONE_BOX_PER_RADIOLOGIST_PER_CLUSTER\n                and row.rad_id in cluster[\"radiologists\"]\n            ):\n                continue\n            overlap = box_iou(candidate, cluster_coordinate(cluster))\n            if overlap >= WBF_IOU_THRESHOLD and overlap > best_iou:\n                best_index, best_iou = index, overlap\n\n        if best_index is None:\n            clusters.append({\n                \"boxes\": [candidate],\n                \"weights\": [1.0],\n                \"radiologists\": {row.rad_id},\n            })\n        else:\n            clusters[best_index][\"boxes\"].append(candidate)\n            clusters[best_index][\"weights\"].append(1.0)\n            clusters[best_index][\"radiologists\"].add(row.rad_id)\n\n    first = ordered.iloc[0]\n    output = []\n    for lesion_index, cluster in enumerate(clusters):\n        radiologists = sorted(cluster[\"radiologists\"])\n        if len(radiologists) < MIN_RADIOLOGISTS:\n            continue\n        x_min, y_min, x_max, y_max = cluster_coordinate(cluster)\n        output.append({\n            \"image_id\": first[\"image_id\"],\n            \"class_name\": first[\"class_name\"],\n            \"class_id\": int(first[\"class_id\"]),\n            \"lesion_index\": lesion_index,\n            \"x_min\": float(x_min),\n            \"y_min\": float(y_min),\n            \"x_max\": float(x_max),\n            \"y_max\": float(y_max),\n            \"n_source_boxes\": len(cluster[\"boxes\"]),\n            \"n_radiologists\": len(radiologists),\n            \"rad_ids\": \"|\".join(radiologists),\n        })\n    return output\n\n\nfused_records = []\ngrouped = clean_boxes.groupby([\"image_id\", \"class_name\"], sort=True)\nfor _, group in tqdm(grouped, total=grouped.ngroups, desc=\"融合多医生框\"):\n    fused_records.extend(fuse_one_image_class(group))\n\nfused_boxes = pd.DataFrame(fused_records)\nif fused_boxes.empty:\n    raise RuntimeError(\"融合后没有检测框。\")\nfused_boxes = fused_boxes.sort_values(\n    [\"image_id\", \"class_id\", \"x_min\", \"y_min\"], kind=\"mergesort\"\n).reset_index(drop=True)\nfused_boxes.to_csv(OUTPUT_ROOT / \"fused_boxes.csv\", index=False)\n\nsingle_radiologist_boxes = fused_boxes[fused_boxes[\"n_radiologists\"] == 1].copy()\nsingle_radiologist_boxes.to_csv(\n    OUTPUT_ROOT / \"reports\" / \"single_radiologist_box_review.csv\", index=False\n)\n\nprint(\"清洗后、融合前框数：\", len(clean_boxes))\nprint(\"融合后框数：\", len(fused_boxes))\nprint(\"单医生框数：\", len(single_radiologist_boxes))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"b1f83819-8fa8-407d-a1a4-c92b09f313ba","cell_type":"markdown","source":"## 6. 与论文WBF后框数量比较\n\n","metadata":{}},{"id":"a35602b6-8412-4210-a0c1-cca942f0b255","cell_type":"code","source":"PAPER_WBF_COUNTS = {\n    \"Aortic enlargement\": 3386,\n    \"Atelectasis\": 230,\n    \"Calcification\": 745,\n    \"Cardiomegaly\": 2402,\n    \"Consolidation\": 435,\n    \"ILD\": 733,\n    \"Infiltration\": 954,\n    \"Lung Opacity\": 2017,\n    \"Nodule/Mass\": 1867,\n    \"Other lesion\": 1836,\n    \"Pleural effusion\": 1771,\n    \"Pleural thickening\": 4058,\n    \"Pneumothorax\": 131,\n    \"Pulmonary fibrosis\": 3369,\n}\nactual_counts = fused_boxes[\"class_name\"].value_counts().to_dict()\ncomparison = pd.DataFrame({\n    \"class_name\": CLASS_NAMES,\n    \"paper_after_wbf\": [PAPER_WBF_COUNTS[name] for name in CLASS_NAMES],\n    \"current_after_fusion\": [actual_counts.get(name, 0) for name in CLASS_NAMES],\n})\ncomparison[\"difference\"] = comparison[\"current_after_fusion\"] - comparison[\"paper_after_wbf\"]\ncomparison[\"exact_match\"] = comparison[\"difference\"] == 0\ncomparison.to_csv(OUTPUT_ROOT / \"reports\" / \"paper_wbf_count_comparison.csv\", index=False)\ndisplay(comparison)\nprint(\"论文WBF后总框数：\", sum(PAPER_WBF_COUNTS.values()))\nprint(\"当前融合后总框数：\", len(fused_boxes))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"a27a5762-544a-402e-9a6e-0325570682aa","cell_type":"markdown","source":"## 7. 生成YOLO标签\n\n","metadata":{}},{"id":"52084920-215c-4753-95b6-8e3fc30e92e6","cell_type":"code","source":"def write_yolo_labels(boxes, split_manifest, output_subdir):\n    label_dir = OUTPUT_ROOT / \"labels\" / output_subdir\n    dimensions = split_manifest.set_index(\"image_id\")[[\"original_width\", \"original_height\"]]\n    grouped_boxes = {image_id: group for image_id, group in boxes.groupby(\"image_id\")}\n    records = []\n\n    for image_id in tqdm(split_manifest[\"image_id\"].astype(str), desc=f\"生成{output_subdir}标签\"):\n        width = float(dimensions.loc[image_id, \"original_width\"])\n        height = float(dimensions.loc[image_id, \"original_height\"])\n        lines = []\n        group = grouped_boxes.get(image_id)\n        if group is not None:\n            for row in group.sort_values([\"class_id\", \"x_min\", \"y_min\"]).itertuples(index=False):\n                x_center = ((row.x_min + row.x_max) / 2.0) / width\n                y_center = ((row.y_min + row.y_max) / 2.0) / height\n                box_width = (row.x_max - row.x_min) / width\n                box_height = (row.y_max - row.y_min) / height\n                values = [x_center, y_center, box_width, box_height]\n                if not all(np.isfinite(values)):\n                    raise ValueError(f\"{image_id}存在非有限YOLO坐标\")\n                if not (\n                    0 <= x_center <= 1 and 0 <= y_center <= 1\n                    and 0 < box_width <= 1 and 0 < box_height <= 1\n                ):\n                    raise ValueError(f\"{image_id}存在越界YOLO坐标：{values}\")\n                lines.append(\n                    f\"{int(row.class_id)} {x_center:.8f} {y_center:.8f} \"\n                    f\"{box_width:.8f} {box_height:.8f}\"\n                )\n        label_path = label_dir / f\"{image_id}.txt\"\n        text = \"\\n\".join(lines) + (\"\\n\" if lines else \"\")\n        label_path.write_text(text, encoding=\"utf-8\")\n        records.append({\n            \"image_id\": image_id,\n            \"label_path\": str(label_path),\n            \"box_count\": len(lines),\n            \"label_sha256\": hashlib.sha256(text.encode(\"utf-8\")).hexdigest(),\n        })\n    return pd.DataFrame(records)\n\n\ntrain_label_manifest = write_yolo_labels(fused_boxes, train_manifest, \"trainval\")\ntrain_label_manifest.to_csv(OUTPUT_ROOT / \"train_label_manifest.csv\", index=False)\n\ntest_label_manifest = None\nif TEST_ANNOTATIONS is not None:\n    test_boxes, _, test_cleaning_report = clean_annotation_csv(\n        TEST_ANNOTATIONS, test_manifest, \"test\", has_radiologist=False\n    )\n    test_label_manifest = write_yolo_labels(test_boxes, test_manifest, \"test\")\n    test_label_manifest.to_csv(OUTPUT_ROOT / \"test_label_manifest.csv\", index=False)\n    (OUTPUT_ROOT / \"reports\" / \"test_annotation_cleaning_summary.json\").write_text(\n        json.dumps(test_cleaning_report, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n    )\nelse:\n    print(\"未发现annotations_test.csv：可以进行测试集推理，但不能在本地计算测试mAP。\")\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"297719da-b705-416f-965d-5ca7fb4d7738","cell_type":"markdown","source":"## 8. 分组分层划分训练集和验证集\n\n精确像素重复图像会被分在同一侧，避免训练/验证泄漏。\n\n","metadata":{}},{"id":"b63c2e11-0bdf-4378-abe0-ce5cec414886","cell_type":"code","source":"image_classes = fused_boxes.groupby(\"image_id\")[\"class_id\"].apply(\n    lambda values: sorted(set(values))\n).to_dict()\nclass_image_frequency = Counter()\nfor classes in image_classes.values():\n    class_image_frequency.update(classes)\n\npixel_groups = []\nfor pixel_sha, group in train_manifest.groupby(\"pixel_sha256\", sort=True):\n    image_ids = sorted(group[\"image_id\"].astype(str).tolist())\n    union_classes = sorted({\n        class_id for image_id in image_ids for class_id in image_classes.get(image_id, [])\n    })\n    if union_classes:\n        rarest = min(union_classes, key=lambda class_id: (class_image_frequency[class_id], class_id))\n        key = f\"class_{rarest}\"\n    else:\n        key = \"negative\"\n    pixel_groups.append({\n        \"pixel_sha256\": pixel_sha,\n        \"image_ids\": image_ids,\n        \"stratify_key\": key,\n    })\n\ngroup_frame = pd.DataFrame(pixel_groups)\nkey_counts = group_frame[\"stratify_key\"].value_counts()\nrare_keys = set(key_counts[key_counts < 2].index)\ngroup_frame.loc[group_frame[\"stratify_key\"].isin(rare_keys), \"stratify_key\"] = \"positive_other\"\n\ntrain_groups, val_groups = train_test_split(\n    group_frame,\n    test_size=VAL_FRACTION,\n    random_state=SEED,\n    shuffle=True,\n    stratify=group_frame[\"stratify_key\"],\n)\ntrain_ids = sorted(image_id for values in train_groups[\"image_ids\"] for image_id in values)\nval_ids = sorted(image_id for values in val_groups[\"image_ids\"] for image_id in values)\ntest_ids = sorted(test_manifest[\"image_id\"].astype(str).tolist())\n\nassert set(train_ids).isdisjoint(val_ids)\nassert len(train_ids) + len(val_ids) == len(train_manifest)\n\ntrain_path_map = train_manifest.set_index(\"image_id\")[\"png_path\"].to_dict()\ntest_path_map = test_manifest.set_index(\"image_id\")[\"png_path\"].to_dict()\n\ndef write_path_list(path, image_ids, path_map):\n    path.write_text(\n        \"\\n\".join(str(Path(path_map[image_id]).resolve()) for image_id in image_ids) + \"\\n\",\n        encoding=\"utf-8\",\n    )\n\n\nwrite_path_list(OUTPUT_ROOT / \"splits\" / \"train.txt\", train_ids, train_path_map)\nwrite_path_list(OUTPUT_ROOT / \"splits\" / \"val.txt\", val_ids, train_path_map)\nwrite_path_list(OUTPUT_ROOT / \"splits\" / \"test.txt\", test_ids, test_path_map)\n\nsplit_manifest = pd.DataFrame(\n    [{\"image_id\": image_id, \"split\": \"train\"} for image_id in train_ids]\n    + [{\"image_id\": image_id, \"split\": \"val\"} for image_id in val_ids]\n    + [{\"image_id\": image_id, \"split\": \"official_test\"} for image_id in test_ids]\n)\nsplit_manifest.to_csv(OUTPUT_ROOT / \"split_manifest.csv\", index=False)\n\nprint(\"训练集：\", len(train_ids))\nprint(\"验证集：\", len(val_ids))\nprint(\"官方测试集：\", len(test_ids))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"4e04bfe2-1205-470f-aa6f-ec03322302bb","cell_type":"markdown","source":"## 9. 最终审计并生成data.yaml\n\n","metadata":{}},{"id":"1498499e-ff81-4959-a8b4-181b045d0ce2","cell_type":"code","source":"def distribution_for_ids(boxes, image_ids, split_name):\n    selected = boxes[boxes[\"image_id\"].isin(set(image_ids))]\n    counts = selected[\"class_name\"].value_counts()\n    return pd.DataFrame({\n        \"split\": split_name,\n        \"class_name\": CLASS_NAMES,\n        \"box_count\": [int(counts.get(name, 0)) for name in CLASS_NAMES],\n    })\n\n\ndistribution = pd.concat([\n    distribution_for_ids(fused_boxes, train_ids, \"train\"),\n    distribution_for_ids(fused_boxes, val_ids, \"val\"),\n], ignore_index=True)\ndistribution.to_csv(OUTPUT_ROOT / \"reports\" / \"split_class_distribution.csv\", index=False)\n\nlabel_errors = []\nfor row in tqdm(\n    train_manifest.itertuples(index=False), total=len(train_manifest), desc=\"审计YOLO标签\"\n):\n    label_path = OUTPUT_ROOT / \"labels\" / \"trainval\" / f\"{row.image_id}.txt\"\n    if not label_path.exists():\n        label_errors.append({\"image_id\": row.image_id, \"error\": \"missing_label_file\"})\n        continue\n    for line_number, line in enumerate(label_path.read_text(encoding=\"utf-8\").splitlines(), 1):\n        fields = line.split()\n        if len(fields) != 5:\n            label_errors.append({\"image_id\": row.image_id, \"line\": line_number, \"error\": \"field_count\"})\n            continue\n        try:\n            class_id = int(fields[0])\n            coordinates = [float(value) for value in fields[1:]]\n        except ValueError:\n            label_errors.append({\"image_id\": row.image_id, \"line\": line_number, \"error\": \"parse_error\"})\n            continue\n        if class_id not in range(14):\n            label_errors.append({\"image_id\": row.image_id, \"line\": line_number, \"error\": \"class_id\"})\n        if not all(np.isfinite(coordinates)) or not all(0 <= value <= 1 for value in coordinates):\n            label_errors.append({\"image_id\": row.image_id, \"line\": line_number, \"error\": \"coordinate_range\"})\n        if coordinates[2] <= 0 or coordinates[3] <= 0:\n            label_errors.append({\"image_id\": row.image_id, \"line\": line_number, \"error\": \"non_positive_size\"})\n\nlabel_error_frame = pd.DataFrame(label_errors)\nlabel_error_frame.to_csv(OUTPUT_ROOT / \"reports\" / \"label_audit_errors.csv\", index=False)\n\n# 检查PNG与标签文件是否完整。\nmissing_final_images = [\n    image_id for image_id, path in {**train_path_map, **test_path_map}.items()\n    if not Path(path).exists()\n]\npd.DataFrame({\"image_id\": missing_final_images}).to_csv(\n    OUTPUT_ROOT / \"reports\" / \"missing_final_images.csv\", index=False\n)\n\nyaml_lines = [\n    f\"path: {OUTPUT_ROOT}\",\n    \"train: splits/train.txt\",\n    \"val: splits/val.txt\",\n    \"test: splits/test.txt\",\n    \"nc: 14\",\n    \"names:\",\n]\nyaml_lines.extend(f\"  {index}: {name}\" for index, name in enumerate(CLASS_NAMES))\n(OUTPUT_ROOT / \"data.yaml\").write_text(\"\\n\".join(yaml_lines) + \"\\n\", encoding=\"utf-8\")\n\nblocking_errors = []\nif len(train_manifest) != 15000:\n    blocking_errors.append(f\"official_train_count={len(train_manifest)}\")\nif len(test_manifest) != 3000:\n    blocking_errors.append(f\"official_test_count={len(test_manifest)}\")\nif not label_error_frame.empty:\n    blocking_errors.append(f\"label_audit_errors={len(label_error_frame)}\")\nif missing_final_images:\n    blocking_errors.append(f\"missing_final_images={len(missing_final_images)}\")\nif not COPY_IMAGES_TO_FINAL_DATASET:\n    blocking_errors.append(\"final_dataset_is_not_self_contained\")\n\nsummary = {\n    \"status\": \"TRAINING_READY\" if not blocking_errors else \"BLOCKED\",\n    \"official_train_images\": int(len(train_manifest)),\n    \"train_images\": int(len(train_ids)),\n    \"validation_images\": int(len(val_ids)),\n    \"official_test_images\": int(len(test_manifest)),\n    \"clean_boxes_before_fusion\": int(len(clean_boxes)),\n    \"boxes_after_fusion\": int(len(fused_boxes)),\n    \"single_radiologist_boxes\": int(len(single_radiologist_boxes)),\n    \"paper_wbf_total_boxes\": int(sum(PAPER_WBF_COUNTS.values())),\n    \"all_paper_class_counts_match\": bool(comparison[\"exact_match\"].all()),\n    \"test_labels_available\": bool(test_label_manifest is not None),\n    \"blocking_errors\": blocking_errors,\n}\n(OUTPUT_ROOT / \"generation_summary.json\").write_text(\n    json.dumps(summary, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n)\n\nprint(json.dumps(summary, ensure_ascii=False, indent=2))\nprint(\"最终YOLO数据集：\", OUTPUT_ROOT)\nprint(\"训练配置：\", OUTPUT_ROOT / \"data.yaml\")\n\nif blocking_errors:\n    raise RuntimeError(\"最终审计未通过：\" + \"; \".join(blocking_errors))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"id":"f28c6701-cefc-4c0f-93a7-f23bcbf9b41c","cell_type":"markdown","source":"## 10. 下一步训练\n\n当`generation_summary.json`显示`TRAINING_READY`后，将整个\n`/kaggle/working/vindr_yolo_clean_v1`保存为私有Kaggle Dataset。\n在训练Notebook中挂载该数据集，并把`data.yaml`中的`path`改为新的只读挂载路径，\n或在训练命令中使用挂载后的`data.yaml`。\n","metadata":{}}]}