{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":null,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 第1步：VinDr-CXR影像预处理（Kaggle）\n\n本Notebook只处理影像，不修改标注：\n\n- 自动发现15,000张训练DICOM和3,000张测试DICOM；\n- 应用Modality LUT和可用的VOI LUT；\n- 处理MONOCHROME1灰度反转；\n- 无VOI时使用0.5%～99.5%百分位窗口；\n- 转换为单通道PNG；\n- 最长边超过1024像素时等比例缩小；\n- 生成尺寸、哈希、对比度和错误报告。\n\n运行完成后，请将整个`/kaggle/working/vindr_preprocessed_v1`保存为一个新的\n私有Kaggle Dataset，然后在第2个Notebook中同时挂载原始数据集和这个输出数据集。\n\n","metadata":{}},{"cell_type":"code","source":"import sys\nimport subprocess\nimport importlib.util\n\nif importlib.util.find_spec(\"pydicom\") is None:\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"pydicom\"])\n\nimport gc\nimport hashlib\nimport json\nimport random\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_modality_lut, apply_voi_lut\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. 配置\n\n如果自动发现多个候选数据集，请手工填写`DATA_ROOT`。\n\n","metadata":{}},{"cell_type":"code","source":"SEED = 2026\nrandom.seed(SEED)\nnp.random.seed(SEED)\n\nINPUT_ROOT = Path(\"/kaggle/input\")\nOUTPUT_ROOT = Path(\"/kaggle/working/vindr_preprocessed_v1\")\n\n# 示例：Path(\"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection\")\nDATA_ROOT = Path(\n    \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection\"\n)\n\n# 以下参数为本实验预处理设置；YOLO-CXR论文未公开这些DICOM转换细节。\nPREPROCESS_MAX_SIDE = 1024\nFALLBACK_PERCENTILES = (0.5, 99.5)\nPNG_COMPRESS_LEVEL = 6\nRESUME = True\nFAIL_ON_IMAGE_ERROR = True\nLOW_CONTRAST_STD_THRESHOLD = 3.0\nEXTREME_PIXEL_FRACTION_THRESHOLD = 0.98\n\nfor directory in [\n    OUTPUT_ROOT / \"images\" / \"trainval\",\n    OUTPUT_ROOT / \"images\" / \"test\",\n    OUTPUT_ROOT / \"reports\",\n]:\n    directory.mkdir(parents=True, exist_ok=True)\n\nconfig = {\n    \"step\": \"01_image_preprocessing\",\n    \"seed\": SEED,\n    \"preprocess_max_side\": PREPROCESS_MAX_SIDE,\n    \"fallback_percentiles\": list(FALLBACK_PERCENTILES),\n    \"png_compress_level\": PNG_COMPRESS_LEVEL,\n    \"low_contrast_std_threshold\": LOW_CONTRAST_STD_THRESHOLD,\n    \"extreme_pixel_fraction_threshold\": EXTREME_PIXEL_FRACTION_THRESHOLD,\n    \"paper_note\": \"These DICOM conversion parameters were not disclosed by the YOLO-CXR paper.\",\n}\n(OUTPUT_ROOT / \"preprocessing_config.json\").write_text(\n    json.dumps(config, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n)\nprint(json.dumps(config, ensure_ascii=False, indent=2))\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. 自动发现DICOM目录\n\n","metadata":{}},{"cell_type":"code","source":"def list_dicom_files(directory):\n    return sorted(\n        path for path in Path(directory).rglob(\"*\")\n        if path.is_file() and path.suffix.lower() in {\".dicom\", \".dcm\"}\n    )\n\n\ndef find_unique(candidates, description):\n    candidates = sorted(set(Path(path) for path in candidates))\n    if len(candidates) == 1:\n        return candidates[0]\n    if not candidates:\n        raise FileNotFoundError(f\"未找到{description}，请检查Kaggle的Add Input。\")\n    raise RuntimeError(\n        f\"发现多个{description}候选，请手工填写DATA_ROOT：\\n\"\n        + \"\\n\".join(str(path) for path in candidates)\n    )\n\n\ndef discover_data_root(configured_root=None):\n    if configured_root is not None:\n        return Path(configured_root)\n    train_csv = find_unique(INPUT_ROOT.rglob(\"train.csv\"), \"train.csv\")\n    return train_csv.parent\n\n\ndef discover_dicom_directory(root, name):\n    direct = [root / name.lower(), root / name.capitalize()]\n    direct = [path for path in direct if path.exists() and list_dicom_files(path)]\n    if direct:\n        return find_unique(direct, f\"{name} DICOM目录\")\n    recursive = [\n        path for path in root.rglob(\"*\")\n        if path.is_dir() and path.name.casefold() == name.casefold() and list_dicom_files(path)\n    ]\n    return find_unique(recursive, f\"{name} DICOM目录\")\n\n\nfrom pathlib import Path\nfrom collections import Counter\n\nINPUT_ROOT = Path(\"/kaggle/input\")\n\n# 1. 查找train.csv\ntrain_csv_candidates = sorted(INPUT_ROOT.rglob(\"train.csv\"))\n\nprint(\"找到的train.csv：\")\nfor path in train_csv_candidates:\n    print(path)\n\nif len(train_csv_candidates) == 0:\n    raise FileNotFoundError(\n        \"没有找到train.csv，请先通过Add Input添加VinBigData完整数据集。\"\n    )\n\nif len(train_csv_candidates) > 1:\n    raise RuntimeError(\n        \"找到多个train.csv，请删除无关Input，或者手动指定TRAIN_CSV。\"\n    )\n\nTRAIN_CSV = train_csv_candidates[0]\nRAW_DATA_ROOT = TRAIN_CSV.parent\n\nprint(\"\\n使用的train.csv：\", TRAIN_CSV)\n\n\n# 2. 在整个/kaggle/input中寻找DICOM\nALL_DICOMS = sorted(\n    path\n    for path in INPUT_ROOT.rglob(\"*\")\n    if path.is_file()\n    and path.suffix.lower() in {\".dicom\", \".dcm\"}\n)\n\nprint(\"\\nDICOM总数：\", len(ALL_DICOMS))\n\nif len(ALL_DICOMS) == 0:\n    suffix_counts = Counter(\n        path.suffix.lower()\n        for path in INPUT_ROOT.rglob(\"*\")\n        if path.is_file()\n    )\n\n    print(\"\\n当前文件扩展名：\")\n    print(suffix_counts.most_common(20))\n\n    raise FileNotFoundError(\n        \"没有找到任何.dicom或.dcm文件。\"\n        \"当前添加的可能只是CSV数据，不是完整VinBigData数据集。\"\n    )\n\n\n# 3. 根据路径中的目录名称区分训练集和测试集\nTRAIN_FOLDER_NAMES = {\n    \"train\",\n    \"train_images\",\n    \"train_image\",\n    \"train_dicom\",\n    \"train_dicoms\",\n}\n\nTEST_FOLDER_NAMES = {\n    \"test\",\n    \"test_images\",\n    \"test_image\",\n    \"test_dicom\",\n    \"test_dicoms\",\n}\n\n\ndef path_belongs_to(path, folder_names):\n    return any(\n        part.casefold() in folder_names\n        for part in path.parts\n    )\n\n\nTRAIN_DICOMS = [\n    path\n    for path in ALL_DICOMS\n    if path_belongs_to(path, TRAIN_FOLDER_NAMES)\n]\n\nTEST_DICOMS = [\n    path\n    for path in ALL_DICOMS\n    if path_belongs_to(path, TEST_FOLDER_NAMES)\n]\n\n\n# 4. 输出识别结果\nprint(\"\\n训练DICOM数量：\", len(TRAIN_DICOMS))\nprint(\"测试DICOM数量：\", len(TEST_DICOMS))\n\nprint(\"\\n前3个训练DICOM：\")\nfor path in TRAIN_DICOMS[:3]:\n    print(path)\n\nprint(\"\\n前3个测试DICOM：\")\nfor path in TEST_DICOMS[:3]:\n    print(path)\n\n\n# 5. 检查数量\nif len(TRAIN_DICOMS) != 15000 or len(TEST_DICOMS) != 3000:\n    print(\"\\nDICOM所在目录及数量：\")\n\n    directory_counts = Counter(\n        str(path.parent)\n        for path in ALL_DICOMS\n    )\n\n    for directory, count in directory_counts.most_common(20):\n        print(count, directory)\n\n    raise RuntimeError(\n        f\"DICOM数量不符合预期：\"\n        f\"训练集={len(TRAIN_DICOMS)}，测试集={len(TEST_DICOMS)}；\"\n        \"预期为训练集15000、测试集3000。\"\n    )\n\n\nprint(\"\\n数据检查通过，可以开始影像预处理。\")\n\nTRAIN_DICOM_DIR = TRAIN_DICOMS[0].parent\nTEST_DICOM_DIR = TEST_DICOMS[0].parent\n\nprint(\"数据根目录：\", RAW_DATA_ROOT)\nprint(\"训练DICOM目录：\", TRAIN_DICOM_DIR)\nprint(\"训练DICOM数量：\", len(TRAIN_DICOMS))\nprint(\"测试DICOM目录：\", TEST_DICOM_DIR)\nprint(\"测试DICOM数量：\", len(TEST_DICOMS))\n\nif len(TRAIN_DICOMS) != 15000:\n    print(f\"警告：官方训练集应为15,000张，当前发现{len(TRAIN_DICOMS):,}张。\")\nif len(TEST_DICOMS) != 3000:\n    print(f\"警告：官方测试集应为3,000张，当前发现{len(TEST_DICOMS):,}张。\")\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. DICOM转PNG函数\n\n","metadata":{}},{"cell_type":"code","source":"def sha256_file(path, chunk_size=1024 * 1024):\n    digest = hashlib.sha256()\n    with open(path, \"rb\") as handle:\n        while True:\n            block = handle.read(chunk_size)\n            if not block:\n                break\n            digest.update(block)\n    return digest.hexdigest()\n\n\ndef pixel_hash(array):\n    digest = hashlib.sha256()\n    digest.update(f\"{array.shape[0]}x{array.shape[1]}|\".encode(\"ascii\"))\n    digest.update(np.ascontiguousarray(array).tobytes())\n    return digest.hexdigest()\n\n\ndef get_padding_mask(raw_pixels, dataset):\n    if \"PixelPaddingValue\" not in dataset:\n        return np.zeros(raw_pixels.shape, dtype=bool)\n    low = float(dataset.PixelPaddingValue)\n    if \"PixelPaddingRangeLimit\" in dataset:\n        high = float(dataset.PixelPaddingRangeLimit)\n        lower, upper = sorted((low, high))\n        return (raw_pixels >= lower) & (raw_pixels <= upper)\n    return raw_pixels == low\n\n\ndef preprocess_dicom(source_path, destination_path):\n    dataset = pydicom.dcmread(str(source_path), force=True)\n    raw = np.asarray(dataset.pixel_array)\n    if raw.ndim != 2:\n        raise ValueError(f\"只支持二维DICOM，实际形状为{raw.shape}\")\n\n    photometric = str(getattr(dataset, \"PhotometricInterpretation\", \"\")).upper()\n    if photometric not in {\"MONOCHROME1\", \"MONOCHROME2\"}:\n        raise ValueError(f\"不支持PhotometricInterpretation={photometric!r}\")\n\n    padding_mask = get_padding_mask(raw, dataset)\n    values = np.asarray(apply_modality_lut(raw, dataset), dtype=np.float32)\n\n    has_voi = (\n        \"VOILUTSequence\" in dataset\n        or (\"WindowCenter\" in dataset and \"WindowWidth\" in dataset)\n    )\n    voi_applied = False\n    if has_voi:\n        try:\n            values = np.asarray(apply_voi_lut(values, dataset), dtype=np.float32)\n            voi_applied = True\n        except Exception:\n            voi_applied = False\n\n    valid = values[~padding_mask & np.isfinite(values)]\n    if valid.size == 0:\n        raise ValueError(\"不存在可用于强度统计的有效像素\")\n\n    if voi_applied:\n        lower, upper = float(valid.min()), float(valid.max())\n    else:\n        lower, upper = np.percentile(valid, FALLBACK_PERCENTILES).astype(float)\n\n    if not np.isfinite(lower) or not np.isfinite(upper) or upper <= lower:\n        raise ValueError(f\"无效强度窗口：lower={lower}, upper={upper}\")\n\n    values = np.nan_to_num(values, nan=lower, posinf=upper, neginf=lower)\n    values = np.clip(values, lower, upper)\n    values = np.rint((values - lower) / (upper - lower) * 255.0).astype(np.uint8)\n\n    if photometric == \"MONOCHROME1\":\n        values = 255 - values\n    values[padding_mask] = 0\n\n    original_height, original_width = values.shape\n    image = Image.fromarray(values, mode=\"L\")\n    if max(original_width, original_height) > PREPROCESS_MAX_SIDE:\n        scale = PREPROCESS_MAX_SIDE / float(max(original_width, original_height))\n        new_size = (\n            max(1, int(round(original_width * scale))),\n            max(1, int(round(original_height * scale))),\n        )\n        image = image.resize(new_size, Image.Resampling.LANCZOS)\n\n    destination_path.parent.mkdir(parents=True, exist_ok=True)\n    image.save(destination_path, format=\"PNG\", compress_level=PNG_COMPRESS_LEVEL)\n    output = np.asarray(image)\n\n    return {\n        \"original_width\": int(original_width),\n        \"original_height\": int(original_height),\n        \"output_width\": int(image.width),\n        \"output_height\": int(image.height),\n        \"photometric\": photometric,\n        \"voi_applied\": bool(voi_applied),\n        \"window_lower\": lower,\n        \"window_upper\": upper,\n        \"pixel_mean\": float(output.mean()),\n        \"pixel_std\": float(output.std()),\n        \"extreme_pixel_fraction\": float(np.mean((output == 0) | (output == 255))),\n        \"pixel_sha256\": pixel_hash(output),\n        \"file_sha256\": sha256_file(destination_path),\n    }\n\n\ndef inspect_existing_png(source_path, destination_path):\n    header = pydicom.dcmread(str(source_path), stop_before_pixels=True, force=True)\n    with Image.open(destination_path) as image:\n        image = image.convert(\"L\")\n        output = np.asarray(image)\n        output_width, output_height = image.size\n    return {\n        \"original_width\": int(header.Columns),\n        \"original_height\": int(header.Rows),\n        \"output_width\": int(output_width),\n        \"output_height\": int(output_height),\n        \"photometric\": str(getattr(header, \"PhotometricInterpretation\", \"\")),\n        \"voi_applied\": None,\n        \"window_lower\": None,\n        \"window_upper\": None,\n        \"pixel_mean\": float(output.mean()),\n        \"pixel_std\": float(output.std()),\n        \"extreme_pixel_fraction\": float(np.mean((output == 0) | (output == 255))),\n        \"pixel_sha256\": pixel_hash(output),\n        \"file_sha256\": sha256_file(destination_path),\n    }\n\n\ndef convert_collection(dicom_paths, official_split, output_subdir):\n    records, failures = [], []\n    destination_dir = OUTPUT_ROOT / \"images\" / output_subdir\n\n    for index, source_path in enumerate(\n        tqdm(dicom_paths, desc=f\"转换{official_split}\"), start=1\n    ):\n        image_id = source_path.stem\n        destination_path = destination_dir / f\"{image_id}.png\"\n        try:\n            if RESUME and destination_path.exists():\n                metadata = inspect_existing_png(source_path, destination_path)\n                status = \"reused\"\n            else:\n                metadata = preprocess_dicom(source_path, destination_path)\n                status = \"processed\"\n\n            original_ratio = metadata[\"original_width\"] / metadata[\"original_height\"]\n            output_ratio = metadata[\"output_width\"] / metadata[\"output_height\"]\n            records.append({\n                \"image_id\": image_id,\n                \"official_split\": official_split,\n                \"source_path\": str(source_path),\n                \"relative_png_path\": f\"images/{output_subdir}/{image_id}.png\",\n                \"status\": status,\n                \"relative_aspect_ratio_error\": abs(output_ratio / original_ratio - 1.0),\n                **metadata,\n            })\n        except Exception as error:\n            failures.append({\n                \"image_id\": image_id,\n                \"official_split\": official_split,\n                \"source_path\": str(source_path),\n                \"error_type\": type(error).__name__,\n                \"error\": str(error),\n            })\n        if index % 25 == 0:\n            gc.collect()\n\n    return pd.DataFrame(records), pd.DataFrame(failures)\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. 正式执行影像预处理\n\n","metadata":{}},{"cell_type":"code","source":"train_manifest, train_failures = convert_collection(\n    TRAIN_DICOMS, \"official_train\", \"trainval\"\n)\ntest_manifest, test_failures = convert_collection(\n    TEST_DICOMS, \"official_test\", \"test\"\n)\n\nmanifest = pd.concat([train_manifest, test_manifest], ignore_index=True)\nfailures = pd.concat([train_failures, test_failures], ignore_index=True)\nmanifest.to_csv(OUTPUT_ROOT / \"image_manifest.csv\", index=False)\nfailures.to_csv(OUTPUT_ROOT / \"reports\" / \"image_conversion_failures.csv\", index=False)\n\nprint(\"成功转换或复用：\", len(manifest))\nprint(\"失败：\", len(failures))\nif FAIL_ON_IMAGE_ERROR and not failures.empty:\n    raise RuntimeError(\"存在DICOM转换失败，请查看image_conversion_failures.csv。\")\n\n\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. 影像质量审计\n\n","metadata":{}},{"cell_type":"code","source":"low_contrast = manifest[manifest[\"pixel_std\"] < LOW_CONTRAST_STD_THRESHOLD].copy()\nextreme_pixels = manifest[\n    manifest[\"extreme_pixel_fraction\"] >= EXTREME_PIXEL_FRACTION_THRESHOLD\n].copy()\nduplicates = manifest[\n    manifest.duplicated(\"pixel_sha256\", keep=False)\n].sort_values([\"pixel_sha256\", \"official_split\", \"image_id\"])\naspect_ratio_failures = manifest[\n    manifest[\"relative_aspect_ratio_error\"] > 0.002\n].copy()\n\nlow_contrast.to_csv(OUTPUT_ROOT / \"reports\" / \"low_contrast_review.csv\", index=False)\nextreme_pixels.to_csv(OUTPUT_ROOT / \"reports\" / \"extreme_pixel_review.csv\", index=False)\nduplicates.to_csv(OUTPUT_ROOT / \"reports\" / \"exact_pixel_duplicates.csv\", index=False)\naspect_ratio_failures.to_csv(\n    OUTPUT_ROOT / \"reports\" / \"aspect_ratio_failures.csv\", index=False\n)\n\ntrain_hashes = set(train_manifest[\"pixel_sha256\"])\ntest_hashes = set(test_manifest[\"pixel_sha256\"])\noverlap_hashes = train_hashes & test_hashes\noverlap = manifest[manifest[\"pixel_sha256\"].isin(overlap_hashes)].copy()\noverlap.to_csv(OUTPUT_ROOT / \"reports\" / \"train_test_exact_pixel_overlap.csv\", index=False)\n\nblocking_errors = []\nif not failures.empty:\n    blocking_errors.append(f\"conversion_failures={len(failures)}\")\nif not aspect_ratio_failures.empty:\n    blocking_errors.append(f\"aspect_ratio_failures={len(aspect_ratio_failures)}\")\naudit_warnings = []\n\nif overlap_hashes:\n    audit_warnings.append(\n        f\"train_test_duplicate_groups={len(overlap_hashes)}\"\n    )\nif len(train_manifest) != 15000:\n    blocking_errors.append(f\"official_train_count={len(train_manifest)}\")\nif len(test_manifest) != 3000:\n    blocking_errors.append(f\"official_test_count={len(test_manifest)}\")\n\nsummary = {\n    \"status\": \"PREPROCESSING_READY\" if not blocking_errors else \"BLOCKED\",\n    \"official_train_images\": int(len(train_manifest)),\n    \"official_test_images\": int(len(test_manifest)),\n    \"low_contrast_review_count\": int(len(low_contrast)),\n    \"extreme_pixel_review_count\": int(len(extreme_pixels)),\n    \"exact_duplicate_image_rows\": int(len(duplicates)),\n    \"blocking_errors\": blocking_errors,\n}\n(OUTPUT_ROOT / \"preprocessing_summary.json\").write_text(\n    json.dumps(summary, ensure_ascii=False, indent=2), encoding=\"utf-8\"\n)\n\nprint(json.dumps(summary, ensure_ascii=False, indent=2))\nprint(\"输出目录：\", OUTPUT_ROOT)\nprint(\"下一步：把此目录保存为私有Kaggle Dataset，再运行02数据清洗Notebook。\")\n\nif blocking_errors:\n    raise RuntimeError(\"影像预处理审计未通过：\" + \"; \".join(blocking_errors))\n","metadata":{},"outputs":[],"execution_count":null}]}