{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"stage1_phase_runner_created_utc":"2026-05-16T00:00:00Z","stage1_phase_runner_source":"notebook\\VinDr_CXR_Stage1_Training.ipynb","stage1_phase_runner_reorganized":true,"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":16316647,"datasetId":10460399,"databundleVersionId":17304846}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# VinDr-CXR Stage 1 Phase Runner\n\nNotebook này chạy Stage 1 + Stage 1.5 Lite theo các phase tách rời: `cache`, `phase1`, `phase2`, `finalize`, hoặc `all`. Mặc định tiền xử lý dùng `modality_voi_uint16` và cache PNG 16-bit trong `/kaggle/temp` để giảm mất mát dải động nhưng không làm phình output Kaggle.\n","metadata":{}},{"cell_type":"markdown","source":"## 1. Packages and Imports\n","metadata":{}},{"cell_type":"code","source":"# import os\n# import shutil\n# import zipfile\n# from pathlib import Path\n\n# RESTORE_DIR = Path(\"/kaggle/input/datasets/mixp123/phase2-restore-efficientnetv2-s-screening/phase2_restore_efficientnetv2_s_screening/latest_valid_phase2_pack\")\n# RESTORE_ZIP_BASE = Path(\"/kaggle/working/latest_valid_phase2_pack\")\n# RESTORE_ZIP = Path(str(RESTORE_ZIP_BASE) + \".zip\")\n\n# assert (RESTORE_DIR / \"run/checkpoints/best_phase1.pt\").exists(), \"Không thấy best_phase1.pt trong thư mục restore\"\n\n# if RESTORE_ZIP.exists():\n#     RESTORE_ZIP.unlink()\n\n# shutil.make_archive(str(RESTORE_ZIP_BASE), \"zip\", root_dir=str(RESTORE_DIR))\n\n# with zipfile.ZipFile(RESTORE_ZIP, \"r\") as z:\n#     names = set(z.namelist())\n#     assert \"run/checkpoints/best_phase1.pt\" in names, [\n#         n for n in names if \"best_phase1\" in n\n#     ]\n\n# os.environ[\"STAGE1_JOB_MODE\"] = \"finalize\"\n# os.environ[\"STAGE1_RUN_ID\"] = \"efficientnetv2_s_screening\"\n# os.environ[\"STAGE1_RESTORE_PACK\"] = str(RESTORE_ZIP)\n# os.environ[\"STAGE1_RESTORE_OVERWRITE\"] = \"1\"\n\n# print(\"Restore pack ready:\", RESTORE_ZIP)\n\n# Đặt chế độ công việc thành tạo cache\n# Đặt tên ID chạy mới để không nhầm lẫn với EfficientNetV2 trước đó\nimport os\n\n# Chỉ dùng được nếu bạn đã sửa notebook để run_model_name đọc từ env.\nos.environ[\"STAGE1_RUN_MODEL_NAME\"] = \"convnextv2_base\"\n\nos.environ[\"STAGE1_RUN_ID\"] = \"convnextv2_base_screening\"\nos.environ[\"STAGE1_JOB_MODE\"] = \"phase1\"\n\nos.environ[\"STAGE1_PREPROCESSING_MODE\"] = \"modality_voi_uint16\"\nos.environ[\"STAGE1_CACHE_READ_ROOTS\"] = \"/kaggle/input/datasets/mixp123/vindr-cxr-1024-uint16-cache\"\n\n# Optional: chỉ backup manifest/cache-job pack nhỏ, không backup toàn bộ PNG.\nos.environ[\"STAGE1_CACHE_BACKUP_DATASET\"] = \"mixp123/vindr-stage1-convnextv2_base-cache-manifest\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.62942Z","iopub.execute_input":"2026-05-20T15:49:25.629671Z","iopub.status.idle":"2026-05-20T15:49:25.638427Z","shell.execute_reply.started":"2026-05-20T15:49:25.629633Z","shell.execute_reply":"2026-05-20T15:49:25.637472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import importlib\n\nimport importlib.metadata as importlib_metadata\n\nimport importlib.util\n\nimport os\n\nimport sys\n\nos.environ.setdefault(\"TOKENIZERS_PARALLELISM\", \"false\")\n\nos.environ.setdefault(\"PYTHONHASHSEED\", \"42\")\n\nREQUIRED_IMPORTS = {\n    \"numpy\": \"numpy\",\n    \"pandas\": \"pandas\",\n    \"torch\": \"torch\",\n    \"timm\": \"timm\",\n    \"albumentations\": \"albumentations\",\n    \"cv2\": \"opencv-python\",\n    \"pydicom\": \"pydicom\",\n    \"sklearn\": \"scikit-learn\",\n    \"iterstrat.ml_stratifiers\": \"iterative-stratification\",\n    \"tqdm\": \"tqdm\",\n}\n\ndef module_available(import_name):\n    try:\n        return importlib.util.find_spec(import_name) is not None\n    except ModuleNotFoundError:\n        return False\n\nmissing = []\n\nPACKAGE_VERSIONS = {\"python\": sys.version.split()[0]}\n\nfor import_name, dist_name in REQUIRED_IMPORTS.items():\n    if not module_available(import_name):\n        missing.append(f\"{import_name} (pip package: {dist_name})\")\n        continue\n    try:\n        PACKAGE_VERSIONS[dist_name] = importlib_metadata.version(dist_name)\n    except importlib_metadata.PackageNotFoundError:\n        module = importlib.import_module(import_name.split(\".\")[0])\n        PACKAGE_VERSIONS[dist_name] = getattr(module, \"__version__\", \"unknown\")\n\nif missing:\n    raise ImportError(\n        \"Thiếu package bắt buộc cho notebook Stage 1: \"\n        + \", \".join(missing)\n        + \". Hãy cài/mount đúng môi trường Kaggle trước khi chạy full training.\"\n    )\n\nprint(\"Package validation passed.\")\n\nfor name, version in PACKAGE_VERSIONS.items():\n    print(f\"{name}: {version}\")\n\nimport concurrent.futures\n\nimport csv\n\nimport gc\n\nimport hashlib\n\nimport inspect\n\nimport json\n\nimport math\n\nimport os\n\nimport platform\n\nimport random\n\nimport re\n\nimport shutil\n\nimport subprocess\n\nimport time\n\nimport zipfile\n\nimport warnings\n\nfrom contextlib import nullcontext\n\nfrom datetime import datetime, timezone\n\nfrom pathlib import Path\n\nimport albumentations as A\n\nimport cv2\n\nimport numpy as np\n\nimport pandas as pd\n\nimport pydicom\n\nimport timm\n\nimport torch\n\nimport torch.nn as nn\n\nimport torch.nn.functional as F\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\n\ntry:\n    from pydicom.pixels import apply_modality_lut, apply_voi_lut\nexcept Exception:\n    from pydicom.pixel_data_handlers.util import apply_modality_lut, apply_voi_lut\n\nfrom sklearn.metrics import (\n    average_precision_score,\n    brier_score_loss,\n    f1_score,\n    log_loss,\n    roc_auc_score,\n)\n\nfrom torch.utils.data import DataLoader, Dataset, WeightedRandomSampler\n\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings(\"default\")\n\nif hasattr(torch, \"set_float32_matmul_precision\"):\n    torch.set_float32_matmul_precision(\"high\")\n\nGLOBAL_START_TIME = time.time()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.639997Z","iopub.execute_input":"2026-05-20T15:49:25.640266Z","iopub.status.idle":"2026-05-20T15:49:25.699021Z","shell.execute_reply.started":"2026-05-20T15:49:25.640239Z","shell.execute_reply":"2026-05-20T15:49:25.69795Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Runtime Helpers\n","metadata":{}},{"cell_type":"code","source":"def to_serializable(obj):\n    if isinstance(obj, Path):\n        return str(obj)\n    if isinstance(obj, (np.integer,)):\n        return int(obj)\n    if isinstance(obj, (np.floating,)):\n        return float(obj)\n    if isinstance(obj, np.ndarray):\n        return obj.tolist()\n    if isinstance(obj, torch.Tensor):\n        return obj.detach().cpu().tolist()\n    if isinstance(obj, dict):\n        return {str(k): to_serializable(v) for k, v in obj.items()}\n    if isinstance(obj, (list, tuple)):\n        return [to_serializable(v) for v in obj]\n    return obj\n\ndef write_json(path, payload):\n    path = Path(path)\n    path.parent.mkdir(parents=True, exist_ok=True)\n    with path.open(\"w\", encoding=\"utf-8\") as handle:\n        json.dump(to_serializable(payload), handle, indent=2, ensure_ascii=False)\n\ndef stable_hash(payload):\n    data = json.dumps(to_serializable(payload), sort_keys=True, ensure_ascii=False).encode(\"utf-8\")\n    return hashlib.sha256(data).hexdigest()\n\ndef env_int(name, default, minimum=None, maximum=None):\n    raw = os.environ.get(name, \"\").strip()\n    value = int(default) if not raw else int(raw)\n    if minimum is not None and value < minimum:\n        raise ValueError(f\"{name} phải >= {minimum}, nhận {value}.\")\n    if maximum is not None and value > maximum:\n        raise ValueError(f\"{name} phải <= {maximum}, nhận {value}.\")\n    return value\n\ndef env_float(name, default, minimum=None, maximum=None):\n    raw = os.environ.get(name, \"\").strip()\n    value = float(default) if not raw else float(raw)\n    if minimum is not None and value < minimum:\n        raise ValueError(f\"{name} phải >= {minimum}, nhận {value}.\")\n    if maximum is not None and value > maximum:\n        raise ValueError(f\"{name} phải <= {maximum}, nhận {value}.\")\n    return value\n\ndef env_bool(name, default=False):\n    raw = os.environ.get(name, \"\").strip().lower()\n    if raw in {\"\", \"none\", \"null\"}:\n        return bool(default)\n    if raw in {\"1\", \"true\", \"yes\", \"y\", \"on\"}:\n        return True\n    if raw in {\"0\", \"false\", \"no\", \"n\", \"off\"}:\n        return False\n    raise ValueError(f\"{name} phải là boolean dạng 1/0, true/false, yes/no; nhận {raw!r}.\")\n\ndef parse_env_paths(raw_value):\n    if not raw_value:\n        return []\n    parts = re.split(r\"[;|]\", raw_value)\n    return [part.strip() for part in parts if part.strip()]\n\ndef preprocessed_cache_dir_name(target_size, cache_format=\"png\", cache_version=\"dicom_modality_voi_lanczos_uint16_v1\"):\n    return f\"img{int(target_size)}_{cache_format}_{cache_version}\"\n\ndef probe_runtime():\n    runtime = {\n        \"timestamp_utc\": datetime.now(timezone.utc).isoformat(),\n        \"platform\": platform.platform(),\n        \"device\": \"cuda\" if torch.cuda.is_available() else \"cpu\",\n        \"gpu_name\": None,\n        \"gpu_vram_gb\": 0.0,\n        \"gpu_compute_capability\": None,\n        \"gpu_count\": 0,\n        \"gpu_names\": [],\n        \"gpu_vram_gb_all\": [],\n        \"supports_bf16\": False,\n        \"supports_weights_only_kwarg\": \"weights_only\" in inspect.signature(torch.load).parameters,\n        \"cpu_count\": os.cpu_count(),\n    }\n    if torch.cuda.is_available():\n        runtime[\"gpu_count\"] = int(torch.cuda.device_count())\n        for gpu_idx in range(runtime[\"gpu_count\"]):\n            props_i = torch.cuda.get_device_properties(gpu_idx)\n            runtime[\"gpu_names\"].append(props_i.name)\n            runtime[\"gpu_vram_gb_all\"].append(props_i.total_memory / (1024 ** 3))\n        props = torch.cuda.get_device_properties(0)\n        runtime[\"gpu_name\"] = props.name\n        runtime[\"gpu_vram_gb\"] = props.total_memory / (1024 ** 3)\n        runtime[\"gpu_compute_capability\"] = f\"{props.major}.{props.minor}\"\n        if hasattr(torch.cuda, \"is_bf16_supported\"):\n            try:\n                runtime[\"supports_bf16\"] = bool(torch.cuda.is_bf16_supported())\n            except Exception:\n                runtime[\"supports_bf16\"] = False\n    cpu_count = os.cpu_count() or 2\n    runtime[\"num_workers\"] = min(6, max(2, cpu_count - 1))\n    runtime[\"pin_memory\"] = runtime[\"device\"] == \"cuda\"\n    runtime[\"persistent_workers\"] = runtime[\"num_workers\"] > 0\n    return runtime\n\ndef probe_cuda_arch_compatibility(runtime):\n    check = {\n        \"checked\": False,\n        \"compatible\": True,\n        \"gpu_sm\": None,\n        \"torch_cuda_arch_list\": [],\n        \"supported_sm\": [],\n        \"message\": \"CUDA is not available; architecture compatibility check skipped.\",\n    }\n    if not torch.cuda.is_available():\n        return check\n    check[\"checked\"] = True\n    try:\n        capability = torch.cuda.get_device_capability(0)\n        gpu_sm = f\"sm_{capability[0]}{capability[1]}\"\n        arch_list = list(torch.cuda.get_arch_list()) if hasattr(torch.cuda, \"get_arch_list\") else []\n        supported_sm = sorted({arch for arch in arch_list if str(arch).startswith(\"sm_\")})\n        compatible = gpu_sm in supported_sm if supported_sm else None\n        check.update(\n            {\n                \"compatible\": compatible,\n                \"gpu_sm\": gpu_sm,\n                \"torch_cuda_arch_list\": arch_list,\n                \"supported_sm\": supported_sm,\n            }\n        )\n        if compatible is True:\n            check[\"message\"] = f\"GPU architecture {gpu_sm} is supported by this PyTorch CUDA build.\"\n        elif compatible is False:\n            check[\"message\"] = (\n                f\"GPU {runtime.get('gpu_name')} ({gpu_sm}) không tương thích với PyTorch CUDA build hiện tại. \"\n                f\"PyTorch build hỗ trợ: {supported_sm}. Hãy đổi Kaggle GPU sang T4/V100/A100/L4 \"\n                f\"hoặc cài PyTorch build hỗ trợ {gpu_sm} trước khi train.\"\n            )\n        else:\n            check[\"message\"] = \"Could not determine PyTorch CUDA architecture list; continuing with runtime smoke tests.\"\n        return check\n    except Exception as exc:\n        check.update({\"compatible\": None, \"message\": f\"CUDA architecture check failed to inspect runtime: {exc}\"})\n        return check\n\ndef assert_cuda_arch_compatibility(runtime):\n    check = runtime.get(\"cuda_arch_check\", {})\n    if check.get(\"compatible\") is False:\n        raise RuntimeError(check.get(\"message\", \"GPU/PyTorch CUDA architecture mismatch.\"))\n\nRUNTIME = probe_runtime()\n\nRUNTIME[\"cuda_arch_check\"] = probe_cuda_arch_compatibility(RUNTIME)\n\nDEVICE = torch.device(RUNTIME[\"device\"])\n\nSTAGE1_SPEED_PERSISTENCE_VERSION = \"stage1_speed_persistence_v3_phase_backup_guard_2026_05_18\"\n\nRUN_TIMESTAMP = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.829513Z","iopub.execute_input":"2026-05-20T15:49:25.830359Z","iopub.status.idle":"2026-05-20T15:49:25.858417Z","shell.execute_reply.started":"2026-05-20T15:49:25.830323Z","shell.execute_reply":"2026-05-20T15:49:25.857026Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Configuration\n","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n    \"spec_version\": \"v4.1\",\n    \"system_spec_version\": \"v6.2\",\n    \"output_schema_version\": \"stage1_v1\",\n    \"stage\": \"stage1_plus_1_5_lite\",\n    \"run_profile\": os.environ.get(\"STAGE1_RUN_PROFILE\", \"screening\").strip() or \"screening\",\n    \"job_mode\": os.environ.get(\"STAGE1_JOB_MODE\", \"all\").strip().lower() or \"all\",\n    \"restore_survival_pack\": os.environ.get(\"STAGE1_RESTORE_PACK\", \"\").strip(),\n    \"restore_overwrite\": env_bool(\"STAGE1_RESTORE_OVERWRITE\", False),\n    \"allow_local_only_training\": env_bool(\"STAGE1_ALLOW_LOCAL_ONLY\", False),\n    \"require_external_backup_for_training\": env_bool(\"STAGE1_REQUIRE_EXTERNAL_BACKUP\", True),\n    \"strict_checkpoint_contract\": env_bool(\"STAGE1_STRICT_CHECKPOINT_CONTRACT\", True),\n    \"force_rerun_completed_phase\": env_bool(\"STAGE1_FORCE_RERUN_COMPLETED_PHASE\", False),\n    \"prebuild_cache_in_training_jobs\": env_bool(\"STAGE1_PREBUILD_CACHE_IN_TRAINING\", False),\n    \"run_smoke_tests\": env_bool(\"STAGE1_RUN_SMOKE\", False),\n    \"run_model_name\": \"vit_small_patch14_dinov2_lvd142m\",\n    \"run_id_prefix\": RUN_TIMESTAMP,\n    \"resume_run_id\": os.environ.get(\"STAGE1_RUN_ID\", \"\").strip(),\n    \"force_new_run\": False,\n    \"data_root_candidates\": [\n        \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection\",\n        \"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection\",\n    ],\n    \"output_root\": \"/kaggle/working/vindr_stage1_v41\",\n    \"timm_cache_dir\": os.environ.get(\"STAGE1_TIMM_CACHE_DIR\", \"/kaggle/working/.cache/timm\").strip() or \"/kaggle/working/.cache/timm\",\n    \"preprocessing_mode\": os.environ.get(\"STAGE1_PREPROCESSING_MODE\", \"modality_voi_uint16\").strip().lower() or \"modality_voi_uint16\",\n    \"preprocessed_cache_root\": os.environ.get(\"STAGE1_CACHE_ROOT\", \"/kaggle/temp/vindr_preprocessed_cache_v41\").strip() or \"/kaggle/temp/vindr_preprocessed_cache_v41\",\n    \"preprocessed_cache_format\": \"png\",\n    \"preprocessed_cache_dtype\": os.environ.get(\"STAGE1_CACHE_DTYPE\", \"uint16\").strip().lower() or \"uint16\",\n    \"preprocessed_cache_max_value\": env_int(\"STAGE1_CACHE_MAX_VALUE\", 65535, minimum=255, maximum=65535),\n    \"preprocessed_cache_version\": os.environ.get(\"STAGE1_CACHE_VERSION\", \"dicom_modality_voi_lanczos_uint16_v1\").strip() or \"dicom_modality_voi_lanczos_uint16_v1\",\n    \"preprocessed_cache_read_roots\": parse_env_paths(os.environ.get(\"STAGE1_CACHE_READ_ROOTS\", \"\")),\n    \"preprocessed_cache_enabled\": True,\n    \"prebuild_preprocessed_cache\": True,\n    \"preprocessed_cache_workers\": env_int(\n        \"STAGE1_CACHE_WORKERS\",\n        min(8, max(2, os.cpu_count() or 2)),\n        minimum=1,\n        maximum=16,\n    ),\n    \"preprocessed_cache_fail_on_error\": False,\n    \"preprocessed_cache_min_free_gb\": env_float(\"STAGE1_CACHE_MIN_FREE_GB\", 8.0, minimum=0.5),\n    \"preprocessed_cache_png_compression\": env_int(\"STAGE1_CACHE_PNG_COMPRESSION\", 1, minimum=0, maximum=9),\n    \"num_classes\": 14,\n    \"n_radiologists\": 3,\n    \"soft_label_values\": [0.0, 1.0 / 3.0, 2.0 / 3.0, 1.0],\n    \"positive_threshold\": 0.50,\n    \"split_seed\": 42,\n    \"train_core_ratio\": 0.80,\n    \"val_select_ratio\": 0.10,\n    \"calib_ratio\": 0.10,\n    \"holdout_eval_policy\": \"official_test_consensus_or_separate_final_audit_not_used_in_stage1_lite\",\n    \"use_internal_holdout_eval\": False,\n    \"effective_batch_size\": 16,\n    \"loss_name\": \"qfl\",\n    \"bce_fallback_available\": True,\n    \"qfl_beta\": 2.0,\n    \"qfl_epsilon\": 1e-6,\n    \"alpha_max\": 5.0,\n    \"bce_pos_weight_max\": 20.0,\n    \"grad_clip_norm\": 1.0,\n    \"use_amp\": True,\n    \"cudnn_benchmark\": True,\n    \"channels_last\": True,\n    \"channels_last_families\": [\"cnn\"],\n    \"prefetch_factor\": 4,\n    \"eval_batch_multiplier\": 3,\n    \"phase2_batch_multiplier\": 2,\n    \"phase2_max_physical_bs\": 16,\n    \"allow_tf32\": True,\n    \"resume\": True,\n    \"soft_time_limit_hours\": 10.5,\n    \"min_eval_positive\": 1,\n    \"temperature_max_iter\": 50,\n    \"threshold_grid\": [round(x, 2) for x in np.arange(0.10, 0.91, 0.05)],\n    \"near_threshold_margin\": 0.05,\n    \"weak_localization_top_k\": 3,\n    \"backup_enabled\": True,\n    \"backup_every_epochs\": 1,\n    \"backup_include_predictions\": False,\n    \"backup_fail_on_error\": False,\n    \"checkpoint_backup_dataset_ref\": os.environ.get(\"STAGE1_CHECKPOINT_BACKUP_DATASET\", os.environ.get(\"STAGE1_BACKUP_DATASET\", \"\")).strip(),\n    \"failure_backup_dataset_ref\": os.environ.get(\"STAGE1_FAILURE_BACKUP_DATASET\", \"\").strip(),\n    \"cache_backup_dataset_ref\": os.environ.get(\"STAGE1_CACHE_BACKUP_DATASET\", \"\").strip(),\n    \"backup_kaggle_dataset_ref\": os.environ.get(\"STAGE1_CHECKPOINT_BACKUP_DATASET\", os.environ.get(\"STAGE1_BACKUP_DATASET\", \"\")).strip(),\n    \"backup_kaggle_dataset_enabled\": bool(os.environ.get(\"STAGE1_CHECKPOINT_BACKUP_DATASET\", os.environ.get(\"STAGE1_BACKUP_DATASET\", \"\")).strip()),\n    \"backup_kaggle_version_timeout_sec\": 900,\n    \"model_configs\": {\n        \"efficientnetv2_s\": {\n            \"display_name\": \"EfficientNetV2-S\",\n            \"family\": \"cnn\",\n            \"timm_candidates\": [\"tf_efficientnetv2_s\", \"efficientnetv2_s\"],\n            \"pretrained_timm_candidates\": [\n                \"tf_efficientnetv2_s.in21k\",\n                \"tf_efficientnetv2_s.in21k_ft_in1k\",\n                \"tf_efficientnetv2_s.in1k\",\n                \"tf_efficientnetv2_s_in21k\",\n                \"tf_efficientnetv2_s_in21ft1k\",\n            ],\n            \"img_size\": 1024,\n            \"physical_bs\": 8,\n            \"grad_checkpoint\": False,\n            \"weak_localization_method\": \"gradcam\",\n            \"timm_kwargs\": {},\n        },\n        \"vit_small_patch14_dinov2_lvd142m\": {\n            \"display_name\": \"DINOv2-ViT-S/14\",\n            \"family\": \"vit\",\n            \"timm_candidates\": [\"vit_small_patch14_dinov2\"],\n            \"pretrained_timm_candidates\": [\n                \"vit_small_patch14_dinov2.lvd142m\",\n                \"vit_small_patch14_dinov2\",\n            ],\n            \"img_size\": 1022,\n            \"physical_bs\": 4,\n            \"grad_checkpoint\": True,\n            \"weak_localization_method\": \"attention_rollout\",\n            \"timm_kwargs\": {\"img_size\": 1022},\n        },\n        \"convnextv2_base\": {\n            \"display_name\": \"ConvNeXtV2-Base\",\n            \"family\": \"cnn\",\n            \"timm_candidates\": [\"convnextv2_base\"],\n            \"pretrained_timm_candidates\": [\n                \"convnextv2_base.fcmae_ft_in22k_in1k\",\n                \"convnextv2_base.fcmae_ft_in1k\",\n                \"convnextv2_base.fcmae\",\n            ],\n            \"img_size\": 1024,\n            \"physical_bs\": 4,\n            \"grad_checkpoint\": True,\n            \"weak_localization_method\": \"gradcam\",\n            \"timm_kwargs\": {},\n        },\n    },\n    \"profiles\": {\n        \"screening_fast\": {\n            \"phase1_epochs\": 6,\n            \"phase2_epochs\": 2,\n            \"patience_phase1\": 2,\n            \"patience_phase2\": 1,\n            \"phase1_lr_backbone\": 1e-4,\n            \"phase1_lr_head\": 1e-3,\n            \"phase2_lr_head\": 5e-4,\n            \"weight_decay_backbone\": 0.01,\n            \"weight_decay_head\": 0.001,\n            \"warmup_ratio\": 0.05,\n        },\n        \"screening\": {\n            \"phase1_epochs\": 18,\n            \"phase2_epochs\": 4,\n            \"patience_phase1\": 5,\n            \"patience_phase2\": 3,\n            \"phase1_lr_backbone\": 1e-4,\n            \"phase1_lr_head\": 1e-3,\n            \"phase2_lr_head\": 5e-4,\n            \"weight_decay_backbone\": 0.01,\n            \"weight_decay_head\": 0.001,\n            \"warmup_ratio\": 0.05,\n        },\n        \"full\": {\n            \"phase1_epochs\": 50,\n            \"phase2_epochs\": 10,\n            \"patience_phase1\": 10,\n            \"patience_phase2\": 5,\n            \"phase1_lr_backbone\": 1e-4,\n            \"phase1_lr_head\": 1e-3,\n            \"phase2_lr_head\": 5e-4,\n            \"weight_decay_backbone\": 0.01,\n            \"weight_decay_head\": 0.001,\n            \"warmup_ratio\": 0.05,\n        },\n    },\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.859268Z","iopub.status.idle":"2026-05-20T15:49:25.85971Z","shell.execute_reply.started":"2026-05-20T15:49:25.859479Z","shell.execute_reply":"2026-05-20T15:49:25.859504Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Derived Paths and Runtime Audit\n","metadata":{}},{"cell_type":"code","source":"CONFIG.update(CONFIG[\"profiles\"][CONFIG[\"run_profile\"]])\n\nif not os.environ.get(\"STAGE1_CACHE_VERSION\", \"\").strip():\n    if CONFIG[\"preprocessing_mode\"] == \"legacy_voi_uint8\":\n        CONFIG[\"preprocessed_cache_version\"] = \"dicom_voi_minmax_lanczos_uint8_v1\"\n        CONFIG[\"preprocessed_cache_dtype\"] = \"uint8\"\n        CONFIG[\"preprocessed_cache_max_value\"] = 255\n    elif CONFIG[\"preprocessing_mode\"] == \"modality_voi_uint8\":\n        CONFIG[\"preprocessed_cache_version\"] = \"dicom_modality_voi_lanczos_uint8_v1\"\n        CONFIG[\"preprocessed_cache_dtype\"] = \"uint8\"\n        CONFIG[\"preprocessed_cache_max_value\"] = 255\n    elif CONFIG[\"preprocessing_mode\"] == \"modality_voi_uint16\":\n        CONFIG[\"preprocessed_cache_version\"] = \"dicom_modality_voi_lanczos_uint16_v1\"\n        CONFIG[\"preprocessed_cache_dtype\"] = \"uint16\"\n        CONFIG[\"preprocessed_cache_max_value\"] = 65535\n\nMODEL_CONFIG = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n\nPHYSICAL_BS_OVERRIDE = os.environ.get(\"STAGE1_PHYSICAL_BS\", \"\").strip()\n\nif PHYSICAL_BS_OVERRIDE:\n    MODEL_CONFIG[\"physical_bs\"] = int(PHYSICAL_BS_OVERRIDE)\n    CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]][\"physical_bs\"] = int(PHYSICAL_BS_OVERRIDE)\n\nNUM_WORKERS_OVERRIDE = os.environ.get(\"STAGE1_NUM_WORKERS\", \"\").strip()\n\nBASE_RUN_ID = f\"{CONFIG['run_model_name']}_{CONFIG['run_profile']}\"\n\nif CONFIG[\"resume_run_id\"]:\n    CONFIG[\"run_id\"] = CONFIG[\"resume_run_id\"]\nelif CONFIG[\"force_new_run\"]:\n    CONFIG[\"run_id\"] = f\"{CONFIG['run_id_prefix']}_{BASE_RUN_ID}\"\nelse:\n    CONFIG[\"run_id\"] = BASE_RUN_ID\n\nCONFIG[\"use_amp\"] = bool(CONFIG[\"use_amp\"] and DEVICE.type == \"cuda\")\n\nCONFIG[\"num_workers\"] = int(NUM_WORKERS_OVERRIDE) if NUM_WORKERS_OVERRIDE else RUNTIME[\"num_workers\"]\n\nCONFIG[\"pin_memory\"] = RUNTIME[\"pin_memory\"]\n\nCONFIG[\"persistent_workers\"] = bool(RUNTIME[\"persistent_workers\"] and CONFIG[\"num_workers\"] > 0)\n\nCONFIG[\"prefetch_factor\"] = int(CONFIG[\"prefetch_factor\"]) if CONFIG[\"num_workers\"] > 0 else None\n\nCONFIG[\"img_size\"] = MODEL_CONFIG[\"img_size\"]\n\nCONFIG[\"channels_last_active\"] = bool(\n    CONFIG[\"channels_last\"]\n    and DEVICE.type == \"cuda\"\n    and MODEL_CONFIG.get(\"family\") in set(CONFIG[\"channels_last_families\"])\n)\n\nCONFIG[\"speed_persistence_version\"] = STAGE1_SPEED_PERSISTENCE_VERSION\n\nCONFIG[\"backup_channel_map\"] = {\n    \"checkpoint\": CONFIG.get(\"checkpoint_backup_dataset_ref\", \"\"),\n    \"failure\": CONFIG.get(\"failure_backup_dataset_ref\", \"\"),\n    \"cache\": CONFIG.get(\"cache_backup_dataset_ref\", \"\"),\n}\n\nDATA_ROOT = next((Path(p) for p in CONFIG[\"data_root_candidates\"] if Path(p).exists()), Path(CONFIG[\"data_root_candidates\"][0]))\n\nTRAIN_CSV = DATA_ROOT / \"train.csv\"\n\nDICOM_DIR = DATA_ROOT / \"train\"\n\nRUN_DIR = Path(CONFIG[\"output_root\"]) / \"runs\" / CONFIG[\"run_id\"]\n\nBEST_DIR = Path(CONFIG[\"output_root\"]) / \"best\"\n\nPATHS = {\n    \"run_dir\": RUN_DIR,\n    \"best_dir\": BEST_DIR,\n    \"checkpoints\": RUN_DIR / \"checkpoints\",\n    \"calibration\": RUN_DIR / \"calibration\",\n    \"predictions\": RUN_DIR / \"predictions\",\n    \"metrics\": RUN_DIR / \"metrics\",\n    \"config\": RUN_DIR / \"config.json\",\n    \"runtime\": RUN_DIR / \"runtime.json\",\n    \"package_versions\": RUN_DIR / \"package_versions.json\",\n    \"split_manifest\": RUN_DIR / \"split_manifest.json\",\n    \"class_weights\": RUN_DIR / \"class_weights.json\",\n    \"epoch_log\": RUN_DIR / \"epoch_log.csv\",\n    \"failure_summary\": RUN_DIR / \"failure_summary.json\",\n    \"last_checkpoint\": RUN_DIR / \"checkpoints\" / \"last.pt\",\n    \"best_phase1_checkpoint\": RUN_DIR / \"checkpoints\" / \"best_phase1.pt\",\n    \"best_phase2_checkpoint\": RUN_DIR / \"checkpoints\" / \"best_phase2.pt\",\n    \"best_stage1_checkpoint\": RUN_DIR / \"checkpoints\" / \"best_stage1.pt\",\n    \"phase1_complete\": RUN_DIR / \"phase1_complete.json\",\n    \"phase2_complete\": RUN_DIR / \"phase2_complete.json\",\n    \"calibration_summary\": RUN_DIR / \"calibration\" / \"calibration_summary.json\",\n    \"thresholds\": RUN_DIR / \"calibration\" / \"thresholds.json\",\n    \"val_predictions\": RUN_DIR / \"predictions\" / \"val_select_predictions.csv\",\n    \"calib_predictions\": RUN_DIR / \"predictions\" / \"calib_predictions.csv\",\n    \"model_summary\": RUN_DIR / \"metrics\" / \"model_summary.csv\",\n    \"per_class_val\": RUN_DIR / \"metrics\" / \"per_class_val_select.csv\",\n    \"per_class_calib\": RUN_DIR / \"metrics\" / \"per_class_calib.csv\",\n    \"stage1_bundle\": RUN_DIR / \"stage1_bundle.json\",\n    \"preprocessed_cache\": Path(CONFIG[\"preprocessed_cache_root\"]) / preprocessed_cache_dir_name(\n        MODEL_CONFIG[\"img_size\"],\n        CONFIG[\"preprocessed_cache_format\"],\n        CONFIG[\"preprocessed_cache_version\"],\n    ),\n    \"preprocessed_cache_summary\": RUN_DIR / \"preprocessed_cache_summary.json\",\n    \"survival_packs\": Path(CONFIG[\"output_root\"]) / \"survival_packs\" / CONFIG[\"run_id\"],\n    \"latest_survival_pack\": Path(CONFIG[\"output_root\"]) / \"survival_packs\" / CONFIG[\"run_id\"] / \"latest_stage1_survival_pack.zip\",\n    \"backup_publish\": Path(CONFIG[\"output_root\"]) / \"dataset_backup_publish\" / CONFIG[\"run_id\"],\n    \"checkpoint_backup_publish\": Path(CONFIG[\"output_root\"]) / \"dataset_backup_publish\" / \"checkpoint\" / CONFIG[\"run_id\"],\n    \"failure_backup_publish\": Path(CONFIG[\"output_root\"]) / \"dataset_backup_publish\" / \"failure\" / CONFIG[\"run_id\"],\n    \"cache_backup_publish\": Path(CONFIG[\"output_root\"]) / \"dataset_backup_publish\" / \"cache\" / CONFIG[\"run_id\"],\n    \"backup_manifest\": RUN_DIR / \"backup_manifest.json\",\n    \"best_stage1_model\": BEST_DIR / \"best_stage1_model.pt\",\n    \"best_stage1_bundle\": BEST_DIR / \"best_stage1_bundle.json\",\n}\n\nfor key in [\"run_dir\", \"best_dir\", \"checkpoints\", \"calibration\", \"predictions\", \"metrics\", \"survival_packs\", \"backup_publish\", \"checkpoint_backup_publish\", \"failure_backup_publish\", \"cache_backup_publish\", \"preprocessed_cache\"]:\n    PATHS[key].mkdir(parents=True, exist_ok=True)\n\nPath(CONFIG[\"timm_cache_dir\"]).mkdir(parents=True, exist_ok=True)\n\nos.environ.setdefault(\"HF_HOME\", CONFIG[\"timm_cache_dir\"])\n\nos.environ.setdefault(\"TORCH_HOME\", CONFIG[\"timm_cache_dir\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.861406Z","iopub.status.idle":"2026-05-20T15:49:25.861823Z","shell.execute_reply.started":"2026-05-20T15:49:25.861597Z","shell.execute_reply":"2026-05-20T15:49:25.861642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def configure_torch_backend(runtime, config):\n    settings = {\n        \"cudnn_benchmark\": bool(config[\"cudnn_benchmark\"] and runtime[\"device\"] == \"cuda\"),\n        \"allow_tf32_requested\": bool(config.get(\"allow_tf32\", False)),\n        \"allow_tf32_enabled\": False,\n        \"channels_last_active\": bool(config.get(\"channels_last_active\", False)),\n        \"note\": \"\",\n    }\n    if runtime[\"device\"] == \"cuda\":\n        torch.backends.cudnn.benchmark = settings[\"cudnn_benchmark\"]\n        torch.backends.cudnn.deterministic = not settings[\"cudnn_benchmark\"]\n        try:\n            major = int(str(runtime.get(\"gpu_compute_capability\", \"0.0\")).split(\".\", 1)[0])\n        except Exception:\n            major = 0\n        if settings[\"allow_tf32_requested\"] and major >= 8:\n            torch.backends.cuda.matmul.allow_tf32 = True\n            torch.backends.cudnn.allow_tf32 = True\n            settings[\"allow_tf32_enabled\"] = True\n            settings[\"note\"] = \"TF32 enabled for Ampere-or-newer GPU.\"\n        elif settings[\"allow_tf32_requested\"]:\n            settings[\"note\"] = \"TF32 requested but not enabled because this GPU is pre-Ampere or capability is unknown.\"\n    return settings\n\nRUNTIME[\"torch_backend_settings\"] = configure_torch_backend(RUNTIME, CONFIG)\n\nSPEC_CONSTRAINTS = {\n    \"stage_first_wave_tracked\": True,\n    \"classifier_type\": \"14_sigmoid_multi_label\",\n    \"soft_label_radiologists_per_image\": 3,\n    \"no_label_smoothing_on_p_soft\": True,\n    \"loss_primary\": \"qfl_plus_per_class_alpha\",\n    \"loss_fallback\": \"bce_with_logits\",\n    \"calibration_split_is_separate\": True,\n    \"checkpoint_selection_metric\": \"val_select_macro_mAP\",\n    \"dino_img_size_constraint\": 1022,\n    \"no_bulk_heatmap_generation_in_training\": True,\n    \"no_fake_box_policy\": True,\n}\n\nwrite_json(PATHS[\"config\"], CONFIG)\n\nwrite_json(PATHS[\"runtime\"], RUNTIME)\n\nwrite_json(PATHS[\"package_versions\"], PACKAGE_VERSIONS)\n\nassert_cuda_arch_compatibility(RUNTIME)\n\nprint(f\"Device: {DEVICE}\")\n\nif DEVICE.type == \"cuda\":\n    print(f\"GPU: {RUNTIME['gpu_name']} | VRAM: {RUNTIME['gpu_vram_gb']:.1f} GB | CC: {RUNTIME['gpu_compute_capability']}\")\n\nprint(f\"Run id: {CONFIG['run_id']}\")\n\nprint(f\"Job mode: {CONFIG['job_mode']}\")\n\nprint(f\"Output root: {CONFIG['output_root']}\")\n\nprint(f\"Run model: {CONFIG['run_model_name']} | image size: {MODEL_CONFIG['img_size']}\")\n\nprint(f\"AMP enabled: {CONFIG['use_amp']}\")\n\nprint(f\"Speed/persistence version: {STAGE1_SPEED_PERSISTENCE_VERSION}\")\n\nprint(f\"channels_last active: {CONFIG['channels_last_active']} | num_workers: {CONFIG['num_workers']} | prefetch_factor: {CONFIG['prefetch_factor']}\")\n\nprint(f\"Preprocessed image cache: enabled={CONFIG['preprocessed_cache_enabled']} prebuild={CONFIG['prebuild_preprocessed_cache']} path={PATHS['preprocessed_cache']}\")\n\nprint(f\"Preprocessing mode: {CONFIG['preprocessing_mode']} | cache dtype: {CONFIG['preprocessed_cache_dtype']} | max: {CONFIG['preprocessed_cache_max_value']}\")\n\nprint(f\"Cache version: {CONFIG['preprocessed_cache_version']} | workers: {CONFIG['preprocessed_cache_workers']} | min_free_gb: {CONFIG['preprocessed_cache_min_free_gb']}\")\n\nif CONFIG[\"preprocessed_cache_read_roots\"]:\n    print(f\"Read-only cache roots: {CONFIG['preprocessed_cache_read_roots']}\")\n\nif RUNTIME.get(\"gpu_count\", 0) > 1:\n    print(f\"Visible GPUs: {RUNTIME['gpu_count']} -> {RUNTIME['gpu_names']} (training still uses single-process cuda:0 in this notebook)\")\n\nif CONFIG[\"backup_kaggle_dataset_enabled\"]:\n    print(f\"Checkpoint Kaggle Dataset backup enabled: {CONFIG['backup_kaggle_dataset_ref']}\")\nelse:\n    print(\"Checkpoint Kaggle Dataset backup disabled. Set STAGE1_CHECKPOINT_BACKUP_DATASET or legacy STAGE1_BACKUP_DATASET.\")\nprint(f\"Backup channel map: {CONFIG['backup_channel_map']}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:25.862994Z","iopub.status.idle":"2026-05-20T15:49:25.863271Z","shell.execute_reply.started":"2026-05-20T15:49:25.863146Z","shell.execute_reply":"2026-05-20T15:49:25.863162Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Spec Guards and Common Utilities\n","metadata":{}},{"cell_type":"code","source":"def assert_spec_contract():\n    legacy_multi_model_key = \"active\" + \"_model_names\"\n    if legacy_multi_model_key in CONFIG:\n        raise AssertionError(\"Notebook Stage 1 Lite không được chạy nhiều model tuần tự mặc định.\")\n    if CONFIG[\"run_profile\"] not in CONFIG[\"profiles\"]:\n        raise AssertionError(f\"run_profile không tồn tại trong profiles: {CONFIG['run_profile']}\")\n    if CONFIG[\"job_mode\"] not in {\"cache\", \"phase1\", \"phase2\", \"finalize\", \"all\"}:\n        raise AssertionError(\"STAGE1_JOB_MODE phải là một trong: cache, phase1, phase2, finalize, all.\")\n    if CONFIG[\"preprocessing_mode\"] not in {\"legacy_voi_uint8\", \"modality_voi_uint8\", \"modality_voi_uint16\"}:\n        raise AssertionError(\"STAGE1_PREPROCESSING_MODE phải là legacy_voi_uint8, modality_voi_uint8 hoặc modality_voi_uint16.\")\n    if CONFIG[\"preprocessed_cache_dtype\"] not in {\"uint8\", \"uint16\"}:\n        raise AssertionError(\"STAGE1_CACHE_DTYPE phải là uint8 hoặc uint16.\")\n    if CONFIG[\"preprocessing_mode\"].endswith(\"uint16\") and CONFIG[\"preprocessed_cache_dtype\"] != \"uint16\":\n        raise AssertionError(\"modality_voi_uint16 yêu cầu preprocessed_cache_dtype=uint16.\")\n    if CONFIG[\"preprocessed_cache_dtype\"] == \"uint16\" and int(CONFIG[\"preprocessed_cache_max_value\"]) != 65535:\n        raise AssertionError(\"uint16 cache phải dùng max value 65535 để tránh co dải động ngoài ý muốn.\")\n    if CONFIG[\"run_model_name\"] not in CONFIG[\"model_configs\"]:\n        raise AssertionError(\"run_model_name không tồn tại trong model_configs.\")\n    if CONFIG[\"model_configs\"][\"vit_small_patch14_dinov2_lvd142m\"][\"img_size\"] != 1022:\n        raise AssertionError(\"DINOv2-ViT-S/14 phải dùng img_size=1022 hoặc dynamic padding rõ ràng.\")\n    ratios = CONFIG[\"train_core_ratio\"] + CONFIG[\"val_select_ratio\"] + CONFIG[\"calib_ratio\"]\n    if abs(ratios - 1.0) > 1e-6:\n        raise AssertionError(\"Stage 1 Lite split ratio phải cộng thành 1.0 cho train_core/val_select/calib.\")\n    if CONFIG[\"n_radiologists\"] != 3:\n        raise AssertionError(\"VinDr-CXR train label phải xuất phát từ R=3 bác sĩ/ảnh.\")\n    if CONFIG[\"loss_name\"] not in {\"qfl\", \"bce\"}:\n        raise AssertionError(\"loss_name phải là qfl hoặc bce fallback.\")\n    selected_physical_bs = int(CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]][\"physical_bs\"])\n    if selected_physical_bs < 1:\n        raise AssertionError(\"physical_bs phải >= 1.\")\n    if int(CONFIG[\"effective_batch_size\"]) < 1:\n        raise AssertionError(\"effective_batch_size phải >= 1.\")\n    if int(CONFIG[\"eval_batch_multiplier\"]) < 1:\n        raise AssertionError(\"eval_batch_multiplier phải >= 1.\")\n    if int(CONFIG[\"backup_every_epochs\"]) < 1:\n        raise AssertionError(\"backup_every_epochs phải >= 1.\")\n    if int(CONFIG.get(\"num_workers\", 0)) < 0:\n        raise AssertionError(\"num_workers phải >= 0.\")\n    if CONFIG.get(\"prefetch_factor\") is not None and int(CONFIG[\"prefetch_factor\"]) < 1:\n        raise AssertionError(\"prefetch_factor phải >= 1 khi num_workers > 0.\")\n    if CONFIG[\"preprocessed_cache_format\"] != \"png\":\n        raise AssertionError(\"Stage 1 cache chỉ dùng PNG lossless; không dùng định dạng lossy cho ảnh y khoa.\")\n    if not str(CONFIG[\"preprocessed_cache_version\"]).strip():\n        raise AssertionError(\"preprocessed_cache_version không được rỗng để tránh stale cache.\")\n    if not (0 <= int(CONFIG[\"preprocessed_cache_png_compression\"]) <= 9):\n        raise AssertionError(\"preprocessed_cache_png_compression phải nằm trong [0, 9].\")\n    if float(CONFIG[\"preprocessed_cache_min_free_gb\"]) < 0.5:\n        raise AssertionError(\"preprocessed_cache_min_free_gb quá thấp, có nguy cơ lấp đầy /kaggle/working.\")\n\ndef set_seed(seed=42, benchmark=False):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = not benchmark\n    torch.backends.cudnn.benchmark = bool(benchmark)\n\ndef elapsed_hours():\n    return (time.time() - GLOBAL_START_TIME) / 3600.0\n\ndef should_stop_for_time():\n    return elapsed_hours() >= float(CONFIG[\"soft_time_limit_hours\"])\n\ndef slugify_class_name(name):\n    return re.sub(r\"[^a-z0-9]+\", \"_\", name.lower()).strip(\"_\")\n\ndef loader_kwargs():\n    kwargs = {\n        \"num_workers\": CONFIG[\"num_workers\"],\n        \"pin_memory\": CONFIG[\"pin_memory\"],\n        \"persistent_workers\": bool(CONFIG[\"persistent_workers\"] and CONFIG[\"num_workers\"] > 0),\n    }\n    if CONFIG[\"num_workers\"] > 0 and CONFIG.get(\"prefetch_factor\") is not None:\n        kwargs[\"prefetch_factor\"] = int(CONFIG[\"prefetch_factor\"])\n    return kwargs\n\ndef use_channels_last_for_config(model_config=None):\n    selected_config = MODEL_CONFIG if model_config is None else model_config\n    return bool(\n        CONFIG.get(\"channels_last_active\", False)\n        and selected_config.get(\"family\") in set(CONFIG.get(\"channels_last_families\", []))\n    )\n\ndef move_images_to_device(images, device, model_config=None):\n    if use_channels_last_for_config(model_config) and images.ndim == 4:\n        return images.to(device, non_blocking=True, memory_format=torch.channels_last)\n    return images.to(device, non_blocking=True)\n\ndef prepare_model_for_runtime(model, model_config, device):\n    model = model.to(device)\n    if use_channels_last_for_config(model_config):\n        model = model.to(memory_format=torch.channels_last)\n    return model\n\ndef autocast_context(device, enabled):\n    if hasattr(torch, \"amp\") and hasattr(torch.amp, \"autocast\"):\n        return torch.amp.autocast(device_type=device.type, enabled=bool(enabled and device.type == \"cuda\"))\n    if device.type == \"cuda\":\n        return torch.cuda.amp.autocast(enabled=bool(enabled))\n    return nullcontext()\n\ndef make_grad_scaler(device, enabled):\n    enabled = bool(enabled and device.type == \"cuda\")\n    if hasattr(torch, \"amp\") and hasattr(torch.amp, \"GradScaler\"):\n        try:\n            return torch.amp.GradScaler(device.type, enabled=enabled)\n        except TypeError:\n            return torch.amp.GradScaler(enabled=enabled)\n    return torch.cuda.amp.GradScaler(enabled=enabled)\n\ndef safe_torch_load(path, map_location):\n    params = inspect.signature(torch.load).parameters\n    if \"weights_only\" in params:\n        return torch.load(path, map_location=map_location, weights_only=False)\n    return torch.load(path, map_location=map_location)\n\ndef append_epoch_log(row):\n    path = PATHS[\"epoch_log\"]\n    path.parent.mkdir(parents=True, exist_ok=True)\n    fieldnames = [\n        \"run_id\",\n        \"spec_version\",\n        \"wave\",\n        \"stage\",\n        \"phase\",\n        \"epoch\",\n        \"global_step\",\n        \"model_name\",\n        \"resolved_timm_name\",\n        \"img_size\",\n        \"batch_size\",\n        \"accum_steps\",\n        \"train_loss\",\n        \"val_loss\",\n        \"val_macro_mAP\",\n        \"val_AUROC_macro\",\n        \"val_F1_macro\",\n        \"val_ECE\",\n        \"val_Brier\",\n        \"best_metric\",\n        \"best_epoch\",\n        \"elapsed_hours\",\n        \"notes\",\n    ]\n    new_file = not path.exists()\n    with path.open(\"a\", newline=\"\", encoding=\"utf-8\") as handle:\n        writer = csv.DictWriter(handle, fieldnames=fieldnames)\n        if new_file:\n            writer.writeheader()\n        writer.writerow({key: row.get(key) for key in fieldnames})\n        handle.flush()\n\ndef write_failure_summary(exc, context):\n    payload = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"context\": context,\n        \"exception_type\": type(exc).__name__,\n        \"exception_message\": str(exc),\n        \"elapsed_hours\": elapsed_hours(),\n        \"runtime\": RUNTIME,\n        \"config\": CONFIG,\n    }\n    write_json(PATHS[\"failure_summary\"], payload)\n\nset_seed(CONFIG[\"split_seed\"], benchmark=CONFIG[\"cudnn_benchmark\"])\n\nassert_spec_contract()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.038472Z","iopub.execute_input":"2026-05-20T15:49:26.039257Z","iopub.status.idle":"2026-05-20T15:49:26.067378Z","shell.execute_reply.started":"2026-05-20T15:49:26.039223Z","shell.execute_reply":"2026-05-20T15:49:26.065571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Labels and Split Manifest\n","metadata":{}},{"cell_type":"code","source":"CLASS_NAMES = [\n    \"Aortic enlargement\",\n    \"Atelectasis\",\n    \"Calcification\",\n    \"Cardiomegaly\",\n    \"Consolidation\",\n    \"ILD\",\n    \"Infiltration\",\n    \"Lung Opacity\",\n    \"Nodule/Mass\",\n    \"Other lesion\",\n    \"Pleural effusion\",\n    \"Pleural thickening\",\n    \"Pneumothorax\",\n    \"Pulmonary fibrosis\",\n]\n\ndef load_and_process_csv(csv_path, num_classes=14, n_radiologists=3):\n    if not Path(csv_path).exists():\n        raise FileNotFoundError(f\"Không tìm thấy train.csv tại {csv_path}\")\n    df = pd.read_csv(csv_path)\n    required_cols = {\"image_id\", \"class_id\", \"rad_id\"}\n    missing_cols = required_cols.difference(df.columns)\n    if missing_cols:\n        raise ValueError(f\"train.csv thiếu cột bắt buộc: {sorted(missing_cols)}\")\n\n    all_image_ids = sorted(df[\"image_id\"].astype(str).unique().tolist())\n    img_to_idx = {image_id: idx for idx, image_id in enumerate(all_image_ids)}\n    label_matrix = np.zeros((len(all_image_ids), num_classes), dtype=np.float32)\n\n    pathology_df = df[df[\"class_id\"] != 14].copy()\n    grouped = (\n        pathology_df.groupby([\"image_id\", \"class_id\"])[\"rad_id\"]\n        .nunique()\n        .reset_index()\n        .rename(columns={\"rad_id\": \"n_positive\"})\n    )\n    grouped[\"n_positive\"] = grouped[\"n_positive\"].clip(lower=0, upper=n_radiologists)\n    for row in grouped.itertuples(index=False):\n        class_id = int(row.class_id)\n        if 0 <= class_id < num_classes:\n            label_matrix[img_to_idx[str(row.image_id)], class_id] = float(row.n_positive) / float(n_radiologists)\n\n    rads_per_image = df.groupby(\"image_id\")[\"rad_id\"].nunique()\n    if int(rads_per_image.min()) != n_radiologists or int(rads_per_image.max()) != n_radiologists:\n        print(\n            \"WARNING: số bác sĩ/ảnh không đồng nhất với R=3. \"\n            f\"min={rads_per_image.min()}, max={rads_per_image.max()}\"\n        )\n\n    unique_soft_values = sorted(np.unique(label_matrix).round(6).tolist())\n    print(f\"CSV loaded: {len(df):,} rows, {len(all_image_ids):,} unique images\")\n    print(f\"Soft label values observed: {unique_soft_values}\")\n    return df, all_image_ids, label_matrix\n\ndef split_train_val_calib(image_ids, label_matrix):\n    y_bin = (label_matrix >= CONFIG[\"positive_threshold\"]).astype(np.int32)\n    dummy_x = np.zeros((len(image_ids), 1), dtype=np.float32)\n\n    first_split = MultilabelStratifiedShuffleSplit(\n        n_splits=1,\n        test_size=CONFIG[\"val_select_ratio\"] + CONFIG[\"calib_ratio\"],\n        random_state=CONFIG[\"split_seed\"],\n    )\n    train_idx, temp_idx = next(first_split.split(dummy_x, y_bin))\n\n    temp_y = y_bin[temp_idx]\n    temp_x = np.zeros((len(temp_idx), 1), dtype=np.float32)\n    relative_calib_size = CONFIG[\"calib_ratio\"] / (CONFIG[\"val_select_ratio\"] + CONFIG[\"calib_ratio\"])\n    second_split = MultilabelStratifiedShuffleSplit(\n        n_splits=1,\n        test_size=relative_calib_size,\n        random_state=CONFIG[\"split_seed\"] + 1,\n    )\n    val_rel_idx, calib_rel_idx = next(second_split.split(temp_x, temp_y))\n    val_idx = temp_idx[val_rel_idx]\n    calib_idx = temp_idx[calib_rel_idx]\n\n    splits = {\n        \"train_core\": {\n            \"ids\": [image_ids[i] for i in train_idx],\n            \"labels\": label_matrix[train_idx],\n            \"indices\": train_idx.tolist(),\n        },\n        \"val_select\": {\n            \"ids\": [image_ids[i] for i in val_idx],\n            \"labels\": label_matrix[val_idx],\n            \"indices\": val_idx.tolist(),\n        },\n        \"calib\": {\n            \"ids\": [image_ids[i] for i in calib_idx],\n            \"labels\": label_matrix[calib_idx],\n            \"indices\": calib_idx.tolist(),\n        },\n        \"holdout_eval\": {\n            \"ids\": [],\n            \"labels\": np.zeros((0, CONFIG[\"num_classes\"]), dtype=np.float32),\n            \"indices\": [],\n            \"status\": CONFIG[\"holdout_eval_policy\"],\n        },\n    }\n    return splits\n\ndef split_report(splits, class_names):\n    rows = []\n    for split_name in [\"train_core\", \"val_select\", \"calib\"]:\n        labels = splits[split_name][\"labels\"]\n        y_bin = (labels >= CONFIG[\"positive_threshold\"]).astype(np.int32)\n        for idx, class_name in enumerate(class_names):\n            positives = int(y_bin[:, idx].sum())\n            negatives = int(y_bin.shape[0] - positives)\n            rows.append(\n                {\n                    \"split\": split_name,\n                    \"class_name\": class_name,\n                    \"positive\": positives,\n                    \"negative\": negatives,\n                    \"n\": int(y_bin.shape[0]),\n                }\n            )\n    report = pd.DataFrame(rows)\n    bad = report[(report[\"split\"].isin([\"val_select\", \"calib\"])) & (report[\"positive\"] == 0)]\n    if len(bad) > 0:\n        raise ValueError(\n            \"Split không hợp lệ: có lớp 0 positive trong val_select hoặc calib. \"\n            + bad[[\"split\", \"class_name\"]].to_string(index=False)\n        )\n    return report\n\ntrain_df, IMAGE_IDS, LABEL_MATRIX = load_and_process_csv(\n    TRAIN_CSV,\n    num_classes=CONFIG[\"num_classes\"],\n    n_radiologists=CONFIG[\"n_radiologists\"],\n)\n\nSPLITS = split_train_val_calib(IMAGE_IDS, LABEL_MATRIX)\n\nSPLIT_REPORT = split_report(SPLITS, CLASS_NAMES)\n\nSPLIT_HASH = stable_hash(\n    {\n        name: {\n            \"ids\": SPLITS[name][\"ids\"],\n            \"label_shape\": list(SPLITS[name][\"labels\"].shape),\n        }\n        for name in [\"train_core\", \"val_select\", \"calib\", \"holdout_eval\"]\n    }\n)\n\nsplit_manifest = {\n    \"split_hash\": SPLIT_HASH,\n    \"method\": \"MultilabelStratifiedShuffleSplit on p_soft >= 0.5\",\n    \"ratios\": {\n        \"train_core\": CONFIG[\"train_core_ratio\"],\n        \"val_select\": CONFIG[\"val_select_ratio\"],\n        \"calib\": CONFIG[\"calib_ratio\"],\n        \"holdout_eval\": CONFIG[\"holdout_eval_policy\"],\n    },\n    \"positive_threshold\": CONFIG[\"positive_threshold\"],\n    \"class_names\": CLASS_NAMES,\n    \"counts\": {\n        split_name: {\n            \"n\": len(SPLITS[split_name][\"ids\"]),\n            \"positive_per_class\": (SPLITS[split_name][\"labels\"] >= CONFIG[\"positive_threshold\"]).sum(axis=0).astype(int).tolist(),\n        }\n        for split_name in [\"train_core\", \"val_select\", \"calib\"]\n    },\n    \"report\": SPLIT_REPORT.to_dict(orient=\"records\"),\n    \"data_leakage_note\": \"VinDr-CXR Kaggle train không có patient_id; không thể tuyên bố patient-level split.\",\n}\n\nwrite_json(PATHS[\"split_manifest\"], split_manifest)\n\nprint(SPLIT_REPORT.pivot(index=\"class_name\", columns=\"split\", values=\"positive\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.068495Z","iopub.status.idle":"2026-05-20T15:49:26.068862Z","shell.execute_reply.started":"2026-05-20T15:49:26.06872Z","shell.execute_reply":"2026-05-20T15:49:26.068738Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. DICOM Preprocessing and PNG Cache\n","metadata":{}},{"cell_type":"code","source":"def resolve_dicom_path(image_id, dicom_dir):\n    for ext in (\".dicom\", \".dcm\"):\n        candidate = Path(dicom_dir) / f\"{image_id}{ext}\"\n        if candidate.exists():\n            return candidate\n    return None\n\n\ndef cache_numpy_dtype():\n    dtype = str(CONFIG.get(\"preprocessed_cache_dtype\", \"uint16\")).lower()\n    if dtype == \"uint16\":\n        return np.uint16\n    if dtype == \"uint8\":\n        return np.uint8\n    raise ValueError(f\"Unsupported preprocessed_cache_dtype: {dtype}\")\n\n\ndef load_cxr_dicom(dicom_path, target_size):\n    ds = pydicom.dcmread(str(dicom_path))\n    pixel = ds.pixel_array\n    mode = CONFIG.get(\"preprocessing_mode\", \"modality_voi_uint16\")\n    if mode.startswith(\"modality_\"):\n        pixel = apply_modality_lut(pixel, ds)\n    pixel = apply_voi_lut(pixel, ds).astype(np.float32)\n\n    if getattr(ds, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n        pixel = float(np.max(pixel)) - pixel\n\n    pmin, pmax = float(np.nanmin(pixel)), float(np.nanmax(pixel))\n    if np.isfinite(pmin) and np.isfinite(pmax) and pmax > pmin:\n        pixel = (pixel - pmin) / (pmax - pmin)\n    else:\n        pixel = np.zeros_like(pixel, dtype=np.float32)\n\n    pixel = cv2.resize(pixel.astype(np.float32), (target_size, target_size), interpolation=cv2.INTER_LANCZOS4)\n    max_value = float(CONFIG[\"preprocessed_cache_max_value\"])\n    return np.rint(np.clip(pixel, 0.0, 1.0) * max_value).astype(cache_numpy_dtype())\n\nclass CacheDiskBudgetError(RuntimeError):\n    pass\n\n\nCACHE_WRITE_DISABLED = False\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.07027Z","iopub.status.idle":"2026-05-20T15:49:26.070589Z","shell.execute_reply.started":"2026-05-20T15:49:26.070447Z","shell.execute_reply":"2026-05-20T15:49:26.070471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def safe_cache_stem(image_id):\n    return re.sub(r\"[^A-Za-z0-9_.-]+\", \"_\", str(image_id))\n\ndef cache_dir_name(target_size):\n    return preprocessed_cache_dir_name(\n        target_size,\n        CONFIG[\"preprocessed_cache_format\"],\n        CONFIG[\"preprocessed_cache_version\"],\n    )\n\ndef normalize_cache_root(root, target_size):\n    root = Path(root)\n    dirname = cache_dir_name(target_size)\n    return root if root.name == dirname else root / dirname\n\ndef writable_cache_dir(target_size):\n    return normalize_cache_root(CONFIG[\"preprocessed_cache_root\"], target_size)\n\ndef candidate_cache_dirs(target_size):\n    if not CONFIG.get(\"preprocessed_cache_enabled\", False):\n        return []\n    dirs = []\n    for root in CONFIG.get(\"preprocessed_cache_read_roots\", []):\n        cache_dir = normalize_cache_root(root, target_size)\n        if cache_dir.exists():\n            dirs.append(cache_dir)\n    write_dir = writable_cache_dir(target_size)\n    if write_dir.exists():\n        dirs.append(write_dir)\n    unique_dirs = []\n    seen = set()\n    for cache_dir in dirs:\n        resolved = str(cache_dir)\n        if resolved not in seen:\n            unique_dirs.append(cache_dir)\n            seen.add(resolved)\n    return unique_dirs\n\ndef preprocessed_cache_path(image_id, target_size, root=None):\n    if not CONFIG.get(\"preprocessed_cache_enabled\", False):\n        return None\n    cache_dir = writable_cache_dir(target_size) if root is None else Path(root)\n    return cache_dir / f\"{safe_cache_stem(image_id)}.{CONFIG['preprocessed_cache_format']}\"\n\ndef cache_disk_status(cache_dir):\n    cache_dir = Path(cache_dir)\n    probe_dir = cache_dir if cache_dir.exists() else cache_dir.parent\n    while not probe_dir.exists() and probe_dir != probe_dir.parent:\n        probe_dir = probe_dir.parent\n    usage = shutil.disk_usage(probe_dir)\n    return {\n        \"path\": str(probe_dir),\n        \"total_gb\": round(usage.total / (1024 ** 3), 3),\n        \"used_gb\": round(usage.used / (1024 ** 3), 3),\n        \"free_gb\": round(usage.free / (1024 ** 3), 3),\n        \"min_free_gb\": float(CONFIG.get(\"preprocessed_cache_min_free_gb\", 0.0)),\n    }\n\ndef ensure_cache_write_budget(cache_dir):\n    status = cache_disk_status(cache_dir)\n    min_free_gb = float(CONFIG.get(\"preprocessed_cache_min_free_gb\", 0.0))\n    if min_free_gb > 0 and status[\"free_gb\"] < min_free_gb:\n        raise CacheDiskBudgetError(\n            f\"Không ghi cache vì free disk {status['free_gb']:.2f} GiB < \"\n            f\"preprocessed_cache_min_free_gb={min_free_gb:.2f} GiB.\"\n        )\n    return status\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.072077Z","iopub.status.idle":"2026-05-20T15:49:26.07308Z","shell.execute_reply.started":"2026-05-20T15:49:26.072902Z","shell.execute_reply":"2026-05-20T15:49:26.072929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_preprocessed_cache_file(image_id, target_size):\n    expected_dtype = cache_numpy_dtype()\n    for cache_dir in candidate_cache_dirs(target_size):\n        cache_path = preprocessed_cache_path(image_id, target_size, root=cache_dir)\n        if cache_path is None or not cache_path.exists():\n            continue\n        image = cv2.imread(str(cache_path), cv2.IMREAD_UNCHANGED)\n        if image is None:\n            continue\n        if image.ndim == 3:\n            image = image[..., 0]\n        if image.shape != (int(target_size), int(target_size)):\n            continue\n        if image.dtype != expected_dtype:\n            print(f\"WARNING: bỏ qua stale cache sai dtype {image.dtype} tại {cache_path}; expected={expected_dtype}.\")\n            continue\n        return cache_path, image\n    return None, None\n\n\ndef read_preprocessed_cache(image_id, target_size):\n    _, image = find_preprocessed_cache_file(image_id, target_size)\n    return image\n\n\ndef write_preprocessed_cache(image_id, image, target_size):\n    global CACHE_WRITE_DISABLED\n    if CACHE_WRITE_DISABLED:\n        return None\n    if image.shape != (int(target_size), int(target_size)):\n        raise ValueError(f\"Cache image shape mismatch for {image_id}: {image.shape}\")\n    expected_dtype = cache_numpy_dtype()\n    if image.dtype != expected_dtype:\n        raise TypeError(f\"Cache image dtype mismatch for {image_id}: {image.dtype} != {expected_dtype}\")\n    cache_dir = writable_cache_dir(target_size)\n    cache_dir.mkdir(parents=True, exist_ok=True)\n    try:\n        ensure_cache_write_budget(cache_dir)\n    except CacheDiskBudgetError:\n        CACHE_WRITE_DISABLED = True\n        raise\n    cache_path = preprocessed_cache_path(image_id, target_size, root=cache_dir)\n    tmp_path = cache_path.with_name(f\"{cache_path.stem}.{os.getpid()}.{time.time_ns()}.tmp{cache_path.suffix}\")\n    params = [int(cv2.IMWRITE_PNG_COMPRESSION), int(CONFIG[\"preprocessed_cache_png_compression\"])]\n    ok = cv2.imwrite(str(tmp_path), image, params)\n    if not ok:\n        if tmp_path.exists():\n            tmp_path.unlink()\n        raise IOError(f\"Không ghi được preprocessed cache: {tmp_path}\")\n    tmp_path.replace(cache_path)\n    return cache_path\n\n\ndef load_cxr_image(image_id, dicom_dir, target_size):\n    cached = read_preprocessed_cache(image_id, target_size)\n    if cached is not None:\n        return cached\n    dicom_path = resolve_dicom_path(image_id, dicom_dir)\n    if dicom_path is None:\n        return None\n    image = load_cxr_dicom(dicom_path, target_size)\n    if CONFIG.get(\"preprocessed_cache_enabled\", False):\n        try:\n            write_preprocessed_cache(image_id, image, target_size)\n        except CacheDiskBudgetError as exc:\n            print(f\"WARNING: dừng ghi cache do thiếu dung lượng: {exc}\")\n        except Exception as exc:\n            print(f\"WARNING: không ghi được cache cho {image_id}: {exc}\")\n    return image\n\ndef build_one_preprocessed_cache(image_id, dicom_dir, target_size):\n    cache_path, _ = find_preprocessed_cache_file(image_id, target_size)\n    if cache_path is not None:\n        return {\"image_id\": image_id, \"status\": \"cached\", \"path\": str(cache_path)}\n    dicom_path = resolve_dicom_path(image_id, dicom_dir)\n    if dicom_path is None:\n        return {\"image_id\": image_id, \"status\": \"missing_dicom\", \"path\": None}\n    image = load_cxr_dicom(dicom_path, target_size)\n    try:\n        written_path = write_preprocessed_cache(image_id, image, target_size)\n    except CacheDiskBudgetError as exc:\n        return {\"image_id\": image_id, \"status\": \"disk_skipped\", \"path\": None, \"error\": str(exc)}\n    if written_path is None:\n        return {\"image_id\": image_id, \"status\": \"disk_skipped\", \"path\": None}\n    return {\"image_id\": image_id, \"status\": \"written\", \"path\": str(written_path)}\n\ndef prebuild_preprocessed_cache(image_ids, dicom_dir, target_size, summary_path=None):\n    if not CONFIG.get(\"preprocessed_cache_enabled\", False) or not CONFIG.get(\"prebuild_preprocessed_cache\", False):\n        summary = {\"enabled\": False, \"status\": \"skipped\"}\n        if summary_path is not None:\n            write_json(summary_path, summary)\n        return summary\n    unique_ids = list(dict.fromkeys([str(image_id) for image_id in image_ids]))\n    workers = max(1, int(CONFIG.get(\"preprocessed_cache_workers\", 4)))\n    write_dir = writable_cache_dir(target_size)\n    summary = {\n        \"enabled\": True,\n        \"cache_version\": CONFIG[\"preprocessed_cache_version\"],\n        \"target_size\": int(target_size),\n        \"format\": CONFIG[\"preprocessed_cache_format\"],\n        \"png_compression\": int(CONFIG[\"preprocessed_cache_png_compression\"]),\n        \"write_cache_dir\": str(write_dir),\n        \"read_cache_dirs\": [str(path) for path in candidate_cache_dirs(target_size)],\n        \"disk_start\": cache_disk_status(write_dir),\n        \"workers\": workers,\n        \"total\": len(unique_ids),\n        \"cached\": 0,\n        \"written\": 0,\n        \"missing_dicom\": 0,\n        \"disk_skipped\": 0,\n        \"error\": 0,\n        \"started_utc\": datetime.now(timezone.utc).isoformat(),\n        \"elapsed_sec\": None,\n        \"errors\": [],\n    }\n    if not unique_ids:\n        summary[\"elapsed_sec\"] = 0.0\n        summary[\"finished_utc\"] = datetime.now(timezone.utc).isoformat()\n        summary[\"disk_end\"] = cache_disk_status(write_dir)\n        if summary_path is not None:\n            write_json(summary_path, summary)\n        return summary\n    start = time.time()\n\n    def task(image_id):\n        try:\n            return build_one_preprocessed_cache(image_id, dicom_dir, target_size)\n        except Exception as exc:\n            return {\"image_id\": image_id, \"status\": \"error\", \"error\": f\"{type(exc).__name__}: {exc}\"}\n\n    with concurrent.futures.ThreadPoolExecutor(max_workers=min(workers, len(unique_ids))) as executor:\n        for result in tqdm(executor.map(task, unique_ids), total=len(unique_ids), desc=\"Prebuild image cache\", leave=False):\n            status = result.get(\"status\", \"error\")\n            if status in {\"cached\", \"written\", \"missing_dicom\", \"disk_skipped\"}:\n                summary[status] += 1\n            else:\n                summary[\"error\"] += 1\n                if len(summary[\"errors\"]) < 20:\n                    summary[\"errors\"].append(result)\n    summary[\"elapsed_sec\"] = round(time.time() - start, 3)\n    summary[\"finished_utc\"] = datetime.now(timezone.utc).isoformat()\n    summary[\"disk_end\"] = cache_disk_status(write_dir)\n    if summary_path is not None:\n        write_json(summary_path, summary)\n    print(\n        \"Preprocessed cache summary: \"\n        f\"cached={summary['cached']} written={summary['written']} \"\n        f\"missing={summary['missing_dicom']} disk_skipped={summary['disk_skipped']} \"\n        f\"error={summary['error']} elapsed={summary['elapsed_sec']}s \"\n        f\"free_disk={summary['disk_end']['free_gb']}GiB\"\n    )\n    if summary[\"error\"] and CONFIG.get(\"preprocessed_cache_fail_on_error\", False):\n        raise RuntimeError(f\"Preprocessed cache build failed: {summary['errors'][:3]}\")\n    return summary\n\nif DICOM_DIR.exists() and SPLITS[\"train_core\"][\"ids\"]:\n    sample_id = SPLITS[\"train_core\"][\"ids\"][0]\n    sample_path = resolve_dicom_path(sample_id, DICOM_DIR)\n    if sample_path is not None:\n        sample_img = load_cxr_dicom(sample_path, MODEL_CONFIG[\"img_size\"])\n        print(\n            f\"DICOM smoke: id={sample_id}, shape={sample_img.shape}, \"\n            f\"dtype={sample_img.dtype}, range=({sample_img.min()}, {sample_img.max()})\"\n        )\n    else:\n        print(\"DICOM smoke skipped: không tìm thấy file DICOM cho image_id đầu tiên.\")\nelse:\n    print(\"DICOM smoke skipped: DICOM_DIR chưa tồn tại trong môi trường hiện tại.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.07408Z","iopub.status.idle":"2026-05-20T15:49:26.074811Z","shell.execute_reply.started":"2026-05-20T15:49:26.074644Z","shell.execute_reply":"2026-05-20T15:49:26.07467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Augmentation and Dataset\n","metadata":{}},{"cell_type":"code","source":"def build_random_resized_crop(img_size, p=0.25):\n    params = inspect.signature(A.RandomResizedCrop.__init__).parameters\n    kwargs = {\n        \"scale\": (0.85, 1.0),\n        \"ratio\": (0.95, 1.05),\n        \"p\": p,\n    }\n    if \"size\" in params:\n        kwargs[\"size\"] = (img_size, img_size)\n    else:\n        kwargs[\"height\"] = img_size\n        kwargs[\"width\"] = img_size\n    return A.RandomResizedCrop(**kwargs)\n\ndef build_rotate(p=0.20):\n    params = inspect.signature(A.Rotate.__init__).parameters\n    kwargs = {\"limit\": 10, \"p\": p}\n    if \"border_mode\" in params:\n        kwargs[\"border_mode\"] = cv2.BORDER_CONSTANT\n    if \"fill\" in params:\n        kwargs[\"fill\"] = 0\n    elif \"value\" in params:\n        kwargs[\"value\"] = 0\n    return A.Rotate(**kwargs)\n\ndef build_gauss_noise(p=0.15):\n    params = inspect.signature(A.GaussNoise.__init__).parameters\n    if \"std_range\" in params:\n        return A.GaussNoise(std_range=(0.01, 0.04), mean_range=(0.0, 0.0), p=p)\n    if \"var_limit\" in params:\n        return A.GaussNoise(var_limit=(2.0, 10.0), p=p)\n    return A.GaussNoise(p=p)\n\n\ndef build_clahe(p=0.15):\n    params = inspect.signature(A.CLAHE.__init__).parameters\n    kwargs = {\"tile_grid_size\": (8, 8), \"p\": p}\n    if \"clip_limit\" in params:\n        kwargs[\"clip_limit\"] = 2.0\n    elif \"clip_range\" in params:\n        kwargs[\"clip_range\"] = (1.0, 2.0)\n    return A.CLAHE(**kwargs)\n\n\ndef build_transforms(img_size):\n    imagenet_mean = (0.485, 0.456, 0.406)\n    imagenet_std = (0.229, 0.224, 0.225)\n    pixel_to_float = A.ToFloat(max_value=float(CONFIG[\"preprocessed_cache_max_value\"]))\n    train_transform = A.Compose(\n        [\n            pixel_to_float,\n            A.HorizontalFlip(p=0.50),\n            build_rotate(p=0.20),\n            build_random_resized_crop(img_size, p=0.25),\n            build_clahe(p=0.15),\n            A.RandomBrightnessContrast(brightness_limit=0.10, contrast_limit=0.10, p=0.20),\n            build_gauss_noise(p=0.10),\n            A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=1.0),\n        ]\n    )\n    eval_transform = A.Compose(\n        [\n            pixel_to_float,\n            A.Normalize(mean=imagenet_mean, std=imagenet_std, max_pixel_value=1.0),\n        ]\n    )\n    return train_transform, eval_transform\n\nclass VinDrCXRDataset(Dataset):\n    def __init__(self, image_ids, labels, dicom_dir, transform, img_size, missing_threshold=0.01):\n        self.image_ids = list(image_ids)\n        self.labels = labels.astype(np.float32)\n        self.dicom_dir = Path(dicom_dir)\n        self.transform = transform\n        self.img_size = int(img_size)\n        self.missing_threshold = float(missing_threshold)\n        self.missing_count = 0\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image = load_cxr_image(image_id, self.dicom_dir, self.img_size)\n        if image is None:\n            self.missing_count += 1\n            if self.missing_count / max(1, len(self.image_ids)) > self.missing_threshold:\n                raise FileNotFoundError(\n                    f\"Quá nhiều DICOM bị thiếu trong {self.dicom_dir}; ví dụ thiếu image_id={image_id}\"\n                )\n            image = np.zeros((self.img_size, self.img_size), dtype=cache_numpy_dtype())\n\n        image = np.stack([image, image, image], axis=-1)\n        if self.transform is not None:\n            image = self.transform(image=image)[\"image\"]\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n        target = self.labels[idx]\n        return {\n            \"image\": torch.from_numpy(image),\n            \"target\": torch.from_numpy(target),\n            \"image_id\": image_id,\n        }\n\n\ntrain_transform, eval_transform = build_transforms(MODEL_CONFIG[\"img_size\"])\nprint(f\"Transforms built with img_size={MODEL_CONFIG['img_size']} dtype={CONFIG['preprocessed_cache_dtype']}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.075696Z","iopub.status.idle":"2026-05-20T15:49:26.076558Z","shell.execute_reply.started":"2026-05-20T15:49:26.076404Z","shell.execute_reply":"2026-05-20T15:49:26.076429Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Loss and Class Weights\n","metadata":{}},{"cell_type":"code","source":"class QualityFocalLoss(nn.Module):\n    def __init__(self, beta=2.0, epsilon=1e-6, class_weights=None):\n        super().__init__()\n        self.beta = float(beta)\n        self.epsilon = float(epsilon)\n        if class_weights is not None:\n            self.register_buffer(\"class_weights\", class_weights.float())\n        else:\n            self.class_weights = None\n\n    @staticmethod\n    def compute_class_weights(label_matrix, epsilon=1e-6, max_weight=5.0):\n        positives = (label_matrix > 0).sum(axis=0).astype(np.float32)\n        positives = np.maximum(positives, 1.0)\n        max_positive = float(np.max(positives))\n        weights = np.sqrt(max_positive / positives)\n        weights = np.clip(weights, 1.0, float(max_weight))\n        weights = weights / weights.mean()\n        return torch.from_numpy(weights.astype(np.float32))\n\n    def forward(self, logits, targets):\n        logits = logits.float()\n        targets = targets.float()\n        probs = torch.sigmoid(logits).clamp(self.epsilon, 1.0 - self.epsilon)\n        focal_weight = torch.abs(targets - probs).pow(self.beta)\n        bce = F.binary_cross_entropy_with_logits(logits, targets, reduction=\"none\")\n        loss = focal_weight * bce\n        if self.class_weights is not None:\n            weights = self.class_weights.to(device=loss.device, dtype=loss.dtype).view(1, -1)\n            loss = loss * weights\n        return loss.mean()\n\ndef compute_bce_pos_weight(label_matrix, positive_threshold=0.5, epsilon=1.0, max_weight=20.0):\n    targets_binary = (label_matrix >= positive_threshold).astype(np.float32)\n    positives = targets_binary.sum(axis=0)\n    negatives = targets_binary.shape[0] - positives\n    pos_weight = (negatives + epsilon) / (positives + epsilon)\n    pos_weight = np.clip(pos_weight, 0.1, float(max_weight))\n    return torch.from_numpy(pos_weight.astype(np.float32))\n\ndef make_criterion(class_weights, device):\n    if CONFIG[\"loss_name\"] == \"qfl\":\n        return QualityFocalLoss(\n            beta=CONFIG[\"qfl_beta\"],\n            epsilon=CONFIG[\"qfl_epsilon\"],\n            class_weights=class_weights[\"qfl_alpha\"].to(device),\n        )\n    if CONFIG[\"loss_name\"] == \"bce\":\n        return nn.BCEWithLogitsLoss(pos_weight=class_weights[\"bce_pos_weight\"].to(device))\n    raise ValueError(f\"Unsupported loss_name={CONFIG['loss_name']}\")\n\ndef compute_and_save_class_weights(train_labels):\n    qfl_alpha = QualityFocalLoss.compute_class_weights(\n        train_labels,\n        epsilon=CONFIG[\"qfl_epsilon\"],\n        max_weight=CONFIG[\"alpha_max\"],\n    )\n    bce_pos_weight = compute_bce_pos_weight(\n        train_labels,\n        positive_threshold=CONFIG[\"positive_threshold\"],\n        epsilon=1.0,\n        max_weight=CONFIG[\"bce_pos_weight_max\"],\n    )\n    payload = {\n        \"qfl_alpha\": {\n            \"formula\": \"alpha_c = clip(sqrt(N_max / N_c), 1.0, 5.0), normalized by mean\",\n            \"positive_definition\": \"p_soft > 0\",\n            \"weights\": qfl_alpha.numpy().tolist(),\n            \"positive_counts\": (train_labels > 0).sum(axis=0).astype(int).tolist(),\n        },\n        \"bce_pos_weight\": {\n            \"formula\": \"pos_weight_c = clip((N_negative + 1) / (N_positive + 1), 0.1, bce_pos_weight_max)\",\n            \"positive_definition\": \"p_soft >= 0.5\",\n            \"weights\": bce_pos_weight.numpy().tolist(),\n            \"positive_counts\": (train_labels >= CONFIG[\"positive_threshold\"]).sum(axis=0).astype(int).tolist(),\n        },\n        \"class_names\": CLASS_NAMES,\n    }\n    write_json(PATHS[\"class_weights\"], payload)\n    return {\"qfl_alpha\": qfl_alpha, \"bce_pos_weight\": bce_pos_weight}\n\nCLASS_WEIGHTS = compute_and_save_class_weights(SPLITS[\"train_core\"][\"labels\"])\n\nprint(\"QFL alpha:\", CLASS_WEIGHTS[\"qfl_alpha\"].numpy().round(3).tolist())\n\nprint(\"BCE pos_weight:\", CLASS_WEIGHTS[\"bce_pos_weight\"].numpy().round(3).tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.07821Z","iopub.status.idle":"2026-05-20T15:49:26.07865Z","shell.execute_reply.started":"2026-05-20T15:49:26.07844Z","shell.execute_reply":"2026-05-20T15:49:26.078466Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 10. Model Factory\n","metadata":{}},{"cell_type":"code","source":"TIMM_RESOLVER_VERSION = \"stage1_timm_resolver_v2_pretrained_candidates_2026_05_14\"\n\nprint(f\"Timm resolver version: {TIMM_RESOLVER_VERSION}\")\n\ndef list_timm_models_compat(pretrained=False):\n    try:\n        return set(timm.list_models(pretrained=pretrained, include_tags=bool(pretrained)))\n    except TypeError:\n        return set(timm.list_models(pretrained=pretrained))\n\nALL_TIMM_MODELS = list_timm_models_compat(pretrained=False)\n\nPRETRAINED_TIMM_MODELS = list_timm_models_compat(pretrained=True)\n\nprint(f\"timm registry: available={len(ALL_TIMM_MODELS)}, pretrained={len(PRETRAINED_TIMM_MODELS)}\")\n\ndef timm_candidate_base_name(candidate):\n    name = str(candidate)\n    if name.startswith(\"hf_hub:\"):\n        name = name.rsplit(\"/\", 1)[-1]\n    return name.split(\".\", 1)[0]\n\ndef candidate_pool_for_model(model_config, require_pretrained):\n    if require_pretrained:\n        return list(model_config.get(\"pretrained_timm_candidates\") or model_config.get(\"timm_candidates\", []))\n    return list(model_config.get(\"timm_candidates\", []))\n\ndef closest_timm_matches(candidates, require_pretrained, limit=12):\n    pool = PRETRAINED_TIMM_MODELS if require_pretrained else ALL_TIMM_MODELS\n    matches = []\n    seen = set()\n    for candidate in candidates:\n        base = timm_candidate_base_name(candidate)\n        for model_name in sorted(pool):\n            if model_name == candidate or model_name.startswith(base + \".\") or model_name.startswith(base + \"_\") or model_name == base:\n                if model_name not in seen:\n                    seen.add(model_name)\n                    matches.append(model_name)\n            if len(matches) >= limit:\n                return matches\n    return matches\n\ndef resolve_timm_model_name(model_config, require_pretrained=True):\n    candidates = candidate_pool_for_model(model_config, require_pretrained=require_pretrained)\n    if not candidates:\n        raise ValueError(f\"Không có timm candidates cho {model_config['display_name']}.\")\n\n    if require_pretrained:\n        for candidate in candidates:\n            if candidate.startswith(\"hf_hub:\"):\n                return candidate\n            if candidate in PRETRAINED_TIMM_MODELS:\n                return candidate\n        pool_name = \"pretrained timm models\"\n        closest = closest_timm_matches(candidates, require_pretrained=True)\n    else:\n        for candidate in candidates:\n            if candidate in ALL_TIMM_MODELS:\n                return candidate\n            base = timm_candidate_base_name(candidate)\n            if base in ALL_TIMM_MODELS:\n                return base\n        pool_name = \"available timm models\"\n        closest = closest_timm_matches(candidates, require_pretrained=False)\n\n    raise ValueError(\n        f\"Không resolve được {pool_name} cho {model_config['display_name']}. \"\n        f\"Candidates={candidates}. \"\n        f\"Closest available matches={closest}. \"\n        f\"timm_version={PACKAGE_VERSIONS.get('timm', getattr(timm, '__version__', 'unknown'))}\"\n    )\n\ndef create_timm_backbone(timm_name, model_config, pretrained=True):\n    kwargs = {\n        \"pretrained\": pretrained,\n        \"num_classes\": 0,\n    }\n    kwargs.update(model_config.get(\"timm_kwargs\", {}))\n    strict_img_size = bool(model_config.get(\"family\") == \"vit\" or kwargs.get(\"img_size\") == model_config.get(\"img_size\"))\n    try:\n        return timm.create_model(timm_name, **kwargs)\n    except TypeError as first_error:\n        if strict_img_size and \"img_size\" in kwargs:\n            raise TypeError(\n                f\"{model_config['display_name']} phải giữ img_size={kwargs['img_size']} theo spec. \"\n                \"Không được fallback bằng cách bỏ img_size vì sẽ phá invariant DINOv2/ViT. \"\n                \"Hãy dùng phiên bản timm hỗ trợ img_size cho backbone này hoặc cấu hình dynamic_img_pad rõ ràng.\"\n            ) from first_error\n        removable = [\"dynamic_img_pad\", \"dynamic_img_size\"]\n        if not strict_img_size:\n            removable.append(\"img_size\")\n        for key in removable:\n            if key in kwargs:\n                retry_kwargs = dict(kwargs)\n                retry_kwargs.pop(key, None)\n                try:\n                    print(f\"Retry timm.create_model without unsupported non-strict kwarg {key}\")\n                    return timm.create_model(timm_name, **retry_kwargs)\n                except TypeError:\n                    pass\n        raise first_error\n\nclass VinDrCXRModel(nn.Module):\n    def __init__(self, model_config, num_classes=14, pretrained=True):\n        super().__init__()\n        self.model_config = dict(model_config)\n        self.timm_name = resolve_timm_model_name(model_config, require_pretrained=pretrained)\n        self.backbone = create_timm_backbone(self.timm_name, model_config, pretrained=pretrained)\n        in_features = getattr(self.backbone, \"num_features\", None)\n        if in_features is None:\n            raise AttributeError(f\"Backbone {self.timm_name} không có num_features sau num_classes=0.\")\n        self.head = nn.Sequential(\n            nn.Dropout(p=0.30),\n            nn.Linear(in_features, 512),\n            nn.GELU(),\n            nn.Dropout(p=0.20),\n            nn.Linear(512, num_classes),\n        )\n        if model_config.get(\"grad_checkpoint\", False) and hasattr(self.backbone, \"set_grad_checkpointing\"):\n            self.backbone.set_grad_checkpointing(enable=True)\n\n    def disable_backbone_checkpointing(self):\n        if hasattr(self.backbone, \"set_grad_checkpointing\"):\n            self.backbone.set_grad_checkpointing(enable=False)\n\n    def forward(self, x):\n        features = self.backbone(x)\n        return self.head(features)\n\nresolved_name = resolve_timm_model_name(MODEL_CONFIG, require_pretrained=False)\n\nresolved_pretrained_name = resolve_timm_model_name(MODEL_CONFIG, require_pretrained=True)\n\nCONFIG[\"timm_resolver_version\"] = TIMM_RESOLVER_VERSION\n\nCONFIG[\"resolved_timm_name_untrained\"] = resolved_name\n\nCONFIG[\"resolved_timm_name_pretrained\"] = resolved_pretrained_name\n\nwrite_json(PATHS[\"config\"], CONFIG)\n\nprint(f\"Resolved selected model for smoke: {CONFIG['run_model_name']} -> {resolved_name}\")\n\nprint(f\"Resolved selected pretrained model: {CONFIG['run_model_name']} -> {resolved_pretrained_name}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.080323Z","iopub.status.idle":"2026-05-20T15:49:26.080688Z","shell.execute_reply.started":"2026-05-20T15:49:26.080515Z","shell.execute_reply":"2026-05-20T15:49:26.080535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 11. Metrics\n","metadata":{}},{"cell_type":"code","source":"def binarize_targets(targets_soft, positive_threshold=None):\n    threshold = CONFIG[\"positive_threshold\"] if positive_threshold is None else positive_threshold\n    return (targets_soft >= threshold).astype(np.int32)\n\ndef sigmoid_numpy(logits):\n    return 1.0 / (1.0 + np.exp(-np.clip(logits, -50, 50)))\n\ndef apply_temperature_numpy(logits, temperature):\n    return logits / max(float(temperature), 1e-6)\n\ndef compute_ece(probs, targets_binary, n_bins=15):\n    num_samples, num_classes = probs.shape\n    bin_bounds = np.linspace(0.0, 1.0, n_bins + 1)\n    values = []\n    for class_idx in range(num_classes):\n        y_true = targets_binary[:, class_idx]\n        if len(np.unique(y_true)) < 2:\n            values.append(np.nan)\n            continue\n        class_ece = 0.0\n        for bin_idx in range(n_bins):\n            left = bin_bounds[bin_idx]\n            right = bin_bounds[bin_idx + 1]\n            if bin_idx == 0:\n                in_bin = (probs[:, class_idx] >= left) & (probs[:, class_idx] <= right)\n            else:\n                in_bin = (probs[:, class_idx] > left) & (probs[:, class_idx] <= right)\n            if in_bin.any():\n                avg_conf = probs[in_bin, class_idx].mean()\n                avg_acc = y_true[in_bin].mean()\n                class_ece += abs(avg_conf - avg_acc) * (in_bin.sum() / num_samples)\n        values.append(float(class_ece))\n    return float(np.nanmean(values)), values\n\ndef compute_metrics_from_probs(\n    probs,\n    targets_soft,\n    class_names,\n    thresholds=None,\n    positive_threshold=None,\n    min_eval_positive=1,\n):\n    targets_binary = binarize_targets(targets_soft, positive_threshold)\n    if thresholds is None:\n        thresholds = np.full(probs.shape[1], 0.5, dtype=np.float32)\n    pred_binary = (probs >= thresholds.reshape(1, -1)).astype(np.int32)\n\n    per_class = []\n    for class_idx, class_name in enumerate(class_names):\n        y_true = targets_binary[:, class_idx]\n        y_prob = probs[:, class_idx]\n        y_pred = pred_binary[:, class_idx]\n        positives = int(y_true.sum())\n        negatives = int(len(y_true) - positives)\n\n        ap = np.nan\n        auroc = np.nan\n        f1 = np.nan\n        if positives >= min_eval_positive and negatives > 0:\n            ap = float(average_precision_score(y_true, y_prob))\n            auroc = float(roc_auc_score(y_true, y_prob))\n            f1 = float(f1_score(y_true, y_pred, zero_division=0))\n        brier_binary = float(brier_score_loss(y_true, y_prob))\n        brier_soft = float(np.mean((targets_soft[:, class_idx] - y_prob) ** 2))\n        per_class.append(\n            {\n                \"class_index\": class_idx,\n                \"class_name\": class_name,\n                \"AP\": ap,\n                \"AUROC\": auroc,\n                \"F1\": f1,\n                \"Brier\": brier_binary,\n                \"Brier_soft\": brier_soft,\n                \"positive\": positives,\n                \"negative\": negatives,\n                \"threshold\": float(thresholds[class_idx]),\n            }\n        )\n\n    ece_macro, ece_per_class = compute_ece(probs, targets_binary)\n    for row, class_ece in zip(per_class, ece_per_class):\n        row[\"ECE\"] = class_ece\n\n    metrics = {\n        \"macro_mAP\": float(np.nanmean([r[\"AP\"] for r in per_class])),\n        \"macro_AUROC\": float(np.nanmean([r[\"AUROC\"] for r in per_class])),\n        \"macro_F1\": float(np.nanmean([r[\"F1\"] for r in per_class])),\n        \"ECE\": ece_macro,\n        \"Brier\": float(np.nanmean([r[\"Brier\"] for r in per_class])),\n        \"Brier_soft\": float(np.nanmean([r[\"Brier_soft\"] for r in per_class])),\n        \"per_class\": per_class,\n        \"nan_AP_classes\": int(np.sum([np.isnan(r[\"AP\"]) for r in per_class])),\n        \"nan_AUROC_classes\": int(np.sum([np.isnan(r[\"AUROC\"]) for r in per_class])),\n    }\n    return metrics\n\ndef build_prediction_dataframe(image_ids, split_name, targets_soft, logits, temperature, thresholds, class_names):\n    logits_np = np.asarray(logits, dtype=np.float32)\n    targets_np = np.asarray(targets_soft, dtype=np.float32)\n    probs_raw = sigmoid_numpy(logits_np)\n    probs_cal = sigmoid_numpy(apply_temperature_numpy(logits_np, temperature))\n    pred_binary = (probs_cal >= thresholds.reshape(1, -1)).astype(np.int32)\n    target_binary = binarize_targets(targets_np)\n\n    rows = []\n    for row_idx, image_id in enumerate(image_ids):\n        row = {\"image_id\": image_id, \"split\": split_name}\n        for class_idx, class_name in enumerate(class_names):\n            slug = slugify_class_name(class_name)\n            row[f\"{slug}_target_soft\"] = float(targets_np[row_idx, class_idx])\n            row[f\"{slug}_target_binary\"] = int(target_binary[row_idx, class_idx])\n            row[f\"{slug}_logit\"] = float(logits_np[row_idx, class_idx])\n            row[f\"{slug}_prob_raw\"] = float(probs_raw[row_idx, class_idx])\n            row[f\"{slug}_prob_cal\"] = float(probs_cal[row_idx, class_idx])\n            row[f\"{slug}_pred_binary\"] = int(pred_binary[row_idx, class_idx])\n        rows.append(row)\n    return pd.DataFrame(rows)\n\ndef per_class_metrics_dataframe(metrics, split_name):\n    rows = []\n    for row in metrics[\"per_class\"]:\n        out = dict(row)\n        out[\"split\"] = split_name\n        rows.append(out)\n    return pd.DataFrame(rows)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.081465Z","iopub.status.idle":"2026-05-20T15:49:26.081843Z","shell.execute_reply.started":"2026-05-20T15:49:26.08165Z","shell.execute_reply":"2026-05-20T15:49:26.081676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 12. Calibration\n","metadata":{}},{"cell_type":"code","source":"class TemperatureScaler(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.log_temperature = nn.Parameter(torch.zeros(1))\n\n    def temperature(self):\n        return self.log_temperature.exp().clamp(1e-2, 20.0)\n\n    def forward(self, logits):\n        return logits / self.temperature()\n\ndef fit_temperature_scaler(calib_logits, calib_targets_soft, device, max_iter=50):\n    scaler = TemperatureScaler().to(device)\n    logits = torch.as_tensor(calib_logits, dtype=torch.float32, device=device)\n    targets = (torch.as_tensor(calib_targets_soft, dtype=torch.float32, device=device) >= CONFIG[\"positive_threshold\"]).float()\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = torch.optim.LBFGS(\n        scaler.parameters(),\n        lr=0.1,\n        max_iter=max_iter,\n        line_search_fn=\"strong_wolfe\",\n    )\n\n    def closure():\n        optimizer.zero_grad(set_to_none=True)\n        loss = criterion(scaler(logits), targets)\n        loss.backward()\n        return loss\n\n    optimizer.step(closure)\n    with torch.no_grad():\n        final_loss = criterion(scaler(logits), targets).item()\n        temperature = scaler.temperature().item()\n    return float(temperature), float(final_loss)\n\ndef tune_per_class_thresholds(probs_cal, targets_soft, threshold_grid, positive_threshold=0.5, min_eval_positive=1):\n    targets_binary = (targets_soft >= positive_threshold).astype(np.int32)\n    thresholds = np.full(probs_cal.shape[1], 0.5, dtype=np.float32)\n    for class_idx in range(probs_cal.shape[1]):\n        y_true = targets_binary[:, class_idx]\n        positives = int(y_true.sum())\n        negatives = int(len(y_true) - positives)\n        if positives < min_eval_positive or negatives == 0:\n            thresholds[class_idx] = 0.5\n            continue\n        best_threshold = 0.5\n        best_f1 = -1.0\n        for threshold in threshold_grid:\n            y_pred = (probs_cal[:, class_idx] >= threshold).astype(np.int32)\n            score = f1_score(y_true, y_pred, zero_division=0)\n            if score > best_f1:\n                best_f1 = float(score)\n                best_threshold = float(threshold)\n        thresholds[class_idx] = best_threshold\n    return thresholds\n\ndef confidence_from_margin(probs_cal, thresholds):\n    margin = np.abs(probs_cal - thresholds.reshape(1, -1))\n    denom = np.maximum(thresholds.reshape(1, -1), 1.0 - thresholds.reshape(1, -1))\n    return np.clip(margin / np.maximum(denom, 1e-6), 0.0, 1.0)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.083583Z","iopub.status.idle":"2026-05-20T15:49:26.083932Z","shell.execute_reply.started":"2026-05-20T15:49:26.083784Z","shell.execute_reply":"2026-05-20T15:49:26.083808Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 13. Training Utilities\n","metadata":{}},{"cell_type":"code","source":"class AverageMeter:\n    def __init__(self, name=\"\"):\n        self.name = name\n        self.reset()\n\n    def reset(self):\n        self.val = 0.0\n        self.avg = 0.0\n        self.total = 0.0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = float(val)\n        self.total += float(val) * int(n)\n        self.count += int(n)\n        self.avg = self.total / max(1, self.count)\n\nclass EarlyStopping:\n    def __init__(self, patience=5, mode=\"max\"):\n        self.patience = int(patience)\n        self.mode = mode\n        self.best_score = None\n        self.counter = 0\n        self.should_stop = False\n\n    def step(self, score):\n        if self.best_score is None:\n            self.best_score = score\n            return True\n        improved = score > self.best_score if self.mode == \"max\" else score < self.best_score\n        if improved:\n            self.best_score = score\n            self.counter = 0\n            return True\n        self.counter += 1\n        if self.counter >= self.patience:\n            self.should_stop = True\n        return False\n\ndef get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps, min_lr_ratio=0.01):\n    def lr_lambda(current_step):\n        if current_step < warmup_steps:\n            return float(current_step) / float(max(1, warmup_steps))\n        progress = float(current_step - warmup_steps) / float(max(1, total_steps - warmup_steps))\n        return max(min_lr_ratio, 0.5 * (1.0 + math.cos(math.pi * progress)))\n    return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n\ndef make_optimizer(model, phase):\n    if phase == \"phase1\":\n        return torch.optim.AdamW(\n            [\n                {\n                    \"params\": model.backbone.parameters(),\n                    \"lr\": CONFIG[\"phase1_lr_backbone\"],\n                    \"weight_decay\": CONFIG[\"weight_decay_backbone\"],\n                },\n                {\n                    \"params\": model.head.parameters(),\n                    \"lr\": CONFIG[\"phase1_lr_head\"],\n                    \"weight_decay\": CONFIG[\"weight_decay_head\"],\n                },\n            ]\n        )\n    if phase == \"phase2\":\n        return torch.optim.AdamW(\n            [\n                {\n                    \"params\": [p for p in model.head.parameters() if p.requires_grad],\n                    \"lr\": CONFIG[\"phase2_lr_head\"],\n                    \"weight_decay\": CONFIG[\"weight_decay_head\"],\n                }\n            ]\n        )\n    raise ValueError(f\"Unknown phase={phase}\")\n\ndef create_class_balanced_sampler(labels):\n    class_freq = (labels > 0).mean(axis=0)\n    sample_weights = np.zeros(len(labels), dtype=np.float32)\n    for idx in range(len(labels)):\n        positive_classes = np.where(labels[idx] > 0)[0]\n        if len(positive_classes) > 0:\n            sample_weights[idx] = float(np.mean(1.0 / np.sqrt(class_freq[positive_classes] + 1e-6)))\n        else:\n            sample_weights[idx] = 1.0\n    sample_weights = sample_weights / sample_weights.mean()\n    return WeightedRandomSampler(\n        weights=torch.from_numpy(sample_weights).double(),\n        num_samples=len(labels),\n        replacement=True,\n    )\n\ndef train_one_epoch(model, loader, criterion, optimizer, scheduler, scaler, device, accum_steps, global_step):\n    model.train()\n    losses = AverageMeter(\"loss\")\n    optimizer.zero_grad(set_to_none=True)\n    for step, batch in enumerate(tqdm(loader, leave=False)):\n        images = move_images_to_device(batch[\"image\"], device)\n        targets = batch[\"target\"].to(device, non_blocking=True)\n        with autocast_context(device, CONFIG[\"use_amp\"]):\n            logits = model(images)\n            loss = criterion(logits, targets)\n            loss_to_backward = loss / accum_steps\n        scaler.scale(loss_to_backward).backward()\n        losses.update(loss.item(), images.size(0))\n\n        should_step = ((step + 1) % accum_steps == 0) or ((step + 1) == len(loader))\n        if should_step:\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_norm_(model.parameters(), CONFIG[\"grad_clip_norm\"])\n            scaler.step(optimizer)\n            scaler.update()\n            scheduler.step()\n            optimizer.zero_grad(set_to_none=True)\n            global_step += 1\n    return losses.avg, global_step\n\n@torch.no_grad()\ndef collect_logits_and_targets(model, loader, device):\n    model.eval()\n    logits_all, targets_all, image_ids = [], [], []\n    for batch in tqdm(loader, leave=False):\n        images = move_images_to_device(batch[\"image\"], device)\n        with autocast_context(device, CONFIG[\"use_amp\"]):\n            logits = model(images)\n        logits_all.append(logits.float().cpu())\n        targets_all.append(batch[\"target\"].float().cpu())\n        image_ids.extend(batch[\"image_id\"])\n    return torch.cat(logits_all).numpy(), torch.cat(targets_all).numpy(), image_ids\n\n@torch.no_grad()\ndef validate(model, loader, criterion, device, class_names, thresholds=None):\n    model.eval()\n    losses = AverageMeter(\"val_loss\")\n    logits_all, targets_all = [], []\n    for batch in tqdm(loader, leave=False):\n        images = move_images_to_device(batch[\"image\"], device)\n        targets = batch[\"target\"].to(device, non_blocking=True)\n        with autocast_context(device, CONFIG[\"use_amp\"]):\n            logits = model(images)\n            loss = criterion(logits, targets)\n        losses.update(loss.item(), images.size(0))\n        logits_all.append(logits.float().cpu().numpy())\n        targets_all.append(targets.float().cpu().numpy())\n\n    logits_np = np.concatenate(logits_all, axis=0)\n    targets_np = np.concatenate(targets_all, axis=0)\n    probs_np = sigmoid_numpy(logits_np)\n    metrics = compute_metrics_from_probs(\n        probs_np,\n        targets_np,\n        class_names=class_names,\n        thresholds=thresholds,\n        positive_threshold=CONFIG[\"positive_threshold\"],\n        min_eval_positive=CONFIG[\"min_eval_positive\"],\n    )\n    return losses.avg, metrics\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.247734Z","iopub.execute_input":"2026-05-20T15:49:26.248161Z","iopub.status.idle":"2026-05-20T15:49:26.277834Z","shell.execute_reply.started":"2026-05-20T15:49:26.248118Z","shell.execute_reply":"2026-05-20T15:49:26.275522Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 14. Checkpoints and Survival Packs\n","metadata":{}},{"cell_type":"code","source":"def checkpoint_payload(\n    model,\n    optimizer,\n    scheduler,\n    scaler,\n    phase,\n    epoch,\n    global_step,\n    best_metric,\n    best_epoch,\n    model_name,\n    resolved_timm_name,\n):\n    return {\n        \"model_state_dict\": model.state_dict(),\n        \"optimizer_state_dict\": optimizer.state_dict() if optimizer is not None else None,\n        \"scheduler_state_dict\": scheduler.state_dict() if scheduler is not None else None,\n        \"scaler_state_dict\": scaler.state_dict() if scaler is not None else None,\n        \"phase\": phase,\n        \"epoch\": int(epoch),\n        \"global_step\": int(global_step),\n        \"best_metric\": float(best_metric) if best_metric is not None else None,\n        \"best_epoch\": int(best_epoch) if best_epoch is not None else None,\n        \"config_snapshot\": to_serializable(CONFIG),\n        \"runtime_snapshot\": to_serializable(RUNTIME),\n        \"class_names\": CLASS_NAMES,\n        \"split_hash\": SPLIT_HASH,\n        \"model_name\": model_name,\n        \"resolved_timm_name\": resolved_timm_name,\n    }\n\ndef save_checkpoint(path, *args, **kwargs):\n    path = Path(path)\n    path.parent.mkdir(parents=True, exist_ok=True)\n    tmp_path = path.with_name(path.name + \".tmp\")\n    torch.save(checkpoint_payload(*args, **kwargs), tmp_path)\n    tmp_path.replace(path)\n\ndef file_sha256(path, chunk_size=1024 * 1024):\n    digest = hashlib.sha256()\n    with Path(path).open(\"rb\") as handle:\n        for chunk in iter(lambda: handle.read(chunk_size), b\"\"):\n            digest.update(chunk)\n    return digest.hexdigest()\n\ndef survival_pack_sources(include_final_artifacts=False):\n    base_sources = [\n        PATHS[\"config\"],\n        PATHS[\"runtime\"],\n        PATHS[\"package_versions\"],\n        PATHS[\"split_manifest\"],\n        PATHS[\"class_weights\"],\n        PATHS[\"epoch_log\"],\n        PATHS[\"failure_summary\"],\n        PATHS[\"preprocessed_cache_summary\"],\n        PATHS[\"last_checkpoint\"],\n        PATHS[\"best_phase1_checkpoint\"],\n        PATHS[\"best_phase2_checkpoint\"],\n        PATHS[\"best_stage1_checkpoint\"],\n        PATHS[\"calibration_summary\"],\n        PATHS[\"thresholds\"],\n        PATHS[\"model_summary\"],\n        PATHS[\"per_class_val\"],\n        PATHS[\"per_class_calib\"],\n        PATHS[\"stage1_bundle\"],\n        PATHS[\"best_stage1_model\"],\n        PATHS[\"best_stage1_bundle\"],\n    ]\n    if include_final_artifacts or CONFIG.get(\"backup_include_predictions\", False):\n        base_sources.extend([PATHS[\"val_predictions\"], PATHS[\"calib_predictions\"]])\n    return [Path(path) for path in base_sources if Path(path).exists()]\n\ndef archive_name_for_path(path):\n    path = Path(path)\n    try:\n        return \"run/\" + str(path.relative_to(RUN_DIR)).replace(\"\\\\\", \"/\")\n    except ValueError:\n        pass\n    try:\n        return \"best/\" + str(path.relative_to(BEST_DIR)).replace(\"\\\\\", \"/\")\n    except ValueError:\n        return path.name\n\ndef normalize_backup_reason(reason):\n    return re.sub(r\"[^a-zA-Z0-9_.-]+\", \"_\", str(reason)).strip(\"_\") or \"manual\"\n\ndef backup_channel_for_reason(reason):\n    reason_slug = normalize_backup_reason(reason)\n    if reason_slug == \"failure\" or reason_slug.startswith(\"failure_\"):\n        return \"failure\"\n    if reason_slug == \"cache_job\" or reason_slug.startswith(\"cache_\"):\n        return \"cache\"\n    return \"checkpoint\"\n\ndef survival_requirements_for_reason(reason, job_mode=None):\n    reason_slug = normalize_backup_reason(reason)\n    channel = backup_channel_for_reason(reason_slug)\n    required_all = []\n    required_any = []\n    valid_checkpoint_pack = channel == \"checkpoint\"\n    if reason_slug.startswith(\"phase1_epoch_\") or reason_slug == \"phase1_complete\":\n        required_any.append([\"run/checkpoints/best_phase1.pt\", \"run/checkpoints/last.pt\"])\n    elif reason_slug.startswith(\"phase2_epoch_\"):\n        required_all.append(\"run/checkpoints/best_phase1.pt\")\n        required_any.append([\"run/checkpoints/best_phase2.pt\", \"run/checkpoints/last.pt\"])\n    elif reason_slug == \"phase2_complete\":\n        required_all.extend([\"run/checkpoints/best_phase1.pt\", \"run/checkpoints/best_phase2.pt\"])\n    elif reason_slug == \"finalize_complete\":\n        required_all.extend([\"run/checkpoints/best_stage1.pt\", \"run/stage1_bundle.json\"])\n    elif reason_slug == \"phase2_restore\":\n        required_all.append(\"run/checkpoints/best_phase1.pt\")\n        valid_checkpoint_pack = True\n        channel = \"checkpoint\"\n    elif reason_slug in {\"cache_job\", \"failure\"}:\n        valid_checkpoint_pack = False\n    elif channel == \"checkpoint\":\n        required_any.append([\"run/checkpoints/best_phase1.pt\", \"run/checkpoints/best_phase2.pt\", \"run/checkpoints/last.pt\"])\n    return {\n        \"reason\": reason_slug,\n        \"channel\": channel,\n        \"valid_checkpoint_pack\": bool(valid_checkpoint_pack),\n        \"required_all\": required_all,\n        \"required_any\": required_any,\n        \"job_mode\": job_mode or CONFIG.get(\"job_mode\"),\n    }\n\ndef inspect_survival_pack(pack_path):\n    pack_path = Path(pack_path)\n    if not pack_path.exists():\n        raise FileNotFoundError(f\"Survival pack not found: {pack_path}\")\n    members = []\n    manifest = {}\n    with zipfile.ZipFile(pack_path, \"r\") as archive:\n        bad_file = archive.testzip()\n        if bad_file is not None:\n            raise RuntimeError(f\"Corrupt survival pack member: {bad_file}\")\n        members = sorted(info.filename.replace(\"\\\\\", \"/\") for info in archive.infolist() if not info.is_dir())\n        if \"survival_manifest.json\" in members:\n            manifest = json.loads(archive.read(\"survival_manifest.json\").decode(\"utf-8\"))\n    return {\n        \"path\": str(pack_path),\n        \"bytes\": int(pack_path.stat().st_size),\n        \"sha256\": file_sha256(pack_path),\n        \"members\": members,\n        \"member_count\": len(members),\n        \"manifest\": manifest,\n        \"has_best_phase1\": \"run/checkpoints/best_phase1.pt\" in members,\n        \"has_best_phase2\": \"run/checkpoints/best_phase2.pt\" in members,\n        \"has_last\": \"run/checkpoints/last.pt\" in members,\n        \"has_failure_summary\": \"run/failure_summary.json\" in members,\n    }\n\ndef validate_survival_pack_for_reason(pack_path, reason, job_mode=None, checkpoint_channel=False):\n    info = inspect_survival_pack(pack_path)\n    requirements = survival_requirements_for_reason(reason, job_mode=job_mode)\n    members = set(info[\"members\"])\n    errors = []\n    if checkpoint_channel and not requirements[\"valid_checkpoint_pack\"]:\n        errors.append(f\"reason {requirements['reason']!r} is not allowed in checkpoint backup channel\")\n    for required in requirements[\"required_all\"]:\n        if required not in members:\n            errors.append(f\"missing required member: {required}\")\n    for choices in requirements[\"required_any\"]:\n        if not any(choice in members for choice in choices):\n            errors.append(\"missing one of required members: \" + \", \".join(choices))\n    status = {\n        \"valid\": not errors,\n        \"errors\": errors,\n        \"requirements\": requirements,\n        \"inspection\": {k: v for k, v in info.items() if k != \"members\"},\n        \"member_count\": info[\"member_count\"],\n    }\n    if errors:\n        raise RuntimeError(\"Unsafe survival pack blocked: \" + \"; \".join(errors))\n    return status\n\ndef create_survival_pack(reason, include_final_artifacts=False):\n    reason_slug = normalize_backup_reason(reason)\n    pack_path = PATHS[\"latest_survival_pack\"]\n    pack_path.parent.mkdir(parents=True, exist_ok=True)\n    tmp_path = pack_path.with_name(pack_path.stem + \".tmp.zip\")\n    sources = survival_pack_sources(include_final_artifacts=include_final_artifacts)\n    manifest = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"reason\": reason_slug,\n        \"created_utc\": datetime.now(timezone.utc).isoformat(),\n        \"elapsed_hours\": elapsed_hours(),\n        \"include_final_artifacts\": bool(include_final_artifacts),\n        \"files\": [],\n        \"backup_channel\": backup_channel_for_reason(reason_slug),\n        \"required_members_policy\": survival_requirements_for_reason(reason_slug),\n        \"warning\": \"This zip is a survival pack. Persist checkpoint packs with STAGE1_CHECKPOINT_BACKUP_DATASET. Failure/cache packs must use separate datasets.\",\n    }\n    with zipfile.ZipFile(tmp_path, mode=\"w\") as archive:\n        for source in sources:\n            stat = source.stat()\n            archive_name = archive_name_for_path(source)\n            compress_type = zipfile.ZIP_STORED if source.suffix in {\".pt\", \".pth\", \".zip\"} else zipfile.ZIP_DEFLATED\n            archive.write(source, arcname=archive_name, compress_type=compress_type)\n            manifest[\"files\"].append(\n                {\n                    \"path\": str(source),\n                    \"archive_name\": archive_name,\n                    \"bytes\": int(stat.st_size),\n                    \"sha256\": file_sha256(source),\n                }\n            )\n        archive.writestr(\"survival_manifest.json\", json.dumps(to_serializable(manifest), indent=2, ensure_ascii=False))\n    tmp_path.replace(pack_path)\n    return pack_path, manifest\n\ndef write_backup_manifest(status):\n    payload = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"updated_utc\": datetime.now(timezone.utc).isoformat(),\n        \"elapsed_hours\": elapsed_hours(),\n        \"backup_status\": status,\n        \"latest_survival_pack\": str(PATHS[\"latest_survival_pack\"]),\n        \"backup_channel_map\": CONFIG.get(\"backup_channel_map\", {}),\n        \"kaggle_dataset_ref\": CONFIG.get(\"backup_kaggle_dataset_ref\", \"\"),\n        \"kaggle_dataset_enabled\": CONFIG.get(\"backup_kaggle_dataset_enabled\", False),\n        \"checkpoint_backup_dataset_ref\": CONFIG.get(\"checkpoint_backup_dataset_ref\", \"\"),\n        \"failure_backup_dataset_ref\": CONFIG.get(\"failure_backup_dataset_ref\", \"\"),\n        \"cache_backup_dataset_ref\": CONFIG.get(\"cache_backup_dataset_ref\", \"\"),\n    }\n    write_json(PATHS[\"backup_manifest\"], payload)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.279137Z","iopub.status.idle":"2026-05-20T15:49:26.279569Z","shell.execute_reply.started":"2026-05-20T15:49:26.279353Z","shell.execute_reply":"2026-05-20T15:49:26.27938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dataset_ref_for_backup_channel(channel):\n    channel = str(channel)\n    if channel == \"checkpoint\":\n        return CONFIG.get(\"checkpoint_backup_dataset_ref\", \"\")\n    if channel == \"failure\":\n        return CONFIG.get(\"failure_backup_dataset_ref\", \"\")\n    if channel == \"cache\":\n        return CONFIG.get(\"cache_backup_dataset_ref\", \"\")\n    raise ValueError(f\"Unknown backup channel: {channel}\")\n\ndef publish_dir_for_backup_channel(channel):\n    channel = str(channel)\n    key = f\"{channel}_backup_publish\"\n    if key not in PATHS:\n        raise ValueError(f\"Missing publish dir path for backup channel: {channel}\")\n    return PATHS[key]\n\ndef prepare_kaggle_dataset_publish_dir(pack_path, reason, channel, validation_status):\n    publish_dir = publish_dir_for_backup_channel(channel)\n    if publish_dir.exists():\n        shutil.rmtree(publish_dir)\n    publish_dir.mkdir(parents=True, exist_ok=True)\n    reason_slug = normalize_backup_reason(reason)\n    timestamp_slug = datetime.now(timezone.utc).strftime(\"%Y%m%dT%H%M%SZ\")\n    immutable_name = f\"stage1_{CONFIG['run_id']}_{reason_slug}_{timestamp_slug}.zip\"\n    immutable_pack = publish_dir / immutable_name\n    shutil.copy2(pack_path, immutable_pack)\n    if channel == \"checkpoint\":\n        alias_name = \"latest_valid_checkpoint_pack.zip\"\n    elif channel == \"failure\":\n        alias_name = \"latest_failure_pack.zip\"\n    else:\n        alias_name = \"latest_cache_pack.zip\"\n    alias_pack = publish_dir / alias_name\n    shutil.copy2(pack_path, alias_pack)\n    dataset_ref = dataset_ref_for_backup_channel(channel)\n    metadata = {\n        \"title\": f\"VinDr Stage1 {channel.title()} Backups\",\n        \"id\": dataset_ref,\n        \"licenses\": [{\"name\": \"unknown\"}],\n        \"subtitle\": f\"Private Stage 1 {channel} artifact packs\",\n        \"description\": (\n            f\"Private {channel} dataset for VinDr-CXR Stage 1. \"\n            \"Checkpoint, failure, and cache artifacts are intentionally separated to prevent unsafe latest-version overwrites.\"\n        ),\n    }\n    publish_manifest = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"reason\": reason_slug,\n        \"channel\": channel,\n        \"created_utc\": datetime.now(timezone.utc).isoformat(),\n        \"source_pack\": str(pack_path),\n        \"source_pack_sha256\": file_sha256(pack_path),\n        \"immutable_pack\": immutable_name,\n        \"alias_pack\": alias_name,\n        \"validation_status\": validation_status,\n    }\n    write_json(publish_dir / \"dataset-metadata.json\", metadata)\n    write_json(publish_dir / \"backup_publish_manifest.json\", publish_manifest)\n    return publish_dir, publish_manifest\n\ndef sync_survival_pack_to_kaggle_dataset(pack_path, reason):\n    reason_slug = normalize_backup_reason(reason)\n    channel = backup_channel_for_reason(reason_slug)\n    dataset_ref = dataset_ref_for_backup_channel(channel)\n    if not dataset_ref:\n        return {\n            \"enabled\": False,\n            \"status\": \"skipped\",\n            \"channel\": channel,\n            \"message\": f\"STAGE1_{channel.upper()}_BACKUP_DATASET is not set.\",\n        }\n    try:\n        validation_status = validate_survival_pack_for_reason(\n            pack_path,\n            reason_slug,\n            job_mode=CONFIG.get(\"job_mode\"),\n            checkpoint_channel=(channel == \"checkpoint\"),\n        )\n    except Exception as exc:\n        return {\n            \"enabled\": True,\n            \"status\": \"failed\",\n            \"channel\": channel,\n            \"dataset_ref\": dataset_ref,\n            \"exception_type\": type(exc).__name__,\n            \"exception_message\": str(exc),\n            \"upload_blocked_before_kaggle\": True,\n        }\n    publish_dir, publish_manifest = prepare_kaggle_dataset_publish_dir(pack_path, reason_slug, channel, validation_status)\n    message = (\n        f\"Stage1 {channel} backup run={CONFIG['run_id']} reason={reason_slug} \"\n        f\"sha256={publish_manifest['source_pack_sha256']} {datetime.now(timezone.utc).isoformat()}\"\n    )\n    command = [\n        \"kaggle\",\n        \"datasets\",\n        \"version\",\n        \"-p\",\n        str(publish_dir),\n        \"-m\",\n        message,\n        \"-q\",\n    ]\n    try:\n        completed = subprocess.run(\n            command,\n            stdout=subprocess.PIPE,\n            stderr=subprocess.STDOUT,\n            text=True,\n            timeout=int(CONFIG[\"backup_kaggle_version_timeout_sec\"]),\n        )\n        return {\n            \"enabled\": True,\n            \"status\": \"uploaded\" if completed.returncode == 0 else \"failed\",\n            \"channel\": channel,\n            \"dataset_ref\": dataset_ref,\n            \"returncode\": int(completed.returncode),\n            \"command\": \" \".join(command),\n            \"publish_manifest\": publish_manifest,\n            \"output_tail\": (completed.stdout or \"\")[-4000:],\n        }\n    except Exception as exc:\n        return {\n            \"enabled\": True,\n            \"status\": \"failed\",\n            \"channel\": channel,\n            \"dataset_ref\": dataset_ref,\n            \"exception_type\": type(exc).__name__,\n            \"exception_message\": str(exc),\n            \"command\": \" \".join(command),\n            \"publish_manifest\": publish_manifest,\n        }\n\ndef run_artifact_backup(reason, include_final_artifacts=False):\n    if not CONFIG.get(\"backup_enabled\", True):\n        return {\"enabled\": False, \"status\": \"disabled\"}\n    pack_path, manifest = create_survival_pack(reason, include_final_artifacts=include_final_artifacts)\n    sync_status = sync_survival_pack_to_kaggle_dataset(pack_path, reason)\n    status = {\n        \"enabled\": True,\n        \"reason\": normalize_backup_reason(reason),\n        \"channel\": backup_channel_for_reason(reason),\n        \"pack_path\": str(pack_path),\n        \"pack_sha256\": file_sha256(pack_path),\n        \"file_count\": len(manifest[\"files\"]),\n        \"bytes\": int(pack_path.stat().st_size),\n        \"sync\": sync_status,\n    }\n    write_backup_manifest(status)\n    if sync_status.get(\"status\") == \"failed\":\n        message = f\"Artifact backup upload failed: {sync_status}\"\n        if CONFIG.get(\"backup_fail_on_error\", False):\n            raise RuntimeError(message)\n        print(\"WARNING:\", message)\n    else:\n        print(\n            f\"Artifact survival pack updated: {pack_path} | \"\n            f\"channel={status['channel']} sync={sync_status.get('status')}\"\n        )\n    return status\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.28105Z","iopub.status.idle":"2026-05-20T15:49:26.281363Z","shell.execute_reply.started":"2026-05-20T15:49:26.281231Z","shell.execute_reply":"2026-05-20T15:49:26.281247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def inspect_last_checkpoint(path=None):\n    checkpoint_path = Path(path or PATHS[\"last_checkpoint\"])\n    if not CONFIG[\"resume\"] or not checkpoint_path.exists():\n        return {\"exists\": False, \"path\": str(checkpoint_path), \"phase\": None}\n    payload = safe_torch_load(checkpoint_path, map_location=\"cpu\")\n    return {\n        \"exists\": True,\n        \"path\": str(checkpoint_path),\n        \"phase\": payload.get(\"phase\"),\n        \"epoch\": payload.get(\"epoch\"),\n        \"global_step\": payload.get(\"global_step\"),\n        \"best_metric\": payload.get(\"best_metric\"),\n        \"best_epoch\": payload.get(\"best_epoch\"),\n        \"model_name\": payload.get(\"model_name\"),\n        \"split_hash\": payload.get(\"split_hash\"),\n    }\n\ndef maybe_resume_from_last(model, optimizer, scheduler, scaler, expected_phase, device):\n    path = PATHS[\"last_checkpoint\"]\n    if not CONFIG[\"resume\"] or not path.exists():\n        return 0, 0, -np.inf, None\n    payload = safe_torch_load(path, map_location=device)\n    if payload.get(\"phase\") != expected_phase:\n        print(f\"Resume skipped: last checkpoint phase={payload.get('phase')} != {expected_phase}\")\n        return 0, 0, -np.inf, None\n    validate_checkpoint_contract(payload, expected_phase=expected_phase, checkpoint_path=path)\n    model.load_state_dict(payload[\"model_state_dict\"])\n    if optimizer is not None and payload.get(\"optimizer_state_dict\") is not None:\n        optimizer.load_state_dict(payload[\"optimizer_state_dict\"])\n    if scheduler is not None and payload.get(\"scheduler_state_dict\") is not None:\n        scheduler.load_state_dict(payload[\"scheduler_state_dict\"])\n    if scaler is not None and payload.get(\"scaler_state_dict\") is not None:\n        scaler.load_state_dict(payload[\"scaler_state_dict\"])\n    start_epoch = int(payload.get(\"epoch\", -1)) + 1\n    global_step = int(payload.get(\"global_step\", 0))\n    best_metric = payload.get(\"best_metric\")\n    best_metric = float(best_metric) if best_metric is not None else -np.inf\n    best_epoch = payload.get(\"best_epoch\")\n    print(f\"Resumed {expected_phase} from {path}: start_epoch={start_epoch}, best_metric={best_metric}\")\n    return start_epoch, global_step, best_metric, best_epoch\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.282021Z","iopub.status.idle":"2026-05-20T15:49:26.282296Z","shell.execute_reply.started":"2026-05-20T15:49:26.282165Z","shell.execute_reply":"2026-05-20T15:49:26.28218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_training_phase(\n    phase,\n    model,\n    train_dataset,\n    val_dataset,\n    model_config,\n    class_weights,\n    device,\n    class_names,\n    sampler=None,\n):\n    if phase == \"phase2\":\n        model.disable_backbone_checkpointing()\n        for parameter in model.backbone.parameters():\n            parameter.requires_grad = False\n        for parameter in model.head.parameters():\n            parameter.requires_grad = True\n\n    if phase == \"phase2\":\n        physical_bs = min(\n            int(CONFIG[\"phase2_max_physical_bs\"]),\n            int(model_config[\"physical_bs\"]) * int(CONFIG[\"phase2_batch_multiplier\"]),\n        )\n    else:\n        physical_bs = int(model_config[\"physical_bs\"])\n    accum_steps = max(1, math.ceil(CONFIG[\"effective_batch_size\"] / physical_bs))\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=physical_bs,\n        shuffle=(sampler is None),\n        sampler=sampler,\n        drop_last=True,\n        **loader_kwargs(),\n    )\n    val_loader = DataLoader(\n        val_dataset,\n        batch_size=max(1, physical_bs * int(CONFIG[\"eval_batch_multiplier\"])),\n        shuffle=False,\n        **loader_kwargs(),\n    )\n\n    epochs = CONFIG[\"phase1_epochs\"] if phase == \"phase1\" else CONFIG[\"phase2_epochs\"]\n    patience = CONFIG[\"patience_phase1\"] if phase == \"phase1\" else CONFIG[\"patience_phase2\"]\n    optimizer = make_optimizer(model, phase)\n    total_optimizer_steps = max(1, math.ceil(len(train_loader) / accum_steps) * epochs)\n    warmup_steps = max(1, int(CONFIG[\"warmup_ratio\"] * total_optimizer_steps))\n    scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_optimizer_steps)\n    scaler = make_grad_scaler(device, CONFIG[\"use_amp\"])\n    criterion = make_criterion(class_weights, device)\n    stopper = EarlyStopping(patience=patience, mode=\"max\")\n\n    start_epoch, global_step, best_metric, best_epoch = maybe_resume_from_last(\n        model, optimizer, scheduler, scaler, phase, device\n    )\n    best_path = PATHS[\"best_phase1_checkpoint\"] if phase == \"phase1\" else PATHS[\"best_phase2_checkpoint\"]\n\n    for epoch in range(start_epoch, epochs):\n        train_loss, global_step = train_one_epoch(\n            model,\n            train_loader,\n            criterion,\n            optimizer,\n            scheduler,\n            scaler,\n            device,\n            accum_steps,\n            global_step,\n        )\n        val_loss, val_metrics = validate(model, val_loader, criterion, device, class_names)\n        val_map = float(val_metrics[\"macro_mAP\"])\n        if np.isnan(val_map):\n            val_map = -np.inf\n\n        improved = (not Path(best_path).exists()) or (val_map > best_metric)\n        if improved:\n            best_metric = val_map\n            best_epoch = epoch\n            save_checkpoint(\n                best_path,\n                model=model,\n                optimizer=optimizer,\n                scheduler=scheduler,\n                scaler=scaler,\n                phase=phase,\n                epoch=epoch,\n                global_step=global_step,\n                best_metric=best_metric,\n                best_epoch=best_epoch,\n                model_name=CONFIG[\"run_model_name\"],\n                resolved_timm_name=model.timm_name,\n            )\n\n        save_checkpoint(\n            PATHS[\"last_checkpoint\"],\n            model=model,\n            optimizer=optimizer,\n            scheduler=scheduler,\n            scaler=scaler,\n            phase=phase,\n            epoch=epoch,\n            global_step=global_step,\n            best_metric=best_metric,\n            best_epoch=best_epoch,\n            model_name=CONFIG[\"run_model_name\"],\n            resolved_timm_name=model.timm_name,\n        )\n\n        append_epoch_log(\n            {\n                \"run_id\": CONFIG[\"run_id\"],\n                \"spec_version\": CONFIG[\"spec_version\"],\n                \"wave\": \"W1/W2\",\n                \"stage\": \"1\",\n                \"phase\": phase,\n                \"epoch\": epoch,\n                \"global_step\": global_step,\n                \"model_name\": CONFIG[\"run_model_name\"],\n                \"resolved_timm_name\": model.timm_name,\n                \"img_size\": model_config[\"img_size\"],\n                \"batch_size\": physical_bs,\n                \"accum_steps\": accum_steps,\n                \"train_loss\": train_loss,\n                \"val_loss\": val_loss,\n                \"val_macro_mAP\": val_metrics[\"macro_mAP\"],\n                \"val_AUROC_macro\": val_metrics[\"macro_AUROC\"],\n                \"val_F1_macro\": val_metrics[\"macro_F1\"],\n                \"val_ECE\": val_metrics[\"ECE\"],\n                \"val_Brier\": val_metrics[\"Brier\"],\n                \"best_metric\": best_metric,\n                \"best_epoch\": best_epoch,\n                \"elapsed_hours\": elapsed_hours(),\n                \"notes\": \"best\" if improved else \"\",\n            }\n        )\n\n        print(\n            f\"{phase} epoch {epoch+1}/{epochs}: train_loss={train_loss:.5f}, \"\n            f\"val_mAP={val_metrics['macro_mAP']:.5f}, val_ECE={val_metrics['ECE']:.5f}, best={best_metric:.5f}\"\n        )\n        if ((epoch + 1) % int(CONFIG[\"backup_every_epochs\"])) == 0 or improved or should_stop_for_time():\n            run_artifact_backup(f\"{phase}_epoch_{epoch+1:03d}\", include_final_artifacts=False)\n        stopper.step(val_map)\n        if stopper.should_stop:\n            print(f\"Early stopping {phase} at epoch {epoch}.\")\n            break\n        if should_stop_for_time():\n            print(f\"Soft time limit reached after epoch {epoch}; last checkpoint saved.\")\n            break\n\n    if Path(best_path).exists():\n        best_payload = safe_torch_load(best_path, map_location=\"cpu\")\n        validate_checkpoint_contract(best_payload, expected_phase=phase, checkpoint_path=best_path)\n        model.load_state_dict(best_payload[\"model_state_dict\"])\n        best_metric = float(best_payload[\"best_metric\"])\n        best_epoch = int(best_payload[\"best_epoch\"])\n    elif start_epoch >= epochs:\n        raise RuntimeError(f\"{phase} đã được đánh dấu hoàn tất nhưng thiếu best checkpoint: {best_path}\")\n    return model, best_metric, best_epoch, str(best_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.283001Z","iopub.status.idle":"2026-05-20T15:49:26.283306Z","shell.execute_reply.started":"2026-05-20T15:49:26.283166Z","shell.execute_reply":"2026-05-20T15:49:26.283183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 15. Resume Guards and Restore\n","metadata":{}},{"cell_type":"code","source":"def log_cuda_memory(label, device=None):\n    if not torch.cuda.is_available():\n        return {\"label\": label, \"device\": \"cpu\"}\n    selected = torch.device(device or DEVICE)\n    idx = selected.index if selected.index is not None else torch.cuda.current_device()\n    allocated = torch.cuda.memory_allocated(idx) / (1024 ** 3)\n    reserved = torch.cuda.memory_reserved(idx) / (1024 ** 3)\n    max_allocated = torch.cuda.max_memory_allocated(idx) / (1024 ** 3)\n    payload = {\n        \"label\": label,\n        \"device_index\": int(idx),\n        \"allocated_gb\": round(float(allocated), 3),\n        \"reserved_gb\": round(float(reserved), 3),\n        \"max_allocated_gb\": round(float(max_allocated), 3),\n    }\n    print(f\"CUDA memory [{label}]: allocated={allocated:.2f}GB reserved={reserved:.2f}GB max_allocated={max_allocated:.2f}GB\")\n    return payload\n\ndef validate_checkpoint_contract(payload, expected_phase=None, checkpoint_path=None):\n    if not CONFIG.get(\"strict_checkpoint_contract\", True):\n        return True\n    errors = []\n    path_text = str(checkpoint_path) if checkpoint_path is not None else \"<checkpoint>\"\n    if expected_phase is not None and payload.get(\"phase\") != expected_phase:\n        errors.append(f\"phase={payload.get('phase')!r} != expected {expected_phase!r}\")\n    if payload.get(\"model_name\") != CONFIG[\"run_model_name\"]:\n        errors.append(f\"model_name={payload.get('model_name')!r} != {CONFIG['run_model_name']!r}\")\n    if payload.get(\"split_hash\") != SPLIT_HASH:\n        errors.append(\"split_hash mismatch\")\n    if list(payload.get(\"class_names\", [])) != list(CLASS_NAMES):\n        errors.append(\"class_names mismatch\")\n    snapshot = payload.get(\"config_snapshot\", {}) or {}\n    checks = {\n        \"img_size\": CONFIG.get(\"img_size\"),\n        \"num_classes\": CONFIG.get(\"num_classes\"),\n        \"positive_threshold\": CONFIG.get(\"positive_threshold\"),\n        \"preprocessing_mode\": CONFIG.get(\"preprocessing_mode\"),\n        \"preprocessed_cache_dtype\": CONFIG.get(\"preprocessed_cache_dtype\"),\n        \"preprocessed_cache_max_value\": CONFIG.get(\"preprocessed_cache_max_value\"),\n        \"preprocessed_cache_version\": CONFIG.get(\"preprocessed_cache_version\"),\n    }\n    for key, expected_value in checks.items():\n        if key in snapshot and snapshot.get(key) != expected_value:\n            errors.append(f\"{key}={snapshot.get(key)!r} != {expected_value!r}\")\n    if errors:\n        raise RuntimeError(\"Unsafe checkpoint resume/load blocked for \" + path_text + \": \" + \"; \".join(errors))\n    return True\n\ndef validate_restored_phase_inputs(job_mode):\n    if job_mode != \"phase2\":\n        return {\"checked\": False, \"job_mode\": job_mode}\n    if not PATHS[\"best_phase1_checkpoint\"].exists():\n        raise FileNotFoundError(\n            f\"Phase 2 requires restored best_phase1.pt but it is missing: {PATHS['best_phase1_checkpoint']}\"\n        )\n    payload = safe_torch_load(PATHS[\"best_phase1_checkpoint\"], map_location=\"cpu\")\n    validate_checkpoint_contract(payload, expected_phase=\"phase1\", checkpoint_path=PATHS[\"best_phase1_checkpoint\"])\n    return {\n        \"checked\": True,\n        \"job_mode\": job_mode,\n        \"best_phase1_checkpoint\": str(PATHS[\"best_phase1_checkpoint\"]),\n        \"phase\": payload.get(\"phase\"),\n        \"epoch\": int(payload.get(\"epoch\", -1)),\n        \"best_metric\": float(payload.get(\"best_metric\", float(\"nan\"))),\n        \"split_hash\": payload.get(\"split_hash\"),\n    }\n\ndef restore_survival_pack_if_requested():\n    pack_value = CONFIG.get(\"restore_survival_pack\", \"\")\n    if not pack_value:\n        return {\"requested\": False, \"status\": \"skipped\"}\n    pack_path = Path(pack_value)\n    if not pack_path.exists():\n        raise FileNotFoundError(f\"STAGE1_RESTORE_PACK không tồn tại: {pack_path}\")\n    restore_validation = None\n    if CONFIG.get(\"job_mode\") == \"phase2\":\n        restore_validation = validate_survival_pack_for_reason(pack_path, \"phase2_restore\", job_mode=\"phase2\", checkpoint_channel=True)\n    restored = []\n    skipped = []\n    with zipfile.ZipFile(pack_path, \"r\") as archive:\n        for member in archive.infolist():\n            name = member.filename.replace(\"\\\\\", \"/\")\n            if member.is_dir() or name.endswith(\"/\"):\n                continue\n            if name.startswith(\"run/\"):\n                target = RUN_DIR / name[len(\"run/\"):]\n            elif name.startswith(\"best/\"):\n                target = BEST_DIR / name[len(\"best/\"):]\n            elif name == \"survival_manifest.json\":\n                target = RUN_DIR / \"restored_survival_manifest.json\"\n            else:\n                target = RUN_DIR / \"restored_misc\" / Path(name).name\n            if target.exists() and not CONFIG.get(\"restore_overwrite\", False):\n                if CONFIG.get(\"job_mode\") == \"phase2\" and name in {\"run/checkpoints/best_phase1.pt\", \"run/checkpoints/last.pt\"}:\n                    raise RuntimeError(\n                        f\"Phase2 restore would skip existing checkpoint {target}. \"\n                        \"Set STAGE1_RESTORE_OVERWRITE=1 or use a clean STAGE1_RUN_ID.\"\n                    )\n                skipped.append(str(target))\n                continue\n            target.parent.mkdir(parents=True, exist_ok=True)\n            with archive.open(member, \"r\") as src, target.open(\"wb\") as dst:\n                shutil.copyfileobj(src, dst)\n            restored.append(str(target))\n    restored_phase_validation = validate_restored_phase_inputs(CONFIG.get(\"job_mode\"))\n    status = {\n        \"requested\": True,\n        \"status\": \"restored\",\n        \"pack_path\": str(pack_path),\n        \"restore_pack_validation\": restore_validation,\n        \"restored_phase_validation\": restored_phase_validation,\n        \"restored_count\": len(restored),\n        \"skipped_existing_count\": len(skipped),\n        \"restored\": restored[:25],\n        \"skipped_existing\": skipped[:25],\n    }\n    write_json(RUN_DIR / \"restore_status.json\", status)\n    print(f\"Restored survival pack: {pack_path} | restored={len(restored)} skipped_existing={len(skipped)}\")\n    return status\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.284649Z","iopub.status.idle":"2026-05-20T15:49:26.285119Z","shell.execute_reply.started":"2026-05-20T15:49:26.284939Z","shell.execute_reply":"2026-05-20T15:49:26.284964Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 16. Optional Smoke Tests\n","metadata":{}},{"cell_type":"code","source":"SMOKE_CELL_VERSION = \"stage1_smoke_v3_speed_persistence_2026_05_16\"\n\nprint(f\"Smoke cell version: {SMOKE_CELL_VERSION}\")\n\ndef smoke_loss_weights(device):\n    ones = torch.ones(CONFIG[\"num_classes\"], dtype=torch.float32)\n    return {\"qfl_alpha\": ones.to(device), \"bce_pos_weight\": ones.to(device)}\n\ndef seed_smoke(seed_offset=0):\n    seed = int(CONFIG[\"split_seed\"]) + int(seed_offset)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed_all(seed)\n\ndef disable_dropout_for_smoke(model):\n    dropout_types = (nn.Dropout, nn.Dropout1d, nn.Dropout2d, nn.Dropout3d, nn.AlphaDropout)\n    for module in model.modules():\n        if isinstance(module, dropout_types):\n            module.p = 0.0\n\ndef qfl_amp_safety_smoke(device):\n    print(\"QFL AMP safety smoke\")\n    seed_smoke(seed_offset=101)\n    logits = torch.randn(2, CONFIG[\"num_classes\"], device=device, requires_grad=True)\n    targets = torch.rand(2, CONFIG[\"num_classes\"], device=device)\n    criterion = QualityFocalLoss(\n        beta=CONFIG[\"qfl_beta\"],\n        epsilon=CONFIG[\"qfl_epsilon\"],\n        class_weights=torch.ones(CONFIG[\"num_classes\"], device=device),\n    )\n    with autocast_context(device, CONFIG[\"use_amp\"]):\n        loss = criterion(logits, targets)\n    if not torch.isfinite(loss):\n        raise RuntimeError(\"QFL AMP safety smoke failed: loss is not finite.\")\n    loss.backward()\n    if logits.grad is None or not torch.isfinite(logits.grad).all():\n        raise RuntimeError(\"QFL AMP safety smoke failed: gradients are invalid.\")\n    print(f\"QFL AMP safety smoke passed: loss={float(loss.detach().cpu()):.6f}\")\n    del logits, targets, criterion, loss\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\ndef quick_test(model_config, device):\n    print(f\"Quick model test: {model_config['display_name']}\")\n    seed_smoke(seed_offset=202)\n    model = prepare_model_for_runtime(VinDrCXRModel(model_config, num_classes=CONFIG[\"num_classes\"], pretrained=False), model_config, device)\n    model.train()\n    dummy_images = move_images_to_device(torch.randn(1, 3, model_config[\"img_size\"], model_config[\"img_size\"]), device, model_config)\n    dummy_targets = torch.zeros(1, CONFIG[\"num_classes\"], device=device)\n    dummy_targets[0, 0] = 1.0\n    criterion = make_criterion(smoke_loss_weights(device), device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n    scaler = make_grad_scaler(device, CONFIG[\"use_amp\"])\n    optimizer.zero_grad(set_to_none=True)\n    with autocast_context(device, CONFIG[\"use_amp\"]):\n        logits = model(dummy_images)\n        loss = criterion(logits, dummy_targets)\n    if logits.shape != (1, CONFIG[\"num_classes\"]):\n        raise RuntimeError(f\"Unexpected logits shape: {tuple(logits.shape)}\")\n    if not torch.isfinite(loss):\n        raise RuntimeError(\"Loss is not finite.\")\n    scaler.scale(loss).backward()\n    has_grad = any(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad)\n    if not has_grad:\n        raise RuntimeError(\"Backward failed: no finite gradients.\")\n    scaler.step(optimizer)\n    scaler.update()\n    del model, dummy_images, dummy_targets, logits, loss\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n    print(\"Quick model test passed: shape=(1, 14), finite loss, backward OK.\")\n\ndef one_batch_overfit_smoke(model_config, device, steps=2):\n    steps = int(steps)\n    if steps < 1:\n        raise ValueError(\"one_batch_overfit_smoke requires at least 1 step.\")\n    print(f\"One-batch FP32 optimizer smoke: {model_config['display_name']} steps={steps}\")\n    seed_smoke(seed_offset=303)\n    model = prepare_model_for_runtime(VinDrCXRModel(model_config, num_classes=CONFIG[\"num_classes\"], pretrained=False), model_config, device)\n    disable_dropout_for_smoke(model)\n    for parameter in model.backbone.parameters():\n        parameter.requires_grad = False\n    for parameter in model.head.parameters():\n        parameter.requires_grad = True\n    model.train()\n    model.backbone.eval()\n    model.head.train()\n\n    images = move_images_to_device(torch.randn(1, 3, model_config[\"img_size\"], model_config[\"img_size\"]), device, model_config)\n    targets = torch.zeros(1, CONFIG[\"num_classes\"], device=device)\n    targets[0, [0, 3, 8]] = torch.tensor([1.0, 2.0 / 3.0, 1.0], device=device)\n    criterion = make_criterion(smoke_loss_weights(device), device)\n    optimizer = torch.optim.AdamW(model.head.parameters(), lr=1e-5, weight_decay=0.0)\n    head_params = [p for p in model.head.parameters() if p.requires_grad]\n    before = [p.detach().clone() for p in head_params]\n    losses = []\n    grad_norms = []\n\n    for _ in range(steps):\n        optimizer.zero_grad(set_to_none=True)\n        with autocast_context(device, False):\n            logits = model(images)\n            loss = criterion(logits, targets)\n        if not torch.isfinite(loss):\n            raise RuntimeError(\"One-batch FP32 optimizer smoke produced non-finite loss.\")\n        loss.backward()\n        grad_sq = 0.0\n        finite_grad_tensors = 0\n        for parameter in head_params:\n            if parameter.grad is None:\n                continue\n            grad = parameter.grad.detach().float()\n            if not torch.isfinite(grad).all():\n                raise RuntimeError(\"One-batch FP32 optimizer smoke produced non-finite head gradients.\")\n            finite_grad_tensors += 1\n            grad_sq += float(grad.pow(2).sum().cpu())\n        grad_norm = math.sqrt(grad_sq)\n        if finite_grad_tensors == 0 or not math.isfinite(grad_norm) or grad_norm <= 0.0:\n            raise RuntimeError(f\"One-batch FP32 optimizer smoke failed: invalid finite head grad_norm={grad_norm}.\")\n        nn.utils.clip_grad_norm_(head_params, max_norm=1.0)\n        optimizer.step()\n        losses.append(float(loss.detach().cpu()))\n        grad_norms.append(grad_norm)\n\n    max_param_delta = 0.0\n    for old_param, new_param in zip(before, head_params):\n        delta = (new_param.detach() - old_param).abs().max()\n        max_param_delta = max(max_param_delta, float(delta.cpu()))\n    if not math.isfinite(max_param_delta) or max_param_delta <= 0.0:\n        raise RuntimeError(\"One-batch FP32 optimizer smoke failed: optimizer did not update head parameters.\")\n\n    trend_note = \"decreased\" if losses[-1] < losses[0] else \"not_monotonic\"\n    print(\n        \"One-batch FP32 optimizer smoke passed: \"\n        f\"initial_loss={losses[0]:.6f}, final_loss={losses[-1]:.6f}, \"\n        f\"trend={trend_note}, max_head_delta={max_param_delta:.3e}, \"\n        f\"last_grad_norm={grad_norms[-1]:.3e}\"\n    )\n    del model, images, targets, logits, loss, before\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.286861Z","iopub.status.idle":"2026-05-20T15:49:26.28743Z","shell.execute_reply.started":"2026-05-20T15:49:26.287206Z","shell.execute_reply":"2026-05-20T15:49:26.287239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CONFIG.get(\"run_smoke_tests\", False):\n    qfl_amp_safety_smoke(DEVICE)\n    quick_test(MODEL_CONFIG, DEVICE)\n    one_batch_overfit_smoke(MODEL_CONFIG, DEVICE, steps=2)\nelse:\n    print(\"Smoke tests skipped. Set STAGE1_RUN_SMOKE=1 to run local safety checks before training.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.288642Z","iopub.status.idle":"2026-05-20T15:49:26.289538Z","shell.execute_reply.started":"2026-05-20T15:49:26.289153Z","shell.execute_reply":"2026-05-20T15:49:26.289179Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 17. Phase Jobs\n","metadata":{}},{"cell_type":"code","source":"def stage1_image_ids():\n    return (\n        list(SPLITS[\"train_core\"][\"ids\"])\n        + list(SPLITS[\"val_select\"][\"ids\"])\n        + list(SPLITS[\"calib\"][\"ids\"])\n    )\n\n\ndef assert_training_persistence_ready(job_mode):\n    training_modes = {\"all\", \"phase1\", \"phase2\"}\n    if job_mode not in training_modes:\n        return True\n    if not CONFIG.get(\"require_external_backup_for_training\", True):\n        return True\n    if CONFIG.get(\"backup_kaggle_dataset_enabled\", False):\n        return True\n    if CONFIG.get(\"allow_local_only_training\", False):\n        print(\"WARNING: External backup is disabled and STAGE1_ALLOW_LOCAL_ONLY=1. Survival pack remains local-only until Save Version/download.\")\n        return True\n    raise RuntimeError(\n        \"External checkpoint backup is required for Stage 1 training jobs. \"\n        \"Set STAGE1_BACKUP_DATASET='username/dataset-slug' or explicitly set STAGE1_ALLOW_LOCAL_ONLY=1.\"\n    )\n\n\ndef maybe_prebuild_cache_for_training_job():\n    if not CONFIG.get(\"prebuild_cache_in_training_jobs\", False):\n        return {\"status\": \"skipped\", \"reason\": \"STAGE1_PREBUILD_CACHE_IN_TRAINING is false\"}\n    return run_cache_job()\n\ndef run_cache_job():\n    model_config = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n    summary = prebuild_preprocessed_cache(\n        stage1_image_ids(),\n        DICOM_DIR,\n        model_config[\"img_size\"],\n        summary_path=PATHS[\"preprocessed_cache_summary\"],\n    )\n    run_artifact_backup(\"cache_job\", include_final_artifacts=False)\n    return {\"job_mode\": \"cache\", \"cache_summary\": summary, \"cache_path\": str(PATHS[\"preprocessed_cache\"])}\n\ndef build_stage1_datasets(include_calib=False):\n    model_config = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n    train_dataset = VinDrCXRDataset(\n        SPLITS[\"train_core\"][\"ids\"],\n        SPLITS[\"train_core\"][\"labels\"],\n        dicom_dir=DICOM_DIR,\n        transform=train_transform,\n        img_size=model_config[\"img_size\"],\n    )\n    val_dataset = VinDrCXRDataset(\n        SPLITS[\"val_select\"][\"ids\"],\n        SPLITS[\"val_select\"][\"labels\"],\n        dicom_dir=DICOM_DIR,\n        transform=eval_transform,\n        img_size=model_config[\"img_size\"],\n    )\n    if not include_calib:\n        return train_dataset, val_dataset, None\n    calib_dataset = VinDrCXRDataset(\n        SPLITS[\"calib\"][\"ids\"],\n        SPLITS[\"calib\"][\"labels\"],\n        dicom_dir=DICOM_DIR,\n        transform=eval_transform,\n        img_size=model_config[\"img_size\"],\n    )\n    return train_dataset, val_dataset, calib_dataset\n\n\ndef build_stage1_model(pretrained=True):\n    model_name = CONFIG[\"run_model_name\"]\n    model_config = CONFIG[\"model_configs\"][model_name]\n    model = VinDrCXRModel(model_config, num_classes=CONFIG[\"num_classes\"], pretrained=pretrained)\n    return prepare_model_for_runtime(model, model_config, DEVICE)\n\ndef checkpoint_summary(path, expected_phase=None):\n    path = Path(path)\n    if not path.exists():\n        return None\n    payload = safe_torch_load(path, map_location=\"cpu\")\n    validate_checkpoint_contract(payload, expected_phase=expected_phase, checkpoint_path=path)\n    return {\n        \"path\": str(path),\n        \"phase\": payload.get(\"phase\"),\n        \"best_metric\": float(payload.get(\"best_metric\", float(\"nan\"))),\n        \"best_epoch\": int(payload.get(\"best_epoch\", -1)) if payload.get(\"best_epoch\") is not None else None,\n        \"epoch\": int(payload.get(\"epoch\", -1)) if payload.get(\"epoch\") is not None else None,\n        \"model_name\": payload.get(\"model_name\"),\n        \"resolved_timm_name\": payload.get(\"resolved_timm_name\"),\n    }\n\n\ndef completed_phase_summary(phase):\n    marker = PATHS[f\"{phase}_complete\"]\n    best_path = PATHS[f\"best_{phase}_checkpoint\"]\n    if CONFIG.get(\"force_rerun_completed_phase\", False):\n        return None\n    if not marker.exists() or not best_path.exists():\n        return None\n    summary = checkpoint_summary(best_path, expected_phase=phase)\n    marker_payload = json.loads(marker.read_text(encoding=\"utf-8\"))\n    summary[\"marker\"] = marker_payload\n    summary[\"status\"] = \"already_complete\"\n    print(f\"{phase} already complete; using {best_path}\")\n    return summary\n\n\ndef write_phase_completion_marker(phase, summary):\n    marker = PATHS[f\"{phase}_complete\"]\n    payload = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"phase\": phase,\n        \"completed_utc\": datetime.now(timezone.utc).isoformat(),\n        \"elapsed_hours\": elapsed_hours(),\n        \"summary\": summary,\n    }\n    write_json(marker, payload)\n    return payload\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.291013Z","iopub.status.idle":"2026-05-20T15:49:26.291371Z","shell.execute_reply.started":"2026-05-20T15:49:26.291206Z","shell.execute_reply":"2026-05-20T15:49:26.291231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_phase1_job():\n    assert_training_persistence_ready(\"phase1\")\n    cached = completed_phase_summary(\"phase1\")\n    if cached is not None:\n        return cached\n    maybe_prebuild_cache_for_training_job()\n    model_config = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n    train_dataset, val_dataset, _ = build_stage1_datasets(include_calib=False)\n    model = build_stage1_model(pretrained=True)\n    log_cuda_memory(\"phase1_model_ready\", DEVICE)\n    model, best_map, best_epoch, best_path = run_training_phase(\n        phase=\"phase1\",\n        model=model,\n        train_dataset=train_dataset,\n        val_dataset=val_dataset,\n        model_config=model_config,\n        class_weights=CLASS_WEIGHTS,\n        device=DEVICE,\n        class_names=CLASS_NAMES,\n        sampler=None,\n    )\n    summary = {\n        \"status\": \"complete\",\n        \"phase\": \"phase1\",\n        \"best_metric\": float(best_map),\n        \"best_epoch\": int(best_epoch),\n        \"best_checkpoint\": str(best_path),\n    }\n    write_phase_completion_marker(\"phase1\", summary)\n    run_artifact_backup(\"phase1_complete\", include_final_artifacts=False)\n    return summary\n\ndef choose_best_stage1_checkpoint():\n    candidates = []\n    phase1 = checkpoint_summary(PATHS[\"best_phase1_checkpoint\"], expected_phase=\"phase1\")\n    phase2 = checkpoint_summary(PATHS[\"best_phase2_checkpoint\"], expected_phase=\"phase2\")\n    if phase1 is not None:\n        candidates.append(phase1)\n    if phase2 is not None:\n        candidates.append(phase2)\n    if not candidates:\n        raise FileNotFoundError(\"Không có best_phase1.pt hoặc best_phase2.pt để chọn best Stage 1.\")\n    best = max(candidates, key=lambda item: item[\"best_metric\"])\n    shutil.copy2(best[\"path\"], PATHS[\"best_stage1_checkpoint\"])\n    write_json(RUN_DIR / \"best_stage1_selection.json\", {\"selected\": best, \"candidates\": candidates})\n    print(f\"Selected best Stage 1 checkpoint from {best['phase']}: {best['path']}\")\n    return best\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.292814Z","iopub.status.idle":"2026-05-20T15:49:26.293183Z","shell.execute_reply.started":"2026-05-20T15:49:26.293002Z","shell.execute_reply":"2026-05-20T15:49:26.293025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_phase1_checkpoint_into_model(model):\n    phase1_path = PATHS[\"best_phase1_checkpoint\"]\n    if not phase1_path.exists():\n        raise FileNotFoundError(\n            f\"Phase 2 cần best Phase 1 checkpoint nhưng không thấy: {phase1_path}. \"\n            \"Chạy STAGE1_JOB_MODE=phase1 trước hoặc restore survival pack.\"\n        )\n    payload = safe_torch_load(phase1_path, map_location=\"cpu\")\n    validate_checkpoint_contract(payload, expected_phase=\"phase1\", checkpoint_path=phase1_path)\n    model.load_state_dict(payload[\"model_state_dict\"])\n    return payload\n\n\ndef run_phase2_job():\n    assert_training_persistence_ready(\"phase2\")\n    cached = completed_phase_summary(\"phase2\")\n    if cached is not None:\n        choose_best_stage1_checkpoint()\n        return cached\n    maybe_prebuild_cache_for_training_job()\n    model_config = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n    train_dataset, val_dataset, _ = build_stage1_datasets(include_calib=False)\n    model = build_stage1_model(pretrained=False)\n    load_phase1_checkpoint_into_model(model)\n    phase2_sampler = create_class_balanced_sampler(SPLITS[\"train_core\"][\"labels\"])\n    log_cuda_memory(\"phase2_model_ready\", DEVICE)\n    model, best_map, best_epoch, best_path = run_training_phase(\n        phase=\"phase2\",\n        model=model,\n        train_dataset=train_dataset,\n        val_dataset=val_dataset,\n        model_config=model_config,\n        class_weights=CLASS_WEIGHTS,\n        device=DEVICE,\n        class_names=CLASS_NAMES,\n        sampler=phase2_sampler,\n    )\n    summary = {\n        \"status\": \"complete\",\n        \"phase\": \"phase2\",\n        \"best_metric\": float(best_map),\n        \"best_epoch\": int(best_epoch),\n        \"best_checkpoint\": str(best_path),\n    }\n    write_phase_completion_marker(\"phase2\", summary)\n    choose_best_stage1_checkpoint()\n    run_artifact_backup(\"phase2_complete\", include_final_artifacts=False)\n    return summary\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.295491Z","iopub.status.idle":"2026-05-20T15:49:26.295982Z","shell.execute_reply.started":"2026-05-20T15:49:26.295704Z","shell.execute_reply":"2026-05-20T15:49:26.295731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 18. Finalization Artifacts\n","metadata":{}},{"cell_type":"code","source":"def build_eval_loaders(val_dataset, calib_dataset, model_config):\n    eval_bs = max(1, int(model_config[\"physical_bs\"]) * int(CONFIG[\"eval_batch_multiplier\"]))\n    val_loader = DataLoader(val_dataset, batch_size=eval_bs, shuffle=False, **loader_kwargs())\n    calib_loader = DataLoader(calib_dataset, batch_size=eval_bs, shuffle=False, **loader_kwargs())\n    return val_loader, calib_loader\n\n\ndef collect_finalize_outputs(model, val_dataset, calib_dataset, model_config):\n    criterion = make_criterion(CLASS_WEIGHTS, DEVICE)\n    val_loader, calib_loader = build_eval_loaders(val_dataset, calib_dataset, model_config)\n    val_loss_raw, val_metrics_raw = validate(model, val_loader, criterion, DEVICE, CLASS_NAMES)\n    val_logits, val_targets, val_ids = collect_logits_and_targets(model, val_loader, DEVICE)\n    calib_logits, calib_targets, calib_ids = collect_logits_and_targets(model, calib_loader, DEVICE)\n    return {\n        \"val_loss_raw\": val_loss_raw,\n        \"val_metrics_raw\": val_metrics_raw,\n        \"val_logits\": val_logits,\n        \"val_targets\": val_targets,\n        \"val_ids\": val_ids,\n        \"calib_logits\": calib_logits,\n        \"calib_targets\": calib_targets,\n        \"calib_ids\": calib_ids,\n    }\n\ndef fit_and_score_stage1_calibration(outputs):\n    temperature, calib_nll = fit_temperature_scaler(\n        calib_logits=outputs[\"calib_logits\"],\n        calib_targets_soft=outputs[\"calib_targets\"],\n        device=DEVICE,\n        max_iter=CONFIG[\"temperature_max_iter\"],\n    )\n    calib_probs_cal = sigmoid_numpy(apply_temperature_numpy(outputs[\"calib_logits\"], temperature))\n    thresholds = tune_per_class_thresholds(\n        calib_probs_cal,\n        outputs[\"calib_targets\"],\n        threshold_grid=CONFIG[\"threshold_grid\"],\n        positive_threshold=CONFIG[\"positive_threshold\"],\n        min_eval_positive=CONFIG[\"min_eval_positive\"],\n    )\n    val_probs_cal = sigmoid_numpy(apply_temperature_numpy(outputs[\"val_logits\"], temperature))\n    val_metrics_cal = compute_metrics_from_probs(\n        val_probs_cal,\n        outputs[\"val_targets\"],\n        CLASS_NAMES,\n        thresholds=thresholds,\n        positive_threshold=CONFIG[\"positive_threshold\"],\n        min_eval_positive=CONFIG[\"min_eval_positive\"],\n    )\n    calib_metrics_cal = compute_metrics_from_probs(\n        calib_probs_cal,\n        outputs[\"calib_targets\"],\n        CLASS_NAMES,\n        thresholds=thresholds,\n        positive_threshold=CONFIG[\"positive_threshold\"],\n        min_eval_positive=CONFIG[\"min_eval_positive\"],\n    )\n    return {\n        \"temperature\": temperature,\n        \"calib_nll\": calib_nll,\n        \"thresholds\": thresholds,\n        \"val_metrics_cal\": val_metrics_cal,\n        \"calib_metrics_cal\": calib_metrics_cal,\n    }\n\ndef write_prediction_artifacts(outputs, calibration):\n    temperature = calibration[\"temperature\"]\n    thresholds = calibration[\"thresholds\"]\n    build_prediction_dataframe(outputs[\"val_ids\"], \"val_select\", outputs[\"val_targets\"], outputs[\"val_logits\"], temperature, thresholds, CLASS_NAMES).to_csv(PATHS[\"val_predictions\"], index=False)\n    build_prediction_dataframe(outputs[\"calib_ids\"], \"calib\", outputs[\"calib_targets\"], outputs[\"calib_logits\"], temperature, thresholds, CLASS_NAMES).to_csv(PATHS[\"calib_predictions\"], index=False)\n    per_class_metrics_dataframe(calibration[\"val_metrics_cal\"], \"val_select\").to_csv(PATHS[\"per_class_val\"], index=False)\n    per_class_metrics_dataframe(calibration[\"calib_metrics_cal\"], \"calib\").to_csv(PATHS[\"per_class_calib\"], index=False)\n\ndef write_calibration_artifacts(outputs, calibration):\n    thresholds_payload = {\n        \"threshold_grid\": CONFIG[\"threshold_grid\"],\n        \"selection_split\": \"calib\",\n        \"selection_metric\": \"per-class F1 on majority-positive p_soft >= 0.5\",\n        \"class_names\": CLASS_NAMES,\n        \"thresholds\": calibration[\"thresholds\"].tolist(),\n    }\n    write_json(PATHS[\"thresholds\"], thresholds_payload)\n    calibration_payload = {\n        \"policy\": \"global_temperature_baseline\",\n        \"fit_split\": \"calib\",\n        \"target\": \"majority_positive_p_soft_ge_0_5\",\n        \"temperature\": calibration[\"temperature\"],\n        \"calib_nll\": calibration[\"calib_nll\"],\n        \"raw_vs_calibrated_on_val_select\": {\n            \"raw\": outputs[\"val_metrics_raw\"],\n            \"calibrated\": calibration[\"val_metrics_cal\"],\n        },\n        \"calibrated_on_calib\": calibration[\"calib_metrics_cal\"],\n        \"refinement_note\": \"Per-class temperature or Platt scaling is conditional refinement, not run in Stage 1 Lite baseline.\",\n    }\n    write_json(PATHS[\"calibration_summary\"], calibration_payload)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.296885Z","iopub.status.idle":"2026-05-20T15:49:26.298822Z","shell.execute_reply.started":"2026-05-20T15:49:26.298572Z","shell.execute_reply":"2026-05-20T15:49:26.298606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def stage1_preprocessing_steps():\n    steps = [\"pydicom.dcmread\"]\n    if str(CONFIG.get(\"preprocessing_mode\", \"\")).startswith(\"modality_\"):\n        steps.append(\"apply_modality_lut\")\n    steps.extend([\n        \"apply_voi_lut\",\n        \"MONOCHROME1 inversion when needed\",\n        \"normalize to configured cache bit depth\",\n        \"Lanczos resize to model img_size\",\n        \"replicate grayscale to 3 RGB-like channels\",\n        \"ImageNet normalization\",\n    ])\n    return steps\n\n\ndef phase_metric_summary():\n    phase1 = checkpoint_summary(PATHS[\"best_phase1_checkpoint\"], expected_phase=\"phase1\")\n    phase2 = checkpoint_summary(PATHS[\"best_phase2_checkpoint\"], expected_phase=\"phase2\")\n    return phase1, phase2\n\ndef write_model_summary(model, calibration):\n    model_name = CONFIG[\"run_model_name\"]\n    model_config = CONFIG[\"model_configs\"][model_name]\n    phase1, phase2 = phase_metric_summary()\n    summary_row = {\n        \"run_id\": CONFIG[\"run_id\"],\n        \"spec_version\": CONFIG[\"spec_version\"],\n        \"stage\": \"1\",\n        \"model_name\": model_name,\n        \"display_name\": model_config[\"display_name\"],\n        \"resolved_timm_name\": model.timm_name,\n        \"img_size\": model_config[\"img_size\"],\n        \"run_profile\": CONFIG[\"run_profile\"],\n        \"phase1_best_map\": None if phase1 is None else phase1[\"best_metric\"],\n        \"phase1_best_epoch\": None if phase1 is None else phase1[\"best_epoch\"],\n        \"phase2_best_map\": None if phase2 is None else phase2[\"best_metric\"],\n        \"phase2_best_epoch\": None if phase2 is None else phase2[\"best_epoch\"],\n        \"val_select_macro_mAP_calibrated\": calibration[\"val_metrics_cal\"][\"macro_mAP\"],\n        \"val_select_AUROC_macro_calibrated\": calibration[\"val_metrics_cal\"][\"macro_AUROC\"],\n        \"val_select_F1_macro_calibrated\": calibration[\"val_metrics_cal\"][\"macro_F1\"],\n        \"val_select_ECE_calibrated\": calibration[\"val_metrics_cal\"][\"ECE\"],\n        \"val_select_Brier_calibrated\": calibration[\"val_metrics_cal\"][\"Brier\"],\n        \"calib_ECE_calibrated\": calibration[\"calib_metrics_cal\"][\"ECE\"],\n        \"calib_Brier_calibrated\": calibration[\"calib_metrics_cal\"][\"Brier\"],\n        \"temperature\": calibration[\"temperature\"],\n        \"best_checkpoint\": str(PATHS[\"best_stage1_checkpoint\"]),\n        \"elapsed_hours\": elapsed_hours(),\n    }\n    pd.DataFrame([summary_row]).to_csv(PATHS[\"model_summary\"], index=False)\n    return summary_row\n\ndef build_stage1_bundle(model, calibration):\n    model_name = CONFIG[\"run_model_name\"]\n    model_config = CONFIG[\"model_configs\"][model_name]\n    return {\n        \"output_schema_version\": CONFIG[\"output_schema_version\"],\n        \"spec_version\": CONFIG[\"spec_version\"],\n        \"system_spec_version\": CONFIG[\"system_spec_version\"],\n        \"run_id\": CONFIG[\"run_id\"],\n        \"stage\": \"stage1_plus_1_5_lite\",\n        \"model\": {\n            \"model_name\": model_name,\n            \"display_name\": model_config[\"display_name\"],\n            \"resolved_timm_name\": model.timm_name,\n            \"family\": model_config[\"family\"],\n            \"checkpoint_path\": str(PATHS[\"best_stage1_checkpoint\"]),\n            \"img_size\": model_config[\"img_size\"],\n        },\n        \"class_names\": CLASS_NAMES,\n        \"input_contract\": {\n            \"source\": \"DICOM CXR\",\n            \"preprocessing_mode\": CONFIG.get(\"preprocessing_mode\"),\n            \"preprocessing\": stage1_preprocessing_steps(),\n        },\n        \"prediction_contract\": {\n            \"outputs\": [\n                \"logit_per_class\",\n                \"probability_raw_per_class\",\n                \"probability_calibrated_per_class\",\n                \"decision_threshold_per_class\",\n                \"pred_binary_per_class\",\n                \"confidence_margin_per_class\",\n                \"location.status_per_class\",\n            ],\n            \"location\": {\n                \"status_allowed_values\": [\"localized\", \"weak_localization\", \"detector_negative\", \"not_routed\", \"not_localizable\", \"deferred\"],\n                \"stage1_default_without_heatmap\": \"not_routed\",\n                \"no_fake_box_policy\": True,\n            },\n            \"confidence_policy\": \"margin_based_stage1_only\",\n            \"confidence_definition\": \"abs(p_cal - threshold) normalized by max(threshold, 1-threshold); later stages may down-weight confidence with conflict/risk flags.\",\n        },\n        \"calibration\": {\n            \"summary_path\": str(PATHS[\"calibration_summary\"]),\n            \"thresholds_path\": str(PATHS[\"thresholds\"]),\n            \"temperature\": calibration[\"temperature\"],\n            \"thresholds\": calibration[\"thresholds\"].tolist(),\n            \"fit_split\": \"calib\",\n        },\n        \"weak_localization_policy\": {\n            \"generation_mode\": \"post_training_only\",\n            \"cnn\": \"gradcam\",\n            \"vit\": \"attention_rollout\",\n            \"selected_method\": model_config[\"weak_localization_method\"],\n            \"top_k\": CONFIG[\"weak_localization_top_k\"],\n            \"near_threshold_margin\": CONFIG[\"near_threshold_margin\"],\n            \"allowed_triggers\": [\"top_k_positive\", \"near_threshold\", \"debug_or_error_analysis\"],\n            \"not_detector_replacement\": True,\n        },\n        \"ambiguity_policy\": {\n            \"ambiguity_flag_conditions\": [\n                \"abs(p_cal - threshold) <= near_threshold_margin\",\n                \"weak map diffuse or unstable in post-training analysis\",\n                \"classifier positive but Stage 2 has no strong location evidence\",\n            ],\n            \"near_threshold_margin\": CONFIG[\"near_threshold_margin\"],\n        },\n        \"preprocessed_cache_policy\": {\n            \"enabled\": CONFIG[\"preprocessed_cache_enabled\"],\n            \"generation_mode\": \"prebuild_cache_job_or_lazy_fill_on_cache_miss\",\n            \"format\": CONFIG[\"preprocessed_cache_format\"],\n            \"lossless_container\": True,\n            \"dtype\": CONFIG[\"preprocessed_cache_dtype\"],\n            \"max_value\": CONFIG[\"preprocessed_cache_max_value\"],\n            \"cache_version\": CONFIG[\"preprocessed_cache_version\"],\n            \"write_cache_dir\": str(PATHS[\"preprocessed_cache\"]),\n            \"read_cache_roots\": CONFIG[\"preprocessed_cache_read_roots\"],\n            \"min_free_gb\": CONFIG[\"preprocessed_cache_min_free_gb\"],\n            \"summary_path\": str(PATHS[\"preprocessed_cache_summary\"]),\n            \"not_in_survival_pack\": True,\n        },\n        \"artifacts\": {\n            \"config\": str(PATHS[\"config\"]),\n            \"runtime\": str(PATHS[\"runtime\"]),\n            \"package_versions\": str(PATHS[\"package_versions\"]),\n            \"split_manifest\": str(PATHS[\"split_manifest\"]),\n            \"class_weights\": str(PATHS[\"class_weights\"]),\n            \"preprocessed_cache_summary\": str(PATHS[\"preprocessed_cache_summary\"]),\n            \"epoch_log\": str(PATHS[\"epoch_log\"]),\n            \"val_select_predictions\": str(PATHS[\"val_predictions\"]),\n            \"calib_predictions\": str(PATHS[\"calib_predictions\"]),\n            \"model_summary\": str(PATHS[\"model_summary\"]),\n            \"per_class_val_select\": str(PATHS[\"per_class_val\"]),\n            \"per_class_calib\": str(PATHS[\"per_class_calib\"]),\n            \"best_stage1_checkpoint\": str(PATHS[\"best_stage1_checkpoint\"]),\n            \"latest_survival_pack\": str(PATHS[\"latest_survival_pack\"]),\n            \"backup_manifest\": str(PATHS[\"backup_manifest\"]),\n        },\n        \"persistence_policy\": {\n            \"survival_pack\": str(PATHS[\"latest_survival_pack\"]),\n            \"backup_manifest\": str(PATHS[\"backup_manifest\"]),\n            \"kaggle_dataset_ref\": CONFIG.get(\"backup_kaggle_dataset_ref\", \"\"),\n            \"kaggle_dataset_enabled\": CONFIG.get(\"backup_kaggle_dataset_enabled\", False),\n        },\n        \"spec_constraints\": SPEC_CONSTRAINTS,\n        \"holdout_eval_policy\": CONFIG[\"holdout_eval_policy\"],\n    }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.299941Z","iopub.status.idle":"2026-05-20T15:49:26.30035Z","shell.execute_reply.started":"2026-05-20T15:49:26.300125Z","shell.execute_reply":"2026-05-20T15:49:26.300163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_finalize_job():\n    if not PATHS[\"best_stage1_checkpoint\"].exists():\n        choose_best_stage1_checkpoint()\n    model_config = CONFIG[\"model_configs\"][CONFIG[\"run_model_name\"]]\n    _, val_dataset, calib_dataset = build_stage1_datasets(include_calib=True)\n    model = build_stage1_model(pretrained=False)\n    best_payload = safe_torch_load(PATHS[\"best_stage1_checkpoint\"], map_location=\"cpu\")\n    validate_checkpoint_contract(best_payload, checkpoint_path=PATHS[\"best_stage1_checkpoint\"])\n    model.load_state_dict(best_payload[\"model_state_dict\"])\n    log_cuda_memory(\"finalize_model_ready\", DEVICE)\n    outputs = collect_finalize_outputs(model, val_dataset, calib_dataset, model_config)\n    calibration = fit_and_score_stage1_calibration(outputs)\n    write_prediction_artifacts(outputs, calibration)\n    write_calibration_artifacts(outputs, calibration)\n    write_model_summary(model, calibration)\n    stage1_bundle = build_stage1_bundle(model, calibration)\n    write_json(PATHS[\"stage1_bundle\"], stage1_bundle)\n    shutil.copy2(PATHS[\"best_stage1_checkpoint\"], PATHS[\"best_stage1_model\"])\n    shutil.copy2(PATHS[\"stage1_bundle\"], PATHS[\"best_stage1_bundle\"])\n    run_artifact_backup(\"finalize_complete\", include_final_artifacts=True)\n    print(f\"Saved Stage 1 checkpoint: {PATHS['best_stage1_checkpoint']}\")\n    print(f\"Saved Stage 1.5 bundle: {PATHS['stage1_bundle']}\")\n    print(f\"Copied best artifacts to: {BEST_DIR}\")\n    return stage1_bundle\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.30192Z","iopub.status.idle":"2026-05-20T15:49:26.302337Z","shell.execute_reply.started":"2026-05-20T15:49:26.302125Z","shell.execute_reply":"2026-05-20T15:49:26.302148Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 19. Run Dispatcher\n","metadata":{}},{"cell_type":"code","source":"def run_stage1_job():\n    restore_status = restore_survival_pack_if_requested()\n    mode = CONFIG[\"job_mode\"]\n    assert_training_persistence_ready(mode)\n    if mode == \"cache\":\n        return {\"restore\": restore_status, \"cache\": run_cache_job()}\n    if mode == \"phase1\":\n        return {\"restore\": restore_status, \"phase1\": run_phase1_job()}\n    if mode == \"phase2\":\n        return {\"restore\": restore_status, \"phase2\": run_phase2_job()}\n    if mode == \"finalize\":\n        return {\"restore\": restore_status, \"finalize\": run_finalize_job()}\n    if mode == \"all\":\n        cache = run_cache_job()\n        phase1 = run_phase1_job()\n        phase2 = run_phase2_job()\n        finalize = run_finalize_job()\n        return {\"restore\": restore_status, \"cache\": cache, \"phase1\": phase1, \"phase2\": phase2, \"finalize\": finalize}\n    raise ValueError(f\"Unsupported STAGE1_JOB_MODE: {mode}\")\n\ntry:\n    STAGE1_RESULT = run_stage1_job()\nexcept Exception as exc:\n    write_failure_summary(exc, context={\"cell\": \"run_stage1_job\", \"run_id\": CONFIG[\"run_id\"], \"job_mode\": CONFIG[\"job_mode\"]})\n    if CONFIG.get(\"failure_backup_dataset_ref\", \"\"):\n        try:\n            run_artifact_backup(\"failure\", include_final_artifacts=False)\n        except Exception as backup_exc:\n            print(f\"WARNING: failure backup could not be created: {backup_exc}\")\n    else:\n        print(\"Failure backup dataset is not set; skipping failure pack upload so checkpoint backups cannot be hidden by failure artifacts.\")\n    raise\nfinally:\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\nprint(\"Stage 1 phase runner completed.\")\nprint(json.dumps(to_serializable(STAGE1_RESULT), indent=2, ensure_ascii=False)[:4000])\nprint(f\"Run directory: {RUN_DIR}\")\nprint(f\"Latest survival pack: {PATHS['latest_survival_pack']}\")\nprint(f\"Best checkpoint: {PATHS['best_stage1_model']}\")\nprint(f\"Bundle: {PATHS['stage1_bundle']}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.303675Z","iopub.status.idle":"2026-05-20T15:49:26.304034Z","shell.execute_reply.started":"2026-05-20T15:49:26.303863Z","shell.execute_reply":"2026-05-20T15:49:26.303884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nimport json\nfrom datetime import datetime, timezone\nfrom IPython.display import FileLink, display\nfrom kaggle_secrets import UserSecretsClient\nfrom kaggle.api.kaggle_api_extended import KaggleApi\n\n# 1. Khai báo thư mục đích\nOUTPUT_DIR = \"/kaggle/working/vindr_stage1_v41\"\nZIP_FILENAME = \"vindr_final_output\"\nZIP_PATH = f\"/kaggle/working/{ZIP_FILENAME}.zip\"\n\n# ==========================================\n# CHỨC NĂNG 1: TẠO LIÊN KẾT TẢI VỀ MÁY LOCAL\n# ==========================================\nif os.path.exists(OUTPUT_DIR):\n    print(f\"Đang nén thư mục {OUTPUT_DIR}...\")\n    shutil.make_archive(f\"/kaggle/working/{ZIP_FILENAME}\", 'zip', OUTPUT_DIR)\n    \n    print(\"\\n[DOWNLOAD LOCAL] Bấm vào liên kết bên dưới để tải về máy tính:\")\n    display(FileLink(f'{ZIP_FILENAME}.zip'))\nelse:\n    print(f\"Không tìm thấy thư mục {OUTPUT_DIR} để nén.\")\n\n# ==========================================\n# CHỨC NĂNG 2: XUẤT RA KAGGLE DATASET\n# ==========================================\nprint(\"\\n[KAGGLE DATASET] Đang đồng bộ lên đám mây...\")\ntry:\n    # Lấy định danh từ Secrets\n    user_secrets = UserSecretsClient()\n    os.environ['KAGGLE_USERNAME'] = user_secrets.get_secret(\"KAGGLE_USERNAME\")\n    os.environ['KAGGLE_KEY'] = user_secrets.get_secret(\"KAGGLE_KEY\")\n    \n    # Xác thực API\n    api = KaggleApi()\n    api.authenticate()\n    \n    # Cấu hình Dataset định danh\n    dataset_slug = \"vindr-stage1-phase1-output-final\"\n    dataset_ref = f\"{os.environ['KAGGLE_USERNAME']}/{dataset_slug}\"\n    \n    # Khởi tạo Metadata trực tiếp trong thư mục Output\n    metadata = {\n        \"title\": \"VinDr Stage 1 Final Output\",\n        \"id\": dataset_ref,\n        \"licenses\": [{\"name\": \"unknown\"}]\n    }\n    with open(os.path.join(OUTPUT_DIR, \"dataset-metadata.json\"), \"w\") as f:\n        json.dump(metadata, f, indent=4)\n        \n    # Kiểm tra trạng thái tồn tại của Dataset trên hệ thống\n    try:\n        api.dataset_status(dataset_ref)\n        dataset_exists = True\n    except Exception:\n        dataset_exists = False\n        \n    # Đẩy dữ liệu qua API (Sử dụng dir_mode=\"zip\" để tối ưu băng thông)\n    timestamp = datetime.now(timezone.utc).isoformat()\n    if dataset_exists:\n        api.dataset_create_version(\n            OUTPUT_DIR, \n            version_notes=f\"Auto final backup at {timestamp}\", \n            dir_mode=\"zip\"\n        )\n        print(f\"[Thành công] Đã cập nhật Version mới tại: https://www.kaggle.com/datasets/{dataset_ref}\")\n    else:\n        api.dataset_create_new(\n            OUTPUT_DIR, \n            dir_mode=\"zip\"\n        )\n        print(f\"[Thành công] Đã khởi tạo Dataset mới tại: https://www.kaggle.com/datasets/{dataset_ref}\")\n\nexcept Exception as e:\n    print(f\"[Thất bại] Quá trình xuất Kaggle Dataset gặp lỗi: {e}\")\n    print(\"Vui lòng kiểm tra lại quyền truy cập Internet của Notebook và cấu hình Secrets.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.304729Z","iopub.status.idle":"2026-05-20T15:49:26.305095Z","shell.execute_reply.started":"2026-05-20T15:49:26.30491Z","shell.execute_reply":"2026-05-20T15:49:26.304933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import shutil\n# from pathlib import Path\n\n# # Xóa thư mục nén lỗi ở vùng working\n# broken_dir = Path(\"/kaggle/working/vindr_dino1022_uint16_cache_dataset\")\n# if broken_dir.exists():\n#     shutil.rmtree(broken_dir)\n#     print(\"Đã dọn sạch file rác trong /kaggle/working!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.305817Z","iopub.status.idle":"2026-05-20T15:49:26.306197Z","shell.execute_reply.started":"2026-05-20T15:49:26.306001Z","shell.execute_reply":"2026-05-20T15:49:26.306025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from pathlib import Path\n# import json, zipfile\n\n# cache_dir = Path(PATHS[\"preprocessed_cache\"])\n# assert cache_dir.exists(), cache_dir\n\n# pngs = sorted(cache_dir.glob(\"*.png\"))\n# print(\"Thư mục cache:\", cache_dir)\n# print(\"Số lượng ảnh:\", len(pngs))\n\n# # QUAN TRỌNG: Đổi đích đến sang vùng /kaggle/temp để không bị giới hạn 20GB\n# publish_dir = Path(\"/kaggle/temp/vindr_dino1022_uint16_cache_dataset\")\n# publish_dir.mkdir(parents=True, exist_ok=True)\n\n# zip_path = publish_dir / f\"{cache_dir.name}.zip\"\n\n# print(f\"Đang tiến hành nén vào vùng an toàn: {zip_path} ...\")\n# with zipfile.ZipFile(zip_path, \"w\", compression=zipfile.ZIP_STORED) as zf:\n#     for p in pngs:\n#         zf.write(p, arcname=f\"{cache_dir.name}/{p.name}\")\n\n# # Tạo Dataset Metadata\n# metadata = {\n#     \"title\": \"VinDr CXR Stage1 DINO 1022 UInt16 Cache\",\n#     \"id\": \"mixp123/vindr-cxr-stage1-dino1022-uint16-cache\",\n#     \"licenses\": [{\"name\": \"unknown\"}],\n#     \"subtitle\": \"Preprocessed uint16 PNG cache for Stage 1 DINO candidate\",\n#     \"description\": f\"Contains {cache_dir.name}.zip for reuse as Stage 1 DINO img_size=1022 cache.\"\n# }\n# (publish_dir / \"dataset-metadata.json\").write_text(json.dumps(metadata, indent=2), encoding=\"utf-8\")\n\n# print(\"Nén hoàn tất! Chuẩn bị upload lên Kaggle Dataset...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.308323Z","iopub.status.idle":"2026-05-20T15:49:26.308728Z","shell.execute_reply.started":"2026-05-20T15:49:26.30851Z","shell.execute_reply":"2026-05-20T15:49:26.308533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Upload thẳng từ /kaggle/temp, không chạm tới vùng /kaggle/working bị giới hạn\n!kaggle datasets create -p /kaggle/temp/vindr_dino1022_uint16_cache_dataset --dir-mode skip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-20T15:49:26.309405Z","iopub.status.idle":"2026-05-20T15:49:26.309804Z","shell.execute_reply.started":"2026-05-20T15:49:26.309575Z","shell.execute_reply":"2026-05-20T15:49:26.309597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}