{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nRSNA ICH — FULL TRAINING RUN (single architecture per run)\n=================================================================\nScaled up from the two-architecture smoke test. Key differences:\n\n  - Uses ALL SIX preprocessed PNG datasets (3 negative parts, 3\n    positive parts) instead of the 2 used for the smoke test.\n  - Only the competition's stage_2_train.csv is read from the\n    competitions directory — no PNGs are pulled from there.\n  - Size filter raised from 5KB to 40KB. The preprocessed PNGs\n    at or below 40KB were found to carry very little usable\n    signal (near-blank / mostly-background slices) — same idea\n    as the smoke test's 5KB filter, just recalibrated after\n    looking at more of the actual data.\n  - Full train/val split (not a 300/100 subsample) — this is a\n    real run, not a pipeline check.\n  - Progress bars (tqdm) on directory scanning and on each\n    train/eval epoch. tqdm's overhead is negligible (a few ms of\n    string formatting per batch) — it does not meaningfully\n    affect training speed.\n  - Trains ONE model per run, chosen via MODEL_NAME below. Per\n    our last exchange: one model at a time, sequentially — not\n    both in the same process. Run this script once per\n    architecture, changing MODEL_NAME between runs.\n\nEverything else — AMP, ViT warmup + cosine annealing, early\nstopping, best-model checkpointing, epoch-wise LR logging — carries\nover unchanged from the smoke test script.\n\"\"\"\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Sampler\nfrom torchvision import models, transforms\nfrom PIL import Image\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.auto import tqdm\n\n# ------------------------------------------------------------------\n# CONFIG\n# ------------------------------------------------------------------\n\n# Pick ONE architecture per run. Run the script again with the other\n# value to train the second model — don't train both in one process.\nMODEL_NAME = \"resnet50\"   # \"resnet50\" or \"vit_b16\"\n\n# All six preprocessed PNG datasets. No PNGs are read from the\n# competitions directory — only its CSV is used (see TRAIN_CSV below).\nINPUT_DIRS = [\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part2\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-positive-part-1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs\",\n]\n\n# Only the CSV is read from this directory — no image files.\nTRAIN_CSV = (\"/kaggle/input/competitions/rsna-intracranial-hemorrhage-detection/\"\n             \"rsna-intracranial-hemorrhage-detection/stage_2_train.csv\")\n\n# Files at or below this size are dropped as near-blank / low-information.\n# Raised from the smoke test's 5KB after a closer look at the actual data —\n# adjust here if you find the cutoff needs further tuning.\nMIN_FILE_SIZE_KB = 40\nMIN_FILE_SIZE_BYTES = MIN_FILE_SIZE_KB * 1024\n\nLABEL_COLS = [\"any\", \"epidural\", \"intraparenchymal\",\n              \"intraventricular\", \"subarachnoid\", \"subdural\"]\n\nBATCH_SIZE = 32\nNUM_EPOCHS = 15           # real budget, not the smoke test's 3 — early stopping\n                           # will likely cut this short before it completes\nLR = 1e-4\nVAL_SPLIT = 0.15\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nUSE_AMP = DEVICE.type == \"cuda\"\n\nEARLY_STOPPING_PATIENCE = 5\nCHECKPOINT_DIR = \"/kaggle/working/checkpoints\"\n\nNUM_WORKERS = 4\n\n\n# ------------------------------------------------------------------\n# LABELS — pivot the raw RSNA csv the same way as the rest of the pipeline\n# ------------------------------------------------------------------\ndef load_labels(csv_path):\n    y = pd.read_csv(csv_path)\n    id_split = y.ID.str.rsplit(\"_\", n=1, expand=True)\n    y = pd.concat([id_split, y.Label], axis=1)\n    y.columns = [\"id\", \"sub_type\", \"label\"]\n    y = y.drop_duplicates(subset=[\"id\", \"sub_type\"])\n    df = y.pivot(index=\"id\", columns=\"sub_type\", values=\"label\")\n    return df\n\n\n# ------------------------------------------------------------------\n# DATASET\n# ------------------------------------------------------------------\nclass ICHDataset(Dataset):\n    def __init__(self, df, id_to_path, transform=None):\n        self.df = df\n        self.id_to_path = id_to_path\n        self.transform = transform\n        self.ids = df.index.values\n        self.labels = df[LABEL_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        img_id = self.ids[idx]\n        path = self.id_to_path[img_id]\n        img = Image.open(path).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        label = torch.tensor(self.labels[idx], dtype=torch.float32)\n        return img, label\n\n\nclass BalancedRandomSampler(Sampler):\n    def __init__(self, labels_any):\n        self.pos_idx = np.where(labels_any == 1)[0]\n        self.neg_idx = np.where(labels_any == 0)[0]\n\n    def __iter__(self):\n        n = min(len(self.neg_idx), max(len(self.pos_idx), 1))\n        neg_sample = np.random.choice(self.neg_idx, n, replace=False)\n        pos_sample = (np.random.choice(self.pos_idx, n, replace=False)\n                      if len(self.pos_idx) > 0 else np.array([], dtype=int))\n        ids = np.concatenate([pos_sample, neg_sample])\n        np.random.shuffle(ids)\n        return iter(ids.tolist())\n\n    def __len__(self):\n        return min(len(self.neg_idx), max(len(self.pos_idx), 1)) * 2\n\n\n# ------------------------------------------------------------------\n# MODELS\n# ------------------------------------------------------------------\ndef build_resnet50(num_classes=len(LABEL_COLS)):\n    model = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2)\n    for param in model.parameters():\n        param.requires_grad = False\n    for param in model.layer4.parameters():\n        param.requires_grad = True\n    model.fc = nn.Sequential(\n        nn.Dropout(0.3),\n        nn.Linear(model.fc.in_features, num_classes),\n    )\n    return model.to(DEVICE)\n\n\ndef build_vit_b16(num_classes=len(LABEL_COLS)):\n    model = models.vit_b_16(weights=models.ViT_B_16_Weights.IMAGENET1K_V1)\n    for param in model.parameters():\n        param.requires_grad = False\n    for param in model.encoder.layers[-1].parameters():\n        param.requires_grad = True\n    model.heads = nn.Sequential(\n        nn.Dropout(0.3),\n        nn.Linear(model.hidden_dim, num_classes),\n    )\n    for param in model.heads.parameters():\n        param.requires_grad = True\n    return model.to(DEVICE)\n\n\nMODEL_BUILDERS = {\n    \"resnet50\": build_resnet50,\n    \"vit_b16\": build_vit_b16,\n}\n\nOPTIMIZER_CONFIG = {\n    \"resnet50\": {\"type\": \"sgd\", \"lr\": 1e-3, \"momentum\": 0.9},\n    \"vit_b16\": {\"type\": \"adamw\", \"lr\": 1e-4, \"weight_decay\": 1e-4},\n}\n\nSCHEDULER_CONFIG = {\n    \"resnet50\": {\"warmup_epochs\": 0},\n    \"vit_b16\": {\"warmup_epochs\": 1},\n}\n\n\ndef build_optimizer(model, model_name):\n    cfg = OPTIMIZER_CONFIG[model_name]\n    trainable_params = filter(lambda p: p.requires_grad, model.parameters())\n    if cfg[\"type\"] == \"sgd\":\n        return torch.optim.SGD(trainable_params, lr=cfg[\"lr\"], momentum=cfg[\"momentum\"])\n    elif cfg[\"type\"] == \"adamw\":\n        return torch.optim.AdamW(trainable_params, lr=cfg[\"lr\"],\n                                  weight_decay=cfg.get(\"weight_decay\", 0.0))\n    else:\n        raise ValueError(f\"Unknown optimizer type: {cfg['type']}\")\n\n\ndef build_scheduler(optimizer, model_name, num_epochs):\n    warmup_epochs = SCHEDULER_CONFIG[model_name][\"warmup_epochs\"]\n    warmup_epochs = min(warmup_epochs, max(num_epochs - 1, 0))\n\n    if warmup_epochs > 0:\n        warmup = torch.optim.lr_scheduler.LinearLR(\n            optimizer, start_factor=0.1, end_factor=1.0, total_iters=warmup_epochs\n        )\n        cosine = torch.optim.lr_scheduler.CosineAnnealingLR(\n            optimizer, T_max=max(num_epochs - warmup_epochs, 1)\n        )\n        scheduler = torch.optim.lr_scheduler.SequentialLR(\n            optimizer, schedulers=[warmup, cosine], milestones=[warmup_epochs]\n        )\n    else:\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n\n    return scheduler\n\n\n# ------------------------------------------------------------------\n# TRAIN / EVAL — architecture-agnostic, AMP-enabled, progress-bar'd\n# ------------------------------------------------------------------\ndef train_one_epoch(model, loader, optimizer, criterion, scaler, epoch, num_epochs):\n    model.train()\n    total_loss = 0.0\n    pbar = tqdm(loader, desc=f\"train epoch {epoch+1}/{num_epochs}\", leave=False)\n    for imgs, labels in pbar:\n        imgs, labels = imgs.to(DEVICE), labels.to(DEVICE)\n        optimizer.zero_grad()\n\n        with torch.autocast(device_type=DEVICE.type, enabled=USE_AMP):\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        total_loss += loss.item() * imgs.size(0)\n        pbar.set_postfix(loss=f\"{loss.item():.4f}\")\n    return total_loss / len(loader.dataset)\n\n\n@torch.no_grad()\ndef evaluate(model, loader, criterion, epoch, num_epochs):\n    model.eval()\n    total_loss = 0.0\n    all_preds, all_labels = [], []\n    pbar = tqdm(loader, desc=f\"val epoch {epoch+1}/{num_epochs}\", leave=False)\n    for imgs, labels in pbar:\n        imgs, labels = imgs.to(DEVICE), labels.to(DEVICE)\n\n        with torch.autocast(device_type=DEVICE.type, enabled=USE_AMP):\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n\n        total_loss += loss.item() * imgs.size(0)\n        all_preds.append(torch.sigmoid(outputs.float()).cpu().numpy())\n        all_labels.append(labels.cpu().numpy())\n\n    all_preds = np.concatenate(all_preds)\n    all_labels = np.concatenate(all_labels)\n\n    aucs = {}\n    for i, name in enumerate(LABEL_COLS):\n        if len(np.unique(all_labels[:, i])) > 1:\n            aucs[name] = roc_auc_score(all_labels[:, i], all_preds[:, i])\n        else:\n            aucs[name] = float(\"nan\")\n\n    return total_loss / len(loader.dataset), aucs\n\n\ndef run_training(model_name, train_ds, val_ds, y_train):\n    print(f\"\\n{'='*60}\")\n    print(f\"FULL TRAINING RUN: {model_name}\")\n    print(f\"{'='*60}\")\n\n    model = MODEL_BUILDERS[model_name]()\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = build_optimizer(model, model_name)\n    scheduler = build_scheduler(optimizer, model_name, NUM_EPOCHS)\n    scaler = torch.amp.GradScaler(device=DEVICE.type, enabled=USE_AMP)\n\n    sampler = BalancedRandomSampler(y_train[\"any\"].values)\n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, sampler=sampler,\n                               num_workers=NUM_WORKERS, pin_memory=True)\n    val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False,\n                             num_workers=NUM_WORKERS, pin_memory=True)\n\n    os.makedirs(CHECKPOINT_DIR, exist_ok=True)\n    checkpoint_path = os.path.join(CHECKPOINT_DIR, f\"{model_name}_best.pt\")\n\n    best_val_loss = float(\"inf\")\n    epochs_no_improve = 0\n\n    for epoch in range(NUM_EPOCHS):\n        current_lr = optimizer.param_groups[0][\"lr\"]\n\n        train_loss = train_one_epoch(model, train_loader, optimizer, criterion,\n                                      scaler, epoch, NUM_EPOCHS)\n        val_loss, val_aucs = evaluate(model, val_loader, criterion, epoch, NUM_EPOCHS)\n\n        auc_str = \", \".join(f\"{k}={v:.3f}\" for k, v in val_aucs.items())\n        print(f\"[{model_name}] Epoch {epoch+1}/{NUM_EPOCHS} | lr={current_lr:.2e} | \"\n              f\"train_loss={train_loss:.4f} | val_loss={val_loss:.4f} | {auc_str}\")\n\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            epochs_no_improve = 0\n            torch.save({\n                \"model_name\": model_name,\n                \"epoch\": epoch + 1,\n                \"model_state_dict\": model.state_dict(),\n                \"optimizer_state_dict\": optimizer.state_dict(),\n                \"val_loss\": val_loss,\n                \"val_aucs\": val_aucs,\n            }, checkpoint_path)\n            print(f\"[{model_name}]   -> val loss improved, checkpoint saved to {checkpoint_path}\")\n        else:\n            epochs_no_improve += 1\n            print(f\"[{model_name}]   -> no improvement ({epochs_no_improve}/{EARLY_STOPPING_PATIENCE})\")\n\n        scheduler.step()\n\n        if epochs_no_improve >= EARLY_STOPPING_PATIENCE:\n            print(f\"[{model_name}] Early stopping triggered at epoch {epoch+1}.\")\n            break\n\n    print(f\"[{model_name}] Training complete. Best val_loss={best_val_loss:.4f}, \"\n          f\"checkpoint at {checkpoint_path}\")\n\n\ndef scan_input_dirs(input_dirs, min_size_bytes):\n    \"\"\"\n    Scans multiple mounted dataset directories for PNGs, filters out\n    files at or below min_size_bytes (near-blank / low-information\n    slices), and returns a dict mapping image_id -> full file path.\n    \"\"\"\n    id_to_path = {}\n    skipped_small = 0\n    for d in input_dirs:\n        if not os.path.isdir(d):\n            print(f\"WARNING: directory not found, skipping: {d}\")\n            continue\n        filenames = [f for f in os.listdir(d) if f.endswith(\".png\")]\n        for f in tqdm(filenames, desc=f\"scanning {os.path.basename(d)}\"):\n            full_path = os.path.join(d, f)\n            if os.path.getsize(full_path) <= min_size_bytes:\n                skipped_small += 1\n                continue\n            img_id = f[:-4]\n            id_to_path[img_id] = full_path\n    print(f\"Scanned {len(input_dirs)} directories: {len(id_to_path)} usable PNGs \"\n          f\"found, {skipped_small} filtered out as low-information (<= \"\n          f\"{min_size_bytes} bytes / {MIN_FILE_SIZE_KB}KB).\")\n    return id_to_path\n\n\ndef main():\n    print(f\"*** FULL TRAINING RUN: {MODEL_NAME} ***\")\n    print(f\"Using device: {DEVICE} | AMP enabled: {USE_AMP}\")\n\n    id_to_path = scan_input_dirs(INPUT_DIRS, MIN_FILE_SIZE_BYTES)\n\n    if len(id_to_path) == 0:\n        print(\"\\nNo usable PNGs found across INPUT_DIRS. Check the paths are \"\n              \"correct and the datasets are actually attached as Input.\")\n        return\n\n    df = load_labels(TRAIN_CSV)\n    df = df[df.index.isin(id_to_path.keys())]\n    print(f\"Matched {len(df)} of those to labels. \"\n          f\"Positive rate: {df['any'].mean():.4f}\")\n\n    train_df, val_df = train_test_split(\n        df, test_size=VAL_SPLIT, stratify=df[\"any\"], random_state=42\n    )\n    print(f\"Full run sizes — train: {len(train_df)}, val: {len(val_df)}\")\n\n    train_transform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                              std=[0.229, 0.224, 0.225]),\n    ])\n    eval_transform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                              std=[0.229, 0.224, 0.225]),\n    ])\n\n    train_ds = ICHDataset(train_df, id_to_path, transform=train_transform)\n    val_ds = ICHDataset(val_df, id_to_path, transform=eval_transform)\n\n    run_training(MODEL_NAME, train_ds, val_ds, train_df)\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-07-19T11:25:40.432657Z","iopub.execute_input":"2026-07-19T11:25:40.433012Z","iopub.status.idle":"2026-07-19T15:10:29.993203Z","shell.execute_reply.started":"2026-07-19T11:25:40.432983Z","shell.execute_reply":"2026-07-19T15:10:29.992089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nPOST-TRAINING EVALUATION — run this in the cell right after training.\n=================================================================\nLoads the best checkpoint saved by full_training_run.py and computes\nthe full metrics table on the held-out val set:\n\n  - AUC-ROC per class + macro/micro summary          (what you asked for)\n  - PR-AUC (Average Precision) per class              (more honest than\n                                                        ROC-AUC under\n                                                        class imbalance,\n                                                        esp. for epidural)\n  - Per-class threshold chosen by max-F1 on the PR curve, then at that\n    threshold: Precision, Recall/Sensitivity, Specificity, F1\n  - Confusion matrix per class at that threshold\n\nNo calibration (ECE/reliability diagram) here — not needed yet, can\nbe added back later as its own cell whenever it's actually wanted.\n\nDoes NOT recompute MSE — not a relevant metric for multi-label\nclassification with BCE loss; dropped as discussed.\n\nTries to reuse val_ds already sitting in memory from the training\ncell (fastest path — no re-scanning). If that's not present (fresh\nkernel / restarted session), rebuilds everything needed from scratch\nusing the definitions inlined below — this file no longer depends on\nfull_training_run.py existing on disk, since you ran that as pasted\ncells rather than a saved .py file.\n\"\"\"\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import (\n    roc_auc_score, average_precision_score, precision_recall_curve,\n    confusion_matrix, f1_score,\n)\n\n# ------------------------------------------------------------------\n# CONFIG — must match full_training_run.py for a valid reload\n# ------------------------------------------------------------------\nMODEL_NAME = \"resnet50\"\nCHECKPOINT_PATH = f\"/kaggle/working/checkpoints/{MODEL_NAME}_best.pt\"\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nUSE_AMP = DEVICE.type == \"cuda\"\n\nINPUT_DIRS = [\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part2\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-positive-part-1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs\",\n]\nTRAIN_CSV = (\"/kaggle/input/competitions/rsna-intracranial-hemorrhage-detection/\"\n             \"rsna-intracranial-hemorrhage-detection/stage_2_train.csv\")\nMIN_FILE_SIZE_KB = 40\nMIN_FILE_SIZE_BYTES = MIN_FILE_SIZE_KB * 1024\nVAL_SPLIT = 0.15\n\nLABEL_COLS = [\"any\", \"epidural\", \"intraparenchymal\",\n              \"intraventricular\", \"subarachnoid\", \"subdural\"]\n\n\nclass ICHDataset(Dataset):\n    def __init__(self, df, id_to_path, transform=None):\n        self.df = df\n        self.id_to_path = id_to_path\n        self.transform = transform\n        self.ids = df.index.values\n        self.labels = df[LABEL_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        img_id = self.ids[idx]\n        path = self.id_to_path[img_id]\n        img = Image.open(path).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        label = torch.tensor(self.labels[idx], dtype=torch.float32)\n        return img, label\n\n\ndef build_resnet50(num_classes=len(LABEL_COLS)):\n    model = models.resnet50(weights=None)  # weights loaded from checkpoint below\n    model.fc = nn.Sequential(nn.Dropout(0.3), nn.Linear(model.fc.in_features, num_classes))\n    return model.to(DEVICE)\n\n\ndef build_vit_b16(num_classes=len(LABEL_COLS)):\n    model = models.vit_b_16(weights=None)  # weights loaded from checkpoint below\n    model.heads = nn.Sequential(nn.Dropout(0.3), nn.Linear(model.hidden_dim, num_classes))\n    return model.to(DEVICE)\n\n\nMODEL_BUILDERS = {\"resnet50\": build_resnet50, \"vit_b16\": build_vit_b16}\n\n\ndef load_labels(csv_path):\n    y = pd.read_csv(csv_path)\n    id_split = y.ID.str.rsplit(\"_\", n=1, expand=True)\n    y = pd.concat([id_split, y.Label], axis=1)\n    y.columns = [\"id\", \"sub_type\", \"label\"]\n    y = y.drop_duplicates(subset=[\"id\", \"sub_type\"])\n    return y.pivot(index=\"id\", columns=\"sub_type\", values=\"label\")\n\n\ndef scan_input_dirs(input_dirs, min_size_bytes):\n    id_to_path = {}\n    for d in input_dirs:\n        if not os.path.isdir(d):\n            print(f\"WARNING: directory not found, skipping: {d}\")\n            continue\n        for f in os.listdir(d):\n            if not f.endswith(\".png\"):\n                continue\n            full_path = os.path.join(d, f)\n            if os.path.getsize(full_path) <= min_size_bytes:\n                continue\n            id_to_path[f[:-4]] = full_path\n    return id_to_path\n\n\n# ------------------------------------------------------------------\n# REBUILD val_ds/val_loader IF NOT ALREADY IN MEMORY, ELSE REUSE\n# ------------------------------------------------------------------\nif \"val_ds\" in globals():\n    print(\"Reusing val_ds already in memory from the training cell.\")\nelse:\n    print(\"val_ds not found in memory — rebuilding the val split from scratch \"\n          \"(same random_state=42, same 40KB filter, same 85/15 split).\")\n    id_to_path = scan_input_dirs(INPUT_DIRS, MIN_FILE_SIZE_BYTES)\n    df = load_labels(TRAIN_CSV)\n    df = df[df.index.isin(id_to_path.keys())]\n    train_df, val_df = train_test_split(\n        df, test_size=VAL_SPLIT, stratify=df[\"any\"], random_state=42\n    )\n    eval_transform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                              std=[0.229, 0.224, 0.225]),\n    ])\n    val_ds = ICHDataset(val_df, id_to_path, transform=eval_transform)\n\nval_loader = DataLoader(val_ds, batch_size=32, shuffle=False,\n                         num_workers=2, pin_memory=True)\n\n\n# ------------------------------------------------------------------\n# LOAD BEST CHECKPOINT\n# ------------------------------------------------------------------\nmodel = MODEL_BUILDERS[MODEL_NAME]()\ncheckpoint = torch.load(CHECKPOINT_PATH, map_location=DEVICE, weights_only=False)\nmodel.load_state_dict(checkpoint[\"model_state_dict\"])\nmodel.eval()\nprint(f\"Loaded checkpoint: epoch {checkpoint['epoch']}, \"\n      f\"val_loss={checkpoint['val_loss']:.4f}\")\n\n\n# ------------------------------------------------------------------\n# RUN INFERENCE ON VAL SET ONCE\n# ------------------------------------------------------------------\nall_probs, all_labels = [], []\nwith torch.no_grad():\n    for imgs, labels in val_loader:\n        imgs = imgs.to(DEVICE)\n        with torch.autocast(device_type=DEVICE.type, enabled=USE_AMP):\n            outputs = model(imgs)\n        probs = torch.sigmoid(outputs.float()).cpu().numpy()\n        all_probs.append(probs)\n        all_labels.append(labels.numpy())\n\nall_probs = np.concatenate(all_probs)     # (N, 6)\nall_labels = np.concatenate(all_labels)   # (N, 6)\nprint(f\"Inference complete on {len(all_probs)} val images.\")\n\n\n# ------------------------------------------------------------------\n# PER-CLASS METRICS\n# ------------------------------------------------------------------\ndef best_f1_threshold(y_true, y_prob):\n    \"\"\"Threshold that maximizes F1 on the PR curve for this class.\"\"\"\n    precisions, recalls, thresholds = precision_recall_curve(y_true, y_prob)\n    f1s = 2 * precisions * recalls / (precisions + recalls + 1e-12)\n    best_idx = np.nanargmax(f1s[:-1])  # last point has no matching threshold\n    return thresholds[best_idx] if len(thresholds) > 0 else 0.5\n\n\nrows = []\nconfusion_matrices = {}\n\nfor i, name in enumerate(LABEL_COLS):\n    y_true = all_labels[:, i]\n    y_prob = all_probs[:, i]\n\n    if len(np.unique(y_true)) < 2:\n        continue  # can't compute ROC/PR AUC with only one class present\n\n    roc_auc = roc_auc_score(y_true, y_prob)\n    pr_auc = average_precision_score(y_true, y_prob)\n    thresh = best_f1_threshold(y_true, y_prob)\n    y_pred = (y_prob >= thresh).astype(int)\n\n    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n    precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0\n    recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0          # sensitivity\n    specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0\n    f1 = f1_score(y_true, y_pred, zero_division=0)\n\n    confusion_matrices[name] = np.array([[tn, fp], [fn, tp]])\n\n    rows.append({\n        \"class\": name,\n        \"roc_auc\": roc_auc,\n        \"pr_auc\": pr_auc,\n        \"threshold\": thresh,\n        \"precision\": precision,\n        \"recall_sensitivity\": recall,\n        \"specificity\": specificity,\n        \"f1\": f1,\n        \"n_positive\": int(y_true.sum()),\n        \"n_total\": len(y_true),\n    })\n\nresults_df = pd.DataFrame(rows).set_index(\"class\")\n\n# Macro / micro AUC summary across all 6 classes at once\nmacro_auc = roc_auc_score(all_labels, all_probs, average=\"macro\")\nmicro_auc = roc_auc_score(all_labels, all_probs, average=\"micro\")\n\nprint(\"\\n\" + \"=\" * 90)\nprint(f\"PER-CLASS METRICS — {MODEL_NAME}, checkpoint epoch {checkpoint['epoch']}\")\nprint(\"=\" * 90)\nprint(results_df.round(4).to_string())\nprint(f\"\\nMacro-average ROC-AUC: {macro_auc:.4f}\")\nprint(f\"Micro-average ROC-AUC: {micro_auc:.4f}\")\n\n\n# ------------------------------------------------------------------\n# CONFUSION MATRICES\n# ------------------------------------------------------------------\nprint(\"\\n\" + \"=\" * 90)\nprint(\"CONFUSION MATRICES (rows=true, cols=predicted, order=[neg, pos])\")\nprint(\"=\" * 90)\nfor name, cm in confusion_matrices.items():\n    print(f\"\\n{name}:\")\n    print(f\"           pred_neg  pred_pos\")\n    print(f\"true_neg   {cm[0,0]:>8}  {cm[0,1]:>8}\")\n    print(f\"true_pos   {cm[1,0]:>8}  {cm[1,1]:>8}\")\n\n\n# ------------------------------------------------------------------\n# SAVE FULL TABLE FOR THE PAPER\n# ------------------------------------------------------------------\nout_path = f\"/kaggle/working/{MODEL_NAME}_eval_metrics.csv\"\nresults_df.to_csv(out_path)\nprint(f\"\\nFull metrics table saved to {out_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T15:48:13.492014Z","iopub.execute_input":"2026-07-19T15:48:13.492584Z","iopub.status.idle":"2026-07-19T15:51:19.454504Z","shell.execute_reply.started":"2026-07-19T15:48:13.492546Z","shell.execute_reply":"2026-07-19T15:51:19.453508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nCALIBRATION ANALYSIS — run after eval_metrics.py, same session.\n=================================================================\nComputes, before and after temperature scaling:\n  - ECE (Expected Calibration Error)\n  - MCE (Maximum Calibration Error)\n  - Brier score\n  - NLL (sanity check that the T fit actually helped)\n  - Reliability diagram + confidence histogram\n  - ROC-AUC and PR-AUC (discrimination doesn't change under temperature\n    scaling — it's a monotonic transform — but reported here anyway so\n    this cell's table stands alone without eval_metrics.py's)\n\nAlso reports the fitted temperature parameter T.\n\nMETHODOLOGY NOTE — read before trusting the numbers:\nTemperature scaling is FIT on one half of the val set and all metrics\n(before AND after) are REPORTED on the other, held-out half. Fitting T\nand reporting ECE on the same data would make the \"after\" numbers\noptimistic by construction — T is chosen specifically to minimize error\non whatever data it's fit on. This splits val 50/50 (stratified on\n\"any\") into calib_df (fit T) and holdout_df (report everything) so\nbefore/after is an honest comparison. This also means \"before\" and\n\"after\" ECE/Brier/ROC-AUC here are on a SMALLER set than the full val\nnumbers eval_metrics.py reported — expect some numerical difference\nfrom that script, not an error.\n\nMULTI-LABEL CAVEAT: standard temperature scaling (Guo et al. 2017) is\ndefined for single-label softmax outputs. This fits ONE scalar T\napplied jointly across all 6 sigmoid logits, minimizing summed BCE\nover the calibration split — the standard multi-label adaptation, but\nworth stating explicitly in the paper since it's not the literal\noriginal formulation.\n\"\"\"\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, average_precision_score, brier_score_loss\n\n# ------------------------------------------------------------------\n# CONFIG\n# ------------------------------------------------------------------\nMODEL_NAME = \"resnet50\"\nCHECKPOINT_PATH = f\"/kaggle/working/checkpoints/{MODEL_NAME}_best.pt\"\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nUSE_AMP = DEVICE.type == \"cuda\"\nN_BINS = 15\nCALIB_SPLIT_SEED = 123   # separate seed from the train/val split (42)\n\nINPUT_DIRS = [\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part2\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-negative-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-positive-part-1\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs-part3\",\n    \"/kaggle/input/datasets/anushakirand/rsna-ich-preprocessed-pngs\",\n]\nTRAIN_CSV = (\"/kaggle/input/competitions/rsna-intracranial-hemorrhage-detection/\"\n             \"rsna-intracranial-hemorrhage-detection/stage_2_train.csv\")\nMIN_FILE_SIZE_BYTES = 40 * 1024\nVAL_SPLIT = 0.15\nLABEL_COLS = [\"any\", \"epidural\", \"intraparenchymal\",\n              \"intraventricular\", \"subarachnoid\", \"subdural\"]\n\n\nclass ICHDataset(Dataset):\n    def __init__(self, df, id_to_path, transform=None):\n        self.df, self.id_to_path, self.transform = df, id_to_path, transform\n        self.ids = df.index.values\n        self.labels = df[LABEL_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.ids)\n\n    def __getitem__(self, idx):\n        img_id = self.ids[idx]\n        img = Image.open(self.id_to_path[img_id]).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        return img, torch.tensor(self.labels[idx], dtype=torch.float32)\n\n\ndef build_resnet50(num_classes=len(LABEL_COLS)):\n    m = models.resnet50(weights=None)\n    m.fc = nn.Sequential(nn.Dropout(0.3), nn.Linear(m.fc.in_features, num_classes))\n    return m.to(DEVICE)\n\n\ndef build_vit_b16(num_classes=len(LABEL_COLS)):\n    m = models.vit_b_16(weights=None)\n    m.heads = nn.Sequential(nn.Dropout(0.3), nn.Linear(m.hidden_dim, num_classes))\n    return m.to(DEVICE)\n\n\nMODEL_BUILDERS = {\"resnet50\": build_resnet50, \"vit_b16\": build_vit_b16}\n\n\ndef load_labels(csv_path):\n    y = pd.read_csv(csv_path)\n    id_split = y.ID.str.rsplit(\"_\", n=1, expand=True)\n    y = pd.concat([id_split, y.Label], axis=1)\n    y.columns = [\"id\", \"sub_type\", \"label\"]\n    y = y.drop_duplicates(subset=[\"id\", \"sub_type\"])\n    return y.pivot(index=\"id\", columns=\"sub_type\", values=\"label\")\n\n\ndef scan_input_dirs(input_dirs, min_size_bytes):\n    id_to_path = {}\n    for d in input_dirs:\n        if not os.path.isdir(d):\n            print(f\"WARNING: directory not found, skipping: {d}\")\n            continue\n        for f in os.listdir(d):\n            if f.endswith(\".png\"):\n                full_path = os.path.join(d, f)\n                if os.path.getsize(full_path) > min_size_bytes:\n                    id_to_path[f[:-4]] = full_path\n    return id_to_path\n\n\n# ------------------------------------------------------------------\n# GET val_df / id_to_path (reuse from memory if present, else rebuild)\n# ------------------------------------------------------------------\nif \"val_df\" in globals() and \"id_to_path\" in globals():\n    print(\"Reusing val_df / id_to_path already in memory.\")\nelse:\n    print(\"val_df not found in memory — rebuilding from scratch \"\n          \"(same random_state=42, same 40KB filter, same 85/15 split).\")\n    id_to_path = scan_input_dirs(INPUT_DIRS, MIN_FILE_SIZE_BYTES)\n    df = load_labels(TRAIN_CSV)\n    df = df[df.index.isin(id_to_path.keys())]\n    _, val_df = train_test_split(df, test_size=VAL_SPLIT, stratify=df[\"any\"], random_state=42)\n\n# Split val into calib (fit T) / holdout (report everything) — stratified, separate seed\ncalib_df, holdout_df = train_test_split(\n    val_df, test_size=0.5, stratify=val_df[\"any\"], random_state=CALIB_SPLIT_SEED\n)\nprint(f\"Calibration split sizes — calib (fit T): {len(calib_df)}, \"\n      f\"holdout (report on): {len(holdout_df)}\")\n\neval_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\ncalib_ds = ICHDataset(calib_df, id_to_path, transform=eval_transform)\nholdout_ds = ICHDataset(holdout_df, id_to_path, transform=eval_transform)\ncalib_loader = DataLoader(calib_ds, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)\nholdout_loader = DataLoader(holdout_ds, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)\n\n\n# ------------------------------------------------------------------\n# LOAD MODEL (reuse if already in memory and matches, else load checkpoint)\n# ------------------------------------------------------------------\nif \"model\" in globals() and \"checkpoint\" in globals() and checkpoint.get(\"model_name\") == MODEL_NAME:\n    print(\"Reusing model already loaded in memory.\")\nelse:\n    model = MODEL_BUILDERS[MODEL_NAME]()\n    checkpoint = torch.load(CHECKPOINT_PATH, map_location=DEVICE, weights_only=False)\n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    print(f\"Loaded checkpoint: epoch {checkpoint['epoch']}, val_loss={checkpoint['val_loss']:.4f}\")\nmodel.eval()\n\n\n@torch.no_grad()\ndef get_logits(loader):\n    all_logits, all_labels = [], []\n    for imgs, labels in loader:\n        imgs = imgs.to(DEVICE)\n        with torch.autocast(device_type=DEVICE.type, enabled=USE_AMP):\n            outputs = model(imgs)\n        all_logits.append(outputs.float().cpu())\n        all_labels.append(labels)\n    return torch.cat(all_logits), torch.cat(all_labels)\n\n\nprint(\"Running inference on calib split...\")\ncalib_logits, calib_labels = get_logits(calib_loader)\nprint(\"Running inference on holdout split...\")\nholdout_logits, holdout_labels = get_logits(holdout_loader)\n\n\n# ------------------------------------------------------------------\n# FIT TEMPERATURE SCALING ON calib SPLIT ONLY\n# ------------------------------------------------------------------\nclass TemperatureScaler(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.log_T = nn.Parameter(torch.zeros(1))  # T starts at 1.0, optimized in log-space for stability\n\n    def forward(self, logits):\n        return logits / torch.exp(self.log_T)\n\n\nscaler = TemperatureScaler()\noptimizer = torch.optim.LBFGS([scaler.log_T], lr=0.05, max_iter=100)\nbce = nn.BCEWithLogitsLoss()\n\ndef closure():\n    optimizer.zero_grad()\n    loss = bce(scaler(calib_logits), calib_labels)\n    loss.backward()\n    return loss\n\noptimizer.step(closure)\nT = torch.exp(scaler.log_T).item()\nprint(f\"\\nFitted temperature: T = {T:.4f}  \"\n      f\"({'softens' if T > 1 else 'sharpens'} the model's confidence)\")\n\n\n# ------------------------------------------------------------------\n# METRICS — computed on holdout ONLY, before vs after scaling\n# ------------------------------------------------------------------\ndef expected_and_max_calibration_error(y_true, y_prob, n_bins=15):\n    bin_edges = np.linspace(0, 1, n_bins + 1)\n    ece, mce = 0.0, 0.0\n    bin_accs, bin_confs, bin_counts = [], [], []\n    for lo, hi in zip(bin_edges[:-1], bin_edges[1:]):\n        mask = (y_prob > lo) & (y_prob <= hi)\n        count = mask.sum()\n        if count == 0:\n            bin_accs.append(np.nan)\n            bin_confs.append((lo + hi) / 2)\n            bin_counts.append(0)\n            continue\n        acc = y_true[mask].mean()\n        conf = y_prob[mask].mean()\n        gap = abs(acc - conf)\n        ece += (count / len(y_prob)) * gap\n        mce = max(mce, gap)\n        bin_accs.append(acc)\n        bin_confs.append(conf)\n        bin_counts.append(count)\n    return ece, mce, bin_edges, bin_accs, bin_confs, bin_counts\n\n\ndef nll(y_true, y_prob):\n    eps = 1e-12\n    p = np.clip(y_prob, eps, 1 - eps)\n    return -np.mean(y_true * np.log(p) + (1 - y_true) * np.log(1 - p))\n\n\nholdout_labels_np = holdout_labels.numpy()\nprobs_before = torch.sigmoid(holdout_logits).numpy()\nprobs_after = torch.sigmoid(holdout_logits / T).numpy()\n\nrows = []\nfor i, name in enumerate(LABEL_COLS):\n    y_true = holdout_labels_np[:, i]\n    if len(np.unique(y_true)) < 2:\n        continue\n\n    p_before, p_after = probs_before[:, i], probs_after[:, i]\n\n    ece_b, mce_b, *_ = expected_and_max_calibration_error(y_true, p_before, N_BINS)\n    ece_a, mce_a, *_ = expected_and_max_calibration_error(y_true, p_after, N_BINS)\n\n    rows.append({\n        \"class\": name,\n        \"roc_auc\": roc_auc_score(y_true, p_before),          # invariant under temp scaling\n        \"pr_auc\": average_precision_score(y_true, p_before),  # invariant under temp scaling\n        \"ece_before\": ece_b, \"ece_after\": ece_a,\n        \"mce_before\": mce_b, \"mce_after\": mce_a,\n        \"brier_before\": brier_score_loss(y_true, p_before),\n        \"brier_after\": brier_score_loss(y_true, p_after),\n        \"nll_before\": nll(y_true, p_before),\n        \"nll_after\": nll(y_true, p_after),\n    })\n\ncalib_results_df = pd.DataFrame(rows).set_index(\"class\")\nprint(\"\\n\" + \"=\" * 100)\nprint(f\"CALIBRATION METRICS (holdout split, n={len(holdout_df)}) — {MODEL_NAME}, T={T:.4f}\")\nprint(\"=\" * 100)\nprint(calib_results_df.round(4).to_string())\n\nmacro_ece_before = calib_results_df[\"ece_before\"].mean()\nmacro_ece_after = calib_results_df[\"ece_after\"].mean()\nmacro_brier_before = calib_results_df[\"brier_before\"].mean()\nmacro_brier_after = calib_results_df[\"brier_after\"].mean()\nprint(f\"\\nMacro-avg ECE:   before={macro_ece_before:.4f} -> after={macro_ece_after:.4f}\")\nprint(f\"Macro-avg Brier: before={macro_brier_before:.4f} -> after={macro_brier_after:.4f}\")\n\n\n# ------------------------------------------------------------------\n# RELIABILITY DIAGRAM + CONFIDENCE HISTOGRAM — \"any\" class, before vs after\n# ------------------------------------------------------------------\nany_idx = LABEL_COLS.index(\"any\")\ny_true_any = holdout_labels_np[:, any_idx]\np_before_any = probs_before[:, any_idx]\np_after_any = probs_after[:, any_idx]\n\nece_b, mce_b, edges_b, accs_b, confs_b, counts_b = expected_and_max_calibration_error(\n    y_true_any, p_before_any, N_BINS)\nece_a, mce_a, edges_a, accs_a, confs_a, counts_a = expected_and_max_calibration_error(\n    y_true_any, p_after_any, N_BINS)\n\nfig, axes = plt.subplots(2, 2, figsize=(11, 9))\nbin_centers = (edges_b[:-1] + edges_b[1:]) / 2\nbin_width = 1 / N_BINS\n\nfor col, (accs, counts, ece, mce, label) in enumerate([\n    (accs_b, counts_b, ece_b, mce_b, \"Before temp scaling (T=1.0)\"),\n    (accs_a, counts_a, ece_a, mce_a, f\"After temp scaling (T={T:.3f})\"),\n]):\n    valid = [not np.isnan(a) for a in accs]\n    ax_rel = axes[0, col]\n    ax_rel.bar(np.array(bin_centers)[valid], np.array(accs)[valid],\n               width=bin_width, edgecolor=\"black\", alpha=0.7, label=\"Model\")\n    ax_rel.plot([0, 1], [0, 1], \"k--\", label=\"Perfect calibration\")\n    ax_rel.set_xlabel(\"Predicted probability\")\n    ax_rel.set_ylabel(\"Observed frequency\")\n    ax_rel.set_title(f\"{label}\\nECE={ece:.4f}  MCE={mce:.4f}\")\n    ax_rel.legend(fontsize=8)\n\n    ax_hist = axes[1, col]\n    ax_hist.bar(bin_centers, counts, width=bin_width, edgecolor=\"black\", alpha=0.7, color=\"gray\")\n    ax_hist.set_xlabel(\"Predicted probability\")\n    ax_hist.set_ylabel(\"Count\")\n    ax_hist.set_title(\"Confidence histogram\")\n\nplt.suptitle(f\"Calibration — {MODEL_NAME}, class 'any' (holdout, n={len(holdout_df)})\", y=1.00)\nplt.tight_layout()\nplt.savefig(f\"/kaggle/working/{MODEL_NAME}_calibration_reliability.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\n\n\n# ------------------------------------------------------------------\n# SAVE FOR THE PAPER\n# ------------------------------------------------------------------\nout_path = f\"/kaggle/working/{MODEL_NAME}_calibration_metrics.csv\"\ncalib_results_df.to_csv(out_path)\nwith open(f\"/kaggle/working/{MODEL_NAME}_temperature.txt\", \"w\") as f:\n    f.write(f\"T={T:.6f}\\n\")\nprint(f\"\\nSaved calibration table to {out_path}\")\nprint(f\"Saved fitted T to /kaggle/working/{MODEL_NAME}_temperature.txt\")\nprint(f\"Saved reliability diagram to /kaggle/working/{MODEL_NAME}_calibration_reliability.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T16:03:15.681604Z","iopub.execute_input":"2026-07-19T16:03:15.682106Z","iopub.status.idle":"2026-07-19T16:06:27.133502Z","shell.execute_reply.started":"2026-07-19T16:03:15.682056Z","shell.execute_reply":"2026-07-19T16:06:27.132707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nZIP + DOWNLOAD EVERYTHING IN /kaggle/working\n=================================================================\nBundles the whole working directory (checkpoints, CSVs, PNGs, the\ntemperature .txt, everything) into one zip file, then gives you a\nclickable download link right in the notebook output.\n\nNote: this includes the model checkpoints (.pt files), which can be\na few hundred MB each (ResNet-50 ~100MB, ViT-B/16 ~330MB, per model).\nIf you only want the small stuff (metrics CSVs, PNGs, temperature\nvalue) and not the checkpoints, see the EXCLUDE_CHECKPOINTS option\nbelow.\n\"\"\"\n\nimport os\nimport shutil\nfrom IPython.display import FileLink\n\nSOURCE_DIR = \"/kaggle/working\"\nZIP_NAME = \"kaggle_working_backup\"   # -> kaggle_working_backup.zip\nEXCLUDE_CHECKPOINTS = False           # set True to skip .pt files (smaller zip)\n\nif EXCLUDE_CHECKPOINTS:\n    # Build the zip manually so we can skip .pt files\n    import zipfile\n    zip_path = f\"/kaggle/working/{ZIP_NAME}.zip\"\n    with zipfile.ZipFile(zip_path, \"w\", zipfile.ZIP_DEFLATED) as zf:\n        for root, dirs, files in os.walk(SOURCE_DIR):\n            for f in files:\n                if f.endswith(\".pt\"):\n                    continue\n                full_path = os.path.join(root, f)\n                if full_path == zip_path:   # don't zip the zip itself\n                    continue\n                arcname = os.path.relpath(full_path, SOURCE_DIR)\n                zf.write(full_path, arcname)\nelse:\n    # Simple full-directory zip, checkpoints included\n    zip_path = shutil.make_archive(f\"/kaggle/working/{ZIP_NAME}\", \"zip\", SOURCE_DIR)\n\nsize_mb = os.path.getsize(zip_path) / (1024 * 1024)\nprint(f\"Zip created: {zip_path}  ({size_mb:.1f} MB)\")\nprint(\"Click below to download:\")\n\nFileLink(zip_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T17:17:14.259193Z","iopub.execute_input":"2026-07-19T17:17:14.259562Z","iopub.status.idle":"2026-07-19T17:17:22.01134Z","shell.execute_reply.started":"2026-07-19T17:17:14.25952Z","shell.execute_reply":"2026-07-19T17:17:22.010399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nROC / PR CURVES + CONFUSION MATRIX HEATMAPS — run after eval_metrics.py.\n=================================================================\nReuses all_probs / all_labels from the eval_metrics.py cell if still\nin memory (fastest — no re-inference). Falls back to recomputing\ninference on val_ds if that's not present.\n\nProduces:\n  1. ROC curves, all 6 classes overlaid (shared 0.5 baseline)\n  2. PR curves, one subplot per class, each with its own no-skill\n     baseline (= that class's prevalence) — NOT overlaid, since the\n     baseline differs per class and overlaying would be misleading\n  3. Grouped bar chart: ROC-AUC vs PR-AUC per class — makes the\n     imbalance-driven gap (e.g. epidural: ROC-AUC 0.88, PR-AUC 0.11)\n     visually obvious rather than just a table row\n  4. Confusion matrix heatmaps, one per class, at the same per-class\n     max-F1 thresholds used in eval_metrics.py\n\"\"\"\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, precision_recall_curve, auc, average_precision_score\n\nMODEL_NAME = \"resnet50\"\nLABEL_COLS = [\"any\", \"epidural\", \"intraparenchymal\",\n              \"intraventricular\", \"subarachnoid\", \"subdural\"]\n\n# ------------------------------------------------------------------\n# GET all_probs / all_labels (reuse if present, else recompute)\n# ------------------------------------------------------------------\nif \"all_probs\" in globals() and \"all_labels\" in globals():\n    print(\"Reusing all_probs / all_labels already in memory from eval_metrics.py.\")\nelse:\n    print(\"all_probs / all_labels not found — recomputing inference on val_ds...\")\n    import torch\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    USE_AMP = DEVICE.type == \"cuda\"\n    if \"model\" not in globals():\n        raise RuntimeError(\n            \"No model in memory either — run eval_metrics.py first so the \"\n            \"checkpoint is loaded, then run this cell.\"\n        )\n    from torch.utils.data import DataLoader\n    val_loader = DataLoader(val_ds, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)\n    all_probs_list, all_labels_list = [], []\n    model.eval()\n    with torch.no_grad():\n        for imgs, labels in val_loader:\n            imgs = imgs.to(DEVICE)\n            with torch.autocast(device_type=DEVICE.type, enabled=USE_AMP):\n                outputs = model(imgs)\n            all_probs_list.append(torch.sigmoid(outputs.float()).cpu().numpy())\n            all_labels_list.append(labels.numpy())\n    all_probs = np.concatenate(all_probs_list)\n    all_labels = np.concatenate(all_labels_list)\n\n\n# ------------------------------------------------------------------\n# 1. ROC CURVES — all classes overlaid\n# ------------------------------------------------------------------\nfig, ax = plt.subplots(figsize=(7, 7))\ncolors = plt.cm.tab10(np.linspace(0, 1, len(LABEL_COLS)))\n\nfor i, (name, color) in enumerate(zip(LABEL_COLS, colors)):\n    y_true = all_labels[:, i]\n    if len(np.unique(y_true)) < 2:\n        continue\n    fpr, tpr, _ = roc_curve(y_true, all_probs[:, i])\n    roc_auc_val = auc(fpr, tpr)\n    ax.plot(fpr, tpr, color=color, label=f\"{name} (AUC={roc_auc_val:.3f})\")\n\nax.plot([0, 1], [0, 1], \"k--\", alpha=0.5, label=\"Random (AUC=0.5)\")\nax.set_xlabel(\"False Positive Rate\")\nax.set_ylabel(\"True Positive Rate\")\nax.set_title(f\"ROC Curves — {MODEL_NAME}, val set\")\nax.legend(loc=\"lower right\", fontsize=9)\nplt.tight_layout()\nplt.savefig(f\"/kaggle/working/{MODEL_NAME}_roc_curves.png\", dpi=150)\nplt.show()\n\n\n# ------------------------------------------------------------------\n# 2. PR CURVES — one subplot per class, own baseline each\n# ------------------------------------------------------------------\nfig, axes = plt.subplots(2, 3, figsize=(15, 9))\naxes = axes.flatten()\n\nfor i, name in enumerate(LABEL_COLS):\n    y_true = all_labels[:, i]\n    ax = axes[i]\n    if len(np.unique(y_true)) < 2:\n        ax.set_title(f\"{name} — only one class present, skipped\")\n        continue\n    precision, recall, _ = precision_recall_curve(y_true, all_probs[:, i])\n    pr_auc_val = average_precision_score(y_true, all_probs[:, i])\n    prevalence = y_true.mean()\n\n    ax.plot(recall, precision, color=\"C0\", label=f\"Model (PR-AUC={pr_auc_val:.3f})\")\n    ax.axhline(prevalence, color=\"gray\", linestyle=\"--\",\n               label=f\"No-skill baseline ({prevalence:.3f})\")\n    ax.set_xlabel(\"Recall\")\n    ax.set_ylabel(\"Precision\")\n    ax.set_title(f\"{name}  (n_pos={int(y_true.sum())}, prevalence={prevalence:.1%})\")\n    ax.legend(fontsize=8)\n    ax.set_ylim(0, 1.02)\n\nplt.suptitle(f\"Precision-Recall Curves — {MODEL_NAME}, val set\", y=1.01)\nplt.tight_layout()\nplt.savefig(f\"/kaggle/working/{MODEL_NAME}_pr_curves.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\n\n\n# ------------------------------------------------------------------\n# 3. ROC-AUC vs PR-AUC bar chart — makes the imbalance gap visible\n# ------------------------------------------------------------------\nroc_aucs, pr_aucs, valid_names = [], [], []\nfor i, name in enumerate(LABEL_COLS):\n    y_true = all_labels[:, i]\n    if len(np.unique(y_true)) < 2:\n        continue\n    fpr, tpr, _ = roc_curve(y_true, all_probs[:, i])\n    roc_aucs.append(auc(fpr, tpr))\n    pr_aucs.append(average_precision_score(y_true, all_probs[:, i]))\n    valid_names.append(name)\n\nx = np.arange(len(valid_names))\nwidth = 0.35\nfig, ax = plt.subplots(figsize=(9, 5))\nax.bar(x - width / 2, roc_aucs, width, label=\"ROC-AUC\", color=\"C0\")\nax.bar(x + width / 2, pr_aucs, width, label=\"PR-AUC\", color=\"C1\")\nax.set_xticks(x)\nax.set_xticklabels(valid_names, rotation=20)\nax.set_ylabel(\"Score\")\nax.set_title(f\"ROC-AUC vs PR-AUC per class — {MODEL_NAME}\\n\"\n             f\"(large gaps flag classes where ROC-AUC overstates performance under imbalance)\")\nax.legend()\nax.set_ylim(0, 1.05)\nplt.tight_layout()\nplt.savefig(f\"/kaggle/working/{MODEL_NAME}_roc_vs_pr_auc.png\", dpi=150)\nplt.show()\n\n\n# ------------------------------------------------------------------\n# 4. CONFUSION MATRIX HEATMAPS — same max-F1 thresholds as eval_metrics.py\n# ------------------------------------------------------------------\nfrom sklearn.metrics import confusion_matrix\n\nfig, axes = plt.subplots(2, 3, figsize=(15, 9))\naxes = axes.flatten()\n\nfor i, name in enumerate(LABEL_COLS):\n    y_true = all_labels[:, i]\n    ax = axes[i]\n    if len(np.unique(y_true)) < 2:\n        ax.set_title(f\"{name} — skipped\")\n        continue\n    y_prob = all_probs[:, i]\n    precisions, recalls, thresholds = precision_recall_curve(y_true, y_prob)\n    f1s = 2 * precisions * recalls / (precisions + recalls + 1e-12)\n    best_idx = np.nanargmax(f1s[:-1])\n    thresh = thresholds[best_idx] if len(thresholds) > 0 else 0.5\n    y_pred = (y_prob >= thresh).astype(int)\n\n    cm = confusion_matrix(y_true, y_pred, labels=[0, 1])\n    im = ax.imshow(cm, cmap=\"Blues\")\n    ax.set_xticks([0, 1]); ax.set_xticklabels([\"pred_neg\", \"pred_pos\"])\n    ax.set_yticks([0, 1]); ax.set_yticklabels([\"true_neg\", \"true_pos\"])\n    for r in range(2):\n        for c in range(2):\n            ax.text(c, r, f\"{cm[r, c]:,}\", ha=\"center\", va=\"center\",\n                     color=\"white\" if cm[r, c] > cm.max() / 2 else \"black\")\n    ax.set_title(f\"{name}  (thresh={thresh:.3f})\")\n\nplt.suptitle(f\"Confusion Matrices — {MODEL_NAME}, val set (max-F1 thresholds)\", y=1.01)\nplt.tight_layout()\nplt.savefig(f\"/kaggle/working/{MODEL_NAME}_confusion_matrices.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\n\nprint(\"\\nSaved: roc_curves.png, pr_curves.png, roc_vs_pr_auc.png, confusion_matrices.png \"\n      f\"(all in /kaggle/working/, prefixed {MODEL_NAME}_)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T17:20:50.56252Z","iopub.execute_input":"2026-07-19T17:20:50.562943Z","iopub.status.idle":"2026-07-19T17:20:54.476061Z","shell.execute_reply.started":"2026-07-19T17:20:50.56291Z","shell.execute_reply":"2026-07-19T17:20:54.475089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nzip_path = \"/kaggle/working/kaggle_working_backup.zip\"\n\nif os.path.exists(zip_path):\n    os.remove(zip_path)\n    print(f\"Deleted: {zip_path}\")\nelse:\n    print(\"No existing zip found.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T17:33:00.316425Z","iopub.execute_input":"2026-07-19T17:33:00.316982Z","iopub.status.idle":"2026-07-19T17:33:00.346935Z","shell.execute_reply.started":"2026-07-19T17:33:00.316947Z","shell.execute_reply":"2026-07-19T17:33:00.346245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nZIP + DOWNLOAD EVERYTHING IN /kaggle/working\n\n\"\"\"\n\nimport os\nimport shutil\nfrom IPython.display import FileLink\n\nSOURCE_DIR = \"/kaggle/working\"\nZIP_NAME = \"kaggle_working_backup\"   # -> kaggle_working_backup.zip\nEXCLUDE_CHECKPOINTS = False           # set True to skip .pt files (smaller zip)\n\nif EXCLUDE_CHECKPOINTS:\n    # Build the zip manually so we can skip .pt files\n    import zipfile\n    zip_path = f\"/kaggle/working/{ZIP_NAME}.zip\"\n    with zipfile.ZipFile(zip_path, \"w\", zipfile.ZIP_DEFLATED) as zf:\n        for root, dirs, files in os.walk(SOURCE_DIR):\n            for f in files:\n                if f.endswith(\".pt\"):\n                    continue\n                full_path = os.path.join(root, f)\n                if full_path == zip_path:   # don't zip the zip itself\n                    continue\n                arcname = os.path.relpath(full_path, SOURCE_DIR)\n                zf.write(full_path, arcname)\nelse:\n    # Simple full-directory zip, checkpoints included\n    zip_path = shutil.make_archive(f\"/kaggle/working/{ZIP_NAME}\", \"zip\", SOURCE_DIR)\n\nsize_mb = os.path.getsize(zip_path) / (1024 * 1024)\nprint(f\"Zip created: {zip_path}  ({size_mb:.1f} MB)\")\nprint(\"Click below to download:\")\n\nFileLink(zip_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-19T17:33:47.681849Z","iopub.execute_input":"2026-07-19T17:33:47.682312Z","iopub.status.idle":"2026-07-19T17:33:55.490956Z","shell.execute_reply.started":"2026-07-19T17:33:47.682277Z","shell.execute_reply":"2026-07-19T17:33:55.490251Z"}},"outputs":[],"execution_count":null}]}