{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":13451,"datasetId":654585,"databundleVersionId":1188070,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":298700524,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":298725060,"isSourceIdPinned":false}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"66e9cc8f","cell_type":"markdown","source":"# Notebook 4 — Ablation Studies\n**RSNA Intracranial Hemorrhage Detection**\n\nThis notebook runs controlled ablation experiments to justify the design decisions\nin the main training pipeline.  Each variant trains for a small number of epochs\non a fixed subset so that all experiments fit in one Kaggle session.\n\n### Experiments\n| # | Name | Variable | Options |\n|---|------|----------|---------|\n| A | Architecture | Backbone | EfficientNet-B0 vs ResNet-50 |\n| B | Windowing | Preprocessing | 3-window stack vs single window (brain only) |\n| C | Augmentation | Data augmentation | ON vs OFF |\n| D | Normalization | Normalize stats | Dataset-specific vs ImageNet |\n\n### Required input\n- Preprocessing cache (Notebook 02 output) — `manifest.csv` + `cache/` NPY arrays\n- **No pre-trained weights needed** — all variants start from ImageNet init","metadata":{}},{"id":"27e09bf7","cell_type":"code","source":"# ── 0. Config ──────────────────────────────────────────────────────────────\nimport os, gc, random, json as _json\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom torch.amp import autocast, GradScaler\nimport torchvision.transforms as T\nimport torchvision.models as models\nfrom sklearn.metrics import roc_auc_score, confusion_matrix, roc_curve\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n# ── Ablation-specific settings ────────────────────────────────────────────\nABLATION_EPOCHS   = 3         # short: just enough to see meaningful differences\nABLATION_SUBSET   = 0.15      # use 15% of data for all ablations (fast + fair)\nBATCH_SIZE        = 16\nBASE_LR           = 1e-4\nSEED              = 42\nIMG_SIZE          = 256\nNUM_WORKERS       = 4\n\nCACHE_INPUT_DIR   = '/kaggle/input/notebooks/harshitghosh/nb02eda'\nMANIFEST_PATH     = f'{CACHE_INPUT_DIR}/manifest.csv'\nNPY_CACHE_DIR     = f'{CACHE_INPUT_DIR}/cache'\n\n# CT windows (for 3-channel preprocessing)\nWINDOWS = [(40, 80), (75, 215), (40, 380)]\n\n# ─── Load dataset-specific normalization (with ImageNet fallback) ────────\n_norm_path = os.path.join(CACHE_INPUT_DIR, 'normalization_stats.json')\nif os.path.exists(_norm_path):\n    with open(_norm_path) as f:\n        _norm = _json.load(f)\n    MEAN = _norm['mean']\n    STD  = _norm['std']\n    print(f'Dataset normalization: mean={MEAN}, std={STD}')\nelse:\n    MEAN = [0.485, 0.456, 0.406]\n    STD  = [0.229, 0.224, 0.225]\n    print(f'Using ImageNet defaults: mean={MEAN}, std={STD}')\n\ndef seed_everything(s):\n    random.seed(s); np.random.seed(s)\n    torch.manual_seed(s); torch.cuda.manual_seed_all(s)\n\nseed_everything(SEED)\nprint(f'Device: {DEVICE}')","metadata":{},"outputs":[],"execution_count":null},{"id":"cb909f6e","cell_type":"code","source":"# ── Dataset classes ───────────────────────────────────────────────────────\n\nclass ICHDataset(Dataset):\n    \"\"\"Standard dataset: reads cached 3-channel windowed NPY arrays.\"\"\"\n    def __init__(self, df, npy_root, transform):\n        self.df = df.reset_index(drop=True)\n        self.npy_root  = npy_root\n        self.transform = transform\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        row  = self.df.iloc[idx]\n        path = os.path.join(self.npy_root, f'{row[\"image_id\"]}.npy')\n        try:\n            img = np.load(path)                        # uint8 H×W×3 [0,255]\n        except Exception:\n            img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)\n        return self.transform(img), torch.tensor(float(row['any']), dtype=torch.float32)\n\n\nclass ICHDatasetGray(Dataset):\n    \"\"\"Ablation: reads the 3-ch NPY but collapses to 1 channel (brain window only),\n    then replicates to 3 channels for compatibility with ImageNet pretrained backbones.\n    This simulates training WITHOUT multi-window preprocessing.\n    \"\"\"\n    def __init__(self, df, npy_root, transform):\n        self.df = df.reset_index(drop=True)\n        self.npy_root  = npy_root\n        self.transform = transform\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        row  = self.df.iloc[idx]\n        path = os.path.join(self.npy_root, f'{row[\"image_id\"]}.npy')\n        try:\n            img = np.load(path)                        # uint8 H×W×3 [0,255]\n        except Exception:\n            img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)\n        ch0 = img[:, :, 0]                             # H×W uint8 (brain window)\n        img = np.stack([ch0, ch0, ch0], axis=-1)       # replicate to 3ch\n        return self.transform(img), torch.tensor(float(row['any']), dtype=torch.float32)\n\n\n# ── Transforms (dataset-specific normalization) ──────────────────────────\naug_transform = T.Compose([\n    T.ToPILImage(),\n    T.RandomHorizontalFlip(),\n    T.RandomRotation(degrees=10),\n    T.ColorJitter(brightness=0.1, contrast=0.1),\n    T.ToTensor(),\n    T.Normalize(mean=MEAN, std=STD),\n])\n\nno_aug_transform = T.Compose([\n    T.ToPILImage(),\n    T.ToTensor(),\n    T.Normalize(mean=MEAN, std=STD),\n])\n\n# ── Transforms (ImageNet normalization — for Experiment D) ───────────────\nIMGNET_MEAN = [0.485, 0.456, 0.406]\nIMGNET_STD  = [0.229, 0.224, 0.225]\n\naug_transform_imgnet = T.Compose([\n    T.ToPILImage(),\n    T.RandomHorizontalFlip(),\n    T.RandomRotation(degrees=10),\n    T.ColorJitter(brightness=0.1, contrast=0.1),\n    T.ToTensor(),\n    T.Normalize(mean=IMGNET_MEAN, std=IMGNET_STD),\n])\n\nno_aug_transform_imgnet = T.Compose([\n    T.ToPILImage(),\n    T.ToTensor(),\n    T.Normalize(mean=IMGNET_MEAN, std=IMGNET_STD),\n])\n\nprint('Dataset classes and transforms defined.')","metadata":{},"outputs":[],"execution_count":null},{"id":"278d06ea","cell_type":"code","source":"# ── Shared utilities ──────────────────────────────────────────────────────\ndef build_model(arch: str) -> nn.Module:\n    if arch == 'efficientnet_b0':\n        m = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)\n        m.classifier = nn.Sequential(nn.Dropout(0.3), nn.Linear(m.classifier[1].in_features, 1))\n    elif arch == 'resnet50':\n        m = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)\n        m.fc = nn.Sequential(nn.Dropout(0.3), nn.Linear(m.fc.in_features, 1))\n    else:\n        raise ValueError(arch)\n    return m.to(DEVICE)\n\n\ndef make_loaders(train_df, val_df, train_ds_cls, train_transform, val_transform):\n    pos  = int(train_df['any'].sum())\n    neg  = len(train_df) - pos\n    w    = np.where(train_df['any'].values == 1, neg/pos, 1.0)\n    sampler = WeightedRandomSampler(w.tolist(), len(train_df), replacement=True)\n    tr_ds = train_ds_cls(train_df, NPY_CACHE_DIR, train_transform)\n    vl_ds = ICHDataset(val_df, NPY_CACHE_DIR, val_transform)\n    pw = dict(persistent_workers=True) if NUM_WORKERS > 0 else {}\n    tr_l  = DataLoader(tr_ds, BATCH_SIZE, sampler=sampler,\n                       num_workers=NUM_WORKERS, pin_memory=True, **pw)\n    vl_l  = DataLoader(vl_ds, BATCH_SIZE * 2, shuffle=False,\n                       num_workers=NUM_WORKERS, pin_memory=True, **pw)\n    return tr_l, vl_l, neg / pos\n\n\n@torch.no_grad()\ndef eval_loader(model, loader, criterion):\n    model.eval()\n    logits_, labels_ = [], []\n    total_loss = 0.0\n    for imgs, lbls in loader:\n        imgs_d, lbls_d = imgs.to(DEVICE), lbls.to(DEVICE)\n        with autocast(device_type='cuda'):\n            out = model(imgs_d).squeeze(1)\n            total_loss += criterion(out, lbls_d).item()\n        logits_.append(out.cpu().float()); labels_.append(lbls)\n    logits = torch.cat(logits_).numpy()\n    labels = torch.cat(labels_).numpy()\n\n    # Guard against NaN / Inf from mixed-precision overflow (seen with\n    # single-channel replicated inputs where logits can reach ~2000+)\n    n_bad = int(np.isnan(logits).sum() + np.isinf(logits).sum())\n    if n_bad:\n        print(f'  ⚠ {n_bad} NaN/Inf logits detected — clamping to [-50, 50]')\n    logits = np.nan_to_num(logits, nan=0.0, posinf=50.0, neginf=-50.0)\n    logits = np.clip(logits, -50.0, 50.0)\n\n    probs  = torch.sigmoid(torch.tensor(logits)).numpy()\n    auc    = roc_auc_score(labels, probs)\n    fpr, tpr, thr = roc_curve(labels, probs)\n    j   = np.argmax(tpr - fpr)\n    t   = thr[j]\n    prd = (probs >= t).astype(int)\n    tn, fp, fn, tp = confusion_matrix(labels, prd).ravel()\n    sens = tp / (tp + fn + 1e-9)\n    spec = tn / (tn + fp + 1e-9)\n    return dict(auc=round(float(auc),4),\n                val_loss=round(total_loss / len(loader), 5),\n                sensitivity=round(float(sens),4),\n                specificity=round(float(spec),4),\n                threshold=round(float(t),4))\n\n\ndef run_experiment(name: str, arch: str, train_ds_cls,\n                   train_transform, val_transform,\n                   train_df, val_df) -> dict:\n    print(f'\\n{\"=\"*55}')\n    print(f'  Experiment: {name}')\n    print(f'  Arch: {arch}  |  Epochs: {ABLATION_EPOCHS}')\n    print(f'{\"=\"*55}')\n    seed_everything(SEED)\n    model = build_model(arch)\n    tr_l, vl_l, pw = make_loaders(train_df, val_df, train_ds_cls,\n                                   train_transform, val_transform)\n    criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([pw]).to(DEVICE))\n\n    # Freeze backbone for epoch 0 — train head only first, then unfreeze\n    for p in model.parameters():\n        p.requires_grad = False\n    head = model.classifier if hasattr(model, 'classifier') else model.fc\n    for p in head.parameters():\n        p.requires_grad = True\n\n    optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n                            lr=BASE_LR, weight_decay=1e-5)\n    scaler    = GradScaler('cuda')\n\n    for epoch in range(ABLATION_EPOCHS):\n        if epoch == 1:\n            # Unfreeze backbone and rebuild optimizer with all params\n            for p in model.parameters():\n                p.requires_grad = True\n            optimizer = optim.AdamW(model.parameters(), lr=BASE_LR, weight_decay=1e-5)\n\n        model.train()\n        ep_loss = 0.0\n        for imgs, lbls in tqdm(tr_l, desc=f'  Ep {epoch}', leave=False):\n            imgs, lbls = imgs.to(DEVICE), lbls.to(DEVICE)\n            optimizer.zero_grad(set_to_none=True)\n            with autocast(device_type='cuda'):\n                loss = criterion(model(imgs).squeeze(1), lbls)\n            scaler.scale(loss).backward()\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            scaler.step(optimizer); scaler.update()\n            ep_loss += loss.item()\n        ep_loss /= len(tr_l)\n\n    result = eval_loader(model, vl_l, criterion)\n    result.update({'name': name, 'arch': arch})\n    print(f'  Result: AUC={result[\"auc\"]:.4f}  '\n          f'Val-loss={result[\"val_loss\"]:.5f}  '\n          f'Sens={result[\"sensitivity\"]:.4f}  '\n          f'Spec={result[\"specificity\"]:.4f}')\n\n    # Free GPU memory\n    del model, tr_l, vl_l\n    gc.collect()\n    torch.cuda.empty_cache()\n    return result\n\n\nprint('Utilities defined.')","metadata":{},"outputs":[],"execution_count":null},{"id":"097d04f6","cell_type":"code","source":"# ── Load manifest + create ablation subset ────────────────────────────────\nmanifest = pd.read_csv(MANIFEST_PATH)\nablation_df = manifest.groupby(['split', 'any'], group_keys=False).apply(\n    lambda x: x.sample(frac=ABLATION_SUBSET, random_state=SEED)\n).reset_index(drop=True)\n\ntrain_df = ablation_df[ablation_df['split'] == 'train'].reset_index(drop=True)\nval_df   = ablation_df[ablation_df['split'] == 'val'].reset_index(drop=True)\nprint(f'Ablation subset — Train: {len(train_df):,}  Val: {len(val_df):,}')","metadata":{},"outputs":[],"execution_count":null},{"id":"8ef2eac3","cell_type":"code","source":"# ── Experiment A: Architecture comparison ────────────────────────────────\nresults = []\n\nres = run_experiment(\n    name='A1 EfficientNet-B0 + 3ch windows + augmentation',\n    arch='efficientnet_b0',\n    train_ds_cls=ICHDataset,\n    train_transform=aug_transform,\n    val_transform=no_aug_transform,\n    train_df=train_df, val_df=val_df\n)\nresults.append(res)\n\nres = run_experiment(\n    name='A2 ResNet-50 + 3ch windows + augmentation',\n    arch='resnet50',\n    train_ds_cls=ICHDataset,\n    train_transform=aug_transform,\n    val_transform=no_aug_transform,\n    train_df=train_df, val_df=val_df\n)\nresults.append(res)","metadata":{},"outputs":[],"execution_count":null},{"id":"a542f99b","cell_type":"code","source":"# ── Experiment B: Windowing ON vs OFF (EfficientNet-B0) ───────────────────\nres = run_experiment(\n    name='B1 EffNet-B0 + single window (brain only, replicated)',\n    arch='efficientnet_b0',\n    train_ds_cls=ICHDatasetGray,   # single brain window replicated to 3ch\n    train_transform=aug_transform,\n    val_transform=no_aug_transform,\n    train_df=train_df, val_df=val_df\n)\nresults.append(res)\n# A1 (with windowing) already collected above","metadata":{},"outputs":[],"execution_count":null},{"id":"07a19112","cell_type":"code","source":"# ── Experiment C: Augmentation ON vs OFF (EfficientNet-B0) ────────────────\nres = run_experiment(\n    name='C1 EffNet-B0 + 3ch windows + NO augmentation',\n    arch='efficientnet_b0',\n    train_ds_cls=ICHDataset,\n    train_transform=no_aug_transform,   # no augmentation\n    val_transform=no_aug_transform,\n    train_df=train_df, val_df=val_df\n)\nresults.append(res)\n# A1 (with augmentation) already collected above","metadata":{},"outputs":[],"execution_count":null},{"id":"bd89ca5a","cell_type":"code","source":"# ── Experiment D: Normalization — Dataset-specific vs ImageNet ────────────\nres = run_experiment(\n    name='D1 EffNet-B0 + ImageNet normalization',\n    arch='efficientnet_b0',\n    train_ds_cls=ICHDataset,\n    train_transform=aug_transform_imgnet,       # ImageNet stats\n    val_transform=no_aug_transform_imgnet,       # ImageNet stats\n    train_df=train_df, val_df=val_df\n)\nresults.append(res)\n# A1 (with dataset normalization) already collected above","metadata":{},"outputs":[],"execution_count":null},{"id":"24d9b81b","cell_type":"code","source":"# ── Compile results table ─────────────────────────────────────────────────\nresults_df = pd.DataFrame(results)[['name', 'arch', 'auc', 'val_loss', 'sensitivity', 'specificity', 'threshold']]\nresults_df.columns = ['Experiment', 'Arch', 'Val AUC', 'Val Loss', 'Sensitivity', 'Specificity', 'Threshold']\n\nprint('\\n' + '='*90)\nprint('ABLATION RESULTS')\nprint('='*90)\nprint(results_df.to_string(index=False))\nprint('='*90)\n\nresults_df.to_csv('/kaggle/working/ablation_results.csv', index=False)","metadata":{},"outputs":[],"execution_count":null},{"id":"4e1d4774","cell_type":"code","source":"# ── Visualise AUC comparison ──────────────────────────────────────────────\nfig, axes = plt.subplots(1, 4, figsize=(20, 5))\n\nmetrics_to_plot = ['Val AUC', 'Val Loss', 'Sensitivity', 'Specificity']\ncolors = sns.color_palette('muted', len(results_df))\n\nfor ax, metric in zip(axes, metrics_to_plot):\n    bars = ax.barh(results_df['Experiment'], results_df[metric], color=colors)\n    ax.set_title(metric)\n    if metric == 'Val Loss':\n        ax.set_xlim(0, max(results_df[metric]) * 1.3)\n    else:\n        ax.set_xlim(0.5, 1.0)\n    ax.set_xlabel(metric)\n    for bar, val in zip(bars, results_df[metric]):\n        ax.text(bar.get_width() + 0.005, bar.get_y() + bar.get_height()/2,\n                f'{val:.4f}', va='center', fontsize=8)\n\nplt.suptitle(f'Ablation Study ({ABLATION_EPOCHS} epochs, {ABLATION_SUBSET*100:.0f}% data subset)',\n             fontsize=12, y=1.01)\nplt.tight_layout()\nplt.savefig('/kaggle/working/ablation_comparison.png', bbox_inches='tight')\nplt.show()\n\nprint('\\nInterpretation guide:')\nprint(' A1 vs A2  → justifies architecture selection')\n\nprint(' A1 vs B1  → justifies multi-window preprocessing')\nprint(' A1 vs C1  → justifies data augmentation')\nprint(' A1 vs D1  → justifies dataset-specific normalization')","metadata":{},"outputs":[],"execution_count":null},{"id":"28989f58","cell_type":"code","source":"# ── HEALTH CHECK — automated output validation ────────────────────────────\nimport json as _json_hc\n\nerrors = []\n\n# Check results CSV\nif not os.path.exists('/kaggle/working/ablation_results.csv'):\n    errors.append('ablation_results.csv is MISSING')\nelse:\n    abl_hc = pd.read_csv('/kaggle/working/ablation_results.csv')\n    if len(abl_hc) < 5:\n        errors.append(f'Only {len(abl_hc)} ablation results — expected 5')\n    if 'Val AUC' in abl_hc.columns and abl_hc['Val AUC'].max() < 0.55:\n        errors.append(f'All ablation AUCs < 0.55 — possible issue')\n\n# Check comparison plot\nif not os.path.exists('/kaggle/working/ablation_comparison.png'):\n    errors.append('ablation_comparison.png is MISSING')\n\nhealth = {\n    'notebook': '04_ablations',\n    'status'  : 'PASS' if not errors else 'FAIL',\n    'errors'  : errors,\n    'n_experiments': len(results),\n    'best_experiment': max(results, key=lambda x: x['auc'])['name'] if results else 'N/A',\n}\n\nwith open('/kaggle/working/health_check_nb04.json', 'w') as f:\n    _json_hc.dump(health, f, indent=2)\n\nif errors:\n    print('❌ HEALTH CHECK FAILED:')\n    for e in errors:\n        print(f'   • {e}')\nelse:\n    print('✅ HEALTH CHECK PASSED')\n    print(f'   {len(results)} experiments completed')\n    print(f'   Best: {health[\"best_experiment\"]}')","metadata":{},"outputs":[],"execution_count":null}]}