{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# RSNA KNEE V13 FINAL — LAST BATTLE\n# Exact paths from your Kaggle inputs. No guessing.\n# ============================================================\n\nimport os, glob, time, warnings, json\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\nimport pydicom\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom tqdm import tqdm\n\nwarnings.filterwarnings('ignore')\ntorch.backends.cudnn.benchmark = True\n\n# --- CONFIG ---\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nLABEL_COLS = ['ACL','MCL','Medial Meniscus','Lateral Meniscus','Medial OA',\n              'Lateral OA','PF OA','Effusion','Synovitis',\"Baker's\",'Contusion','Fracture']\nN_FOLDS = 5\nSEED = 42\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\n\n# PATHS — EXACT FROM YOUR KAGGLE INPUTS\nTRAIN_DIR = '/kaggle/input/competitions/rsna-knee-abnormality-detection/train_series'\nTEST_DIR  = '/kaggle/input/competitions/rsna-knee-abnormality-detection/test_series'\nTRAIN_CSV = '/kaggle/input/competitions/rsna-knee-abnormality-detection/train.csv'\nSAMPLE_SUB = '/kaggle/input/competitions/rsna-knee-abnormality-detection/sample_submission.csv'\n\n# LABEL SOURCES — ALL 4 SOURCES FUSED\nLABEL_SOURCES = {\n    'merged_4source': '/kaggle/input/datasets/yunusgmsoy/rsna-knee-llm-labels-4-source-merged/report_labels_v5.csv',\n    'steven_blend':   '/kaggle/input/datasets/stevenleehans/rsna-knee-llm-report-labels/llm_labels_v4_blend.csv',\n    'rodilla':        '/kaggle/input/datasets/giovannicr/rodilla-pseudo-labels/pseudo_labels.csv',\n}\n\n# DINOv3 CHECKPOINTS — 5 FOLD\nDINOV3_DIR = '/kaggle/input/datasets/mattiaangeli/knee-mri-fold-weights'\n\n# Model configs\nEFFNETB3_CFG = {\n    'name': 'efficientnet_b3', 'timm_name': 'efficientnet_b3',\n    'img_size': 224, 'batch_size': 12, 'epochs': 5, 'lr': 8e-5, 'dropout': 0.4,\n}\nDINOV3_CFG = {\n    'name': 'dinov3', 'img_size': 224, 'batch_size': 8, 'epochs': 5,\n    'lr': 5e-4, 'dropout': 0.3, 'freeze_backbone': True,\n}\n\n# ============================================================\n# 1. LABEL FUSION — 4 SOURCE + GOLD CALIBRATION\n# ============================================================\ndef standardize_columns(df):\n    \"\"\"Map various column names to standard LABEL_COLS.\"\"\"\n    col_map = {}\n    for std_col in LABEL_COLS:\n        for c in df.columns:\n            c_clean = c.strip().lower().replace('_', ' ').replace('-', ' ')\n            std_clean = std_col.lower().replace(\"'\", \"\")\n            if c_clean == std_clean or c_clean in std_clean or std_clean in c_clean:\n                col_map[c] = std_col\n                break\n    # Also check exact matches\n    for c in df.columns:\n        if c in LABEL_COLS and c not in col_map.values():\n            col_map[c] = c\n    return col_map\n\ndef fuse_labels():\n    gold = pd.read_csv(TRAIN_CSV)\n    gold['StudyInstanceUID'] = gold['StudyInstanceUID'].astype(str)\n    gold = gold.set_index('StudyInstanceUID')[LABEL_COLS]\n    \n    sources = {}\n    for name, path in LABEL_SOURCES.items():\n        if not os.path.exists(path):\n            print(f\"[!] MISSING: {path}\")\n            continue\n        df = pd.read_csv(path)\n        df['StudyInstanceUID'] = df['StudyInstanceUID'].astype(str)\n        col_map = standardize_columns(df)\n        df = df.rename(columns=col_map)\n        available = [c for c in LABEL_COLS if c in df.columns]\n        if len(available) == 0:\n            print(f\"[!] {name}: no matching columns found\")\n            continue\n        sources[name] = df.set_index('StudyInstanceUID')[available]\n        print(f\"[+] {name}: {len(sources[name])} studies, {len(available)}/12 labels\")\n    \n    if len(sources) == 0:\n        raise ValueError(\"NO LABEL SOURCES FOUND!\")\n    \n    all_studies = sorted(set().union(*[df.index for df in sources.values()]))\n    fused = pd.DataFrame(index=all_studies, columns=LABEL_COLS, dtype=np.float32)\n    \n    # Compute per-class, per-source reliability on gold studies\n    weights = {}\n    for name, src in sources.items():\n        common = gold.index.intersection(src.index)\n        w = {}\n        for col in LABEL_COLS:\n            if col not in src.columns:\n                w[col] = 0.3\n            elif len(common) > 5 and len(gold.loc[common, col].unique()) > 1:\n                try:\n                    w[col] = max(0.2, roc_auc_score(gold.loc[common, col], src.loc[common, col]))\n                except:\n                    w[col] = 0.5\n            else:\n                w[col] = 0.5\n        weights[name] = w\n        print(f\"  {name} gold AUCs: \" + \" \".join([f\"{k}={v:.3f}\" for k,v in list(w.items())[:4]]) + \"...\")\n    \n    # Weighted fusion\n    for col in LABEL_COLS:\n        numer = np.zeros(len(all_studies))\n        denom = np.zeros(len(all_studies))\n        for name, src in sources.items():\n            if col not in src.columns:\n                continue\n            aligned = src[col].reindex(all_studies).fillna(0.5).values\n            numer += aligned * weights[name][col]\n            denom += weights[name][col]\n        fused[col] = numer / np.maximum(denom, 1e-6)\n    \n    # Override gold studies\n    for uid in gold.index:\n        if uid in fused.index:\n            fused.loc[uid, LABEL_COLS] = gold.loc[uid, LABEL_COLS].values\n    \n    fused = fused.reset_index().rename(columns={'index': 'StudyInstanceUID'})\n    fused.to_csv('/kaggle/working/fused_labels.csv', index=False)\n    print(f\"\\n[+] FUSED: {len(fused)} studies\")\n    return fused\n\n# ============================================================\n# 2. DATASET\n# ============================================================\nclass KneeDataset(Dataset):\n    def __init__(self, df, img_dir, img_size=224, is_train=True):\n        self.df = df\n        self.img_dir = Path(img_dir)\n        self.img_size = img_size\n        self.is_train = is_train\n        self.studies = df['StudyInstanceUID'].unique().tolist()\n    \n    def __len__(self):\n        return len(self.studies)\n    \n    def load_slices(self, study_id):\n        study_dir = self.img_dir / study_id\n        slices = []\n        for dcm_path in sorted(study_dir.rglob('*.dcm')):\n            try:\n                dcm = pydicom.dcmread(str(dcm_path), force=True)\n                img = dcm.pixel_array.astype(np.float32)\n                img = (img - img.min()) / (img.max() - img.min() + 1e-6)\n                img = torch.tensor(img).unsqueeze(0)\n                img = F.interpolate(img.unsqueeze(0), size=(self.img_size, self.img_size),\n                                    mode='bilinear', align_corners=False).squeeze(0)\n                slices.append(img)\n            except:\n                continue\n        if len(slices) == 0:\n            return torch.zeros(1, self.img_size, self.img_size)\n        if len(slices) > 32:\n            idx = np.linspace(0, len(slices)-1, 32, dtype=int)\n            slices = [slices[i] for i in idx]\n        return torch.stack(slices)\n    \n    def __getitem__(self, idx):\n        study_id = self.studies[idx]\n        images = self.load_slices(study_id)\n        images = images.repeat(1, 3, 1, 1)\n        row = self.df[self.df['StudyInstanceUID'] == study_id].iloc[0]\n        labels = torch.tensor(row[LABEL_COLS].values.astype(np.float32))\n        return {'images': images, 'labels': labels, 'study_id': study_id}\n\ndef collate_varlen(batch):\n    return batch\n\n# ============================================================\n# 3. MODELS\n# ============================================================\nclass EffNetB3(nn.Module):\n    def __init__(self, num_classes=12, dropout=0.4):\n        super().__init__()\n        self.backbone = timm.create_model('efficientnet_b3', pretrained=False, num_classes=0)\n        self.pool = nn.AdaptiveAvgPool2d(1)\n        self.head = nn.Sequential(\n            nn.Dropout(dropout), nn.Linear(self.backbone.num_features, 256),\n            nn.ReLU(), nn.Dropout(dropout), nn.Linear(256, num_classes)\n        )\n    def forward(self, x):\n        B, N, C, H, W = x.shape\n        x = x.view(B*N, C, H, W)\n        f = self.backbone.forward_features(x)\n        f = self.pool(f).view(B, N, -1).mean(dim=1)\n        return self.head(f)\n\nclass AttentionMIL(nn.Module):\n    def __init__(self, dim=768, hidden=512, num_classes=12, dropout=0.3):\n        super().__init__()\n        self.attn = nn.Sequential(nn.Linear(dim, hidden), nn.Tanh(), nn.Linear(hidden, 1))\n        self.head = nn.Sequential(\n            nn.Dropout(dropout), nn.Linear(dim, hidden), nn.ReLU(),\n            nn.Dropout(dropout), nn.Linear(hidden, num_classes)\n        )\n    def forward(self, x):\n        a = F.softmax(self.attn(x), dim=1)\n        return self.head((x * a).sum(dim=1))\n\nclass DINOv3Model(nn.Module):\n    def __init__(self, num_classes=12, dropout=0.3, checkpoint_path=None):\n        super().__init__()\n        self.checkpoint_path = checkpoint_path\n        \n        # Try DINOv3, DINOv2, then vanilla ViT\n        self.backbone = None\n        for arch_name in ['vit_base_patch14_reg4_dinov3.lvd142m', 'vit_base_patch14_dinov2']:\n            try:\n                self.backbone = timm.create_model(arch_name, pretrained=False, num_classes=0, img_size=224)\n                print(f\"[+] Loaded architecture: {arch_name}\")\n                break\n            except:\n                continue\n        if self.backbone is None:\n            self.backbone = timm.create_model('vit_base_patch16_224', pretrained=False, num_classes=0)\n            print(\"[!] Fallback to vanilla ViT\")\n        \n        # Load checkpoint if exists\n        if checkpoint_path and os.path.exists(checkpoint_path):\n            state = torch.load(checkpoint_path, map_location='cpu')\n            if isinstance(state, dict):\n                if 'model' in state: state = state['model']\n                if 'state_dict' in state: state = state['state_dict']\n            if isinstance(state, dict):\n                state = {k.replace('module.', ''): v for k, v in state.items()}\n                # Try loading full state\n                try:\n                    self.load_state_dict(state, strict=False)\n                    print(f\"[+] Loaded FULL checkpoint: {checkpoint_path}\")\n                except:\n                    # Try loading just backbone\n                    try:\n                        self.backbone.load_state_dict(state, strict=False)\n                        print(f\"[+] Loaded BACKBONE from: {checkpoint_path}\")\n                    except:\n                        print(f\"[!] Could not load checkpoint: {checkpoint_path}\")\n            else:\n                print(f\"[!] Unexpected checkpoint format\")\n        \n        for p in self.backbone.parameters():\n            p.requires_grad = False\n        \n        self.mil = AttentionMIL(dim=768, num_classes=num_classes, dropout=dropout)\n    \n    def forward(self, x):\n        B, N, C, H, W = x.shape\n        x = x.view(B*N, C, H, W)\n        with torch.cuda.amp.autocast():\n            if hasattr(self.backbone, 'forward_features'):\n                f = self.backbone.forward_features(x)\n            else:\n                f = self.backbone(x)\n        f = f.view(B, N, -1)\n        return self.mil(f)\n\n# ============================================================\n# 4. ASYMMETRIC LOSS (Top teams use this)\n# ============================================================\nclass AsymmetricLoss(nn.Module):\n    def __init__(self, gamma_neg=4, gamma_pos=1, clip=0.05, eps=1e-8):\n        super().__init__()\n        self.gamma_neg = gamma_neg\n        self.gamma_pos = gamma_pos\n        self.clip = clip\n        self.eps = eps\n    \n    def forward(self, logits, targets):\n        xs_pos = torch.sigmoid(logits)\n        xs_neg = 1 - xs_pos\n        if self.clip and self.clip > 0:\n            xs_neg = (xs_neg + self.clip).clamp(max=1)\n        los_pos = targets * torch.log(xs_pos.clamp(min=self.eps))\n        los_neg = (1 - targets) * torch.log(xs_neg.clamp(min=self.eps))\n        loss = los_pos + los_neg\n        \n        xs_pos = xs_pos * targets\n        xs_neg = xs_neg * (1 - targets)\n        anti_target = 1 - targets\n        xs_pos_anti = xs_pos + anti_target\n        xs_neg_anti = xs_neg + targets\n        loss_pos = (torch.pow(xs_pos_anti, self.gamma_pos) * loss * targets).sum() / (targets.sum() + self.eps)\n        loss_neg = (torch.pow(xs_neg_anti, self.gamma_neg) * loss * anti_target).sum() / (anti_target.sum() + self.eps)\n        return loss_pos + loss_neg\n\n# ============================================================\n# 5. TRAINING\n# ============================================================\ndef train_one_fold(model, train_loader, val_loader, cfg, fold):\n    model = model.to(DEVICE)\n    opt = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n                             lr=cfg['lr'], weight_decay=1e-5)\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=cfg['epochs'])\n    criterion = AsymmetricLoss()\n    scaler = torch.cuda.amp.GradScaler()\n    best_auc, best_state = 0, None\n    best_preds, best_labels, best_ids = None, None, None\n    \n    for epoch in range(cfg['epochs']):\n        model.train()\n        losses = []\n        for batch in tqdm(train_loader, desc=f\"{cfg['name']} F{fold} E{epoch+1}\", leave=False):\n            imgs = batch['images'].to(DEVICE)\n            lbls = batch['labels'].to(DEVICE)\n            opt.zero_grad()\n            with torch.cuda.amp.autocast():\n                loss = criterion(model(imgs), lbls)\n            scaler.scale(loss).backward()\n            scaler.step(opt)\n            scaler.update()\n            losses.append(loss.item())\n        scheduler.step()\n        \n        model.eval()\n        preds, labels, ids = [], [], []\n        with torch.no_grad():\n            for batch in val_loader:\n                imgs = batch['images'].to(DEVICE)\n                p = torch.sigmoid(model(imgs)).cpu().numpy()\n                preds.append(p)\n                labels.append(batch['labels'].numpy())\n                ids.extend(batch['study_id'])\n        \n        preds = np.concatenate(preds)\n        labels = np.concatenate(labels)\n        aucs = [roc_auc_score(labels[:,i], preds[:,i]) for i in range(12) \n                if len(np.unique(labels[:,i])) > 1]\n        macro = np.mean(aucs) if aucs else 0\n        print(f\"  [{cfg['name']}] F{fold} E{epoch+1}/{cfg['epochs']} loss={np.mean(losses):.4f} val_auc={macro:.4f}\")\n        \n        if macro > best_auc:\n            best_auc = macro\n            best_state = model.state_dict()\n            best_preds, best_labels, best_ids = preds, labels, ids\n    \n    torch.save(best_state, f\"/kaggle/working/{cfg['name']}_fold{fold}.pth\")\n    oof = pd.DataFrame(best_preds, columns=LABEL_COLS)\n    oof['StudyInstanceUID'] = best_ids\n    oof.to_csv(f\"/kaggle/working/oof_{cfg['name']}_fold{fold}.csv\", index=False)\n    return best_auc, best_preds, best_labels, best_ids\n\n# ============================================================\n# 6. ENSEMBLE WEIGHTS\n# ============================================================\ndef optimize_ensemble_weights(oof_dict, labels_arr):\n    names = list(oof_dict.keys())\n    weights = {}\n    for i, col in enumerate(LABEL_COLS):\n        yt = labels_arr[:, i]\n        valid = ~np.isnan(yt)\n        if valid.sum() < 10 or len(names) < 2:\n            weights[col] = {n: 1.0/len(names) for n in names}\n            continue\n        best_auc, best_w = -1, None\n        if len(names) == 2:\n            for w in np.linspace(0, 1, 51):\n                pred = w * oof_dict[names[0]][valid, i] + (1-w) * oof_dict[names[1]][valid, i]\n                auc = roc_auc_score(yt[valid], pred)\n                if auc > best_auc:\n                    best_auc, best_w = auc, {names[0]: w, names[1]: 1-w}\n        else:\n            best_w = {n: 1.0/len(names) for n in names}\n        weights[col] = best_w\n    return weights\n\n# ============================================================\n# 7. MAIN\n# ============================================================\ndef main():\n    t0 = time.time()\n    print(\"=\" * 70)\n    print(\"RSNA KNEE V13 FINAL — LAST BATTLE\")\n    print(\"=\" * 70)\n    \n    labels_df = fuse_labels()\n    \n    gold = pd.read_csv(TRAIN_CSV)\n    gold['StudyInstanceUID'] = gold['StudyInstanceUID'].astype(str)\n    train_df = labels_df.copy()\n    for _, row in gold.iterrows():\n        mask = train_df['StudyInstanceUID'] == row['StudyInstanceUID']\n        if mask.any():\n            train_df.loc[mask, LABEL_COLS] = row[LABEL_COLS].values\n    \n    groups = train_df['StudyInstanceUID'].values\n    gkf = GroupKFold(n_splits=N_FOLDS)\n    \n    oof_preds = {}\n    oof_labels = {}\n    \n    for fold, (tr_idx, val_idx) in enumerate(gkf.split(train_df, groups=groups)):\n        print(f\"\\n{'#'*70}\\nFOLD {fold+1}/{N_FOLDS}\\n{'#'*70}\")\n        tr = train_df.iloc[tr_idx].reset_index(drop=True)\n        vl = train_df.iloc[val_idx].reset_index(drop=True)\n        \n        # --- EFFNET-B3 ---\n        tr_ds = KneeDataset(tr, TRAIN_DIR, img_size=EFFNETB3_CFG['img_size'])\n        vl_ds = KneeDataset(vl, TRAIN_DIR, img_size=EFFNETB3_CFG['img_size'])\n        tr_ld = DataLoader(tr_ds, batch_size=EFFNETB3_CFG['batch_size'], shuffle=True, num_workers=2, collate_fn=collate_varlen)\n        vl_ld = DataLoader(vl_ds, batch_size=EFFNETB3_CFG['batch_size'], shuffle=False, num_workers=2, collate_fn=collate_varlen)\n        m = EffNetB3(dropout=EFFNETB3_CFG['dropout'])\n        auc, preds, labels, ids = train_one_fold(m, tr_ld, vl_ld, EFFNETB3_CFG, fold)\n        if 'efficientnet_b3' not in oof_preds:\n            oof_preds['efficientnet_b3'] = {}\n        for uid, p in zip(ids, preds):\n            oof_preds['efficientnet_b3'][uid] = p\n        for uid, l in zip(ids, labels):\n            oof_labels[uid] = l\n        \n        # --- DINOv3 ---\n        ckpt = f\"{DINOV3_DIR}/m_f{fold}.pt\"\n        tr_ld2 = DataLoader(KneeDataset(tr, TRAIN_DIR, img_size=224), batch_size=8, shuffle=True, num_workers=2, collate_fn=collate_varlen)\n        vl_ld2 = DataLoader(KneeDataset(vl, TRAIN_DIR, img_size=224), batch_size=8, shuffle=False, num_workers=2, collate_fn=collate_varlen)\n        m2 = DINOv3Model(checkpoint_path=ckpt)\n        auc2, preds2, labels2, ids2 = train_one_fold(m2, tr_ld2, vl_ld2, DINOV3_CFG, fold)\n        if 'dinov3' not in oof_preds:\n            oof_preds['dinov3'] = {}\n        for uid, p in zip(ids2, preds2):\n            oof_preds['dinov3'][uid] = p\n    \n    # --- Build OOF arrays ---\n    n = len(train_df)\n    oof_arr = {k: np.zeros((n, 12)) for k in oof_preds}\n    labels_arr = np.zeros((n, 12))\n    for i, uid in enumerate(train_df['StudyInstanceUID']):\n        for k in oof_preds:\n            if uid in oof_preds[k]:\n                oof_arr[k][i] = oof_preds[k][uid]\n        if uid in oof_labels:\n            labels_arr[i] = oof_labels[uid]\n    \n    # --- Optimize ensemble ---\n    print(\"\\n\" + \"=\"*70 + \"\\nENSEMBLE WEIGHTS\\n\" + \"=\"*70)\n    weights = optimize_ensemble_weights(oof_arr, labels_arr)\n    for col, w in weights.items():\n        print(f\"  {col:20s}: {w}\")\n    \n    with open('/kaggle/working/ensemble_weights.json', 'w') as f:\n        json.dump({k: {kk: float(vv) for kk, vv in v.items()} for k, v in weights.items()}, f)\n    \n    # --- TEST INFERENCE ---\n    print(\"\\n\" + \"=\"*70 + \"\\nTEST INFERENCE\\n\" + \"=\"*70)\n    sub = pd.read_csv(SAMPLE_SUB)\n    sub['StudyInstanceUID'] = sub['StudyInstanceUID'].astype(str)\n    test_ds = KneeDataset(sub, TEST_DIR, img_size=224, is_train=False)\n    test_ld = DataLoader(test_ds, batch_size=8, shuffle=False, num_workers=2, collate_fn=collate_varlen)\n    \n    model_folds = {name: [] for name in oof_preds.keys()}\n    for fold in range(N_FOLDS):\n        # EffNet-B3\n        e = EffNetB3().to(DEVICE)\n        e.load_state_dict(torch.load(f\"/kaggle/working/efficientnet_b3_fold{fold}.pth\", map_location=DEVICE))\n        e.eval()\n        model_folds['efficientnet_b3'].append(e)\n        \n        # DINOv3\n        ckpt = f\"{DINOV3_DIR}/m_f{fold}.pt\"\n        d = DINOv3Model(checkpoint_path=ckpt).to(DEVICE)\n        d.load_state_dict(torch.load(f\"/kaggle/working/dinov3_fold{fold}.pth\", map_location=DEVICE))\n        d.eval()\n        model_folds['dinov3'].append(d)\n    \n    final_preds = np.zeros((len(sub), 12))\n    with torch.no_grad():\n        for batch in tqdm(test_ld, desc=\"Test\"):\n            imgs = batch['images'].to(DEVICE)\n            B = imgs.shape[0]\n            \n            model_avgs = {}\n            for name, models in model_folds.items():\n                avg = torch.zeros(B, 12).to(DEVICE)\n                for m in models:\n                    with torch.cuda.amp.autocast():\n                        avg += torch.sigmoid(m(imgs))\n                model_avgs[name] = avg / len(models)\n            \n            for i, col in enumerate(LABEL_COLS):\n                w = weights[col]\n                p = torch.zeros(B).to(DEVICE)\n                for name in model_avgs:\n                    if name in w:\n                        p += w[name] * model_avgs[name][:, i]\n                final_preds[:, i] = p.cpu().numpy()\n    \n    sub[LABEL_COLS] = final_preds\n    sub.to_csv('/kaggle/working/submission.csv', index=False)\n    \n    elapsed = (time.time() - t0) / 3600\n    print(f\"\\n{'='*70}\")\n    print(f\"DONE. Runtime: {elapsed:.2f}h\")\n    print(f\"Submission: /kaggle/working/submission.csv\")\n    print(f\"{'='*70}\")\n\nif __name__ == '__main__':\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}