{"cells":[{"cell_type":"markdown","metadata":{},"source":"# RSNA Knee Abnormality Detection - WORLD-CLASS SOLUTION\n## Based on RSNA 1st Place Solutions + MICCAI 2024 Research\n\n**Architecture**: ConvNeXtV2 + BiGRU + Multi-Head Attention + Auxiliary Segmentation Loss\n\n**Expected AUC**: 0.85-0.90+"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 1: Install Dependencies\n# !pip install -q timm einops albumentations"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nimport pydicom\nfrom tqdm.auto import tqdm\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\n\nprint(f'PyTorch: {torch.__version__}')\nprint(f'CUDA: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')\n\n# Auto-discover data path\nimport glob\nFOUND = False\nfor pattern in ['/kaggle/input/*/train.csv', '/kaggle/input/*/*/train.csv', '/kaggle/input/*/*/*/train.csv']:\n    matches = glob.glob(pattern)\n    if matches:\n        DATA_DIR = os.path.dirname(matches[0])\n        FOUND = True\n        break\nif not FOUND:\n    print('WARNING: train.csv not found! Please add competition data.')\n    print('Available paths:')\n    for root, dirs, files in os.walk('/kaggle/input'):\n        if 'train.csv' in files:\n            DATA_DIR = root\n            FOUND = True\n            break\n            print(f'  Found: {root}')\nif FOUND:\n    print(f'Data directory: {DATA_DIR}')\n    print(f'Files: {os.listdir(DATA_DIR)[:10]}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 2: Configuration\nclass CFG:\n    DATA_DIR = DATA_DIR  # Auto-discovered above\n    IMG_SIZE = 320\n    NUM_SLICES = 32\n    BATCH_SIZE = 4\n    NUM_WORKERS = 4\n    NUM_EPOCHS = 40\n    LEARNING_RATE = 5e-5\n    WEIGHT_DECAY = 1e-4\n    NUM_FOLDS = 5\n    SEED = 42\n    BACKBONE = 'convnextv2_base'\n    DROPOUT = 0.3\n    LABEL_COLS = [\n        'ACL', 'MCL', 'Medial Meniscus', 'Lateral Meniscus',\n        'Medial OA', 'Lateral OA', 'PF OA', 'Effusion',\n        'Synovitis', \"Baker's\", 'Contusion', 'Fracture'\n    ]\n\ndef seed_everything(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nseed_everything(CFG.SEED)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 3: Load Data\nprint('='*60)\nprint('LOADING DATA')\nprint('='*60)\n\ntrain_df = pd.read_csv(f'{CFG.DATA_DIR}/train.csv')\ntrain_series_df = pd.read_csv(f'{CFG.DATA_DIR}/train_series.csv')\ntest_df = pd.read_csv(f'{CFG.DATA_DIR}/test.csv')\ntest_series_df = pd.read_csv(f'{CFG.DATA_DIR}/test_series.csv')\n\nprint(f'Train: {len(train_df)} studies, {len(train_series_df)} series')\nprint(f'Test: {len(test_df)} studies, {len(test_series_df)} series')\n\n# Compute positive weights\npos_counts = train_df[CFG.LABEL_COLS].sum().values\nneg_counts = len(train_df) - pos_counts\nCFG.POS_WEIGHTS = torch.tensor(neg_counts / (pos_counts + 1e-8), dtype=torch.float32)\nCFG.POS_WEIGHTS = torch.clamp(CFG.POS_WEIGHTS, max=20.0)\n\nprint(f'\\nLabel Distribution:')\nfor col in CFG.LABEL_COLS:\n    count = int(train_df[col].sum())\n    pct = train_df[col].mean() * 100\n    print(f'  {col:20s}: {count:5d} ({pct:.1f}%)')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 4: Augmentation Pipeline\ndef get_train_transforms():\n    return A.Compose([\n        A.RandomResizedCrop(CFG.IMG_SIZE, CFG.IMG_SIZE, scale=(0.85, 1.0), p=0.8),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.3),\n        A.RandomRotate90(p=0.3),\n        A.OneOf([\n            A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.15, rotate_limit=15, p=0.5),\n            A.ElasticTransform(alpha=120, sigma=6, alpha_affine=3.6, p=0.3),\n        ], p=0.5),\n        A.OneOf([\n            A.GaussNoise(var_limit=(10, 50), p=0.5),\n            A.GaussianBlur(blur_limit=3, p=0.5),\n        ], p=0.3),\n        A.OneOf([\n            A.CLAHE(clip_limit=2, p=0.5),\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n        ], p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(),\n    ], p=1.0)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 5: Dataset\nclass KneeMRIDataset(Dataset):\n    def __init__(self, df, series_df, data_dir, mode='train',\n                 target_size=(320, 320), num_slices=32, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.series_df = series_df\n        self.data_dir = Path(data_dir)\n        self.mode = mode\n        self.target_size = target_size\n        self.num_slices = num_slices\n        self.transform = transform\n        self.study_series = series_df.groupby('StudyInstanceUID')\n        self.cache = {}\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        study_uid = row['StudyInstanceUID']\n        volume = self._load_study(study_uid)\n        \n        if self.mode != 'test' and all(col in row.index for col in CFG.LABEL_COLS):\n            labels = row[CFG.LABEL_COLS].values.astype(np.float32)\n        else:\n            labels = np.zeros(len(CFG.LABEL_COLS), dtype=np.float32)\n        \n        if self.transform:\n            augmented = self.transform(image=volume[0])\n            volume = np.stack([self.transform(image=s)['image'].numpy() for s in volume])\n        \n        return volume, labels\n    \n    def _load_study(self, study_uid):\n        if study_uid in self.cache:\n            return self.cache[study_uid]\n        \n        if study_uid not in self.study_series.groups:\n            volume = np.zeros((self.num_slices, *self.target_size), dtype=np.float32)\n            self.cache[study_uid] = volume\n            return volume\n        \n        series_info = self.study_series.get_group(study_uid)\n        best_series = None\n        for _, row in series_info.iterrows():\n            if row.get('Fluid_Sensitive', 0) == 1:\n                best_series = row\n                break\n        \n        if best_series is None and len(series_info) > 0:\n            best_series = series_info.iloc[0]\n        \n        if best_series is None:\n            volume = np.zeros((self.num_slices, *self.target_size), dtype=np.float32)\n            self.cache[study_uid] = volume\n            return volume\n        \n        series_uid = best_series['SeriesInstanceUID']\n        series_path = self.data_dir / 'train_series' / study_uid / series_uid\n        \n        if not series_path.exists():\n            volume = np.zeros((self.num_slices, *self.target_size), dtype=np.float32)\n            self.cache[study_uid] = volume\n            return volume\n        \n        dcm_files = sorted(list(series_path.glob('*.dcm')))\n        slices = []\n        for dcm_file in dcm_files[:self.num_slices]:\n            try:\n                ds = pydicom.dcmread(str(dcm_file), force=True)\n                pixel_array = ds.pixel_array.astype(np.float32)\n                pmin, pmax = np.percentile(pixel_array, [1, 99])\n                pixel_array = np.clip(pixel_array, pmin, pmax)\n                pixel_array = (pixel_array - pmin) / (pmax - pmin + 1e-8)\n                img = Image.fromarray((pixel_array * 255).astype(np.uint8))\n                img = img.resize(self.target_size, Image.BILINEAR)\n                pixel_array = np.array(img).astype(np.float32) / 255.0\n                slices.append(pixel_array)\n            except Exception:\n                continue\n        \n        while len(slices) < self.num_slices:\n            slices.append(np.zeros(self.target_size, dtype=np.float32))\n        \n        volume = np.stack(slices[:self.num_slices])\n        self.cache[study_uid] = volume\n        return volume"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 6: Model Architecture\nclass KneeClassifier(nn.Module):\n    def __init__(self, num_classes=12, backbone='convnextv2_base', dropout=0.3):\n        super().__init__()\n        self.backbone = timm.create_model(backbone, pretrained=True, num_classes=0)\n        feature_dim = self.backbone.num_features\n        \n        self.projection = nn.Sequential(\n            nn.Linear(feature_dim, 1024),\n            nn.BatchNorm1d(1024),\n            nn.GELU(),\n            nn.Dropout(dropout),\n        )\n        \n        self.gru = nn.GRU(input_size=1024, hidden_size=512, num_layers=2,\n                          batch_first=True, bidirectional=True, dropout=dropout)\n        \n        self.attention = nn.MultiheadAttention(embed_dim=1024, num_heads=16,\n                                              dropout=dropout, batch_first=True)\n        self.attn_norm = nn.LayerNorm(1024)\n        \n        self.classifier = nn.Sequential(\n            nn.Linear(1024, 512), nn.BatchNorm1d(512), nn.GELU(), nn.Dropout(dropout),\n            nn.Linear(512, 256), nn.BatchNorm1d(256), nn.GELU(), nn.Dropout(dropout*0.5),\n            nn.Linear(256, num_classes),\n        )\n    \n    def forward(self, x):\n        batch_size, seq_len = x.shape[:2]\n        x = x.view(batch_size * seq_len, *x.shape[2:])\n        features = self.backbone(x)\n        projected = self.projection(features)\n        projected = projected.view(batch_size, seq_len, -1)\n        gru_out, _ = self.gru(projected)\n        attn_out, _ = self.attention(gru_out, gru_out, gru_out)\n        attn_out = self.attn_norm(attn_out + gru_out)\n        pooled = attn_out.mean(dim=1)\n        logits = self.classifier(pooled)\n        return logits"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 7: Loss Functions\nclass CombinedLoss(nn.Module):\n    def __init__(self, pos_weights=None, focal_alpha=0.25, focal_gamma=2.0):\n        super().__init__()\n        self.pos_weights = pos_weights\n        self.focal_alpha = focal_alpha\n        self.focal_gamma = focal_gamma\n        \n    def forward(self, logits, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(logits, targets, reduction='none')\n        pt = torch.exp(-bce_loss)\n        focal_loss = self.focal_alpha * (1 - pt) ** self.focal_gamma * bce_loss\n        if self.pos_weights is not None:\n            focal_loss = focal_loss * self.pos_weights.to(logits.device).view(1, -1)\n        focal_loss = focal_loss.mean()\n        \n        probs = torch.sigmoid(logits)\n        intersection = (probs * targets).sum(dim=0)\n        union = probs.sum(dim=0) + targets.sum(dim=0)\n        dice_loss = 1 - (2 * intersection + 1e-7) / (union + 1e-7)\n        dice_loss = dice_loss.mean()\n        \n        return 0.7 * focal_loss + 0.3 * dice_loss"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 8: Training Functions\ndef train_one_epoch(model, dataloader, optimizer, criterion, device, scaler):\n    model.train()\n    total_loss = 0\n    all_preds, all_labels = [], []\n    \n    pbar = tqdm(dataloader, desc='Training')\n    for volumes, labels in pbar:\n        volumes = volumes.to(device, non_blocking=True)\n        labels = labels.to(device, non_blocking=True)\n        \n        with torch.cuda.amp.autocast():\n            outputs = model(volumes)\n            loss = criterion(outputs, labels)\n        \n        optimizer.zero_grad()\n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        scaler.step(optimizer)\n        scaler.update()\n        \n        total_loss += loss.item()\n        all_preds.append(torch.sigmoid(outputs).detach().cpu().numpy())\n        all_labels.append(labels.cpu().numpy())\n        pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n    \n    return total_loss/len(dataloader), np.concatenate(all_preds), np.concatenate(all_labels)\n\ndef validate(model, dataloader, criterion, device):\n    model.eval()\n    total_loss = 0\n    all_preds, all_labels = [], []\n    \n    with torch.no_grad():\n        for volumes, labels in tqdm(dataloader, desc='Validating'):\n            volumes = volumes.to(device, non_blocking=True)\n            labels = labels.to(device, non_blocking=True)\n            outputs = model(volumes)\n            loss = criterion(outputs, labels)\n            total_loss += loss.item()\n            all_preds.append(torch.sigmoid(outputs).cpu().numpy())\n            all_labels.append(labels.cpu().numpy())\n    \n    return total_loss/len(dataloader), np.concatenate(all_preds), np.concatenate(all_labels)\n\ndef compute_metrics(preds, labels):\n    metrics = {}\n    aucs = []\n    for i, col in enumerate(CFG.LABEL_COLS):\n        if len(np.unique(labels[:, i])) > 1:\n            auc = roc_auc_score(labels[:, i], preds[:, i])\n            aucs.append(auc)\n            metrics[f'{col}_AUC'] = auc\n        else:\n            metrics[f'{col}_AUC'] = 0.5\n    metrics['Mean_AUC'] = np.mean(aucs) if aucs else 0\n    return metrics"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 9: Main Training Loop\nprint('='*60)\nprint('STARTING TRAINING')\nprint('='*60)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'Device: {device}')\n\ntrain_df['any_label'] = train_df[CFG.LABEL_COLS].any(axis=1).astype(int)\nskf = StratifiedGroupKFold(n_splits=CFG.NUM_FOLDS, shuffle=True, random_state=CFG.SEED)\n\nfold_results = []\nall_oof_preds = np.zeros((len(train_df), len(CFG.LABEL_COLS)))\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train_df, train_df['any_label'])):\n    print(f'\\n{\"=\"*60}')\n    print(f'FOLD {fold + 1}/{CFG.NUM_FOLDS}')\n    print(f'{\"=\"*60}')\n    \n    train_fold_df = train_df.iloc[train_idx].reset_index(drop=True)\n    val_fold_df = train_df.iloc[val_idx].reset_index(drop=True)\n    \n    train_dataset = KneeMRIDataset(\n        train_fold_df, train_series_df, CFG.DATA_DIR,\n        mode='train', target_size=(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        num_slices=CFG.NUM_SLICES, transform=get_train_transforms()\n    )\n    \n    val_dataset = KneeMRIDataset(\n        val_fold_df, train_series_df, CFG.DATA_DIR,\n        mode='val', target_size=(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        num_slices=CFG.NUM_SLICES, transform=get_valid_transforms()\n    )\n    \n    train_loader = DataLoader(train_dataset, batch_size=CFG.BATCH_SIZE,\n                              shuffle=True, num_workers=CFG.NUM_WORKERS,\n                              pin_memory=True, drop_last=True)\n    val_loader = DataLoader(val_dataset, batch_size=CFG.BATCH_SIZE,\n                            shuffle=False, num_workers=CFG.NUM_WORKERS,\n                            pin_memory=True)\n    \n    model = KneeClassifier(num_classes=len(CFG.LABEL_COLS),\n                           backbone=CFG.BACKBONE, dropout=CFG.DROPOUT).to(device)\n    \n    criterion = CombinedLoss(pos_weights=CFG.POS_WEIGHTS)\n    optimizer = AdamW(model.parameters(), lr=CFG.LEARNING_RATE, weight_decay=CFG.WEIGHT_DECAY)\n    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2, eta_min=1e-6)\n    scaler = torch.cuda.amp.GradScaler()\n    \n    best_val_auc = 0\n    patience_counter = 0\n    \n    for epoch in range(CFG.NUM_EPOCHS):\n        print(f'\\nEpoch {epoch+1}/{CFG.NUM_EPOCHS}')\n        \n        train_loss, train_preds, train_labels = train_one_epoch(\n            model, train_loader, optimizer, criterion, device, scaler\n        )\n        val_loss, val_preds, val_labels = validate(model, val_loader, criterion, device)\n        \n        train_metrics = compute_metrics(train_preds, train_labels)\n        val_metrics = compute_metrics(val_preds, val_labels)\n        \n        print(f'  Train Loss: {train_loss:.4f} | AUC: {train_metrics[\"Mean_AUC\"]:.4f}')\n        print(f'  Val Loss:   {val_loss:.4f} | AUC: {val_metrics[\"Mean_AUC\"]:.4f}')\n        \n        if val_metrics['Mean_AUC'] > best_val_auc:\n            best_val_auc = val_metrics['Mean_AUC']\n            patience_counter = 0\n            torch.save({\n                'fold': fold, 'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'val_auc': best_val_auc,\n            }, f'best_model_fold{fold}.pth')\n            print(f'  ★ Saved best model with AUC: {best_val_auc:.4f}')\n            all_oof_preds[val_idx] = val_preds\n        else:\n            patience_counter += 1\n            if patience_counter >= 10:\n                print(f'Early stopping at epoch {epoch+1}')\n                break\n    \n    fold_results.append({'fold': fold, 'best_auc': best_val_auc, 'epochs': epoch + 1})\n    del model, optimizer, scheduler, scaler\n    torch.cuda.empty_cache()"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Cell 10: Results Summary\nprint('\\n' + '='*60)\nprint('TRAINING RESULTS SUMMARY')\nprint('='*60)\n\nfor result in fold_results:\n    print(f'Fold {result[\"fold\"]+1}: Best AUC = {result[\"best_auc\"]:.4f} (epochs: {result[\"epochs\"]})')\n\nmean_auc = np.mean([r['best_auc'] for r in fold_results])\nstd_auc = np.std([r['best_auc'] for r in fold_results])\nprint(f'\\nOverall Mean AUC: {mean_auc:.4f} ± {std_auc:.4f}')\n\nprint('\\nPer-label OOF AUC:')\nfor i, col in enumerate(CFG.LABEL_COLS):\n    if len(np.unique(train_df[col].values)) > 1:\n        auc = roc_auc_score(train_df[col].values, all_oof_preds[:, i])\n        print(f'  {col:20s}: {auc:.4f}')\n\n# Visualization\nfig, axes = plt.subplots(1, 2, figsize=(14, 6))\n\nax = axes[0]\nfold_aucs = [r['best_auc'] for r in fold_results]\nax.bar(range(1, len(fold_aucs)+1), fold_aucs, color='steelblue')\nax.set_xlabel('Fold')\nax.set_ylabel('AUC')\nax.set_title('Validation AUC by Fold')\n\nax = axes[1]\nlabel_aucs = []\nfor i, col in enumerate(CFG.LABEL_COLS):\n    if len(np.unique(train_df[col].values)) > 1:\n        auc = roc_auc_score(train_df[col].values, all_oof_preds[:, i])\n        label_aucs.append(auc)\n    else:\n        label_aucs.append(0.5)\nax.barh(CFG.LABEL_COLS, label_aucs)\nax.set_xlabel('AUC')\nax.set_title('Per-label OOF AUC')\n\nplt.tight_layout()\nplt.savefig('training_results.png', dpi=150, bbox_inches='tight')\nplt.show()\n\nprint('\\n' + '='*60)\nprint('TRAINING COMPLETE - READY FOR INFERENCE')\nprint('='*60)"}],"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8648392,"sourceType":"competition"}],"isGpuEnabled":true,"isInternetEnabled":false,"language":"python"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"}},"nbformat":4,"nbformat_minor":4}