{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ================================================================\n# Grand X-Ray Multilabel — ConvNeXt Base @512 with DataParallel\n# ================================================================\nimport os, math, random, warnings, cv2\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\nimport torchvision.transforms as T\nimport timm\nfrom tqdm import tqdm\n\n# -------------------------\n# Config\n# -------------------------\nSEED = 42\nIMG_SIZE = 512\nBATCH_SIZE = 8  # safe for T4; try 16 if enough VRAM\nEPOCHS = 7\nWARMUP_EPOCHS = 1\nBASE_LR = 2e-5\nHEAD_LR = 8e-5\nWEIGHT_DECAY = 1e-2\nGRAD_CLIP_NORM = 1.0\nEMA_DECAY = 0.999\nFOCAL_GAMMA = 2.0\nNUM_WORKERS = 4\n\nTRAIN_CSV = \"/kaggle/input/grand-xray-slam-division-a/train1.csv\"\nTRAIN_DIR = \"/kaggle/input/grand-xray-slam-division-a/train1\"\nTEST_DIR  = \"/kaggle/input/grand-xray-slam-division-a/test1\"\n\nLABEL_COLS = [\n    'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Enlarged Cardiomediastinum',\n    'Fracture', 'Lung Lesion', 'Lung Opacity', 'No Finding', 'Pleural Effusion',\n    'Pleural Other', 'Pneumonia', 'Pneumothorax', 'Support Devices'\n]\n\nSAVE_PATH = \"convnext_best.pth\"  # where to save model weights\n\n# -------------------------\n# Repro\n# -------------------------\ndef set_seed(seed=SEED):\n    random.seed(seed); np.random.seed(seed); torch.manual_seed(seed); torch.cuda.manual_seed_all(seed)\nset_seed()\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Using device:\", device)\n\n# -------------------------\n# Dataset\n# -------------------------\nclass XRayDataset(Dataset):\n    def __init__(self, df, image_dir, img_size=IMG_SIZE, is_train=True):\n        self.df = df.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.is_train = is_train\n        self.img_size = img_size\n        if is_train:\n            self.tf = T.Compose([\n                T.ToTensor(),\n                T.RandomHorizontalFlip(p=0.5),\n                T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n            ])\n        else:\n            self.tf = T.Compose([\n                T.ToTensor(),\n                T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n            ])\n    def __len__(self): return len(self.df)\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        path = os.path.join(self.image_dir, row['Image_name'])\n        img = cv2.imread(path, cv2.IMREAD_COLOR)\n        if img is None:\n            img = np.zeros((self.img_size, self.img_size, 3), dtype=np.uint8)\n        img = cv2.resize(img, (self.img_size, self.img_size), interpolation=cv2.INTER_CUBIC)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = self.tf(img)\n        if self.is_train:\n            y = torch.tensor(row[LABEL_COLS].values.astype(np.float32), dtype=torch.float32)\n            return img, y\n        else:\n            return img, row['Image_name']\n\n# -------------------------\n# Focal Loss\n# -------------------------\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=None, gamma=2.0, reduction=\"mean\"):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n    def forward(self, logits, targets):\n        bce = nn.functional.binary_cross_entropy_with_logits(logits, targets, reduction=\"none\")\n        pt = torch.exp(-bce)\n        loss = (1 - pt)**self.gamma * bce\n        if self.alpha is not None:\n            loss = loss * self.alpha.to(logits.device)\n        if self.reduction == \"mean\":\n            return loss.mean()\n        return loss.sum()\n\n# -------------------------\n# EMA\n# -------------------------\nclass EMA:\n    def __init__(self, model, decay=EMA_DECAY):\n        self.decay = decay\n        self.shadow = {n: p.detach().clone() for n, p in model.named_parameters() if p.requires_grad}\n    @torch.no_grad()\n    def update(self, model):\n        for n, p in model.named_parameters():\n            if n in self.shadow:\n                self.shadow[n].mul_(self.decay).add_(p.detach(), alpha=1-self.decay)\n    @torch.no_grad()\n    def apply_to(self, model):\n        for n, p in model.named_parameters():\n            if n in self.shadow:\n                p.copy_(self.shadow[n])\n\n# -------------------------\n# Data prep\n# -------------------------\ndf = pd.read_csv(TRAIN_CSV)\ndf[LABEL_COLS] = df[LABEL_COLS].apply(pd.to_numeric, errors=\"coerce\").fillna(0)\n\nif 'No Finding' in LABEL_COLS:\n    others = [c for c in LABEL_COLS if c != 'No Finding']\n    df['No Finding'] = (df[others].sum(axis=1) == 0).astype(int)\n\ndf['sum_labels'] = df[LABEL_COLS].sum(axis=1)\ntrain_df, val_df = train_test_split(df, test_size=0.1, random_state=SEED,\n                                   stratify=np.clip(df['sum_labels'], 0, 5))\ntrain_df = train_df.drop(columns=['sum_labels']).reset_index(drop=True)\nval_df = val_df.drop(columns=['sum_labels']).reset_index(drop=True)\n\npos_counts = train_df[LABEL_COLS].sum()\nneg_counts = len(train_df) - pos_counts\nalpha = torch.tensor((neg_counts/(pos_counts+1e-6)).values, dtype=torch.float32)\n\ntrain_ds = XRayDataset(train_df, TRAIN_DIR, is_train=True)\nval_ds = XRayDataset(val_df, TRAIN_DIR, is_train=True)\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True,\n                          num_workers=NUM_WORKERS, pin_memory=True, drop_last=True)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False,\n                        num_workers=NUM_WORKERS, pin_memory=True)\n\n# -------------------------\n# Model + training helpers\n# -------------------------\ndef make_model():\n    return timm.create_model(\"convnext_base.fb_in22k_ft_in1k\", pretrained=True, num_classes=len(LABEL_COLS))\n\ndef param_groups(m):\n    module = m.module if isinstance(m, nn.DataParallel) else m\n    head = module.get_classifier()\n    head_ids = set(map(id, head.parameters()))\n    base_params = [p for p in module.parameters() if id(p) not in head_ids]\n    return [\n        {\"params\": base_params, \"lr\": BASE_LR, \"weight_decay\": WEIGHT_DECAY},\n        {\"params\": head.parameters(), \"lr\": HEAD_LR, \"weight_decay\": WEIGHT_DECAY},\n    ]\n\ncriterion = FocalLoss(alpha=alpha, gamma=FOCAL_GAMMA)\n\ndef evaluate(model):\n    model.eval()\n    logits_all, targets_all = [], []\n    with torch.no_grad():\n        for imgs, y in val_loader:\n            imgs, y = imgs.to(device), y.to(device)\n            with torch.cuda.amp.autocast(enabled=device.type==\"cuda\"):\n                out = model(imgs)\n            logits_all.append(out.detach().cpu().numpy())\n            targets_all.append(y.detach().cpu().numpy())\n    logits_all = np.concatenate(logits_all,0)\n    targets_all = np.concatenate(targets_all,0)\n    probs = 1/(1+np.exp(-logits_all))\n    auc = roc_auc_score(targets_all, probs, average=\"macro\")\n    return auc\n\n# -------------------------\n# Train ConvNeXt Base (with DataParallel)\n# -------------------------\nmodel = make_model().to(device)\n\n# ✅ wrap with DataParallel if multiple GPUs\nif torch.cuda.device_count() > 1:\n    print(f\"✅ Using {torch.cuda.device_count()} GPUs for DataParallel\")\n    model = nn.DataParallel(model)\n\noptimizer = optim.AdamW(param_groups(model))\nscheduler = optim.lr_scheduler.LambdaLR(\n    optimizer,\n    lr_lambda=lambda e: (e+1)/WARMUP_EPOCHS if e<WARMUP_EPOCHS\n    else 0.5*(1+math.cos(math.pi*(e-WARMUP_EPOCHS)/max(1,EPOCHS-WARMUP_EPOCHS)))\n)\nscaler = torch.cuda.amp.GradScaler(enabled=device.type==\"cuda\")\nema = EMA(model)\nbest_auc=-1\n\nfor epoch in range(EPOCHS):\n    model.train()\n    running=0\n    pbar=tqdm(train_loader,desc=f\"ConvNeXt Epoch {epoch+1}/{EPOCHS}\")\n    for imgs,y in pbar:\n        imgs,y=imgs.to(device),y.to(device)\n        optimizer.zero_grad(set_to_none=True)\n        with torch.cuda.amp.autocast(enabled=device.type==\"cuda\"):\n            out=model(imgs)\n            loss=criterion(out,y)\n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP_NORM)\n        scaler.step(optimizer); scaler.update()\n        ema.update(model)\n        running+=loss.item()\n        pbar.set_postfix(loss=running/max(1,pbar.n))\n    scheduler.step()\n    ema.apply_to(model)\n    auc=evaluate(model)\n    print(f\"ConvNeXt Epoch {epoch+1}: AUC={auc:.4f}\")\n    # ✅ save correctly with DataParallel\n    state_dict = model.module.state_dict() if isinstance(model, nn.DataParallel) else model.state_dict()\n    if auc>best_auc:\n        best_auc=auc\n        torch.save(state_dict, SAVE_PATH)\nprint(f\"Best AUC for ConvNeXt: {best_auc:.4f}\")\n\n# -------------------------\n# Inference with TTA\n# -------------------------\nclass TestDataset(Dataset):\n    def __init__(self, df, image_dir):\n        self.df=df.reset_index(drop=True)\n        self.image_dir=image_dir\n        self.tf=T.Compose([\n            T.ToTensor(),\n            T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n        ])\n    def __len__(self):return len(self.df)\n    def __getitem__(self,idx):\n        row=self.df.iloc[idx]\n        path=os.path.join(self.image_dir,row['Image_name'])\n        img=cv2.imread(path,cv2.IMREAD_COLOR)\n        if img is None:\n            img=np.zeros((IMG_SIZE,IMG_SIZE,3),dtype=np.uint8)\n        img=cv2.resize(img,(IMG_SIZE,IMG_SIZE))\n        img=cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n        img=self.tf(img)\n        return img,row['Image_name']\n\n# Load best weights for inference\nmodel_infer = make_model().to(device)\nif torch.cuda.device_count() > 1:\n    model_infer = nn.DataParallel(model_infer)\n\nstate_dict = torch.load(SAVE_PATH,map_location=device)\nif isinstance(model_infer, nn.DataParallel):\n    model_infer.module.load_state_dict(state_dict)\nelse:\n    model_infer.load_state_dict(state_dict)\nmodel_infer.eval()\n\ntest_names=sorted(os.listdir(TEST_DIR))\ntest_df=pd.DataFrame({'Image_name':test_names})\ntest_ds=TestDataset(test_df,TEST_DIR)\ntest_loader=DataLoader(test_ds,batch_size=BATCH_SIZE,shuffle=False,num_workers=NUM_WORKERS,pin_memory=True)\n\n@torch.no_grad()\ndef predict_tta(model,loader):\n    preds_all=[];names_all=[]\n    for imgs,names in tqdm(loader,desc=\"Predicting ConvNeXt\"):\n        imgs=imgs.to(device)\n        logits1=model(imgs)\n        imgs_flipped=torch.flip(imgs,dims=[3])\n        logits2=model(imgs_flipped)\n        logits=0.5*(logits1+logits2)\n        probs=torch.sigmoid(logits).cpu().numpy()\n        preds_all.append(probs)\n        names_all.extend(names)\n    return np.concatenate(preds_all,axis=0),names_all\n\npreds,names=predict_tta(model_infer,test_loader)\nsub=pd.DataFrame(preds,columns=LABEL_COLS)\nsub.insert(0,\"Image_name\",names)\nsub.to_csv(\"submission.csv\",index=False)\nprint(\"✅ Created submission.csv\")\nprint(sub.head())\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-14T06:23:32.946326Z","iopub.execute_input":"2025-09-14T06:23:32.946897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}