{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":113002,"databundleVersionId":13471427,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# === Grand X-Ray Slam: Division B ===\n# Uses:\n#   DATA_ROOT = \"/kaggle/input/grand-xray-slam-division-b\"\n#   TRAIN_CSV = \"/kaggle/input/grand-xray-slam-division-b/train2.csv\"\n#   OUT_DIR   = \"/kaggle/working/outputs_effv2s_fold0\"\n\nimport os, json, math, random, gc\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport torch, torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, f1_score\nimport timm\n\n# ----------------------- USER PATHS (fixed) -----------------------\nDATA_ROOT = \"/kaggle/input/grand-xray-slam-division-b\"        # contains train2/ and test2/\nTRAIN_CSV = \"/kaggle/input/grand-xray-slam-division-b/train2.csv\"\nOUT_DIR   = \"/kaggle/working/outputs_effv2s_fold0\"\nos.makedirs(OUT_DIR, exist_ok=True)\n\n# ----------------------- CONFIG -----------------------\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\n\nLABELS = [\n    \"Atelectasis\",\"Cardiomegaly\",\"Consolidation\",\"Edema\",\"Enlarged Cardiomediastinum\",\n    \"Fracture\",\"Lung Lesion\",\"Lung Opacity\",\"No Finding\",\"Pleural Effusion\",\n    \"Pleural Other\",\"Pneumonia\",\"Pneumothorax\",\"Support Devices\"\n]\n\nFOLD         = 0\nIMG_SIZE     = 256\nBATCH_SIZE   = 4\nNUM_WORKERS  = 0    # start with 0 to surface errors clearly; bump to 2/4 later\nMODEL_NAME   = \"tf_efficientnetv2_s\"\nLR           = 1e-4\nWEIGHT_DECAY = 1e-4\nEPOCHS       = 10    # increase to 10–15 for real training\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", device)\nprint(\"DATA_ROOT:\", DATA_ROOT)\nprint(\"TRAIN_CSV exists?\", os.path.isfile(TRAIN_CSV))\nprint(\"OUT_DIR:\", OUT_DIR)\n\n# ----------------------- ARTIFACTS (pos_weight, folds, manifest) -----------------------\ndf = pd.read_csv(TRAIN_CSV)\ndf.columns = df.columns.str.strip()\n\n# Ensure label columns exist; if any are missing, create as 0.0\nfor c in LABELS:\n    if c not in df.columns:\n        df[c] = 0.0\n\n# Sanitize label dtypes to float32 in [0,1]\nfor c in LABELS:\n    df[c] = pd.to_numeric(df[c], errors=\"coerce\")\ndf[LABELS] = df[LABELS].fillna(0.0).clip(0, 1).astype(\"float32\")\n\n# pos_weight for BCEWithLogitsLoss\ntotal = len(df)\npos = df[LABELS].sum().astype(float)\nneg = (total - pos).astype(float)\npos_weight = (neg / pos.replace(0, np.nan)).fillna(100.0)\nwith open(os.path.join(OUT_DIR, \"pos_weight.json\"), \"w\") as f:\n    json.dump({c: float(pos_weight[c]) for c in LABELS}, f, indent=2)\n\n# Patient-wise 5-fold split\ngkf = GroupKFold(n_splits=5)\nfold_ids = np.zeros(total, dtype=int)\nfor fold, (_, val_idx) in enumerate(gkf.split(df, groups=df[\"Patient_ID\"].astype(str).values)):\n    fold_ids[val_idx] = fold\ndf_folds = df[[\"Image_name\",\"Patient_ID\"]].copy()\ndf_folds[\"fold\"] = fold_ids\ndf_folds.to_csv(os.path.join(OUT_DIR, \"train_folds.csv\"), index=False)\n\n# Minimal manifest for dataloader\nmanifest = df[[\"Image_name\",\"Patient_ID\",\"Study\",\"Sex\",\"Age\",\"ViewCategory\",\"ViewPosition\"] + LABELS].copy()\nmanifest.to_csv(os.path.join(OUT_DIR, \"train_manifest.csv\"), index=False)\n\nprint(\"Artifacts saved:\",\n      os.path.join(OUT_DIR, \"pos_weight.json\"),\n      os.path.join(OUT_DIR, \"train_folds.csv\"),\n      os.path.join(OUT_DIR, \"train_manifest.csv\"))\n\n# ----------------------- DATASET (hardened) + SAFE COLLATE -----------------------\nclass CXRDataset(Dataset):\n    \"\"\"\n    - Forces square resize with OpenCV to (size, size) BEFORE transforms.\n    - Pre-sanitizes label array to float32.\n    - Robustly resolves image root (DATA_ROOT/img_dir or DATA_ROOT itself if already pointing to images).\n    \"\"\"\n    def __init__(self, df: pd.DataFrame, data_root: str, img_dir: str, train: bool=True, size: int=512):\n        self.df = df.reset_index(drop=True)\n        # pre-store labels as float32 array (N,14)\n        lab = self.df[LABELS].copy()\n        for c in LABELS: lab[c] = pd.to_numeric(lab[c], errors=\"coerce\")\n        lab = lab.fillna(0.0).clip(0,1).astype(\"float32\")\n        self.labels = lab.values\n\n        root_candidate = Path(data_root) / img_dir\n        if not root_candidate.exists():\n            if Path(data_root).exists() and any(Path(data_root).glob(\"*.jpg\")):\n                root_candidate = Path(data_root)\n        if not root_candidate.exists():\n            raise FileNotFoundError(f\"Image root not found: tried '{Path(data_root)/img_dir}' and '{data_root}'\")\n        self.root = root_candidate\n\n        self.train = train\n        self.size  = int(size)\n        self.train_tf = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomRotation(7),\n            transforms.ToTensor()\n        ])\n        self.val_tf = transforms.Compose([\n            transforms.ToPILImage(),\n            transforms.ToTensor()\n        ])\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        try:\n            r = self.df.iloc[idx]\n            p = self.root / r[\"Image_name\"]\n            img = cv2.imread(str(p), cv2.IMREAD_GRAYSCALE)\n            if img is None:\n                return None\n            img = cv2.resize(img, (self.size, self.size), interpolation=cv2.INTER_AREA)\n            img = np.stack([img, img, img], axis=-1)\n            tf = self.train_tf if self.train else self.val_tf\n            x = tf(img)\n            y = torch.from_numpy(self.labels[idx])\n            return x, y, r[\"Image_name\"]\n        except Exception:\n            return None\n\ndef safe_collate(batch):\n    batch = [b for b in batch if b is not None]\n    if len(batch) == 0:\n        return torch.empty(0), torch.empty(0), []\n    xs, ys, names = zip(*batch)\n    return torch.stack(xs, 0), torch.stack(ys, 0), list(names)\n\n# ----------------------- BUILD LOADERS -----------------------\ndfm = pd.read_csv(os.path.join(OUT_DIR, \"train_manifest.csv\"))\ndff = pd.read_csv(os.path.join(OUT_DIR, \"train_folds.csv\"))\ndf_all = dfm.merge(dff, on=[\"Image_name\",\"Patient_ID\"], how=\"left\")\n\ntr_df = df_all[df_all.fold != FOLD].copy()\nva_df = df_all[df_all.fold == FOLD].copy()\n\ntr_ds = CXRDataset(tr_df, DATA_ROOT, \"train2\", train=True,  size=IMG_SIZE)\nva_ds = CXRDataset(va_df, DATA_ROOT, \"train2\", train=False, size=IMG_SIZE)\n\n# smoke test a mini-batch\nchk = DataLoader(tr_ds, batch_size=4, shuffle=False, num_workers=0, collate_fn=safe_collate)\nxb, yb, names = next(iter(chk))\nprint(\"Sanity batch:\", xb.shape, yb.shape, \"first:\", names[0] if names else None)\n\ntr_ld = DataLoader(tr_ds, batch_size=BATCH_SIZE, shuffle=True,  num_workers=NUM_WORKERS, pin_memory=True, collate_fn=safe_collate)\nva_ld = DataLoader(va_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True, collate_fn=safe_collate)\n\n# ----------------------- METRICS -----------------------\n@torch.no_grad()\ndef evaluate_logits(y_true, y_pred_logits, search_thresholds=True):\n    y_true = np.asarray(y_true)\n    if y_true.size == 0:\n        return np.nan, np.nan, [0.5]*len(LABELS)\n    p = torch.sigmoid(torch.tensor(y_pred_logits)).numpy()\n    aucs=[]\n    for i in range(y_true.shape[1]):\n        try:\n            aucs.append(roc_auc_score(y_true[:,i], p[:,i]))\n        except Exception:\n            aucs.append(np.nan)\n    thresholds = [0.5]*y_true.shape[1]\n    macro_f1 = np.nan\n    if search_thresholds:\n        ts=[]\n        for i in range(y_true.shape[1]):\n            best_t, best_f1 = 0.5, 0.0\n            for t in np.linspace(0.05,0.95,19):\n                if y_true[:,i].max() > 0:\n                    f1 = f1_score(y_true[:,i], (p[:,i]>=t).astype(int))\n                else:\n                    f1 = 0.0\n                if f1>best_f1: best_f1, best_t = f1, t\n            ts.append(best_t)\n        thresholds = ts\n        macro_f1 = np.nanmean([f1_score(y_true[:,i], (p[:,i]>=thresholds[i]).astype(int)) for i in range(y_true.shape[1])])\n    return np.nanmean(aucs), macro_f1, thresholds\n\n# ----------------------- MODEL / LOSS / OPT -----------------------\nwith open(os.path.join(OUT_DIR, \"pos_weight.json\")) as f:\n    pos_weight_dict = json.load(f)\npos_weight_vec = torch.tensor([pos_weight_dict[c] for c in LABELS], dtype=torch.float32)\n\nmodel = timm.create_model(MODEL_NAME, pretrained=True, num_classes=len(LABELS)).to(device)\ncriterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight_vec.to(device))\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\nuse_amp = (device==\"cuda\")\nfrom torch.cuda.amp import autocast, GradScaler\nscaler = GradScaler(enabled=use_amp)\n\n# ----------------------- TRAIN (quick) -----------------------\nbest_f1 = -1.0\nhistory = []\nfor epoch in range(1, EPOCHS+1):\n    model.train(); train_loss, n = 0.0, 0\n    for x, y, _ in tr_ld:\n        if x.numel()==0: continue  # all skipped\n        x, y = x.to(device, non_blocking=True), y.to(device, non_blocking=True)\n        optimizer.zero_grad(set_to_none=True)\n        with autocast(enabled=use_amp):\n            logits = model(x)\n            loss   = criterion(logits, y)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer); scaler.update()\n        train_loss += loss.item()*x.size(0); n += x.size(0)\n    train_loss /= max(n,1)\n\n    # validate\n    model.eval(); ys, ps = [], []\n    with torch.no_grad():\n        for x, y, _ in va_ld:\n            if x.numel()==0: continue\n            x = x.to(device, non_blocking=True)\n            with autocast(enabled=use_amp):\n                logits = model(x)\n            ps.append(logits.float().cpu().numpy())\n            ys.append(y.numpy())\n    y_true   = np.concatenate(ys) if ys else np.zeros((0,len(LABELS)))\n    y_logits = np.concatenate(ps) if ps else np.zeros((0,len(LABELS)))\n    auc, mf1, thresholds = evaluate_logits(y_true, y_logits, search_thresholds=True)\n\n    history.append({\"epoch\": epoch, \"train_loss\": float(train_loss), \"val_auc\": float(auc), \"val_macro_f1\": float(mf1)})\n    print(f\"Epoch {epoch}: train_loss={train_loss:.4f} val_auc={auc:.4f} val_macro_f1={mf1:.4f}\")\n\n    ckpt = {\"model\": model.state_dict(), \"cfg\": {\"MODEL_NAME\": MODEL_NAME, \"IMG_SIZE\": IMG_SIZE, \"LABELS\": LABELS}, \"epoch\": epoch}\n    torch.save(ckpt, os.path.join(OUT_DIR, \"last.ckpt\"))\n    if (mf1==mf1) and (mf1>best_f1):   # mf1==mf1 guards against NaN\n        best_f1 = mf1\n        torch.save(ckpt, os.path.join(OUT_DIR, \"best.ckpt\"))\n        with open(os.path.join(OUT_DIR, \"thresholds.json\"), \"w\") as f:\n            json.dump({\"thresholds\": thresholds, \"labels\": LABELS}, f, indent=2)\n    with open(os.path.join(OUT_DIR, \"val_metrics.json\"), \"w\") as f:\n        json.dump(history, f, indent=2)\n\nprint(\"Training done. Best macro-F1:\", best_f1, \" Checkpoints @\", OUT_DIR)\n\n# ----------------------- INFERENCE ON test2/ -----------------------\nfrom timm import create_model as _cm\n\nckpt_path = Path(OUT_DIR)/\"best.ckpt\"\nif not ckpt_path.exists():\n    print(\"best.ckpt not found; using last.ckpt\")\n    ckpt_path = Path(OUT_DIR)/\"last.ckpt\"\nassert ckpt_path.exists(), f\"No checkpoint in {OUT_DIR}\"\n\nckpt = torch.load(ckpt_path, map_location=\"cpu\")\nlabels = ckpt.get(\"cfg\", {}).get(\"LABELS\", LABELS)\nmodel_name = ckpt.get(\"cfg\", {}).get(\"MODEL_NAME\", MODEL_NAME)\nimg_size = ckpt.get(\"cfg\", {}).get(\"IMG_SIZE\", IMG_SIZE)\n\nmodel_inf = _cm(model_name, pretrained=False, num_classes=len(labels))\nmodel_inf.load_state_dict(ckpt[\"model\"], strict=True)\nmodel_inf = model_inf.to(device).eval()\n\nimg_folder = Path(DATA_ROOT) / \"test2\"\nassert img_folder.exists(), f\"Image folder not found: {img_folder}\"\nimage_names = sorted([f.name for f in img_folder.iterdir() if f.suffix.lower() in (\".jpg\",\".png\")])\nassert len(image_names) > 0, f\"No images in {img_folder}\"\n\n# minimal DF with dummy labels\nd = {\"Image_name\": image_names}\nfor lab in labels: d[lab] = 0\ndf_dummy = pd.DataFrame(d)\n\nds = CXRDataset(df_dummy, DATA_ROOT, \"test2\", train=False, size=img_size)\ndl = DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True, collate_fn=safe_collate)\n\nprobs_all = []\nwith torch.no_grad():\n    for x, _, _ in dl:\n        if x.numel()==0: continue\n        x = x.to(device, non_blocking=True)\n        logits = model_inf(x)\n        probs_all.append(torch.sigmoid(logits).cpu().numpy())\nprobs = np.concatenate(probs_all, axis=0)\n\nsub = pd.DataFrame({\"Image_name\": image_names})\nfor i, lab in enumerate(labels):\n    sub[lab] = probs[:, i]\nsub_soft = str(Path(OUT_DIR)/\"submission.csv\")\nsub.to_csv(sub_soft, index=False)\nprint(\"Saved soft-prob submission to\", sub_soft)\n\nthr_path = Path(OUT_DIR)/\"thresholds.json\"\nif thr_path.exists():\n    th = np.array(json.load(open(thr_path))[\"thresholds\"], dtype=float)\n    hard = (probs >= th).astype(int)\n    hard_df = pd.DataFrame({\"Image_name\": image_names})\n    for i, lab in enumerate(labels):\n        hard_df[lab] = hard[:, i]\n    sub_hard = str(Path(OUT_DIR)/\"submission_hard.csv\")\n    hard_df.to_csv(sub_hard, index=False)\n    print(\"Saved hard-label CSV to\", sub_hard)\n\nprint(\"DONE \")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T18:59:05.653069Z","iopub.execute_input":"2025-10-15T18:59:05.653334Z","iopub.status.idle":"2025-10-16T00:52:29.771933Z","shell.execute_reply.started":"2025-10-15T18:59:05.653312Z","shell.execute_reply":"2025-10-16T00:52:29.768003Z"}},"outputs":[],"execution_count":null}]}