{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"},{"sourceId":12780021,"sourceType":"datasetVersion","datasetId":8079690}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc, warnings\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nimport torch.nn as nn\nimport timm\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\nwarnings.filterwarnings(\"ignore\")\n\n# ====================================================\n# Constants\n# ====================================================\nID_COL = \"SeriesInstanceUID\"\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\n# ====================================================\n# Config\n# ====================================================\nclass CFG:\n    model_name = \"tf_efficientnetv2_s.in21k_ft_in1k\"\n    size = 384\n    batch_size = 8\n    epochs = 5\n    lr = 2e-4\n    n_folds = 5\n    seed = 42\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# ====================================================\n# Dummy dataset class (replace with your dataset)\n# ====================================================\nclass RSNADataset(Dataset):\n    def __init__(self, df, labels):\n        self.df = df\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # X = image tensor [3, H, W] (replace with DICOM preproc if needed)\n        X = torch.randn(3, CFG.size, CFG.size)\n        y = torch.tensor(self.labels[idx], dtype=torch.float32)\n        return X, y\n\n# ====================================================\n# Model\n# ====================================================\ndef get_model():\n    model = timm.create_model(\n        CFG.model_name,\n        pretrained=False,   # ⬅️ change this to False (no HF download)\n        in_chans=3,\n        num_classes=len(LABEL_COLS),\n        drop_rate=0.2,\n        drop_path_rate=0.2\n    )\n    return model\n\n\n# ====================================================\n# Training loop\n# ====================================================\ndef train_and_save_oof(train_df, targets):\n    oof_preds = np.zeros((len(train_df), len(LABEL_COLS)))\n\n    skf = StratifiedKFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed)\n\n    for fold, (trn_idx, val_idx) in enumerate(skf.split(train_df, targets[:, -1])):  # stratify on \"Aneurysm Present\"\n        print(f\"===== Fold {fold} =====\")\n\n        model = get_model().to(CFG.device)\n        optimizer = AdamW(model.parameters(), lr=CFG.lr)\n        criterion = nn.BCEWithLogitsLoss()\n\n        trn_dataset = RSNADataset(train_df.iloc[trn_idx], targets[trn_idx])\n        val_dataset = RSNADataset(train_df.iloc[val_idx], targets[val_idx])\n\n        trn_loader = DataLoader(trn_dataset, batch_size=CFG.batch_size, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False)\n\n        best_auc = 0\n        for epoch in range(CFG.epochs):\n            model.train()\n            for X, y in trn_loader:\n                X, y = X.to(CFG.device), y.to(CFG.device)\n                optimizer.zero_grad()\n                out = model(X)\n                loss = criterion(out, y)\n                loss.backward()\n                optimizer.step()\n\n            # Validation\n            model.eval()\n            val_preds, val_targets = [], []\n            with torch.no_grad():\n                for X, y in val_loader:\n                    X = X.to(CFG.device)\n                    out = model(X)\n                    val_preds.append(torch.sigmoid(out).cpu().numpy())\n                    val_targets.append(y.numpy())\n            val_preds = np.concatenate(val_preds)\n            val_targets = np.concatenate(val_targets)\n\n            auc = roc_auc_score(val_targets[:, -1], val_preds[:, -1])\n            print(f\"Epoch {epoch} - Fold {fold} - Aneurysm AUC: {auc:.4f}\")\n\n            if auc > best_auc:\n                best_auc = auc\n                torch.save(model.state_dict(), f\"fold{fold}_best.pth\")\n\n        # Save OOF predictions\n        model.load_state_dict(torch.load(f\"fold{fold}_best.pth\"))\n        model.eval()\n        val_preds = []\n        with torch.no_grad():\n            for X, _ in val_loader:\n                X = X.to(CFG.device)\n                out = model(X)\n                val_preds.append(torch.sigmoid(out).cpu().numpy())\n        val_preds = np.concatenate(val_preds)\n        oof_preds[val_idx] = val_preds\n\n    # Save to CSV\n    oof_df = pd.DataFrame(oof_preds, columns=LABEL_COLS)\n    oof_df.insert(0, ID_COL, train_df[ID_COL].values)\n    oof_df[\"target\"] = targets[:, -1]  # ground truth for aneurysm\n    oof_df.to_csv(\"/kaggle/working/oof.csv\", index=False)\n    print(\"Saved OOF predictions to /kaggle/working/oof.csv\")\n'''\n# Example usage (replace with your real train dataframe + labels)\ndummy_df = pd.DataFrame({\"SeriesInstanceUID\": [f\"id_{i}\" for i in range(100)]})\ndummy_labels = np.random.randint(0, 2, (100, len(LABEL_COLS)))\ntrain_and_save_oof(dummy_df, dummy_labels)\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:34:53.786515Z","iopub.execute_input":"2025-09-26T13:34:53.786745Z","iopub.status.idle":"2025-09-26T13:35:58.363036Z","shell.execute_reply.started":"2025-09-26T13:34:53.786726Z","shell.execute_reply":"2025-09-26T13:35:58.362168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# fit_calibrators.py\ndef_calibrate():\n    import pandas as pd\n    import joblib\n    import numpy as np\n    from sklearn.linear_model import LogisticRegression\n    from sklearn.isotonic import IsotonicRegression\n    from pathlib import Path\n    \n    # Paths\n    oof_path = Path(\"/kaggle/working/oof.csv\")\n    out_path = Path(\"/kaggle/working/calibrators.pkl\")\n    \n    LABEL_COLS = [\n        'Left Infraclinoid Internal Carotid Artery',\n        'Right Infraclinoid Internal Carotid Artery',\n        'Left Supraclinoid Internal Carotid Artery',\n        'Right Supraclinoid Internal Carotid Artery',\n        'Left Middle Cerebral Artery',\n        'Right Middle Cerebral Artery',\n        'Anterior Communicating Artery',\n        'Left Anterior Cerebral Artery',\n        'Right Anterior Cerebral Artery',\n        'Left Posterior Communicating Artery',\n        'Right Posterior Communicating Artery',\n        'Basilar Tip',\n        'Other Posterior Circulation',\n        'Aneurysm Present',\n    ]\n    '''\n    if not oof_path.exists():\n        raise FileNotFoundError(f\"OOF not found: {oof_path}\")\n        \n    oof = pd.read_csv(oof_path)\n    '''\n    print(\"Loaded OOF:\", oof.shape)\n    print(\"OOF columns:\", oof.columns.tolist()[:30])\n    \n    # Determine ground-truth column name(s)\n    # We'll assume the OOF has a 'target' or 'Aneurysm_gt' or similar.\n    # Try to detect a ground truth column automatically.\n    possible_gt_names = [\"target\", \"AneurysmPresent_gt\", \"aneurysm_label\", \"Aneurysm_gt\",\n                         \"Aneurysm Present_gt\", \"aneurysm\", \"y_true\"]\n    gt_col = None\n    for cand in possible_gt_names:\n        if cand in oof.columns:\n            gt_col = cand\n            break\n\n    # Heuristic: if an explicit per-row ground truth column not found, try common patterns:\n    if gt_col is None:\n        # If oof contains a column matching 'Aneurysm Present' with integer values 0/1,\n        # we may assume that is the prediction column, and underlying ground truth may be stored as 'target'\n        # Fallback: if there's a 'target' column use it.\n        if \"target\" in oof.columns:\n            gt_col = \"target\"\n    \n    if gt_col is None:\n        # Let user know and raise error for clarity.\n        raise ValueError(\"Could not find a ground-truth column in OOF. Please ensure your OOF has a column named 'target' (or similar).\")\n    \n    print(\"Using ground-truth column:\", gt_col)\n    \n    calibrators = {}\n    \n    for col in LABEL_COLS:\n        if col not in oof.columns:\n            print(f\"[Skip] Predictions column not found in OOF: {col}\")\n            calibrators[col] = None\n            continue\n\n        y_scores = oof[col].values\n        y_true = oof[gt_col].values  # NOTE: uses same GT for all labels; replace per-label GT if you have them\n    \n        # Check that y_true contains at least two classes\n        unique = np.unique(y_true)\n        if unique.shape[0] < 2:\n            print(f\"[Skip] Ground truth for column '{col}' has only one class ({unique}). Skipping calibrator.\")\n            calibrators[col] = None\n            continue\n    \n        # Also ensure scores are not degenerate\n        if np.allclose(y_scores, y_scores[0]):\n            print(f\"[Skip] Predictions for '{col}' are constant; skipping calibrator.\")\n            calibrators[col] = None\n            continue\n\n        # Prefer Platt scaling (LogisticRegression). If it fails, fallback to isotonic\n        try:\n            lr = LogisticRegression(max_iter=2000)\n            lr.fit(y_scores.reshape(-1, 1), y_true)\n            calibrators[col] = (\"platt\", lr)\n            print(f\"[OK] Trained Platt calibrator for '{col}'\")\n        except Exception as e:\n            print(f\"[Warn] Platt failed for '{col}' ({e}), trying isotonic.\")\n            try:\n                iso = IsotonicRegression(out_of_bounds='clip')\n                iso.fit(y_scores, y_true)\n                calibrators[col] = (\"isotonic\", iso)\n                print(f\"[OK] Trained isotonic calibrator for '{col}'\")\n            except Exception as e2:\n                print(f\"[Skip] Both calibrators failed for '{col}': {e2}. Saving None.\")\n                calibrators[col] = None\n    \n    # Save calibrators dictionary\n    joblib.dump(calibrators, out_path)\n    print(\"Saved calibrators to:\", out_path) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:35:58.364789Z","iopub.execute_input":"2025-09-26T13:35:58.365255Z","iopub.status.idle":"2025-09-26T13:35:58.501926Z","shell.execute_reply.started":"2025-09-26T13:35:58.365234Z","shell.execute_reply":"2025-09-26T13:35:58.500768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------------\n# Build and save submission.parquet (robust, runs predict() per series)\n# Paste this into your inference notebook AFTER models are loaded\n# and predict(series_path) is defined.\n# ---------------------------\n\nimport os\nfrom pathlib import Path\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nimport traceback\nfrom tqdm import tqdm\n\n# Make sure LABEL_COLS matches your model output order\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\n# Conservative fallback prediction if a series fails\nFALLBACK_ROW = [0.1] * len(LABEL_COLS)\n'''\ndef find_series_dirs(search_roots=None):\n    \"\"\"Search for DICOM series directories (contains at least one .dcm file).\"\"\"\n    roots = search_roots or ['/kaggle/input', '/kaggle/working']\n    series_dirs = []\n    for root in roots:\n        if not os.path.exists(root): \n            continue\n        for entry in Path(root).rglob('*'):\n            if entry.is_dir():\n                try:\n                    # quick check: any .dcm file in directory (non-recursive)\n                    for f in entry.iterdir():\n                        if f.is_file() and f.suffix.lower() == '.dcm':\n                            series_dirs.append(str(entry))\n                            break\n                except Exception:\n                    # skip unreadable dirs\n                    continue\n    return sorted(set(series_dirs))\n\n# Find candidates automatically\nseries_dirs = find_series_dirs()\n\n# If none found automatically, also try direct children of /kaggle/input\nif not series_dirs:\n    # fallback: any subdirectory of input that contains files\n    for root in ['/kaggle/input', '/kaggle/working']:\n        if not os.path.exists(root): continue\n        for d in Path(root).iterdir():\n            if d.is_dir():\n                # treat the folder as a series if it has files\n                files = list(d.rglob('*'))\n                if files:\n                    series_dirs.append(str(d))\n    series_dirs = sorted(set(series_dirs))\n\nprint(f\"Found {len(series_dirs)} candidate series directories (first 10):\")\nfor d in series_dirs[:10]:\n    print(\"  \", d)\n\nif not series_dirs:\n    raise RuntimeError(\"No series directories found. If your test series are located in a different path, set `series_dirs` manually.\")\n\n# Run predictions and collect rows\nrows = []\nids = []\n\nfor series_path in tqdm(series_dirs, desc=\"Predicting series\"):\n    sid = os.path.basename(series_path.rstrip('/')) or series_path  # fallback UID\n    try:\n        # predict should return a polars.DataFrame with columns = LABEL_COLS (no ID col)\n        pred_df = predict(series_path)  # your existing function\n        # If function returns a polars.DataFrame with ID column, handle that\n        if isinstance(pred_df, pl.DataFrame):\n            pdf = pred_df.to_pandas()\n        elif isinstance(pred_df, pd.DataFrame):\n            pdf = pred_df\n        else:\n            # if predict returns a numpy array\n            if isinstance(pred_df, np.ndarray):\n                arr = pred_df\n                if arr.ndim == 1:\n                    arr = arr.reshape(1, -1)\n                pdf = pd.DataFrame(arr, columns=LABEL_COLS)\n        # If pdf has the ID column, drop it and use our sid\n        if 'ID' in pdf.columns or 'SeriesInstanceUID' in pdf.columns:\n            # find label columns intersection\n            label_cols_present = [c for c in LABEL_COLS if c in pdf.columns]\n            row_values = pdf[label_cols_present].iloc[0].astype(float).values\n            # ensure correct column order; if some missing, fill with fallback\n            final_row = []\n            for col in LABEL_COLS:\n                if col in pdf.columns:\n                    final_row.append(float(pdf[col].iloc[0]))\n                else:\n                    final_row.append(0.1)\n        else:\n            # assume pdf columns are exactly LABEL_COLS order\n            try:\n                row_values = pdf.values.flatten().astype(float)\n                if row_values.size != len(LABEL_COLS):\n                    # mismatch — try to map by column names\n                    final_row = []\n                    for col in LABEL_COLS:\n                        if col in pdf.columns:\n                            final_row.append(float(pdf[col].iloc[0]))\n                        else:\n                            final_row.append(0.1)\n                else:\n                    final_row = row_values.tolist()\n            except Exception:\n                final_row = FALLBACK_ROW\n        # ensure numeric and length-correct\n        if len(final_row) != len(LABEL_COLS):\n            final_row = FALLBACK_ROW\n        rows.append(final_row)\n        ids.append(sid)\n    except Exception as e:\n        # on any failure, log and append fallback\n        print(f\"[Error] series {sid} failed: {e}\")\n        traceback.print_exc()\n        rows.append(FALLBACK_ROW)\n        ids.append(sid)\n\n# Build submission DataFrame\nsub_df = pd.DataFrame(rows, columns=LABEL_COLS)\nsub_df.insert(0, \"ID\", ids)\n\n# Save as submission.parquet exactly where Kaggle expects it\nout_path = \"/kaggle/working/submission.parquet\"\nsub_df.to_parquet(out_path, index=False)\nprint(f\"✅ Saved submission to {out_path} (rows: {len(sub_df)})\")\nprint(sub_df.head())'''\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:41:11.607374Z","iopub.execute_input":"2025-09-26T13:41:11.60765Z","iopub.status.idle":"2025-09-26T13:52:50.495474Z","shell.execute_reply.started":"2025-09-26T13:41:11.607627Z","shell.execute_reply":"2025-09-26T13:52:50.494434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\nimport pydicom\n\nimport kaggle_evaluation.rsna_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:52:57.684104Z","iopub.execute_input":"2025-09-26T13:52:57.684846Z","iopub.status.idle":"2025-09-26T13:52:58.225649Z","shell.execute_reply.started":"2025-09-26T13:52:57.684823Z","shell.execute_reply":"2025-09-26T13:52:58.225072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ID_COL = 'SeriesInstanceUID'\n\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\n# All tags (other than PixelData and SeriesInstanceUID) that may be in a test set dcm file\nDICOM_TAG_ALLOWLIST = [\n    'BitsAllocated',\n    'BitsStored',\n    'Columns',\n    'FrameOfReferenceUID',\n    'HighBit',\n    'ImageOrientationPatient',\n    'ImagePositionPatient',\n    'InstanceNumber',\n    'Modality',\n    'PatientID',\n    'PhotometricInterpretation',\n    'PixelRepresentation',\n    'PixelSpacing',\n    'PlanarConfiguration',\n    'RescaleIntercept',\n    'RescaleSlope',\n    'RescaleType',\n    'Rows',\n    'SOPClassUID',\n    'SOPInstanceUID',\n    'SamplesPerPixel',\n    'SliceThickness',\n    'SpacingBetweenSlices',\n    'StudyInstanceUID',\n    'TransferSyntaxUID',\n]\n\n# Replace this function with your inference code.\n# You can return either a Pandas or Polars dataframe, though Polars is recommended.\n# Each prediction (except the very first) must be returned within 30 minutes of the series being provided.\ndef predict(series_path: str) -> pl.DataFrame | pd.DataFrame:\n    \"\"\"Make a prediction.\"\"\"\n    # --------- Replace this section with your own prediction code ---------\n    series_id = os.path.basename(series_path)\n    \n    all_filepaths = []\n    for root, _, files in os.walk(series_path):\n        for file in files:\n            if file.endswith('.dcm'):\n                all_filepaths.append(os.path.join(root, file))\n    all_filepaths.sort()\n    \n    # Collect tags from the dicoms\n    tags = defaultdict(list)\n    tags['SeriesInstanceUID'] = series_id\n    global dcms\n    for filepath in all_filepaths:\n        ds = pydicom.dcmread(filepath, force=True)\n        tags['filepath'].append(filepath)\n        for tag in DICOM_TAG_ALLOWLIST:\n            tags[tag].append(getattr(ds, tag, None))\n        # The image is in ds.PixelData\n\n    # ... do some machine learning magic ...\n    predictions = pl.DataFrame(\n        data=[[series_id] + [0.5] * len(LABEL_COLS)],\n        schema=[ID_COL, *LABEL_COLS],\n        orient='row',\n    )\n    # ----------------------------------------------------------------------\n\n    if isinstance(predictions, pl.DataFrame):\n        assert predictions.columns == [ID_COL, *LABEL_COLS]\n    elif isinstance(predictions, pd.DataFrame):\n        assert (predictions.columns == [ID_COL, *LABEL_COLS]).all()\n    else:\n        raise TypeError('The predict function must return a DataFrame')\n\n    # ----------------------------- IMPORTANT ------------------------------\n    # You MUST have the following code in your `predict` function\n    # to prevent \"out of disk space\" errors. This is a temporary workaround\n    # as we implement improvements to our evaluation system.\n    shutil.rmtree('/kaggle/shared', ignore_errors=True)\n    # ----------------------------------------------------------------------\n    \n    return predictions.drop(ID_COL)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:52:59.141321Z","iopub.execute_input":"2025-09-26T13:52:59.142028Z","iopub.status.idle":"2025-09-26T13:52:59.15058Z","shell.execute_reply.started":"2025-09-26T13:52:59.142007Z","shell.execute_reply":"2025-09-26T13:52:59.149668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.rsna_inference_server.RSNAInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway()\n    display(pl.read_parquet('/kaggle/working/submission.parquet'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T13:53:00.101242Z","iopub.execute_input":"2025-09-26T13:53:00.101966Z","iopub.status.idle":"2025-09-26T13:53:13.176683Z","shell.execute_reply.started":"2025-09-26T13:53:00.10194Z","shell.execute_reply":"2025-09-26T13:53:13.176062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}