{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13190393,"sourceType":"competition"},{"sourceId":12637336,"sourceType":"datasetVersion","datasetId":7981664}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## References\n- https://www.kaggle.com/code/dennisfong/dicom-pngs-for-rsna-intracranial-aneurysm/notebook\n\n## Notebooks\n- My Train Notebook: here\n- My Inference Notebook: https://www.kaggle.com/code/ichigoe/inference-224x224-dicom-pngs-efficientnetb0\n\n## Training Overview\n### What to Train:\n- 5-Frame EfficientNetV2-S (Optimized CNN Architecture)\n- 14-class Multi-label Classifier (13 anatomical locations + Aneurysm Present)\n\n### What to Train With:\n- 5 frames simultaneous input (224×224 PNG/DICOM)\n- Weighted BCE Loss with Focal Loss (Aneurysm Present weight=3.0)\n\n### Key Features:\n- **True Patient Separation**: DICOM StudyInstanceUID-based cross-validation for patient-level separation\n- **Strategic 5-frame Sampling**: Central region focus (middle 60%) where aneurysms are most common\n- **3-Channel Input Processing**: Middle slice + Maximum Intensity Projection + Standard deviation projection\n- **CLAHE Contrast Adaptation**: Modality-specific enhancement for CTA/MRA/MRI variations\n- **Strong Augmentation**: 15° rotation, elastic transforms, noise simulation for scanner robustness\n- **Robust Percentile Normalization**: Outlier-resistant preprocessing using 1st-99th percentile clipping\n- **Medical Metadata Integration**: Patient age and sex features for enhanced classification\n- **Modality-specific Windowing**: Optimized intensity windows (CTA: 50/350, MRA: 600/1200, MRI: 40/80)\n- **Mixed Precision Training**: GPU-optimized training with gradient accumulation (batch size 8, accumulation 4)\n- **LRU Caching**: Performance optimization for frequently accessed DICOM data\n\n### Improvements:\n- **Patient Leakage Prevention**: DICOM metadata extraction ensures no patient overlap between train/validation\n\n### Evaluation Metric:\n- **Weighted Multi-label AUC ROC**\n- **Final Score = (Aneurysm AUC + Individual AUC Average) / 2**\n\n### Expected Performance:\n- **Improved CV/LB Alignment**: True patient separation should reduce overfitting\n- **Better Generalization**: Strategic sampling and robust preprocessing for real-world variation\n- **Reduced CV/LB Gap**: From ~0.44 gap to healthy 0.10-0.15 range through proper validation","metadata":{}},{"cell_type":"code","source":"# Environment setup and library imports\nimport os\nimport glob\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport functools\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom typing import List, Tuple, Optional\nimport gc\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import roc_auc_score\nimport pydicom\n\nwarnings.filterwarnings('ignore')\n\ndef set_seed(seed=42):\n    \"\"\"Set all random seeds for reproducibility\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nset_seed(42)\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"Memory: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n    print(f\"CUDA version: {torch.version.cuda}\")\n    torch.cuda.empty_cache()\nelse:\n    raise RuntimeError(\"CUDA is not available! This code requires GPU.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:15:48.600198Z","iopub.execute_input":"2025-08-07T07:15:48.600368Z","iopub.status.idle":"2025-08-07T07:16:09.465445Z","shell.execute_reply.started":"2025-08-07T07:15:48.600352Z","shell.execute_reply":"2025-08-07T07:16:09.464685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    # Data paths\n    DATA_DIR = \"/kaggle/input/rsna-2025-intracranial-aneurysm-png-224x224\"\n    CVT_PNG_DIR = os.path.join(DATA_DIR, \"cvt_png\")\n    SERIES_MAPPING_PATH = os.path.join(DATA_DIR, \"series_index_mapping.csv\")\n    LOCALIZERS_PATH = os.path.join(DATA_DIR, \"train_localizers_with_relative.csv\")\n    TRAIN_CSV_PATH = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\n    \n    # Model parameters for 8-frame processing\n    NUM_FRAMES = 8\n    IMAGE_SIZE = 224\n    NUM_CLASSES = 14\n    BATCH_SIZE = 6  # Reduced for 8-frame processing\n    NUM_EPOCHS = 50\n    LEARNING_RATE = 5e-5\n    \n    # Model configuration\n    MODEL_NAME_BACKBONE = \"tf_efficientnetv2_s.in1k\"\n    USE_METADATA = True\n    USE_WINDOWING = True\n    USE_3CHANNEL_INPUT = True\n    USE_IMPROVED_LOSS = True\n    USE_CLAHE = True\n    USE_STRONG_AUGMENTATION = True\n    \n    # GPU optimization settings\n    NUM_WORKERS = 2\n    PIN_MEMORY = True\n    PREFETCH_FACTOR = 2\n    PERSISTENT_WORKERS = True\n    \n    # Training parameters with robust cross-validation\n    NUM_FOLDS = 5\n    FOLD = 0\n    ACCUMULATION_STEPS = 5  # Adjusted for smaller batch size\n    EARLY_STOPPING_PATIENCE = 3\n    USE_GROUP_CV = True\n    \n    # Data loading optimization\n    CACHE_SIZE = 100\n    \n    # Output\n    OUTPUT_DIR = \"/kaggle/working\"\n    MODEL_NAME = \"eightframe_efficientnetv2s\"\n\nconfig = Config()\n\nprint(\"=== Configuration Summary ===\")\nprint(f\"Model Backbone: {config.MODEL_NAME_BACKBONE}\")\nprint(f\"Number of Frames: {config.NUM_FRAMES}\")\nprint(f\"Batch Size: {config.BATCH_SIZE}\")\nprint(f\"Accumulation Steps: {config.ACCUMULATION_STEPS}\")\nprint(f\"Effective Batch Size: {config.BATCH_SIZE * config.ACCUMULATION_STEPS}\")\nprint(f\"CLAHE Enabled: {config.USE_CLAHE}\")\nprint(f\"Strong Augmentation: {config.USE_STRONG_AUGMENTATION}\")\nprint(f\"Group Cross-Validation: {config.USE_GROUP_CV}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:16:09.466245Z","iopub.execute_input":"2025-08-07T07:16:09.466515Z","iopub.status.idle":"2025-08-07T07:16:09.473667Z","shell.execute_reply.started":"2025-08-07T07:16:09.466489Z","shell.execute_reply":"2025-08-07T07:16:09.473025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\nprint(\"Loading data...\")\ntrain_df = pd.read_csv(config.TRAIN_CSV_PATH)\nseries_mapping_df = pd.read_csv(config.SERIES_MAPPING_PATH)\nlocalizers_df = pd.read_csv(config.LOCALIZERS_PATH)\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Series mapping shape: {series_mapping_df.shape}\")\nprint(f\"Localizers shape: {localizers_df.shape}\")\n\n# Define target columns\nTARGET_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery', \n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery', \n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present'\n]\n\nprint(f\"Target columns: {len(TARGET_COLS)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:16:09.475844Z","iopub.execute_input":"2025-08-07T07:16:09.476461Z","iopub.status.idle":"2025-08-07T07:16:16.64669Z","shell.execute_reply.started":"2025-08-07T07:16:09.476442Z","shell.execute_reply":"2025-08-07T07:16:16.646057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_windowing_params(modality: str) -> Tuple[float, float]:\n    \"\"\"Get optimal windowing parameters for different modalities\"\"\"\n    windows = {\n        'CT': (40, 80),\n        'CTA': (50, 350), \n        'MRA': (600, 1200),\n        'MRI': (40, 80),\n        'MR': (40, 80)\n    }\n    return windows.get(modality, (40, 80))\n\ndef apply_dicom_windowing(img: np.ndarray, window_center: float, window_width: float) -> np.ndarray:\n    \"\"\"Apply DICOM windowing to normalize image intensities\"\"\"\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    img = (img - img_min) / (img_max - img_min + 1e-7)\n    return (img * 255).astype(np.uint8)\n\ndef apply_clahe_normalization(img: np.ndarray, modality: str) -> np.ndarray:\n    \"\"\"Apply CLAHE with modality-specific optimization\"\"\"\n    if not config.USE_CLAHE:\n        return img\n        \n    if modality in ['CTA', 'MRA']:\n        # Vascular imaging: stronger contrast improvement\n        clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n        img_clahe = cv2.convertScaleAbs(img_clahe, alpha=1.1, beta=5)\n    elif modality in ['MRI', 'MR']:\n        # MRI: gentler improvement with gamma correction\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n        img_clahe = np.power(img_clahe / 255.0, 0.9) * 255\n        img_clahe = img_clahe.astype(np.uint8)\n    else:\n        # CT: standard CLAHE\n        clahe = cv2.createCLAHE(clipLimit=2.5, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n    \n    return img_clahe\n\ndef robust_normalization(volume: np.ndarray) -> np.ndarray:\n    \"\"\"Apply robust normalization using percentiles\"\"\"\n    p1, p99 = np.percentile(volume.flatten(), [1, 99])\n    volume_norm = np.clip(volume, p1, p99)\n    \n    if p99 > p1:\n        volume_norm = (volume_norm - p1) / (p99 - p1 + 1e-7)\n    else:\n        volume_norm = np.zeros_like(volume_norm)\n        \n    return (volume_norm * 255).astype(np.uint8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:16:16.647387Z","iopub.execute_input":"2025-08-07T07:16:16.647656Z","iopub.status.idle":"2025-08-07T07:16:16.656822Z","shell.execute_reply.started":"2025-08-07T07:16:16.647635Z","shell.execute_reply":"2025-08-07T07:16:16.655945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_3channel_input_8frame(volume: np.ndarray) -> np.ndarray:\n    \"\"\"Create 3-channel input from 8-frame volume optimized for aneurysm detection\"\"\"\n    if len(volume) == 0:\n        return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n    \n    # Middle slice (most important for anatomical reference)\n    middle_slice = volume[len(volume) // 2]\n    \n    # Maximum Intensity Projection (MIP) - optimized for vascular structures\n    mip = np.max(volume, axis=0)\n    \n    # Standard deviation projection for texture analysis\n    std_proj = np.std(volume, axis=0).astype(np.float32)\n    \n    # Normalize standard deviation projection with robust method\n    if std_proj.max() > std_proj.min():\n        p1, p99 = np.percentile(std_proj, [5, 95])\n        std_proj = np.clip(std_proj, p1, p99)\n        std_proj = ((std_proj - p1) / (p99 - p1 + 1e-7) * 255).astype(np.uint8)\n    else:\n        std_proj = np.zeros_like(std_proj, dtype=np.uint8)\n    \n    return np.stack([middle_slice, mip, std_proj], axis=-1)\n\ndef smart_8_frame_sampling(volume_paths: List[str], series_uid: str = None) -> List[str]:\n    \"\"\"Intelligent 8-frame sampling strategy using every other frame\"\"\"\n    n = len(volume_paths)\n    \n    if n <= 8:\n        # If we have 8 or fewer frames, use all available\n        result = volume_paths[:]\n        # Pad with repetitions if needed\n        while len(result) < 8:\n            result.extend(volume_paths[:8-len(result)])\n        return result[:8]\n    \n    # Skip every other frame starting from a strategic position\n    # Start from 10% into the volume to avoid empty slices at the beginning\n    start_idx = max(0, int(n * 0.1))\n    \n    # Calculate step size to get 8 frames with good coverage\n    available_frames = n - start_idx\n    step = max(1, available_frames // 8)\n    \n    indices = []\n    current_idx = start_idx\n    while len(indices) < 8 and current_idx < n:\n        indices.append(current_idx)\n        current_idx += step\n    \n    # If we need more frames, fill from the remaining\n    while len(indices) < 8:\n        remaining = [i for i in range(n) if i not in indices]\n        if remaining:\n            indices.append(remaining[len(indices) % len(remaining)])\n        else:\n            indices.append(indices[-1])  # Duplicate last frame\n    \n    return [volume_paths[i] for i in indices[:8]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:16:16.657791Z","iopub.execute_input":"2025-08-07T07:16:16.658098Z","iopub.status.idle":"2025-08-07T07:16:16.693421Z","shell.execute_reply.started":"2025-08-07T07:16:16.658073Z","shell.execute_reply":"2025-08-07T07:16:16.69271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_dicom_patient_info(series_uid: str) -> Tuple[str, str]:\n    \"\"\"Extract StudyInstanceUID and PatientID from DICOM metadata\"\"\"\n    try:\n        dicom_dir = f\"/kaggle/input/rsna-intracranial-aneurysm-detection/series/{series_uid}\"\n        if os.path.exists(dicom_dir):\n            dcm_files = [f for f in os.listdir(dicom_dir) if f.endswith('.dcm')]\n            if dcm_files:\n                ds = pydicom.dcmread(\n                    os.path.join(dicom_dir, dcm_files[0]), \n                    stop_before_pixels=True, \n                    force=True\n                )\n                study_uid = getattr(ds, 'StudyInstanceUID', None)\n                patient_id = getattr(ds, 'PatientID', None)\n                return study_uid or f\"fallback_{series_uid[:32]}\", patient_id\n    except Exception:\n        pass\n    \n    # Fallback: use longer prefix from series UID\n    return f\"fallback_{series_uid[:32]}\", f\"fallback_{series_uid[:32]}\"\n\n@functools.lru_cache(maxsize=5000)\ndef get_patient_group_cached(series_uid: str) -> str:\n    \"\"\"Get patient group with caching for performance\"\"\"\n    study_uid, patient_id = extract_dicom_patient_info(series_uid)\n    # Use StudyInstanceUID as primary identifier\n    return study_uid if study_uid and not study_uid.startswith('fallback_') else patient_id\n\ndef create_frame_paths_8frame():\n    \"\"\"Create mapping from series to frame paths optimized for 8-frame processing\"\"\"\n    frame_paths = {}\n    \n    print(\"Creating 8-frame optimized paths from series_index_mapping.csv...\")\n    \n    for series_uid in tqdm(train_df['SeriesInstanceUID'].unique(), desc=\"Processing series\"):\n        # Get series data from mapping\n        series_data = series_mapping_df[series_mapping_df['SeriesInstanceUID'] == series_uid]\n        \n        if len(series_data) == 0:\n            frame_paths[series_uid] = []\n            continue\n            \n        # Get row from train_df to check which diseases are present\n        train_row = train_df[train_df['SeriesInstanceUID'] == series_uid].iloc[0]\n        \n        # Find any disease location that has this series\n        found_paths = []\n        \n        # Check each target column (except Aneurysm Present)\n        for target_col in TARGET_COLS[:-1]:\n            if train_row[target_col] == 1:\n                location_clean = target_col.replace('/', '_')\n                series_dir = os.path.join(config.CVT_PNG_DIR, location_clean, series_uid)\n                \n                if os.path.exists(series_dir):\n                    png_files = sorted(glob.glob(os.path.join(series_dir, \"*.png\")))\n                    if png_files:\n                        found_paths = png_files\n                        break\n        \n        # If no paths found in disease folders, use DICOM structure\n        if not found_paths:\n            dicom_dir = f\"/kaggle/input/rsna-intracranial-aneurysm-detection/series/{series_uid}\"\n            if os.path.exists(dicom_dir):\n                num_frames = len(series_data)\n                found_paths = [f\"dummy_path_{i:04d}.png\" for i in range(num_frames)]\n        \n        # Apply smart 8-frame sampling\n        if found_paths:\n            found_paths = smart_8_frame_sampling(found_paths, series_uid)\n        \n        frame_paths[series_uid] = found_paths\n    \n    return frame_paths\n\n# Create optimized paths\nframe_paths_dict = create_frame_paths_8frame()\nprint(f\"Created 8-frame optimized paths for {len(frame_paths_dict)} series\")\n\n# Filter data\nvalid_series = [uid for uid, paths in frame_paths_dict.items() if len(paths) > 0]\ntrain_df_filtered = train_df[train_df['SeriesInstanceUID'].isin(valid_series)].copy()\nprint(f\"Filtered train data shape: {train_df_filtered.shape}\")\n\n# Check distribution\naneurysm_dist_filtered = train_df_filtered['Aneurysm Present'].value_counts()\nprint(f\"Aneurysm Present distribution: {aneurysm_dist_filtered.to_dict()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:16:16.694124Z","iopub.execute_input":"2025-08-07T07:16:16.694298Z","iopub.status.idle":"2025-08-07T07:21:56.705385Z","shell.execute_reply.started":"2025-08-07T07:16:16.694284Z","shell.execute_reply":"2025-08-07T07:21:56.70469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_robust_cv_split(train_df, n_splits=5):\n    \"\"\"Create robust cross-validation split with true patient separation from DICOM\"\"\"\n    \n    print(\"Creating patient-separated cross-validation split...\")\n    print(\"Extracting true patient IDs from DICOM metadata...\")\n    print(\"This will take a few minutes but ensures proper patient separation.\")\n    \n    # Extract true patient groups from DICOM metadata\n    patient_groups = []\n    for series_uid in tqdm(train_df['SeriesInstanceUID'], desc=\"Reading DICOM patient info\"):\n        patient_group = get_patient_group_cached(series_uid)\n        patient_groups.append(patient_group)\n    \n    # Add patient groups to dataframe\n    train_df = train_df.copy()\n    train_df['patient_id'] = patient_groups\n    \n    n_groups = train_df['patient_id'].nunique()\n    print(f\"True patient groups found: {n_groups}\")\n    \n    # Check if we have enough patient groups\n    if n_groups < n_splits:\n        print(f\"Not enough patient groups ({n_groups}) for {n_splits}-fold CV.\")\n        print(\"Falling back to StratifiedKFold...\")\n        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n        return list(skf.split(train_df, train_df['Aneurysm Present']))\n    \n    # Create stratification key combining modality and aneurysm presence\n    train_df['stratify_key'] = (\n        train_df['Modality'].astype(str) + '_' + \n        train_df['Aneurysm Present'].astype(str)\n    )\n    \n    print(f\"Stratification keys: {train_df['stratify_key'].unique()}\")\n    \n    # Use GroupKFold to ensure patient-level separation\n    group_kfold = GroupKFold(n_splits=n_splits)\n    \n    splits = []\n    for fold_idx, (train_idx, val_idx) in enumerate(group_kfold.split(\n        train_df, \n        groups=train_df['patient_id']\n    )):\n        # Validate patient separation\n        train_fold = train_df.iloc[train_idx]\n        val_fold = train_df.iloc[val_idx]\n        \n        # Check for patient overlap (should be 0)\n        train_patients = set(train_fold['patient_id'])\n        val_patients = set(val_fold['patient_id'])\n        overlap = train_patients.intersection(val_patients)\n        \n        train_dist = train_fold['Aneurysm Present'].value_counts(normalize=True)\n        val_dist = val_fold['Aneurysm Present'].value_counts(normalize=True)\n        \n        print(f\"Fold {fold_idx}:\")\n        print(f\"  Train: {len(train_fold)} samples ({len(train_patients)} patients)\")\n        print(f\"  Val: {len(val_fold)} samples ({len(val_patients)} patients)\")\n        print(f\"  Patient overlap: {len(overlap)} (should be 0!)\")\n        print(f\"  Aneurysm Present - Train: {train_dist.get(1, 0):.3f}, Val: {val_dist.get(1, 0):.3f}\")\n        \n        if len(overlap) > 0:\n            print(f\"  WARNING: Found {len(overlap)} overlapping patients!\")\n        \n        splits.append((train_idx, val_idx))\n    \n    return splits\n\n# Create robust train/validation split\ncv_splits = create_robust_cv_split(train_df_filtered, config.NUM_FOLDS)\ntrain_indices, val_indices = cv_splits[config.FOLD]\n\ntrain_fold_df = train_df_filtered.iloc[train_indices]\nval_fold_df = train_df_filtered.iloc[val_indices]\n\nprint(f\"\\nRobust CV Fold {config.FOLD} Summary:\")\nprint(f\"Train fold size: {len(train_fold_df)}\")\nprint(f\"Validation fold size: {len(val_fold_df)}\")\n\n# Check distributions\nprint(f\"Train Aneurysm Present: {train_fold_df['Aneurysm Present'].value_counts().to_dict()}\")\nprint(f\"Val Aneurysm Present: {val_fold_df['Aneurysm Present'].value_counts().to_dict()}\")\n\n# Check modality distribution\nprint(f\"Train Modality distribution: {train_fold_df['Modality'].value_counts().to_dict()}\")\nprint(f\"Val Modality distribution: {val_fold_df['Modality'].value_counts().to_dict()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:21:56.706316Z","iopub.execute_input":"2025-08-07T07:21:56.706599Z","iopub.status.idle":"2025-08-07T07:27:15.234518Z","shell.execute_reply.started":"2025-08-07T07:21:56.70657Z","shell.execute_reply":"2025-08-07T07:27:15.233824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data transforms with strong augmentation\nif config.USE_STRONG_AUGMENTATION:\n    print(\"Using strong augmentation for better generalization...\")\n    train_transform = A.Compose([\n        # Geometric transformations (safe for medical images)\n        A.Rotate(limit=15, p=0.7),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=10, p=0.6),\n        \n        # Advanced geometric transformations for robustness\n        A.ElasticTransform(alpha=50, sigma=5, p=0.3),\n        A.GridDistortion(num_steps=3, distort_limit=0.1, p=0.3),\n        \n        # Image quality variations (simulate different scanners/protocols)\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.6),\n        A.CLAHE(clip_limit=2.0, tile_grid_size=(8,8), p=0.5),\n        A.RandomGamma(gamma_limit=(80, 120), p=0.4),\n        \n        # Noise simulation (scanner differences)\n        A.GaussNoise(var_limit=(10, 80), p=0.4),\n        A.ISONoise(color_shift=(0.01, 0.05), intensity=(0.1, 0.5), p=0.3),\n        A.Blur(blur_limit=3, p=0.2),\n        \n        # Medical image specific augmentations\n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        \n        # Normalization\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\nelse:\n    print(\"Using standard augmentation...\")\n    train_transform = A.Compose([\n        A.Rotate(limit=10, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.3),\n        A.GaussNoise(var_limit=(10, 50), p=0.2),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\nval_transform = A.Compose([\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:15.23529Z","iopub.execute_input":"2025-08-07T07:27:15.235564Z","iopub.status.idle":"2025-08-07T07:27:15.256271Z","shell.execute_reply.started":"2025-08-07T07:27:15.235543Z","shell.execute_reply":"2025-08-07T07:27:15.255698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EightFrameDataset(Dataset):\n    \"\"\"Dataset optimized for 8-frame processing with CLAHE\"\"\"\n    def __init__(self, df, frame_paths_dict, series_mapping_df, num_frames=8, \n                 transform=None, is_training=True):\n        self.df = df.reset_index(drop=True)\n        self.frame_paths_dict = frame_paths_dict\n        self.series_mapping_df = series_mapping_df\n        self.num_frames = num_frames\n        self.transform = transform\n        self.is_training = is_training\n        \n        # Simple LRU cache for recently accessed data\n        self._cache = {}\n        self._cache_keys = []\n        self._max_cache_size = config.CACHE_SIZE\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        # Check cache first\n        if idx in self._cache:\n            return self._cache[idx]\n        \n        row = self.df.iloc[idx]\n        series_uid = row['SeriesInstanceUID']\n        \n        # Get labels\n        labels = torch.tensor(row[TARGET_COLS].values.astype(np.float32))\n        \n        # Extract metadata\n        metadata = self._extract_metadata(row)\n        \n        # Load 8-frame 3-channel image\n        image = self._load_8frame_3channel_image(series_uid, row)\n        \n        result = (image, labels, metadata)\n        \n        # Update cache\n        self._update_cache(idx, result)\n        \n        return result\n    \n    def _update_cache(self, idx, data):\n        \"\"\"Update LRU cache\"\"\"\n        if len(self._cache) >= self._max_cache_size:\n            # Remove oldest entry\n            oldest_idx = self._cache_keys.pop(0)\n            del self._cache[oldest_idx]\n        \n        self._cache[idx] = data\n        self._cache_keys.append(idx)\n    \n    def _extract_metadata(self, row) -> torch.Tensor:\n        \"\"\"Extract and normalize metadata\"\"\"\n        if not config.USE_METADATA:\n            return torch.tensor([0.0, 0.0], dtype=torch.float32)\n        \n        # Age processing\n        age = row.get('PatientAge', 50)\n        if pd.isna(age):\n            age = 50\n        elif isinstance(age, str):\n            age = int(''.join(filter(str.isdigit, age[:3])) or '50')\n        age = min(float(age), 100.0) / 100.0\n        \n        # Sex processing\n        sex = row.get('PatientSex', 'M')\n        sex = 1.0 if sex == 'M' else 0.0\n        \n        return torch.tensor([age, sex], dtype=torch.float32)\n    \n    def _load_8frame_3channel_image(self, series_uid: str, row) -> torch.Tensor:\n        \"\"\"Load 8-frame 3-channel image with processing\"\"\"\n        paths = self.frame_paths_dict.get(series_uid, [])\n        \n        try:\n            if len(paths) == 0 or paths[0].startswith('dummy_path'):\n                volume = self._load_volume_from_dicom_8frame(series_uid, row)\n            else:\n                volume = self._load_volume_from_png_8frame(paths)\n            \n            # Apply robust normalization\n            volume = robust_normalization(volume)\n            \n            # Create 3-channel input optimized for 8 frames\n            image = create_3channel_input_8frame(volume)\n            \n            # Apply transforms\n            if self.transform:\n                transformed = self.transform(image=image)\n                image = transformed['image']\n            \n            return image\n            \n        except Exception as e:\n            print(f\"Error loading {series_uid}: {e}\")\n            # Return dummy image\n            dummy_image = np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n            if self.transform:\n                transformed = self.transform(image=dummy_image)\n                return transformed['image']\n            return torch.zeros(3, config.IMAGE_SIZE, config.IMAGE_SIZE)\n    \n    def _load_volume_from_png_8frame(self, paths: List[str]) -> np.ndarray:\n        \"\"\"Load PNG volume optimized for 8 frames\"\"\"\n        volume = []\n        \n        # Ensure we have exactly 8 paths\n        if len(paths) != 8:\n            paths = smart_8_frame_sampling(paths)\n        \n        for path in paths:\n            try:\n                img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n                if img is not None:\n                    img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), \n                                   interpolation=cv2.INTER_AREA)\n                    volume.append(img)\n            except:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        return np.array(volume) if volume else np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n    \n    def _load_volume_from_dicom_8frame(self, series_uid: str, row) -> np.ndarray:\n        \"\"\"Load DICOM volume optimized for 8 frames with CLAHE\"\"\"\n        series_data = self.series_mapping_df[\n            self.series_mapping_df['SeriesInstanceUID'] == series_uid\n        ].sort_values('relative_index')\n        \n        if len(series_data) == 0:\n            return np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n        \n        volume = []\n        modality = row.get('Modality', 'CT')\n        \n        # Sample exactly 8 slices using every-other-frame strategy\n        if len(series_data) <= 8:\n            sampled_data = series_data\n        else:\n            # Apply smart 8-frame sampling logic to indices\n            all_indices = list(range(len(series_data)))\n            sampled_indices = smart_8_frame_sampling([str(i) for i in all_indices])\n            sampled_indices = [int(i) for i in sampled_indices]\n            sampled_data = series_data.iloc[sampled_indices]\n        \n        for _, dicom_row in sampled_data.iterrows():\n            try:\n                ds = pydicom.dcmread(dicom_row['dicom_filename'])\n                img = ds.pixel_array.astype(np.float32)\n                \n                # Handle multi-frame/color images\n                if img.ndim == 3:\n                    if img.shape[-1] == 3:\n                        img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2GRAY).astype(np.float32)\n                    else:\n                        img = img[:, :, 0]\n                \n                # Apply rescale if available\n                if hasattr(ds, 'RescaleSlope') and hasattr(ds, 'RescaleIntercept'):\n                    img = img * ds.RescaleSlope + ds.RescaleIntercept\n                \n                # Apply windowing\n                if config.USE_WINDOWING:\n                    window_center, window_width = get_windowing_params(modality)\n                    img = apply_dicom_windowing(img, window_center, window_width)\n                else:\n                    img_min, img_max = img.min(), img.max()\n                    if img_max > img_min:\n                        img = ((img - img_min) / (img_max - img_min) * 255).astype(np.uint8)\n                    else:\n                        img = np.zeros_like(img, dtype=np.uint8)\n                \n                # Apply CLAHE improvement\n                img = apply_clahe_normalization(img, modality)\n                \n                # High quality resize\n                img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), \n                               interpolation=cv2.INTER_AREA)\n                volume.append(img)\n                \n            except Exception as e:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n                continue\n        \n        # Ensure we have exactly 8 frames\n        while len(volume) < 8:\n            if volume:\n                volume.append(volume[-1])  # Duplicate last frame\n            else:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        return np.array(volume[:8])  # Take exactly 8 frames","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:15.258264Z","iopub.execute_input":"2025-08-07T07:27:15.258452Z","iopub.status.idle":"2025-08-07T07:27:15.282843Z","shell.execute_reply.started":"2025-08-07T07:27:15.258437Z","shell.execute_reply":"2025-08-07T07:27:15.282328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create 8-frame datasets\nprint(\"Creating 8-frame datasets with CLAHE...\")\ntrain_dataset = EightFrameDataset(\n    train_fold_df, \n    frame_paths_dict, \n    series_mapping_df,\n    num_frames=config.NUM_FRAMES,\n    transform=train_transform,\n    is_training=True\n)\n\nval_dataset = EightFrameDataset(\n    val_fold_df,\n    frame_paths_dict,\n    series_mapping_df,\n    num_frames=config.NUM_FRAMES, \n    transform=val_transform,\n    is_training=False\n)\n\n# Create optimized data loaders\nprint(\"Creating optimized data loaders for 8-frame processing...\")\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=config.BATCH_SIZE,\n    shuffle=True,\n    num_workers=config.NUM_WORKERS,\n    pin_memory=config.PIN_MEMORY,\n    drop_last=True,\n    prefetch_factor=config.PREFETCH_FACTOR,\n    persistent_workers=config.PERSISTENT_WORKERS\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=config.BATCH_SIZE,\n    shuffle=False,\n    num_workers=config.NUM_WORKERS,\n    pin_memory=config.PIN_MEMORY,\n    prefetch_factor=config.PREFETCH_FACTOR,\n    persistent_workers=config.PERSISTENT_WORKERS\n)\n\nprint(f\"Train batches: {len(train_loader)}\")\nprint(f\"Validation batches: {len(val_loader)}\")\n\n# Test 8-frame data loading speed\nprint(\"Testing 8-frame data loading speed...\")\nimport time\n\nstart_time = time.time()\nfor i, batch in enumerate(train_loader):\n    if i >= 5:  # Test first 5 batches\n        break\n    images, labels, metadata = batch\n    print(f\"Batch {i+1}: Images shape: {images.shape}, Device: {images.device}\")\n\nelapsed = time.time() - start_time\nprint(f\"Loaded 5 batches in {elapsed:.2f} seconds ({elapsed/5:.2f}s per batch)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:15.283454Z","iopub.execute_input":"2025-08-07T07:27:15.283688Z","iopub.status.idle":"2025-08-07T07:27:19.703122Z","shell.execute_reply.started":"2025-08-07T07:27:15.283666Z","shell.execute_reply":"2025-08-07T07:27:19.702021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ImprovedMultiFrameModel(nn.Module):\n    \"\"\"Model with EfficientNetV2-S and metadata integration for 8-frame processing\"\"\"\n    def __init__(self, num_frames=8, num_classes=14, pretrained=True):\n        super(ImprovedMultiFrameModel, self).__init__()\n        self.num_frames = num_frames\n        self.num_classes = num_classes\n        self.use_3channel = config.USE_3CHANNEL_INPUT\n        self.use_metadata = config.USE_METADATA\n        \n        # Backbone: EfficientNetV2-S\n        print(f\"Loading backbone: {config.MODEL_NAME_BACKBONE}\")\n        self.backbone = timm.create_model(\n            config.MODEL_NAME_BACKBONE,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        self.feature_dim = self.backbone.num_features\n        print(f\"Backbone {config.MODEL_NAME_BACKBONE}: {self.feature_dim} features\")\n        \n        # Metadata processing\n        if self.use_metadata:\n            self.meta_fc = nn.Sequential(\n                nn.Linear(2, 16),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(16, 32),\n                nn.ReLU()\n            )\n            classifier_input_dim = self.feature_dim + 32\n        else:\n            classifier_input_dim = self.feature_dim\n        \n        # Classifier\n        self.classifier = nn.Sequential(\n            nn.Linear(classifier_input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, num_classes)\n        )\n        \n    def forward(self, x, meta=None):\n        # 3-channel input processing (optimized for 8-frame data)\n        features = self.backbone(x)  # (batch_size, feature_dim)\n        \n        # Metadata integration\n        if self.use_metadata and meta is not None:\n            meta_features = self.meta_fc(meta)\n            features = torch.cat([features, meta_features], dim=1)\n        \n        # Classification\n        output = self.classifier(features)\n        return output\n\n# Initialize 8-frame model\nprint(\"Initializing 8-frame model...\")\nmodel = ImprovedMultiFrameModel(\n    num_frames=config.NUM_FRAMES,\n    num_classes=config.NUM_CLASSES,\n    pretrained=True\n)\n\nmodel = model.to(device)\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")\nprint(f\"Model device: {next(model.parameters()).device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:19.704957Z","iopub.execute_input":"2025-08-07T07:27:19.705287Z","iopub.status.idle":"2025-08-07T07:27:21.035537Z","shell.execute_reply.started":"2025-08-07T07:27:19.705242Z","shell.execute_reply":"2025-08-07T07:27:21.034799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    \"\"\"Focal Loss for addressing class imbalance\"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        \n    def forward(self, inputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-bce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * bce_loss\n        return focal_loss.mean()\n\nclass WeightedMultiLabelLoss(nn.Module):\n    \"\"\"Weighted multi-label loss\"\"\"\n    def __init__(self, aneurysm_weight=3.0):\n        super(WeightedMultiLabelLoss, self).__init__()\n        self.weights = torch.ones(config.NUM_CLASSES, device=device)\n        self.weights[-1] = aneurysm_weight\n        \n    def forward(self, outputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction='none')\n        weighted_loss = bce_loss * self.weights\n        return weighted_loss.mean()\n\nclass ImprovedLoss(nn.Module):\n    \"\"\"Advanced combined loss function\"\"\"\n    def __init__(self, aneurysm_weight=3.0, focal_weight=0.3):\n        super(ImprovedLoss, self).__init__()\n        self.aneurysm_weight = aneurysm_weight\n        self.focal_weight = focal_weight\n        \n        self.weights = torch.ones(config.NUM_CLASSES, device=device)\n        self.weights[-1] = aneurysm_weight\n        \n        self.focal_loss = FocalLoss(alpha=1, gamma=2)\n        \n    def forward(self, outputs, targets):\n        # Weighted BCE\n        bce_loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction='none')\n        weighted_bce = (bce_loss * self.weights).mean()\n        \n        # Focal Loss\n        focal_loss = self.focal_loss(outputs, targets)\n        \n        # Combination\n        return (1 - self.focal_weight) * weighted_bce + self.focal_weight * focal_loss\n\ndef get_loss_function():\n    \"\"\"Get loss function based on configuration\"\"\"\n    if config.USE_IMPROVED_LOSS:\n        return ImprovedLoss(aneurysm_weight=3.0, focal_weight=0.3)\n    else:\n        return WeightedMultiLabelLoss(aneurysm_weight=3.0)\n\ndef calculate_competition_metric(y_true, y_pred):\n    \"\"\"Calculate competition metric: weighted multilabel AUC ROC\"\"\"\n    individual_aucs = []\n    \n    # Calculate AUC for first 13 classes\n    for i in range(13):\n        try:\n            if len(np.unique(y_true[:, i])) > 1:\n                auc = roc_auc_score(y_true[:, i], y_pred[:, i])\n            else:\n                auc = 0.5\n            individual_aucs.append(auc)\n        except:\n            individual_aucs.append(0.5)\n    \n    # Calculate AUC for Aneurysm Present\n    try:\n        if len(np.unique(y_true[:, 13])) > 1:\n            aneurysm_present_auc = roc_auc_score(y_true[:, 13], y_pred[:, 13])\n        else:\n            aneurysm_present_auc = 0.5\n    except:\n        aneurysm_present_auc = 0.5\n    \n    # Final score\n    avg_individual = np.mean(individual_aucs)\n    final_score = (aneurysm_present_auc + avg_individual) / 2\n    \n    return final_score, aneurysm_present_auc, avg_individual, individual_aucs\n\n# Training setup\ncriterion = get_loss_function()\noptimizer = AdamW(model.parameters(), lr=config.LEARNING_RATE, weight_decay=1e-4)\nscheduler = CosineAnnealingLR(optimizer, T_max=config.NUM_EPOCHS, eta_min=1e-6)\n\n# Mixed precision training\nscaler = torch.cuda.amp.GradScaler()\n\nprint(\"Training setup complete\")\nprint(f\"Using loss function: {type(criterion).__name__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:21.036649Z","iopub.execute_input":"2025-08-07T07:27:21.037017Z","iopub.status.idle":"2025-08-07T07:27:21.112319Z","shell.execute_reply.started":"2025-08-07T07:27:21.036992Z","shell.execute_reply":"2025-08-07T07:27:21.111443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch_optimized(model, train_loader, criterion, optimizer, scaler, device, accumulation_steps):\n    \"\"\"Optimized training function for 8-frame processing\"\"\"\n    model.train()\n    running_loss = 0.0\n    \n    optimizer.zero_grad()\n    \n    for batch_idx, (images, targets, metadata) in enumerate(tqdm(train_loader, desc=\"Training 8-Frame\")):\n        # Move data to GPU efficiently\n        images = images.to(device, non_blocking=True)\n        targets = targets.to(device, non_blocking=True)\n        metadata = metadata.to(device, non_blocking=True)\n        \n        with torch.cuda.amp.autocast():\n            outputs = model(images, metadata)\n            loss = criterion(outputs, targets)\n            loss = loss / accumulation_steps\n        \n        scaler.scale(loss).backward()\n        \n        if (batch_idx + 1) % accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        \n        running_loss += loss.item() * accumulation_steps\n    \n    # Handle remaining gradients\n    if len(train_loader) % accumulation_steps != 0:\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n    \n    return running_loss / len(train_loader)\n\ndef validate_epoch_optimized(model, val_loader, criterion, device):\n    \"\"\"Optimized validation function for 8-frame processing\"\"\"\n    model.eval()\n    running_loss = 0.0\n    all_outputs = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for images, targets, metadata in tqdm(val_loader, desc=\"Validating 8-Frame\"):\n            # Move data to GPU efficiently\n            images = images.to(device, non_blocking=True)\n            targets = targets.to(device, non_blocking=True)\n            metadata = metadata.to(device, non_blocking=True)\n                \n            with torch.cuda.amp.autocast():\n                logits = model(images, metadata)\n                loss = criterion(logits, targets)\n            \n            outputs = torch.sigmoid(logits)\n            \n            running_loss += loss.item()\n            all_outputs.append(outputs.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    \n    final_score, aneurysm_auc, avg_individual, individual_aucs = calculate_competition_metric(\n        all_targets, all_outputs\n    )\n    \n    return running_loss / len(val_loader), final_score, aneurysm_auc, avg_individual\n\ndef check_gpu_utilization():\n    \"\"\"Check current GPU memory usage\"\"\"\n    if torch.cuda.is_available():\n        allocated = torch.cuda.memory_allocated() / 1024**3\n        reserved = torch.cuda.memory_reserved() / 1024**3\n        max_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3\n        print(f\"GPU Memory - Allocated: {allocated:.2f}GB, Reserved: {reserved:.2f}GB, Total: {max_memory:.2f}GB\")\n        utilization = (allocated/max_memory)*100\n        print(f\"GPU Utilization: {utilization:.1f}%\")\n        return utilization\n    return 0\n\nprint(\"Initial GPU status for 8-frame processing:\")\ncheck_gpu_utilization()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:21.11319Z","iopub.execute_input":"2025-08-07T07:27:21.113849Z","iopub.status.idle":"2025-08-07T07:27:21.133799Z","shell.execute_reply.started":"2025-08-07T07:27:21.113824Z","shell.execute_reply":"2025-08-07T07:27:21.133053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training loop with 8-frame optimization and true patient separation\nbest_score = 0.0\nbest_epoch = 0\npatience_counter = 0\ntrain_losses = []\nval_losses = []\nval_scores = []\n\nprint(\"Starting 8-frame training with patient-separated CV...\")\nprint(f\"Batch size: {config.BATCH_SIZE}, Workers: {config.NUM_WORKERS}\")\nprint(f\"Frames per sample: {config.NUM_FRAMES}\")\nprint(f\"CLAHE enabled: {config.USE_CLAHE}\")\nprint(f\"Strong augmentation: {config.USE_STRONG_AUGMENTATION}\")\nprint(f\"True patient separation: {config.USE_GROUP_CV}\")\n\nfor epoch in range(config.NUM_EPOCHS):\n    print(f\"\\nEpoch {epoch+1}/{config.NUM_EPOCHS}\")\n    print(\"-\" * 50)\n    \n    # Training with 8-frame processing\n    train_loss = train_epoch_optimized(\n        model, train_loader, criterion, optimizer, scaler, device, config.ACCUMULATION_STEPS\n    )\n    \n    # Validation with 8-frame processing\n    val_loss, val_score, aneurysm_auc, avg_individual = validate_epoch_optimized(\n        model, val_loader, criterion, device\n    )\n    \n    # Learning rate scheduling\n    scheduler.step()\n    \n    # Log metrics\n    train_losses.append(train_loss)\n    val_losses.append(val_loss)\n    val_scores.append(val_score)\n    \n    print(f\"Train Loss: {train_loss:.6f}\")\n    print(f\"Val Loss: {val_loss:.6f}\")\n    print(f\"Val Score: {val_score:.6f}\")\n    print(f\"Aneurysm AUC: {aneurysm_auc:.6f}\")\n    print(f\"Avg Individual AUC: {avg_individual:.6f}\")\n    print(f\"Learning Rate: {optimizer.param_groups[0]['lr']:.8f}\")\n    \n    # GPU utilization\n    gpu_util = check_gpu_utilization()\n    \n    # Early stopping and model saving\n    if val_score > best_score:\n        best_score = val_score\n        best_epoch = epoch + 1\n        patience_counter = 0\n        \n        # Save model\n        model_path = os.path.join(config.OUTPUT_DIR, f\"{config.MODEL_NAME}_best.pth\")\n        torch.save({\n            'epoch': epoch + 1,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'scheduler_state_dict': scheduler.state_dict(),\n            'best_score': best_score,\n            'val_loss': val_loss,\n            'aneurysm_auc': aneurysm_auc,\n            'avg_individual_auc': avg_individual,\n            'config': config,\n            'model_config': {\n                'backbone': config.MODEL_NAME_BACKBONE,\n                'num_frames': config.NUM_FRAMES,\n                'use_3channel': config.USE_3CHANNEL_INPUT,\n                'use_metadata': config.USE_METADATA,\n                'use_windowing': config.USE_WINDOWING,\n                'use_improved_loss': config.USE_IMPROVED_LOSS,\n                'use_clahe': config.USE_CLAHE,\n                'use_strong_augmentation': config.USE_STRONG_AUGMENTATION,\n                'use_group_cv': config.USE_GROUP_CV\n            }\n        }, model_path)\n        \n        print(f\"New best model saved! Score: {best_score:.6f}\")\n    else:\n        patience_counter += 1\n        print(f\"No improvement. Patience: {patience_counter}/{config.EARLY_STOPPING_PATIENCE}\")\n        \n        if patience_counter >= config.EARLY_STOPPING_PATIENCE:\n            print(f\"Early stopping triggered at epoch {epoch + 1}\")\n            break\n    \n    # Memory cleanup\n    torch.cuda.empty_cache()\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"8-FRAME TRAINING WITH PATIENT SEPARATION COMPLETED\")\nprint(\"=\"*70)\nprint(f\"Best Score: {best_score:.6f} at Epoch {best_epoch}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:27:21.134646Z","iopub.execute_input":"2025-08-07T07:27:21.134822Z","iopub.status.idle":"2025-08-07T07:40:08.403129Z","shell.execute_reply.started":"2025-08-07T07:27:21.134809Z","shell.execute_reply":"2025-08-07T07:40:08.401869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training results visualization and summary\nfig, axes = plt.subplots(1, 3, figsize=(18, 5))\n\n# Loss curves\naxes[0].plot(range(1, len(train_losses)+1), train_losses, 'b-', label='Train Loss', linewidth=2)\naxes[0].plot(range(1, len(val_losses)+1), val_losses, 'r-', label='Val Loss', linewidth=2)\naxes[0].set_xlabel('Epoch')\naxes[0].set_ylabel('Loss')\naxes[0].set_title('8-Frame Training: Loss Curves')\naxes[0].legend()\naxes[0].grid(True, alpha=0.3)\n\n# Validation score\naxes[1].plot(range(1, len(val_scores)+1), val_scores, 'g-', label='Val Score', linewidth=2)\naxes[1].axhline(y=best_score, color='r', linestyle='--', alpha=0.7, \n                label=f'Best: {best_score:.6f}')\naxes[1].set_xlabel('Epoch')\naxes[1].set_ylabel('Competition Score')\naxes[1].set_title('8-Frame Training: Competition Score')\naxes[1].legend()\naxes[1].grid(True, alpha=0.3)\n\n# Learning rate schedule\nlr_values = []\ntemp_optimizer = AdamW(model.parameters(), lr=config.LEARNING_RATE, weight_decay=1e-4)\ntemp_scheduler = CosineAnnealingLR(temp_optimizer, T_max=config.NUM_EPOCHS, eta_min=1e-6)\nfor _ in range(config.NUM_EPOCHS):\n    lr_values.append(temp_optimizer.param_groups[0]['lr'])\n    temp_scheduler.step()\n\naxes[2].plot(range(1, len(lr_values)+1), lr_values, 'purple', linewidth=2, label='Learning Rate')\naxes[2].set_xlabel('Epoch')\naxes[2].set_ylabel('Learning Rate')\naxes[2].set_title('Learning Rate Schedule')\naxes[2].legend()\naxes[2].grid(True, alpha=0.3)\naxes[2].set_yscale('log')\n\nplt.tight_layout()\nplt.show()\n\n# Final model summary\nmodel_path = os.path.join(config.OUTPUT_DIR, f\"{config.MODEL_NAME}_best.pth\")\nif os.path.exists(model_path):\n    checkpoint = torch.load(model_path, map_location='cpu', weights_only=False)\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"8-FRAME MODEL WITH PATIENT SEPARATION SUMMARY\")\n    print(\"=\"*60)\n    print(f\"Best Epoch: {checkpoint['epoch']}\")\n    print(f\"Best Score: {checkpoint['best_score']:.6f}\")\n    print(f\"Aneurysm AUC: {checkpoint['aneurysm_auc']:.6f}\")\n    print(f\"Avg Individual AUC: {checkpoint['avg_individual_auc']:.6f}\")\n    print(f\"Model Size: {os.path.getsize(model_path) / (1024*1024):.1f} MB\")\n    print(f\"- CLAHE contrast adaptation: {config.USE_CLAHE}\")\n    print(f\"- Strong augmentation: {config.USE_STRONG_AUGMENTATION}\")\n\n\nprint(\"=\"*60)\nprint(\"TRAINING COMPLETE WITH PATIENT-SEPARATED CV!\")\nprint(\"This should significantly improve LB performance.\")\nprint(\"=\"*60)\n\n# Final cleanup\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-07T07:40:43.615336Z","iopub.execute_input":"2025-08-07T07:40:43.616008Z","iopub.status.idle":"2025-08-07T07:40:44.816084Z","shell.execute_reply.started":"2025-08-07T07:40:43.615971Z","shell.execute_reply":"2025-08-07T07:40:44.815521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}