{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nMEDICAL MODEL - GUARANTEED SAVE\nFull medical features + location knowledge + guaranteed to save\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nimport pydicom\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport pickle\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nimport gc\nimport os\n\nDATA_PATH = '/kaggle/input/rsna-intracranial-aneurysm-detection'\nSERIES_PATH = Path(DATA_PATH) / 'series'\n\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\n# MEDICAL KNOWLEDGE from your research\nLOCATION_PREVALENCE = {\n    'Anterior Communicating Artery': 0.325,  # 30-35% most common\n    'Left Posterior Communicating Artery': 0.125,\n    'Right Posterior Communicating Artery': 0.125,\n    'Left Middle Cerebral Artery': 0.10,\n    'Right Middle Cerebral Artery': 0.10,\n    'Basilar Tip': 0.06,\n    'Left Supraclinoid Internal Carotid Artery': 0.04,\n    'Right Supraclinoid Internal Carotid Artery': 0.04,\n    'Left Anterior Cerebral Artery': 0.025,\n    'Right Anterior Cerebral Artery': 0.025,\n    'Other Posterior Circulation': 0.05,\n    'Left Infraclinoid Internal Carotid Artery': 0.015,\n    'Right Infraclinoid Internal Carotid Artery': 0.015,\n}\n\n# Visibility scores (higher = better visibility on that modality)\nCT_VISIBILITY = {\n    'Anterior Communicating Artery': 0.9,\n    'Left Middle Cerebral Artery': 0.9,\n    'Right Middle Cerebral Artery': 0.9,\n    'Left Posterior Communicating Artery': 0.85,\n    'Right Posterior Communicating Artery': 0.85,\n    'Left Supraclinoid Internal Carotid Artery': 0.85,\n    'Right Supraclinoid Internal Carotid Artery': 0.85,\n    'Basilar Tip': 0.8,\n    'Left Anterior Cerebral Artery': 0.75,\n    'Right Anterior Cerebral Artery': 0.75,\n    'Other Posterior Circulation': 0.6,\n    'Left Infraclinoid Internal Carotid Artery': 0.5,\n    'Right Infraclinoid Internal Carotid Artery': 0.5,\n}\n\ndef extract_medical_features(series_id):\n    \"\"\"Extract full medical features.\"\"\"\n    \n    series_path = SERIES_PATH / series_id\n    dcm_files = sorted(list(series_path.glob('*.dcm')))\n    \n    if not dcm_files:\n        return None\n    \n    features = {'SeriesInstanceUID': series_id}\n    \n    try:\n        ds = pydicom.dcmread(dcm_files[0], force=True)\n        \n        # === MODALITY (Critical for visibility) ===\n        modality = str(getattr(ds, 'Modality', 'Unknown'))\n        features['is_ct'] = 1.0 if modality == 'CT' else 0.0\n        features['is_mr'] = 1.0 if modality == 'MR' else 0.0\n        \n        # === PROTOCOL QUALITY ===\n        features['num_slices'] = float(len(dcm_files))\n        features['num_slices_log'] = np.log(len(dcm_files) + 1)\n        \n        slice_thickness = float(getattr(ds, 'SliceThickness', 1.0))\n        features['slice_thickness'] = slice_thickness\n        features['is_thin'] = 1.0 if slice_thickness < 0.7 else 0.0\n        features['is_good'] = 1.0 if slice_thickness < 1.0 else 0.0\n        \n        pixel_spacing = getattr(ds, 'PixelSpacing', [0.5, 0.5])\n        features['pixel_spacing'] = float(pixel_spacing[0])\n        features['is_high_res'] = 1.0 if float(pixel_spacing[0]) < 0.4 else 0.0\n        \n        # Protocol quality score\n        features['protocol_quality'] = (\n            (1.0 if features['is_thin'] else 0.5) *\n            (1.0 if features['is_high_res'] else 0.7) *\n            (1.0 if features['num_slices'] > 100 else 0.8)\n        )\n        \n        # === INTENSITY ANALYSIS (Sample 3 regions) ===\n        n_slices = len(dcm_files)\n        sample_indices = [\n            int(n_slices * 0.3),\n            int(n_slices * 0.5),\n            int(n_slices * 0.7),\n        ]\n        \n        intensities = []\n        for idx in sample_indices:\n            if 0 <= idx < n_slices:\n                ds_sample = pydicom.dcmread(dcm_files[idx], force=True)\n                if hasattr(ds_sample, 'pixel_array'):\n                    pixels = ds_sample.pixel_array.astype(float)\n                    intercept = getattr(ds_sample, 'RescaleIntercept', 0)\n                    slope = getattr(ds_sample, 'RescaleSlope', 1)\n                    pixels = pixels * slope + intercept\n                    intensities.append(pixels)\n        \n        if intensities:\n            all_pixels = np.concatenate([img.flatten() for img in intensities])\n            \n            features['intensity_mean'] = np.mean(all_pixels)\n            features['intensity_std'] = np.std(all_pixels)\n            features['intensity_max'] = np.max(all_pixels)\n            features['intensity_p95'] = np.percentile(all_pixels, 95)\n            features['intensity_p99'] = np.percentile(all_pixels, 99)\n            \n            # CRITICAL: Enhancement ratio (aneurysms are bright!)\n            features['enhancement_ratio'] = features['intensity_p99'] / (features['intensity_mean'] + 1.0)\n            features['contrast_quality'] = (features['intensity_p95'] - features['intensity_mean']) / (features['intensity_mean'] + 1.0)\n            features['has_strong_enhancement'] = 1.0 if features['enhancement_ratio'] > 3.0 else 0.0\n            \n        else:\n            features.update({\n                'intensity_mean': 0.0, 'intensity_std': 0.0, 'intensity_max': 0.0,\n                'intensity_p95': 0.0, 'intensity_p99': 0.0,\n                'enhancement_ratio': 1.0, 'contrast_quality': 0.0,\n                'has_strong_enhancement': 0.0,\n            })\n        \n        # === MEDICAL INTERACTION FEATURES ===\n        features['ct_with_enhancement'] = features['is_ct'] * features['enhancement_ratio']\n        features['quality_coverage'] = features['protocol_quality'] * features['num_slices_log']\n        features['optimal_for_small'] = features['is_thin'] * features['has_strong_enhancement']\n        \n    except Exception as e:\n        return None\n    \n    return features\n\ndef train_medical_model():\n    \"\"\"Train with medical knowledge + guaranteed save.\"\"\"\n    \n    print(\"=\"*80)\n    print(\"MEDICAL MODEL TRAINING - WITH GUARANTEED SAVE\")\n    print(\"=\"*80)\n    \n    # Load\n    print(\"\\n[1/5] Loading data...\")\n    train_df = pd.read_csv(f'{DATA_PATH}/train.csv')\n    print(f\"   ✅ {len(train_df):,} samples\")\n    \n    # Save checkpoint\n    with open('checkpoint_loaded.pkl', 'wb') as f:\n        pickle.dump({'n_samples': len(train_df)}, f)\n    \n    # Extract\n    print(\"\\n[2/5] Extracting medical features...\")\n    feature_list = []\n    \n    for series_id in tqdm(train_df['SeriesInstanceUID'], desc=\"   Progress\"):\n        feats = extract_medical_features(series_id)\n        if feats:\n            feature_list.append(feats)\n        if len(feature_list) % 100 == 0:\n            gc.collect()\n    \n    features_df = pd.DataFrame(feature_list)\n    train_data = train_df.merge(features_df, on='SeriesInstanceUID', how='inner')\n    \n    print(f\"\\n   ✅ {len(features_df)} samples, {len(features_df.columns)-1} features\")\n    print(f\"   Features include:\")\n    print(f\"      - Modality (CT/MR)\")\n    print(f\"      - Protocol quality (slice thickness, resolution)\")\n    print(f\"      - Intensity analysis (enhancement detection)\")\n    print(f\"      - Medical interactions (optimal for small aneurysms)\")\n    \n    feature_cols = [col for col in features_df.columns if col != 'SeriesInstanceUID']\n    \n    # Save checkpoint\n    with open('checkpoint_features.pkl', 'wb') as f:\n        pickle.dump({'feature_cols': feature_cols, 'n_features': len(feature_cols)}, f)\n    \n    # Train\n    print(\"\\n[3/5] Training location-specific models...\")\n    \n    models = {}\n    cv_scores = {}\n    n_trained = 0\n    n_skipped = 0\n    \n    for target in LABEL_COLS:\n        X = train_data[feature_cols].fillna(0).values\n        y = train_data[target].values\n        \n        n_pos = y.sum()\n        \n        if n_pos < 10:\n            print(f\"   {target:50s}: SKIP ({n_pos} pos)\")\n            n_skipped += 1\n            continue\n        \n        # Get medical info\n        expected_prev = LOCATION_PREVALENCE.get(target, 0.1)\n        \n        # Train with 3-fold (faster)\n        skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n        fold_scores = []\n        fold_models = []\n        \n        for train_idx, val_idx in skf.split(X, y):\n            X_train, X_val = X[train_idx], X[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n            \n            scaler = StandardScaler()\n            X_train_scaled = scaler.fit_transform(X_train)\n            X_val_scaled = scaler.transform(X_val)\n            \n            n_neg = len(y_train) - y_train.sum()\n            scale_pos_weight = n_neg / (y_train.sum() + 1e-6)\n            \n            lgbm = lgb.LGBMClassifier(\n                n_estimators=200,\n                max_depth=4,\n                learning_rate=0.05,\n                scale_pos_weight=scale_pos_weight,\n                random_state=42,\n                verbose=-1\n            )\n            \n            lgbm.fit(X_train_scaled, y_train)\n            \n            if len(np.unique(y_val)) > 1:\n                pred = lgbm.predict_proba(X_val_scaled)[:, 1]\n                auc = roc_auc_score(y_val, pred)\n                fold_scores.append(auc)\n                fold_models.append({'model': lgbm, 'scaler': scaler})\n        \n        if fold_scores:\n            mean_auc = np.mean(fold_scores)\n            print(f\"   {target:50s}: {mean_auc:.4f} (prev:{expected_prev:.1%})\")\n            cv_scores[target] = mean_auc\n            models[target] = fold_models\n            n_trained += 1\n    \n    print(f\"\\n   ✅ Trained {n_trained} models, Skipped {n_skipped}\")\n    \n    # Save checkpoint\n    with open('checkpoint_trained.pkl', 'wb') as f:\n        pickle.dump({'n_trained': n_trained, 'cv_scores': cv_scores}, f)\n    \n    # Calculate\n    print(\"\\n[4/5] Results...\")\n    \n    aneurysm_score = cv_scores.get('Aneurysm Present', 0.5)\n    location_scores = [cv_scores[col] for col in LABEL_COLS[:-1] if col in cv_scores]\n    \n    if location_scores:\n        location_avg = np.mean(location_scores)\n        comp_score = (aneurysm_score + location_avg) / 2\n        print(f\"\\n   Aneurysm Present: {aneurysm_score:.4f}\")\n        print(f\"   Location Average: {location_avg:.4f}\")\n        print(f\"   🏆 COMPETITION: {comp_score:.4f}\")\n    else:\n        comp_score = 0.5\n        print(f\"   ⚠️  No scores\")\n    \n    # GUARANTEED SAVE\n    print(\"\\n[5/5] SAVING (trying multiple methods)...\")\n    \n    model_data = {\n        'models': models,\n        'feature_cols': feature_cols,\n        'cv_scores': cv_scores,\n        'prevalence': LOCATION_PREVALENCE,\n        'ct_visibility': CT_VISIBILITY,\n    }\n    \n    saved_files = []\n    \n    # Method 1\n    try:\n        with open('medical_model.pkl', 'wb') as f:\n            pickle.dump(model_data, f)\n        if os.path.exists('medical_model.pkl'):\n            size = os.path.getsize('medical_model.pkl') / (1024*1024)\n            print(f\"   ✅ medical_model.pkl ({size:.2f} MB)\")\n            saved_files.append('medical_model.pkl')\n    except Exception as e:\n        print(f\"   ❌ Method 1: {e}\")\n    \n    # Method 2\n    try:\n        with open('medical_model_v2.pkl', 'wb') as f:\n            pickle.dump(model_data, f, protocol=4)\n        if os.path.exists('medical_model_v2.pkl'):\n            size = os.path.getsize('medical_model_v2.pkl') / (1024*1024)\n            print(f\"   ✅ medical_model_v2.pkl ({size:.2f} MB)\")\n            saved_files.append('medical_model_v2.pkl')\n    except Exception as e:\n        print(f\"   ❌ Method 2: {e}\")\n    \n    # List all\n    print(\"\\n📁 All .pkl files:\")\n    for file in sorted(os.listdir('.')):\n        if file.endswith('.pkl'):\n            size = os.path.getsize(file) / 1024\n            print(f\"   {file:40s} {size:8.2f} KB\")\n    \n    if saved_files:\n        print(f\"\\n✅ SUCCESS! Saved: {saved_files[0]}\")\n        print(f\"\\n🎯 Next steps:\")\n        print(f\"   1. Download {saved_files[0]}\")\n        print(f\"   2. Create Kaggle dataset\")\n        print(f\"   3. Use inference code\")\n    else:\n        print(f\"\\n❌ Save failed! But you have checkpoints\")\n    \n    return model_data\n\nif __name__ == \"__main__\":\n    train_medical_model()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}