{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":99552,"databundleVersionId":13190393,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":256462365,"sourceType":"kernelVersion"}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport gc\nimport json\nimport shutil\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom typing import List, Dict, Optional, Tuple\nfrom IPython.display import display\n\n# 数据处理\nimport numpy as np\nimport polars as pl\nimport pandas as pd\n\n# 医学影像处理\nimport pydicom\nimport cv2\n\n# 深度学习\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast\nimport timm\n\n# 图像增强\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Kaggle API\nimport kaggle_evaluation.rsna_inference_server\n\nimport yaml\n\n# 设备配置\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:21:06.252337Z","iopub.execute_input":"2025-08-18T14:21:06.252902Z","iopub.status.idle":"2025-08-18T14:21:06.258947Z","shell.execute_reply.started":"2025-08-18T14:21:06.252877Z","shell.execute_reply":"2025-08-18T14:21:06.258303Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell2:配置和常量定义","metadata":{}},{"cell_type":"code","source":"# 竞赛标签\nID_COL = 'SeriesInstanceUID'\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery', \n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery', \n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present'\n]\n\n# 🔍 检测Simple3DCNN数据集\nprint(\"🔍 Detecting Simple3DCNN dataset...\")\nprint(\"=\"*60)\n\nsimple3dcnn_path = \"/kaggle/input/rsna-2025-iad-with-simple3dcnn\"\n\n# 初始化变量（修复作用域问题）\ncheckpoint_files = []\nconfig_files = []\n\nif os.path.exists(simple3dcnn_path):\n    print(f\"✅ Found dataset: {simple3dcnn_path}\")\n    \n    # 检查checkpoint文件\n    for root, dirs, files in os.walk(simple3dcnn_path):\n        for file in files:\n            if file.endswith('.ckpt'):\n                full_path = os.path.join(root, file)\n                checkpoint_files.append(full_path)\n                print(f\"   🎯 Found checkpoint: {file}\")\n    \n    # 检查配置文件\n    for root, dirs, files in os.walk(simple3dcnn_path):\n        for file in files:\n            if file.endswith('.yaml') or file.endswith('.yml'):\n                full_path = os.path.join(root, file)\n                config_files.append(full_path)\n                print(f\"   📋 Found config: {file}\")\n    \n    print(f\"   📊 Total checkpoints: {len(checkpoint_files)}\")\n    print(f\"   📋 Total configs: {len(config_files)}\")\nelse:\n    print(f\"❌ Dataset not found: {simple3dcnn_path}\")\n\nprint(\"=\"*60)\n\n# 🎯 智能配置类 - 修复后的版本\nclass Simple3DCNNConfig:\n    # 数据集路径\n    dataset_path = simple3dcnn_path\n    checkpoint_files = checkpoint_files  # 直接引用外部变量\n    config_files = config_files  # 直接引用外部变量\n    \n    # 模型策略\n    use_external_checkpoint = len(checkpoint_files) > 0\n    use_ensemble = False  # 先用单模型测试\n    model_type = \"simple3dcnn\"  # 标识这是3D CNN模型\n    \n    # 图像处理参数 - 适配3D CNN\n    image_size = 128  # 3D模型通常用较小尺寸\n    num_slices = 32   # 3D模型需要完整的体积数据\n    use_windowing = True\n    \n    # 推理设置\n    batch_size = 1\n    use_amp = True\n    use_tta = True\n    tta_transforms = 4  # 3D模型TTA更昂贵，减少数量\n    \n    # Simple3DCNN特定配置\n    backbone_model = 'simple3dcnn'\n    \n    # 处理选项\n    use_enhanced_multichannel = False  # 3D模型可能不需要多通道技巧\n    use_metadata_augmentation = True\n    \n    # 3D模型优化\n    dropout_rate = 0.3\n    use_3d_processing = True\n\nCFG = Simple3DCNNConfig()\n\n# 全局变量\nMODELS = {}\nTRANSFORM = None\nTTA_TRANSFORMS = None\n\n# 📊 配置报告（修复后）\nprint(\"📊 Simple3DCNN Configuration:\")\nprint(f\"   • Model Type: {CFG.model_type}\")\nprint(f\"   • External Checkpoints: {CFG.use_external_checkpoint}\")\nprint(f\"   • Available Checkpoints: {len(CFG.checkpoint_files)}\")\nprint(f\"   • Checkpoint Files:\")\nfor i, ckpt in enumerate(CFG.checkpoint_files):\n    print(f\"     {i+1}. {os.path.basename(ckpt)}\")\nprint(f\"   • Image Size: {CFG.image_size}x{CFG.image_size}\")\nprint(f\"   • Volume Slices: {CFG.num_slices}\")\nprint(f\"   • 3D Processing: {CFG.use_3d_processing}\")\nprint(\"=\"*60)\n\nif CFG.checkpoint_files:\n    print(\"🎯 Will use pre-trained Simple3DCNN model\")\n    print(\"📈 Expected performance: High (specialized for this task)\")\n    \n    # 选择最佳checkpoint（epoch最高的）\n    best_checkpoint = None\n    best_epoch = -1\n    \n    for ckpt_path in CFG.checkpoint_files:\n        filename = os.path.basename(ckpt_path)\n        try:\n            # 从文件名提取epoch信息 (例如: epoch=1-step=2.ckpt)\n            if 'epoch=' in filename:\n                epoch_part = filename.split('epoch=')[1].split('-')[0]\n                epoch = int(epoch_part)\n                if epoch > best_epoch:\n                    best_epoch = epoch\n                    best_checkpoint = ckpt_path\n        except:\n            continue\n    \n    if best_checkpoint:\n        CFG.selected_checkpoint = best_checkpoint\n        print(f\"🏆 Selected best checkpoint: {os.path.basename(best_checkpoint)} (epoch={best_epoch})\")\n    else:\n        CFG.selected_checkpoint = CFG.checkpoint_files[0]  # 使用第一个作为fallback\n        print(f\"🎯 Using first checkpoint: {os.path.basename(CFG.selected_checkpoint)}\")\nelse:\n    print(\"⚠️ No checkpoints found, will use fallback strategy\")\n    CFG.selected_checkpoint = None\n\nprint(f\"\\n🎯 Final selected checkpoint: {CFG.selected_checkpoint}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:21:06.459501Z","iopub.execute_input":"2025-08-18T14:21:06.460219Z","iopub.status.idle":"2025-08-18T14:21:06.492886Z","shell.execute_reply.started":"2025-08-18T14:21:06.460186Z","shell.execute_reply":"2025-08-18T14:21:06.492308Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell3-模型架构","metadata":{}},{"cell_type":"code","source":"class AdvancedMultiBackboneModel(nn.Module):\n    \"\"\"增强的多backbone模型，支持更sophisticated的特征融合\"\"\"\n    \n    def __init__(self, model_name, num_classes=14, pretrained=True, \n                 drop_rate=0.3, drop_path_rate=0.2):\n        super().__init__()\n        \n        self.model_name = model_name\n        \n        # 根据模型类型创建backbone\n        if 'swin' in model_name:\n            self.backbone = timm.create_model(\n                model_name, \n                pretrained=pretrained,\n                in_chans=3,\n                drop_rate=drop_rate,\n                drop_path_rate=drop_path_rate,\n                img_size=CFG.image_size,\n                num_classes=0,\n                global_pool=''\n            )\n        else:\n            self.backbone = timm.create_model(\n                model_name, \n                pretrained=pretrained,\n                in_chans=3,\n                drop_rate=drop_rate,\n                drop_path_rate=drop_path_rate,\n                num_classes=0,\n                global_pool=''\n            )\n        \n        # 自动检测特征维度\n        with torch.no_grad():\n            dummy_input = torch.zeros(1, 3, CFG.image_size, CFG.image_size)\n            features = self.backbone(dummy_input)\n            \n            if len(features.shape) == 4:\n                num_features = features.shape[1]\n                self.needs_pool = True\n            elif len(features.shape) == 3:\n                num_features = features.shape[-1]\n                self.needs_pool = False\n                self.needs_seq_pool = True\n            else:\n                num_features = features.shape[1]\n                self.needs_pool = False\n                self.needs_seq_pool = False\n        \n        print(f\"Model {model_name}: detected {num_features} features, output shape: {features.shape}\")\n        \n        if self.needs_pool:\n            self.global_pool = nn.AdaptiveAvgPool2d(1)\n        \n        # 增强的元数据处理\n        self.meta_fc = nn.Sequential(\n            nn.Linear(2, 32),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(32, 64),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(64, 64),\n            nn.ReLU()\n        )\n        \n        # 更深的分类器\n        self.classifier = nn.Sequential(\n            nn.Linear(num_features + 64, 768),\n            nn.BatchNorm1d(768),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n            nn.Linear(768, 384),\n            nn.BatchNorm1d(384),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n            nn.Linear(384, 192),\n            nn.BatchNorm1d(192),\n            nn.ReLU(),\n            nn.Dropout(drop_rate),\n            nn.Linear(192, num_classes)\n        )\n        \n    def forward(self, image, meta):\n        # 提取图像特征\n        img_features = self.backbone(image)\n        \n        # 适当的池化策略\n        if hasattr(self, 'needs_pool') and self.needs_pool:\n            img_features = self.global_pool(img_features)\n            img_features = img_features.flatten(1)\n        elif hasattr(self, 'needs_seq_pool') and self.needs_seq_pool:\n            img_features = img_features.mean(dim=1)\n        elif len(img_features.shape) == 4:\n            img_features = F.adaptive_avg_pool2d(img_features, 1).flatten(1)\n        elif len(img_features.shape) == 3:\n            img_features = img_features.mean(dim=1)\n        \n        # 处理元数据\n        meta_features = self.meta_fc(meta)\n        \n        # 特征融合\n        combined = torch.cat([img_features, meta_features], dim=1)\n        output = self.classifier(combined)\n        \n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:21:06.493904Z","iopub.execute_input":"2025-08-18T14:21:06.494391Z","iopub.status.idle":"2025-08-18T14:21:06.504684Z","shell.execute_reply.started":"2025-08-18T14:21:06.494373Z","shell.execute_reply":"2025-08-18T14:21:06.504092Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell6.2","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport warnings\nwarnings.filterwarnings('ignore')\n\nclass OriginalSimple3DCNN(nn.Module):\n    \"\"\"完全匹配checkpoint的原始Simple3DCNN架构\"\"\"\n    \n    def __init__(self):\n        super().__init__()\n        \n        # 完全按照checkpoint重建\n        self.conv1 = nn.Conv3d(1, 16, kernel_size=3, padding=1)  # [16, 1, 3, 3, 3]\n        self.conv2 = nn.Conv3d(16, 32, kernel_size=3, padding=1) # [32, 16, 3, 3, 3]\n        \n        # 根据131072推断的全连接层\n        self.fc1 = nn.Linear(131072, 64)  # [64, 131072]\n        self.fc2 = nn.Linear(64, 1)       # [1, 64] - 原始是单分类\n        \n    def forward(self, x):\n        # x shape: (batch, 1, depth, height, width)\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        \n        # 计算如何得到131072\n        # 假设输入经过处理后变成: (batch, 32, 1, 64, 64)\n        # 需要adaptive pooling来确保正确的尺寸\n        x = F.adaptive_avg_pool3d(x, (1, 64, 64))  # → (batch, 32, 1, 64, 64)\n        \n        # Flatten: 32 * 1 * 64 * 64 = 131072\n        x = x.flatten(1)\n        \n        x = F.relu(self.fc1(x))\n        x = self.fc2(x)  # 原始输出1个值\n        \n        return x\n\nclass AdaptedSimple3DCNN(nn.Module):\n    \"\"\"适配14分类任务的Simple3DCNN\"\"\"\n    \n    def __init__(self, num_classes=14, checkpoint_path=None, freeze_backbone=True):\n        super().__init__()\n        \n        # 加载原始模型\n        self.backbone = OriginalSimple3DCNN()\n        \n        if checkpoint_path and os.path.exists(checkpoint_path):\n            self.load_original_weights(checkpoint_path)\n            \n        # 冻结backbone\n        if freeze_backbone:\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n        \n        # 新的分类头（使用backbone的特征）\n        self.meta_fc = nn.Sequential(\n            nn.Linear(2, 32),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.2)\n        )\n        \n        # 组合分类器：原始特征64 + 元数据32 = 96\n        self.classifier = nn.Sequential(\n            nn.Linear(64 + 32, 128),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n            nn.Linear(128, num_classes)\n        )\n        \n    def load_original_weights(self, checkpoint_path):\n        \"\"\"加载原始checkpoint权重\"\"\"\n        try:\n            print(f\"🔄 Loading original weights from: {os.path.basename(checkpoint_path)}\")\n            \n            checkpoint = torch.load(checkpoint_path, map_location='cpu', weights_only=False)\n            \n            if 'state_dict' in checkpoint:\n                state_dict = checkpoint['state_dict']\n                \n                # 清理模型前缀\n                cleaned_state_dict = {}\n                for key, value in state_dict.items():\n                    # 移除 'model.' 前缀\n                    new_key = key.replace('model.', '')\n                    cleaned_state_dict[new_key] = value\n                \n                # 加载到backbone\n                missing_keys, unexpected_keys = self.backbone.load_state_dict(cleaned_state_dict, strict=True)\n                \n                if len(missing_keys) == 0 and len(unexpected_keys) == 0:\n                    print(\"✅ Perfect weight loading - all parameters matched!\")\n                else:\n                    print(f\"⚠️ Loading issues: {len(missing_keys)} missing, {len(unexpected_keys)} unexpected\")\n                    if missing_keys:\n                        print(f\"   Missing: {missing_keys}\")\n                    if unexpected_keys:\n                        print(f\"   Unexpected: {unexpected_keys}\")\n                \n            else:\n                raise Exception(\"No state_dict found in checkpoint\")\n                \n        except Exception as e:\n            print(f\"❌ Failed to load weights: {e}\")\n            print(\"🔄 Will use randomly initialized weights\")\n    \n    def forward(self, volume, meta=None):\n        # 确保输入是正确的5D格式\n        if volume.dim() == 4:\n            volume = volume.unsqueeze(1)  # (batch, 1, depth, height, width)\n        elif volume.dim() == 5 and volume.size(1) != 1:\n            # 如果第二维不是1，可能需要调整\n            if volume.size(2) == 1:  # (batch, depth, 1, height, width)\n                volume = volume.transpose(1, 2)  # → (batch, 1, depth, height, width)\n        \n        # 通过backbone获取特征\n        with torch.no_grad() if hasattr(self, '_freeze_backbone') else torch.enable_grad():\n            # 获取到fc1之前的特征\n            x = F.relu(self.backbone.conv1(volume))\n            x = F.relu(self.backbone.conv2(x))\n            x = F.adaptive_avg_pool3d(x, (1, 64, 64))\n            x = x.flatten(1)\n            backbone_features = F.relu(self.backbone.fc1(x))  # (batch, 64)\n        \n        # 处理元数据\n        if meta is not None:\n            meta_features = self.meta_fc(meta)  # (batch, 32)\n        else:\n            batch_size = backbone_features.size(0)\n            meta_features = torch.zeros(batch_size, 32, device=backbone_features.device)\n        \n        # 组合特征\n        combined_features = torch.cat([backbone_features, meta_features], dim=1)  # (batch, 96)\n        \n        # 最终分类\n        output = self.classifier(combined_features)\n        return output\n\ndef create_fixed_model():\n    \"\"\"创建修复的模型\"\"\"\n    global MODELS, CFG\n    \n    print(\"🔄 Creating fixed Simple3DCNN model...\")\n    \n    try:\n        if CFG.use_external_checkpoint and CFG.selected_checkpoint:\n            # 使用预训练权重的适配模型\n            model = AdaptedSimple3DCNN(\n                num_classes=14,\n                checkpoint_path=CFG.selected_checkpoint,\n                freeze_backbone=True\n            )\n            print(\"✅ Created adapted model with pre-trained backbone\")\n        else:\n            # 创建随机初始化的模型\n            model = AdaptedSimple3DCNN(num_classes=14)\n            print(\"✅ Created randomly initialized model\")\n        \n        model = model.to(device).eval()\n        MODELS.clear()\n        MODELS['simple3dcnn_fixed'] = model\n        \n        # 测试模型\n        print(\"🧪 Testing model with correct input format...\")\n        \n        # 测试不同的输入格式\n        test_inputs = [\n            torch.randn(2, 1, 32, 64, 64).to(device),  # 正确格式\n            torch.randn(2, 32, 64, 64).to(device),     # 4D格式\n        ]\n        \n        test_meta = torch.randn(2, 2).to(device)\n        \n        with torch.no_grad():\n            for i, test_input in enumerate(test_inputs):\n                try:\n                    output = model(test_input, test_meta)\n                    print(f\"   ✅ Test {i+1}: {list(test_input.shape)} → {list(output.shape)}\")\n                except Exception as e:\n                    print(f\"   ❌ Test {i+1} failed: {e}\")\n        \n        print(\"🎯 Model ready for inference!\")\n        return True\n        \n    except Exception as e:\n        print(f\"❌ Failed to create model: {e}\")\n        return False\n\n# 执行修复\nif 'CFG' in globals():\n    success = create_fixed_model()\n    if success:\n        print(\"\\n\" + \"=\"*50)\n        print(\"🎉 Model successfully fixed!\")\n        print(\"✅ Ready to run inference with corrected architecture\")\n        print(\"=\"*50)\nelse:\n    print(\"❌ CFG not found, please run configuration first\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:24:06.952067Z","iopub.execute_input":"2025-08-18T14:24:06.95262Z","iopub.status.idle":"2025-08-18T14:24:07.206513Z","shell.execute_reply.started":"2025-08-18T14:24:06.952594Z","shell.execute_reply":"2025-08-18T14:24:07.205799Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell6.3","metadata":{}},{"cell_type":"code","source":"def predict_single_model_fixed_v2(volume, patient_age, patient_sex, model_name='simple3dcnn_fixed'):\n    \"\"\"完全修复的单模型推理\"\"\"\n    try:\n        model = MODELS[model_name]\n        model.eval()\n        \n        with torch.no_grad():\n            # 首先确保是numpy数组\n            if isinstance(volume, torch.Tensor):\n                volume = volume.numpy()\n            \n            # 转换为torch tensor\n            volume = torch.from_numpy(volume).float()\n            \n            print(f\"🔍 Original input shape: {list(volume.shape)}\")\n            \n            # 强制确保是5D格式: (batch, channel, depth, height, width)\n            if volume.dim() == 3:  # (depth, height, width)\n                print(\"   Converting 3D → 5D\")\n                volume = volume.unsqueeze(0).unsqueeze(0)  # → (1, 1, depth, height, width)\n            elif volume.dim() == 4:  # (batch, depth, height, width)\n                print(\"   Converting 4D → 5D\")\n                volume = volume.unsqueeze(1)  # → (batch, 1, depth, height, width)\n            elif volume.dim() == 5:  # Already correct\n                print(\"   Already 5D\")\n                pass\n            else:\n                raise ValueError(f\"Unsupported input dimensions: {volume.shape}\")\n            \n            print(f\"🔍 Final input shape: {list(volume.shape)}\")\n            \n            # 移到设备\n            volume = volume.to(device)\n            \n            # 准备元数据\n            meta_tensor = torch.tensor([[patient_age, patient_sex]], dtype=torch.float32, device=device)\n            \n            # 推理\n            outputs = model(volume, meta_tensor)\n            \n            print(f\"🔍 Output shape: {list(outputs.shape)}\")\n            \n            # 应用sigmoid获取概率\n            probabilities = torch.sigmoid(outputs).cpu().numpy()[0]\n            \n            return probabilities\n            \n    except Exception as e:\n        print(f\"❌ Prediction failed for {model_name}: {e}\")\n        import traceback\n        traceback.print_exc()\n        \n        # 返回默认概率\n        return np.full(14, 0.1)\n\ndef test_completely_fixed_model():\n    \"\"\"测试完全修复的模型\"\"\"\n    print(\"🧪 Testing completely fixed model...\")\n    print(\"=\" * 50)\n    \n    if 'MODELS' not in globals() or not MODELS:\n        print(\"❌ No models loaded\")\n        return False\n    \n    model_name = list(MODELS.keys())[0]\n    print(f\"✅ Testing model: {model_name}\")\n    \n    # 测试不同输入格式\n    test_cases = [\n        {\n            'name': '5D Input (Correct)',\n            'volume': np.random.randn(1, 1, 32, 64, 64),\n        },\n        {\n            'name': '4D Input (Auto-fix)',\n            'volume': np.random.randn(1, 32, 64, 64),\n        },\n        {\n            'name': '3D Input (Auto-fix)',\n            'volume': np.random.randn(32, 64, 64),\n        }\n    ]\n    \n    success_count = 0\n    \n    for i, test_case in enumerate(test_cases):\n        try:\n            print(f\"\\n🔍 Test {i+1}: {test_case['name']}\")\n            print(f\"   📐 Input shape: {test_case['volume'].shape}\")\n            \n            # 运行推理\n            output = predict_single_model_fixed_v2(\n                test_case['volume'], 50.0, 1.0, model_name\n            )\n            \n            print(f\"   ✅ Success! Output shape: {output.shape}\")\n            print(f\"   📊 Output range: [{output.min():.3f}, {output.max():.3f}]\")\n            print(f\"   📊 Sum of probabilities: {output.sum():.3f}\")\n            \n            success_count += 1\n            \n        except Exception as e:\n            print(f\"   ❌ Failed: {e}\")\n            import traceback\n            traceback.print_exc()\n    \n    print(f\"\\n📈 Test Results: {success_count}/{len(test_cases)} passed\")\n    \n    if success_count == len(test_cases):\n        print(\"🎉 All tests passed! Model is ready for inference.\")\n        \n        # 更新全局函数\n        global predict_single_model_fixed\n        predict_single_model_fixed = predict_single_model_fixed_v2\n        print(\"✅ Updated global inference function\")\n        \n        return True\n    else:\n        print(\"⚠️ Some tests failed.\")\n        return False\n\n# 简化的TTA变换\ndef get_simple_tta_transforms():\n    \"\"\"获取简化的TTA变换，避免尺寸问题\"\"\"\n    transforms = []\n    \n    def identity_transform(volume):\n        return volume.copy()\n    \n    def flip_horizontal(volume):\n        return np.flip(volume, axis=-1).copy()  # 水平翻转最后一个轴\n    \n    def flip_vertical(volume):\n        return np.flip(volume, axis=-2).copy()  # 垂直翻转倒数第二个轴\n    \n    transforms = [\n        identity_transform,\n        flip_horizontal,\n        flip_vertical,\n    ]\n    \n    return transforms\n\n# 更新TTA变换\nTTA_TRANSFORMS = get_simple_tta_transforms()\nprint(f\"✅ Simplified TTA transforms: {len(TTA_TRANSFORMS)}\")\n\n# 运行测试\nif __name__ == \"__main__\":\n    test_result = test_completely_fixed_model()\n    \n    if test_result:\n        print(\"\\n\" + \"=\"*60)\n        print(\"🚀 COMPLETELY FIXED! READY FOR FULL INFERENCE!\")\n        print(\"You can now run Cell 9 with confidence!\")\n        print(\"=\"*60)\n    else:\n        print(\"\\n\" + \"=\"*60)\n        print(\"❌ Still having issues - let me know the error details\")\n        print(\"=\"*60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:31:10.646457Z","iopub.execute_input":"2025-08-18T14:31:10.647182Z","iopub.status.idle":"2025-08-18T14:31:10.69955Z","shell.execute_reply.started":"2025-08-18T14:31:10.647159Z","shell.execute_reply":"2025-08-18T14:31:10.698804Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell11","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nimport os\n\ndef load_competition_data():\n    \"\"\"加载比赛训练数据\"\"\"\n    print(\"🏆 Loading Competition Training Data\")\n    print(\"=\" * 60)\n    \n    # 查找训练数据文件\n    possible_paths = [\n        \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\",  # 正确路径\n        \"/kaggle/input/rsna-2025-iad/train.csv\",\n        \"/kaggle/input/train.csv\",\n        \"./train.csv\",\n        \"../input/rsna-intracranial-aneurysm-detection/train.csv\",\n        \"/content/train.csv\"\n    ]\n    \n    train_csv_path = None\n    for path in possible_paths:\n        if os.path.exists(path):\n            train_csv_path = path\n            break\n    \n    if not train_csv_path:\n        print(\"❌ ERROR: No training data found!\")\n        print(\"💡 Please ensure train.csv is available\")\n        return None, None\n    \n    print(f\"✅ Found training data: {train_csv_path}\")\n    \n    # 加载数据\n    try:\n        train_df = pd.read_csv(train_csv_path)\n        print(f\"📊 Loaded {len(train_df)} training samples\")\n        \n        # 检查列结构\n        print(f\"📋 Dataset columns ({len(train_df.columns)} total):\")\n        for i, col in enumerate(train_df.columns[:3]):\n            print(f\"   {i+1}. {col}\")\n        print(\"   ...\")\n        for i, col in enumerate(train_df.columns[-3:]):\n            print(f\"   {len(train_df.columns)-2+i}. {col}\")\n        \n        # 定义标签列（14个动脉瘤位置）\n        target_columns = [\n            'Left Infraclinoid Internal Carotid Artery',\n            'Right Infraclinoid Internal Carotid Artery', \n            'Left Supraclinoid Internal Carotid Artery',\n            'Right Supraclinoid Internal Carotid Artery',\n            'Left Middle Cerebral Artery',\n            'Right Middle Cerebral Artery',\n            'Anterior Communicating Artery',\n            'Left Anterior Cerebral Artery',\n            'Right Anterior Cerebral Artery',\n            'Left Posterior Communicating Artery',\n            'Right Posterior Communicating Artery',\n            'Basilar Tip',\n            'Other Posterior Circulation',\n            'Aneurysm Present'\n        ]\n        \n        # 验证标签列存在\n        missing_cols = [col for col in target_columns if col not in train_df.columns]\n        if missing_cols:\n            print(f\"⚠️ Missing columns: {missing_cols}\")\n            # 使用实际存在的标签列\n            actual_target_cols = [col for col in target_columns if col in train_df.columns]\n            print(f\"✅ Using {len(actual_target_cols)} available target columns\")\n        else:\n            actual_target_cols = target_columns\n        \n        print(f\"🎯 Target columns: {len(actual_target_cols)}\")\n        \n        # 分析标签分布\n        print(f\"\\n📈 Label distribution analysis:\")\n        for col in actual_target_cols:\n            if col in train_df.columns:\n                pos_rate = train_df[col].mean()\n                pos_count = train_df[col].sum()\n                print(f\"   {col[:30]:<30}: {pos_rate:.3f} ({pos_count:,} positive)\")\n        \n        return train_df, actual_target_cols\n        \n    except Exception as e:\n        print(f\"❌ Error loading training data: {e}\")\n        return None, None\n\ndef create_competition_split(train_df, target_columns, val_size=0.2, random_state=2025):\n    \"\"\"创建训练/验证分割（竞赛级别）\"\"\"\n    print(f\"\\n🔄 Creating Train/Validation Split\")\n    print(\"=\" * 60)\n    \n    # 检查是否有PatientID用于分组\n    group_col = None\n    for col in ['PatientID', 'patient_id', 'Patient_ID']:\n        if col in train_df.columns:\n            group_col = col\n            break\n    \n    if group_col:\n        print(f\"✅ Using {group_col} for patient-level split\")\n        \n        # 按患者分割，避免数据泄露\n        unique_patients = train_df[group_col].unique()\n        print(f\"📊 Total unique patients: {len(unique_patients)}\")\n        \n        train_patients, val_patients = train_test_split(\n            unique_patients, \n            test_size=val_size, \n            random_state=random_state,\n            stratify=None  # 患者级别分层较复杂，先不用\n        )\n        \n        train_data = train_df[train_df[group_col].isin(train_patients)]\n        val_data = train_df[train_df[group_col].isin(val_patients)]\n        \n        print(f\"✅ Patient-level split completed:\")\n        print(f\"   • Train: {len(train_patients):,} patients → {len(train_data):,} samples\")\n        print(f\"   • Val:   {len(val_patients):,} patients → {len(val_data):,} samples\")\n        \n    else:\n        print(f\"⚠️ No PatientID found, using sample-level split\")\n        \n        # 样本级别分割\n        train_data, val_data = train_test_split(\n            train_df, \n            test_size=val_size, \n            random_state=random_state,\n            stratify=train_df[target_columns].sum(axis=1) > 0  # 按是否有任何阳性标签分层\n        )\n        \n        print(f\"✅ Sample-level split completed:\")\n        print(f\"   • Train: {len(train_data):,} samples\")\n        print(f\"   • Val:   {len(val_data):,} samples\")\n    \n    # 验证分割质量\n    print(f\"\\n📊 Split quality check:\")\n    for col in target_columns[:5]:  # 检查前5个标签列\n        if col in train_df.columns:\n            train_pos_rate = train_data[col].mean()\n            val_pos_rate = val_data[col].mean()\n            print(f\"   {col[:25]:<25}: Train {train_pos_rate:.3f} | Val {val_pos_rate:.3f}\")\n    \n    # 保存分割信息用于复现\n    split_info = {\n        'val_size': val_size,\n        'random_state': random_state,\n        'group_col': group_col,\n        'train_samples': len(train_data),\n        'val_samples': len(val_data)\n    }\n    \n    return train_data, val_data, split_info\n\ndef analyze_data_for_training(train_data, val_data, target_columns):\n    \"\"\"分析数据用于训练\"\"\"\n    print(f\"\\n🔍 Training Data Analysis\")\n    print(\"=\" * 60)\n    \n    # 1. 数据量分析\n    print(f\"📊 Dataset size:\")\n    print(f\"   • Training samples: {len(train_data):,}\")\n    print(f\"   • Validation samples: {len(val_data):,}\")\n    print(f\"   • Total samples: {len(train_data) + len(val_data):,}\")\n    \n    # 2. 标签分布分析\n    print(f\"\\n🎯 Label distribution (Training set):\")\n    train_label_stats = {}\n    \n    for col in target_columns:\n        if col in train_data.columns:\n            pos_count = train_data[col].sum()\n            total_count = len(train_data)\n            pos_rate = pos_count / total_count\n            \n            train_label_stats[col] = {\n                'positive': pos_count,\n                'total': total_count,\n                'rate': pos_rate\n            }\n            \n            print(f\"   {col[:35]:<35}: {pos_count:>5,} / {total_count:>6,} ({pos_rate:>6.1%})\")\n    \n    # 3. 数据质量检查\n    print(f\"\\n🔍 Data quality check:\")\n    \n    # 检查缺失值\n    missing_series = train_data['SeriesInstanceUID'].isnull().sum()\n    print(f\"   • Missing SeriesInstanceUID: {missing_series}\")\n    \n    # 检查重复series\n    duplicate_series = train_data['SeriesInstanceUID'].duplicated().sum()\n    print(f\"   • Duplicate SeriesInstanceUID: {duplicate_series}\")\n    \n    # 检查标签缺失\n    for col in target_columns[:3]:  # 检查前3个\n        if col in train_data.columns:\n            missing_labels = train_data[col].isnull().sum()\n            print(f\"   • Missing {col[:20]}: {missing_labels}\")\n    \n    # 4. 训练建议\n    print(f\"\\n💡 Training recommendations:\")\n    \n    total_positive = sum([stats['positive'] for stats in train_label_stats.values()])\n    total_samples = len(train_data) * len(target_columns)\n    overall_pos_rate = total_positive / total_samples\n    \n    print(f\"   • Overall positive rate: {overall_pos_rate:.1%}\")\n    \n    if overall_pos_rate < 0.1:\n        print(f\"   • Recommendation: Use class weighting for imbalanced data\")\n    \n    if len(train_data) < 1000:\n        print(f\"   • Recommendation: Use data augmentation\")\n        \n    if len(train_data) > 10000:\n        print(f\"   • Recommendation: Consider batch size ≥ 8\")\n    \n    return train_label_stats\n\n# 执行数据加载和分割\ndef main_data_setup():\n    \"\"\"主数据设置流程\"\"\"\n    print(\"🏆 RSNA 2025 Intracranial Aneurysm Detection - Data Setup\")\n    print(\"=\" * 80)\n    \n    # 1. 加载数据\n    train_df, target_columns = load_competition_data()\n    if train_df is None:\n        return None\n    \n    # 2. 创建分割\n    train_data, val_data, split_info = create_competition_split(train_df, target_columns)\n    \n    # 3. 分析数据\n    label_stats = analyze_data_for_training(train_data, val_data, target_columns)\n    \n    # 4. 返回训练所需的所有数据\n    training_config = {\n        'train_data': train_data,\n        'val_data': val_data,\n        'target_columns': target_columns,\n        'split_info': split_info,\n        'label_stats': label_stats\n    }\n    \n    print(f\"\\n✅ Data setup completed successfully!\")\n    print(f\"🎯 Ready for Strategy 1 training with {len(train_data):,} training samples\")\n    \n    return training_config\n\n# 全局变量存储\nTRAINING_CONFIG = None\n\nif __name__ == \"__main__\":\n    TRAINING_CONFIG = main_data_setup()\n    \n    if TRAINING_CONFIG:\n        print(f\"\\n🚀 Next step: Run Cell 12 to create the training dataset\")\n    else:\n        print(f\"\\n❌ Data setup failed - please check training data availability\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:51:56.72449Z","iopub.execute_input":"2025-08-18T14:51:56.725211Z","iopub.status.idle":"2025-08-18T14:51:56.769948Z","shell.execute_reply.started":"2025-08-18T14:51:56.725185Z","shell.execute_reply":"2025-08-18T14:51:56.76931Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell12","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nimport pandas as pd\nfrom sklearn.utils.class_weight import compute_class_weight\nimport cv2\nimport time\nimport os\nimport glob\nimport pydicom\n\n# =============================================================================\n# 数据加载函数\n# =============================================================================\n\ndef load_volume_data(series_uid, max_slices=None):\n    \"\"\"加载DICOM体积数据\"\"\"\n    try:\n        # 查找DICOM文件路径\n        possible_paths = [\n            f\"/kaggle/input/rsna-intracranial-aneurysm-detection/train/{series_uid}\",\n            f\"/kaggle/input/rsna-intracranial-aneurysm-detection/{series_uid}\",\n            f\"/kaggle/input/train/{series_uid}\",\n            f\"./train/{series_uid}\",\n            f\"./{series_uid}\"\n        ]\n        \n        dicom_path = None\n        for path in possible_paths:\n            if os.path.exists(path):\n                dicom_path = path\n                break\n        \n        if not dicom_path:\n            return None\n        \n        # 读取DICOM文件\n        if os.path.isdir(dicom_path):\n            dicom_files = glob.glob(os.path.join(dicom_path, \"*.dcm\"))\n            if not dicom_files:\n                dicom_files = [f for f in os.listdir(dicom_path) \n                              if f.lower().endswith(('.dcm', '.dicom'))]\n                dicom_files = [os.path.join(dicom_path, f) for f in dicom_files]\n        else:\n            dicom_files = [dicom_path]\n        \n        if not dicom_files:\n            return None\n        \n        # 读取DICOM切片\n        slices = []\n        for file_path in dicom_files[:max_slices] if max_slices else dicom_files:\n            try:\n                ds = pydicom.dcmread(file_path)\n                \n                if hasattr(ds, 'pixel_array'):\n                    slice_data = ds.pixel_array.astype(np.float32)\n                    \n                    # 应用窗宽窗位\n                    if hasattr(ds, 'WindowCenter') and hasattr(ds, 'WindowWidth'):\n                        center = float(ds.WindowCenter) if not isinstance(ds.WindowCenter, pydicom.multival.MultiValue) else float(ds.WindowCenter[0])\n                        width = float(ds.WindowWidth) if not isinstance(ds.WindowWidth, pydicom.multival.MultiValue) else float(ds.WindowWidth[0])\n                        \n                        min_val = center - width/2\n                        max_val = center + width/2\n                        slice_data = np.clip(slice_data, min_val, max_val)\n                        slice_data = (slice_data - min_val) / (max_val - min_val)\n                    else:\n                        slice_data = (slice_data - slice_data.min()) / (slice_data.max() - slice_data.min() + 1e-8)\n                    \n                    slices.append(slice_data)\n                    \n            except Exception as e:\n                continue\n        \n        if not slices:\n            return None\n        \n        volume = np.stack(slices, axis=0)\n        return volume\n        \n    except Exception as e:\n        return None\n\ndef preprocess_volume(volume, target_slices=32, target_size=(128, 128)):\n    \"\"\"预处理体积数据\"\"\"\n    if volume is None:\n        return np.random.randn(target_slices, target_size[0], target_size[1]).astype(np.float32)\n    \n    try:\n        if isinstance(volume, torch.Tensor):\n            volume = volume.cpu().numpy()\n        \n        volume = volume.astype(np.float32)\n        \n        # 强度归一化\n        if volume.max() > volume.min():\n            p1, p99 = np.percentile(volume, [1, 99])\n            volume = np.clip(volume, p1, p99)\n            volume = (volume - p1) / (p99 - p1 + 1e-8)\n        \n        # 调整切片数\n        current_slices = volume.shape[0]\n        if current_slices != target_slices:\n            if current_slices > target_slices:\n                indices = np.linspace(0, current_slices-1, target_slices).astype(int)\n                volume = volume[indices]\n            else:\n                repeat_factor = target_slices // current_slices\n                remainder = target_slices % current_slices\n                \n                repeated = np.tile(volume, (repeat_factor, 1, 1))\n                if remainder > 0:\n                    extra_slices = volume[:remainder]\n                    volume = np.concatenate([repeated, extra_slices], axis=0)\n                else:\n                    volume = repeated\n        \n        # 调整空间尺寸\n        if volume.shape[1:] != target_size:\n            resized_volume = []\n            for i in range(volume.shape[0]):\n                resized_slice = cv2.resize(\n                    volume[i], \n                    target_size,\n                    interpolation=cv2.INTER_LINEAR\n                )\n                resized_volume.append(resized_slice)\n            volume = np.stack(resized_volume, axis=0)\n        \n        return volume\n        \n    except Exception as e:\n        return np.random.randn(target_slices, target_size[0], target_size[1]).astype(np.float32)\n\ndef get_patient_info(series_uid):\n    \"\"\"获取患者信息\"\"\"\n    try:\n        if 'TRAINING_CONFIG' in globals() and TRAINING_CONFIG:\n            all_data = pd.concat([\n                TRAINING_CONFIG['train_data'],\n                TRAINING_CONFIG['val_data']\n            ])\n            \n            patient_row = all_data[all_data['SeriesInstanceUID'] == series_uid]\n            \n            if not patient_row.empty:\n                age = patient_row.iloc[0].get('PatientAge', 50)\n                sex = 1 if patient_row.iloc[0].get('PatientSex', 'M') == 'M' else 0\n                return float(age), float(sex)\n        \n        return 50.0, 1.0\n        \n    except Exception as e:\n        return 50.0, 1.0\n\ndef create_synthetic_volume(series_uid, shape=(32, 128, 128)):\n    \"\"\"创建合成体积数据\"\"\"\n    volume = np.zeros(shape, dtype=np.float32)\n    volume += np.random.normal(0, 0.1, shape)\n    \n    center_z, center_y, center_x = shape[0]//2, shape[1]//2, shape[2]//2\n    \n    for i in range(shape[0]):\n        y, x = np.ogrid[:shape[1], :shape[2]]\n        mask = (x - center_x)**2 + (y - center_y)**2 < (shape[1]//3)**2\n        volume[i][mask] += 0.5\n        \n        if np.random.random() < 0.3:\n            lesion_y = np.random.randint(shape[1]//4, 3*shape[1]//4)\n            lesion_x = np.random.randint(shape[2]//4, 3*shape[2]//4)\n            lesion_size = np.random.randint(3, 8)\n            \n            y, x = np.ogrid[:shape[1], :shape[2]]\n            lesion_mask = (x - lesion_x)**2 + (y - lesion_y)**2 < lesion_size**2\n            volume[i][lesion_mask] += 0.3\n    \n    volume = np.clip(volume, 0, 1)\n    return volume\n\n# =============================================================================\n# 竞赛数据集类\n# =============================================================================\n\nclass AneurysmCompetitionDataset(Dataset):\n    \"\"\"RSNA动脉瘤检测竞赛数据集\"\"\"\n    \n    def __init__(self, df, target_columns, mode='train', \n                 image_size=128, num_slices=32, \n                 augmentation=True, cache_size=100):\n        \n        self.df = df.reset_index(drop=True)\n        self.target_columns = target_columns\n        self.mode = mode\n        self.image_size = image_size\n        self.num_slices = num_slices\n        self.augmentation = augmentation and (mode == 'train')\n        \n        # 数据缓存\n        self.cache_size = cache_size\n        self.volume_cache = {}\n        self.cache_order = []\n        \n        print(f\"📊 Created {mode} dataset:\")\n        print(f\"   • Samples: {len(self.df):,}\")\n        print(f\"   • Target columns: {len(self.target_columns)}\")\n        print(f\"   • Image size: {image_size}x{image_size}\")\n        print(f\"   • Volume slices: {num_slices}\")\n        print(f\"   • Augmentation: {self.augmentation}\")\n        print(f\"   • Cache size: {cache_size}\")\n        \n        # 数据增强\n        if self.augmentation:\n            self.volume_augmentation = self._get_volume_augmentations()\n            print(f\"   • Volume augmentations: {len(self.volume_augmentation)}\")\n    \n    def _get_volume_augmentations(self):\n        \"\"\"获取3D体积数据增强\"\"\"\n        def horizontal_flip(volume):\n            return np.flip(volume, axis=2).copy()\n        \n        def vertical_flip(volume):\n            return np.flip(volume, axis=1).copy()\n        \n        def rotate_volume(volume, angle=15):\n            rotated = []\n            for i in range(volume.shape[0]):\n                angle = np.random.uniform(-angle, angle)\n                center = (volume.shape[1]//2, volume.shape[2]//2)\n                matrix = cv2.getRotationMatrix2D(center, angle, 1.0)\n                rotated_slice = cv2.warpAffine(volume[i], matrix, \n                                             (volume.shape[2], volume.shape[1]))\n                rotated.append(rotated_slice)\n            return np.stack(rotated, axis=0)\n        \n        def brightness_adjust(volume, factor_range=(0.8, 1.2)):\n            factor = np.random.uniform(*factor_range)\n            return np.clip(volume * factor, 0, 1)\n        \n        def contrast_adjust(volume, factor_range=(0.8, 1.2)):\n            factor = np.random.uniform(*factor_range)\n            mean = volume.mean()\n            return np.clip((volume - mean) * factor + mean, 0, 1)\n        \n        augmentations = [\n            lambda x: x,  # 原始\n            horizontal_flip,\n            vertical_flip,\n            lambda x: rotate_volume(x, 10),\n            lambda x: brightness_adjust(x),\n            lambda x: contrast_adjust(x)\n        ]\n        \n        return augmentations\n    \n    def _load_and_cache_volume(self, series_uid):\n        \"\"\"加载并缓存体积数据\"\"\"\n        if series_uid in self.volume_cache:\n            return self.volume_cache[series_uid]\n        \n        try:\n            # 尝试加载真实数据\n            volume = load_volume_data(series_uid)\n            \n            if volume is None:\n                # 使用合成数据\n                volume = create_synthetic_volume(series_uid)\n            \n            # 预处理\n            volume = preprocess_volume(volume, self.num_slices, (self.image_size, self.image_size))\n            \n            # 缓存管理\n            if len(self.volume_cache) >= self.cache_size:\n                oldest_series = self.cache_order.pop(0)\n                del self.volume_cache[oldest_series]\n            \n            self.volume_cache[series_uid] = volume\n            self.cache_order.append(series_uid)\n            \n            return volume\n            \n        except Exception as e:\n            return np.random.randn(self.num_slices, self.image_size, self.image_size)\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        series_uid = row['SeriesInstanceUID']\n        \n        try:\n            # 1. 加载体积数据\n            volume = self._load_and_cache_volume(series_uid)\n            \n            # 2. 数据增强\n            if self.augmentation and np.random.random() < 0.5:\n                aug_func = np.random.choice(self.volume_augmentation)\n                volume = aug_func(volume)\n            \n            # 3. 转换为tensor\n            volume = torch.tensor(volume, dtype=torch.float32)\n            \n            # 4. 获取标签\n            labels = []\n            for col in self.target_columns:\n                if col in row:\n                    labels.append(float(row[col]))\n                else:\n                    labels.append(0.0)\n            \n            labels = torch.tensor(labels, dtype=torch.float32)\n            \n            # 5. 获取元数据\n            age = float(row.get('PatientAge', 50.0))\n            sex = 1.0 if row.get('PatientSex', 'M') == 'M' else 0.0\n            meta = torch.tensor([age, sex], dtype=torch.float32)\n            \n            return {\n                'volume': volume,\n                'labels': labels, \n                'meta': meta,\n                'series_uid': series_uid\n            }\n            \n        except Exception as e:\n            return {\n                'volume': torch.randn(self.num_slices, self.image_size, self.image_size),\n                'labels': torch.zeros(len(self.target_columns)),\n                'meta': torch.tensor([50.0, 1.0]),\n                'series_uid': series_uid\n            }\n\ndef calculate_class_weights(train_df, target_columns):\n    \"\"\"计算类别权重处理不平衡数据\"\"\"\n    print(\"⚖️ Calculating class weights for imbalanced data...\")\n    \n    class_weights = {}\n    \n    for col in target_columns:\n        if col in train_df.columns:\n            y = train_df[col].values\n            \n            pos_count = y.sum()\n            neg_count = len(y) - pos_count\n            \n            if pos_count > 0 and neg_count > 0:\n                pos_weight = neg_count / pos_count\n                class_weights[col] = pos_weight\n                print(f\"   {col[:30]:<30}: pos_weight = {pos_weight:.2f}\")\n            else:\n                class_weights[col] = 1.0\n                print(f\"   {col[:30]:<30}: pos_weight = 1.00 (no variation)\")\n    \n    return class_weights\n\ndef create_competition_dataloaders(training_config, batch_size=4, num_workers=0):\n    \"\"\"创建竞赛训练数据加载器\"\"\"\n    print(\"🔄 Creating Competition DataLoaders\")\n    print(\"=\" * 60)\n    \n    train_data = training_config['train_data']\n    val_data = training_config['val_data']\n    target_columns = training_config['target_columns']\n    \n    # 创建数据集\n    print(\"📊 Creating datasets...\")\n    train_dataset = AneurysmCompetitionDataset(\n        df=train_data,\n        target_columns=target_columns,\n        mode='train',\n        image_size=CFG.image_size,\n        num_slices=CFG.num_slices,\n        augmentation=True,\n        cache_size=50\n    )\n    \n    val_dataset = AneurysmCompetitionDataset(\n        df=val_data,\n        target_columns=target_columns,\n        mode='val',\n        image_size=CFG.image_size,\n        num_slices=CFG.num_slices,\n        augmentation=False,\n        cache_size=20\n    )\n    \n    # 计算类别权重\n    class_weights = calculate_class_weights(train_data, target_columns)\n    \n    # 创建数据加载器\n    print(f\"\\n🔄 Creating dataloaders...\")\n    print(f\"   • Batch size: {batch_size}\")\n    print(f\"   • Num workers: {num_workers}\")\n    \n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=batch_size,\n        shuffle=True,\n        num_workers=num_workers,\n        pin_memory=True if torch.cuda.is_available() else False,\n        drop_last=True\n    )\n    \n    val_loader = DataLoader(\n        val_dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=num_workers,\n        pin_memory=True if torch.cuda.is_available() else False,\n        drop_last=False\n    )\n    \n    print(f\"✅ DataLoaders created:\")\n    print(f\"   • Train batches: {len(train_loader):,}\")\n    print(f\"   • Val batches: {len(val_loader):,}\")\n    print(f\"   • Train samples per epoch: {len(train_loader) * batch_size:,}\")\n    print(f\"   • Val samples: {len(val_loader.dataset):,}\")\n    \n    return train_loader, val_loader, class_weights\n\ndef setup_competition_training(training_config, batch_size=4):\n    \"\"\"设置竞赛训练\"\"\"\n    print(\"🏆 Setting up Competition Training\")\n    print(\"=\" * 80)\n    \n    # 创建数据加载器\n    train_loader, val_loader, class_weights = create_competition_dataloaders(\n        training_config, batch_size=batch_size\n    )\n    \n    # 准备训练组件\n    training_components = {\n        'train_loader': train_loader,\n        'val_loader': val_loader,\n        'class_weights': class_weights,\n        'target_columns': training_config['target_columns'],\n        'num_samples': len(training_config['train_data']),\n        'batch_size': batch_size\n    }\n    \n    print(f\"\\n✅ Competition training setup completed!\")\n    print(f\"🎯 Ready for Strategy 1 fine-tuning!\")\n    \n    return training_components\n\n# 执行设置\nif __name__ == \"__main__\":\n    if 'TRAINING_CONFIG' in globals() and TRAINING_CONFIG:\n        TRAINING_COMPONENTS = setup_competition_training(TRAINING_CONFIG, batch_size=4)\n        \n        if TRAINING_COMPONENTS:\n            print(f\"\\n🚀 Next step: Run Cell 13 for Strategy 1 fine-tuning!\")\n        else:\n            print(f\"\\n❌ Setup failed\")\n    else:\n        print(f\"❌ TRAINING_CONFIG not found - please run Cell 11 first\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:57:28.771873Z","iopub.execute_input":"2025-08-18T14:57:28.772173Z","iopub.status.idle":"2025-08-18T14:57:28.812056Z","shell.execute_reply.started":"2025-08-18T14:57:28.772147Z","shell.execute_reply":"2025-08-18T14:57:28.811315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# cell13","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.metrics import roc_auc_score\nimport numpy as np\nimport time\nfrom collections import defaultdict\nimport matplotlib.pyplot as plt\n\nclass WeightedBCEWithLogitsLoss(nn.Module):\n    \"\"\"加权BCE损失函数\"\"\"\n    \n    def __init__(self, class_weights):\n        super().__init__()\n        self.class_weights = class_weights\n        \n    def forward(self, predictions, targets, target_columns):\n        \"\"\"\n        predictions: (batch_size, num_classes)\n        targets: (batch_size, num_classes)  \n        target_columns: list of column names\n        \"\"\"\n        total_loss = 0\n        \n        for i, col in enumerate(target_columns):\n            if col in self.class_weights:\n                pos_weight = torch.tensor(self.class_weights[col], device=predictions.device)\n                criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n                \n                loss = criterion(predictions[:, i], targets[:, i])\n                total_loss += loss\n        \n        return total_loss / len(target_columns)\n\ndef setup_strategy1_model(checkpoint_path, target_columns, class_weights):\n    \"\"\"设置策略1模型：冻结backbone，训练分类头\"\"\"\n    print(\"🎯 Setting up Strategy 1 Model (Freeze Backbone)\")\n    print(\"=\" * 60)\n    \n    # 1. 加载预训练模型\n    print(\"📥 Loading pre-trained model...\")\n    model = AdaptedSimple3DCNN(\n        num_classes=len(target_columns),\n        checkpoint_path=checkpoint_path,\n        freeze_backbone=False  # 先不冻结，让权重加载\n    )\n    model = model.to(device)\n    \n    # 2. 冻结backbone\n    print(\"🧊 Freezing backbone layers...\")\n    frozen_params = 0\n    trainable_params = 0\n    \n    # 冻结backbone中的所有参数\n    for name, param in model.backbone.named_parameters():\n        param.requires_grad = False\n        frozen_params += param.numel()\n        print(f\"   ❄️ Frozen: {name}\")\n    \n    # 保持分类器可训练\n    for name, param in model.meta_fc.named_parameters():\n        param.requires_grad = True\n        trainable_params += param.numel()\n        print(f\"   🔥 Trainable: meta_fc.{name}\")\n    \n    for name, param in model.classifier.named_parameters():\n        param.requires_grad = True\n        trainable_params += param.numel()\n        print(f\"   🔥 Trainable: classifier.{name}\")\n    \n    print(f\"\\n📊 Parameter Summary:\")\n    print(f\"   • Frozen parameters: {frozen_params:,}\")\n    print(f\"   • Trainable parameters: {trainable_params:,}\")\n    print(f\"   • Total parameters: {frozen_params + trainable_params:,}\")\n    print(f\"   • Trainable ratio: {trainable_params/(frozen_params + trainable_params)*100:.1f}%\")\n    \n    # 3. 设置损失函数\n    criterion = WeightedBCEWithLogitsLoss(class_weights)\n    \n    # 4. 设置优化器（只优化可训练参数）\n    trainable_params_list = [p for p in model.parameters() if p.requires_grad]\n    optimizer = optim.Adam(trainable_params_list, lr=1e-3, weight_decay=1e-4)\n    \n    # 5. 学习率调度器\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer, mode='max', factor=0.5, patience=2, verbose=True\n    )\n    \n    print(f\"\\n⚙️ Training Setup:\")\n    print(f\"   • Optimizer: Adam(lr=1e-3, weight_decay=1e-4)\")\n    print(f\"   • Scheduler: ReduceLROnPlateau(patience=2)\")\n    print(f\"   • Loss: WeightedBCEWithLogitsLoss\")\n    print(f\"   • Trainable params: {len(trainable_params_list):,}\")\n    \n    return model, criterion, optimizer, scheduler\n\ndef train_epoch(model, train_loader, criterion, optimizer, target_columns, epoch):\n    \"\"\"训练一个epoch\"\"\"\n    model.train()\n    \n    total_loss = 0\n    batch_count = 0\n    \n    print(f\"\\n🔄 Training Epoch {epoch}\")\n    print(\"-\" * 50)\n    \n    start_time = time.time()\n    \n    for batch_idx, batch in enumerate(train_loader):\n        try:\n            # 数据移动到设备\n            volume = batch['volume'].to(device)\n            meta = batch['meta'].to(device)\n            labels = batch['labels'].to(device)\n            \n            # 前向传播\n            optimizer.zero_grad()\n            outputs = model(volume, meta)\n            \n            # 计算损失\n            loss = criterion(outputs, labels, target_columns)\n            \n            # 反向传播\n            loss.backward()\n            \n            # 梯度裁剪\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            \n            optimizer.step()\n            \n            total_loss += loss.item()\n            batch_count += 1\n            \n            # 进度显示\n            if batch_idx % 100 == 0:\n                elapsed = time.time() - start_time\n                batches_done = batch_idx + 1\n                eta = elapsed / batches_done * (len(train_loader) - batches_done)\n                \n                print(f\"   Batch {batch_idx:3d}/{len(train_loader):3d} | \"\n                      f\"Loss: {loss.item():.4f} | \"\n                      f\"ETA: {eta:.0f}s\")\n                \n        except Exception as e:\n            print(f\"   ❌ Batch {batch_idx} failed: {e}\")\n            continue\n    \n    avg_loss = total_loss / batch_count if batch_count > 0 else 0\n    elapsed = time.time() - start_time\n    \n    print(f\"✅ Epoch {epoch} completed:\")\n    print(f\"   • Average Loss: {avg_loss:.4f}\")\n    print(f\"   • Time: {elapsed:.1f}s\")\n    print(f\"   • Samples/sec: {len(train_loader.dataset)/elapsed:.1f}\")\n    \n    return avg_loss\n\ndef validate_epoch(model, val_loader, target_columns, epoch):\n    \"\"\"验证一个epoch\"\"\"\n    model.eval()\n    \n    all_predictions = []\n    all_labels = []\n    \n    print(f\"\\n📊 Validating Epoch {epoch}\")\n    print(\"-\" * 50)\n    \n    start_time = time.time()\n    \n    with torch.no_grad():\n        for batch_idx, batch in enumerate(val_loader):\n            try:\n                volume = batch['volume'].to(device)\n                meta = batch['meta'].to(device)\n                labels = batch['labels'].to(device)\n                \n                outputs = model(volume, meta)\n                \n                # 应用sigmoid获取概率\n                predictions = torch.sigmoid(outputs)\n                \n                all_predictions.append(predictions.cpu().numpy())\n                all_labels.append(labels.cpu().numpy())\n                \n            except Exception as e:\n                print(f\"   ❌ Validation batch {batch_idx} failed: {e}\")\n                continue\n    \n    if not all_predictions:\n        print(\"❌ No successful validation batches!\")\n        return 0.0, {}\n    \n    # 合并所有预测\n    predictions = np.vstack(all_predictions)\n    labels = np.vstack(all_labels)\n    \n    # 计算AUC指标\n    aucs = {}\n    valid_aucs = []\n    \n    print(f\"\\n📈 Per-location AUC scores:\")\n    \n    for i, col in enumerate(target_columns):\n        try:\n            col_labels = labels[:, i]\n            col_preds = predictions[:, i]\n            \n            # 需要至少有正负样本才能计算AUC\n            if len(np.unique(col_labels)) > 1:\n                auc = roc_auc_score(col_labels, col_preds)\n                aucs[col] = auc\n                valid_aucs.append(auc)\n                \n                status = \"🟢\" if auc > 0.8 else \"🟡\" if auc > 0.6 else \"🔴\"\n                print(f\"   {status} {col[:30]:<30}: {auc:.4f}\")\n            else:\n                aucs[col] = 0.5  # 默认值\n                print(f\"   ⚪ {col[:30]:<30}: N/A (no variation)\")\n                \n        except Exception as e:\n            aucs[col] = 0.5\n            print(f\"   ❌ {col[:30]:<30}: Error ({e})\")\n    \n    # 计算平均AUC\n    mean_auc = np.mean(valid_aucs) if valid_aucs else 0.5\n    \n    elapsed = time.time() - start_time\n    \n    print(f\"\\n📊 Validation Summary:\")\n    print(f\"   • Mean AUC: {mean_auc:.4f}\")\n    print(f\"   • Valid locations: {len(valid_aucs)}/{len(target_columns)}\")\n    print(f\"   • Time: {elapsed:.1f}s\")\n    \n    return mean_auc, aucs\n\ndef run_strategy1_training(training_components, num_epochs=8):\n    \"\"\"运行策略1训练\"\"\"\n    print(\"🏆 Starting Strategy 1 Fine-tuning Training\")\n    print(\"=\" * 80)\n    \n    # 获取训练组件\n    train_loader = training_components['train_loader']\n    val_loader = training_components['val_loader']\n    class_weights = training_components['class_weights']\n    target_columns = training_components['target_columns']\n    \n    # 设置模型\n    model, criterion, optimizer, scheduler = setup_strategy1_model(\n        CFG.selected_checkpoint, target_columns, class_weights\n    )\n    \n    # 训练历史\n    history = {\n        'train_loss': [],\n        'val_auc': [],\n        'learning_rate': []\n    }\n    \n    best_val_auc = 0.0\n    best_epoch = 0\n    \n    print(f\"\\n🚀 Starting training for {num_epochs} epochs...\")\n    \n    for epoch in range(1, num_epochs + 1):\n        print(f\"\\n{'='*60}\")\n        print(f\"EPOCH {epoch}/{num_epochs}\")\n        print(f\"{'='*60}\")\n        \n        # 训练\n        train_loss = train_epoch(model, train_loader, criterion, optimizer, target_columns, epoch)\n        \n        # 验证\n        val_auc, location_aucs = validate_epoch(model, val_loader, target_columns, epoch)\n        \n        # 学习率调度\n        scheduler.step(val_auc)\n        current_lr = optimizer.param_groups[0]['lr']\n        \n        # 记录历史\n        history['train_loss'].append(train_loss)\n        history['val_auc'].append(val_auc)\n        history['learning_rate'].append(current_lr)\n        \n        # 保存最佳模型\n        if val_auc > best_val_auc:\n            best_val_auc = val_auc\n            best_epoch = epoch\n            \n            # 保存最佳模型\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'val_auc': val_auc,\n                'location_aucs': location_aucs,\n                'target_columns': target_columns\n            }, 'best_strategy1_model.pth')\n            \n            print(f\"   🏆 New best model saved! AUC: {val_auc:.4f}\")\n        \n        # 当前epoch总结\n        print(f\"\\n📊 Epoch {epoch} Summary:\")\n        print(f\"   • Train Loss: {train_loss:.4f}\")\n        print(f\"   • Val AUC: {val_auc:.4f}\")\n        print(f\"   • Best AUC: {best_val_auc:.4f} (epoch {best_epoch})\")\n        print(f\"   • Learning Rate: {current_lr:.6f}\")\n        \n        # 早停检查\n        if epoch - best_epoch > 4:\n            print(f\"\\n⏹️ Early stopping: No improvement for 4 epochs\")\n            break\n    \n    # 训练完成\n    print(f\"\\n🎉 Training completed!\")\n    print(f\"   • Best validation AUC: {best_val_auc:.4f}\")\n    print(f\"   • Best epoch: {best_epoch}\")\n    print(f\"   • Total epochs: {epoch}\")\n    \n    # 加载最佳模型\n    if best_val_auc > 0:\n        print(f\"\\n📥 Loading best model...\")\n        checkpoint = torch.load('best_strategy1_model.pth')\n        model.load_state_dict(checkpoint['model_state_dict'])\n        print(f\"✅ Best model loaded (AUC: {checkpoint['val_auc']:.4f})\")\n    \n    return model, history, best_val_auc\n\n# 执行训练\nif __name__ == \"__main__\":\n    if 'TRAINING_COMPONENTS' in globals() and TRAINING_COMPONENTS:\n        print(\"🎯 All components ready - starting Strategy 1 training...\")\n        \n        TRAINED_MODEL, TRAINING_HISTORY, FINAL_AUC = run_strategy1_training(\n            TRAINING_COMPONENTS, num_epochs=8\n        )\n        \n        print(f\"\\n🏆 FINAL RESULTS:\")\n        print(f\"   • Final AUC Score: {FINAL_AUC:.4f}\")\n        \n        if FINAL_AUC > 0.7:\n            print(f\"   • Status: 🎉 EXCELLENT! Ready for submission!\")\n        elif FINAL_AUC > 0.6:\n            print(f\"   • Status: ✅ GOOD! Should submit!\")\n        elif FINAL_AUC > 0.5:\n            print(f\"   • Status: 🟡 Moderate. Consider improvements.\")\n        else:\n            print(f\"   • Status: ❌ Poor. Needs more work.\")\n            \n        print(f\"\\n🚀 Next step: Run Cell 14 for final submission generation!\")\n        \n    else:\n        print(f\"❌ TRAINING_COMPONENTS not found - please run Cell 12 first\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-18T14:59:20.700408Z","iopub.execute_input":"2025-08-18T14:59:20.700693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}