{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =================================================================\n# 🫁 CliniScan: Unified Kaggle Training Script (Final v3)\n# =================================================================\n# Instructions:\n# 1. Create a New Notebook on Kaggle.\n# 2. Add the dataset: `vinbigdata-chest-xray-abnormalities-detection`.\n# 3. Turn ON \"GPU T4 x2\" in the right sidebar.\n# 4. Paste this entire file into a single cell and click \"Run All\".\n# 5. When it finishes, hit the Refresh (🔄) button next to `/kaggle/working`.\n# 6. Download `my_trained_models.zip`!\n\n# --- CELL 1: SETUP & INSTALLATIONS ---\n!pip install -q pydicom opencv-python-headless ultralytics pandas numpy scikit-learn\n\nimport os\nimport glob\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport torch\nimport shutil\n\n# --- CELL 2: GPU SETUP ---\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\nif not torch.cuda.is_available():\n    print(\"⚠️ WARNING: No GPU found. Turn on 'GPU T4 x2' in Kaggle Settings!\")\n\n# --- CELL 3: BULLETPROOF PATH SETUP ---\n# We use glob to aggressively search for `train.csv` anywhere inside /kaggle/input.\n# This prevents ANY Kaggle folder naming issues!\nsearch_paths = glob.glob('/kaggle/input/**/train.csv', recursive=True)\n\nif len(search_paths) > 0:\n    BASE_DIR = os.path.dirname(search_paths[0])\n    print(f\"🎯 Dataset successfully found at: {BASE_DIR}\")\nelse:\n    print(\"Folders currently in /kaggle/input/:\", os.listdir('/kaggle/input'))\n    raise FileNotFoundError(\"Could not find train.csv anywhere! Please ensure the dataset is attached in the right sidebar.\")\n\nTRAIN_DIR = os.path.join(BASE_DIR, 'train')\nCSV_PATH = os.path.join(BASE_DIR, 'train.csv')\n\nOUTPUT_DIR = '/kaggle/working/cliniscan'\nIMG_TRAIN = os.path.join(OUTPUT_DIR, 'dataset/images/train')\nLBL_TRAIN = os.path.join(OUTPUT_DIR, 'dataset/labels/train')\nIMG_VAL   = os.path.join(OUTPUT_DIR, 'dataset/images/val')\nLBL_VAL   = os.path.join(OUTPUT_DIR, 'dataset/labels/val')\n\nfor d in [IMG_TRAIN, IMG_VAL, LBL_TRAIN, LBL_VAL]:\n    os.makedirs(d, exist_ok=True)\n\n# --- CELL 4: DATA PREPARATION (DICOM -> PNG & YOLO LABELS) ---\nprint(\"\\n--- Starting Data Prep ---\")\ndf = pd.read_csv(CSV_PATH)\ndf.fillna(0, inplace=True) \n\n# Using all 15,000 images for maximum accuracy!\nunique_images = df['image_id'].unique()\nnp.random.seed(42)\nnp.random.shuffle(unique_images)\n\nsplit_idx = int(len(unique_images) * 0.8) # 80% train, 20% validation\nsubset_images = unique_images # Keep variable name same so tqdm loop works\ntrain_samples = set(unique_images[:split_idx])\nval_samples = set(unique_images[split_idx:])\n\ndf_subset = df # Use the full dataframe\nIMG_SIZE = 512\n\ndef process_dicom_to_png_and_yolo(img_id, split_name):\n    dicom_path = os.path.join(TRAIN_DIR, f\"{img_id}.dicom\")\n    if not os.path.exists(dicom_path): return False\n        \n    dicom = pydicom.dcmread(dicom_path)\n    data = apply_voi_lut(dicom.pixel_array, dicom)\n    if getattr(dicom, \"PhotometricInterpretation\", \"\") == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    original_h, original_w = data.shape\n    resized_img = cv2.resize(data, (IMG_SIZE, IMG_SIZE))\n    \n    img_folder = IMG_TRAIN if split_name == 'train' else IMG_VAL\n    cv2.imwrite(os.path.join(img_folder, f\"{img_id}.png\"), resized_img)\n    \n    label_folder = LBL_TRAIN if split_name == 'train' else LBL_VAL\n    img_annotations = df[df['image_id'] == img_id]\n    \n    yolo_lines = []\n    for _, row in img_annotations.iterrows():\n        class_id = int(row['class_id'])\n        if class_id == 14: continue # Skip \"No finding\" for detection boxes\n            \n        x_min, y_min = row['x_min'] / original_w, row['y_min'] / original_h\n        x_max, y_max = row['x_max'] / original_w, row['y_max'] / original_h\n        \n        x_center, y_center = (x_min + x_max) / 2, (y_min + y_max) / 2\n        width, height = x_max - x_min, y_max - y_min\n        yolo_lines.append(f\"{class_id} {x_center:.6f} {y_center:.6f} {width:.6f} {height:.6f}\")\n        \n    with open(os.path.join(label_folder, f\"{img_id}.txt\"), \"w\") as f:\n        f.write(\"\\n\".join(yolo_lines))\n    return True\n\nprint(\"Converting DICOM to PNG and generating YOLO labels...\")\nfor img_id in tqdm(subset_images):\n    split = 'train' if img_id in train_samples else 'val'\n    process_dicom_to_png_and_yolo(img_id, split)\n\n# --- CELL 5: TRAIN EFFICIENTNET CLASSIFIER ---\nprint(\"\\n--- Training EfficientNet Classifier ---\")\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass CliniScanBinaryDataset(Dataset):\n    def __init__(self, df_subset, img_ids, base_img_dir):\n        self.img_ids = list(img_ids)\n        self.labels = {}\n        for img_id in self.img_ids:\n            img_rows = df_subset[df_subset['image_id'] == img_id]\n            is_normal = all(img_rows['class_name'] == 'No finding')\n            self.labels[img_id] = 0 if is_normal else 1\n            \n        self.transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])\n\n    def __len__(self): return len(self.img_ids)\n        \n    def __getitem__(self, idx):\n        img_id = self.img_ids[idx]\n        split = 'train' if img_id in train_samples else 'val'\n        folder = IMG_TRAIN if split == 'train' else IMG_VAL\n        \n        img = cv2.imread(os.path.join(folder, f\"{img_id}.png\"))\n        img = cv2.resize(img, (224, 224))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        if self.transform: img = self.transform(img)\n        return img, torch.tensor(self.labels[img_id], dtype=torch.float32)\n\ntrain_loader = DataLoader(CliniScanBinaryDataset(df_subset, train_samples, OUTPUT_DIR), batch_size=16, shuffle=True)\nval_loader = DataLoader(CliniScanBinaryDataset(df_subset, val_samples, OUTPUT_DIR), batch_size=16, shuffle=False)\n\nmodel_cls = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)\nmodel_cls.classifier[1] = nn.Linear(model_cls.classifier[1].in_features, 2) # Binary output\nmodel_cls = model_cls.to(device)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model_cls.parameters(), lr=0.001)\n\n# Train Classifier (3 Epochs)\nfor epoch in range(3):\n    model_cls.train()\n    total_loss = 0\n    for images, labels in train_loader:\n        images, labels = images.to(device), labels.to(device).long()\n        optimizer.zero_grad()\n        loss = criterion(model_cls(images), labels)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n    print(f\"Classifier Epoch {epoch+1}/3 - Loss: {total_loss/len(train_loader):.4f}\")\n\n# ✅ SAVE CLASSIFIER WEIGHTS\ntorch.save(model_cls.state_dict(), '/kaggle/working/best_classifier.pth')\nprint(\"✅ Saved best_classifier.pth\")\n\n# --- CELL 6: TRAIN YOLOv8 DETECTOR ---\nfrom ultralytics import YOLO\nprint(\"\\n--- Training YOLOv8 Detector ---\")\nyaml_content = f\"\"\"\npath: {OUTPUT_DIR}/dataset\ntrain: images/train\nval: images/val\nnc: 14\nnames: ['Aortic enlargement', 'Atelectasis', 'Calcification', 'Cardiomegaly', 'Consolidation', 'ILD', 'Infiltration', 'Lung Opacity', 'Nodule/Mass', 'Other lesion', 'Pleural effusion', 'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis']\n\"\"\"\nwith open(os.path.join(OUTPUT_DIR, \"data.yaml\"), \"w\") as f:\n    f.write(yaml_content)\n\nmodel_det = YOLO('yolov8s.pt')\n# Train YOLO logic (3 Epochs)\nmodel_det.train(data=os.path.join(OUTPUT_DIR, \"data.yaml\"), epochs=3, imgsz=IMG_SIZE, project='/kaggle/working', name=\"yolo_baseline\")\nprint(\"✅ YOLO training complete. Weights auto-saved to /kaggle/working/yolo_baseline/weights/best.pt\")\n\n# --- CELL 7: ZIP EVERYTHING FOR EASY DOWNLOAD ---\nprint(\"\\n--- Zipping trained models ---\")\nos.makedirs('/kaggle/working/download_models', exist_ok=True)\n\n# Copy EfficientNet\nshutil.copy('/kaggle/working/best_classifier.pth', '/kaggle/working/download_models/best_classifier.pth')\n\n# Copy YOLOv8\nyolo_path = '/kaggle/working/yolo_baseline/weights/best.pt'\nif os.path.exists(yolo_path):\n    shutil.copy(yolo_path, '/kaggle/working/download_models/best_detector.pt')\n\nshutil.make_archive('/kaggle/working/my_trained_models', 'zip', '/kaggle/working/download_models')\nprint(\"\\n🎉 ALL DONE! Refresh the Kaggle Output panel and download 'my_trained_models.zip'!\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-24T06:45:15.226094Z","iopub.execute_input":"2026-03-24T06:45:15.226951Z"}},"outputs":[],"execution_count":null}]}