{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"sourceType":"competition"}],"dockerImageVersionId":31091,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification albumentations","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nfrom tqdm.auto import tqdm\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\n\nfrom transformers import ViTForImageClassification\n\n# Check for multiple GPUs\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}, Number of GPUs: {torch.cuda.device_count()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ndf = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/train1.csv')\ndf.dropna(inplace=True)\nlabel_cols = ['Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Enlarged Cardiomediastinum',\n              'Fracture', 'Lung Lesion', 'Lung Opacity', 'No Finding', 'Pleural Effusion',\n              'Pleural Other', 'Pneumonia', 'Pneumothorax', 'Support Devices']\ndf = df[['Image_name', 'Patient_ID'] + label_cols]\n\n# Aggregate by patient\npatient_df = df.groupby('Patient_ID')[label_cols].max().reset_index()\npatient_ids = patient_df['Patient_ID']\npatient_labels = patient_df[label_cols]\n\n# Multi-label stratified split (single 80-20 split)\nmsss = MultilabelStratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\nfor train_idx, val_idx in msss.split(patient_ids, patient_labels):\n    train_patients = patient_ids.iloc[train_idx]\n    val_patients = patient_ids.iloc[val_idx]\n    train_df = df[df['Patient_ID'].isin(train_patients)]\n    val_df = df[df['Patient_ID'].isin(val_patients)]\nprint(f\"Train shape {train_df.shape}, Val shape {val_df.shape}\")\nprint(\"Train label distribution:\\n\", train_df[label_cols].mean())\nprint(\"Val label distribution:\\n\", val_df[label_cols].mean())\n\n# Verify no patient overlap\nassert len(set(train_df['Patient_ID']) & set(val_df['Patient_ID'])) == 0, \"Patient overlap detected!\"\n\n# Remove Patient_ID and save to CSV\ntrain_df = train_df[['Image_name'] + label_cols]\nval_df = val_df[['Image_name'] + label_cols]\ntrain_df.to_csv('/kaggle/working/train_fold_1.csv', index=False)\nval_df.to_csv('/kaggle/working/val_fold_1.csv', index=False)\n\n# Verify saved files\nprint(\"\\nSaved train CSV columns:\", pd.read_csv('/kaggle/working/train_fold_1.csv').columns.tolist())\nprint(\"Saved val CSV columns:\", pd.read_csv('/kaggle/working/val_fold_1.csv').columns.tolist())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define separate pipelines for training (with augmentation) and validation (only resizing/normalization)\ndef get_transforms(train=False):\n    if train:\n        return A.Compose([\n            A.Resize(224, 224),\n            A.HorizontalFlip(p=0.5),\n            A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.1, rotate_limit=5, p=0.5),\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n            A.CLAHE(p=0.3),\n            A.Normalize(mean=np.array([0.485, 0.456, 0.406]), std=np.array([0.229, 0.224, 0.225])),\n            ToTensorV2()\n        ])\n    else:\n        return A.Compose([\n            A.Resize(224, 224),\n            A.Normalize(mean=np.array([0.485, 0.456, 0.406]), std=np.array([0.229, 0.224, 0.225])),\n            ToTensorV2()\n        ])\n\nclass ChestXrayDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, image_dir: str, label_cols: list, transforms=None, is_test=False):\n        self.df = df\n        self.image_dir = image_dir\n        self.transforms = transforms\n        self.labels = label_cols\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = os.path.join(self.image_dir, row['Image_name'])\n        image = cv2.imread(image_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        labels = torch.tensor(row[self.labels].values.astype(float), dtype=torch.float32)\n        augmented = self.transforms(image=image)\n        image = augmented['image']\n        return image if self.is_test else (image, labels)\n\n# Create dataset instances\ntrain_dataset = ChestXrayDataset(\n    df=train_df,\n    image_dir='/kaggle/input/grand-xray-slam-division-a/train1',\n    label_cols=label_cols,\n    transforms=get_transforms(train=True)\n)\n\nval_dataset = ChestXrayDataset(\n    df=val_df,\n    image_dir='/kaggle/input/grand-xray-slam-division-a/train1',\n    label_cols=label_cols,\n    transforms=get_transforms(train=False)\n)\n\n# --- Multi-GPU change: Increase batch size to utilize both GPUs ---\nbatch_size = 64  # Doubled from 32 to leverage GPU memory (adjust if OOM occurs)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load model\nmodel = ViTForImageClassification.from_pretrained(\n    'codewithdark/vit-chest-xray',\n    num_labels=len(label_cols),\n    problem_type='multi_label_classification',\n    ignore_mismatched_sizes=True\n)\n\nif torch.cuda.device_count() > 1:\n    print(f\"Using {torch.cuda.device_count()} GPUs\")\n    model = nn.DataParallel(model)\n\nmodel.to(device)\n\n# Print parameter stats\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f\"Total Parameters: {total_params}\")\nprint(f\"Trainable Parameters: {trainable_params}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# weighted focal loss\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=None, gamma=2.0, reduction='mean'):\n        super().__init__()\n        self.alpha=alpha,\n        self.gamma=gamma,\n        self.reduction=reduction\n        \n    def forward(self, inputs, targets):\n        BCE_loss = nn.functional.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-BCE_loss)\n        if self.alpha is not None:\n            BCE_loss = self.alpha[0] * BCE_loss\n        F_loss = (1-pt)**self.gamma[0] * BCE_loss\n        if self.reduction == 'mean':\n            return F_loss.mean()\n        else :\n            return F_loss.sum()\n\n\n# Loss function with positive weights\npos_weights = torch.tensor([len(train_df) / (train_df[col].sum() + 1e-6) for col in label_cols]).float().to(device)\nloss_fn = FocalLoss(alpha=pos_weights)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=2)\n\n# Training loop\nepochs = 10  # Define epochs (was missing in original code)\nbest_auc = 0.0","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(1, epochs+1):\n    model.train()\n    train_loss = 0.0\n    train_preds, train_labels = [], []\n\n    train_loop = tqdm(train_loader, leave=True, desc=f\"Epoch {epoch}/{epochs} Training\")\n\n    for images, labels in train_loop:\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images).logits\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item() * images.size(0)\n        \n        train_preds.append(torch.sigmoid(outputs).detach().cpu().numpy())\n        train_labels.append(labels.detach().cpu().numpy())\n        \n        train_loop.set_postfix(loss=loss.item())\n    epoch_train_loss = train_loss / len(train_dataset)\n    train_preds = np.vstack(train_preds)\n    train_labels = np.vstack(train_labels)\n    train_auc = roc_auc_score(train_labels, train_preds, average='macro')\n\n    # Validation\n    model.eval()\n    val_loss = 0.0\n    val_preds, val_labels = [], []\n\n    val_loop = tqdm(val_loader, leave=True, desc=f\"Epoch {epoch}/{epochs} Validation\")\n\n    with torch.no_grad():\n        for images, labels in val_loop:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images).logits\n            loss = loss_fn(outputs, labels)\n            val_loss += loss.item() * images.size(0)\n\n            val_preds.append(torch.sigmoid(outputs).cpu().numpy())\n            val_labels.append(labels.cpu().numpy())\n\n        epoch_val_loss = val_loss / len(val_dataset)\n        val_preds = np.vstack(val_preds)\n        val_labels = np.vstack(val_labels)\n        val_auc = roc_auc_score(val_labels, val_preds, average='macro')\n\n        print(f\"\\nEpoch {epoch}/{epochs} Complete: Train Loss: {epoch_train_loss:.4f}, Train AUC: {train_auc:.4f}, Val Loss: {epoch_val_loss:.4f}, Val AUC: {val_auc:.4f}\")\n\n        scheduler.step(val_auc)\n\n        # Save best model\n        if val_auc > best_auc:\n            best_auc = val_auc\n            if isinstance(model, nn.DataParallel):\n                torch.save(model.module.state_dict(), '/kaggle/working/best_model.pth')\n            else:\n                torch.save(model.state_dict(), '/kaggle/working/best_model.pth')\n            print(f\"Checkpoint Model saved. best AUC: {best_auc}\")\n        ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate Sample Submission\nsample_submission = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/sample_submission_1.csv')\n\n\ntest_dataset = ChestXrayDataset(\n    df=sample_submission,\n    image_dir='/kaggle/input/grand-xray-slam-division-a/test1',\n    label_cols=label_cols,\n    transforms=get_transforms(train=False),\n    is_test=True\n)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n# test_loader = pl.MpDeviceLoader(test_loader, device)\n\n# Load model\nmodel = ViTForImageClassification.from_pretrained(\n    'codewithdark/vit-chest-xray',\n    num_labels=len(label_cols),\n    problem_type='multi_label_classification',\n    ignore_mismatched_sizes=True\n)\n\nmodel.load_state_dict(torch.load('best_model.pth'))\nprint(\"model laoded successfully.\")\nif torch.cuda.device_count() > 1:\n    print(f\"Using {torch.cuda.device_count()} GPUs\")\n    model = nn.DataParallel(model)\nmodel = model.to(device)\n\nmodel.eval()\npredictions = []\nwith torch.no_grad():\n    for images in test_loader:\n        images = images.to(device)\n        outputs = model(images).logits\n        batch_preds = torch.sigmoid(outputs).cpu().numpy()\n        predictions.append(batch_preds)\n\npredictions = np.vstack(predictions)\npredictions = predictions[:len(sample_submission)]\n\nsubmission_df = sample_submission.copy()\nsubmission_df[label_cols] = predictions\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\nprint(\"Submission file created: submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}