{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112899,"databundleVersionId":13449579,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Fine-Tuning DenseNet121 on Chest X-Rays\n\n### Fine-Tuning DenseNet121 for Multi-Label Thoracic Disease Classification\n\n---\n\n**Project Goals**\n\n* **Model:** Fine-tune a **DenseNet121** on chest X-rays.\n* **Task:** Multi-label classification of thoracic diseases.\n* **Dataset:** Grand X-Ray Slam (Kaggle).\n\n---\n\n**Key Techniques & Features**\n\n* **Data Analysis:**\n    * Label distribution and correlations.\n* **Data Augmentation:**\n    * Rotations, flips, and CLAHE-like normalization.\n* **Training & Optimization:**\n    * Weighted Binary Cross-Entropy (BCE) loss to address class imbalance.\n    * `ReduceLROnPlateau` scheduler.\n    * Mixed precision training for efficiency.\n    * Early stopping to prevent overfitting.\n* **Evaluation:**\n    * Loss and macro ROC-AUC metrics.\n* **Deliverable:**\n    * Creation of a submission file.","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import roc_auc_score\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom tqdm.auto import tqdm\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom collections import Counter\n\n# Set device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T15:01:43.486206Z","iopub.execute_input":"2025-09-01T15:01:43.48643Z","iopub.status.idle":"2025-09-01T15:01:53.86781Z","shell.execute_reply.started":"2025-09-01T15:01:43.486411Z","shell.execute_reply":"2025-09-01T15:01:53.867124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load and analyze data\ntry:\n    train_df = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/train1.csv')\n    print(f\"Loaded train1.csv with {len(train_df)} rows\")\nexcept FileNotFoundError:\n    print(\"Error: train1.csv not found. Ensure dataset is attached.\")\n    raise\n\nlabel_columns = [\n    'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Enlarged Cardiomediastinum',\n    'Fracture', 'Lung Lesion', 'Lung Opacity', 'No Finding', 'Pleural Effusion',\n    'Pleural Other', 'Pneumonia', 'Pneumothorax', 'Support Devices'\n]\n\n# Check for missing columns\nmissing_cols = [col for col in label_columns if col not in train_df.columns]\nif missing_cols:\n    print(f\"Error: Missing columns in train1.csv: {missing_cols}\")\n    raise KeyError(f\"Missing columns: {missing_cols}\")\n\n# --- Data Visualization ---\n\n# 1. Label Distribution Plot\nprint(\"\\nGenerating Label Distribution Plot...\")\nlabel_counts = train_df[label_columns].sum().sort_values(ascending=False)\nplt.figure(figsize=(12, 6))\nsns.barplot(x=label_counts.index, y=label_counts.values, palette=\"viridis\")\nplt.title('Distribution of Disease Labels in Training Data')\nplt.xlabel('Disease Label')\nplt.ylabel('Number of Cases')\nplt.xticks(rotation=45, ha='right')\nplt.tight_layout()\nplt.show()\n\n# 2. Disease Correlation Plot\nprint(\"\\nGenerating Disease Correlation Plot...\")\nplt.figure(figsize=(10, 8))\ncorrelation_matrix = train_df[label_columns].corr()\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=.5)\nplt.title('Correlation between Disease Labels')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T15:01:53.869124Z","iopub.execute_input":"2025-09-01T15:01:53.869561Z","iopub.status.idle":"2025-09-01T15:01:55.113647Z","shell.execute_reply.started":"2025-09-01T15:01:53.86954Z","shell.execute_reply":"2025-09-01T15:01:55.112696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform iterative stratified (multi-label) split\nX = train_df[['Image_name']]\ny = train_df[label_columns].values\n\nX_train, y_train, X_val, y_val = iterative_train_test_split(X.values, y, test_size=0.2)\n\ntrain_data = pd.DataFrame(X_train, columns=['Image_name'])\ntrain_data[label_columns] = y_train\n\nval_data = pd.DataFrame(X_val, columns=['Image_name'])\nval_data[label_columns] = y_val\n\nprint(f\"Train samples after stratified split: {len(train_data)}\")\nprint(f\"Validation samples after stratified split: {len(val_data)}\")\nprint(\"\\nDistribution of labels in training set:\", Counter(np.where(y_train == 1)[1]))\nprint(\"Distribution of labels in validation set:\", Counter(np.where(y_val == 1)[1]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T15:01:55.114442Z","iopub.execute_input":"2025-09-01T15:01:55.11493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define Dataset and DataLoader classes\nclass ChestXRayDataset(Dataset):\n    def __init__(self, df, img_size=(224, 224), is_test=False, transforms=None):\n        self.df = df\n        self.img_size = img_size\n        self.is_test = is_test\n        self.label_columns = label_columns\n        self.image_dir = '/kaggle/input/grand-xray-slam-division-a/train1/' if not is_test else '/kaggle/input/grand-xray-slam-division-a/test1/'\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.image_dir, row['Image_name'])\n        \n        try:\n            img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n            if img is None:\n                raise FileNotFoundError\n        except FileNotFoundError:\n            print(f\"Warning: Image not found at {img_path}. Returning black image.\")\n            img = np.zeros(self.img_size, dtype=np.uint8)\n        \n        img = cv2.resize(img, self.img_size)\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n        \n        if self.transforms:\n            img = self.transforms(img)\n        \n        if not self.is_test:\n            labels = row[self.label_columns].values.astype(np.float32)\n            return img, torch.tensor(labels)\n        \n        return img\n\n# Define image transformations with data augmentation for training\ntrain_transforms = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomRotation(degrees=10),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Define transformations for validation/testing (no augmentation)\nval_transforms = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Create DataLoader instances\nbatch_size = 32\ntrain_dataset = ChestXRayDataset(train_data, img_size=(224, 224), transforms=train_transforms)\nval_dataset = ChestXRayDataset(val_data, img_size=(224, 224), transforms=val_transforms)\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n\nprint(\"PyTorch DataLoaders created successfully.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate class weights for Weighted BCE Loss\nprint(\"Calculating class weights...\")\ntotal_samples = len(train_data)\npositive_counts = train_data[label_columns].sum()\nnegative_counts = total_samples - positive_counts\npos_weight = negative_counts / positive_counts\npos_weight = torch.tensor(pos_weight.values, dtype=torch.float).to(device)\n\nprint(\"Calculated positive weights:\", pos_weight)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build and fine-tune DenseNet-121\ndef create_model(num_classes=14, dropout_rate=0.5):\n    # Load pretrained DenseNet-121\n    model = models.densenet121(weights=models.DenseNet121_Weights.IMAGENET1K_V1)\n    \n    # Freeze all layers except the last dense block and the classifier\n    for name, param in model.named_parameters():\n        if 'denseblock4' not in name and 'norm5' not in name:\n            param.requires_grad = False\n\n    # Get the number of input features for the classifier\n    num_ftrs = model.classifier.in_features\n    \n    # Redefine the classifier to be a sequential block with Dropout and a linear layer\n    model.classifier = nn.Sequential(\n        nn.Dropout(dropout_rate),\n        nn.Linear(num_ftrs, num_classes)\n    )\n    return model\n\nmodel = create_model().to(device)\n# The criterion is now initialized with the calculated pos_weight\ncriterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode='max', factor=0.1, patience=3\n)\nprint(\"Model, Loss, Optimizer, and Scheduler created.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train model with Early Stopping\nnum_epochs = 30\nbest_val_auc = 0.0\npatience = 5\nepochs_no_improve = 0\nearly_stop = False\n\n# Lists to store training history\ntrain_loss_history = []\nval_loss_history = []\ntrain_auc_history = []\nval_auc_history = []\n\nfor epoch in range(num_epochs):\n    if early_stop:\n        print(\"Early stopping triggered.\")\n        break\n\n    # --- Training loop ---\n    model.train()\n    running_train_loss = 0.0\n    train_preds, train_labels = [], []\n    train_loop = tqdm(train_loader, leave=True, desc=f\"Epoch {epoch+1}/{num_epochs} Training\")\n    for images, labels in train_loop:\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_train_loss += loss.item() * images.size(0)\n        train_preds.append(torch.sigmoid(outputs).detach().cpu().numpy())\n        train_labels.append(labels.detach().cpu().numpy())\n        train_loop.set_postfix(loss=loss.item())\n    epoch_train_loss = running_train_loss / len(train_data)\n    train_loss_history.append(epoch_train_loss)\n    train_auc = roc_auc_score(np.vstack(train_labels), np.vstack(train_preds), average='macro')\n    train_auc_history.append(train_auc)\n\n\n    # --- Validation loop ---\n    model.eval()\n    running_val_loss = 0.0\n    val_preds, val_labels = [], []\n    val_loop = tqdm(val_loader, leave=True, desc=f\"Epoch {epoch+1}/{num_epochs} Validation\")\n    with torch.no_grad():\n        for images, labels in val_loop:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            running_val_loss += loss.item() * images.size(0)\n            val_preds.append(torch.sigmoid(outputs).cpu().numpy())\n            val_labels.append(labels.cpu().numpy())\n    epoch_val_loss = running_val_loss / len(val_data)\n    val_loss_history.append(epoch_val_loss)\n    val_preds = np.vstack(val_preds)\n    val_labels = np.vstack(val_labels)\n    val_auc = roc_auc_score(val_labels, val_preds, average='macro')\n    val_auc_history.append(val_auc)\n\n    # Log metrics\n    print(f\"\\nEpoch {epoch+1}/{num_epochs} Complete: Train Loss: {epoch_train_loss:.4f}, Val Loss: {epoch_val_loss:.4f}, Train AUC: {train_auc:.4f}, Val AUC: {val_auc:.4f}\")\n\n    # Early stopping and saving the best model\n    if val_auc > best_val_auc:\n        print(f\"Validation AUC improved from {best_val_auc:.4f} to {val_auc:.4f}. Saving model...\")\n        best_val_auc = val_auc\n        torch.save(model.state_dict(), 'best_model.pth')\n        epochs_no_improve = 0\n    else:\n        epochs_no_improve += 1\n        print(f\"Validation AUC did not improve. Patience: {epochs_no_improve}/{patience}\")\n        if epochs_no_improve >= patience:\n            early_stop = True\n\n    # Step the scheduler\n    scheduler.step(val_auc)\n\nprint(\"Training complete.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot training history for Loss and AUC\nepochs_ran = len(train_loss_history)\nepochs = range(1, epochs_ran + 1)\n\n# Plot Loss History\nplt.figure(figsize=(12, 6))\nplt.plot(epochs, train_loss_history, 'b', label='Training Loss')\nplt.plot(epochs, val_loss_history, 'r', label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.grid(True)\nplt.show()\n\n# Plot AUC History\nplt.figure(figsize=(12, 6))\nplt.plot(epochs, train_auc_history, 'b', label='Training AUC')\nplt.plot(epochs, val_auc_history, 'r', label='Validation AUC')\nplt.title('Training and Validation AUC')\nplt.xlabel('Epochs')\nplt.ylabel('AUC')\nplt.legend()\nplt.grid(True)\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Generate submission file\ntry:\n    sample_submission = pd.read_csv('/kaggle/input/grand-xray-slam-division-a/sample_submission_1.csv')\n    print(f\"Loaded sample_submission_1.csv with {len(sample_submission)} rows\")\nexcept FileNotFoundError:\n    print(\"Error: sample_submission_1.csv not found.\")\n    raise\n\n# Create test dataset and loader\ntest_dataset = ChestXRayDataset(sample_submission, img_size=(224, 224), is_test=True, transforms=val_transforms)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n\n# Load the best model and set to evaluation mode\nmodel.load_state_dict(torch.load('best_model.pth'))\nmodel.eval()\n\n# Generate predictions\npredictions = []\nwith torch.no_grad():\n    for images in tqdm(test_loader, desc=\"Generating predictions\"):\n        images = images.to(device)\n        outputs = model(images)\n        batch_preds = torch.sigmoid(outputs).cpu().numpy()\n        predictions.append(batch_preds)\n\n# Combine predictions and create submission DataFrame\npredictions = np.vstack(predictions)\npredictions = predictions[:len(sample_submission)]\nsubmission_df = sample_submission.copy()\nsubmission_df[label_columns] = predictions\n\n# Save the submission file\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Submission file created: submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}