{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"import os\nimport pandas as pd\n\n# Specify the path to the folder containing segmentation images\nfolder_path = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations'\n\n# Path to the CSV file\ncsv_file_path = '/kaggle/working/updated_csv'\n\n# Read the CSV file into a DataFrame\ndata = pd.read_csv(csv_file_path)\n\n# Get a list of all files in the folder\nfile_names = os.listdir(folder_path)\nprint(data.head())\n\n# Loop through the file names\nfor filename in file_names:\n # Remove the file extension to get the patient_id\n patient_id = os.path.splitext(filename)[0]\n \n # Check if the patient_id is present in the CSV\n if any(data['sub_patient_id'] == patient_id):\n # Update the 'mask_path' column with the new file path\n mask_path = os.path.join(folder_path, filename)\n data.loc[data['sub_patient_id'] == patient_id, 'mask_path'] = mask_path\n print(f\"Updated 'mask_path' for sub_patient_id: {patient_id}\")\n\n# Save the updated DataFrame back to the CSV file\ndata.to_csv(csv_file_path, index=False)\nprint(\"Done\")\nprint(data.head())\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import shutil\n\n# Specify the source path of the CSV file\nsource_path = '/kaggle/input/sample/updated_csv_file (12).csv'\n\n# Specify the destination path in the Kaggle working directory\ndestination_path = '/kaggle/working/updation.csv'\n\n# Copy the CSV file\nshutil.copy(source_path, destination_path)\n\nprint(f\"CSV file copied to: {destination_path}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T14:43:08.864755Z","iopub.execute_input":"2023-09-27T14:43:08.865179Z","iopub.status.idle":"2023-09-27T14:43:08.886177Z","shell.execute_reply.started":"2023-09-27T14:43:08.865141Z","shell.execute_reply":"2023-09-27T14:43:08.885215Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"CSV file copied to: /kaggle/working/updation.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# Specify the path to the folder containing segmentation images\nfolder_path = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations'\n\n# Path to the CSV file\ncsv_file_path = '/kaggle/working/updation.csv'\n\n# Read the CSV file into a DataFrame\ndata = pd.read_csv(csv_file_path)\n\n# Get a list of all files in the folder\nfile_names = os.listdir(folder_path)\nprint(data.head())\n\n# Loop through the file names\nfor filename in file_names:\n # Remove the file extension to get the sub_patient_id\n sub_patient_id = os.path.splitext(filename)[0]\n \n # Check if the sub_patient_id is present in the CSV\n if int(sub_patient_id) in data['sub_patient_id'].values.tolist():\n # Update the 'mask_path' column with the new file path\n mask_path = os.path.join(folder_path, filename)\n data.loc[data['sub_patient_id'] == sub_patient_id, 'mask_path'] = mask_path\n print(f\"Updated 'mask_path' for sub_patient_id: {sub_patient_id}\")\n\n# Save the updated DataFrame back to the CSV file\ndata.to_csv(csv_file_path, index=False)\nprint(\"Done\")\nprint(data.head())\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support\nfrom sklearn.model_selection import train_test_split\nimport torchvision.models as models\n\n# Define a custom dataset class\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Load the segmentation mask if available\n segmentation_mask = None\n segmentation_mask_affine = None # Store the affine information\n if pd.notna(mask_path):\n segmentation_mask_data = nib.load(mask_path).get_fdata()\n segmentation_mask_affine = nib.load(mask_path).affine # Store the affine information\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask = nib.Nifti1Image(resized_data, affine=segmentation_mask_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n # Ensure that the image has a single channel (1)\n image = image[None, ...]\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n\n return image, segmentation_mask, label\n\n\n# Function to preprocess the 3D image\ndef preprocess_image(image_path):\n # Load your NIfTI image using nibabel or any other library\n # Apply necessary preprocessing such as resizing, normalization, etc.\n # Return the processed image as a torch tensor\n return processed_image\n\n# Function to preprocess the segmentation mask\ndef preprocess_segmentation_mask(mask_path):\n # Load your segmentation mask using nibabel or any other library\n # Apply necessary preprocessing such as resizing, normalization, etc.\n # Return the processed mask as a torch tensor\n return processed_mask\n\n# Define a custom 3D CNN model using a pre-trained ResNet as the backbone\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import models\n\nclass EnhancedMultiLabel3DAttentionModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation):\n super(EnhancedMultiLabel3DAttentionModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n \n # Modify the stem to accept the correct input channels (128)\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Attention block\n self.attention = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, 1, kernel_size=1),\n nn.Sigmoid()\n )\n \n # Additional convolutional layers for feature refinement\n self.conv1 = nn.Conv3d(64, 128, kernel_size=3, padding=1)\n self.conv2 = nn.Conv3d(128, 256, kernel_size=3, padding=1)\n \n # Classification head\n self.classification_head = nn.Sequential(\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n nn.Linear(256, 128),\n nn.ReLU(inplace=True),\n nn.Linear(128, num_classes),\n nn.Sigmoid()\n )\n \n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(64, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n \n def forward(self, x, segmentation_mask):\n features = self.backbone(x)\n \n # Apply additional convolutional layers for feature refinement\n features = F.relu(self.conv1(features))\n features = F.relu(self.conv2(features))\n \n # Apply attention to features\n features = features.view(features.size(0), 1, 1, 1, features.size(1))\n attention_weights = self.attention(features)\n attended_features = features * attention_weights\n \n # Classification branch\n classification_output = self.classification_head(attended_features)\n \n # Modify the segmentation head to handle the different number of channels\n segmentation_output = self.segmentation_head(features)\n \n # Interpolate segmentation output to match segmentation mask's shape\n segmentation_output = F.interpolate(segmentation_output, size=segmentation_mask.shape[2:], mode='trilinear')\n \n # Element-wise multiplication with segmentation mask\n segmentation_output = segmentation_output * segmentation_mask\n \n return classification_output, segmentation_output\n\n\n# Paths and settings (update with your paths and settings)\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations'\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv'\nbatch_size = 16\nnum_workers = 4\nnum_classes_classification = 14\nnum_classes_segmentation = 1\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(),\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets and data loaders\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Instantiate the model\nmodel = EnhancedMultiLabel3DAttentionModel(num_classes_classification, num_classes_segmentation)\nmodel.to(device)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop, validation loop, and test loop (similar to your previous code)\n# ...\n# Training loop\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n\n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n\n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n # Convert it to shape (batch_size, 1, height, width, depth) by taking the first channel\n batch_images = batch_images[:, :, 0:1, :, :]\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n\n # Rest of your training code...\n\n # Add the code to define and return processed_image here if needed\n\n\n\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(2)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n \n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n \n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n \n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n \n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n \n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n \n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T17:47:12.115483Z","iopub.execute_input":"2023-09-28T17:47:12.115903Z","iopub.status.idle":"2023-09-28T17:47:24.53758Z","shell.execute_reply.started":"2023-09-28T17:47:12.115863Z","shell.execute_reply":"2023-09-28T17:47:24.536186Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=R3D_18_Weights.KINETICS400_V1`. You can also use `weights=R3D_18_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[15], line 224\u001b[0m\n\u001b[1;32m 221\u001b[0m batch_images \u001b[38;5;241m=\u001b[39m batch_images[:, :, \u001b[38;5;241m0\u001b[39m:\u001b[38;5;241m1\u001b[39m, :, :]\n\u001b[1;32m 223\u001b[0m \u001b[38;5;66;03m# Forward pass\u001b[39;00m\n\u001b[0;32m--> 224\u001b[0m classification_outputs, segmentation_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_images\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_segmentation_masks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;66;03m# Rest of your training code...\u001b[39;00m\n\u001b[1;32m 227\u001b[0m \n\u001b[1;32m 228\u001b[0m \u001b[38;5;66;03m# Add the code to define and return processed_image here if needed\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 232\u001b[0m \n\u001b[1;32m 233\u001b[0m \u001b[38;5;66;03m# Apply sigmoid activation to the classification outputs\u001b[39;00m\n\u001b[1;32m 234\u001b[0m classification_outputs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msigmoid(classification_outputs)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[15], line 124\u001b[0m, in \u001b[0;36mEnhancedMultiLabel3DAttentionModel.forward\u001b[0;34m(self, x, segmentation_mask)\u001b[0m\n\u001b[1;32m 121\u001b[0m features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbackbone(x)\n\u001b[1;32m 123\u001b[0m \u001b[38;5;66;03m# Apply additional convolutional layers for feature refinement\u001b[39;00m\n\u001b[0;32m--> 124\u001b[0m features \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 125\u001b[0m features \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv2(features))\n\u001b[1;32m 127\u001b[0m \u001b[38;5;66;03m# Apply attention to features\u001b[39;00m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:613\u001b[0m, in \u001b[0;36mConv3d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 612\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 613\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:608\u001b[0m, in \u001b[0;36mConv3d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 597\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv3d(\n\u001b[1;32m 598\u001b[0m F\u001b[38;5;241m.\u001b[39mpad(\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 606\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups,\n\u001b[1;32m 607\u001b[0m )\n\u001b[0;32m--> 608\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv3d\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 609\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\n\u001b[1;32m 610\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","\u001b[0;31mRuntimeError\u001b[0m: Expected 4D (unbatched) or 5D (batched) input to conv3d, but got input of size: [16, 400]"],"ename":"RuntimeError","evalue":"Expected 4D (unbatched) or 5D (batched) input to conv3d, but got input of size: [16, 400]","output_type":"error"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\nclass MultiLabel3DAttentionModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation):\n super(MultiLabel3DAttentionModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n \n \n # Modify the stem to accept the correct input channels (128)\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Attention block\n self.attention = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, 1, kernel_size=1),\n nn.Sigmoid()\n )\n \n # Classification head\n self.classification_head = nn.Sequential(\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n nn.Linear(1, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes),\n nn.Sigmoid()\n )\n \n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n \n def forward(self, x, segmentation_mask):\n #print( '...............', x.dtype, segmentation_mask.dtype)\n print(\"X shape\", x.shape)\n \n\n features = self.backbone(x)\n print(\"features shape\", features.shape)\n # Apply attention to features\n # Assuming features has shape [batch_size, num_features]\n # Reshape features to [batch_size, 1, 1, 1, num_features]\n features = features.view(features.size(0), 1, 1, 1, features.size(1))\n \n attention_weights = self.attention(features)\n \n print('attention_weights shape', attention_weights.shape)\n attended_features = features * attention_weights\n print('attended_features', attended_features.shape)\n \n # Classification branch\n classification_output = self.classification_head(attended_features)\n \n # Reshape attended_features to match segmentation_mask's shape along dimensions 2, 3, and 4\n #attended_features = attended_features.expand(-1, -1, 128, 128, 128)\n\n print('segmentation_mask', segmentation_mask.shape)\n \n # Segmentation branch\n # Reshape attended_features to match segmentation_mask's shape along dimensions 2, 3, and 4\n #attended_features = attended_features.expand(-1, -1, segmentation_mask.size(2), segmentation_mask.size(3), segmentation_mask.size(4))\n \n #segmentation_output = self.segmentation_head(attended_features) * segmentation_mask\n \n # Modify the segmentation head to handle the different number of channels\n segmentation_output = self.segmentation_head(attended_features)\n # You can use interpolation or other methods to match the shape.\n segmentation_output = F.interpolate(segmentation_output, size=segmentation_mask.shape[2:], mode='trilinear')\n \n print('segmentation_output', segmentation_output.shape)\n \n segmentation_output = segmentation_output * segmentation_mask # Element-wise multiplication\n \n \n return classification_output, segmentation_output\n\n# Paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n#data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n# print(data)\n# print(data.index)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n# print(\"Indices:\", train_loader.index) # Print the indices\n \n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = MultiLabel3DAttentionModel(num_classes_classification, num_classes_segmentation)\n\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# def print_parameter_data_types(module):\n# for name, param in module.named_parameters():\n# print(f\"Parameter: {name}, Data Type: {param.dtype}\")\n\n# # Call the function to print data types of parameters in the backbone module\n# print_parameter_data_types(model.backbone)\n\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(2) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(2)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n \n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n \n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(2)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n \n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n \n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n \n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n \n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T14:54:03.793787Z","iopub.execute_input":"2023-09-27T14:54:03.794216Z","iopub.status.idle":"2023-09-27T15:01:02.803534Z","shell.execute_reply.started":"2023-09-27T14:54:03.794183Z","shell.execute_reply":"2023-09-27T15:01:02.80201Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Length of DataFrame: 4711\nlen of train_dataset 3297\ntrain_loader 206\n\n","output_type":"stream"},{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=R3D_18_Weights.KINETICS400_V1`. You can also use `weights=R3D_18_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\nDownloading: \"https://download.pytorch.org/models/r3d_18-b3b3357e.pth\" to /root/.cache/torch/hub/checkpoints/r3d_18-b3b3357e.pth\n100%|██████████| 127M/127M [00:00<00:00, 228MB/s] \n","output_type":"stream"},{"name":"stdout","text":"X shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[6], line 319\u001b[0m\n\u001b[1;32m 316\u001b[0m total_train \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 318\u001b[0m \u001b[38;5;66;03m# Backpropagation and optimization\u001b[39;00m\n\u001b[0;32m--> 319\u001b[0m \u001b[43mtotal_loss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 320\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 322\u001b[0m \u001b[38;5;66;03m# Calculate and print average training accuracy and loss\u001b[39;00m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 479\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 480\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 485\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 486\u001b[0m )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 488\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 195\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m 198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-27T14:52:32.58577Z","iopub.execute_input":"2023-09-27T14:52:32.586122Z","iopub.status.idle":"2023-09-27T14:52:39.142383Z","shell.execute_reply.started":"2023-09-27T14:52:32.586093Z","shell.execute_reply":"2023-09-27T14:52:39.141123Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Collecting nibabel\n Downloading nibabel-5.1.0-py3-none-any.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\nInstalling collected packages: nibabel\nSuccessfully installed nibabel-5.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\n# Define paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n# Custom Dataset class\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\ndata = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Define the model class with attention mechanism\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass MultiLabel3DAttentionModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation):\n super(MultiLabel3DAttentionModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n\n # Modify the stem to accept the correct input channels (128)\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Attention block\n self.attention = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, 1, kernel_size=1),\n nn.Sigmoid())\n\n # RNN block\n self.rnn = nn.LSTM(input_size=64, hidden_size=128, num_layers=2, batch_first=True)\n\n # Classification head\n self.classification_head = nn.Sequential(\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n nn.Linear(128, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes),\n nn.Sigmoid())\n\n def forward(self, x,segmentation_mask):\n #print( '...............', x.dtype, segmentation_mask.dtype)\n print(\"X shape\", x.shape)\n\n features = self.backbone(x)\n print(\"features shape\", features.shape)\n\n # Apply attention to features\n # Assuming features has shape [batch_size, num_features]\n # Reshape features to [batch_size, 1, 1, 1, num_features]\n features = features.view(features.size(0), 1, 1, 1, features.size(1))\n\n attention_weights = self.attention(features)\n\n print('attention_weights shape', attention_weights.shape)\n attended_features = features * attention_weights\n print('attended_features', attended_features.shape)\n\n # Apply RNN to attended features\n # Assuming attended_features has shape [batch_size, 1, 1, 1, num_features]\n # Reshape attended_features to [batch_size, num_features]\n attended_features = attended_features.view(attended_features.size(0), attended_features.size(4))\n\n rnn_output, _ = self.rnn(attended_features)\n\n # Classification branch\n classification_output = self.classification_head(rnn_output)\n\n # Segmentation branch\n # segmentation_output = self.segmentation_head(rnn_output)\n\n # Reshape the segmentation output to match the input shape\n # segmentation_output = segmentation_output.view(x.shape)\n\n return classification_output, segmentation_output\n\n\n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = MultiLabel3DAttentionModel(num_classes_classification, num_classes_segmentation)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 10\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(1) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n# Save the trained model\ntorch.save(model.state_dict(), 'trained_model.pth')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T11:26:45.709325Z","iopub.execute_input":"2023-09-28T11:26:45.709773Z","iopub.status.idle":"2023-09-28T11:29:36.36903Z","shell.execute_reply.started":"2023-09-28T11:26:45.709728Z","shell.execute_reply":"2023-09-28T11:29:36.367686Z"},"trusted":true},"execution_count":23,"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=R3D_18_Weights.KINETICS400_V1`. You can also use `weights=R3D_18_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\n","output_type":"stream"},{"name":"stdout","text":"X shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[23], line 217\u001b[0m\n\u001b[1;32m 214\u001b[0m batch_segmentation_masks \u001b[38;5;241m=\u001b[39m batch_segmentation_masks\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 216\u001b[0m \u001b[38;5;66;03m# Forward pass\u001b[39;00m\n\u001b[0;32m--> 217\u001b[0m classification_outputs, segmentation_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_images\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_segmentation_masks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 219\u001b[0m \u001b[38;5;66;03m# Apply sigmoid activation to the classification outputs\u001b[39;00m\n\u001b[1;32m 220\u001b[0m classification_outputs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msigmoid(classification_outputs)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[23], line 172\u001b[0m, in \u001b[0;36mMultiLabel3DAttentionModel.forward\u001b[0;34m(self, x, segmentation_mask)\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[38;5;66;03m# Apply RNN to attended features\u001b[39;00m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;66;03m# Assuming attended_features has shape [batch_size, 1, 1, 1, num_features]\u001b[39;00m\n\u001b[1;32m 169\u001b[0m \u001b[38;5;66;03m# Reshape attended_features to [batch_size, num_features]\u001b[39;00m\n\u001b[1;32m 170\u001b[0m attended_features \u001b[38;5;241m=\u001b[39m attended_features\u001b[38;5;241m.\u001b[39mview(attended_features\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m0\u001b[39m), attended_features\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m4\u001b[39m))\n\u001b[0;32m--> 172\u001b[0m rnn_output, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrnn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mattended_features\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 174\u001b[0m \u001b[38;5;66;03m# Classification branch\u001b[39;00m\n\u001b[1;32m 175\u001b[0m classification_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclassification_head(rnn_output)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/rnn.py:810\u001b[0m, in \u001b[0;36mLSTM.forward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m 806\u001b[0m \u001b[38;5;66;03m# Each batch of the hidden state should match the input sequence that\u001b[39;00m\n\u001b[1;32m 807\u001b[0m \u001b[38;5;66;03m# the user believes he/she is passing in.\u001b[39;00m\n\u001b[1;32m 808\u001b[0m hx \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpermute_hidden(hx, sorted_indices)\n\u001b[0;32m--> 810\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_forward_args\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_sizes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 811\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m batch_sizes \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 812\u001b[0m result \u001b[38;5;241m=\u001b[39m _VF\u001b[38;5;241m.\u001b[39mlstm(\u001b[38;5;28minput\u001b[39m, hx, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flat_weights, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_layers,\n\u001b[1;32m 813\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbidirectional, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_first)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/rnn.py:730\u001b[0m, in \u001b[0;36mLSTM.check_forward_args\u001b[0;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m 725\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_forward_args\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;66;03m# type: ignore[override]\u001b[39;00m\n\u001b[1;32m 726\u001b[0m \u001b[38;5;28minput\u001b[39m: Tensor,\n\u001b[1;32m 727\u001b[0m hidden: Tuple[Tensor, Tensor],\n\u001b[1;32m 728\u001b[0m batch_sizes: Optional[Tensor],\n\u001b[1;32m 729\u001b[0m ):\n\u001b[0;32m--> 730\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_input\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_sizes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 731\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcheck_hidden_size(hidden[\u001b[38;5;241m0\u001b[39m], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_expected_hidden_size(\u001b[38;5;28minput\u001b[39m, batch_sizes),\n\u001b[1;32m 732\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mExpected hidden[0] size \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, got \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 733\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcheck_hidden_size(hidden[\u001b[38;5;241m1\u001b[39m], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_expected_cell_size(\u001b[38;5;28minput\u001b[39m, batch_sizes),\n\u001b[1;32m 734\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mExpected hidden[1] size \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, got \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/rnn.py:218\u001b[0m, in \u001b[0;36mRNNBase.check_input\u001b[0;34m(self, input, batch_sizes)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 215\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput must have \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m dimensions, got \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 216\u001b[0m expected_input_dim, \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39mdim()))\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_size \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m--> 218\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 219\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput.size(-1) must be equal to input_size. Expected \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m, got \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_size, \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)))\n","\u001b[0;31mRuntimeError\u001b[0m: input.size(-1) must be equal to input_size. Expected 64, got 400"],"ename":"RuntimeError","evalue":"input.size(-1) must be equal to input_size. Expected 64, got 400","output_type":"error"}]},{"cell_type":"code","source":"\nimport nibabel as nib\n\n# Specify the path to your .nii file\nnii_file_path = \"/kaggle/input/abdominal-trauma-nii-dataset/output-1/10004_21057.nii\"\n\n# Load the .nii image\nnii_image = nib.load(nii_file_path)\n\n# Get the shape of the image data\nimage_shape = nii_image.get_fdata().shape\n\n# Print the shape\nprint(\"Image Shape:\", image_shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T15:09:26.442884Z","iopub.execute_input":"2023-09-27T15:09:26.443292Z","iopub.status.idle":"2023-09-27T15:09:26.456122Z","shell.execute_reply.started":"2023-09-27T15:09:26.443263Z","shell.execute_reply":"2023-09-27T15:09:26.455053Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stdout","text":"Image Shape: (128, 128, 128)\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv3D, MaxPooling3D, Flatten, Dense, Dropout\nfrom tensorflow.keras.optimizers import Adam\n\n# Step 1: Load and preprocess the data\n\n# Load the CSV file\ndf = pd.read_csv('/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv')\n\n# Define paths to segmentation images folder\nsegmentations_folder = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations'\n\n# Initialize empty lists for image and label data\nimages = []\nlabels = []\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n# Iterate through the CSV entries\nfor index, row in df.iterrows():\n nii_path = os.path.join(segmentations_folder, row['file_path'])\n labels_row = row[['bowel_healthy', 'bowel_injury', 'extravasation_healthy', 'extravasation_injury',\n 'kidney_healthy', 'kidney_low', 'kidney_high', 'liver_healthy', 'liver_low', 'liver_high',\n 'spleen_healthy', 'spleen_low', 'spleen_high', 'any_injury']].values\n\n # Load and preprocess .nii files\n nii_data = nib.load(nii_path).get_fdata()\n \n # Resize or preprocess the data if needed\n # For example, if you want to resize to (128, 128, 128):\n nii_data = resize_nifti(nii_data, target_shape=(128, 128, 128))\n \n images.append(nii_data)\n labels.append(labels_row)\n\n# Convert lists to NumPy arrays\nimages = np.array(images)\nlabels = np.array(labels)\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(images, labels, test_size=0.2, random_state=42)\n\n# Add a channel dimension to the input data (required for 3D CNN)\nX_train = X_train[..., np.newaxis]\nX_val = X_val[..., np.newaxis]\n\n# Step 2: Build the model\n\nmodel = Sequential()\nmodel.add(Conv3D(32, (3, 3, 3), activation='relu', input_shape=(128, 128, 128, 1)))\nmodel.add(MaxPooling3D((2, 2, 2)))\nmodel.add(Conv3D(64, (3, 3, 3), activation='relu'))\nmodel.add(MaxPooling3D((2, 2, 2)))\nmodel.add(Conv3D(128, (3, 3, 3), activation='relu'))\nmodel.add(MaxPooling3D((2, 2, 2)))\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(14, activation='sigmoid')) # 14 output nodes for 14 labels, sigmoid activation for multilabel classification\n\n# Step 3: Compile and train the model\n\nmodel.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy'])\n\nhistory = model.fit(X_train, y_train, batch_size=4, epochs=10, validation_data=(X_val, y_val))\n\n# Step 4: Test the model (you can use a separate test dataset or validation dataset for testing)\n\ntest_loss, test_accuracy = model.evaluate(X_test, y_test)\nprint(\"Test Loss:\", test_loss)\nprint(\"Test Accuracy:\", test_accuracy)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T15:31:44.664249Z","iopub.execute_input":"2023-09-27T15:31:44.664652Z","iopub.status.idle":"2023-09-27T15:43:31.547979Z","shell.execute_reply.started":"2023-09-27T15:31:44.664619Z","shell.execute_reply":"2023-09-27T15:43:31.546769Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stderr","text":"D0927 15:32:16.633262349 15 config.cc:119] gRPC EXPERIMENT tcp_frame_size_tuning OFF (default:OFF)\nD0927 15:32:16.633288434 15 config.cc:119] gRPC EXPERIMENT tcp_rcv_lowat OFF (default:OFF)\nD0927 15:32:16.633292846 15 config.cc:119] gRPC EXPERIMENT peer_state_based_framing OFF (default:OFF)\nD0927 15:32:16.633296236 15 config.cc:119] gRPC EXPERIMENT flow_control_fixes ON (default:ON)\nD0927 15:32:16.633299265 15 config.cc:119] gRPC EXPERIMENT memory_pressure_controller OFF (default:OFF)\nD0927 15:32:16.633302792 15 config.cc:119] gRPC EXPERIMENT unconstrained_max_quota_buffer_size OFF (default:OFF)\nD0927 15:32:16.633313885 15 config.cc:119] gRPC EXPERIMENT new_hpack_huffman_decoder ON (default:ON)\nD0927 15:32:16.633316943 15 config.cc:119] gRPC EXPERIMENT event_engine_client OFF (default:OFF)\nD0927 15:32:16.633320003 15 config.cc:119] gRPC EXPERIMENT monitoring_experiment ON (default:ON)\nD0927 15:32:16.633322986 15 config.cc:119] gRPC EXPERIMENT promise_based_client_call OFF (default:OFF)\nD0927 15:32:16.633325923 15 config.cc:119] gRPC EXPERIMENT free_large_allocator OFF (default:OFF)\nD0927 15:32:16.633329110 15 config.cc:119] gRPC EXPERIMENT promise_based_server_call OFF (default:OFF)\nD0927 15:32:16.633332203 15 config.cc:119] gRPC EXPERIMENT transport_supplies_client_latency OFF (default:OFF)\nD0927 15:32:16.633335170 15 config.cc:119] gRPC EXPERIMENT event_engine_listener OFF (default:OFF)\nI0927 15:32:16.633607110 15 ev_epoll1_linux.cc:122] grpc epoll fd: 67\nD0927 15:32:16.633634647 15 ev_posix.cc:144] Using polling engine: epoll1\nD0927 15:32:16.633732843 15 dns_resolver_ares.cc:822] Using ares dns resolver\nD0927 15:32:16.634355603 15 lb_policy_registry.cc:46] registering LB policy factory for \"priority_experimental\"\nD0927 15:32:16.634375847 15 lb_policy_registry.cc:46] registering LB policy factory for \"outlier_detection_experimental\"\nD0927 15:32:16.634383404 15 lb_policy_registry.cc:46] registering LB policy factory for \"weighted_target_experimental\"\nD0927 15:32:16.634387130 15 lb_policy_registry.cc:46] registering LB policy factory for \"pick_first\"\nD0927 15:32:16.634394506 15 lb_policy_registry.cc:46] registering LB policy factory for \"round_robin\"\nD0927 15:32:16.634397927 15 lb_policy_registry.cc:46] registering LB policy factory for \"weighted_round_robin_experimental\"\nD0927 15:32:16.634405831 15 lb_policy_registry.cc:46] registering LB policy factory for \"ring_hash_experimental\"\nD0927 15:32:16.634442414 15 lb_policy_registry.cc:46] registering LB policy factory for \"grpclb\"\nD0927 15:32:16.634533664 15 lb_policy_registry.cc:46] registering LB policy factory for \"rls_experimental\"\nD0927 15:32:16.634564272 15 lb_policy_registry.cc:46] registering LB policy factory for \"xds_cluster_manager_experimental\"\nD0927 15:32:16.634571896 15 lb_policy_registry.cc:46] registering LB policy factory for \"xds_cluster_impl_experimental\"\nD0927 15:32:16.634575517 15 lb_policy_registry.cc:46] registering LB policy factory for \"cds_experimental\"\nD0927 15:32:16.634586165 15 lb_policy_registry.cc:46] registering LB policy factory for \"xds_cluster_resolver_experimental\"\nD0927 15:32:16.634593332 15 lb_policy_registry.cc:46] registering LB policy factory for \"xds_override_host_experimental\"\nD0927 15:32:16.634609847 15 lb_policy_registry.cc:46] registering LB policy factory for \"xds_wrr_locality_experimental\"\nD0927 15:32:16.634613939 15 certificate_provider_registry.cc:35] registering certificate provider factory for \"file_watcher\"\nI0927 15:32:16.637347854 15 socket_utils_common_posix.cc:408] Disabling AF_INET6 sockets because ::1 is not available.\nI0927 15:32:16.653689987 2921 socket_utils_common_posix.cc:337] TCP_USER_TIMEOUT is available. TCP_USER_TIMEOUT will be used thereafter\nE0927 15:32:16.660814001 2921 oauth2_credentials.cc:236] oauth_fetch: UNKNOWN:C-ares status is not ARES_SUCCESS qtype=A name=metadata.google.internal. is_balancer=0: Domain name not found {created_time:\"2023-09-27T15:32:16.660787467+00:00\", grpc_status:2}\n\nKeyboardInterrupt\n\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport cv2\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import binary_crossentropy\nfrom tensorflow.keras.metrics import binary_accuracy\n\n# Load your CSV file\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv'\ndf = pd.read_csv(csv_file)\n\n# Define the path to the mask folder\n\n\n# Initialize empty lists for image and mask data\nimages = []\nmasks = []\n\n# Iterate through the CSV entries\nfor index, row in df.iterrows():\n image_path = row['file_path']\n mask_path = row['mask_path']\n \n # Check if the mask file exists\n if mask_path!=\"\":\n # Load and preprocess the image\n image = nib.load(image_path).get_fdata()\n image = image / np.max(image) # Normalize\n \n # Load the mask\n mask = nib.load(mask_path).get_fdata()\n mask = cv2.resize(mask, (128, 128, 128))\n mask = mask / np.max(mask) # Normalize\n \n images.append(image)\n masks.append(mask)\n\n# Convert lists to NumPy arrays\nimages = np.array(images)\nmasks = np.array(masks)\nprint(images)\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(images, masks, test_size=0.2, random_state=42)\n\n# Step 2: Build the model\n\nmodel = models.Sequential()\nmodel.add(layers.Conv3D(32, (3, 3, 3), activation='relu', input_shape=(128, 128, 128, 1)))\nmodel.add(layers.MaxPooling3D((2, 2, 2)))\nmodel.add(layers.Conv3D(64, (3, 3, 3), activation='relu'))\nmodel.add(layers.MaxPooling3D((2, 2, 2)))\nmodel.add(layers.Conv3D(128, (3, 3, 3), activation='relu'))\nmodel.add(layers.MaxPooling3D((2, 2, 2)))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid')) # Modify output layer for your specific task\n\n# Compile the model\nmodel.compile(optimizer=Adam(learning_rate=0.001), loss=binary_crossentropy, metrics=[binary_accuracy])\n\n# Step 3: Train the model\nfor epoch in range(10): # Modify the number of epochs as needed\n print(f\"Epoch {epoch + 1}/{10}\") # Replace 10 with the total number of epochs\n \n for batch_start in range(0, len(X_train), 4): # Batch size = 4, adjust as needed\n batch_end = min(batch_start + 4, len(X_train))\n X_batch = X_train[batch_start:batch_end]\n y_batch = y_train[batch_start:batch_end]\n \n # Training step\n loss = model.train_on_batch(X_batch[..., np.newaxis], y_batch)\n print(f\"Batch {batch_start // 4 + 1}/{len(X_train) // 4} - Loss: {loss[0]:.4f} - Binary Accuracy: {loss[1]:.4f}\")\n \n # Validation step (print validation metrics if needed)\n val_loss = model.evaluate(X_val[..., np.newaxis], y_val)\n print(f\"Validation Loss: {val_loss[0]:.4f} - Validation Binary Accuracy: {val_loss[1]:.4f}\")\n\n# Step 4: Evaluate the model and make predictions (not included here, as it depends on your specific task)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T15:46:40.687946Z","iopub.execute_input":"2023-09-27T15:46:40.688374Z","iopub.status.idle":"2023-09-27T15:47:08.808736Z","shell.execute_reply.started":"2023-09-27T15:46:40.688335Z","shell.execute_reply":"2023-09-27T15:47:08.807471Z"},"trusted":true},"execution_count":17,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31merror\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[17], line 37\u001b[0m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;66;03m# Load the mask\u001b[39;00m\n\u001b[1;32m 36\u001b[0m mask \u001b[38;5;241m=\u001b[39m nib\u001b[38;5;241m.\u001b[39mload(mask_path)\u001b[38;5;241m.\u001b[39mget_fdata()\n\u001b[0;32m---> 37\u001b[0m mask \u001b[38;5;241m=\u001b[39m \u001b[43mcv2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresize\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 38\u001b[0m mask \u001b[38;5;241m=\u001b[39m mask \u001b[38;5;241m/\u001b[39m np\u001b[38;5;241m.\u001b[39mmax(mask) \u001b[38;5;66;03m# Normalize\u001b[39;00m\n\u001b[1;32m 40\u001b[0m images\u001b[38;5;241m.\u001b[39mappend(image)\n","\u001b[0;31merror\u001b[0m: OpenCV(4.8.0) :-1: error: (-5:Bad argument) in function 'resize'\n> Overload resolution failed:\n> - Can't parse 'dsize'. Expected sequence length 2, got 3\n> - Can't parse 'dsize'. Expected sequence length 2, got 3\n"],"ename":"error","evalue":"OpenCV(4.8.0) :-1: error: (-5:Bad argument) in function 'resize'\n> Overload resolution failed:\n> - Can't parse 'dsize'. Expected sequence length 2, got 3\n> - Can't parse 'dsize'. Expected sequence length 2, got 3\n","output_type":"error"}]},{"cell_type":"code","source":"pip install torch\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:30:13.483044Z","iopub.execute_input":"2023-09-28T10:30:13.483563Z","iopub.status.idle":"2023-09-28T10:30:19.297016Z","shell.execute_reply.started":"2023-09-28T10:30:13.483531Z","shell.execute_reply":"2023-09-28T10:30:19.296028Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Requirement already satisfied: torch in /usr/local/lib/python3.8/site-packages (2.0.0)\nRequirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.8/site-packages (from torch) (11.7.99)\nRequirement already satisfied: networkx in /usr/local/lib/python3.8/site-packages (from torch) (3.1)\nRequirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /usr/local/lib/python3.8/site-packages (from torch) (11.7.4.91)\nRequirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /usr/local/lib/python3.8/site-packages (from torch) (10.9.0.58)\nRequirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /usr/local/lib/python3.8/site-packages (from torch) (11.7.91)\nRequirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.8/site-packages (from torch) (11.10.3.66)\nRequirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /usr/local/lib/python3.8/site-packages (from torch) (11.4.0.1)\nRequirement already satisfied: filelock in /usr/local/lib/python3.8/site-packages (from torch) (3.12.2)\nRequirement already satisfied: sympy in /usr/local/lib/python3.8/site-packages (from torch) (1.12)\nRequirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /usr/local/lib/python3.8/site-packages (from torch) (10.2.10.91)\nRequirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.8/site-packages (from torch) (2.0.0)\nRequirement already satisfied: nvidia-nccl-cu11==2.14.3 in /usr/local/lib/python3.8/site-packages (from torch) (2.14.3)\nRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/site-packages (from torch) (4.7.1)\nRequirement already satisfied: jinja2 in /usr/local/lib/python3.8/site-packages (from torch) (3.1.2)\nRequirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.8/site-packages (from torch) (11.7.99)\nRequirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.8/site-packages (from torch) (8.5.0.96)\nRequirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /usr/local/lib/python3.8/site-packages (from torch) (11.7.101)\nRequirement already satisfied: setuptools in /usr/local/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (57.5.0)\nRequirement already satisfied: wheel in /usr/local/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.40.0)\nRequirement already satisfied: lit in /usr/local/lib/python3.8/site-packages (from triton==2.0.0->torch) (16.0.6)\nRequirement already satisfied: cmake in /usr/local/lib/python3.8/site-packages (from triton==2.0.0->torch) (3.26.4)\nRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/site-packages (from jinja2->torch) (2.1.3)\nRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.8/site-packages (from sympy->torch) (1.3.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\nNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass RNNModel(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes):\n super(RNNModel, self).__init__()\n \n # Define the RNN layer\n self.rnn = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n \n # Define the fully connected layer for classification\n self.fc = nn.Linear(hidden_size, num_classes)\n \n def forward(self, x):\n # Forward pass through the RNN layer\n out, _ = self.rnn(x)\n \n # Get the output from the last time step\n out = out[:, -1, :]\n \n # Apply the fully connected layer for classification\n out = self.fc(out)\n \n return out\n\n# Define hyperparameters\ninput_size = 128 # You may need to adjust this based on your input data\nhidden_size = 256\nnum_layers = 2\nnum_classes = 14 # Number of classes in your multi-label classification problem\n\n# Create the RNN model\nmodel = RNNModel(input_size, hidden_size, num_layers, num_classes)\n\n# Print the model architecture\nprint(model)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\n# Function to resize NIfTI data\n\n# ... (Previous code for dataset loading and preprocessing)\n# Define paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n# Define an RNN model for your task\nclass RNNModel(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes):\n super(RNNModel, self).__init__()\n self.hidden_size = hidden_size\n self.num_layers = num_layers\n self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True)\n self.fc = nn.Linear(hidden_size, num_classes)\n \n def forward(self, x):\n # Initialize hidden state with zeros\n h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(x.device)\n \n # Forward propagate RNN\n out, _ = self.rnn(x, h0)\n \n # Decode the hidden state of the last time step\n out = self.fc(out[:, -1, :])\n return out\n\n# Define the RNN model\ninput_size = 128 * 128 * 128 # Assuming input images are of shape (128, 128, 128)\nhidden_size = 128\nnum_layers = 2 # You can adjust the number of layers\nnum_classes = 7 # Number of output classes\nrnn_model = RNNModel(input_size, hidden_size, num_layers, num_classes).to(device)\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(rnn_model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 10 # You can adjust the number of epochs\n\nfor epoch in range(num_epochs):\n rnn_model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, _, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Reshape 3D images into a sequence format for RNN\n batch_size, _, height, width, depth = batch_images.shape\n batch_images = batch_images.view(batch_size, -1, input_size) # Reshape\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_labels = batch_labels.to(device)\n\n # Forward pass\n outputs = rnn_model(batch_images)\n \n # Calculate loss\n loss = criterion(outputs, torch.argmax(batch_labels, dim=1))\n running_loss += loss.item()\n \n # Backpropagation and optimization\n loss.backward()\n optimizer.step()\n\n # Calculate accuracy\n _, predicted = torch.max(outputs.data, 1)\n total_train += batch_labels.size(0)\n correct_train += (predicted == torch.argmax(batch_labels, dim=1)).sum().item()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\n\n\nfrom torchvision import transforms\n\n# Define your dataset class\nfrom torchvision import transforms\n\n# Define your dataset class\nclass CustomDataset(Dataset):\n def __init__(self, csv_file, transform=None, target_size=(128, 128, 128)):\n self.data = pd.read_csv(csv_file)\n self.transform = transform\n self.target_size = target_size # Specify the target size for images\n\n def __len__(self):\n return len(self.data)\n\n def __getitem__(self, idx):\n image_path = self.data.loc[idx, 'file_path']\n label = self.data.loc[idx, ['bowel_healthy', 'bowel_injury', 'extravasation_healthy', 'extravasation_injury',\n 'kidney_healthy', 'kidney_low', 'kidney_high', 'liver_healthy', 'liver_low', 'liver_high',\n 'spleen_healthy', 'spleen_low', 'spleen_high', 'any_injury']].values.astype(np.float32) # Convert to numpy array\n image = nib.load(image_path).get_fdata()\n\n # Ensure the image is of the target size\n if image.shape != self.target_size:\n # Resize the image to the target size\n image = zoom(image, (self.target_size[0] / image.shape[0], self.target_size[1] / image.shape[1], self.target_size[2] / image.shape[2]))\n\n # Load and preprocess the mask if available\n mask_path = self.data.loc[idx, 'mask_path']\n if pd.notna(mask_path):\n mask = nib.load(mask_path).get_fdata()\n\n # Ensure the mask is of the target size\n if mask.shape != self.target_size:\n # Resize the mask to the target size\n mask = zoom(mask, (self.target_size[0] / mask.shape[0], self.target_size[1] / mask.shape[1], self.target_size[2] / mask.shape[2]))\n else:\n # If mask is not available, create an empty mask of the target size\n mask = np.zeros(self.target_size, dtype=np.float32)\n \n if self.transform:\n # Apply additional transformations if needed\n pass\n \n return image, mask, label\n\n\n \n\n \n\n\n# Define your neural network model\nclass MultiLabelClassifier(nn.Module):\n def __init__(self, input_channels, num_classes):\n super(MultiLabelClassifier, self).__init__()\n self.conv1 = nn.Conv3d(input_channels, 64, kernel_size=3, padding=1)\n self.pool = nn.MaxPool3d(kernel_size=2, stride=2)\n self.fc1 = nn.Linear(64 * 32 * 32 * 32, 512)\n self.fc2 = nn.Linear(512, num_classes)\n\n def forward(self, x):\n x = F.relu(self.conv1(x))\n x = self.pool(x)\n x = x.view(-1, 64 * 32 * 32 * 32)\n x = F.relu(self.fc1(x))\n x = self.fc2(x)\n return torch.sigmoid(x)\n\n\n# Paths and settings\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv'\nbatch_size = 16\nnum_workers = 4\nnum_classes = 14 # Adjust based on your labels\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(),\n # Add more transformations if necessary\n])\n\n# Load the dataset\ndataset = CustomDataset(csv_file, transform=transform)\n\n# Split the dataset into training and testing sets\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])\n\n# Create data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Initialize the model\nmodel = MultiLabelClassifier(input_channels=16, num_classes=num_classes).to(device)\nmodel = model.float() # Convert model weights to float32\n\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 10\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n\n for batch in train_loader:\n images, masks, labels = batch\n images = images.to(device)\n labels = labels.to(device)\n\n optimizer.zero_grad()\n\n outputs = model(images)\n\n loss = criterion(outputs, labels)\n loss.backward()\n optimizer.step()\n\n running_loss += loss.item()\n\n # Calculate and print average training loss\n avg_train_loss = running_loss / len(train_loader)\n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Loss: {avg_train_loss:.4f}\")\n\n# Testing loop\nmodel.eval()\ntest_labels = []\ntest_predictions = []\n\nwith torch.no_grad():\n for batch in test_loader:\n images, masks, labels = batch\n images = images.to(device)\n labels = labels.to(device)\n\n outputs = model(images)\n\n test_labels.append(labels.cpu().numpy())\n test_predictions.append(outputs.cpu().numpy())\n\n# Calculate evaluation metrics\ntest_labels = np.vstack(test_labels)\ntest_predictions = np.vstack(test_predictions)\n\naccuracy = accuracy_score(test_labels, (test_predictions > 0.5).astype(int))\nprecision, recall, f1, _ = precision_recall_fscore_support(test_labels, (test_predictions > 0.5).astype(int), average='micro')\nroc_auc = roc_auc_score(test_labels, test_predictions)\n\nprint(f\"Test Accuracy: {accuracy:.4f}\")\nprint(f\"Test Precision: {precision:.4f}\")\nprint(f\"Test Recall: {recall:.4f}\")\nprint(f\"Test F1 Score: {f1:.4f}\")\nprint(f\"Test ROC AUC: {roc_auc:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:58:10.014504Z","iopub.execute_input":"2023-09-28T10:58:10.014961Z","iopub.status.idle":"2023-09-28T10:58:39.931483Z","shell.execute_reply.started":"2023-09-28T10:58:10.014911Z","shell.execute_reply":"2023-09-28T10:58:39.93008Z"},"trusted":true},"execution_count":18,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[18], line 135\u001b[0m\n\u001b[1;32m 131\u001b[0m labels \u001b[38;5;241m=\u001b[39m labels\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 133\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m--> 135\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimages\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 137\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(outputs, labels)\n\u001b[1;32m 138\u001b[0m loss\u001b[38;5;241m.\u001b[39mbackward()\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[18], line 79\u001b[0m, in \u001b[0;36mMultiLabelClassifier.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m---> 79\u001b[0m x \u001b[38;5;241m=\u001b[39m F\u001b[38;5;241m.\u001b[39mrelu(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 80\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool(x)\n\u001b[1;32m 81\u001b[0m x \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m64\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m32\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m32\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m32\u001b[39m)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:613\u001b[0m, in \u001b[0;36mConv3d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 612\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 613\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:608\u001b[0m, in \u001b[0;36mConv3d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 597\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv3d(\n\u001b[1;32m 598\u001b[0m F\u001b[38;5;241m.\u001b[39mpad(\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 606\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups,\n\u001b[1;32m 607\u001b[0m )\n\u001b[0;32m--> 608\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv3d\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 609\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\n\u001b[1;32m 610\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","\u001b[0;31mRuntimeError\u001b[0m: expected scalar type Double but found Float"],"ename":"RuntimeError","evalue":"expected scalar type Double but found Float","output_type":"error"}]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-28T10:30:34.006679Z","iopub.execute_input":"2023-09-28T10:30:34.007028Z","iopub.status.idle":"2023-09-28T10:30:39.926514Z","shell.execute_reply.started":"2023-09-28T10:30:34.006999Z","shell.execute_reply":"2023-09-28T10:30:39.925552Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Collecting nibabel\n Downloading nibabel-5.1.0-py3-none-any.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\nInstalling collected packages: nibabel\nSuccessfully installed nibabel-5.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nimport nibabel as nib\n\n# Define paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n# Custom Dataset class\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path).get_fdata()\n\n # Resize the segmentation mask to match the shape of the input image\n segmentation_mask = resize_nifti(segmentation_mask, image.shape)\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\ndata = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Define the new model class using RNNs and LSTMs\nclass CustomRNNModel(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes, sequence_length):\n super(CustomRNNModel, self).__init__()\n self.sequence_length = sequence_length\n self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n self.fc = nn.Linear(hidden_size, num_classes)\n\n def forward(self, x, mask):\n # Forward pass through the LSTM layer\n out, _ = self.lstm(x)\n\n # Take the output from the last time step\n out = self.fc(out[:, -1, :])\n\n # Resize mask to match the size of the input images\n mask = F.interpolate(mask.unsqueeze(1), size=(128, 128, 128), mode='trilinear')\n mask = mask.squeeze(1) # Remove the channel dimension\n\n return out, mask\n\n# Instantiate the new model\ninput_size = 128 # Replace with the appropriate input size\nhidden_size = 64\nnum_layers = 2\nnum_classes = 14 # Number of classes\nsequence_length = 128 # Length of the sequences\n\nrnn_model = CustomRNNModel(input_size, hidden_size, num_layers, num_classes, sequence_length)\n# Move the model to the GPU if available\nrnn_model.to(device)\n\n# Define loss function and optimizer\ncriterion = nn.BCEWithLogitsLoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(rnn_model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 1\n\nfor epoch in range(num_epochs):\n rnn_model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, _, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n\n # Create an initial mask (you may need to adjust this based on your requirements)\n initial_mask = torch.zeros(batch_images.size(0), 128, 128, 128).to(device)\n\n # Forward pass\n outputs, mask = rnn_model(batch_images, initial_mask)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_loss = criterion(outputs[:, class_index], class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_logits = outputs[:, class_index] # Select model logits for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (torch.sigmoid(class_logits) > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T13:20:29.91614Z","iopub.execute_input":"2023-09-28T13:20:29.916632Z","iopub.status.idle":"2023-09-28T13:22:15.190219Z","shell.execute_reply.started":"2023-09-28T13:20:29.91658Z","shell.execute_reply":"2023-09-28T13:22:15.189063Z"},"trusted":true},"execution_count":6,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[6], line 168\u001b[0m\n\u001b[1;32m 165\u001b[0m initial_mask \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mzeros(batch_images\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;241m128\u001b[39m, \u001b[38;5;241m128\u001b[39m, \u001b[38;5;241m128\u001b[39m)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m 167\u001b[0m \u001b[38;5;66;03m# Forward pass\u001b[39;00m\n\u001b[0;32m--> 168\u001b[0m outputs, mask \u001b[38;5;241m=\u001b[39m \u001b[43mrnn_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_images\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitial_mask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 170\u001b[0m \u001b[38;5;66;03m# Calculate binary cross-entropy loss for each class separately\u001b[39;00m\n\u001b[1;32m 171\u001b[0m losses \u001b[38;5;241m=\u001b[39m []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[6], line 123\u001b[0m, in \u001b[0;36mCustomRNNModel.forward\u001b[0;34m(self, x, mask)\u001b[0m\n\u001b[1;32m 121\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x, mask):\n\u001b[1;32m 122\u001b[0m \u001b[38;5;66;03m# Forward pass through the LSTM layer\u001b[39;00m\n\u001b[0;32m--> 123\u001b[0m out, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlstm\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 125\u001b[0m \u001b[38;5;66;03m# Take the output from the last time step\u001b[39;00m\n\u001b[1;32m 126\u001b[0m out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc(out[:, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, :])\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/rnn.py:773\u001b[0m, in \u001b[0;36mLSTM.forward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 772\u001b[0m batch_sizes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 773\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m (\u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39mdim() \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m3\u001b[39m)), \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLSTM: Expected input to be 2-D or 3-D but received \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39mdim()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m-D tensor\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 774\u001b[0m is_batched \u001b[38;5;241m=\u001b[39m \u001b[38;5;28minput\u001b[39m\u001b[38;5;241m.\u001b[39mdim() \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m3\u001b[39m\n\u001b[1;32m 775\u001b[0m batch_dim \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbatch_first \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;241m1\u001b[39m\n","\u001b[0;31mAssertionError\u001b[0m: LSTM: Expected input to be 2-D or 3-D but received 4-D tensor"],"ename":"AssertionError","evalue":"LSTM: Expected input to be 2-D or 3-D but received 4-D tensor","output_type":"error"}]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-28T12:54:23.110638Z","iopub.execute_input":"2023-09-28T12:54:23.111028Z","iopub.status.idle":"2023-09-28T12:54:29.844557Z","shell.execute_reply.started":"2023-09-28T12:54:23.110998Z","shell.execute_reply":"2023-09-28T12:54:29.843348Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Collecting nibabel\n Downloading nibabel-5.1.0-py3-none-any.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m29.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\nInstalling collected packages: nibabel\nSuccessfully installed nibabel-5.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\n# Define the CustomDataset class\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Load the segmentation mask if available\n segmentation_mask = None\n segmentation_mask_affine = None # Store the affine information\n if pd.notna(mask_path):\n segmentation_mask_data = nib.load(mask_path).get_fdata()\n segmentation_mask_affine = nib.load(mask_path).affine # Store the affine information\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask = nib.Nifti1Image(resized_data, affine=segmentation_mask_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n\n return image, segmentation_mask, label\n\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n# Define the VGG3DModel class\nclass VGG3DModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation):\n super(VGG3DModel, self).__init__()\n\n # Load a pre-trained VGG16 backbone\n self.backbone = models.vgg16(pretrained=True)\n\n # Modify the classifier head to accept the correct input channels (1)\n self.backbone.features[0] = nn.Conv3d(1, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1))\n\n # Remove the last fully connected layer (classifier)\n self.backbone.classifier = nn.Sequential(\n nn.Linear(512, 256),\n nn.ReLU(inplace=True),\n nn.Dropout(0.5),\n nn.Linear(256, num_classes),\n nn.Sigmoid()\n )\n\n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(1, 1, 1)),\n nn.ReLU(inplace=True),\n nn.Conv3d(64, num_classes_segmentation, kernel_size=(1, 1, 1)),\n nn.Sigmoid()\n )\n\n def forward(self, x, segmentation_mask):\n # Remove the extra batch dimension\n x = x.squeeze(1)\n\n # Forward pass through the VGG backbone\n features = self.backbone.features(x)\n\n # Classification branch\n classification_output = self.backbone.classifier(features.view(features.size(0), -1))\n\n # Reshape segmentation mask to match the feature dimensions\n segmentation_mask = F.interpolate(segmentation_mask, size=features.size()[2:], mode='trilinear', align_corners=False)\n\n # Segmentation branch\n segmentation_output = self.segmentation_head(segmentation_mask)\n\n return classification_output, segmentation_output\n\n \n\n# Paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n# data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n\n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = VGG3DModel(num_classes=14, num_classes_segmentation=1)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n\n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n\n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(2) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n\n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n\n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n\n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n\n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n\n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n\n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n batch_images = batch_images.unsqueeze(2)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n\n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n\n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n\n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n\n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n batch_images = batch_images.unsqueeze(2)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n\n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n\n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n\n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n\n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n\n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-28T17:48:33.281922Z","iopub.execute_input":"2023-09-28T17:48:33.282397Z","iopub.status.idle":"2023-09-28T17:48:44.489836Z","shell.execute_reply.started":"2023-09-28T17:48:33.282359Z","shell.execute_reply":"2023-09-28T17:48:44.488456Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"Length of DataFrame: 4711\nlen of train_dataset 3297\ntrain_loader 206\n\n","output_type":"stream"},{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[16], line 216\u001b[0m\n\u001b[1;32m 213\u001b[0m batch_segmentation_masks \u001b[38;5;241m=\u001b[39m batch_segmentation_masks\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 215\u001b[0m \u001b[38;5;66;03m# Forward pass\u001b[39;00m\n\u001b[0;32m--> 216\u001b[0m classification_outputs, segmentation_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_images\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_segmentation_masks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;66;03m# Apply sigmoid activation to the classification outputs\u001b[39;00m\n\u001b[1;32m 219\u001b[0m classification_outputs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msigmoid(classification_outputs)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[16], line 99\u001b[0m, in \u001b[0;36mVGG3DModel.forward\u001b[0;34m(self, x, segmentation_mask)\u001b[0m\n\u001b[1;32m 96\u001b[0m x \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39msqueeze(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 98\u001b[0m \u001b[38;5;66;03m# Forward pass through the VGG backbone\u001b[39;00m\n\u001b[0;32m---> 99\u001b[0m features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackbone\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeatures\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# Classification branch\u001b[39;00m\n\u001b[1;32m 102\u001b[0m classification_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbackbone\u001b[38;5;241m.\u001b[39mclassifier(features\u001b[38;5;241m.\u001b[39mview(features\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m0\u001b[39m), \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/container.py:217\u001b[0m, in \u001b[0;36mSequential.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[0;32m--> 217\u001b[0m \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:613\u001b[0m, in \u001b[0;36mConv3d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 612\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 613\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_conv_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py:608\u001b[0m, in \u001b[0;36mConv3d._conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 597\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv3d(\n\u001b[1;32m 598\u001b[0m F\u001b[38;5;241m.\u001b[39mpad(\n\u001b[1;32m 599\u001b[0m \u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 606\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups,\n\u001b[1;32m 607\u001b[0m )\n\u001b[0;32m--> 608\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv3d\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 609\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstride\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdilation\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroups\u001b[49m\n\u001b[1;32m 610\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","\u001b[0;31mRuntimeError\u001b[0m: Given groups=1, weight of size [64, 1, 3, 3, 3], expected input[16, 128, 1, 128, 128] to have 1 channels, but got 128 channels instead"],"ename":"RuntimeError","evalue":"Given groups=1, weight of size [64, 1, 3, 3, 3], expected input[16, 128, 1, 128, 128] to have 1 channels, but got 128 channels instead","output_type":"error"}]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-28T17:04:23.949139Z","iopub.execute_input":"2023-09-28T17:04:23.949546Z","iopub.status.idle":"2023-09-28T17:04:30.825602Z","shell.execute_reply.started":"2023-09-28T17:04:23.949513Z","shell.execute_reply":"2023-09-28T17:04:30.824575Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Collecting nibabel\n Downloading nibabel-5.1.0-py3-none-any.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m21.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\nInstalling collected packages: nibabel\nSuccessfully installed nibabel-5.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass CustomCNNModel(nn.Module):\n def __init__(self, num_classes):\n super(CustomCNNModel, self).__init__()\n self.conv1 = nn.Conv3d(1, 32, kernel_size=3, padding=1)\n self.relu1 = nn.ReLU()\n self.pool1 = nn.MaxPool3d(kernel_size=2, stride=2)\n \n self.conv2 = nn.Conv3d(32, 64, kernel_size=3, padding=1)\n self.relu2 = nn.ReLU()\n self.pool2 = nn.MaxPool3d(kernel_size=2, stride=2)\n \n self.conv3 = nn.Conv3d(64, 128, kernel_size=3, padding=1)\n self.relu3 = nn.ReLU()\n self.pool3 = nn.MaxPool3d(kernel_size=2, stride=2)\n \n self.fc1 = nn.Linear(128 * 16 * 16 * 16, 256)\n self.relu4 = nn.ReLU()\n self.fc2 = nn.Linear(256, num_classes)\n self.sigmoid = nn.Sigmoid()\n \n def forward(self, x):\n x = self.pool1(self.relu1(self.conv1(x)))\n x = self.pool2(self.relu2(self.conv2(x)))\n x = self.pool3(self.relu3(self.conv3(x)))\n x = x.view(x.size(0), -1)\n x = self.relu4(self.fc1(x))\n x = self.fc2(x)\n x = self.sigmoid(x)\n return x\n\n# Instantiate the model\nnum_classes_classification = 14 # Number of classes for classification\nmodel = CustomCNNModel(num_classes_classification)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop, validation loop, and test loop (similar to the code you provided) should remain the same with this new model.\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n\nimport torch\nimport torch.nn as nn\n\nclass CustomCNNModel(nn.Module):\n def __init__(self, num_classes_classification, num_classes_segmentation):\n super(CustomCNNModel, self).__init__()\n self.features = nn.Sequential(\n nn.Conv3d(1, 32, kernel_size=3, padding=1),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=2, stride=2),\n nn.Conv3d(32, 64, kernel_size=3, padding=1),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=2, stride=2),\n nn.Conv3d(64, 128, kernel_size=3, padding=1),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=2, stride=2)\n )\n \n self.classification_head = nn.Sequential(\n nn.Linear(128 * 16 * 16 * 16, 256),\n nn.ReLU(),\n nn.Linear(256, num_classes_classification),\n nn.Sigmoid()\n )\n self.features = nn.Sequential(\n nn.Conv3d(1, 32, kernel_size=(3, 3, 3), padding=(1, 1, 1)),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)),\n nn.Conv3d(32, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1)),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)),\n nn.Conv3d(64, 128, kernel_size=(3, 3, 3), padding=(1, 1, 1)),\n nn.ReLU(),\n nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))\n)\n\n\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(128, 64, kernel_size=3, padding=1),\n nn.ReLU(),\n nn.Conv3d(64, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n # Assuming x is a 5D tensor with shape [batch_size, 1, depth, height, width]\n x = self.features(x)\n x = x.view(x.size(0), -1)\n classification_output = self.classification_head(x)\n segmentation_output = self.segmentation_head(x)\n return classification_output, segmentation_output\n\n# Instantiate the model\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\n\n\n# Define loss function and optimizer\n\n\n# Training loop, validation loop, and test loop (similar to the code you provided) should remain the same with this new model.\n\n\n# Paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n#data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n# print(data)\n# print(data.index)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n# print(\"Indices:\", train_loader.index) # Print the indices\n \n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = CustomCNNModel(num_classes_classification, num_classes_segmentation)\n\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n# Define loss function and optimizer\n\n# def print_parameter_data_types(module):\n# for name, param in module.named_parameters():\n# print(f\"Parameter: {name}, Data Type: {param.dtype}\")\n\n# # Call the function to print data types of parameters in the backbone module\n# print_parameter_data_types(model.backbone)\n\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(1) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model((batch_images, batch_segmentation_masks))\n\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n \n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n \n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n \n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n \n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n \n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n \n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-29T09:32:32.433115Z","iopub.status.idle":"2023-09-29T09:32:32.43343Z","shell.execute_reply.started":"2023-09-29T09:32:32.433279Z","shell.execute_reply":"2023-09-29T09:32:32.433297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-29T09:19:09.46937Z","iopub.execute_input":"2023-09-29T09:19:09.47011Z","iopub.status.idle":"2023-09-29T09:19:14.949901Z","shell.execute_reply.started":"2023-09-29T09:19:09.470076Z","shell.execute_reply":"2023-09-29T09:19:14.948298Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Collecting nibabel\n Downloading nibabel-5.1.0-py3-none-any.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m31.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\nInstalling collected packages: nibabel\nSuccessfully installed nibabel-5.1.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\nclass MultiLabel3DAttentionModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation):\n super(MultiLabel3DAttentionModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n \n \n # Modify the stem to accept the correct input channels (128)\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Attention block\n self.attention = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, 1, kernel_size=1),\n nn.Sigmoid()\n )\n \n # Classification head\n self.classification_head = nn.Sequential(\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n nn.Linear(1, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes),\n nn.Sigmoid()\n )\n \n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n \n def forward(self, x, segmentation_mask):\n #print( '...............', x.dtype, segmentation_mask.dtype)\n print(\"X shape\", x.shape)\n \n\n features = self.backbone(x)\n print(\"features shape\", features.shape)\n # Apply attention to features\n # Assuming features has shape [batch_size, num_features]\n # Reshape features to [batch_size, 1, 1, 1, num_features]\n features = features.view(features.size(0), 1, 1, 1, features.size(1))\n \n attention_weights = self.attention(features)\n \n print('attention_weights shape', attention_weights.shape)\n attended_features = features * attention_weights\n print('attended_features', attended_features.shape)\n \n # Classification branch\n classification_output = self.classification_head(attended_features)\n \n # Reshape attended_features to match segmentation_mask's shape along dimensions 2, 3, and 4\n #attended_features = attended_features.expand(-1, -1, 128, 128, 128)\n\n print('segmentation_mask', segmentation_mask.shape)\n \n # Segmentation branch\n # Reshape attended_features to match segmentation_mask's shape along dimensions 2, 3, and 4\n #attended_features = attended_features.expand(-1, -1, segmentation_mask.size(2), segmentation_mask.size(3), segmentation_mask.size(4))\n \n #segmentation_output = self.segmentation_head(attended_features) * segmentation_mask\n \n # Modify the segmentation head to handle the different number of channels\n segmentation_output = self.segmentation_head(attended_features)\n # You can use interpolation or other methods to match the shape.\n segmentation_output = F.interpolate(segmentation_output, size=segmentation_mask.shape[2:], mode='trilinear')\n \n print('segmentation_output', segmentation_output.shape)\n \n segmentation_output = segmentation_output * segmentation_mask # Element-wise multiplication\n \n \n return classification_output, segmentation_output\n\n# Paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n#data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n# print(data)\n# print(data.index)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n# print(\"Indices:\", train_loader.index) # Print the indices\n \n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = MultiLabel3DAttentionModel(num_classes_classification, num_classes_segmentation)\n\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# def print_parameter_data_types(module):\n# for name, param in module.named_parameters():\n# print(f\"Parameter: {name}, Data Type: {param.dtype}\")\n\n# # Call the function to print data types of parameters in the backbone module\n# print_parameter_data_types(model.backbone)\n\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(1) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n \n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n \n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n \n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n \n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n \n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n \n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-29T09:42:58.302066Z","iopub.execute_input":"2023-09-29T09:42:58.302431Z","iopub.status.idle":"2023-09-29T10:09:01.777627Z","shell.execute_reply.started":"2023-09-29T09:42:58.302403Z","shell.execute_reply":"2023-09-29T10:09:01.776188Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"Length of DataFrame: 4711\nlen of train_dataset 3297\ntrain_loader 206\n\n","output_type":"stream"},{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=R3D_18_Weights.KINETICS400_V1`. You can also use `weights=R3D_18_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\nDownloading: \"https://download.pytorch.org/models/r3d_18-b3b3357e.pth\" to /root/.cache/torch/hub/checkpoints/r3d_18-b3b3357e.pth\n100%|██████████| 127M/127M [00:00<00:00, 243MB/s] \n","output_type":"stream"},{"name":"stdout","text":"X shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\nX shape torch.Size([16, 1, 128, 128, 128])\nfeatures shape torch.Size([16, 400])\nattention_weights shape torch.Size([16, 1, 1, 1, 400])\nattended_features torch.Size([16, 1, 1, 1, 400])\nsegmentation_mask torch.Size([16, 1, 128, 128, 128])\nsegmentation_output torch.Size([16, 1, 128, 128, 128])\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[8], line 319\u001b[0m\n\u001b[1;32m 316\u001b[0m total_train \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 318\u001b[0m \u001b[38;5;66;03m# Backpropagation and optimization\u001b[39;00m\n\u001b[0;32m--> 319\u001b[0m \u001b[43mtotal_loss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 320\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 322\u001b[0m \u001b[38;5;66;03m# Calculate and print average training accuracy and loss\u001b[39;00m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 477\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 478\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 479\u001b[0m Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m 480\u001b[0m (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 485\u001b[0m inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m 486\u001b[0m )\n\u001b[0;32m--> 487\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 488\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 195\u001b[0m retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m 197\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m 198\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 199\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"execution":{"iopub.status.busy":"2023-09-29T10:13:24.111401Z","iopub.execute_input":"2023-09-29T10:13:24.111787Z","iopub.status.idle":"2023-09-29T10:13:28.603399Z","shell.execute_reply.started":"2023-09-29T10:13:24.111756Z","shell.execute_reply":"2023-09-29T10:13:28.602133Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"Requirement already satisfied: nibabel in /usr/local/lib/python3.8/site-packages (5.1.0)\nRequirement already satisfied: importlib-resources>=1.3 in /usr/local/lib/python3.8/site-packages (from nibabel) (6.0.0)\nRequirement already satisfied: numpy>=1.19 in /usr/local/lib/python3.8/site-packages (from nibabel) (1.23.5)\nRequirement already satisfied: packaging>=17 in /usr/local/lib/python3.8/site-packages (from nibabel) (23.1)\nRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/site-packages (from importlib-resources>=1.3->nibabel) (3.15.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\n\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import models\n\nclass TransformerModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation, input_channels=1, d_model=64): # Define d_model here\n super(TransformerModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n\n # Modify the stem to accept the correct input channels\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(input_channels, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Embedding layer to match d_model\n self.embedding = nn.Linear(d_model, d_model)\n\n # Transformer encoder layer\n self.transformer = nn.Transformer(\n d_model=d_model, # Set d_model here\n nhead=4, # You can adjust this based on your data and requirements\n num_encoder_layers=4, # You can adjust the number of layers\n dim_feedforward=256, # You can adjust this\n dropout=0.1\n )\n\n # Classification head\n self.classification_head = nn.Sequential(\n nn.Linear(d_model, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes),\n nn.Sigmoid()\n )\n\n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(d_model, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n\n def forward(self, x, segmentation_mask):\n # Feature extraction using the backbone\n features = self.backbone(x)\n features = features.view(features.size(0), features.size(1), -1) # Flatten the spatial dimensions\n features = features.permute(2, 0, 1) # Transpose to (sequence_length, batch_size, d_model)\n\n # Apply the embedding layer to match d_model\n features = self.embedding(features) # Apply embedding\n\n # Assuming features as both source and target for self-attention\n # Assuming features as both source and target for self-attention\n src = features.permute(1, 0, 2) # Transpose to (batch_size, sequence_length, d_model)\n tgt = features.permute(1, 0, 2) # Transpose to (batch_size, sequence_length, d_model)\n\n# Apply the Transformer encoder\n features = self.transformer(src, tgt)\n\n # Transpose back to (batch_size, sequence_length, d_model)\n features = features.permute(1, 0, 2)\n\n # Classification branch\n classification_output = self.classification_head(features.mean(dim=1)) # Global average pooling\n\n # Segmentation branch\n segmentation_output = self.segmentation_head(features)\n segmentation_output = F.interpolate(segmentation_output, size=segmentation_mask.shape[2:], mode='trilinear')\n segmentation_output = segmentation_output * segmentation_mask\n\n return classification_output, segmentation_output\n\n\n\n\n\n\n#paths\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n#data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n# print(data)\n# print(data.index)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n# print(\"Indices:\", train_loader.index) # Print the indices\n \n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = TransformerModel(num_classes_classification, num_classes_segmentation).to(device)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# def print_parameter_data_types(module):\n# for name, param in module.named_parameters():\n# print(f\"Parameter: {name}, Data Type: {param.dtype}\")\n\n# # Call the function to print data types of parameters in the backbone module\n# print_parameter_data_types(model.backbone)\n\n# Instantiate the model with the appropriate number of classes for both classification and segmentation\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(1) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-29T11:00:18.307378Z","iopub.execute_input":"2023-09-29T11:00:18.307771Z","iopub.status.idle":"2023-09-29T11:00:48.183258Z","shell.execute_reply.started":"2023-09-29T11:00:18.307742Z","shell.execute_reply":"2023-09-29T11:00:48.181794Z"},"trusted":true},"execution_count":29,"outputs":[{"name":"stdout","text":"Length of DataFrame: 4711\nlen of train_dataset 3297\ntrain_loader 206\n\n","output_type":"stream"},{"name":"stderr","text":"/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n warnings.warn(\n/usr/local/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=R3D_18_Weights.KINETICS400_V1`. You can also use `weights=R3D_18_Weights.DEFAULT` to get the most up-to-date weights.\n warnings.warn(msg)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)","Cell \u001b[0;32mIn[29], line 269\u001b[0m\n\u001b[1;32m 266\u001b[0m batch_segmentation_masks \u001b[38;5;241m=\u001b[39m batch_segmentation_masks\u001b[38;5;241m.\u001b[39munsqueeze(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 268\u001b[0m \u001b[38;5;66;03m# Forward pass\u001b[39;00m\n\u001b[0;32m--> 269\u001b[0m classification_outputs, segmentation_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch_images\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_segmentation_masks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[38;5;66;03m# Apply sigmoid activation to the classification outputs\u001b[39;00m\n\u001b[1;32m 272\u001b[0m classification_outputs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msigmoid(classification_outputs)\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","Cell \u001b[0;32mIn[29], line 123\u001b[0m, in \u001b[0;36mTransformerModel.forward\u001b[0;34m(self, x, segmentation_mask)\u001b[0m\n\u001b[1;32m 120\u001b[0m features \u001b[38;5;241m=\u001b[39m features\u001b[38;5;241m.\u001b[39mpermute(\u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m) \u001b[38;5;66;03m# Transpose to (sequence_length, batch_size, d_model)\u001b[39;00m\n\u001b[1;32m 122\u001b[0m \u001b[38;5;66;03m# Apply the embedding layer to match d_model\u001b[39;00m\n\u001b[0;32m--> 123\u001b[0m features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# Apply embedding\u001b[39;00m\n\u001b[1;32m 125\u001b[0m \u001b[38;5;66;03m# Assuming features as both source and target for self-attention\u001b[39;00m\n\u001b[1;32m 126\u001b[0m \u001b[38;5;66;03m# Assuming features as both source and target for self-attention\u001b[39;00m\n\u001b[1;32m 127\u001b[0m src \u001b[38;5;241m=\u001b[39m features\u001b[38;5;241m.\u001b[39mpermute(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m2\u001b[39m) \u001b[38;5;66;03m# Transpose to (batch_size, sequence_length, d_model)\u001b[39;00m\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n","File \u001b[0;32m/usr/local/lib/python3.8/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n","\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (16x400 and 64x64)"],"ename":"RuntimeError","evalue":"mat1 and mat2 shapes cannot be multiplied (16x400 and 64x64)","output_type":"error"}]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import train_test_split\nfrom scipy.ndimage import zoom\n\n# Define a custom dataset for 3D .nii images\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, labels, transform=None):\n self.image_paths = image_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n\n return image, label\n\n# Define transformations if needed\ntransform = transforms.Compose([\n # You can add additional transformations here if needed\n transforms.ToTensor(), # Convert to tensor\n])\n\n# Define paths and other parameters\n\n\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' \ndata_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the directory containing your .nii files\n\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Extract file paths and labels from the data\nimage_paths = [os.path.join(data_dir, filename) for filename in data['file_path']]\nlabels = data[['bowel_healthy', 'bowel_injury', 'extravasation_healthy', 'extravasation_injury',\n 'kidney_healthy', 'kidney_low', 'kidney_high', 'liver_healthy', 'liver_low',\n 'liver_high', 'spleen_healthy', 'spleen_low', 'spleen_high', 'any_injury']].values\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(list(zip(image_paths, labels)), test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(*zip(*train_data), transform=transform)\nval_dataset = CustomDataset(*zip(*val_data), transform=transform)\ntest_dataset = CustomDataset(*zip(*test_data), transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Define a 3D CNN model\nclass CNN3DModel(nn.Module):\n def __init__(self, num_classes):\n super(CNN3DModel, self).__init__()\n self.conv1 = nn.Conv3d(1, 32, kernel_size=3, padding=1)\n self.conv2 = nn.Conv3d(32, 64, kernel_size=3, padding=1)\n self.pool = nn.MaxPool3d(kernel_size=2, stride=2)\n self.fc1 = nn.Linear(64 * 32 * 32 * 32, 128)\n self.fc2 = nn.Linear(128, num_classes)\n\n def forward(self, x):\n x = self.pool(F.relu(self.conv1(x)))\n x = self.pool(F.relu(self.conv2(x)))\n x = x.view(-1, 64 * 32 * 32 * 32)\n x = F.relu(self.fc1(x))\n x = self.fc2(x)\n return x\n\n# Instantiate the model\nmodel = CNN3DModel(num_classes)\n\n# Define loss function and optimizer\ncriterion = nn.BCEWithLogitsLoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n\n for batch_images, batch_labels in train_loader:\n optimizer.zero_grad()\n\n # Move data to the GPU if available\n batch_images = batch_images.unsqueeze(1).to(torch.float32)\n batch_labels = batch_labels.to(torch.float32)\n\n # Forward pass\n outputs = model(batch_images)\n\n # Calculate the loss\n loss = criterion(outputs, batch_labels)\n\n # Backpropagation and optimization\n loss.backward()\n optimizer.step()\n\n running_loss += loss.item()\n\n # Calculate accuracy\n predicted = (torch.sigmoid(outputs) > 0.5).float()\n batch_accuracy = (predicted == batch_labels).all(dim=1).float().mean()\n correct_train += batch_accuracy.item()\n total_train += 1\n\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n\n print(f\"Epoch [{epoch + 1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-09-29T11:20:57.591987Z","iopub.status.idle":"2023-09-29T11:20:57.592285Z","shell.execute_reply.started":"2023-09-29T11:20:57.592139Z","shell.execute_reply":"2023-09-29T11:20:57.592156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\n\n# Define a custom dataset for 3D .nii images\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path).get_fdata()\n\n # Apply transformations if provided to the image and mask\n if self.transform:\n image = self.transform(image)\n if segmentation_mask is not None:\n segmentation_mask = self.transform(segmentation_mask)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n# Define paths and other parameters\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n # You can add additional transformations here if needed\n transforms.ToTensor(), # Convert to tensor\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Extract file paths and labels from the data\nimage_paths = [os.path.join(segmentation_dir, filename) for filename in data['file_path']]\nmask_paths = [os.path.join(segmentation_dir, filename) if pd.notna(filename) else None for filename in data['mask_path']]\nlabels = data[['bowel_healthy', 'bowel_injury', 'extravasation_healthy', 'extravasation_injury',\n 'kidney_healthy', 'kidney_low', 'kidney_high', 'liver_healthy', 'liver_low',\n 'liver_high', 'spleen_healthy', 'spleen_low', 'spleen_high', 'any_injury']].values\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(list(zip(image_paths, mask_paths, labels)), test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(*zip(*train_data), transform=transform)\nval_dataset = CustomDataset(*zip(*val_data), transform=transform)\ntest_dataset = CustomDataset(*zip(*test_data), transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Define a 3D Transformer model\nclass TransformerModel(nn.Module):\n def __init__(self, num_classes, num_classes_segmentation, input_channels=1, d_model=64):\n super(TransformerModel, self).__init__()\n\n # Load a pre-trained ResNet3D backbone\n self.backbone = models.video.r3d_18(pretrained=True)\n\n # Modify the stem to accept the correct input channels\n self.backbone.stem[0] = nn.Sequential(\n nn.Conv3d(input_channels, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True))\n\n # Transformer encoder layer\n self.transformer = nn.Transformer(\n d_model=d_model,\n nhead=4,\n num_encoder_layers=4,\n dim_feedforward=256,\n dropout=0.1\n )\n\n # Classification head\n self.classification_head = nn.Sequential(\n nn.Linear(d_model, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes),\n nn.Sigmoid()\n )\n\n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Conv3d(d_model, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.Conv3d(128, num_classes_segmentation, kernel_size=1),\n nn.Sigmoid()\n )\n\n def forward(self, x, segmentation_mask):\n # Feature extraction using the backbone\n features = self.backbone(x)\n features = features.view(features.size(0), features.size(1), -1)\n features = features.permute(2, 0, 1)\n\n # Apply the embedding layer to match d_model\n features = self.embedding(features)\n\n # Assuming features as both source and target for self-attention\n src = features.permute(1, 0, 2)\n tgt = features.permute(1, 0, 2)\n\n # Apply the Transformer encoder\n features = self.transformer(src, tgt)\n\n # Revert the permutation\n features = features.permute(1, 0, 2)\n\n # Classification branch\n classification_output = self.classification_head(features)\n\n # Reshape features to match segmentation_mask's shape\n features = features.view(features.size(0), -1, *segmentation_mask.size()[2:])\n\n # Segmentation branch\n segmentation_output = self.segmentation_head(features)\n\n return classification_output, segmentation_output\n\n# Instantiate the model\nmodel = TransformerModel(num_classes, num_classes_segmentation)\n\n# Move the model to the selected device\nmodel.to(device)\n\n# Define loss function and optimizer\ncriterion = nn.BCEWithLogitsLoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 20\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n\n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n\n # Move data to the GPU if available\n batch_images = batch_images.unsqueeze(1).to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1).to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Forward pass\n outputs = model(batch_images, batch_segmentation_masks)\n\n # Calculate the loss\n loss = criterion(outputs[0], batch_labels) # Classification loss\n loss += criterion(outputs[1], batch_segmentation_masks) # Segmentation loss\n\n # Backpropagation and optimization\n loss.backward()\n optimizer.step()\n\n running_loss += loss.item()\n\n # Calculate accuracy\n predicted = (torch.sigmoid(outputs[0]) > 0.5).float()\n batch_accuracy = (predicted == batch_labels).all(dim=1).float().mean()\n correct_train += batch_accuracy.item()\n total_train += 1\n\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n\n print(f\"Epoch [{epoch + 1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install nibabel","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import transforms\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\n\n# Define your Transformer model\nclass TransformerModel(nn.Module):\n def __init__(self, num_classes_classification, num_classes_segmentation, d_model=128, nhead=4, num_encoder_layers=6):\n super(TransformerModel, self).__init__()\n\n # Define the image encoder (Convolutional layers)\n self.image_encoder = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True),\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n )\n\n # Define the segmentation encoder (Convolutional layers)\n self.segmentation_encoder = nn.Sequential(\n nn.Conv3d(1, 128, kernel_size=1),\n nn.ReLU(inplace=True),\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n )\n\n # Define the Transformer encoder\n self.transformer_encoder = nn.TransformerEncoder(\n nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead),\n num_layers=num_encoder_layers,\n )\n\n # Classification head\n self.classification_head = nn.Sequential(\n nn.Linear(d_model, 64),\n nn.ReLU(inplace=True),\n nn.Linear(64, num_classes_classification),\n nn.Sigmoid()\n )\n\n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Linear(d_model, num_classes_segmentation),\n nn.Sigmoid()\n )\n\n def forward(self, image, segmentation_mask):\n # Encode the image and segmentation mask\n image_features = self.image_encoder(image)\n segmentation_features = self.segmentation_encoder(segmentation_mask)\n\n # Concatenate the features\n features = torch.cat((image_features, segmentation_features), dim=1)\n\n # Apply Transformer encoder\n transformer_output = self.transformer_encoder(features)\n\n # Classification branch\n classification_output = self.classification_head(transformer_output)\n\n # Segmentation branch\n segmentation_output = self.segmentation_head(transformer_output)\n\n return classification_output, segmentation_output\n\n# Define a custom dataset\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load and preprocess the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Load and preprocess the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path).get_fdata()\n # Apply transformations if provided to the segmentation mask\n if self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n\n return image, segmentation_mask, label\n\n# Set up your data paths, hyperparameters, and device\n# ...\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file and split the data\n# ...\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n# Instantiate the model with the appropriate number of classes\nmodel = TransformerModel(num_classes_classification, num_classes_segmentation)\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training, validation, and test loops\n# ...\n\n# Note: You can reuse the training, validation, and test loops from your previous code with minor modifications to\n# adapt them to the new Transformer model architecture.\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install nibabel\nimport os\nimport pandas as pd\nimport numpy as np\nimport nibabel as nib\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models\nfrom scipy.ndimage import zoom\nimport torch.nn.functional as F\nfrom PIL import Image\n\nclass CustomDataset(Dataset):\n def __init__(self, image_paths, mask_paths, labels, transform=None):\n self.image_paths = image_paths\n self.mask_paths = mask_paths\n self.labels = labels\n self.transform = transform\n\n def __len__(self):\n return len(self.image_paths)\n\n def __getitem__(self, idx):\n image_path = self.image_paths[idx]\n mask_path = self.mask_paths[idx]\n\n # Load the 3D NIfTI image using nibabel\n image = nib.load(image_path).get_fdata()\n# print(image.shape)\n# print(\"Data shape *********:\", image.dtype)\n\n # Load the segmentation mask if available\n segmentation_mask = None\n if pd.notna(mask_path):\n segmentation_mask = nib.load(mask_path)\n segmentation_mask_data = segmentation_mask.get_fdata()\n resized_data = resize_nifti(segmentation_mask_data, desired_shape)\n segmentation_mask_data_affine = segmentation_mask.affine\n resized_affine = segmentation_mask_data_affine\n segmentation_mask = nib.Nifti1Image(resized_data, affine=resized_affine).get_fdata()\n\n # Apply transformations if provided to the image\n if self.transform:\n image = self.transform(image)\n\n # Apply transformations if provided to the segmentation mask\n if segmentation_mask is not None and self.transform:\n segmentation_mask = self.transform(segmentation_mask)\n else:\n segmentation_mask = torch.zeros_like(image)\n\n label = torch.tensor(self.labels[idx], dtype=torch.float32)\n \n return image, segmentation_mask, label\n\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\n\n# Function to resize NIfTI data\ndef resize_nifti(nifti_data, target_shape):\n factors = (target_shape[0] / nifti_data.shape[0],\n target_shape[1] / nifti_data.shape[1],\n target_shape[2] / nifti_data.shape[2])\n resized_data = zoom(nifti_data, factors, order=3) # Cubic interpolation (higher quality)\n return resized_data\n\n\nclass TransformerModel(nn.Module):\n def __init__(self, num_classes_classification, num_classes_segmentation, d_model=128, nhead=4, num_encoder_layers=6):\n super(TransformerModel, self).__init__()\n\n # Define the image encoder (Convolutional layers)\n self.image_encoder = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True),\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n )\n\n # Define the segmentation encoder (Convolutional layers)\n self.segmentation_encoder = nn.Sequential(\n nn.Conv3d(1, 64, kernel_size=(3, 7, 7), stride=(1, 2, 2), padding=(1, 3, 3)),\n nn.BatchNorm3d(64),\n nn.ReLU(inplace=True),\n nn.AdaptiveAvgPool3d(1),\n nn.Flatten(),\n )\n\n # Define the Transformer encoder\n self.transformer_encoder = nn.TransformerEncoder(\n nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead),\n num_layers=num_encoder_layers,\n )\n\n # Add a linear layer to adjust the output dimension to match the classification head\n self.transformer_encoder_output_linear = nn.Linear(2 * d_model, 128)\n\n # Classification head\n self.classification_head = nn.Sequential(\n nn.Linear(128, num_classes_classification), # Adjust input dimension to match concatenated features\n nn.ReLU(inplace=True),\n nn.Linear(128, num_classes_classification),\n nn.Sigmoid()\n )\n\n # Segmentation head\n self.segmentation_head = nn.Sequential(\n nn.Linear(d_model, num_classes_segmentation),\n nn.Sigmoid()\n )\n\n def forward(self, image, segmentation_mask):\n # Encode the image and segmentation mask\n image_features = self.image_encoder(image)\n segmentation_features = self.segmentation_encoder(segmentation_mask)\n\n # Concatenate the features\n features = torch.cat((image_features, segmentation_features), dim=1)\n\n # Apply Transformer encoder\n transformer_output = self.transformer_encoder(features)\n print(\"Image features shape:\", image_features.shape)\n print(\"Segmentation features shape:\", segmentation_features.shape)\n\n # Get the matrix `mat2`\n mat2 = transformer_output.transpose(0, 1)\n\n # Classification branch\n classification_output = self.classification_head(mat2)\n\n # Segmentation branch\n segmentation_output = self.segmentation_head(mat2)\n\n return classification_output, segmentation_output\n\n# Paths and settings\nsegmentation_dir = '/kaggle/input/rsna-2023-abdominal-trauma-detection/segmentations' # Update with the correct path\ncsv_file = '/kaggle/input/abdominal-trauma-nii-csv/abdominal_trauma_nii.csv' # Update with the correct path\nbatch_size = 16\nnum_workers = 4 # Number of CPU cores to use for data loading\nnum_classes = 14 # Number of classes\ndesired_shape = (128, 128, 128)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# Define transformations if needed\ntransform = transforms.Compose([\n transforms.ToTensor(), # Convert to tensor\n # Add more transformations if necessary\n])\n\n# Load the CSV file\ndata = pd.read_csv(csv_file)\n\n# Filter rows where the 'mask_path' column is not empty\n#data = data[pd.notna(data['mask_path'])]\n\n# Remove the extra space from the column name\ndata.columns = data.columns.str.strip()\n\n# Assuming 'data' is your DataFrame\ndata_length = len(data)\nprint(\"Length of DataFrame:\", data_length)\n# print(data)\n# print(data.index)\n\n# Split the data into training, validation, and test sets\ntrain_data, temp_data = train_test_split(data, test_size=0.3, random_state=42)\nval_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)\n\n# Set the display option to show all rows\npd.set_option('display.max_rows', None)\n\nindex_values = train_data.index.values\n\n# chunk_size = 200 # You can adjust the chunk size\n# for i in range(0, len(index_values), chunk_size):\n# print(index_values[i:i+chunk_size])\n\n# Reset the display option to its default value (if needed)\npd.reset_option('display.max_rows')\n\n# Extract file paths and labels from the data\ntrain_paths = train_data['file_path'].values\ntrain_mask_paths = train_data['mask_path'].values\ntrain_labels = train_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\nval_paths = val_data['file_path'].values\nval_mask_paths = val_data['mask_path'].values\nval_labels = val_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\ntest_paths = test_data['file_path'].values\ntest_mask_paths = test_data['mask_path'].values\ntest_labels = test_data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury','kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low','liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n# Instantiate the datasets\ntrain_dataset = CustomDataset(train_paths, train_mask_paths, train_labels, transform=transform)\nprint('len of train_dataset', len(train_dataset))\nval_dataset = CustomDataset(val_paths, val_mask_paths, val_labels, transform=transform)\ntest_dataset = CustomDataset(test_paths, test_mask_paths, test_labels, transform=transform)\n\n# Instantiate the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\nprint('train_loader', len(train_loader))\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\nprint(train_loader)\n# print(\"Indices:\", train_loader.index) # Print the indices\n \n# Instantiate the model with the appropriate number of classes for both classification and segmentation\nnum_classes_classification = 14 # Number of classes for classification\nnum_classes_segmentation = 1 # Number of classes for segmentation (change this according to your task)\nmodel = TransformerModel(num_classes_classification, num_classes_segmentation).to(device)\n\n\n# Define loss function and optimizer\ncriterion = nn.BCELoss() # Binary Cross-Entropy loss\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# def print_parameter_data_types(module):\n# for name, param in module.named_parameters():\n# print(f\"Parameter: {name}, Data Type: {param.dtype}\")\n\n# # Call the function to print data types of parameters in the backbone module\n# print_parameter_data_types(model.backbone)\n\n\n# Training loop\nnum_epochs = 1\n\nfor epoch in range(num_epochs):\n model.train()\n running_loss = 0.0\n correct_train = 0\n total_train = 0\n \n for batch_images, batch_segmentation_masks, batch_labels in train_loader:\n optimizer.zero_grad()\n \n # Move data to the GPU if available\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n\n # Assuming batch_images has shape (batch_size, num_frames, num_channels, height, width)\n batch_images = batch_images.unsqueeze(1) # Add a singleton dimension for channels\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n # Calculate the total loss as the sum of individual class losses\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n # Ensure that both input and target tensors are of type torch.float32\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float32)\n\n # Calculate segmentation loss\n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss += segmentation_loss\n\n running_loss += total_loss.item()\n \n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index] # Select labels for the current class\n class_outputs = classification_outputs[:, class_index] # Select model outputs for the current class\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n # Calculate overall accuracy\n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_train += batch_accuracy\n total_train += 1\n \n # Backpropagation and optimization\n total_loss.backward()\n optimizer.step()\n\n # Calculate and print average training accuracy and loss\n avg_train_accuracy = correct_train / total_train\n avg_train_loss = running_loss / len(train_loader)\n \n print(f\"Epoch [{epoch+1}/{num_epochs}]\")\n print(f\"Train Accuracy: {avg_train_accuracy:.4f} | Train Loss: {avg_train_loss:.4f}\")\n\n # Validation loop\n model.eval()\n total_val_loss = 0.0\n correct_val = 0\n total_val = 0\n\n with torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in val_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n\n # Calculate binary cross-entropy loss for each class separately\n losses = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n class_loss = criterion(class_outputs, class_labels)\n losses.append(class_loss)\n\n total_loss = sum(losses)\n\n # Check if segmentation mask is available\n if batch_segmentation_masks is not None:\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float64)\n \n # Apply sigmoid activation to segmentation_outputs\n segmentation_outputs = torch.sigmoid(segmentation_outputs)\n segmentation_outputs = segmentation_outputs.to(torch.float64)\n \n segmentation_loss = criterion(segmentation_outputs, batch_segmentation_masks)\n total_loss = total_loss + segmentation_loss\n \n total_val_loss += total_loss.item()\n\n # Calculate accuracy for each class separately\n accuracies = []\n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n\n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n accuracies.append(class_accuracy)\n \n batch_accuracy = sum(accuracies) / num_classes_classification\n correct_val += batch_accuracy\n total_val += batch_labels.size(0)\n\n val_accuracy = correct_val / total_val\n avg_val_loss = total_val_loss / len(val_loader)\n\n print(f\"Validation Accuracy: {val_accuracy:.4f} | Validation Loss: {avg_val_loss:.4f}\")\n\n# Test loop\nmodel.eval()\ntotal_correct = 0\ntotal_samples = 0\n# Initialize lists to store per-class metrics\nprecision_list = []\nrecall_list = []\nf1_list = []\n\nwith torch.no_grad():\n for batch_images, batch_segmentation_masks, batch_labels in test_loader:\n batch_images = batch_images.to(torch.float32).to(device)\n batch_segmentation_masks = batch_segmentation_masks.to(torch.float32).to(device)\n batch_labels = batch_labels.to(torch.float32).to(device)\n \n batch_images = batch_images.unsqueeze(1)\n batch_segmentation_masks = batch_segmentation_masks.unsqueeze(1)\n\n # Forward pass\n classification_outputs, segmentation_outputs = model(batch_images, batch_segmentation_masks)\n \n # Apply sigmoid activation to the classification outputs\n classification_outputs = torch.sigmoid(classification_outputs)\n \n # Initialize batch-level variables for accuracy calculation\n batch_correct = 0\n batch_samples = batch_labels.size(0)\n \n for class_index in range(num_classes_classification):\n class_labels = batch_labels[:, class_index]\n class_outputs = classification_outputs[:, class_index]\n \n # Calculate binary predictions based on a threshold (e.g., 0.5)\n predicted = (class_outputs > 0.5).float()\n \n class_accuracy = accuracy_score(class_labels.cpu(), predicted.cpu())\n batch_correct += class_accuracy\n \n # Calculate precision, recall, and F1-score for the current class\n precision, recall, f1, _ = precision_recall_fscore_support(\n class_labels.cpu(), predicted.cpu(), average='binary')\n \n precision_list.append(precision)\n recall_list.append(recall)\n f1_list.append(f1)\n\n # Accumulate batch-level accuracy\n total_correct += batch_correct\n total_samples += batch_samples\n \n test_accuracy = total_correct / total_samples\n print(f\"Test Accuracy: {test_accuracy:.4f}\")\n\n # Calculate average precision, recall, and F1-score across all classes\n avg_precision = sum(precision_list) / num_classes_classification\n avg_recall = sum(recall_list) / num_classes_classification\n avg_f1 = sum(f1_list) / num_classes_classification\n\n print(f\"Average Precision: {avg_precision:.4f}\")\n print(f\"Average Recall: {avg_recall:.4f}\")\n print(f\"Average F1 Score: {avg_f1:.4f}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}