{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

⚕️RSNA Breast Cancer Classification⚕️

\n

Screening Mammography

","metadata":{}},{"cell_type":"markdown","source":"
\n 📌   If thou find it beneficial, pray cast thy vote upward. 'Tis of aid to me. :)\n
","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import sys\n\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:26.485133Z","iopub.execute_input":"2023-01-26T16:16:26.485764Z","iopub.status.idle":"2023-01-26T16:16:30.940107Z","shell.execute_reply.started":"2023-01-26T16:16:26.485662Z","shell.execute_reply":"2023-01-26T16:16:30.938833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check timm version\ntimm.__version__","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:30.946367Z","iopub.execute_input":"2023-01-26T16:16:30.94694Z","iopub.status.idle":"2023-01-26T16:16:30.970379Z","shell.execute_reply.started":"2023-01-26T16:16:30.94691Z","shell.execute_reply":"2023-01-26T16:16:30.969315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport shutil\nimport cv2\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport timm\n\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedGroupKFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-26T16:16:30.97341Z","iopub.execute_input":"2023-01-26T16:16:30.975856Z","iopub.status.idle":"2023-01-26T16:16:31.605964Z","shell.execute_reply.started":"2023-01-26T16:16:30.975818Z","shell.execute_reply":"2023-01-26T16:16:31.604993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric - Probabilistic F1","metadata":{}},{"cell_type":"code","source":"def probabilistic_f1(labels, predictions, beta=0.5):\n y_true_count = 0\n ctp = 0\n cfp = 0\n\n for idx in range(len(labels)):\n prediction = min(max(predictions[idx], 0), 1)\n if (labels[idx]):\n y_true_count += 1\n ctp += prediction\n cfp += 1 - prediction\n else:\n cfp += prediction\n\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return result\n else:\n return 0","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:31.60873Z","iopub.execute_input":"2023-01-26T16:16:31.609165Z","iopub.status.idle":"2023-01-26T16:16:31.617029Z","shell.execute_reply.started":"2023-01-26T16:16:31.609119Z","shell.execute_reply":"2023-01-26T16:16:31.615989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = \"/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512\"\nTEST_PATH = \"/kaggle/input/rsna-breast-cancer-detection/test_images\"","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:31.618568Z","iopub.execute_input":"2023-01-26T16:16:31.619608Z","iopub.status.idle":"2023-01-26T16:16:31.627427Z","shell.execute_reply.started":"2023-01-26T16:16:31.619573Z","shell.execute_reply":"2023-01-26T16:16:31.626426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"markdown","source":"## Get and Undersample Data","metadata":{}},{"cell_type":"code","source":"# read raw file\ndf = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\n# add image filename columns\ndf[\"img_name\"] = df[\"patient_id\"].astype(str) + \"/\" + df[\"image_id\"].astype(str) + \".png\"\n# shuffle it\ndf = df.sample(frac=1).reset_index(drop=True)\n\n# undersample according to the cancer patients since they are minority\nundersample_amount = len(df[df[\"cancer\"]==1])\n\ndfnotcancer = df[df[\"cancer\"]==0].sample(undersample_amount).reset_index(drop=True)\ndfcancer = df[df[\"cancer\"]==1].reset_index(drop=True)\n\n# concat and then shuffle, reset index\ndff = pd.concat([dfcancer, dfnotcancer]).sample(frac=1).reset_index(drop=True)\n\nprint(f\"Old data shape is {df.shape} and new data shape is: {dff.shape}\")\n\ndff.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:31.628859Z","iopub.execute_input":"2023-01-26T16:16:31.629288Z","iopub.status.idle":"2023-01-26T16:16:31.863118Z","shell.execute_reply.started":"2023-01-26T16:16:31.629254Z","shell.execute_reply":"2023-01-26T16:16:31.861997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process Data - Cut the ROI","metadata":{}},{"cell_type":"code","source":"def img2roi(img):\n \"\"\"\n Returns ROI area in other words \n cuts the image to a desired one\n \n Because there are machine label tags,\n undesired details out of the breast image.\n \"\"\"\n img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n \n img = np.array(img * 255, dtype = np.uint8)\n # Binarize the image\n bin_img = cv2.threshold(img, 20, 255, cv2.THRESH_BINARY)[1]\n\n # Make contours around the binarized image, keep only the largest contour\n contours, _ = cv2.findContours(bin_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n contour = max(contours, key=cv2.contourArea)\n\n # Find ROI from largest contour\n ys = contour.squeeze()[:, 0]\n xs = contour.squeeze()[:, 1]\n roi = img[np.min(xs):np.max(xs), np.min(ys):np.max(ys)]\n \n #print(f\"Shape of ROI image: {roi.shape}\")\n \n return roi","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:31.864542Z","iopub.execute_input":"2023-01-26T16:16:31.867215Z","iopub.status.idle":"2023-01-26T16:16:31.874428Z","shell.execute_reply.started":"2023-01-26T16:16:31.867169Z","shell.execute_reply":"2023-01-26T16:16:31.873423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img2roi(cv2.imread(TRAIN_PATH+\"/\"+dff.img_name[0])), cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:31.876058Z","iopub.execute_input":"2023-01-26T16:16:31.876652Z","iopub.status.idle":"2023-01-26T16:16:32.155965Z","shell.execute_reply.started":"2023-01-26T16:16:31.876615Z","shell.execute_reply":"2023-01-26T16:16:32.154985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose([\n transforms.ToPILImage(),\n transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.2, hue=0.2),\n transforms.GaussianBlur(kernel_size=3, sigma=(0.1, 2.0)),\n transforms.RandomVerticalFlip(),\n transforms.RandomEqualize(),\n transforms.ToTensor()\n ])","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:32.157459Z","iopub.execute_input":"2023-01-26T16:16:32.158097Z","iopub.status.idle":"2023-01-26T16:16:32.164526Z","shell.execute_reply.started":"2023-01-26T16:16:32.15806Z","shell.execute_reply":"2023-01-26T16:16:32.163476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Dataset","metadata":{}},{"cell_type":"code","source":"class RSNADataset(Dataset):\n def __init__(self, df, img_folder, transform=transform, is_test=False):\n self.df = df\n self.img_folder = img_folder\n self.transform = transform\n self.is_test = is_test\n self.test_folder = \"/kaggle/tmp/out/\"\n \n def __getitem__(self, idx):\n if self.is_test:\n img_path = os.path.join(self.test_folder, self.df[\"img_name\"][idx])\n else:\n img_path = os.path.join(self.img_folder, self.df[\"img_name\"][idx])\n img = cv2.imread(img_path)\n img = img2roi(img)\n \n #print(img.shape)\n img = cv2.resize(img, (224, 224))\n if self.transform is not None:\n img = self.transform(img) \n img = torch.tensor(img, dtype=torch.float)\n #img = img.permute(2, 1, 0)\n if not self.is_test:\n target = self.df[\"cancer\"][idx]\n target = torch.tensor(target, dtype=torch.float)\n return img, target\n img = img.unsqueeze(0)\n #print(img.shape)\n return img\n \n def __len__(self):\n return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:23.547452Z","iopub.execute_input":"2023-01-26T16:49:23.547822Z","iopub.status.idle":"2023-01-26T16:49:23.558907Z","shell.execute_reply.started":"2023-01-26T16:49:23.547789Z","shell.execute_reply":"2023-01-26T16:49:23.557879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataLoaders","metadata":{}},{"cell_type":"code","source":"split = .95\ndf = dff.sample(frac=1).reset_index(drop=True)\ntrain_samples = int(len(df) * split)\ntrain_df = df[:train_samples+1].reset_index(drop=True)\nval_df = df[train_samples:].reset_index(drop=True)\n\ntrain_dataset = RSNADataset(df=train_df, img_folder=TRAIN_PATH, transform=transform)\nvalid_dataset = RSNADataset(df=val_df, img_folder=TRAIN_PATH, transform=transform)\n \ntrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\nval_loader = DataLoader(valid_dataset, batch_size=16, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:32.182331Z","iopub.execute_input":"2023-01-26T16:16:32.18282Z","iopub.status.idle":"2023-01-26T16:16:32.200203Z","shell.execute_reply.started":"2023-01-26T16:16:32.182785Z","shell.execute_reply":"2023-01-26T16:16:32.199335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Augmentations","metadata":{}},{"cell_type":"code","source":"def show_aug(inputs, targets, nrows=4, ncols=4, norm=False):\n plt.figure(figsize=(10, 10))\n plt.subplots_adjust(wspace=0.2, hspace=0.2)\n i_ = 0\n \n if len(inputs) > 25:\n inputs = inputs[:25]\n \n for idx in range(len(inputs)):\n \n # normalization\n if norm: \n img = inputs[idx].numpy().transpose(1,2,0)\n mean = [0.485, 0.456, 0.406]\n std = [0.229, 0.224, 0.225] \n img = (img*std+mean).astype(np.float32)\n \n else:\n img = inputs[idx].numpy().astype(np.float32)\n img = img[0,:,:]\n \n plt.subplot(nrows, ncols, i_+1)\n plt.title(f\"Label: {targets[idx].item()}\")\n plt.imshow(img, cmap=\"bone\"); \n plt.axis('off')\n \n i_ += 1\n \n return plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:32.201743Z","iopub.execute_input":"2023-01-26T16:16:32.202237Z","iopub.status.idle":"2023-01-26T16:16:32.212352Z","shell.execute_reply.started":"2023-01-26T16:16:32.202195Z","shell.execute_reply":"2023-01-26T16:16:32.211032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets = next(iter(train_loader))\nprint(images.shape, targets.shape)\n\nshow_aug(images, targets)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:32.213905Z","iopub.execute_input":"2023-01-26T16:16:32.214265Z","iopub.status.idle":"2023-01-26T16:16:33.559909Z","shell.execute_reply.started":"2023-01-26T16:16:32.214231Z","shell.execute_reply":"2023-01-26T16:16:33.559038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets = next(iter(val_loader))\nprint(images.shape, targets.shape)\n\nshow_aug(images, targets)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:33.560855Z","iopub.execute_input":"2023-01-26T16:16:33.561182Z","iopub.status.idle":"2023-01-26T16:16:34.686489Z","shell.execute_reply.started":"2023-01-26T16:16:33.56115Z","shell.execute_reply":"2023-01-26T16:16:34.682254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class EfficientNetV2_xl(torch.nn.Module):\n def __init__(self, model_name=\"efficientnet_b0\",pretrained=False, out_dim=1):\n super().__init__()\n self.backbone = timm.create_model(model_name, pretrained=pretrained, in_chans=1)\n # adjust classifier\n self.backbone.classifier = nn.Linear(self.backbone.classifier.in_features, \n out_dim)\n def forward(self, x):\n return self.backbone(x)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.687745Z","iopub.execute_input":"2023-01-26T16:16:34.688847Z","iopub.status.idle":"2023-01-26T16:16:34.696074Z","shell.execute_reply.started":"2023-01-26T16:16:34.688808Z","shell.execute_reply":"2023-01-26T16:16:34.695174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EfficientNetV2_xl()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.697666Z","iopub.execute_input":"2023-01-26T16:16:34.698366Z","iopub.status.idle":"2023-01-26T16:16:34.81986Z","shell.execute_reply.started":"2023-01-26T16:16:34.69833Z","shell.execute_reply":"2023-01-26T16:16:34.818994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.821102Z","iopub.execute_input":"2023-01-26T16:16:34.821842Z","iopub.status.idle":"2023-01-26T16:16:34.895486Z","shell.execute_reply.started":"2023-01-26T16:16:34.821805Z","shell.execute_reply":"2023-01-26T16:16:34.894431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 1","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.899353Z","iopub.execute_input":"2023-01-26T16:16:34.899663Z","iopub.status.idle":"2023-01-26T16:16:34.906285Z","shell.execute_reply.started":"2023-01-26T16:16:34.899634Z","shell.execute_reply":"2023-01-26T16:16:34.905408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit(model, epochs, train_loader, valid_loader, optimizer, loss_fn):\n \n train_loss, val_loss = [], []\n f1_score_hist = []\n \n for ep in tqdm(range(epochs+1)):\n print(f\"{35*'='} Epoch:{ep+1} {35*'='}\")\n model.train()\n \n running_loss = 0\n correct = 0\n total = 0\n # training\n for idx, (images, targets) in tqdm(enumerate(train_loader), total=len(train_loader)):\n images = images.to(device)\n targets = targets.to(device)\n \n outputs = model(images).view(-1)\n #targets = targets.unsqueeze(1)\n loss = loss_fn(outputs, targets.float())\n \n \n loss.backward()\n optimizer.step()\n optimizer.zero_grad()\n \n running_loss += loss.item()\n \n if idx % 100 == 0:\n print(f\"Batch: {idx} | Train Batch Loss: {loss.item():.4f}\")\n \n train_loss.append(running_loss / len(train_loader))\n \n print(f\"\\nEpoch: {ep+1} / {epochs} | Training Loss: {np.mean(train_loss):.4f}\\n\")\n \n \n # validation \n all_outputs, all_targets = [], []\n correct_t = 0\n batch_loss = 0\n total_t = 0\n with torch.no_grad():\n #model.eval()\n for idx, (images_t, targets_t) in enumerate(valid_loader):\n images_t = images_t.to(device)\n targets_t = targets_t.to(device)\n \n outputs_t = model(images_t).view(-1)\n #targets_t = targets_t.unsqueeze(1)\n loss_t = loss_fn(outputs_t, targets_t.float())\n \n batch_loss += loss_t.item()\n \n if idx % 10 == 0:\n print(f\"\\nBatch: {idx} | Valid Batch Loss: {loss_t.item():.4f}\")\n \n all_outputs.extend(torch.sigmoid(outputs_t).cpu().detach().tolist())\n all_targets.extend(targets_t.cpu().detach().tolist())\n prob_f1_score = probabilistic_f1(all_targets, all_outputs)\n \n f1_score_hist.append(prob_f1_score)\n val_loss.append(batch_loss/len(val_loader))\n \n print(f\"\\nEpoch: {ep+1} / {epochs} | Validation Loss: {np.mean(val_loss):.4f}\\n\")\n print(f\"\\nF1 Score for Epoch {ep+1}: {prob_f1_score:.4f}\\n\")\n \n \n torch.save(model.state_dict(), \"/kaggle/working/first.pth\")\n \n return train_loss, val_loss, f1_score_hist","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.909522Z","iopub.execute_input":"2023-01-26T16:16:34.90981Z","iopub.status.idle":"2023-01-26T16:16:34.928333Z","shell.execute_reply.started":"2023-01-26T16:16:34.909785Z","shell.execute_reply":"2023-01-26T16:16:34.927458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(train_loader, val_loader, epochs=n_epochs):\n model = EfficientNetV2_xl().to(device)\n \n optimizer = torch.optim.Adamax(params=model.parameters(), weight_decay=1e-2)\n loss_fn = nn.BCEWithLogitsLoss()\n \n train_loss, val_loss, f1_score_hist = fit(model=model, epochs=epochs, train_loader=train_loader,\n valid_loader=val_loader, optimizer=optimizer,\n loss_fn=loss_fn)\n return train_loss, val_loss, f1_score_hist","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.929655Z","iopub.execute_input":"2023-01-26T16:16:34.930107Z","iopub.status.idle":"2023-01-26T16:16:34.942073Z","shell.execute_reply.started":"2023-01-26T16:16:34.930073Z","shell.execute_reply":"2023-01-26T16:16:34.941187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the Model","metadata":{}},{"cell_type":"code","source":"train_loss, val_loss, f1_score_hist = run(train_loader, val_loader, n_epochs)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:16:34.943392Z","iopub.execute_input":"2023-01-26T16:16:34.9438Z","iopub.status.idle":"2023-01-26T16:18:03.218105Z","shell.execute_reply.started":"2023-01-26T16:16:34.943765Z","shell.execute_reply":"2023-01-26T16:18:03.2171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_history(model_name,\n train_history, val_history, \n num_epochs):\n \n x = np.arange(num_epochs)\n\n fig = plt.figure(figsize=(10, 6))\n plt.plot(x, train_history, label='train loss', lw=3, c=\"springgreen\")\n plt.plot(x, val_history, label='validation loss', lw=3, c=\"deeppink\")\n\n plt.title(f\"{model_name}\", fontsize=15)\n plt.legend(fontsize=12)\n plt.xlabel(\"Epoch\", fontsize=15)\n plt.ylabel(\"DICE\", fontsize=15)\n\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:18:03.22025Z","iopub.execute_input":"2023-01-26T16:18:03.22079Z","iopub.status.idle":"2023-01-26T16:18:03.229681Z","shell.execute_reply.started":"2023-01-26T16:18:03.220751Z","shell.execute_reply":"2023-01-26T16:18:03.228649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_history(\"EfficientNet_XL\", train_loss, val_loss, num_epochs=n_epochs+1)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:18:03.231175Z","iopub.execute_input":"2023-01-26T16:18:03.231588Z","iopub.status.idle":"2023-01-26T16:18:03.453797Z","shell.execute_reply.started":"2023-01-26T16:18:03.231553Z","shell.execute_reply":"2023-01-26T16:18:03.452844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use(\"dark_background\")\n\nplt.plot(range(n_epochs+1), f1_score_hist)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:18:03.455216Z","iopub.execute_input":"2023-01-26T16:18:03.455813Z","iopub.status.idle":"2023-01-26T16:18:03.650804Z","shell.execute_reply.started":"2023-01-26T16:18:03.455777Z","shell.execute_reply":"2023-01-26T16:18:03.649871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission | Inference","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/rsna-bcd-whl-ds/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:44:14.276389Z","iopub.execute_input":"2023-01-26T16:44:14.276818Z","iopub.status.idle":"2023-01-26T16:44:43.522315Z","shell.execute_reply.started":"2023-01-26T16:44:14.276782Z","shell.execute_reply":"2023-01-26T16:44:43.521075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl\nimport cv2\n\nfrom joblib import Parallel, delayed\nimport glob\nimport time\nimport numpy as np\nimport os\nfrom matplotlib import pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:44:43.524761Z","iopub.execute_input":"2023-01-26T16:44:43.525176Z","iopub.status.idle":"2023-01-26T16:44:43.54357Z","shell.execute_reply.started":"2023-01-26T16:44:43.525132Z","shell.execute_reply":"2023-01-26T16:44:43.542562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir_dicomsdl = '/kaggle/tmp/out/'\n\nos.makedirs(image_dir_dicomsdl, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:44:47.429335Z","iopub.execute_input":"2023-01-26T16:44:47.429695Z","iopub.status.idle":"2023-01-26T16:44:47.43656Z","shell.execute_reply.started":"2023-01-26T16:44:47.429659Z","shell.execute_reply":"2023-01-26T16:44:47.435535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/test_images/*/*.dcm\")\nlen(test_images)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:45:10.964927Z","iopub.execute_input":"2023-01-26T16:45:10.966011Z","iopub.status.idle":"2023-01-26T16:45:10.974725Z","shell.execute_reply.started":"2023-01-26T16:45:10.96594Z","shell.execute_reply":"2023-01-26T16:45:10.973535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512, save_folder=None, dicom_process = False, extension=\"png\"):\n \n patient = f.split('/')[-2]\n image_name = f.split('/')[-1][:-4]\n if dicom_process:\n dicom = pydicom.dcmread(f)\n img = dicom.pixel_array\n\n img = (img - img.min()) / (img.max() - img.min())\n\n if dicom.PhotometricInterpretation == \"MONOCHROME1\": \n img = 1 - img\n \n image = (img * 255).astype(np.uint8)\n else:\n \n dicom = dicomsdl.open(f)\n img = dicom.pixelData()\n\n img = (img - img.min()) / (img.max() - img.min())\n\n if dicom.getPixelDataInfo()['PhotometricInterpretation'] == \"MONOCHROME1\":\n img = 1 - img\n\n image = (img * 255).astype(np.uint8)\n \n img = cv2.resize(image, (size, size))\n\n file_name = f'{save_folder}' + f\"{patient}_{image_name}.{extension}\"\n\n cv2.imwrite(file_name, img)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:46:40.575777Z","iopub.execute_input":"2023-01-26T16:46:40.576166Z","iopub.status.idle":"2023-01-26T16:46:40.585703Z","shell.execute_reply.started":"2023-01-26T16:46:40.576131Z","shell.execute_reply":"2023-01-26T16:46:40.584754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Parallel(n_jobs=4)(\n delayed(process)(f, size = 512, save_folder = image_dir_dicomsdl, dicom_process = False)\n for f in tqdm(test_images)\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:46:41.134375Z","iopub.execute_input":"2023-01-26T16:46:41.134748Z","iopub.status.idle":"2023-01-26T16:46:44.334342Z","shell.execute_reply.started":"2023-01-26T16:46:41.134716Z","shell.execute_reply":"2023-01-26T16:46:44.333181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\ndf_test[\"img_name\"] = df_test[\"patient_id\"].astype(str) + \"_\" + df_test[\"image_id\"].astype(str) + \".png\"\ndf_test[\"dcm_path\"] = df_test[\"patient_id\"].astype(str) + \"_\" + df_test[\"image_id\"].astype(str) + \".dcm\"\n\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:48:03.256577Z","iopub.execute_input":"2023-01-26T16:48:03.256962Z","iopub.status.idle":"2023-01-26T16:48:03.280901Z","shell.execute_reply.started":"2023-01-26T16:48:03.256916Z","shell.execute_reply":"2023-01-26T16:48:03.279943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset=RSNADataset(df=df_test, img_folder=TEST_PATH, transform=None, is_test=True)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:32.419391Z","iopub.execute_input":"2023-01-26T16:49:32.419757Z","iopub.status.idle":"2023-01-26T16:49:32.425555Z","shell.execute_reply.started":"2023-01-26T16:49:32.419724Z","shell.execute_reply":"2023-01-26T16:49:32.424265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(next(iter(test_loader))[0][0], cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:32.592019Z","iopub.execute_input":"2023-01-26T16:49:32.592934Z","iopub.status.idle":"2023-01-26T16:49:32.836799Z","shell.execute_reply.started":"2023-01-26T16:49:32.592891Z","shell.execute_reply":"2023-01-26T16:49:32.835847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"\"\"\"\ndef prediction_model(model, test_loader): \n model.eval()\n \n with torch.no_grad():\n preds = []\n for idx in tqdm(range(len(next(iter(test_loader))))):\n X = next(iter(test_loader))\n #print(X[idx].shape)\n pred = model(X[idx].unsqueeze(0).float())\n #pred = pred.clone().detach().cpu()\n preds.append(pred)\n return torch.concat(preds).numpy()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:18:39.088493Z","iopub.status.idle":"2023-01-26T16:18:39.089632Z","shell.execute_reply.started":"2023-01-26T16:18:39.089373Z","shell.execute_reply":"2023-01-26T16:18:39.089397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_model(model, test_loader): \n model.eval()\n \n with torch.no_grad():\n preds = []\n if len(test_loader) != 1:\n # there are images and targets in loader in batches\n for images, targets in tqdm(test_loader, total=len(test_loader)):\n for im in tqdm(images):\n pred = model(im.unsqueeze(0).float())\n preds.append(pred)\n else:\n # just images\n for batch in tqdm(test_loader, total=len(test_loader)):\n for im in tqdm(batch):\n pred = model(im.unsqueeze(0).float())\n preds.append(pred)\n \n return torch.concat(preds).numpy()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:36.947339Z","iopub.execute_input":"2023-01-26T16:49:36.947717Z","iopub.status.idle":"2023-01-26T16:49:36.955643Z","shell.execute_reply.started":"2023-01-26T16:49:36.947683Z","shell.execute_reply":"2023-01-26T16:49:36.954554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = prediction_model(EfficientNetV2_xl(), test_loader)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:37.623986Z","iopub.execute_input":"2023-01-26T16:49:37.624653Z","iopub.status.idle":"2023-01-26T16:49:37.992978Z","shell.execute_reply.started":"2023-01-26T16:49:37.624619Z","shell.execute_reply":"2023-01-26T16:49:37.992014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:39.123043Z","iopub.execute_input":"2023-01-26T16:49:39.123418Z","iopub.status.idle":"2023-01-26T16:49:39.130853Z","shell.execute_reply.started":"2023-01-26T16:49:39.123385Z","shell.execute_reply":"2023-01-26T16:49:39.129757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nss = MinMaxScaler()\nscaled_preds = ss.fit_transform(preds)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:41.481602Z","iopub.execute_input":"2023-01-26T16:49:41.482006Z","iopub.status.idle":"2023-01-26T16:49:41.488358Z","shell.execute_reply.started":"2023-01-26T16:49:41.481964Z","shell.execute_reply":"2023-01-26T16:49:41.486888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\")\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:53.459632Z","iopub.execute_input":"2023-01-26T16:49:53.460013Z","iopub.status.idle":"2023-01-26T16:49:53.475725Z","shell.execute_reply.started":"2023-01-26T16:49:53.459976Z","shell.execute_reply":"2023-01-26T16:49:53.474604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_id = df_test['patient_id'].astype(str) + \"_\" + df_test['laterality']","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:53.753084Z","iopub.execute_input":"2023-01-26T16:49:53.753447Z","iopub.status.idle":"2023-01-26T16:49:53.759074Z","shell.execute_reply.started":"2023-01-26T16:49:53.753417Z","shell.execute_reply":"2023-01-26T16:49:53.758008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = .45\n\nsub_preds = scaled_preds.copy()\n\nsub_preds[sub_preds > threshold] = 1\nsub_preds[sub_preds <= threshold] = 0","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:55.549449Z","iopub.execute_input":"2023-01-26T16:49:55.549802Z","iopub.status.idle":"2023-01-26T16:49:55.555264Z","shell.execute_reply.started":"2023-01-26T16:49:55.549771Z","shell.execute_reply":"2023-01-26T16:49:55.554136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {\n \"prediction_id\": np.array((prediction_id)),\n \"cancer\": sub_preds.T[0]\n}\n\nsub_df = pd.DataFrame(data=data)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:57.72663Z","iopub.execute_input":"2023-01-26T16:49:57.727042Z","iopub.status.idle":"2023-01-26T16:49:57.733285Z","shell.execute_reply.started":"2023-01-26T16:49:57.726988Z","shell.execute_reply":"2023-01-26T16:49:57.732128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subb = sub_df.drop_duplicates(\"prediction_id\")\nsubb","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:49:58.933513Z","iopub.execute_input":"2023-01-26T16:49:58.933917Z","iopub.status.idle":"2023-01-26T16:49:58.946889Z","shell.execute_reply.started":"2023-01-26T16:49:58.933884Z","shell.execute_reply":"2023-01-26T16:49:58.945878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subb.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-26T16:50:01.461052Z","iopub.execute_input":"2023-01-26T16:50:01.461421Z","iopub.status.idle":"2023-01-26T16:50:01.470256Z","shell.execute_reply.started":"2023-01-26T16:50:01.461391Z","shell.execute_reply":"2023-01-26T16:50:01.469309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}