Importing Libraries
","metadata":{}},{"cell_type":"code","source":"! pip install einops\n! pip install dicomsdl\n! pip install pylibjpeg\n! pip install python-gdcm","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:14:40.839984Z","iopub.execute_input":"2022-12-28T12:14:40.840905Z","iopub.status.idle":"2022-12-28T12:15:30.720978Z","shell.execute_reply.started":"2022-12-28T12:14:40.840803Z","shell.execute_reply":"2022-12-28T12:15:30.71978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#CONFIG
","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport torch\nimport dicomsdl\nimport pylibjpeg\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom glob import glob\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport pydicom as dicom\nimport albumentations as A\nfrom pydicom import dcmread\nfrom einops import rearrange\nfrom IPython import display as ipd\nfrom torchvision import transforms\nfrom matplotlib import pyplot as plt\nfrom joblib import Parallel, delayed\nfrom albumentations.pytorch import ToTensorV2\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.augmentations.dropout.coarse_dropout import CoarseDropout ","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:30.72377Z","iopub.execute_input":"2022-12-28T12:15:30.724577Z","iopub.status.idle":"2022-12-28T12:15:34.993361Z","shell.execute_reply.started":"2022-12-28T12:15:30.724529Z","shell.execute_reply":"2022-12-28T12:15:34.992063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n class data:\n fold=0\n batch_size=40\n resize_dim=1024\n aspect_ratio=True\n img_size=(1024, 512)\n image_dir='/tmp/dataset/rsna-bcd'\n base_path = '/kaggle/input/rsna-breast-cancer-detection'\n \n path_to_train=\"../input/split-folds-rsna/5_folds_data.csv\"\n path_to_train_images=\"/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512/\"","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:34.994758Z","iopub.execute_input":"2022-12-28T12:15:34.995508Z","iopub.status.idle":"2022-12-28T12:15:35.001984Z","shell.execute_reply.started":"2022-12-28T12:15:34.99547Z","shell.execute_reply":"2022-12-28T12:15:35.000625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#LOADING DATASET
","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.data.path_to_train).head(20)\nprint(f\"train.shape = {train_df.shape}\")\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:35.004684Z","iopub.execute_input":"2022-12-28T12:15:35.005044Z","iopub.status.idle":"2022-12-28T12:15:35.17116Z","shell.execute_reply.started":"2022-12-28T12:15:35.004988Z","shell.execute_reply":"2022-12-28T12:15:35.169883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['dicom_path'] = f'{CFG.data.base_path}/train_images'\\\n + '/' + train_df.patient_id.astype(str)\\\n + '/' + train_df.image_id.astype(str)\\\n + '.dcm'\ntrain_df['image_path'] = train_df.dicom_path.str.replace('.dcm','.png').str.replace(CFG.data.base_path, CFG.data.image_dir)\n\ndisplay(train_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:35.172815Z","iopub.execute_input":"2022-12-28T12:15:35.173623Z","iopub.status.idle":"2022-12-28T12:15:35.204778Z","shell.execute_reply.started":"2022-12-28T12:15:35.173574Z","shell.execute_reply":"2022-12-28T12:15:35.203414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#Convert DCM to PNG (use ROI)
","metadata":{}},{"cell_type":"code","source":"! rm -r /tmp/Dataset/rsna-bcdRegoin Of Interest","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:35.206394Z","iopub.execute_input":"2022-12-28T12:15:35.206876Z","iopub.status.idle":"2022-12-28T12:15:36.314619Z","shell.execute_reply.started":"2022-12-28T12:15:35.206831Z","shell.execute_reply":"2022-12-28T12:15:36.313462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/tmp/dataset/rsna-bcd/train_images', exist_ok = True)\nos.makedirs('/tmp/dataset/rsna-bcd/test_images', exist_ok = True)\n\n! ls /tmp/dataset/rsna-bcd","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:36.316265Z","iopub.execute_input":"2022-12-28T12:15:36.316765Z","iopub.status.idle":"2022-12-28T12:15:37.417473Z","shell.execute_reply.started":"2022-12-28T12:15:36.316716Z","shell.execute_reply":"2022-12-28T12:15:37.416455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ROI\ndef img2roi(img):\n # Binarize the image\n bin_img = cv2.threshold(img, 20, 255, cv2.THRESH_BINARY)[1]\n\n # Make contours around the binarized image, keep only the largest contour\n contours, _ = cv2.findContours(bin_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n contour = max(contours, key=cv2.contourArea)\n\n # Find ROI from largest contour\n ys = contour.squeeze()[:, 0]\n xs = contour.squeeze()[:, 1]\n roi = img[np.min(xs):np.max(xs), np.min(ys):np.max(ys)]\n \n return roi","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:37.419028Z","iopub.execute_input":"2022-12-28T12:15:37.419374Z","iopub.status.idle":"2022-12-28T12:15:37.427588Z","shell.execute_reply.started":"2022-12-28T12:15:37.419341Z","shell.execute_reply":"2022-12-28T12:15:37.426775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DCM \ndef read_xray(path, fix_monochrome = True):\n dicom = dicomsdl.open(path)\n data = dicom.pixelData(storedvalue=False) # storedvalue = True for int16 return otherwise float32\n data = data - np.min(data)\n data = data / np.max(data)\n if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n data = 1.0 - data\n return data\n\ndef resize_and_save(file_path):\n img = read_xray(file_path)\n h, w = img.shape[:2] # orig hw\n if CFG.data.aspect_ratio:\n r = CFG.data.resize_dim / max(h, w) # resize image to img_size\n interp = cv2.INTER_LINEAR\n if r != 1: # always resize down, only resize up if training with augmentation\n img = cv2.resize(img, (int(w * r), int(h * r)), interpolation=interp)\n else:\n img = cv2.resize(img, (CFG.resize_dim, CFG.resize_dim), cv2.INTER_LINEAR)\n \n img = (img * 255).astype(np.uint8)\n img = img2roi(img)\n img = cv2.resize(img, CFG.data.img_size[::-1], cv2.INTER_LINEAR)\n \n sub_path = file_path.split(\"/\",4)[-1].split('.dcm')[0] + '.png'\n infos = sub_path.split('/')\n pid = infos[-2]\n iid = infos[-1]; iid = iid.replace('.png','')\n new_path = os.path.join(CFG.data.image_dir, sub_path)\n \n os.makedirs(new_path.rsplit('/',1)[0], exist_ok=True)\n cv2.imwrite(new_path, img)\n \n return pid, iid , w, h","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:37.428603Z","iopub.execute_input":"2022-12-28T12:15:37.42888Z","iopub.status.idle":"2022-12-28T12:15:37.445985Z","shell.execute_reply.started":"2022-12-28T12:15:37.428853Z","shell.execute_reply":"2022-12-28T12:15:37.444869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_paths = train_df.dicom_path.tolist()\nlen(file_paths)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:37.450604Z","iopub.execute_input":"2022-12-28T12:15:37.450936Z","iopub.status.idle":"2022-12-28T12:15:37.461806Z","shell.execute_reply.started":"2022-12-28T12:15:37.450906Z","shell.execute_reply":"2022-12-28T12:15:37.460477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgsize = Parallel(n_jobs=2, backend='threading')(delayed(resize_and_save)(file_path) for file_path in tqdm(file_paths, leave=True, position=0))\nlen(imgsize)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:37.463382Z","iopub.execute_input":"2022-12-28T12:15:37.463724Z","iopub.status.idle":"2022-12-28T12:15:51.579324Z","shell.execute_reply.started":"2022-12-28T12:15:37.463693Z","shell.execute_reply":"2022-12-28T12:15:51.578246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#Train/Validation
","metadata":{}},{"cell_type":"code","source":"train = train_df.query(f'fold != {CFG.data.fold}').reset_index(drop=True)\nvalid = train_df.query(f'fold == {CFG.data.fold}').reset_index(drop=True)\n\ntrain.shape, valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:51.580817Z","iopub.execute_input":"2022-12-28T12:15:51.581203Z","iopub.status.idle":"2022-12-28T12:15:51.606692Z","shell.execute_reply.started":"2022-12-28T12:15:51.58117Z","shell.execute_reply":"2022-12-28T12:15:51.605544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#Transforms & Data Generator
","metadata":{}},{"cell_type":"code","source":"class RSNAData(Dataset):\n def __init__(self, df, img_folder, transform=None, is_test=False):\n self.df = df\n self.is_test = is_test\n self.transform = transform\n self.img_folder = img_folder\n \n def __getitem__(self, idx):\n img_path = self.df['image_path'][idx]\n img = cv2.imread(img_path)\n\n if self.transform:\n img = self.transform(image=img)[\"image\"]\n \n if not self.is_test:\n target = self.df['cancer'][idx]\n target = torch.tensor(target, dtype=torch.float)\n return {\n \"X\": img,\n \"y\": target,\n }\n return {\"X\": img,}\n \n def __len__(self):\n return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:15:51.60868Z","iopub.execute_input":"2022-12-28T12:15:51.609048Z","iopub.status.idle":"2022-12-28T12:15:51.617749Z","shell.execute_reply.started":"2022-12-28T12:15:51.609018Z","shell.execute_reply":"2022-12-28T12:15:51.61644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmantation(image_size, is_train=True):\n if is_train:\n transforms = [\n A.HorizontalFlip(p=0.5),\n A.Rotate(limit=5, p=0.5),\n A.Resize(*image_size),\n ]\n else:\n transforms = [\n A.HorizontalFlip(p=0.5),\n A.Resize(*image_size)\n ]\n \n transforms.extend([\n ToTensorV2(), \n ])\n \n manipulation = A.Compose(transforms, p=1)\n \n return manipulation","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:16:43.098846Z","iopub.execute_input":"2022-12-28T12:16:43.099276Z","iopub.status.idle":"2022-12-28T12:16:43.106466Z","shell.execute_reply.started":"2022-12-28T12:16:43.099242Z","shell.execute_reply":"2022-12-28T12:16:43.105343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = RSNAData(df=train_df, img_folder=CFG.data.path_to_train_images, transform=augmantation(CFG.data.img_size, True))\n# valid_dataset = RSNAData(df=valid, img_folder=CFG.data.path_to_train_images, transform=augmantation(CFG.data.image_size, False))\n\ntrain_loader = DataLoader(train_dataset, batch_size=CFG.data.batch_size, shuffle=True) \n# valid_loader = DataLoader(valid_dataset, batch_size=CFG.data.batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:16:43.48746Z","iopub.execute_input":"2022-12-28T12:16:43.488499Z","iopub.status.idle":"2022-12-28T12:16:43.49393Z","shell.execute_reply.started":"2022-12-28T12:16:43.48846Z","shell.execute_reply":"2022-12-28T12:16:43.492998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#Example image samples
","metadata":{}},{"cell_type":"code","source":"batch_sample_images = next(iter(train_loader))\nbatch_sample_images[\"X\"].size()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:16:44.697349Z","iopub.execute_input":"2022-12-28T12:16:44.697728Z","iopub.status.idle":"2022-12-28T12:16:44.892736Z","shell.execute_reply.started":"2022-12-28T12:16:44.697697Z","shell.execute_reply":"2022-12-28T12:16:44.891598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.imshow(batch_sample_images[\"X\"][0].permute(1, 2, 0))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:16:45.743111Z","iopub.execute_input":"2022-12-28T12:16:45.743524Z","iopub.status.idle":"2022-12-28T12:16:45.748329Z","shell.execute_reply.started":"2022-12-28T12:16:45.743488Z","shell.execute_reply":"2022-12-28T12:16:45.747073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(30, 20))\ngrid = ImageGrid(fig, 111,\n nrows_ncols=(2, 10),\n axes_pad=0.25\n)\n\nfor ax, img in zip(grid, batch_sample_images[\"X\"]):\n ax.imshow(img.permute(1, 2, 0))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T12:17:35.48023Z","iopub.execute_input":"2022-12-28T12:17:35.480653Z","iopub.status.idle":"2022-12-28T12:17:39.275769Z","shell.execute_reply.started":"2022-12-28T12:17:35.480618Z","shell.execute_reply":"2022-12-28T12:17:39.274658Z"},"trusted":true},"execution_count":null,"outputs":[]}]}