{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":1799797,"datasetId":1069654,"databundleVersionId":1837254},{"sourceType":"datasetVersion","sourceId":7361200,"datasetId":4275869,"databundleVersionId":7451697},{"sourceType":"datasetVersion","sourceId":1805632,"datasetId":1070494,"databundleVersionId":1843099}],"dockerImageVersionId":30043,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mask-RCNN for Chest X-ray Diagnostic - Starter","metadata":{}},{"cell_type":"markdown","source":"The goal of this notebook is to help everyone getting started with the VinBigData chest x-ray abnormalities detection competition. This is essentially an object detection problem and there is several tricky bits to it:\n* Groundtruth is provided by different experts and diagnostics can vary from one expert to another.\n* Images are massive\n* There is a significant amount of metadata\n\nTo start in a simple way, I will resize all the images and only selected groundtruth from one expert and from the diseases with the largest bounding boxes. This data will then be fed to a MaskRCNN which, hopefully, will show some preliminary results.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport time\nimport json\nimport glob\nimport random\nfrom pathlib import Path\nimport pandas as pd\n\nfrom PIL import Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom imgaug import augmenters as iaa\n\nimport itertools\nfrom tqdm import tqdm\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-08T08:57:09.308935Z","iopub.execute_input":"2024-01-08T08:57:09.309292Z","iopub.status.idle":"2024-01-08T08:57:10.870272Z","shell.execute_reply.started":"2024-01-08T08:57:09.309259Z","shell.execute_reply":"2024-01-08T08:57:10.869586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{}},{"cell_type":"code","source":"training_folder = \"../input/vinbigdata-chest-xray-abnormalities-detection/train/\"\ndf = pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ndf = df.query(\"class_id<14\")\ndf = df.query(\"rad_id=='R9'\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2024-01-08T08:57:10.872436Z","iopub.execute_input":"2024-01-08T08:57:10.872818Z","iopub.status.idle":"2024-01-08T08:57:11.153348Z","shell.execute_reply.started":"2024-01-08T08:57:10.872779Z","shell.execute_reply":"2024-01-08T08:57:11.152686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"several_issues\"] = df.duplicated(subset=['image_id'])\ndf[\"box_size\"] = [(row.y_max-row.y_min)*(row.x_max-row.x_min) for idx, row in df.iterrows()]\ndf.head","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:11.154576Z","iopub.execute_input":"2024-01-08T08:57:11.154938Z","iopub.status.idle":"2024-01-08T08:57:12.918559Z","shell.execute_reply.started":"2024-01-08T08:57:11.154901Z","shell.execute_reply":"2024-01-08T08:57:12.917749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.class_name.unique()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.919797Z","iopub.execute_input":"2024-01-08T08:57:12.920077Z","iopub.status.idle":"2024-01-08T08:57:12.926274Z","shell.execute_reply.started":"2024-01-08T08:57:12.920047Z","shell.execute_reply":"2024-01-08T08:57:12.925491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\")[\"box_size\"].mean()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.929522Z","iopub.execute_input":"2024-01-08T08:57:12.929807Z","iopub.status.idle":"2024-01-08T08:57:12.941032Z","shell.execute_reply.started":"2024-01-08T08:57:12.929779Z","shell.execute_reply":"2024-01-08T08:57:12.940306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\")[\"box_size\"].std()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.944034Z","iopub.execute_input":"2024-01-08T08:57:12.944361Z","iopub.status.idle":"2024-01-08T08:57:12.952376Z","shell.execute_reply.started":"2024-01-08T08:57:12.944333Z","shell.execute_reply":"2024-01-08T08:57:12.951622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\").image_id.count()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.953893Z","iopub.execute_input":"2024-01-08T08:57:12.954218Z","iopub.status.idle":"2024-01-08T08:57:12.96307Z","shell.execute_reply.started":"2024-01-08T08:57:12.95419Z","shell.execute_reply":"2024-01-08T08:57:12.962245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking the box sizes per abnormality, their standard deviation, and the number of examples, I decided to pick the 5 abnormalities below. I am carefully picking abnormalities with bounding boxes large enough as I will be significantly downsizing the images.","metadata":{}},{"cell_type":"code","source":"selected_classes = [0,3,5,7,10]\ncategory_list = [\"Aortic enlargement\", \"Cardiomegaly\", \"ILD\", \"Lung Opacity\", \"Pleural effusion\"]\nfiltered_df = df.query(\"class_id in @selected_classes\")","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.964278Z","iopub.execute_input":"2024-01-08T08:57:12.9646Z","iopub.status.idle":"2024-01-08T08:57:12.975846Z","shell.execute_reply.started":"2024-01-08T08:57:12.964572Z","shell.execute_reply":"2024-01-08T08:57:12.975025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_classes_dict = {\"0\":0,\"3\":1,\"5\":2,\"7\":3,\"10\":4}\nfiltered_df[\"reclass_id\"] = [selected_classes_dict[str(row.class_id)] for idx, row in filtered_df.iterrows()]","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:12.976864Z","iopub.execute_input":"2024-01-08T08:57:12.977212Z","iopub.status.idle":"2024-01-08T08:57:13.612936Z","shell.execute_reply.started":"2024-01-08T08:57:12.977172Z","shell.execute_reply":"2024-01-08T08:57:13.612098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.614091Z","iopub.execute_input":"2024-01-08T08:57:13.614355Z","iopub.status.idle":"2024-01-08T08:57:13.644635Z","shell.execute_reply.started":"2024-01-08T08:57:13.614329Z","shell.execute_reply":"2024-01-08T08:57:13.643843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The 2 functions below allows to go from bounding boxes to the right format for a MaskRCNN.","metadata":{}},{"cell_type":"code","source":"def get_mask(img_dimensions, x_min, y_min, x_max, y_max):\n    img_height, img_width = img_dimensions\n    img_mask = np.full((img_height,img_width),0)\n    img_mask[y_min:y_max,x_min:x_max] = 255\n    \n    return img_mask.astype(np.float32)\n\n\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 255)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join([str(x) for x in run_lengths])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.645997Z","iopub.execute_input":"2024-01-08T08:57:13.646299Z","iopub.status.idle":"2024-01-08T08:57:13.654968Z","shell.execute_reply.started":"2024-01-08T08:57:13.646256Z","shell.execute_reply":"2024-01-08T08:57:13.654139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following function is taken from raddar's [notebook](https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way) and helps to load the images in the cleanest way possible.","metadata":{}},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.656087Z","iopub.execute_input":"2024-01-08T08:57:13.656373Z","iopub.status.idle":"2024-01-08T08:57:13.664639Z","shell.execute_reply.started":"2024-01-08T08:57:13.656347Z","shell.execute_reply":"2024-01-08T08:57:13.663784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resized_folder = \"../working/resized_train/\"\nos.mkdir(resized_folder)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.665758Z","iopub.execute_input":"2024-01-08T08:57:13.666047Z","iopub.status.idle":"2024-01-08T08:57:13.677945Z","shell.execute_reply.started":"2024-01-08T08:57:13.666021Z","shell.execute_reply":"2024-01-08T08:57:13.677237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df.groupby(\"class_id\").image_id.count()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.679072Z","iopub.execute_input":"2024-01-08T08:57:13.679389Z","iopub.status.idle":"2024-01-08T08:57:13.693175Z","shell.execute_reply.started":"2024-01-08T08:57:13.679362Z","shell.execute_reply":"2024-01-08T08:57:13.692152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As I am not going to use the full dataset in this notebook due to the processing time allowed on Kaggle Notebooks, I am sampling an even number of images per class (depending on the number of images available) to ensure I don't end up with a highly imbalanced training set.","metadata":{}},{"cell_type":"code","source":"balanced_filtered_df = pd.DataFrame()\nsamples_per_class = 500\nfor class_name in filtered_df.class_name.unique():\n    balanced_filtered_df = balanced_filtered_df.append(filtered_df.query(\"class_name==@class_name\")[:samples_per_class], \n                                                       ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.694489Z","iopub.execute_input":"2024-01-08T08:57:13.69477Z","iopub.status.idle":"2024-01-08T08:57:13.733358Z","shell.execute_reply.started":"2024-01-08T08:57:13.694743Z","shell.execute_reply":"2024-01-08T08:57:13.732744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"balanced_filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.734858Z","iopub.execute_input":"2024-01-08T08:57:13.735224Z","iopub.status.idle":"2024-01-08T08:57:13.762663Z","shell.execute_reply.started":"2024-01-08T08:57:13.735188Z","shell.execute_reply":"2024-01-08T08:57:13.761696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diagnostic_per_image = []\n\nimage_size=512\nwith tqdm(total=len(balanced_filtered_df)) as pbar:\n    for idx,row in balanced_filtered_df.iterrows():\n        image_id = row.image_id\n        image_df = balanced_filtered_df.query(\"image_id==@image_id\")\n        class_list = []\n        RLE_list = []\n        \n        for diagnostic_id, diagnostic in image_df.iterrows():\n            class_list.append(diagnostic.reclass_id)\n\n            dicom_image = read_xray(training_folder+image_id+\".dicom\")\n            image_dimensions = dicom_image.shape\n            \n            resized_img = cv2.resize(dicom_image, (image_size,image_size), interpolation = cv2.INTER_AREA)\n            cv2.imwrite(resized_folder+image_id+\".jpg\", resized_img) \n            \n            mask = get_mask(image_dimensions, int(diagnostic.x_min), int(diagnostic.y_min), int(diagnostic.x_max), int(diagnostic.y_max))\n            resized_mask = cv2.resize(mask, (image_size,image_size))\n            RLE_list.append(rle_encoding(resized_mask))\n        diagnostic_per_image.append({\"image_id\":image_id,\n                                     \"CategoryId\":class_list,\n                                     \"EncodedPixels\":RLE_list})\n        pbar.update(1)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:13.763667Z","iopub.execute_input":"2024-01-08T08:57:13.763917Z","iopub.status.idle":"2024-01-08T08:57:39.779139Z","shell.execute_reply.started":"2024-01-08T08:57:13.763893Z","shell.execute_reply":"2024-01-08T08:57:39.776891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_df = pd.DataFrame(diagnostic_per_image)\nsamples_df[\"Height\"] = image_size\nsamples_df[\"Width\"] = image_size","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:09.913356Z","iopub.execute_input":"2024-01-08T09:04:09.913727Z","iopub.status.idle":"2024-01-08T09:04:09.921646Z","shell.execute_reply.started":"2024-01-08T09:04:09.913696Z","shell.execute_reply":"2024-01-08T09:04:09.920686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_df","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:13.685203Z","iopub.execute_input":"2024-01-08T09:04:13.685552Z","iopub.status.idle":"2024-01-08T09:04:13.713113Z","shell.execute_reply.started":"2024-01-08T09:04:13.685516Z","shell.execute_reply":"2024-01-08T09:04:13.712293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train a Mask-RCNN model","metadata":{}},{"cell_type":"markdown","source":"I struggled a bit to make the Matterport MaskRCNN repo work with Tensorflow 2.3. In the end, I used a [PR created by tomgross](https://github.com/matterport/Mask_RCNN/pull/1896/commits/a3be0c2c8654628f10736c4dd88060440fab3968) as a base and only had a couple of fixes left to get everything running!","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/maskrcnn-tf2-keras ../working/maskrcnn-tf2-keras","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:24.866095Z","iopub.execute_input":"2024-01-08T09:04:24.866432Z","iopub.status.idle":"2024-01-08T09:04:25.969703Z","shell.execute_reply.started":"2024-01-08T09:04:24.866393Z","shell.execute_reply":"2024-01-08T09:04:25.968305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('../working/')\nROOT_DIR = \"../../working\"\n\nNUM_CATS = len(selected_classes)\nIMAGE_SIZE = 512\nos.chdir('../working/maskrcnn-tf2-keras')\nsys.path.append(ROOT_DIR+'/maskrcnn-tf2-keras')\nfrom mrcnn.config import Config\n\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:27.804726Z","iopub.execute_input":"2024-01-08T09:04:27.805104Z","iopub.status.idle":"2024-01-08T09:04:32.348159Z","shell.execute_reply.started":"2024-01-08T09:04:27.805058Z","shell.execute_reply":"2024-01-08T09:04:32.347346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COCO_WEIGHTS_PATH = '../../input/mask-rcnn-coco/mask_rcnn_coco.h5'\n\nclass DiagnosticConfig(Config):\n    NAME = \"Diagnostic\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 2 #That is the maximum with the memory available on kernels\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n\n    POST_NMS_ROIS_TRAINING = 250\n    POST_NMS_ROIS_INFERENCE = 150\n    MAX_GROUNDTRUTH_INSTANCES = 5\n    BACKBONE_STRIDES = [4, 8, 16, 32, 64]\n    BACKBONESHAPE = (8, 16, 24, 32, 48)\n    RPN_ANCHOR_SCALES = (8,16,24,32,48)\n    ROI_POSITIVE_RATIO = 0.33\n    DETECTION_MAX_INSTANCES = 300\n    DETECTION_MIN_CONFIDENCE = 0.7    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = int(len(samples_df)*0.9/IMAGES_PER_GPU)\n    VALIDATION_STEPS = len(samples_df)-int(len(samples_df)*0.9/IMAGES_PER_GPU)\n    \nconfig = DiagnosticConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:05:37.687502Z","iopub.execute_input":"2024-01-08T09:05:37.687967Z","iopub.status.idle":"2024-01-08T09:05:37.704898Z","shell.execute_reply.started":"2024-01-08T09:05:37.687928Z","shell.execute_reply":"2024-01-08T09:05:37.704021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DiagnosticDataset(utils.Dataset):\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(category_list):\n            self.add_class(\"diagnostic\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"diagnostic\", \n                           image_id=row.name,\n                           path=\"../\"+resized_folder+str(row.image_id)+\".jpg\", \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [category_list[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return cv2.imread(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n        return mask, np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:41.669954Z","iopub.execute_input":"2024-01-08T09:04:41.670311Z","iopub.status.idle":"2024-01-08T09:04:41.689629Z","shell.execute_reply.started":"2024-01-08T09:04:41.670278Z","shell.execute_reply":"2024-01-08T09:04:41.688545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_percentage = 0.9\n\ntraining_set_size = int(training_percentage*len(samples_df))\nvalidation_set_size = int((1-training_percentage)*len(samples_df))\n\ntrain_dataset = DiagnosticDataset(samples_df[:training_set_size])\ntrain_dataset.prepare()\n\nvalid_dataset = DiagnosticDataset(samples_df[training_set_size:training_set_size+validation_set_size])\nvalid_dataset.prepare()\n\nfor i in range(10):\n    image_id = random.choice(train_dataset.image_ids)\n    image = train_dataset.load_image(image_id)\n    mask, class_ids = train_dataset.load_mask(image_id)\n    \n    visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:04:47.407921Z","iopub.execute_input":"2024-01-08T09:04:47.408299Z","iopub.status.idle":"2024-01-08T09:04:50.705193Z","shell.execute_reply.started":"2024-01-08T09:04:47.408257Z","shell.execute_reply":"2024-01-08T09:04:50.704201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LR = 1e-4\nEPOCHS = [1,18]\n\nmodel = modellib.MaskRCNN(mode='training', config=config, model_dir=\"\")\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=['mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:05:48.206034Z","iopub.execute_input":"2024-01-08T09:05:48.206386Z","iopub.status.idle":"2024-01-08T09:05:59.583369Z","shell.execute_reply.started":"2024-01-08T09:05:48.206351Z","shell.execute_reply":"2024-01-08T09:05:59.582619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please note that there seems to be a bug which resets the loss displayed after each of the training blocks below. It did not happen in the original Matterport repo.","metadata":{}},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[0],\n            layers='heads')\n\nhistory = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:06:04.771568Z","iopub.execute_input":"2024-01-08T09:06:04.771907Z","iopub.status.idle":"2024-01-08T09:06:59.671964Z","shell.execute_reply.started":"2024-01-08T09:06:04.771878Z","shell.execute_reply":"2024-01-08T09:06:59.67107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR/10,\n            epochs=EPOCHS[1],\n            layers='all')\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:07:18.164671Z","iopub.execute_input":"2024-01-08T09:07:18.165064Z","iopub.status.idle":"2024-01-08T09:10:20.597713Z","shell.execute_reply.started":"2024-01-08T09:07:18.165024Z","shell.execute_reply":"2024-01-08T09:10:20.596851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:10:32.76505Z","iopub.execute_input":"2024-01-08T09:10:32.765374Z","iopub.status.idle":"2024-01-08T09:10:33.231954Z","shell.execute_reply.started":"2024-01-08T09:10:32.765346Z","shell.execute_reply":"2024-01-08T09:10:33.231112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I am not including the loss from the training of the heads when picking the best loss because of the bug mentioned earlier.","metadata":{}},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"][1:]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][1:][best_epoch-1])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:12:12.491084Z","iopub.execute_input":"2024-01-08T09:12:12.491467Z","iopub.status.idle":"2024-01-08T09:12:12.498249Z","shell.execute_reply.started":"2024-01-08T09:12:12.491424Z","shell.execute_reply":"2024-01-08T09:12:12.497279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test images","metadata":{}},{"cell_type":"code","source":"resized_test_folder = \"../../working/resized_test/\"\nos.mkdir(resized_test_folder)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:16:13.292828Z","iopub.execute_input":"2024-01-08T09:16:13.29321Z","iopub.status.idle":"2024-01-08T09:16:13.319011Z","shell.execute_reply.started":"2024-01-08T09:16:13.293176Z","shell.execute_reply":"2024-01-08T09:16:13.317772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(DiagnosticConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    DETECTION_MIN_CONFIDENCE = 0.8\n    DETECTION_NMS_THRESHOLD = 0.5\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=\"\")","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:12:29.044871Z","iopub.execute_input":"2024-01-08T09:12:29.045232Z","iopub.status.idle":"2024-01-08T09:12:32.737556Z","shell.execute_reply.started":"2024-01-08T09:12:29.045197Z","shell.execute_reply":"2024-01-08T09:12:32.736726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'diagnostic*/mask_rcnn_diagnostic_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:12:36.479536Z","iopub.execute_input":"2024-01-08T09:12:36.479875Z","iopub.status.idle":"2024-01-08T09:12:40.962324Z","shell.execute_reply.started":"2024-01-08T09:12:36.479848Z","shell.execute_reply":"2024-01-08T09:12:40.961508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from skimage.measure import find_contours\nfrom matplotlib.patches import Polygon\n\n\n# Fix overlapping masks\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois\n\ndef decode_rle(rle, height, width):\n    s = rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(height*width, dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape((height, width)).T\n\ndef annotations_to_mask(annotations, height, width):\n    if isinstance(annotations, list):\n        # The annotation consists in a list of RLE codes\n        mask = np.zeros((height, width, len(annotations)))\n        for i, rle_code in enumerate(annotations):\n            mask[:, :, i] = decode_rle(rle_code, height, width)\n    else:\n        error_message = \"{} is expected to be a list or str but received {}\".format(annotation, type(annotation))\n        raise TypeError(error_message)\n    return mask\n\ndef find_anomalies(dicom_image, display=False):\n\n    image_dimensions = dicom_image.shape\n\n    resized_img = cv2.resize(dicom_image, (image_size,image_size), interpolation = cv2.INTER_AREA)\n    saved_filename = resized_test_folder+\"temp_image.jpg\"\n    cv2.imwrite(saved_filename, resized_img) \n    img = cv2.imread(saved_filename)\n\n    result = model.detect([img])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = image_dimensions[0]/IMAGE_SIZE\n        x_scale = image_dimensions[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    if display:\n        visualize.display_instances(img, rois, masks, r['class_ids'], \n                                    ['bg']+category_list, r['scores'],\n                                    title=\"prediction\", figsize=(12, 12))\n    return rois, r['class_ids'], r['scores']","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:12:46.197237Z","iopub.execute_input":"2024-01-08T09:12:46.197599Z","iopub.status.idle":"2024-01-08T09:12:46.227657Z","shell.execute_reply.started":"2024-01-08T09:12:46.197568Z","shell.execute_reply":"2024-01-08T09:12:46.226509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we run a test on several images and will display the results.","metadata":{}},{"cell_type":"code","source":"test_folder = \"../../input/vinbigdata-chest-xray-abnormalities-detection/test/\"\ntest_file_list = os.listdir(test_folder)[:5]\n\nfor test_file in test_file_list:\n    dicom_image = read_xray(test_folder+test_file)\n    find_anomalies(dicom_image, display=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:21:38.815959Z","iopub.execute_input":"2024-01-08T09:21:38.816301Z","iopub.status.idle":"2024-01-08T09:21:49.365046Z","shell.execute_reply.started":"2024-01-08T09:21:38.816268Z","shell.execute_reply":"2024-01-08T09:21:49.364036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and generating the submission file","metadata":{}},{"cell_type":"markdown","source":"Below, we run the prediction on the entire test set and format the results into a dataframe that will then be saved for submission. As it appeared that the model mistakenly gave several bounding boxes for Cardiomegaly, I am adding a function to keep only the best bounding box.","metadata":{}},{"cell_type":"code","source":"def keep_best_cardiomegaly_box(bbox_list, class_list, confidence_list):\n    '''\n    go through the boxes and keep only one box for \n    cardiomegaly with the highest confidence score\n    '''\n    best_cardiomegaly_score = -1\n    best_cardiomegaly_bbox = []\n    clean_bbox_list, clean_class_list, clean_confidence_list = [],[],[]\n    \n    for bbox, class_id, confidence in zip(bbox_list, class_list, confidence_list):\n        #While the class number if 3 in the dataset, it is 2 in the maskrcnn training process\n        # as I have excluded some classes\n        if class_id==2:\n            if confidence>best_cardiomegaly_score:\n                best_cardiomegaly_score = confidence\n                best_cardiomegaly_bbox = bbox\n        else:\n            clean_bbox_list.append(bbox)\n            clean_class_list.append(class_id)\n            clean_confidence_list.append(confidence)\n            \n    if best_cardiomegaly_score>0:\n        clean_bbox_list.append(best_cardiomegaly_bbox)\n        clean_class_list.append(2)\n        clean_confidence_list.append(best_cardiomegaly_score)\n        \n    return clean_bbox_list, clean_class_list, clean_confidence_list","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:22:00.214468Z","iopub.execute_input":"2024-01-08T09:22:00.214829Z","iopub.status.idle":"2024-01-08T09:22:00.22368Z","shell.execute_reply.started":"2024-01-08T09:22:00.214799Z","shell.execute_reply":"2024-01-08T09:22:00.222907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\ntest_file_list = os.listdir(test_folder)\nwith tqdm(total=len(test_file_list)) as pbar:\n    for image_filename in test_file_list:\n        dicom_image = read_xray(test_folder+image_filename)\n        image_dimensions = dicom_image.shape\n        bbox_list, class_list, confidence_list = find_anomalies(dicom_image, display=False)\n        prediction_string = \"\"\n        \n        if len(bbox_list)>0:\n                    \n            bbox_list, class_list, confidence_list = keep_best_cardiomegaly_box(bbox_list, class_list, confidence_list)\n            \n            for bbox, class_id, confidence in zip(bbox_list, class_list, confidence_list):\n                class_id = next(key for key, value in selected_classes_dict.items() if value == int(class_id)-1)\n                confidence_score = str(round(confidence,3))\n\n                #HACK: I had to rescale the bounding box here. For some reason,\n                #It did not do it in the prediction function.\n                y_scale = image_dimensions[0]/image_size\n                x_scale = image_dimensions[1]/image_size\n                rescaled_bbox = (bbox * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n\n                #organise the bbox into xmin, ymin, xmax, ymax\n                ymin = image_dimensions[0]-rescaled_bbox[2]\n                ymax = image_dimensions[0]-rescaled_bbox[0]\n                xmin = rescaled_bbox[1]\n                xmax = rescaled_bbox[3]\n\n                prediction_string += \"{} {} {} {} {} {} \".format(class_id, confidence_score, xmin, ymin, xmax, ymax)\n            results.append({\"image_id\":image_filename.replace(\".dicom\",\"\"), \"PredictionString\":prediction_string.strip()})\n        else:\n            results.append({\"image_id\":image_filename.replace(\".dicom\",\"\"), \"PredictionString\":\"14 1.0 0 0 1 1\"})\n        pbar.update(1)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T09:22:04.43935Z","iopub.execute_input":"2024-01-08T09:22:04.439733Z","iopub.status.idle":"2024-01-08T10:21:00.567199Z","shell.execute_reply.started":"2024-01-08T09:22:04.4397Z","shell.execute_reply":"2024-01-08T10:21:00.566282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(results)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T10:21:47.360592Z","iopub.execute_input":"2024-01-08T10:21:47.360959Z","iopub.status.idle":"2024-01-08T10:21:47.370188Z","shell.execute_reply.started":"2024-01-08T10:21:47.360929Z","shell.execute_reply":"2024-01-08T10:21:47.369083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-01-08T10:21:51.573734Z","iopub.execute_input":"2024-01-08T10:21:51.574161Z","iopub.status.idle":"2024-01-08T10:21:51.586727Z","shell.execute_reply.started":"2024-01-08T10:21:51.574115Z","shell.execute_reply":"2024-01-08T10:21:51.585859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.loc[:10]","metadata":{"execution":{"iopub.status.busy":"2024-01-08T10:27:04.57514Z","iopub.execute_input":"2024-01-08T10:27:04.575529Z","iopub.status.idle":"2024-01-08T10:27:04.58654Z","shell.execute_reply.started":"2024-01-08T10:27:04.575493Z","shell.execute_reply":"2024-01-08T10:27:04.585615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission_df.to_csv('../submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:39.8117Z","iopub.status.idle":"2024-01-08T08:57:39.812186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thanks for reading this notebook! If you found this notebook helpful, please give it an upvote. It is always greatly appreciated","metadata":{}},{"cell_type":"code","source":"#clear all the images from the working directory\n#!rm -rf ../../working/resized_train/\n#!rm -rf ../../working/resized_test/","metadata":{"execution":{"iopub.status.busy":"2024-01-08T08:57:39.813218Z","iopub.status.idle":"2024-01-08T08:57:39.813708Z"},"trusted":true},"execution_count":null,"outputs":[]}]}