{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":7361200,"datasetId":4275869,"databundleVersionId":7451697},{"sourceType":"datasetVersion","sourceId":1799797,"datasetId":1069654,"databundleVersionId":1837254},{"sourceType":"datasetVersion","sourceId":1805632,"datasetId":1070494,"databundleVersionId":1843099}],"dockerImageVersionId":30043,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport time\nimport json\nimport glob\nimport random\nfrom pathlib import Path\nimport pandas as pd\n\nfrom PIL import Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom imgaug import augmenters as iaa\n\nimport itertools\nfrom tqdm import tqdm\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport warnings \nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-17T15:05:19.762276Z","iopub.execute_input":"2024-03-17T15:05:19.762646Z","iopub.status.idle":"2024-03-17T15:05:20.82276Z","shell.execute_reply.started":"2024-03-17T15:05:19.762611Z","shell.execute_reply":"2024-03-17T15:05:20.821975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{}},{"cell_type":"code","source":"training_folder = \"../input/vinbigdata-chest-xray-abnormalities-detection/train/\"\ndf = pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ndf = df.query(\"class_id<14\")\ndf = df.query(\"rad_id=='R9'\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2024-03-17T15:05:20.825121Z","iopub.execute_input":"2024-03-17T15:05:20.8255Z","iopub.status.idle":"2024-03-17T15:05:21.012648Z","shell.execute_reply.started":"2024-03-17T15:05:20.82546Z","shell.execute_reply":"2024-03-17T15:05:21.011743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"several_issues\"] = df.duplicated(subset=['image_id'])\ndf[\"box_size\"] = [(row.y_max-row.y_min)*(row.x_max-row.x_min) for idx, row in df.iterrows()]\ndf.head","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:21.013952Z","iopub.execute_input":"2024-03-17T15:05:21.014349Z","iopub.status.idle":"2024-03-17T15:05:22.847268Z","shell.execute_reply.started":"2024-03-17T15:05:21.014311Z","shell.execute_reply":"2024-03-17T15:05:22.846329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.class_name.unique()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.848523Z","iopub.execute_input":"2024-03-17T15:05:22.848804Z","iopub.status.idle":"2024-03-17T15:05:22.854963Z","shell.execute_reply.started":"2024-03-17T15:05:22.848775Z","shell.execute_reply":"2024-03-17T15:05:22.85411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\")[\"box_size\"].mean()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.857686Z","iopub.execute_input":"2024-03-17T15:05:22.857945Z","iopub.status.idle":"2024-03-17T15:05:22.869792Z","shell.execute_reply.started":"2024-03-17T15:05:22.85792Z","shell.execute_reply":"2024-03-17T15:05:22.869047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\")[\"box_size\"].std()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.872823Z","iopub.execute_input":"2024-03-17T15:05:22.873108Z","iopub.status.idle":"2024-03-17T15:05:22.880996Z","shell.execute_reply.started":"2024-03-17T15:05:22.873071Z","shell.execute_reply":"2024-03-17T15:05:22.880134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"class_id\").image_id.count()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.882281Z","iopub.execute_input":"2024-03-17T15:05:22.882656Z","iopub.status.idle":"2024-03-17T15:05:22.892175Z","shell.execute_reply.started":"2024-03-17T15:05:22.88262Z","shell.execute_reply":"2024-03-17T15:05:22.891343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking the box sizes per abnormality, their standard deviation, and the number of examples, I decided to pick the 5 abnormalities below. I am carefully picking abnormalities with bounding boxes large enough as I will be significantly downsizing the images.","metadata":{}},{"cell_type":"code","source":"selected_classes = [0,3,5,7,10]\ncategory_list = [\"Aortic enlargement\", \"Cardiomegaly\", \"ILD\", \"Lung Opacity\", \"Pleural effusion\"]\nfiltered_df = df.query(\"class_id in @selected_classes\")","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.893318Z","iopub.execute_input":"2024-03-17T15:05:22.893565Z","iopub.status.idle":"2024-03-17T15:05:22.902228Z","shell.execute_reply.started":"2024-03-17T15:05:22.893541Z","shell.execute_reply":"2024-03-17T15:05:22.901496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_classes_dict = {\"0\":0,\"3\":1,\"5\":2,\"7\":3,\"10\":4}\nfiltered_df[\"reclass_id\"] = [selected_classes_dict[str(row.class_id)] for idx, row in filtered_df.iterrows()]","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:22.903394Z","iopub.execute_input":"2024-03-17T15:05:22.903759Z","iopub.status.idle":"2024-03-17T15:05:23.542995Z","shell.execute_reply.started":"2024-03-17T15:05:22.90372Z","shell.execute_reply":"2024-03-17T15:05:23.542332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.543966Z","iopub.execute_input":"2024-03-17T15:05:23.544246Z","iopub.status.idle":"2024-03-17T15:05:23.574418Z","shell.execute_reply.started":"2024-03-17T15:05:23.54422Z","shell.execute_reply":"2024-03-17T15:05:23.573517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The 2 functions below allows to go from bounding boxes to the right format for a MaskRCNN.","metadata":{}},{"cell_type":"code","source":"def get_mask(img_dimensions, x_min, y_min, x_max, y_max):\n    img_height, img_width = img_dimensions\n    img_mask = np.full((img_height,img_width),0)\n    img_mask[y_min:y_max,x_min:x_max] = 255\n    \n    return img_mask.astype(np.float32)\n\n\ndef rle_encoding(x):\n    dots = np.where(x.T.flatten() == 255)[0]\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b>prev+1): run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return ' '.join([str(x) for x in run_lengths])","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.575589Z","iopub.execute_input":"2024-03-17T15:05:23.575853Z","iopub.status.idle":"2024-03-17T15:05:23.584654Z","shell.execute_reply.started":"2024-03-17T15:05:23.575827Z","shell.execute_reply":"2024-03-17T15:05:23.583854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following function is taken from raddar's [notebook](https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way) and helps to load the images in the cleanest way possible.","metadata":{}},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.585725Z","iopub.execute_input":"2024-03-17T15:05:23.586032Z","iopub.status.idle":"2024-03-17T15:05:23.594916Z","shell.execute_reply.started":"2024-03-17T15:05:23.586002Z","shell.execute_reply":"2024-03-17T15:05:23.59398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resized_folder = \"../working/resized_train/\"\nos.mkdir(resized_folder)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.596162Z","iopub.execute_input":"2024-03-17T15:05:23.596468Z","iopub.status.idle":"2024-03-17T15:05:23.605222Z","shell.execute_reply.started":"2024-03-17T15:05:23.596438Z","shell.execute_reply":"2024-03-17T15:05:23.604435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df.groupby(\"class_id\").image_id.count()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.606338Z","iopub.execute_input":"2024-03-17T15:05:23.606626Z","iopub.status.idle":"2024-03-17T15:05:23.618629Z","shell.execute_reply.started":"2024-03-17T15:05:23.606587Z","shell.execute_reply":"2024-03-17T15:05:23.617858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As I am not going to use the full dataset in this notebook due to the processing time allowed on Kaggle Notebooks, I am sampling an even number of images per class (depending on the number of images available) to ensure I don't end up with a highly imbalanced training set.","metadata":{}},{"cell_type":"code","source":"balanced_filtered_df = pd.DataFrame()\nsamples_per_class = 500\nfor class_name in filtered_df.class_name.unique():\n    balanced_filtered_df = balanced_filtered_df.append(filtered_df.query(\"class_name==@class_name\")[:samples_per_class], \n                                                       ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.620055Z","iopub.execute_input":"2024-03-17T15:05:23.620458Z","iopub.status.idle":"2024-03-17T15:05:23.659399Z","shell.execute_reply.started":"2024-03-17T15:05:23.620419Z","shell.execute_reply":"2024-03-17T15:05:23.658801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"balanced_filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.660627Z","iopub.execute_input":"2024-03-17T15:05:23.660903Z","iopub.status.idle":"2024-03-17T15:05:23.689308Z","shell.execute_reply.started":"2024-03-17T15:05:23.660875Z","shell.execute_reply":"2024-03-17T15:05:23.6884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diagnostic_per_image = []\n\nimage_size=512\nwith tqdm(total=len(balanced_filtered_df)) as pbar:\n    for idx,row in balanced_filtered_df.iterrows():\n        image_id = row.image_id\n        image_df = balanced_filtered_df.query(\"image_id==@image_id\")\n        class_list = []\n        RLE_list = []\n        \n        for diagnostic_id, diagnostic in image_df.iterrows():\n            class_list.append(diagnostic.reclass_id)\n\n            dicom_image = read_xray(training_folder+image_id+\".dicom\")\n            image_dimensions = dicom_image.shape\n            \n            resized_img = cv2.resize(dicom_image, (image_size,image_size), interpolation = cv2.INTER_AREA)\n            cv2.imwrite(resized_folder+image_id+\".jpg\", resized_img) \n            \n            mask = get_mask(image_dimensions, int(diagnostic.x_min), int(diagnostic.y_min), int(diagnostic.x_max), int(diagnostic.y_max))\n            resized_mask = cv2.resize(mask, (image_size,image_size))\n            RLE_list.append(rle_encoding(resized_mask))\n        diagnostic_per_image.append({\"image_id\":image_id,\n                                     \"CategoryId\":class_list,\n                                     \"EncodedPixels\":RLE_list})\n        pbar.update(1)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T15:05:23.690753Z","iopub.execute_input":"2024-03-17T15:05:23.69116Z","iopub.status.idle":"2024-03-17T16:12:04.713045Z","shell.execute_reply.started":"2024-03-17T15:05:23.691118Z","shell.execute_reply":"2024-03-17T16:12:04.712064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_df = pd.DataFrame(diagnostic_per_image)\nsamples_df[\"Height\"] = image_size\nsamples_df[\"Width\"] = image_size","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:04.715282Z","iopub.execute_input":"2024-03-17T16:12:04.715655Z","iopub.status.idle":"2024-03-17T16:12:04.728672Z","shell.execute_reply.started":"2024-03-17T16:12:04.715618Z","shell.execute_reply":"2024-03-17T16:12:04.72776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:04.730044Z","iopub.execute_input":"2024-03-17T16:12:04.730471Z","iopub.status.idle":"2024-03-17T16:12:04.766705Z","shell.execute_reply.started":"2024-03-17T16:12:04.73043Z","shell.execute_reply":"2024-03-17T16:12:04.76583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train a Mask-RCNN model","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/maskrcnn-tf2-keras ../working/maskrcnn-tf2-keras","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:04.768229Z","iopub.execute_input":"2024-03-17T16:12:04.768534Z","iopub.status.idle":"2024-03-17T16:12:05.788791Z","shell.execute_reply.started":"2024-03-17T16:12:04.768479Z","shell.execute_reply":"2024-03-17T16:12:05.787661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = Path('../working/')\nROOT_DIR = \"../../working\"\n\nNUM_CATS = len(selected_classes)\nIMAGE_SIZE = 512\nos.chdir('../working/maskrcnn-tf2-keras')\nsys.path.append(ROOT_DIR+'/maskrcnn-tf2-keras')\nfrom mrcnn.config import Config\n\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:05.790519Z","iopub.execute_input":"2024-03-17T16:12:05.790828Z","iopub.status.idle":"2024-03-17T16:12:10.858396Z","shell.execute_reply.started":"2024-03-17T16:12:05.790795Z","shell.execute_reply":"2024-03-17T16:12:10.857552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COCO_WEIGHTS_PATH = '../../input/mask-rcnn-coco/mask_rcnn_coco.h5'\n\nclass DiagnosticConfig(Config):\n    NAME = \"Diagnostic\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n    \n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 2 #That is the maximum with the memory available on kernels\n    \n    BACKBONE = 'resnet50'\n    \n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n\n    POST_NMS_ROIS_TRAINING = 250\n    POST_NMS_ROIS_INFERENCE = 150\n    MAX_GROUNDTRUTH_INSTANCES = 5\n    BACKBONE_STRIDES = [4, 8, 16, 32, 64]\n    BACKBONESHAPE = (8, 16, 24, 32, 48)\n    RPN_ANCHOR_SCALES = (8,16,24,32,48)\n    ROI_POSITIVE_RATIO = 0.33\n    DETECTION_MAX_INSTANCES = 300\n    DETECTION_MIN_CONFIDENCE = 0.7    \n    # STEPS_PER_EPOCH should be the number of instances \n    # divided by (GPU_COUNT*IMAGES_PER_GPU), and so should VALIDATION_STEPS;\n    # however, due to the time limit, I set them so that this kernel can be run in 9 hours\n    STEPS_PER_EPOCH = int(len(samples_df)*0.9/IMAGES_PER_GPU)\n    VALIDATION_STEPS = len(samples_df)-int(len(samples_df)*0.9/IMAGES_PER_GPU)\n    \nconfig = DiagnosticConfig()\nconfig.display()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:10.859723Z","iopub.execute_input":"2024-03-17T16:12:10.859991Z","iopub.status.idle":"2024-03-17T16:12:10.872693Z","shell.execute_reply.started":"2024-03-17T16:12:10.859964Z","shell.execute_reply":"2024-03-17T16:12:10.871734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DiagnosticDataset(utils.Dataset):\n    def __init__(self, df):\n        super().__init__(self)\n        \n        # Add classes\n        for i, name in enumerate(category_list):\n            self.add_class(\"diagnostic\", i+1, name)\n        \n        # Add images \n        for i, row in df.iterrows():\n            self.add_image(\"diagnostic\", \n                           image_id=row.name,\n                           path=\"../\"+resized_folder+str(row.image_id)+\".jpg\", \n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'], \n                           height=row['Height'], width=row['Width'])\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [category_list[int(x)] for x in info['labels']]\n    \n    def load_image(self, image_id):\n        return cv2.imread(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n                \n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        \n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n            annotation = [int(x) for x in annotation.split(' ')]\n            \n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n            \n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n        return mask, np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:10.87389Z","iopub.execute_input":"2024-03-17T16:12:10.874186Z","iopub.status.idle":"2024-03-17T16:12:10.896923Z","shell.execute_reply.started":"2024-03-17T16:12:10.874155Z","shell.execute_reply":"2024-03-17T16:12:10.895854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_percentage = 0.9\n\ntraining_set_size = int(training_percentage*len(samples_df))\nvalidation_set_size = int((1-training_percentage)*len(samples_df))\n\ntrain_dataset = DiagnosticDataset(samples_df[:training_set_size])\ntrain_dataset.prepare()\n\nvalid_dataset = DiagnosticDataset(samples_df[training_set_size:training_set_size+validation_set_size])\nvalid_dataset.prepare()\n\nfor i in range(10):\n    image_id = random.choice(train_dataset.image_ids)\n    image = train_dataset.load_image(image_id)\n    mask, class_ids = train_dataset.load_mask(image_id)\n    \n    visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:12:10.898319Z","iopub.execute_input":"2024-03-17T16:12:10.898689Z","iopub.status.idle":"2024-03-17T16:12:14.563503Z","shell.execute_reply.started":"2024-03-17T16:12:10.89865Z","shell.execute_reply":"2024-03-17T16:12:14.562581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"LR = 1e-4\nEPOCHS = [1,5]\n\nmodel = modellib.MaskRCNN(mode='training', config=config, model_dir=\"\")\nmodel.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=['mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])","metadata":{"execution":{"iopub.status.busy":"2024-03-17T17:18:28.935375Z","iopub.execute_input":"2024-03-17T17:18:28.935737Z","iopub.status.idle":"2024-03-17T17:18:32.339884Z","shell.execute_reply.started":"2024-03-17T17:18:28.935703Z","shell.execute_reply":"2024-03-17T17:18:32.337873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR,\n            epochs=EPOCHS[0],\n            layers='heads')\n\nhistory = model.keras_model.history.history","metadata":{"execution":{"iopub.status.busy":"2024-03-17T17:18:39.905433Z","iopub.execute_input":"2024-03-17T17:18:39.905777Z","iopub.status.idle":"2024-03-17T17:28:55.586745Z","shell.execute_reply.started":"2024-03-17T17:18:39.905749Z","shell.execute_reply":"2024-03-17T17:28:55.585513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.train(train_dataset, valid_dataset,\n            learning_rate=LR/10,\n            epochs=EPOCHS[1],\n            layers='all')\n\nnew_history = model.keras_model.history.history\nfor k in new_history: history[k] = history[k] + new_history[k]","metadata":{"execution":{"iopub.status.busy":"2024-03-17T17:28:55.589865Z","iopub.execute_input":"2024-03-17T17:28:55.59019Z","iopub.status.idle":"2024-03-17T18:09:10.983985Z","shell.execute_reply.started":"2024-03-17T17:28:55.590156Z","shell.execute_reply":"2024-03-17T18:09:10.983115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(EPOCHS[-1])\n\nplt.figure(figsize=(18, 6))\n\nplt.subplot(131)\nplt.plot(epochs, history['loss'], label=\"train loss\")\nplt.plot(epochs, history['val_loss'], label=\"valid loss\")\nplt.legend()\nplt.subplot(132)\nplt.plot(epochs, history['mrcnn_class_loss'], label=\"train class loss\")\nplt.plot(epochs, history['val_mrcnn_class_loss'], label=\"valid class loss\")\nplt.legend()\nplt.subplot(133)\nplt.plot(epochs, history['mrcnn_mask_loss'], label=\"train mask loss\")\nplt.plot(epochs, history['val_mrcnn_mask_loss'], label=\"valid mask loss\")\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:09:10.986329Z","iopub.execute_input":"2024-03-17T18:09:10.986747Z","iopub.status.idle":"2024-03-17T18:09:11.517612Z","shell.execute_reply.started":"2024-03-17T18:09:10.986702Z","shell.execute_reply":"2024-03-17T18:09:11.516781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_epoch = np.argmin(history[\"val_loss\"][1:]) + 1\nprint(\"Best epoch: \", best_epoch)\nprint(\"Valid loss: \", history[\"val_loss\"][1:][best_epoch-1])","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:31:26.666791Z","iopub.execute_input":"2024-03-17T18:31:26.667227Z","iopub.status.idle":"2024-03-17T18:31:26.673537Z","shell.execute_reply.started":"2024-03-17T18:31:26.667179Z","shell.execute_reply":"2024-03-17T18:31:26.672525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport numpy as np\n\n# Ground truth labels (actual classes)\nactual_labels = np.array([\"loss\",\"train loss\",\"mrcnn_class_loss\",\"train class loss\",\"mrcnn_mask_loss\",\"train mask loss\"])  # Example ground truth labels\n\n# Predicted labels (predicted classes)\npredicted_labels = np.array([\"val_loss\",\"valid loss\",\"val_mrcnn_class_loss\",\"valid class loss\",\"val_mrcnn_mask_loss\",\"valid mask loss\"])  # Example predicted labels\n\n# Compute confusion matrix\nconf_matrix = confusion_matrix(actual_labels, predicted_labels)\n\n# Display the confusion matrix\nprint(\"Confusion Matrix:\")\nprint(conf_matrix)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:31:29.467631Z","iopub.execute_input":"2024-03-17T18:31:29.467966Z","iopub.status.idle":"2024-03-17T18:31:29.477838Z","shell.execute_reply.started":"2024-03-17T18:31:29.467936Z","shell.execute_reply":"2024-03-17T18:31:29.47683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Plot confusion matrix\nplt.figure(figsize=(8, 6))\nsns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False)\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"Actual\")\nplt.title(\"Confusion Matrix\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:41:07.792916Z","iopub.execute_input":"2024-03-17T18:41:07.793285Z","iopub.status.idle":"2024-03-17T18:41:08.431115Z","shell.execute_reply.started":"2024-03-17T18:41:07.793255Z","shell.execute_reply":"2024-03-17T18:41:08.430272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test images","metadata":{}},{"cell_type":"code","source":"resized_test_folder = \"../../working/resized_test/\"\nos.mkdir(resized_test_folder)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:42:41.511958Z","iopub.execute_input":"2024-03-17T18:42:41.512337Z","iopub.status.idle":"2024-03-17T18:42:41.516616Z","shell.execute_reply.started":"2024-03-17T18:42:41.512304Z","shell.execute_reply":"2024-03-17T18:42:41.515643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InferenceConfig(DiagnosticConfig):\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE    \n    IMAGE_RESIZE_MODE = 'none'\n    DETECTION_MIN_CONFIDENCE = 0.8\n    DETECTION_NMS_THRESHOLD = 0.5\n\ninference_config = InferenceConfig()\n\nmodel = modellib.MaskRCNN(mode='inference', \n                          config=inference_config,\n                          model_dir=\"\")","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:42:45.839526Z","iopub.execute_input":"2024-03-17T18:42:45.839898Z","iopub.status.idle":"2024-03-17T18:42:49.341907Z","shell.execute_reply.started":"2024-03-17T18:42:45.839866Z","shell.execute_reply":"2024-03-17T18:42:49.341036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob_list = glob.glob(f'diagnostic*/mask_rcnn_diagnostic_{best_epoch:04d}.h5')\nmodel_path = glob_list[0] if glob_list else ''\nmodel.load_weights(model_path, by_name=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:44:05.489284Z","iopub.execute_input":"2024-03-17T18:44:05.48964Z","iopub.status.idle":"2024-03-17T18:44:13.601434Z","shell.execute_reply.started":"2024-03-17T18:44:05.489609Z","shell.execute_reply":"2024-03-17T18:44:13.60055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from skimage.measure import find_contours\nfrom matplotlib.patches import Polygon\n\n\n# Fix overlapping masks\ndef refine_masks(masks, rois):\n    areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n    mask_index = np.argsort(areas)\n    union_mask = np.zeros(masks.shape[:-1], dtype=bool)\n    for m in mask_index:\n        masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))\n        union_mask = np.logical_or(masks[:, :, m], union_mask)\n    for m in range(masks.shape[-1]):\n        mask_pos = np.where(masks[:, :, m]==True)\n        if np.any(mask_pos):\n            y1, x1 = np.min(mask_pos, axis=1)\n            y2, x2 = np.max(mask_pos, axis=1)\n            rois[m, :] = [y1, x1, y2, x2]\n    return masks, rois\n\ndef decode_rle(rle, height, width):\n    s = rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(height*width, dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape((height, width)).T\n\ndef annotations_to_mask(annotations, height, width):\n    if isinstance(annotations, list):\n        # The annotation consists in a list of RLE codes\n        mask = np.zeros((height, width, len(annotations)))\n        for i, rle_code in enumerate(annotations):\n            mask[:, :, i] = decode_rle(rle_code, height, width)\n    else:\n        error_message = \"{} is expected to be a list or str but received {}\".format(annotation, type(annotation))\n        raise TypeError(error_message)\n    return mask\n\ndef find_anomalies(dicom_image, display=False):\n\n    image_dimensions = dicom_image.shape\n\n    resized_img = cv2.resize(dicom_image, (image_size,image_size), interpolation = cv2.INTER_AREA)\n    saved_filename = resized_test_folder+\"temp_image.jpg\"\n    cv2.imwrite(saved_filename, resized_img) \n    img = cv2.imread(saved_filename)\n\n    result = model.detect([img])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)\n        for m in range(r['masks'].shape[-1]):\n            masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), \n                                        (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)\n        \n        y_scale = image_dimensions[0]/IMAGE_SIZE\n        x_scale = image_dimensions[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n        masks, rois = refine_masks(masks, rois)\n    else:\n        masks, rois = r['masks'], r['rois']\n        \n    if display:\n        visualize.display_instances(img, rois, masks, r['class_ids'], \n                                    ['bg']+category_list, r['scores'],\n                                    title=\"prediction\", figsize=(12, 12))\n    return rois, r['class_ids'], r['scores']","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:44:28.068691Z","iopub.execute_input":"2024-03-17T18:44:28.069017Z","iopub.status.idle":"2024-03-17T18:44:28.099703Z","shell.execute_reply.started":"2024-03-17T18:44:28.068988Z","shell.execute_reply":"2024-03-17T18:44:28.098498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we run a test on several images and will display the results.","metadata":{}},{"cell_type":"code","source":"test_folder = \"../../input/vinbigdata-chest-xray-abnormalities-detection/test/\"\ntest_file_list = os.listdir(test_folder)[:5]\n\nfor test_file in test_file_list:\n    dicom_image = read_xray(test_folder+test_file)\n    find_anomalies(dicom_image, display=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:44:34.449362Z","iopub.execute_input":"2024-03-17T18:44:34.449724Z","iopub.status.idle":"2024-03-17T18:44:45.725413Z","shell.execute_reply.started":"2024-03-17T18:44:34.449689Z","shell.execute_reply":"2024-03-17T18:44:45.724524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction and generating the submission file","metadata":{}},{"cell_type":"code","source":"def keep_best_cardiomegaly_box(bbox_list, class_list, confidence_list):\n    '''\n    go through the boxes and keep only one box for \n    cardiomegaly with the highest confidence score\n    '''\n    best_cardiomegaly_score = -1\n    best_cardiomegaly_bbox = []\n    clean_bbox_list, clean_class_list, clean_confidence_list = [],[],[]\n    \n    for bbox, class_id, confidence in zip(bbox_list, class_list, confidence_list):\n        #While the class number if 3 in the dataset, it is 2 in the maskrcnn training process\n        # as I have excluded some classes\n        if class_id==2:\n            if confidence>best_cardiomegaly_score:\n                best_cardiomegaly_score = confidence\n                best_cardiomegaly_bbox = bbox\n        else:\n            clean_bbox_list.append(bbox)\n            clean_class_list.append(class_id)\n            clean_confidence_list.append(confidence)\n            \n    if best_cardiomegaly_score>0:\n        clean_bbox_list.append(best_cardiomegaly_bbox)\n        clean_class_list.append(2)\n        clean_confidence_list.append(best_cardiomegaly_score)\n        \n    return clean_bbox_list, clean_class_list, clean_confidence_list","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:45:01.219388Z","iopub.execute_input":"2024-03-17T18:45:01.219749Z","iopub.status.idle":"2024-03-17T18:45:01.229127Z","shell.execute_reply.started":"2024-03-17T18:45:01.219714Z","shell.execute_reply":"2024-03-17T18:45:01.228192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\ntest_file_list = os.listdir(test_folder)\nwith tqdm(total=len(test_file_list)) as pbar:\n    for image_filename in test_file_list:\n        dicom_image = read_xray(test_folder+image_filename)\n        image_dimensions = dicom_image.shape\n        bbox_list, class_list, confidence_list = find_anomalies(dicom_image, display=False)\n        prediction_string = \"\"\n        \n        if len(bbox_list)>0:\n                    \n            bbox_list, class_list, confidence_list = keep_best_cardiomegaly_box(bbox_list, class_list, confidence_list)\n            \n            for bbox, class_id, confidence in zip(bbox_list, class_list, confidence_list):\n                class_id = next(key for key, value in selected_classes_dict.items() if value == int(class_id)-1)\n                confidence_score = str(round(confidence,3))\n\n                #HACK: I had to rescale the bounding box here. For some reason,\n                #It did not do it in the prediction function.\n                y_scale = image_dimensions[0]/image_size\n                x_scale = image_dimensions[1]/image_size\n                rescaled_bbox = (bbox * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n\n                #organise the bbox into xmin, ymin, xmax, ymax\n                ymin = image_dimensions[0]-rescaled_bbox[2]\n                ymax = image_dimensions[0]-rescaled_bbox[0]\n                xmin = rescaled_bbox[1]\n                xmax = rescaled_bbox[3]\n\n                prediction_string += \"{} {} {} {} {} {} \".format(class_id, confidence_score, xmin, ymin, xmax, ymax)\n            results.append({\"image_id\":image_filename.replace(\".dicom\",\"\"), \"PredictionString\":prediction_string.strip()})\n        else:\n            results.append({\"image_id\":image_filename.replace(\".dicom\",\"\"), \"PredictionString\":\"14 1.0 0 0 1 1\"})\n        pbar.update(1)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T18:45:08.979595Z","iopub.execute_input":"2024-03-17T18:45:08.979931Z","iopub.status.idle":"2024-03-17T19:42:39.699441Z","shell.execute_reply.started":"2024-03-17T18:45:08.979902Z","shell.execute_reply":"2024-03-17T19:42:39.698676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(results)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T19:42:58.405878Z","iopub.execute_input":"2024-03-17T19:42:58.407502Z","iopub.status.idle":"2024-03-17T19:42:58.415886Z","shell.execute_reply.started":"2024-03-17T19:42:58.407428Z","shell.execute_reply":"2024-03-17T19:42:58.415014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T19:43:01.835757Z","iopub.execute_input":"2024-03-17T19:43:01.836085Z","iopub.status.idle":"2024-03-17T19:43:01.848786Z","shell.execute_reply.started":"2024-03-17T19:43:01.836056Z","shell.execute_reply":"2024-03-17T19:43:01.847855Z"},"trusted":true},"execution_count":null,"outputs":[]}]}