{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Mask-RCNN for Chest X-ray Abnormalities Detection"},{"metadata":{},"cell_type":"markdown","source":"Thank you for these excellent notebooks:\n\n1. [Mask-RCNN and Medical Transfer Learning SIIM-ACR](https://www.kaggle.com/hmendonca/mask-rcnn-and-medical-transfer-learning-siim-acr)\n2. [MaskRCNN for Chest X-ray Anomaly Detection](https://www.kaggle.com/frlemarchand/maskrcnn-for-chest-x-ray-anomaly-detection)\n3. [Segmenting Nuclei in Microscopy Images](https://github.com/matterport/Mask_RCNN/blob/master/samples/nucleus/nucleus.py) \n\nThis notebook aims to inspect the trained model and the stage of submission. The model was trained with backbone ResNet101 + augmentation data + stratied k-fold.\nI look forward to receiving feedback from you on how to boost the efficiency of the MaskRCNN model. \n\nThank you."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.makedirs(\"../working/mrcnn\")\nos.makedirs(\"../working/test_img\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from shutil import copyfile\ncopyfile(src = \"../input/mrcnn-tf2/config.py\", dst = \"../working/mrcnn/config.py\")\ncopyfile(src = \"../input/mrcnn-tf2/model.py\", dst = \"../working/mrcnn/model.py\")\ncopyfile(src = \"../input/mrcnn-tf2/visualize.py\", dst = \"../working/mrcnn/visualize.py\")\ncopyfile(src = \"../input/mrcnn-tf2/utils.py\", dst = \"../working/mrcnn/utils.py\")\ncopyfile(src = \"../input/mrcnn-tf2/parallel_model.py\", dst = \"../working/mrcnn/parallel_model.py\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !rm -rf ../working/mrcnn/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random\nfrom random import randint\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nimport cv2\nfrom tqdm import tqdm\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom skimage import exposure\nfrom imgaug import augmenters as iaa\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\ntf.__version__\n\nfrom tensorflow.python.client import device_lib\nprint(\"GPU sample processing: \")\nprint(device_lib.list_local_devices())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = \"../input/vinbigdata-chest-xray-abnormalities-detection\"\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\nTRAIN_CSV_DIR = os.path.join(DATA_DIR, \"train.csv\")\nMASKRCNN_CSV_DIR = '../input/trainingdf/sample_df.csv'\n\nPREPROCESSED_TRAINING_IMAGE_FOLDER = \"../input/512-jpg/512_jpg/\"\nTEMP_TEST_FOLDER = \"../working/test_img/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"org_df = pd.read_csv(TRAIN_CSV_DIR)\norg_df = org_df.query('class_id != 14')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_df = pd.read_csv(MASKRCNN_CSV_DIR, converters ={'EncodedPixels': eval, 'CategoryId': eval})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from mrcnn.config import Config\nfrom mrcnn import utils\nimport mrcnn.model as modellib\nfrom mrcnn import visualize\nfrom mrcnn.model import log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_CATS = 14\nIMAGE_SIZE = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DiagnosticConfig(Config):\n    NAME = \"Diagnostic\"\n    NUM_CLASSES = NUM_CATS + 1 # +1 for the background class\n\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 10 #That is the maximum with the memory available on kernels\n\n    BACKBONE = 'resnet101'\n\n    IMAGE_MIN_DIM = IMAGE_SIZE\n    IMAGE_MAX_DIM = IMAGE_SIZE\n    IMAGE_RESIZE_MODE = 'none'\n\n    POST_NMS_ROIS_TRAINING = 250\n    POST_NMS_ROIS_INFERENCE = 150\n    MAX_GROUNDTRUTH_INSTANCES = 5\n    BACKBONE_STRIDES = [4, 8, 16, 32, 64]\n    BACKBONESHAPE = (8, 16, 24, 32, 48)\n    RPN_ANCHOR_SCALES = (8,16,24,32,48)\n    ROI_POSITIVE_RATIO = 0.33\n    DETECTION_MAX_INSTANCES = 300\n    DETECTION_MIN_CONFIDENCE = 0.7\n\n\n    STEPS_PER_EPOCH = int(len(training_df)*0.8/IMAGES_PER_GPU)\n    VALIDATION_STEPS = int(len(training_df)/IMAGES_PER_GPU)-int(len(training_df)*0.9/IMAGES_PER_GPU)\n\nconfig = DiagnosticConfig()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Inference Config\nclass InferenceConfig(config.__class__):\n    # Run detection on one image at a time\n    GPU_COUNT = 1\n    IMAGES_PER_GPU = 1\n\nconfig = InferenceConfig()\nconfig.display()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"category_list = [\"Aortic enlargement\", \"Atelectasis\",\"Calcification\",\"Cardiomegaly\",\"Consolidation\",\"ILD\",\n                \"Infiltration\", \"Lung opacity\", \"Nodule/ Mass\",\"Other lesion\",\"Pleural effusion\",\n                \"Pleural thickening\", \"Pneumothorax\",\"Pulmonary fibrosis\"]\ncategory_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create Mask RCNN formart dataset\nclass DiagnosticDataset(utils.Dataset):\n    def __init__(self, df):\n        super().__init__(self)\n\n        # Add classes\n        for i, name in enumerate(category_list):\n            self.add_class(\"diagnostic\", i+1, name)\n\n        # Add images\n        for i, row in df.iterrows():\n            self.add_image(\"diagnostic\",\n                           image_id=row.name,\n                           path= PREPROCESSED_TRAINING_IMAGE_FOLDER+str(row.image_id)+\".jpg\",\n                           labels=row['CategoryId'],\n                           annotations=row['EncodedPixels'],\n                           height=row['Height'], width=row['Width'],\n                           img_org_id = row.image_id)\n\n    def image_reference(self, image_id):\n        info = self.image_info[image_id]\n        return info['path'], [category_list[int(x)] for x in info['labels']]\n\n    def load_image(self, image_id):\n\n        return cv2.imread(self.image_info[image_id]['path'])\n\n    def load_mask(self, image_id):\n        info = self.image_info[image_id]\n\n        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)\n        labels = []\n        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n            sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)\n\n            annotation = [int(x) for x in annotation.split(' ')]\n\n            for i, start_pixel in enumerate(annotation[::2]):\n                sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1\n\n            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')\n            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)\n\n            mask[:, :, m] = sub_mask\n            labels.append(int(label)+1)\n        return mask, np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load dataset\n# Split with train = 80% samples and val = 10% and test = 10%\ntraining_percentage = 0.8\n\ntraining_set_size = int(training_percentage*len(training_df))\nvalidation_set_size = int((0.9-training_percentage)*len(training_df))\ntest_set_size = int((0.9-training_percentage)*len(training_df))\n\ntrain_dataset = DiagnosticDataset(training_df[:training_set_size])\ntrain_dataset.prepare()\n\nvalid_dataset = DiagnosticDataset(training_df[training_set_size:training_set_size+validation_set_size])\nvalid_dataset.prepare()\n\ntest_dataset = DiagnosticDataset(training_df[training_set_size + validation_set_size:])\ntest_dataset.prepare()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show test_dataset information\nprint(\"Images: {}\\nClasses: {}\".format(len(test_dataset.image_ids), test_dataset.class_names))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inspect training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_random_images(dataset):\n    # Load and display random samples\n    image_ids = np.random.choice(dataset.image_ids, 5)\n    for image_id in image_ids:\n        image = dataset.load_image(image_id)\n        mask, class_ids = dataset.load_mask(image_id)\n        visualize.display_top_masks(image, mask, class_ids, dataset.class_names, limit=8)\ndisplay_random_images(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_with_image(dataset, image_id = 95):\n#     image_id = np.random.choice(dataset.image_ids)\n\n    # Load and display\n    image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(\n            dataset, config, image_id, use_mini_mask=False)\n    log(\"molded_image\", image)\n    log(\"mask\", mask)\n    visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names,\n                                show_bbox=False)\ndisplay_with_image(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# AUGMENTATION IMPLEMENTATION\n# Image augmentation (light but constant)\naugmentation = iaa.Sequential([\n    iaa.OneOf([ ## geometric transform\n        iaa.Affine(\n            scale={\"x\": (0.98, 1.02), \"y\": (0.98, 1.04)},\n            translate_percent={\"x\": (-0.02, 0.02), \"y\": (-0.04, 0.04)},\n            rotate=(-2, 2),\n            shear=(-1, 1),\n        ),\n#         iaa.PiecewiseAffine(scale=(0.001, 0.025)),\n    ]),\n    iaa.OneOf([ ## brightness or contrast\n        iaa.Multiply((0.9, 1.1)),\n        iaa.ContrastNormalization((0.9, 1.1)),\n    ]),\n    iaa.OneOf([ ## blur or sharpen\n        iaa.GaussianBlur(sigma=(0.0, 0.1)),\n        iaa.Sharpen(alpha=(0.0, 0.1)),\n    ]),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ax(rows=1, cols=1, size=16):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    _, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image_augmentation(dataset, image_id = 95):\n    # Load the image multiple times to show augmentations\n    limit = 4\n    ax = get_ax(rows=2, cols=limit//2)\n    for i in range(limit):\n        image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(\n            dataset, config, image_id, use_mini_mask=False, augment=False, augmentation=augmentation)\n        visualize.display_instances(image, bbox, mask, class_ids,\n                                    dataset.class_names, ax=ax[i//2, i % 2],\n                                    show_mask=False, show_bbox=False)\ndisplay_image_augmentation(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trained Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Call model\nmodel = modellib.MaskRCNN(mode=\"inference\", model_dir=\"\",\n                              config=config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pretrained weight\nmodel_path = '../input/trained-weight-updated/mask_rcnn_diagnostic_0015.h5'\nmodel.load_weights(model_path, by_name=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## [Run Detection](https://github.com/matterport/Mask_RCNN/blob/master/samples/nucleus/inspect_nucleus_model.ipynb)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_detection(dataset):\n    image_id = random.choice(dataset.image_ids)\n    image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)\n    info = dataset.image_info[image_id]\n    print(\"image ID: {}.{} ({}) {}\".format(info[\"source\"], info[\"id\"], image_id, \n                                           dataset.image_reference(image_id)))\n    print(\"Original image shape: \", modellib.parse_image_meta(image_meta[np.newaxis,...])[\"original_image_shape\"][0])\n\n    # Run object detection\n    results = model.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1)\n\n    # Display results\n    r = results[0]\n    log(\"gt_class_id\", gt_class_id)\n    log(\"gt_bbox\", gt_bbox)\n    log(\"gt_mask\", gt_mask)\n\n    # Compute AP over range 0.5 to 0.95 and print it\n    utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,\n                           r['rois'], r['class_ids'], r['scores'], r['masks'],\n                           verbose=1)\n\n    visualize.display_differences(\n        image,\n        gt_bbox, gt_class_id, gt_mask,\n        r['rois'], r['class_ids'], r['scores'], r['masks'],\n        dataset.class_names, ax=get_ax(),\n        show_box=False, show_mask=False,\n        iou_threshold=0.5, score_threshold=0.5)\nfor _ in range(5):\n    run_detection(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_bbox(img_id , bbox_df = org_df, normalize = True):\n\n    img_ids = bbox_df['image_id'].values\n    class_ids = bbox_df['class_id'].unique()\n\n    label2color = {class_id:[randint(0,255) for i in range(3)] for class_id in class_ids}\n\n    plt.figure(figsize=(20,8))\n    sub_num =1\n\n    img_id = img_id\n\n    img_path = os.path.join(TRAIN_DIR, img_id + \".dicom\")\n    img = dicom2array(img_path)\n\n    if normalize:\n        # normalize\n        img = exposure.equalize_adapthist(img/np.max(img))\n        img = (img * 255).astype(np.uint8)\n\n    # convert from single-channel grayscale to 3-channel RGB\n    img = np.stack([img] * 3, axis=2)\n\n    # add bounding boxes\n    box_coordinates = bbox_df.loc[bbox_df['image_id'] == img_id, ['x_min', 'y_min', 'x_max', 'y_max']].values\n    labels = bbox_df.loc[bbox_df['image_id'] == img_id, ['class_id']].values.squeeze()\n    if not labels.shape:\n        labels = np.expand_dims(labels, axis =0)\n\n    for label_id, box in zip(labels, box_coordinates):\n        color = label2color[label_id]\n        img_bbox = cv2.rectangle(\n            img,\n            (int(box[0]), int(box[1])),\n            (int(box[2]), int(box[3])),\n            color = color, thickness= 8\n        )\n        # add labels\n        cv2.putText(img_bbox, str(label_id), (int(box[0]), int(box[1])), cv2.FONT_HERSHEY_SIMPLEX, 5, (36,255,12), 5)\n\n    plt.subplot(1,3,sub_num)\n    sub_num += 1\n    plt.imshow(img_bbox, cmap = 'gray')\n    plt.title('Finding contains in image')\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Display original\ndef display_test_result(dataset):\n    image_id = random.choice(dataset.image_ids)\n    \n#     print(image_id)\n    # Display original Dicom\n    \n    print('GT - Dicom')\n    plot_bbox(img_id = dataset.image_info[image_id]['img_org_id'])\n\n    # Display original in training form\n\n    original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\\\n        modellib.load_image_gt(dataset, config, \n                               image_id, use_mini_mask=False)\n    \n    print('GT')\n    visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, \n                                dataset.class_names, figsize=(5, 5))\n    \n    # Display test prediction\n\n    results = model.detect([original_image], verbose=0)\n    r = results[0]\n    \n    print('Predict')\n\n    visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], \n                                dataset.class_names, r['scores'], figsize=(5, 5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _ in range(5):\n    display_test_result(dataset = test_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_anomalies(dicom_image):\n\n    image_dimensions = dicom_image.shape\n\n    resized_img = cv2.resize(dicom_image, (IMAGE_SIZE,IMAGE_SIZE), interpolation = cv2.INTER_AREA)\n    saved_filename = TEMP_TEST_FOLDER+\"temp_image.jpg\"\n    \n    cv2.imwrite(saved_filename, resized_img) \n    \n    img = cv2.imread(saved_filename)\n\n    result = model.detect([img])\n    r = result[0]\n    \n    if r['masks'].size > 0:\n        y_scale = image_dimensions[0]/IMAGE_SIZE\n        x_scale = image_dimensions[1]/IMAGE_SIZE\n        rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int)\n        \n    else:\n        rois = r['rois']\n        \n    return rois, r['class_ids'], r['scores']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"selected_classes_dict = {}\nkey_value = 0\nfor _ in train_dataset.class_names:\n#     print(key_value)\n    selected_classes_dict[str(key_value)] = key_value \n    key_value += 1\n# selected_classes_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submission_results(visualize_mask = True, num_test = 5):\n    results = []\n    test_file_list = os.listdir(TEST_DIR)\n\n\n    for image_file_name in tqdm(test_file_list[:num_test]):\n\n        dicom_image = dicom2array(TEST_DIR + '/' + image_file_name)\n        image_dimensions = dicom_image.shape\n\n        # extracrt results\n        bbox_list, class_list, confidence_list = find_anomalies(dicom_image)\n\n        # convert from single-channel grayscale to 3-channel RGB\n        img = np.stack([dicom_image] * 3, axis=2)\n        resized_img = cv2.resize(img, (IMAGE_SIZE,IMAGE_SIZE), interpolation = cv2.INTER_AREA)\n\n        # visualize\n        result = model.detect([resized_img])\n        r = result[0]\n\n        if visualize_mask:\n            visualize.display_instances(resized_img, r['rois'], r['masks'], r['class_ids'], \n                                        train_dataset.class_names, r['scores'], show_mask = False,\n                                       figsize=(5,5))\n\n        # found abnormalities\n        prediction_string = \"\"\n\n        if len(bbox_list) > 0:\n\n            for bbox, class_id, confidence in zip(bbox_list, class_list, confidence_list):\n\n                # Convert to submission class id\n                for key, value in selected_classes_dict.items():\n                    if class_id == int(key):\n                        class_id_correct = value -1 # minus 1 because start from 0 for abnormal cases\n\n                confidence_score = str(round(confidence,3))\n\n                #organise the bbox into xmin, ymin, xmax, ymax\n                ymin = bbox[2]\n                ymax = bbox[0]\n                xmin = bbox[1]\n                xmax = bbox[3]\n                if xmin > xmax:\n                    xmin, xmax = xmax, xmin\n                if ymin > ymax:\n                    ymin, ymax = ymax, ymin\n\n                prediction_string += \"{} {} {} {} {} {} \".format(class_id_correct, confidence_score, xmin, ymin, xmax, ymax)\n\n            results.append({\"image_id\":image_file_name.replace(\".dicom\",\"\"), \"PredictionString\":prediction_string.strip()})\n\n        else:\n            results.append({\"image_id\":image_file_name.replace(\".dicom\",\"\"), \"PredictionString\":\"14 1.0 0 0 1 1\"})\n    submission_df = pd.DataFrame(results)\n    return submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_results(visualize_mask = True, num_test = 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = submission_results(visualize_mask = False, num_test = 3000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}