{"cells":[{"metadata":{},"cell_type":"markdown","source":"It is discovered that there is extreme foreground-background class imbalance problem in one-stage detector. And it is believed that this is the central cause which makes the performance of one-stage detectors inferior to two-stage detectors.\nIn RetinaNet, an one-stage detector, by using focal loss, lower loss is contributed by “easy” negative samples so that the loss is focusing on “hard” samples, which improves the prediction accuracy. With ResNet+FPN as backbone for feature extraction, plus two task-specific subnetworks for classification and bounding box regression, forming the RetinaNet, which achieves state-of-the-art performance, outperforms Faster R-CNN, the well-known two-stage detectors. It is a 2017 ICCV Best Student Paper Award paper with more than 500 citations. (The first author, Tsung-Yi Lin, has become Research Scientist at Google Brain when he was presenting RetinaNet in 2017 ICCV.) \n\nFor more info read this [article](https://towardsdatascience.com/review-retinanet-focal-loss-object-detection-38fba6afabe4)."},{"metadata":{},"cell_type":"markdown","source":"![](https://miro.medium.com/max/1010/1*0-GVAp6WCzPMR6puuaYQTQ.png)"},{"metadata":{},"cell_type":"markdown","source":"## Load Detectron2"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":false,"collapsed":true},"cell_type":"code","source":"# install dependencies: \n!pip install pyyaml==5.1\nimport torch, torchvision\nprint(torch.__version__, torch.cuda.is_available())\n!gcc --version\nimport torch\nassert torch.__version__.startswith(\"1.7\")\n!pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Some basic setup:\n# Setup detectron2 logger\nimport detectron2\nfrom detectron2.utils.logger import setup_logger\nsetup_logger()\n\n\n# import some common libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import  StratifiedShuffleSplit\nimport os, json, cv2, random\n\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.config import get_cfg\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2.data import MetadataCatalog, DatasetCatalog \n\nimport copy\nimport logging\nimport numpy as np\nfrom typing import Callable, List, Union\nimport torch\n\nfrom detectron2.config import configurable\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.data import detection_utils as utils\nfrom detectron2.data import transforms as T\n\nfrom detectron2.data import detection_utils as utils\nimport copy\nimport detectron2.data.transforms as T\nimport matplotlib.pyplot as plt\nfrom detectron2.data import DatasetMapper\nimport torch\nimport os\nimport numpy as np\n\nfrom detectron2.config import configurable","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"Dataset_Path = '../input/vinbigdata-chest-xray-abnormalities-detection'\n\ndf = pd.read_csv(f'{Dataset_Path}/train.csv')\ndf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['w'], df['h'] = df['x_max'] - df['x_min'], df['y_max'] - df['y_min']\ndf['area'] = df['w'] * df['h']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total Images: ', len(os.listdir(f'{Dataset_Path}/train')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Spliting Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)\ndf_dd = df.drop_duplicates('image_id')\ndf_dd = df_dd.reset_index()\nsss.get_n_splits(df_dd['image_id'], df_dd['class_id'])\nfor train_index, test_index in sss.split(df_dd['image_id'], df_dd['class_id']):\n    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n    X_train, X_test = df_dd['image_id'][train_index], df_dd['image_id'][test_index]\n    y_train, y_test = df_dd['class_id'][train_index], df_dd['class_id'][test_index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = df.drop_duplicates('class_id').sort_values('class_id')[['class_name']].values[:-1].ravel().tolist()\nprint(classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### DATASET PREPARING: CONERTING CSV TO DICTIONARY WHICH CONTAINS ANNOTATION AND IMAGE PATH ETC.\n\nfrom detectron2.structures import BoxMode\n\ndef chest_dicts(images, img_dir = '../input/vinbigdata-chest-xray-abnormalities-detection/train'):\n    \n    dataset_dicts = []\n    for idx, v in enumerate(images):\n        record = {}\n        \n        filename = os.path.join(img_dir, v + '.dicom')\n        \n        record[\"file_name\"] = filename\n        record[\"image_id\"] = idx\n        record[\"height\"] = 2500 # RANDOM Not Req\n        record[\"width\"] = 2500 # RANDOM Not Req\n      \n        annos = df[df.image_id == v]\n        objs = []\n        for _, anno in annos.iterrows():\n            if anno.class_id != 14:\n\n                obj = {\n                    \"bbox\": [int(anno.x_min), int(anno.y_min), int(anno.w), int(anno.h)],\n                    \"bbox_mode\": BoxMode.XYWH_ABS,\n                    \"category_id\": int(anno.class_id)\n                }\n                objs.append(obj)\n        record[\"annotations\"] = objs\n        dataset_dicts.append(record)\n    return dataset_dicts\ndef train():\n    return chest_dicts(X_train)\ndef val():\n    return chest_dicts(X_test)\nDatasetCatalog.register(\"chest_Train\", train)\nMetadataCatalog.get(\"chest_Train\").set(thing_classes=classes)\nDatasetCatalog.register(\"chest_Val\",val)\nMetadataCatalog.get(\"chest_Val\").set(thing_classes=classes)\nChest_metadata = MetadataCatalog.get(\"chest_Train\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dicts = DatasetCatalog.get(\"chest_Val\")\nfor d in random.sample(dataset_dicts, 3):\n    img = read_xray(d[\"file_name\"])\n    visualizer = Visualizer(img, metadata=Chest_metadata, scale=1.0)\n    out = visualizer.draw_dataset_dict(d)\n    plt.imshow(out.get_image(), cmap = 'gray')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset Mapper"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n##### AS THE DATASET CONTAIN DICOM IMAGES WE NEED DIFFERENT DATSETMAPPER.\n\nclass DatasetMapper:\n\n    @configurable\n    def __init__(\n        self,\n        is_train: bool,\n        *,\n        augmentations: List[Union[T.Augmentation, T.Transform]],\n        image_format: str = 'BGR',\n        use_instance_mask: bool = False,\n        use_keypoint: bool = False,\n        instance_mask_format = \"polygon\",\n        keypoint_hflip_indices = None,\n        precomputed_proposal_topk= None,\n        recompute_boxes: bool = False,\n    ):\n        \"\"\"\n        NOTE: this interface is experimental.\n\n        Args:\n            is_train: whether it's used in training or inference\n            augmentations: a list of augmentations or deterministic transforms to apply\n            image_format: an image format supported by :func:`detection_utils.read_image`.\n            use_instance_mask: whether to process instance segmentation annotations, if available\n            use_keypoint: whether to process keypoint annotations if available\n            instance_mask_format: one of \"polygon\" or \"bitmask\". Process instance segmentation\n                masks into this format.\n            keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices`\n            precomputed_proposal_topk: if given, will load pre-computed\n                proposals from dataset_dict and keep the top k proposals for each image.\n            recompute_boxes: whether to overwrite bounding box annotations\n                by computing tight bounding boxes from instance mask annotations.\n        \"\"\"\n\n        # fmt: off\n        self.is_train               = is_train\n        self.augmentations          = T.AugmentationList(  augmentations)\n        self.image_format           = image_format\n        \n        # fmt: on\n        logger = logging.getLogger(__name__)\n        mode = \"training\" if is_train else \"inference\"\n        logger.info(f\"[DatasetMapper] Augmentations used in {mode}: {augmentations}\")\n\n    @classmethod\n    def from_config(cls, cfg, is_train: bool = True):\n        augs = utils.build_augmentation(cfg, is_train)\n        if cfg.INPUT.CROP.ENABLED and is_train:\n            augs.insert(0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE))\n            recompute_boxes = cfg.MODEL.MASK_ON\n        else:\n            recompute_boxes = False\n\n\n        ret = {\n            \"is_train\": is_train,\n            \"augmentations\": augs,\n            \"image_format\": cfg.INPUT.FORMAT,\n            \"use_instance_mask\": cfg.MODEL.MASK_ON,\n            \"instance_mask_format\": cfg.INPUT.MASK_FORMAT,\n            \"use_keypoint\": cfg.MODEL.KEYPOINT_ON,\n            \"recompute_boxes\": recompute_boxes,\n        }\n\n        return ret\n\n    def __call__(self, dataset_dict):\n        # print(dataset_dict)\n        dataset_dict = copy.deepcopy(dataset_dict)\n        image = read_xray(dataset_dict[\"file_name\"])\n        \n        auginput = T.AugInput(image)\n        transform = self.augmentations(auginput)\n        image = np.expand_dims(auginput.image, axis=2).copy()\n        image = torch.from_numpy(image.transpose(2, 0, 1))\n        annos = [\n            utils.transform_instance_annotations(annotation, [transform], image.shape[1:])\n            for annotation in dataset_dict.pop(\"annotations\")\n        ]\n        return {\n        # create the format that the model expects\n        \"image\": image,\n        \"instances\": utils.annotations_to_instances(annos, image.shape[1:])\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#### WE ALSO NEED CUSTOM TRAINER TO TELL DETECTRON TO USE OUR CUSTOM DATASETMAPPER\nfrom detectron2.engine import DefaultTrainer\n\n\nclass Trainer(DefaultTrainer):\n    \n    @classmethod\n    def build_test_loader(cls, cfg, dataset_name):\n        return build_detection_test_loader(cfg,dataset_name , mapper=DatasetMapper(cfg, is_train = True))\n\n    @classmethod\n    def build_train_loader(cls, cfg):\n        return build_detection_train_loader(dataset =train(), mapper=DatasetMapper(cfg, is_train = True),aspect_ratio_grouping=False, total_batch_size = cfg.SOLVER.IMS_PER_BATCH)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Change Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.data import MetadataCatalog, build_detection_train_loader,build_detection_test_loader\nBatch = 10\nEpochs = 3\nsteps = 1000  #  ### INCREASE THE STEPS   (len(X_train) // Batch) * Epochs \ncfg = get_cfg()\nNAME = \"COCO-Detection/retinanet_R_50_FPN_1x.yaml\"\ncfg.merge_from_file(model_zoo.get_config_file(NAME))\ncfg.DATASETS.TRAIN = (\"chest_Train\",)\ncfg.DATASETS.TEST = ('chest_Val', )\ncfg.DATALOADER.NUM_WORKERS = 2\ncfg.MODEL.WEIGHTS =  model_zoo.get_checkpoint_url(NAME)\ncfg.SOLVER.IMS_PER_BATCH = Batch\ncfg.CUDNN_BENCHMARK =  True\ncfg.MODEL.RETINANET.NUM_CLASSES  = len(classes)\ncfg.SOLVER.BASE_LR = 0.00025\ncfg.SOLVER.LR_SCHEDULER_NAME = \"WarmupCosineLR\"\ncfg.SOLVER.MAX_ITER = steps  \ncfg.OUTPUT_DIR = './output'\ncfg.MODEL.PIXEL_MEAN = [103.530]\ncfg.MODEL.PIXEL_STD = [1.0]\ncfg.SOLVER.CHECKPOINT_PERIOD = 1000\ncfg.SOLVER.CLIP_GRADIENTS.CLIP_VALUE = 0.95\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n\ntrainer = Trainer(cfg) \ntrainer.resume_or_load(resume=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")  # path to the model we just trained\ncfg.MODEL.RETINANET.SCORE_THRESH_TEST = 0.1   # set a custom testing threshold\npredictor = DefaultPredictor(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.utils.visualizer import ColorMode\nfor d in random.sample(dataset_dicts, 3):    \n    im = read_xray(d[\"file_name\"])\n    im = np.expand_dims(im, axis=2)\n    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format\n    v = Visualizer(im[:, :, 0],\n                   metadata=Chest_metadata, \n                   scale=0.5,  \n    )\n    out = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n    plt.imshow(out.get_image()[:,:,::-1])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}