{"cells":[{"metadata":{},"cell_type":"markdown","source":"## VinBigData Chest X-ray\n\nTeam Name : Overfitters\n\n\nWe classify 14 types of thoracic abnormalities from chest radiographs using detectron model. \nWe use Fast RCNN-50"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualisation of VinBigData"},{"metadata":{},"cell_type":"markdown","source":"* Import the libraries for the visualisation of Dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport glob\nimport pydicom ### to conver dicom to png images\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut #\nimport cv2 ## OpenCV package\nfrom skimage import exposure \n### imports for the bbox section\nfrom PIL import Image\nfrom sklearn import preprocessing\nimport random\nfrom random import randint","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Read data using pandas."},{"metadata":{"trusted":true},"cell_type":"code","source":"## Columns \nprimary_dir = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/'\nsample_sub = pd.read_csv(primary_dir +'sample_submission.csv')\ntrain_csv = pd.read_csv(primary_dir +'train.csv')\ntest_csv = pd.read_csv(primary_dir +'train.csv')\n\nprint (train_csv.columns)\nprint (test_csv.columns)\n\nprint (sample_sub.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Total number of rows in traning images are :-', len((train_csv['image_id']))) \nprint ('Unique training images :- ', len(set(train_csv['image_id'])))\nprint ('Number of total radiologists:- ', len(set(train_csv['rad_id'])))\nprint ('Number of total classes of diagnosis', len(set(train_csv['class_name'])))\nprint (set(train_csv['class_name']))\nfig,ax = plt.subplots(1,2, figsize = (15, 6))\nax[0].hist(train_csv['rad_id'])\nax[1].hist(train_csv['class_id'])\n### \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The figure on the right side show the distribution of radiologists. The other figure show the classes of diagnosis."},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.express as px\nimport plotly\nimport plotly.express as px\nimport seaborn as sns\nimport plotly.graph_objects as go\n\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"coolwarm\", 15)]\n\n\nfig = go.Figure() ### PLOTLY graph object interesting usage \nint_2_str = {i:train_csv[train_csv[\"class_id\"]==i].iloc[0][\"class_name\"] for i in range(15)}\nstr_2_int = {v:k for k,v in int_2_str.items()}\nint_2_clr = {str_2_int[k]:LABEL_COLORS[i] for i,k in enumerate(sorted(str_2_int.keys()))}\n\nfor i in range(15):\n    fig.add_trace(go.Histogram(\n        y=train_csv[train_csv[\"class_id\"]==i][\"rad_id\"],\n        marker_color=int_2_clr[i],\n        name=f\"<b>{i, int_2_str[i]}</b>\", orientation='h'))\n\nfig.update_xaxes(categoryorder=\"total descending\")\nfig.update_layout(title=\"<b>Radiologsits annotation for each anamoly</b>\",\n                  barmode='stack',\n                  xaxis_title=\"<b>Radiologist ID</b>\",\n                  yaxis_title=\"<b>Number of Annotations Made</b>\",\n                  font=FIG_FONT)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**5-Conclusions from radiologists data skews**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox_area(row):\n    return (row['x_max']-row['x_min'])*(row['y_max']-row['y_min'])\ndef get_bbox_height(row):\n    return ((row['y_max']-row['y_min']))\ndef get_bbox_width(row):\n    return (row['x_max']-row['x_min'])\ndef get_bbox_aspect_ratio(row):\n    return ((row['x_max']-row['x_min'])/(row['y_max']-row['y_min']))\n\nle = preprocessing.LabelEncoder()  # encode rad_id\ntrain_csv['rad_label'] = le.fit_transform(train_csv['rad_id'])\n\ntrain_csv_good = train_csv[train_csv['class_id'] != 14]\ntrain_csv_good['bbox_area'] = train_csv_good.apply(get_bbox_area, axis=1)\ntrain_csv_good['bbox_height'] = train_csv_good.apply(get_bbox_height, axis=1)\ntrain_csv_good['bbox_width'] = train_csv_good.apply(get_bbox_width, axis=1)\ntrain_csv_good['bbox_aspect_ratio'] = train_csv_good.apply(get_bbox_aspect_ratio, axis=1)\n\ntrain_csv_good.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The table above show the bboxes of the images."},{"metadata":{},"cell_type":"markdown","source":"* Reference https://www.kaggle.com/trungthanhnguyen0502/eda-vinbigdata-chest-x-ray-abnormalities  for code below."},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=None): #(500,500)\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    #plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Visualize the X-ray images."},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndicom_paths = [primary_dir + '/train/'+train_csv['image_id'][x]+'.dicom' for x in range(4)]\nprint (dicom_paths)\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs, size=9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. * The visualization of anomalies on the images."},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nimg_ids = train_csv_good['image_id'].values\nclass_ids = train_csv_good['class_id'].unique()\n\n# map label_id to specify color\nlabel2color = {class_id:[randint(0,255) for i in range(3)] for class_id in class_ids}\nthickness = 3\nscale = 5\n\n\nfor i in range(8):\n    img_id = random.choice(img_ids)\n    img_path = primary_dir+f'/train/{img_id}.dicom'\n    img = dicom2array(path=img_path)\n    img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n    img = np.stack([img, img, img], axis=-1)\n    \n    boxes = train_csv_good.loc[train_csv_good['image_id'] == img_id, ['x_min', 'y_min', 'x_max', 'y_max']].values/scale\n    labels = train_csv_good.loc[train_csv_good['image_id'] == img_id, ['class_id']].values.squeeze()\n    \n    for label_id, box in zip(labels, boxes):\n        color = label2color[label_id]\n        img = cv2.rectangle(\n            img,\n            (int(box[0]), int(box[1])),\n            (int(box[2]), int(box[3])),\n            color, thickness\n    )\n        \n    img = cv2.resize(img, (500,500))\n    imgs.append(img)\n    \nplot_imgs(imgs, size = 9, cmap=None)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* **Discussion**: "},{"metadata":{},"cell_type":"markdown","source":"# ****Resize the images****"},{"metadata":{},"cell_type":"markdown","source":"The resize of the images step is too long and due to kaggle, we could not run the these code lines. As a result, we use the resized images. The image sizes are 256X256.\nReference : [Resized Images here](https://www.kaggle.com/xhlulu/vinbigdata-chest-xray-resized-png-256x256)"},{"metadata":{},"cell_type":"markdown","source":"# **Model Training and Prediction with Detectron**"},{"metadata":{},"cell_type":"markdown","source":"* Discussion :\nThe model that we prefer to use here is Fast RCNN and we exploit from detectron2"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\n\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.core.display import display, HTML\n\n# --- plotly ---\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.io as pio\npio.templates.default = \"plotly_dark\"\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\n\n# --- setup ---\npd.set_option('max_columns', 50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvcc --version","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\n\ntorch.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* install the detectron2."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install detectron2 -f \\\n  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Get data using the methods below."},{"metadata":{},"cell_type":"markdown","source":"# Model Training "},{"metadata":{},"cell_type":"markdown","source":"Import the libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nfrom pathlib import Path\nfrom typing import Optional\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom detectron2.structures import BoxMode\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom typing import Any, Union\n\nimport contextlib\nimport copy\nimport io\nimport itertools\nimport json\nimport logging\nimport numpy as np\nimport os\nimport pickle\nfrom collections import OrderedDict\nimport pycocotools.mask as mask_util\nimport torch\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval\nfrom tabulate import tabulate\nimport detectron2.data.transforms as T\nfrom detectron2.data import detection_utils as utils\nimport detectron2\nimport detectron2.utils.comm as comm\nfrom detectron2.config import CfgNode\nfrom detectron2.data import MetadataCatalog\nfrom detectron2.data.datasets.coco import convert_to_coco_json\nfrom detectron2.evaluation.evaluator import DatasetEvaluator\nfrom detectron2.evaluation.fast_eval_api import COCOeval_opt\nfrom detectron2.structures import Boxes, BoxMode, pairwise_iou\nfrom detectron2.utils.file_io import PathManager\nfrom detectron2.utils.logger import create_small_table\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer, launch\nfrom detectron2.evaluation import COCOEvaluator\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.utils.visualizer import Visualizer\nfrom detectron2 import model_zoo\nfrom detectron2.config import get_cfg\nfrom detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_test_loader\nfrom detectron2.engine import DefaultPredictor\nfrom detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.structures import BoxMode\nfrom detectron2.utils.logger import setup_logger\nfrom detectron2.utils.visualizer import ColorMode, Visualizer\nfrom detectron2.engine.hooks import HookBase\nfrom detectron2.utils.logger import log_every_n_seconds\nimport detectron2.utils.comm as comm\nfrom tqdm import tqdm\nfrom math import ceil\nfrom typing import Any, Dict, List\nimport argparse\nimport dataclasses\nimport json\nimport os\nimport pickle\nimport random\nimport sys\nfrom dataclasses import dataclass\nfrom distutils.util import strtobool\nfrom pathlib import Path\n\nimport cv2\n\nimport numpy as np\nfrom numpy import ndarray\nimport pandas as pd\n\nfrom tqdm import tqdm\nsetup_logger()\nimport yaml\n\nimport copy\nimport logging\n\n\nimport albumentations as A\nimport time\nimport datetime\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The helper functions for metadata catalog."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_vinbigdata_dicts(\n    imgdir: Path,\n    train_df: pd.DataFrame,\n    train_data_type: str = \"original\",\n    use_cache: bool = True,\n    debug: bool = True,\n    target_indices: Optional[np.ndarray] = None,\n    use_class14: bool = False,\n):\n    debug_str = f\"_debug{int(debug)}\"\n    train_data_type_str = f\"_{train_data_type}\"\n    class14_str = f\"_14class{int(use_class14)}\"\n    cache_path = Path(\".\") / f\"dataset_dicts_cache{train_data_type_str}{class14_str}{debug_str}.pkl\"\n    if not use_cache or not cache_path.exists():\n        print(\"Creating data...\")\n        train_meta = pd.read_csv(imgdir / \"train_meta.csv\")\n        if debug:\n            train_meta = train_meta.iloc[:500]  # For debug....\n\n        # Load 1 image to get image size.\n        image_id = train_meta.loc[0, \"image_id\"]\n        image_path = str(imgdir / \"train\" / f\"{image_id}.png\")\n        image = cv2.imread(image_path)\n        resized_height, resized_width, ch = image.shape\n        print(f\"image shape: {image.shape}\")\n\n        dataset_dicts = []\n        for index, train_meta_row in tqdm(train_meta.iterrows(), total=len(train_meta)):\n            record = {}\n\n            image_id, height, width = train_meta_row.values\n            filename = str(imgdir / \"train\" / f\"{image_id}.png\")\n            record[\"file_name\"] = filename\n            record[\"image_id\"] = image_id\n            record[\"height\"] = resized_height\n            record[\"width\"] = resized_width\n            objs = []\n            for index2, row in train_df.query(\"image_id == @image_id\").iterrows():\n                # print(row)\n                # print(row[\"class_name\"])\n                # class_name = row[\"class_name\"]\n                class_id = row[\"class_id\"]\n                if class_id == 14:\n                    # It is \"No finding\"\n                    if use_class14:\n                        # Use this No finding class with the bbox covering all image area.\n                        bbox_resized = [0, 0, resized_width, resized_height]\n                        obj = {\n                            \"bbox\": bbox_resized,\n                            \"bbox_mode\": BoxMode.XYXY_ABS,\n                            \"category_id\": class_id,\n                        }\n                        objs.append(obj)\n                    else:\n                        # This annotator does not find anything, skip.\n                        pass\n                else:\n                    # bbox_original = [int(row[\"x_min\"]), int(row[\"y_min\"]), int(row[\"x_max\"]), int(row[\"y_max\"])]\n                    h_ratio = resized_height / height\n                    w_ratio = resized_width / width\n                    bbox_resized = [\n                        float(row[\"x_min\"]) * w_ratio,\n                        float(row[\"y_min\"]) * h_ratio,\n                        float(row[\"x_max\"]) * w_ratio,\n                        float(row[\"y_max\"]) * h_ratio,\n                    ]\n                    obj = {\n                        \"bbox\": bbox_resized,\n                        \"bbox_mode\": BoxMode.XYXY_ABS,\n                        \"category_id\": class_id,\n                    }\n                    objs.append(obj)\n            record[\"annotations\"] = objs\n            dataset_dicts.append(record)\n        with open(cache_path, mode=\"wb\") as f:\n            pickle.dump(dataset_dicts, f)\n\n    print(f\"Load from cache {cache_path}\")\n    with open(cache_path, mode=\"rb\") as f:\n        dataset_dicts = pickle.load(f)\n    if target_indices is not None:\n        dataset_dicts = [dataset_dicts[i] for i in target_indices]\n    return dataset_dicts\n\n\ndef get_vinbigdata_dicts_test(\n    imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True,\n):\n    debug_str = f\"_debug{int(debug)}\"\n    cache_path = Path(\".\") / f\"dataset_dicts_cache_test{debug_str}.pkl\"\n    if not use_cache or not cache_path.exists():\n        print(\"Creating data...\")\n        # test_meta = pd.read_csv(imgdir / \"test_meta.csv\")\n        if debug:\n            test_meta = test_meta.iloc[:500]  # For debug....\n\n        # Load 1 image to get image size.\n        image_id = test_meta.loc[0, \"image_id\"]\n        image_path = str(imgdir / \"test\" / f\"{image_id}.png\")\n        image = cv2.imread(image_path)\n        resized_height, resized_width, ch = image.shape\n        print(f\"image shape: {image.shape}\")\n\n        dataset_dicts = []\n        for index, test_meta_row in tqdm(test_meta.iterrows(), total=len(test_meta)):\n            record = {}\n\n            image_id, height, width = test_meta_row.values\n            filename = str(imgdir / \"test\" / f\"{image_id}.png\")\n            record[\"file_name\"] = filename\n            # record[\"image_id\"] = index\n            record[\"image_id\"] = image_id\n            record[\"height\"] = resized_height\n            record[\"width\"] = resized_width\n            # objs = []\n            # record[\"annotations\"] = objs\n            dataset_dicts.append(record)\n        with open(cache_path, mode=\"wb\") as f:\n            pickle.dump(dataset_dicts, f)\n\n    print(f\"Load from cache {cache_path}\")\n    with open(cache_path, mode=\"rb\") as f:\n        dataset_dicts = pickle.load(f)\n    return dataset_dicts\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef save_yaml(filepath: Union[str, Path], content: Any, width: int = 120):\n    with open(filepath, \"w\") as f:\n        yaml.dump(content, f, width=width)\n\n\ndef load_yaml(filepath: Union[str, Path]) -> Any:\n    with open(filepath, \"r\") as f:\n        content = yaml.full_load(f)\n    return content\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Referenced:\n - https://detectron2.readthedocs.io/en/latest/tutorials/data_loading.html\n - https://www.kaggle.com/dhiiyaur/detectron-2-compare-models-augmentation/#data"},{"metadata":{},"cell_type":"markdown","source":"The augmentation for the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass MyMapper:\n    \"\"\"Mapper which uses `detectron2.data.transforms` augmentations\"\"\"\n\n    def __init__(self, cfg, is_train: bool = True):\n        aug_kwargs = cfg.aug_kwargs\n        aug_list = [\n            # T.Resize((800, 800)),\n        ]\n        if is_train:\n            aug_list.extend([getattr(T, name)(**kwargs) for name, kwargs in aug_kwargs.items()])\n        self.augmentations = T.AugmentationList(aug_list)\n        self.is_train = is_train\n\n        mode = \"training\" if is_train else \"inference\"\n        print(f\"[MyDatasetMapper] Augmentations used in {mode}: {self.augmentations}\")\n\n    def __call__(self, dataset_dict):\n        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below\n        image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n        aug_input = T.AugInput(image)\n        transforms = self.augmentations(aug_input)\n        image = aug_input.image\n\n\n        image_shape = image.shape[:2]  # h, w\n        dataset_dict[\"image\"] = torch.as_tensor(image.transpose(2, 0, 1).astype(\"float32\"))\n        annos = [\n            utils.transform_instance_annotations(obj, transforms, image_shape)\n            for obj in dataset_dict.pop(\"annotations\")\n            if obj.get(\"iscrowd\", 0) == 0\n        ]\n        instances = utils.annotations_to_instances(annos, image_shape)\n        dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n        return dataset_dict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nReferenced:\n - https://detectron2.readthedocs.io/en/latest/tutorials/data_loading.html\n - https://www.kaggle.com/dhiiyaur/detectron-2-compare-models-augmentation/#data"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AlbumentationsMapper:\n    \"\"\"Mapper which uses `albumentations` augmentations\"\"\"\n    def __init__(self, cfg, is_train: bool = True):\n        aug_kwargs = cfg.aug_kwargs\n        aug_list = [\n        ]\n        if is_train:\n            aug_list.extend([getattr(A, name)(**kwargs) for name, kwargs in aug_kwargs.items()])\n        self.transform = A.Compose(\n            aug_list, bbox_params=A.BboxParams(format=\"pascal_voc\", label_fields=[\"category_ids\"])\n        )\n        self.is_train = is_train\n\n        mode = \"training\" if is_train else \"inference\"\n        print(f\"[AlbumentationsMapper] Augmentations used in {mode}: {self.transform}\")\n\n    def __call__(self, dataset_dict):\n        dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below\n        image = utils.read_image(dataset_dict[\"file_name\"], format=\"BGR\")\n\n        prev_anno = dataset_dict[\"annotations\"]\n        bboxes = np.array([obj[\"bbox\"] for obj in prev_anno], dtype=np.float32)\n        # category_id = np.array([obj[\"category_id\"] for obj in dataset_dict[\"annotations\"]], dtype=np.int64)\n        category_id = np.arange(len(dataset_dict[\"annotations\"]))\n\n        transformed = self.transform(image=image, bboxes=bboxes, category_ids=category_id)\n        image = transformed[\"image\"]\n        annos = []\n        for i, j in enumerate(transformed[\"category_ids\"]):\n            d = prev_anno[j]\n            d[\"bbox\"] = transformed[\"bboxes\"][i]\n            annos.append(d)\n        dataset_dict.pop(\"annotations\", None)  # Remove unnecessary field.\n\n        image_shape = image.shape[:2]  # h, w\n        dataset_dict[\"image\"] = torch.as_tensor(image.transpose(2, 0, 1).astype(\"float32\"))\n        instances = utils.annotations_to_instances(annos, image_shape)\n        dataset_dict[\"instances\"] = utils.filter_empty_instances(instances)\n        return dataset_dict\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Original code from https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py\n* Just modified to show AP@40"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef vin_summarize(self):\n    '''\n    Compute and display summary metrics for evaluation results.\n    Note this functin can *only* be applied on the default parameter setting\n    '''\n\n    def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):\n        p = self.params\n        iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'\n        titleStr = 'Average Precision' if ap == 1 else 'Average Recall'\n        typeStr = '(AP)' if ap == 1 else '(AR)'\n        iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \\\n            if iouThr is None else '{:0.2f}'.format(iouThr)\n\n        aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]\n        mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]\n        if ap == 1:\n            # dimension of precision: [TxRxKxAxM]\n            s = self.eval['precision']\n            # IoU\n            if iouThr is not None:\n                t = np.where(iouThr == p.iouThrs)[0]\n                s = s[t]\n            s = s[:, :, :, aind, mind]\n        else:\n            # dimension of recall: [TxKxAxM]\n            s = self.eval['recall']\n            if iouThr is not None:\n                t = np.where(iouThr == p.iouThrs)[0]\n                s = s[t]\n            s = s[:, :, aind, mind]\n        if len(s[s > -1]) == 0:\n            mean_s = -1\n        else:\n            mean_s = np.mean(s[s > -1])\n        print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))\n        return mean_s\n\n    def _summarizeDets():\n        stats = np.zeros((12,))\n        stats[0] = _summarize(1)\n        stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])\n        # stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2])\n        stats[2] = _summarize(1, iouThr=.4, maxDets=self.params.maxDets[2])\n        stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2])\n        stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2])\n        stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2])\n        stats[6] = _summarize(0, maxDets=self.params.maxDets[0])\n        stats[7] = _summarize(0, maxDets=self.params.maxDets[1])\n        stats[8] = _summarize(0, maxDets=self.params.maxDets[2])\n        stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2])\n        stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2])\n        stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2])\n        return stats\n\n    def _summarizeKps():\n        stats = np.zeros((10,))\n        stats[0] = _summarize(1, maxDets=20)\n        stats[1] = _summarize(1, maxDets=20, iouThr=.5)\n        stats[2] = _summarize(1, maxDets=20, iouThr=.75)\n        stats[3] = _summarize(1, maxDets=20, areaRng='medium')\n        stats[4] = _summarize(1, maxDets=20, areaRng='large')\n        stats[5] = _summarize(0, maxDets=20)\n        stats[6] = _summarize(0, maxDets=20, iouThr=.5)\n        stats[7] = _summarize(0, maxDets=20, iouThr=.75)\n        stats[8] = _summarize(0, maxDets=20, areaRng='medium')\n        stats[9] = _summarize(0, maxDets=20, areaRng='large')\n        return stats\n\n    if not self.eval:\n        raise Exception('Please run accumulate() first')\n    iouType = self.params.iouType\n    if iouType == 'segm' or iouType == 'bbox':\n        summarize = _summarizeDets\n    elif iouType == 'keypoints':\n        summarize = _summarizeKps\n    self.stats = summarize()\n\n\nprint(\"HACKING: overriding COCOeval.summarize = vin_summarize...\")\nCOCOeval.summarize = vin_summarize\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*  COCO Evaluator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class VinbigdataEvaluator(DatasetEvaluator):\n    \"\"\"\n    Evaluate AR for object proposals, AP for instance detection/segmentation, AP\n    for keypoint detection outputs using COCO's metrics.\n    See http://cocodataset.org/#detection-eval and\n    http://cocodataset.org/#keypoints-eval to understand its metrics.\n\n    In addition to COCO, this evaluator is able to support any bounding box detection,\n    instance segmentation, or keypoint detection dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        dataset_name,\n        tasks=None,\n        distributed=True,\n        output_dir=None,\n        *,\n        use_fast_impl=True,\n        kpt_oks_sigmas=(),\n    ):\n        \n        self._logger = logging.getLogger(__name__)\n        self._distributed = distributed\n        self._output_dir = output_dir\n        self._use_fast_impl = use_fast_impl\n\n        if tasks is not None and isinstance(tasks, CfgNode):\n            kpt_oks_sigmas = (\n                tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas\n            )\n            self._logger.warn(\n                \"COCO Evaluator instantiated using config, this is deprecated behavior.\"\n                \" Please pass in explicit arguments instead.\"\n            )\n            self._tasks = None  # Infering it from predictions should be better\n        else:\n            self._tasks = tasks\n\n        self._cpu_device = torch.device(\"cpu\")\n\n        self._metadata = MetadataCatalog.get(dataset_name)\n        if not hasattr(self._metadata, \"json_file\"):\n            self._logger.info(\n                f\"'{dataset_name}' is not registered by `register_coco_instances`.\"\n                \" Therefore trying to convert it to COCO format ...\"\n            )\n\n            cache_path = os.path.join(output_dir, f\"{dataset_name}_coco_format.json\")\n            self._metadata.json_file = cache_path\n            convert_to_coco_json(dataset_name, cache_path)\n\n        json_file = PathManager.get_local_path(self._metadata.json_file)\n        with contextlib.redirect_stdout(io.StringIO()):\n            self._coco_api = COCO(json_file)\n\n        # Test set json files do not contain annotations (evaluation must be\n        # performed using the COCO evaluation server).\n        self._do_evaluation = \"annotations\" in self._coco_api.dataset\n        if self._do_evaluation:\n            self._kpt_oks_sigmas = kpt_oks_sigmas\n\n    def reset(self):\n        self._predictions = []\n\n    def process(self, inputs, outputs):\n        \"\"\"\n        Args:\n            inputs: the inputs to a COCO model (e.g., GeneralizedRCNN).\n                It is a list of dict. Each dict corresponds to an image and\n                contains keys like \"height\", \"width\", \"file_name\", \"image_id\".\n            outputs: the outputs of a COCO model. It is a list of dicts with key\n                \"instances\" that contains :class:`Instances`.\n        \"\"\"\n        for input, output in zip(inputs, outputs):\n            prediction = {\"image_id\": input[\"image_id\"]}\n\n            if \"instances\" in output:\n                instances = output[\"instances\"].to(self._cpu_device)\n                prediction[\"instances\"] = instances_to_coco_json(instances, input[\"image_id\"])\n            if \"proposals\" in output:\n                prediction[\"proposals\"] = output[\"proposals\"].to(self._cpu_device)\n            if len(prediction) > 1:\n                self._predictions.append(prediction)\n\n    def evaluate(self, img_ids=None):\n        \"\"\"\n        Args:\n            img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset\n        \"\"\"\n        if self._distributed:\n            comm.synchronize()\n            predictions = comm.gather(self._predictions, dst=0)\n            predictions = list(itertools.chain(*predictions))\n\n            if not comm.is_main_process():\n                return {}\n        else:\n            predictions = self._predictions\n\n        if len(predictions) == 0:\n            self._logger.warning(\"[VinbigdataEvaluator] Did not receive valid predictions.\")\n            return {}\n\n        if self._output_dir:\n            PathManager.mkdirs(self._output_dir)\n            file_path = os.path.join(self._output_dir, \"instances_predictions.pth\")\n            with PathManager.open(file_path, \"wb\") as f:\n                torch.save(predictions, f)\n\n        self._results = OrderedDict()\n        if \"proposals\" in predictions[0]:\n            self._eval_box_proposals(predictions)\n        if \"instances\" in predictions[0]:\n            self._eval_predictions(predictions, img_ids=img_ids)\n        # Copy so the caller can do whatever with results\n        return copy.deepcopy(self._results)\n\n    def _tasks_from_predictions(self, predictions):\n        \"\"\"\n        Get COCO API \"tasks\" (i.e. iou_type) from COCO-format predictions.\n        \"\"\"\n        tasks = {\"bbox\"}\n        for pred in predictions:\n            if \"segmentation\" in pred:\n                tasks.add(\"segm\")\n            if \"keypoints\" in pred:\n                tasks.add(\"keypoints\")\n        return sorted(tasks)\n\n    def _eval_predictions(self, predictions, img_ids=None):\n        \"\"\"\n        Evaluate predictions. Fill self._results with the metrics of the tasks.\n        \"\"\"\n        self._logger.info(\"Preparing results for COCO format ...\")\n        coco_results = list(itertools.chain(*[x[\"instances\"] for x in predictions]))\n        tasks = self._tasks or self._tasks_from_predictions(coco_results)\n\n        # unmap the category ids for COCO\n        if hasattr(self._metadata, \"thing_dataset_id_to_contiguous_id\"):\n            dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id\n            all_contiguous_ids = list(dataset_id_to_contiguous_id.values())\n            num_classes = len(all_contiguous_ids)\n            assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1\n\n            reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()}\n            for result in coco_results:\n                category_id = result[\"category_id\"]\n                assert category_id < num_classes, (\n                    f\"A prediction has class={category_id}, \"\n                    f\"but the dataset only has {num_classes} classes and \"\n                    f\"predicted class id should be in [0, {num_classes - 1}].\"\n                )\n                result[\"category_id\"] = reverse_id_mapping[category_id]\n\n        if self._output_dir:\n            file_path = os.path.join(self._output_dir, \"coco_instances_results.json\")\n            self._logger.info(\"Saving results to {}\".format(file_path))\n            with PathManager.open(file_path, \"w\") as f:\n                f.write(json.dumps(coco_results))\n                f.flush()\n\n        if not self._do_evaluation:\n            self._logger.info(\"Annotations are not available for evaluation.\")\n            return\n\n        self._logger.info(\n            \"Evaluating predictions with {} COCO API...\".format(\n                \"unofficial\" if self._use_fast_impl else \"official\"\n            )\n        )\n        for task in sorted(tasks):\n            coco_eval = (\n                _evaluate_predictions_on_coco(\n                    self._coco_api,\n                    coco_results,\n                    task,\n                    kpt_oks_sigmas=self._kpt_oks_sigmas,\n                    use_fast_impl=self._use_fast_impl,\n                    img_ids=img_ids,\n                )\n                if len(coco_results) > 0\n                else None  # cocoapi does not handle empty results very well\n            )\n\n            res = self._derive_coco_results(\n                coco_eval, task, class_names=self._metadata.get(\"thing_classes\")\n            )\n            self._results[task] = res\n\n    def _eval_box_proposals(self, predictions):\n        \"\"\"\n        Evaluate the box proposals in predictions.\n        Fill self._results with the metrics for \"box_proposals\" task.\n        \"\"\"\n        if self._output_dir:\n            # Saving generated box proposals to file.\n            # Predicted box_proposals are in XYXY_ABS mode.\n            bbox_mode = BoxMode.XYXY_ABS.value\n            ids, boxes, objectness_logits = [], [], []\n            for prediction in predictions:\n                ids.append(prediction[\"image_id\"])\n                boxes.append(prediction[\"proposals\"].proposal_boxes.tensor.numpy())\n                objectness_logits.append(prediction[\"proposals\"].objectness_logits.numpy())\n\n            proposal_data = {\n                \"boxes\": boxes,\n                \"objectness_logits\": objectness_logits,\n                \"ids\": ids,\n                \"bbox_mode\": bbox_mode,\n            }\n            with PathManager.open(os.path.join(self._output_dir, \"box_proposals.pkl\"), \"wb\") as f:\n                pickle.dump(proposal_data, f)\n\n        if not self._do_evaluation:\n            self._logger.info(\"Annotations are not available for evaluation.\")\n            return\n\n        self._logger.info(\"Evaluating bbox proposals ...\")\n        res = {}\n        areas = {\"all\": \"\", \"small\": \"s\", \"medium\": \"m\", \"large\": \"l\"}\n        for limit in [100, 1000]:\n            for area, suffix in areas.items():\n                stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit)\n                key = \"AR{}@{:d}\".format(suffix, limit)\n                res[key] = float(stats[\"ar\"].item() * 100)\n        self._logger.info(\"Proposal metrics: \\n\" + create_small_table(res))\n        self._results[\"box_proposals\"] = res\n\n    def _derive_coco_results(self, coco_eval, iou_type, class_names=None):\n        \"\"\"\n        Derive the desired score numbers from summarized COCOeval.\n\n        \"\"\"\n\n        metrics = {\n            \"bbox\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n            \"segm\": [\"AP\", \"AP50\", \"AP75\", \"APs\", \"APm\", \"APl\"],\n            \"keypoints\": [\"AP\", \"AP50\", \"AP75\", \"APm\", \"APl\"],\n        }[iou_type]\n\n        if coco_eval is None:\n            self._logger.warn(\"No predictions from the model!\")\n            return {metric: float(\"nan\") for metric in metrics}\n\n        # the standard metrics\n        results = {\n            metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else \"nan\")\n            for idx, metric in enumerate(metrics)\n        }\n        self._logger.info(\n            \"Evaluation results for {}: \\n\".format(iou_type) + create_small_table(results)\n        )\n        if not np.isfinite(sum(results.values())):\n            self._logger.info(\"Some metrics cannot be computed and is shown as NaN.\")\n\n        if class_names is None or len(class_names) <= 1:\n            return results\n      \n        precisions = coco_eval.eval[\"precision\"]\n        # precision has dims (iou, recall, cls, area range, max dets)\n        assert len(class_names) == precisions.shape[2]\n\n        results_per_category = []\n        for idx, name in enumerate(class_names):\n            # area range index 0: all area ranges\n\n            precision = precisions[:, :, idx, 0, -1]\n            precision = precision[precision > -1]\n            ap = np.mean(precision) if precision.size else float(\"nan\")\n            results_per_category.append((\"{}\".format(name), float(ap * 100)))\n\n        # tabulate it\n        N_COLS = min(6, len(results_per_category) * 2)\n        results_flatten = list(itertools.chain(*results_per_category))\n        results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)])\n        table = tabulate(\n            results_2d,\n            tablefmt=\"pipe\",\n            floatfmt=\".3f\",\n            headers=[\"category\", \"AP\"] * (N_COLS // 2),\n            numalign=\"left\",\n        )\n        self._logger.info(\"Per-category {} AP: \\n\".format(iou_type) + table)\n\n        results.update({\"AP-\" + name: ap for name, ap in results_per_category})\n        return results\n\n\ndef instances_to_coco_json(instances, img_id):\n    \"\"\"\n    Dump an \"Instances\" object to a COCO-format json that's used for evaluation.\n\n    \"\"\"\n    num_instance = len(instances)\n    if num_instance == 0:\n        return []\n\n    boxes = instances.pred_boxes.tensor.numpy()\n    boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS)\n    boxes = boxes.tolist()\n    scores = instances.scores.tolist()\n    classes = instances.pred_classes.tolist()\n\n    has_mask = instances.has(\"pred_masks\")\n    if has_mask:\n        # use RLE to encode the masks, because they are too large and takes memory\n        rles = [\n            mask_util.encode(np.array(mask[:, :, None], order=\"F\", dtype=\"uint8\"))[0]\n            for mask in instances.pred_masks\n        ]\n        for rle in rles:\n\n            rle[\"counts\"] = rle[\"counts\"].decode(\"utf-8\")\n\n    has_keypoints = instances.has(\"pred_keypoints\")\n    if has_keypoints:\n        keypoints = instances.pred_keypoints\n\n    results = []\n    for k in range(num_instance):\n        result = {\n            \"image_id\": img_id,\n            \"category_id\": classes[k],\n            \"bbox\": boxes[k],\n            \"score\": scores[k],\n        }\n        if has_mask:\n            result[\"segmentation\"] = rles[k]\n        if has_keypoints:\n            keypoints[k][:, :2] -= 0.5\n            result[\"keypoints\"] = keypoints[k].flatten().tolist()\n        results.append(result)\n    return results\n\n\n# inspired from Detectron:\n# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa\ndef _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area=\"all\", limit=None):\n    \"\"\"\n    Evaluate detection proposal recall metrics. This function is a much\n    faster alternative to the official COCO API recall evaluation code. However,\n    it produces slightly different results.\n    \"\"\"\n    # Record max overlap value for each gt box\n    # Return vector of overlap values\n    areas = {\n        \"all\": 0,\n        \"small\": 1,\n        \"medium\": 2,\n        \"large\": 3,\n        \"96-128\": 4,\n        \"128-256\": 5,\n        \"256-512\": 6,\n        \"512-inf\": 7,\n    }\n    area_ranges = [\n        [0 ** 2, 1e5 ** 2],  # all\n        [0 ** 2, 32 ** 2],  # small\n        [32 ** 2, 96 ** 2],  # medium\n        [96 ** 2, 1e5 ** 2],  # large\n        [96 ** 2, 128 ** 2],  # 96-128\n        [128 ** 2, 256 ** 2],  # 128-256\n        [256 ** 2, 512 ** 2],  # 256-512\n        [512 ** 2, 1e5 ** 2],\n    ]  # 512-inf\n    assert area in areas, \"Unknown area range: {}\".format(area)\n    area_range = area_ranges[areas[area]]\n    gt_overlaps = []\n    num_pos = 0\n\n    for prediction_dict in dataset_predictions:\n        predictions = prediction_dict[\"proposals\"]\n\n        # sort predictions in descending order\n        # TODO maybe remove this and make it explicit in the documentation\n        inds = predictions.objectness_logits.sort(descending=True)[1]\n        predictions = predictions[inds]\n\n        ann_ids = coco_api.getAnnIds(imgIds=prediction_dict[\"image_id\"])\n        anno = coco_api.loadAnns(ann_ids)\n        gt_boxes = [\n            BoxMode.convert(obj[\"bbox\"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)\n            for obj in anno\n            if obj[\"iscrowd\"] == 0\n        ]\n        gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4)  # guard against no boxes\n        gt_boxes = Boxes(gt_boxes)\n        gt_areas = torch.as_tensor([obj[\"area\"] for obj in anno if obj[\"iscrowd\"] == 0])\n\n        if len(gt_boxes) == 0 or len(predictions) == 0:\n            continue\n\n        valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])\n        gt_boxes = gt_boxes[valid_gt_inds]\n\n        num_pos += len(gt_boxes)\n\n        if len(gt_boxes) == 0:\n            continue\n\n        if limit is not None and len(predictions) > limit:\n            predictions = predictions[:limit]\n\n        overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes)\n\n        _gt_overlaps = torch.zeros(len(gt_boxes))\n        for j in range(min(len(predictions), len(gt_boxes))):\n            # find which proposal box maximally covers each gt box\n            # and get the iou amount of coverage for each gt box\n            max_overlaps, argmax_overlaps = overlaps.max(dim=0)\n\n            # find which gt box is 'best' covered (i.e. 'best' = most iou)\n            gt_ovr, gt_ind = max_overlaps.max(dim=0)\n            assert gt_ovr >= 0\n            # find the proposal box that covers the best covered gt box\n            box_ind = argmax_overlaps[gt_ind]\n            # record the iou coverage of this gt box\n            _gt_overlaps[j] = overlaps[box_ind, gt_ind]\n            assert _gt_overlaps[j] == gt_ovr\n            # mark the proposal box and the gt box as used\n            overlaps[box_ind, :] = -1\n            overlaps[:, gt_ind] = -1\n\n        # append recorded iou coverage level\n        gt_overlaps.append(_gt_overlaps)\n    gt_overlaps = (\n        torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32)\n    )\n    gt_overlaps, _ = torch.sort(gt_overlaps)\n\n    if thresholds is None:\n        step = 0.05\n        # thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)\n        thresholds = torch.arange(0.4, 0.95 + 1e-5, step, dtype=torch.float32)\n    recalls = torch.zeros_like(thresholds)\n    # compute recall for each iou threshold\n    for i, t in enumerate(thresholds):\n        recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)\n    # ar = 2 * np.trapz(recalls, thresholds)\n    ar = recalls.mean()\n    return {\n        \"ar\": ar,\n        \"recalls\": recalls,\n        \"thresholds\": thresholds,\n        \"gt_overlaps\": gt_overlaps,\n        \"num_pos\": num_pos,\n    }\n\n\ndef _evaluate_predictions_on_coco(\n    coco_gt, coco_results, iou_type, kpt_oks_sigmas=None, use_fast_impl=True, img_ids=None\n):\n    \"\"\"\n    Evaluate the coco results using COCOEval API.\n    \"\"\"\n    assert len(coco_results) > 0\n\n    if iou_type == \"segm\":\n        coco_results = copy.deepcopy(coco_results)\n        # When evaluating mask AP, if the results contain bbox, cocoapi will\n        # use the box area as the area of the instance, instead of the mask area.\n        # This leads to a different definition of small/medium/large.\n        # We remove the bbox field to let mask AP use mask area.\n        for c in coco_results:\n            c.pop(\"bbox\", None)\n\n    coco_dt = coco_gt.loadRes(coco_results)\n    coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type)\n\n    # HACKING: overwrite iouThrs to calc ious 0.4\n    coco_eval.params.iouThrs = np.linspace(\n        .4, 0.95, int(np.round((0.95 - .4) / .05)) + 1, endpoint=True)\n\n    if img_ids is not None:\n        coco_eval.params.imgIds = img_ids\n\n    if iou_type == \"keypoints\":\n        # Use the COCO default keypoint OKS sigmas unless overrides are specified\n        if kpt_oks_sigmas:\n            assert hasattr(coco_eval.params, \"kpt_oks_sigmas\"), \"pycocotools is too old!\"\n            coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas)\n        # COCOAPI requires every detection and every gt to have keypoints, so\n        # we just take the first entry from both\n        num_keypoints_dt = len(coco_results[0][\"keypoints\"]) // 3\n        num_keypoints_gt = len(next(iter(coco_gt.anns.values()))[\"keypoints\"]) // 3\n        num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas)\n        assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, (\n            f\"[VinbigdataEvaluator] Prediction contain {num_keypoints_dt} keypoints. \"\n            f\"Ground truth contains {num_keypoints_gt} keypoints. \"\n            f\"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. \"\n            \"They have to agree with each other. For meaning of OKS, please refer to \"\n            \"http://cocodataset.org/#keypoints-eval.\"\n        )\n\n    coco_eval.evaluate()\n    coco_eval.accumulate()\n    coco_eval.summarize()\n\n    return coco_eval\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* To calculate & record validation loss\n\n* Original code from https://medium.com/@apofeniaco/training-on-detectron2-with-a-validation-set-and-plot-loss-on-it-to-avoid-overfitting-6449418fbf4e\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class LossEvalHook(HookBase):\n    def __init__(self, eval_period, model, data_loader):\n        self._model = model\n        self._period = eval_period\n        self._data_loader = data_loader\n\n    def _do_loss_eval(self):\n        # Copying inference_on_dataset from evaluator.py\n        total = len(self._data_loader)\n        num_warmup = min(5, total - 1)\n\n        start_time = time.perf_counter()\n        total_compute_time = 0\n        losses = []\n        for idx, inputs in enumerate(self._data_loader):\n            if idx == num_warmup:\n                start_time = time.perf_counter()\n                total_compute_time = 0\n            start_compute_time = time.perf_counter()\n            if torch.cuda.is_available():\n                torch.cuda.synchronize()\n            total_compute_time += time.perf_counter() - start_compute_time\n            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)\n            seconds_per_img = total_compute_time / iters_after_start\n            if idx >= num_warmup * 2 or seconds_per_img > 5:\n                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start\n                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))\n                log_every_n_seconds(\n                    logging.INFO,\n                    \"Loss on Validation  done {}/{}. {:.4f} s / img. ETA={}\".format(\n                        idx + 1, total, seconds_per_img, str(eta)\n                    ),\n                    n=5,\n                )\n            loss_batch = self._get_loss(inputs)\n            losses.append(loss_batch)\n        mean_loss = np.mean(losses)\n        \n        comm.synchronize()\n\n        return mean_loss\n\n    def _get_loss(self, data):\n        # How loss is calculated on train_loop\n        metrics_dict = self._model(data)\n        metrics_dict = {\n            k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v)\n            for k, v in metrics_dict.items()\n        }\n        total_losses_reduced = sum(loss for loss in metrics_dict.values())\n        return total_losses_reduced\n\n    def after_step(self):\n        next_iter = int(self.trainer.iter) + 1\n        is_final = next_iter == self.trainer.max_iter\n        if is_final or (self._period > 0 and next_iter % self._period == 0):\n            mean_loss = self._do_loss_eval()\n            self.trainer.storage.put_scalars(validation_loss=mean_loss)\n            print(\"validation do loss eval\", mean_loss)\n        else:\n            pass\n          \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Definition of custom trainer of detectron2"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nfrom detectron2.data import build_detection_test_loader, build_detection_train_loader\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer, launch\n\n# from detectron2.evaluation import COCOEvaluator, PascalVOCDetectionEvaluator\n\n\nclass MyTrainer(DefaultTrainer):\n    @classmethod\n    def build_train_loader(cls, cfg, sampler=None):\n        return build_detection_train_loader(\n            cfg, mapper=AlbumentationsMapper(cfg, True), sampler=sampler\n        )\n\n    @classmethod\n    def build_test_loader(cls, cfg, dataset_name):\n        return build_detection_test_loader(\n            cfg, dataset_name, mapper=AlbumentationsMapper(cfg, False)\n        )\n\n    @classmethod\n    def build_evaluator(cls, cfg, dataset_name, output_folder=None):\n        if output_folder is None:\n            output_folder = os.path.join(cfg.OUTPUT_DIR, \"inference\")\n\n        return VinbigdataEvaluator(dataset_name, (\"bbox\",), False, output_dir=output_folder)\n\n    def build_hooks(self):\n        hooks = super(MyTrainer, self).build_hooks()\n        cfg = self.cfg\n        if len(cfg.DATASETS.TEST) > 0:\n            loss_eval_hook = LossEvalHook(\n                cfg.TEST.EVAL_PERIOD,\n                self.model,\n                MyTrainer.build_test_loader(cfg, cfg.DATASETS.TEST[0]),\n            )\n            hooks.insert(-1, loss_eval_hook)\n\n        return hooks\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Flags class consists of the all parameters of model for training phase.\n1. The Learning rate scheduler here: \"WarmupMultiStepLR\""},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- flags ---\nfrom dataclasses import dataclass, field\nfrom typing import Dict\n\n\n@dataclass\nclass Flags:\n\n    debug: bool = True\n    outdir: str = \"results/det\"\n\n    imgdir_name: str = \"vinbigdata-chest-xray-resized-png-256x256\"\n    split_mode: str = \"all_train\"  \n    seed: int = 111\n    train_data_type: str = \"original\" \n    use_class14: bool = False\n   \n    iter: int = 10000\n    ims_per_batch: int = 2 \n    num_workers: int = 4\n    lr_scheduler_name: str = \"WarmupMultiStepLR\" \n    base_lr: float = 0.00025\n    roi_batch_size_per_image: int = 512\n    eval_period: int = 10000\n    aug_kwargs: Dict = field(default_factory=lambda: {})\n\n    def update(self, param_dict: Dict) -> \"Flags\":\n        # Overwrite by `param_dict`\n        for key, value in param_dict.items():\n            if not hasattr(self, key):\n                raise ValueError(f\"[ERROR] Unexpected key for flag = {key}\")\n            setattr(self, key, value)\n        return self","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"flags_dict = {\n    \"debug\": False,\n    \"outdir\": \"results/v9\", \n    \"imgdir_name\": \"vinbigdata-chest-xray-resized-png-256x256\",\n    \"split_mode\": \"valid20\",\n    \"iter\": 30000,\n    \"roi_batch_size_per_image\": 512,\n    \"eval_period\": 1000,\n    \"lr_scheduler_name\": \"WarmupCosineLR\",\n    \"base_lr\": 0.001,\n    \"num_workers\": 4,\n    \"aug_kwargs\": {\n        \"HorizontalFlip\": {\"p\": 0.5},\n        \"ShiftScaleRotate\": {\"scale_limit\": 0.15, \"rotate_limit\": 10, \"p\": 0.5},\n        \"RandomBrightnessContrast\": {\"p\": 0.5}\n    }\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- configs ---\nthing_classes = [\n    \"Aortic enlargement\",\n    \"Atelectasis\",\n    \"Calcification\",\n    \"Cardiomegaly\",\n    \"Consolidation\",\n    \"ILD\",\n    \"Infiltration\",\n    \"Lung Opacity\",\n    \"Nodule/Mass\",\n    \"Other lesion\",\n    \"Pleural effusion\",\n    \"Pleural thickening\",\n    \"Pneumothorax\",\n    \"Pulmonary fibrosis\"\n]\ncategory_name_to_id = {class_name: index for index, class_name in enumerate(thing_classes)}\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The image directory refers to resized images with  256 X 256"},{"metadata":{"trusted":true},"cell_type":"code","source":"# args = parse()\nprint(\"torch\", torch.__version__)\nflags = Flags().update(flags_dict)\nprint(\"flags\", flags)\ndebug = flags.debug\noutdir = Path(flags.outdir)\nos.makedirs(str(outdir), exist_ok=True)\nflags_dict = dataclasses.asdict(flags)\nsave_yaml(outdir / \"flags.yaml\", flags_dict)\n\n# --- Read data ---\ninputdir = Path(\"/kaggle/input\")\ndatadir = inputdir / \"vinbigdata-chest-xray-abnormalities-detection\"\n\nimgdir = inputdir / flags.imgdir_name\n\n# Read in the data CSV files\ntrain_df = pd.read_csv(datadir / \"train.csv\")\ntrain = train_df  # alias\nsample_submission = pd.read_csv(datadir / 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Register the training data to MetaDataCatalog and DatasetCatalog"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"train_data_type = flags.train_data_type\nif flags.use_class14:\n    thing_classes.append(\"No finding\")\n\nsplit_mode = flags.split_mode\nif split_mode == \"all_train\":\n    DatasetCatalog.register(\n        \"vinbigdata_train\",\n        lambda: get_vinbigdata_dicts(\n            imgdir, train_df, train_data_type, debug=debug, use_class14=flags.use_class14\n        ),\n    )\n    MetadataCatalog.get(\"vinbigdata_train\").set(thing_classes=thing_classes)\nelif split_mode == \"valid20\":\n    # To get number of data...\n    n_dataset = len(\n        get_vinbigdata_dicts(\n            imgdir, train_df, train_data_type, debug=debug, use_class14=flags.use_class14\n        )\n    )\n    n_train = int(n_dataset * 0.8)\n    print(\"n_dataset\", n_dataset, \"n_train\", n_train)\n    rs = np.random.RandomState(flags.seed)\n    inds = rs.permutation(n_dataset)\n    train_inds, valid_inds = inds[:n_train], inds[n_train:]\n    DatasetCatalog.register(\n        \"vinbigdata_train\",\n        lambda: get_vinbigdata_dicts(\n            imgdir,\n            train_df,\n            train_data_type,\n            debug=debug,\n            target_indices=train_inds,\n            use_class14=flags.use_class14,\n        ),\n    )\n    MetadataCatalog.get(\"vinbigdata_train\").set(thing_classes=thing_classes)\n    DatasetCatalog.register(\n        \"vinbigdata_valid\",\n        lambda: get_vinbigdata_dicts(\n            imgdir,\n            train_df,\n            train_data_type,\n            debug=debug,\n            target_indices=valid_inds,\n            use_class14=flags.use_class14,\n        ),\n    )\n    MetadataCatalog.get(\"vinbigdata_valid\").set(thing_classes=thing_classes)\nelse:\n    raise ValueError(f\"[ERROR] Unexpected value split_mode={split_mode}\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"dataset_dicts = get_vinbigdata_dicts(imgdir, train, debug=debug)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\n\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.core.display import display, HTML\n\n# --- plotly ---\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.io as pio\npio.templates.default = \"plotly_dark\"\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\n\n# --- setup ---\npd.set_option('max_columns', 50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize data...\nanomaly_image_ids = train.query(\"class_id != 14\")[\"image_id\"].unique()\ntrain_meta = pd.read_csv(imgdir/\"train_meta.csv\")\nanomaly_inds = np.argwhere(train_meta[\"image_id\"].isin(anomaly_image_ids).values)[:, 0]\n\nvinbigdata_metadata = MetadataCatalog.get(\"vinbigdata_train\")\n\ncols = 3\nrows = 3\nfig, axes = plt.subplots(rows, cols, figsize=(18, 18))\naxes = axes.flatten()\n\nfor index, anom_ind in enumerate(anomaly_inds[:cols * rows]):\n    ax = axes[index]\n    # print(anom_ind)\n    d = dataset_dicts[anom_ind]\n    img = cv2.imread(d[\"file_name\"])\n    visualizer = Visualizer(img[:, :, ::-1], metadata=vinbigdata_metadata, scale=0.5)\n    out = visualizer.draw_dataset_dict(d)\n\n    ax.imshow(out.get_image()[:, :, ::-1])\n    ax.set_title(f\"{anom_ind}: image_id {anomaly_image_ids[index]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The visulisation of dataset above gives an overview regarding annomalies on X-ray images."},{"metadata":{},"cell_type":"markdown","source":"# Training "},{"metadata":{},"cell_type":"markdown","source":"* We use the detectron2 to detect the anomalies."},{"metadata":{"trusted":true},"cell_type":"code","source":"from detectron2.config.config import CfgNode as CN\n\ncfg = get_cfg()\ncfg.aug_kwargs = CN(flags.aug_kwargs)  # pass aug_kwargs to cfg\n\noriginal_output_dir = cfg.OUTPUT_DIR\ncfg.OUTPUT_DIR = str(outdir)\nprint(f\"cfg.OUTPUT_DIR {original_output_dir} -> {cfg.OUTPUT_DIR}\")\n\nconfig_name = \"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"\ncfg.merge_from_file(model_zoo.get_config_file(config_name))\ncfg.DATASETS.TRAIN = (\"vinbigdata_train\",)\nif split_mode == \"all_train\":\n    cfg.DATASETS.TEST = ()\nelse:\n    cfg.DATASETS.TEST = (\"vinbigdata_valid\",)\n    cfg.TEST.EVAL_PERIOD = flags.eval_period\n\ncfg.DATALOADER.NUM_WORKERS = flags.num_workers\n\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(config_name)\ncfg.SOLVER.IMS_PER_BATCH = flags.ims_per_batch\ncfg.SOLVER.LR_SCHEDULER_NAME = flags.lr_scheduler_name\ncfg.SOLVER.BASE_LR = flags.base_lr  \ncfg.SOLVER.MAX_ITER = flags.iter\ncfg.SOLVER.CHECKPOINT_PERIOD = 100000 \ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = flags.roi_batch_size_per_image\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = len(thing_classes)\n\n\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train the  model using custom trainer."},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = MyTrainer(cfg)\ntrainer.resume_or_load(resume=False)\ntrainer.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Calculate the metrics of trained model."},{"metadata":{"trusted":true},"cell_type":"code","source":"metrics_df = pd.read_json(outdir / \"metrics.json\", orient=\"records\", lines=True)\nmdf = metrics_df.sort_values(\"iteration\")\nmdf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 1. Loss curve\nfig, ax = plt.subplots()\n\nmdf1 = mdf[~mdf[\"total_loss\"].isna()]\nax.plot(mdf1[\"iteration\"], mdf1[\"total_loss\"], c=\"C0\", label=\"train\")\nif \"validation_loss\" in mdf.columns:\n    mdf2 = mdf[~mdf[\"validation_loss\"].isna()]\n    ax.plot(mdf2[\"iteration\"], mdf2[\"validation_loss\"], c=\"C1\", label=\"validation\")\n\n# ax.set_ylim([0, 0.5])\nax.legend()\nax.set_title(\"Loss curve\")\nplt.show()\nplt.savefig(outdir/\"loss.png\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Methods for prediction of test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_pred(labels: ndarray, boxes: ndarray, scores: ndarray) -> str:\n    pred_strings = []\n    for label, score, bbox in zip(labels, scores, boxes):\n        xmin, ymin, xmax, ymax = bbox.astype(np.int64)\n        pred_strings.append(f\"{label} {score} {xmin} {ymin} {xmax} {ymax}\")\n    return \" \".join(pred_strings)\n\n\ndef predict_batch(predictor: DefaultPredictor, im_list: List[ndarray]) -> List:\n    with torch.no_grad():  # https://github.com/sphinx-doc/sphinx/issues/4258\n        inputs_list = []\n        for original_image in im_list:\n            # Apply pre-processing to image.\n            if predictor.input_format == \"RGB\":\n                # whether the model expects BGR inputs or RGB\n                original_image = original_image[:, :, ::-1]\n            height, width = original_image.shape[:2]\n            image = predictor.aug.get_transform(original_image).apply_image(original_image)\n            image = torch.as_tensor(image.astype(\"float32\").transpose(2, 0, 1))\n            inputs = {\"image\": image, \"height\": height, \"width\": width}\n            inputs_list.append(inputs)\n        predictions = predictor.model(inputs_list)\n        return predictions\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We use the trained model in models folder under kaggle/input"},{"metadata":{"trusted":true},"cell_type":"code","source":"inputdir = Path(\"/kaggle/input\")\ntraineddir = inputdir / \"models\"\n\nflags: Flags = Flags().update(load_yaml(str(traineddir/\"flags.yaml\")))\nprint(\"flags\", flags)\ndebug = flags.debug\n\noutdir = Path(flags.outdir)\nos.makedirs(str(outdir), exist_ok=True)\n\n# --- Read data ---\ndatadir = inputdir / \"vinbigdata-chest-xray-abnormalities-detection\"\nimgdir = inputdir / flags.imgdir_name\n\ntest_meta = pd.read_csv(inputdir / \"vinbigdata-testmeta\" / \"test_meta.csv\")\nsample_submission = pd.read_csv(datadir / \"sample_submission.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction of Model"},{"metadata":{},"cell_type":"markdown","source":"* 1. imgdir : ../input/vinbigdata-chest-xray-resized-png-256x256"},{"metadata":{"trusted":true},"cell_type":"code","source":"cfg = get_cfg()\noriginal_output_dir = cfg.OUTPUT_DIR\ncfg.OUTPUT_DIR = str(outdir)\nprint(f\"cfg.OUTPUT_DIR {original_output_dir} -> {cfg.OUTPUT_DIR}\")\n\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\"))\ncfg.DATASETS.TRAIN = (\"vinbigdata_train\",)\ncfg.DATASETS.TEST = ()\n\ncfg.DATALOADER.NUM_WORKERS = 2\n\ncfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml\")\ncfg.SOLVER.IMS_PER_BATCH = 2\ncfg.SOLVER.BASE_LR = flags.base_lr  # pick a good LR\ncfg.SOLVER.MAX_ITER = flags.iter\ncfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = flags.roi_batch_size_per_image\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = len(thing_classes)\n\n### --- Inference & Evaluation ---\ncfg.MODEL.WEIGHTS = str(traineddir/\"model_final.pth\")\nprint(\"Original thresh\", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)  # 0.05\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.0 \nprint(\"Changed  thresh\", cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictor = DefaultPredictor(cfg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"DatasetCatalog.register(\n    \"vinbigdata_test\", lambda: get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MetadataCatalog.get(\"vinbigdata_test\").set(thing_classes=thing_classes)\nmetadata = MetadataCatalog.get(\"vinbigdata_test\")\ndataset_dicts = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)\n\nif debug:\n    dataset_dicts = dataset_dicts[:100]\n\nresults_list = []\nindex = 0\nbatch_size = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nif debug:\n    dataset_dicts = dataset_dicts[:100]\n\nresults_list = []\nindex = 0\nbatch_size = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor i in tqdm(range(ceil(len(dataset_dicts) / batch_size))):\n    inds = list(range(batch_size * i, min(batch_size * (i + 1), len(dataset_dicts))))\n    dataset_dicts_batch = [dataset_dicts[i] for i in inds]\n    im_list = [cv2.imread(d[\"file_name\"]) for d in dataset_dicts_batch]\n    outputs_list = predict_batch(predictor, im_list)\n\n    for im, outputs, d in zip(im_list, outputs_list, dataset_dicts_batch):\n        resized_height, resized_width, ch = im.shape\n    \n        if index < 5:\n           \n            v = Visualizer(\n                im[:, :, ::-1],\n                metadata=metadata,\n                scale=0.5,\n                instance_mode=ColorMode.IMAGE_BW\n               \n            )\n            out = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n            \n            cv2.imwrite(str(outdir / f\"pred_{index}.jpg\"), out.get_image()[:, :, ::-1])\n\n        image_id, dim0, dim1 = test_meta.iloc[index].values\n\n        instances = outputs[\"instances\"]\n        if len(instances) == 0:\n            \n            result = {\"image_id\": image_id, \"PredictionString\": \"14 1.0 0 0 1 1\"}\n        else:\n            # Find some bbox...\n            fields: Dict[str, Any] = instances.get_fields()\n            pred_classes = fields[\"pred_classes\"]  # (n_boxes,)\n            pred_scores = fields[\"scores\"]\n            # shape (n_boxes, 4). (xmin, ymin, xmax, ymax)\n            pred_boxes = fields[\"pred_boxes\"].tensor\n\n            h_ratio = dim0 / resized_height\n            w_ratio = dim1 / resized_width\n            pred_boxes[:, [0, 2]] *= w_ratio\n            pred_boxes[:, [1, 3]] *= h_ratio\n\n            pred_classes_array = pred_classes.cpu().numpy()\n            pred_boxes_array = pred_boxes.cpu().numpy()\n            pred_scores_array = pred_scores.cpu().numpy()\n\n            result = {\n                \"image_id\": image_id,\n                \"PredictionString\": format_pred(\n                    pred_classes_array, pred_boxes_array, pred_scores_array\n                ),\n            }\n        results_list.append(result)\n        index += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" * This submission includes only detection model's predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_det = pd.DataFrame(results_list, columns=['image_id', 'PredictionString'])\nsubmission_det.to_csv(\"submission_det.csv\", index=False)\nsubmission_det","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Improve the predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_2class = pd.read_csv(inputdir/\"vinbigdata-2class-prediction/2-cls test pred.csv\")\npred_2class","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NORMAL = \"14 1 0 0 1 1\"\nlow_threshold = 0.0\nhigh_threshold = 0.95\n\npred_det_df = submission_det  \nn_normal_before = len(pred_det_df.query(\"PredictionString == @NORMAL\"))\nmerged_df = pd.merge(pred_det_df, pred_2class, on=\"image_id\", how=\"left\")\n\n# 1. p < low_threshold                   -> \"Keep\": Do nothing, Keep det prediction.\n# 2. low_threshold <= p < high_threshold -> \"Add\": Just \"Add\" Normal prediction\n# 3. high_threshold <= p                 -> \"Replace\": Replace with Normal prediction\n\nif \"target\" in merged_df.columns:\n    merged_df[\"class0\"] = 1 - merged_df[\"target\"]\n\nc0, c1, c2 = 0, 0, 0\nfor i in range(len(merged_df)):\n    p0 = merged_df.loc[i, \"class0\"]\n    if p0 < low_threshold:\n        # Keep, do nothing.\n        c0 += 1\n    elif low_threshold <= p0 and p0 < high_threshold:\n        # Add, keep \"det\" preds and add normal pred.\n        merged_df.loc[i, \"PredictionString\"] += f\" 14 {p0} 0 0 1 1\"\n        c1 += 1\n    else:\n        # Replace, remove all \"det\" preds.\n        merged_df.loc[i, \"PredictionString\"] = NORMAL\n        c2 += 1\n\nn_normal_after = len(merged_df.query(\"PredictionString == @NORMAL\"))\nprint(\n    f\"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}\"\n)\nprint(f\"Keep {c0} Add {c1} Replace {c2}\")\nsubmission_filepath = str(\"submission.csv\")\nsubmission_df = merged_df[[\"image_id\", \"PredictionString\"]]\nsubmission_df.to_csv(submission_filepath, index=False)\nprint(f\"Saved to {submission_filepath}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Conclusion: If we use more iteration with different parameter like  in \"https://www.kaggle.com/corochann/vinbigdata-alb-aug-512-cos\"\n* The submission result is 0.221\n* Due to the kaggle constraint and  model training time, we use 10000 iteration and the result is 0.076: \n* Lastly, we trained the model with  30000. File is  [here](https://drive.google.com/file/d/1443L8_zw9wINEhkGx8_jaWZFN9lrSRf-/view?usp=sharing)"},{"metadata":{},"cell_type":"markdown","source":"You can find the submission.cv : [file here](https://drive.google.com/file/d/1COveby6u6WFo6HO4W07E_KG4IOsetbGs/view?usp=sharing)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}