{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Competiton metric calculator\n\n> The challenge uses the standard [PASCAL VOC 2010 mean Average Precision (mAP)](http://host.robots.ox.ac.uk/pascal/VOC/voc2010/devkit_doc_08-May-2010.pdf) at IoU > 0.4."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pycocotools","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport plotly.express as px\nimport plotly.graph_objects as go\n\nfrom pycocotools.coco import COCO\nfrom pycocotools.cocoeval import COCOeval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class VinBigDataEval:\n    \"\"\"Helper class for calculating the competition metric.\n    \n    You should remove the duplicated annoatations from the `true_df` dataframe\n    before using this script. Otherwise it may give incorrect results.\n\n        >>> vineval = VinBigDataEval(valid_df)\n        >>> cocoEvalResults = vineval.evaluate(pred_df)\n\n    Arguments:\n        true_df: pd.DataFrame Clean (no duplication) Training/Validating dataframe.\n\n    Authors:\n        Peter (https://kaggle.com/pestipeti)\n\n    See:\n        https://www.kaggle.com/pestipeti/competition-metric-map-0-4\n\n    Returns: None\n    \n    \"\"\"\n    def __init__(self, true_df):\n        \n        self.true_df = true_df\n\n        self.image_ids = true_df[\"image_id\"].unique()\n        self.annotations = {\n            \"type\": \"instances\",\n            \"images\": self.__gen_images(self.image_ids),\n            \"categories\": self.__gen_categories(self.true_df),\n            \"annotations\": self.__gen_annotations(self.true_df, self.image_ids)\n        }\n        \n        self.predictions = {\n            \"images\": self.annotations[\"images\"].copy(),\n            \"categories\": self.annotations[\"categories\"].copy(),\n            \"annotations\": None\n        }\n        \n    def __gen_categories(self, df):\n        print(\"Generating category data...\")\n        \n        if \"class_name\" not in df.columns:\n            df[\"class_name\"] = df[\"class_id\"]\n        \n        cats = df[[\"class_name\", \"class_id\"]]\n        cats = cats.drop_duplicates().sort_values(by='class_id').values\n        \n        results = []\n        \n        for cat in cats:\n            results.append({\n                \"id\": cat[1],\n                \"name\": cat[0],\n                \"supercategory\": \"none\",\n            })\n            \n        return results\n\n    def __gen_images(self, image_ids):\n        print(\"Generating image data...\")\n        results = []\n\n        for idx, image_id in enumerate(image_ids):\n\n            # Add image identification.\n            results.append({\n                \"id\": idx,\n            })\n            \n        return results\n    \n    def __gen_annotations(self, df, image_ids):\n        print(\"Generating annotation data...\")\n        k = 0\n        results = []\n        \n        for idx, image_id in enumerate(image_ids):\n\n            # Add image annotations\n            for i, row in df[df[\"image_id\"] == image_id].iterrows():\n\n                results.append({\n                    \"id\": k,\n                    \"image_id\": idx,\n                    \"category_id\": row[\"class_id\"],\n                    \"bbox\": np.array([\n                        row[\"x_min\"],\n                        row[\"y_min\"],\n                        row[\"x_max\"],\n                        row[\"y_max\"]]\n                    ),\n                    \"segmentation\": [],\n                    \"ignore\": 0,\n                    \"area\":(row[\"x_max\"] - row[\"x_min\"]) * (row[\"y_max\"] - row[\"y_min\"]),\n                    \"iscrowd\": 0,\n                })\n\n                k += 1\n                \n        return results\n\n    def __decode_prediction_string(self, pred_str):\n        data = list(map(float, pred_str.split(\" \")))\n        data = np.array(data)\n\n        return data.reshape(-1, 6)    \n    \n    def __gen_predictions(self, df, image_ids):\n        print(\"Generating prediction data...\")\n        k = 0\n        results = []\n        \n        for i, row in df.iterrows():\n            \n            image_id = row[\"image_id\"]\n            preds = self.__decode_prediction_string(row[\"PredictionString\"])\n\n            for j, pred in enumerate(preds):\n\n                results.append({\n                    \"id\": k,\n                    \"image_id\": int(np.where(image_ids == image_id)[0]),\n                    \"category_id\": int(pred[0]),\n                    \"bbox\": np.array([\n                        pred[2], pred[3], pred[4], pred[5]\n                    ]),\n                    \"segmentation\": [],\n                    \"ignore\": 0,\n                    \"area\": (pred[4] - pred[2]) * (pred[5] - pred[3]),\n                    \"iscrowd\": 0,\n                    \"score\": pred[1]\n                })\n\n                k += 1\n                \n        return results\n                \n    def evaluate(self, pred_df, n_imgs = -1):\n        \"\"\"Evaluating your results\n        \n        Arguments:\n            pred_df: pd.DataFrame your predicted results in the\n                     competition output format.\n\n            n_imgs:  int Number of images use for calculating the\n                     result.All of the images if `n_imgs` <= 0\n                     \n        Returns:\n            COCOEval object\n        \"\"\"\n        \n        if pred_df is not None:\n            self.predictions[\"annotations\"] = self.__gen_predictions(pred_df, self.image_ids)\n\n        coco_ds = COCO()\n        coco_ds.dataset = self.annotations\n        coco_ds.createIndex()\n        \n        coco_dt = COCO()\n        coco_dt.dataset = self.predictions\n        coco_dt.createIndex()\n        \n        imgIds=sorted(coco_ds.getImgIds())\n        \n        if n_imgs > 0:\n            imgIds = np.random.choice(imgIds, n_imgs)\n\n        cocoEval = COCOeval(coco_ds, coco_dt, 'bbox')\n        cocoEval.params.imgIds  = imgIds\n        cocoEval.params.useCats = True\n        cocoEval.params.iouType = \"bbox\"\n        cocoEval.params.iouThrs = np.array([0.4])\n\n        cocoEval.evaluate()\n        cocoEval.accumulate()\n        cocoEval.summarize()\n        \n        return cocoEval","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ndf.fillna(0, inplace=True)\ndf.loc[df[\"class_id\"] == 14, ['x_max', 'y_max']] = 1.0\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing duplications! DO NOT USE THIS in your training!!!\ndf = df.groupby(by=['image_id', 'class_id']).first().reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You only need to run this once.\nvineval = VinBigDataEval(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You will (hopefully) have better predictions\n# after every training epochs.\n#\n# The format is the same as we have in the submission.csv\npred_df = df[[\"image_id\"]]\npred_df = pred_df.drop_duplicates()\npred_df[\"PredictionString\"] = \"14 1.0 0 0 1 1\"\npred_df.reset_index(drop=True, inplace=True)\n\npred_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You should evaluate after every n epochs.\ncocoEvalRes = vineval.evaluate(pred_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cocoEvalRes.stats","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Recalculating with random samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\nstats = []\n\n# Recalculate the validation score using randomly selected images\nfor i in range(100):\n    cocoEvalRes = vineval.evaluate(pred_df = None, n_imgs = 300)\n    stats.append(cocoEvalRes.stats[0])\n    \navg = np.array(stats).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Scatter(x=[x for x in range(len(stats))], y=stats, mode=\"markers\", name=\"Stats\"))\nfig.add_trace(go.Scatter(x=[0, 100], y=[avg, avg], mode=\"lines\", name=\"Mean\"))\nfig.add_trace(go.Scatter(x=[0, 100], y=[0.052, 0.052], mode=\"lines\", name=\"Public Baseline\"))\n\nfig.update_yaxes(\n    range=[0.03, 0.07]\n)\n\nfig.update_layout(title='Results of mAP@0.4 (randomly selected 300 images)',\n                  yaxis_title='Score',\n                  xaxis_title='')\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}