{"cells":[{"metadata":{},"cell_type":"markdown","source":"# YOLOv5 Detection evaluation\n\n\nThis code is based on [VinBigData-CXR-AD YOLOv5 14 Class [infer]](https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-infer) and [a Few Tips on mAP](https://www.kaggle.com/its7171/a-few-tips-on-map)\n\nSince this model is trained by this notebook [VinBigData-CXR-AD YOLOv5 14 Class [train]](https://www.kaggle.com/awsaf49/vinbigdata-cxr-ad-yolov5-14-class-train). Its validation set has no normal data.\n\nSo be careful when you compare this CV with LB.\n\n\nThe Goal of notebook is to analyze data with YOLOv5.\n\n**Please upvote if this notebook was helpful to you. Thank you:)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport shutil, os\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dim = 512 #1024, 256, 'original'\ntest_dir = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/test'\nweights_dir = '/kaggle/input/vinbigdata-cxr-ad-yolov5-14-class-train/yolov5/runs/train/exp/weights/best.pt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(f'../input/vinbigdata-{dim}-image-dataset/vinbigdata/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['image_path'] = f'/kaggle/input/vinbigdata-{dim}-image-dataset/vinbigdata/train/'+train_df.image_id+('.png' if dim!='original' else '.jpg')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df[train_df.class_id!=14].reset_index(drop = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 4\ngkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.image_id.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\nval_df = train_df[train_df['fold']==4]\nval_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = []\nval_files   = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"os.makedirs('/kaggle/working/vinbigdata/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/vinbigdata/images/val', exist_ok = True)\nlabel_dir = '/kaggle/input/vinbigdata-yolo-labels-dataset/labels'\nfor file in train_files:\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/train')\n    \nfor file in val_files:\n    shutil.copy(file, '/kaggle/working/vinbigdata/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename+'.txt'), '/kaggle/working/vinbigdata/labels/val')\n    \nval_dir = f'/kaggle/working/vinbigdata/images/val'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# YOLOv5 Set up"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\nos.chdir('/kaggle/working/yolov5') # install dependencies\n\nimport torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":true},"cell_type":"code","source":"!python detect.py --weights $weights_dir\\\n--img 640\\\n--conf 0.15\\\n--iou 0.4\\\n--source $val_dir\\\n--save-txt --save-conf --exist-ok","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def yolo2voc(image_height, image_width, bboxes):\n    \"\"\"\n    yolo => [xmid, ymid, w, h] (normalized)\n    voc  => [x1, y1, x2, y1]\n    \n    \"\"\" \n    bboxes = bboxes.copy().astype(float) # otherwise all value will be 0 as voc_pascal dtype is np.int\n    \n    bboxes[..., [0, 2]] = bboxes[..., [0, 2]]* image_width\n    bboxes[..., [1, 3]] = bboxes[..., [1, 3]]* image_height\n    \n    bboxes[..., [0, 1]] = bboxes[..., [0, 1]] - bboxes[..., [2, 3]]/2\n    bboxes[..., [2, 3]] = bboxes[..., [0, 1]] + bboxes[..., [2, 3]]\n    \n    return bboxes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"image_ids = []\nPredictionStrings = []\nclasses = []\nscores = []\nx_min = []\ny_min = []\nx_max = []\ny_max = []\n\n\nfor file_path in glob('runs/detect/exp/labels/*txt'):\n    image_id = file_path.split('/')[-1].split('.')[0]\n    w, h = val_df.loc[val_df.image_id==image_id,['width', 'height']].values[0]\n    f = open(file_path, 'r')\n    data = np.array(f.read().replace('\\n', ' ').strip().split(' ')).astype(np.float32).reshape(-1, 6)\n    data = data[:, [0, 5, 1, 2, 3, 4]]\n    bboxes = list(np.concatenate((data[:, :2], np.round(yolo2voc(h, w, data[:, 2:]))), axis =1).reshape(-1))#.astype(str))\n    for i in range(len(bboxes)//6):\n        image_ids.append(image_id)\n        classes.append(int(bboxes[i*6]))\n        scores.append(bboxes[i*6+1])\n        x_min.append(int(bboxes[i*6+2]))\n        y_min.append(int(bboxes[i*6+3]))\n        x_max.append(int(bboxes[i*6+4]))\n        y_max.append(int(bboxes[i*6+5]))\n#         bboxes[idx] = str(int(float(bboxes[idx]))) if idx%6!=1 else bboxes[idx]\n#     image_ids.append(image_id)\n#     PredictionStrings.append(' '.join(bboxes))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"df_pred = pd.DataFrame({'image_id':image_ids,\n                        'class_id':classes,\n                        'conf':scores,\n                        'x_min':x_min,\n                        'y_min':y_min,\n                        'x_max':x_max,\n                        'y_max':y_max\n                       })\ndf_pred = df_pred.sort_values('conf', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# this code is copied from https://github.com/ZFTurbo/Mean-Average-Precision-for-Boxes\n# you can install map-boxes package by pip.\n# I just copied it because I want to make small changes for visualization.\n\n\"\"\"\nAuthor: Roman Solovyev, IPPM RAS\nURL: https://github.com/ZFTurbo\nCode based on: https://github.com/fizyr/keras-retinanet/blob/master/keras_retinanet/utils/eval.py\n\"\"\"\ndef get_real_annotations(table):\n    res = dict()\n    ids = table['ImageID'].values.astype(np.str)\n    labels = table['LabelName'].values.astype(np.str)\n    xmin = table['XMin'].values.astype(np.float32)\n    xmax = table['XMax'].values.astype(np.float32)\n    ymin = table['YMin'].values.astype(np.float32)\n    ymax = table['YMax'].values.astype(np.float32)\n\n    for i in range(len(ids)):\n        id = ids[i]\n        label = labels[i]\n        if id not in res:\n            res[id] = dict()\n        if label not in res[id]:\n            res[id][label] = []\n        box = [xmin[i], ymin[i], xmax[i], ymax[i]]\n        res[id][label].append(box)\n\n    return res\n\n\ndef get_detections(table):\n    res = dict()\n    ids = table['ImageID'].values.astype(np.str)\n    labels = table['LabelName'].values.astype(np.str)\n    scores = table['Conf'].values.astype(np.float32)\n    xmin = table['XMin'].values.astype(np.float32)\n    xmax = table['XMax'].values.astype(np.float32)\n    ymin = table['YMin'].values.astype(np.float32)\n    ymax = table['YMax'].values.astype(np.float32)\n\n    for i in range(len(ids)):\n        id = ids[i]\n        label = labels[i]\n        if id not in res:\n            res[id] = dict()\n        if label not in res[id]:\n            res[id][label] = []\n        box = [xmin[i], ymin[i], xmax[i], ymax[i], scores[i]]\n        res[id][label].append(box)\n\n    return res\n\n\ndef _compute_ap(recall, precision):\n    \"\"\" Compute the average precision, given the recall and precision curves.\n    Code originally from https://github.com/rbgirshick/py-faster-rcnn.\n    # Arguments\n        recall:    The recall curve (list).\n        precision: The precision curve (list).\n    # Returns\n        The average precision as computed in py-faster-rcnn.\n    \"\"\"\n    # correct AP calculation\n    # first append sentinel values at the end\n    mrec = np.concatenate(([0.], recall, [1.]))\n    mpre = np.concatenate(([0.], precision, [0.]))\n\n    # compute the precision envelope\n    for i in range(mpre.size - 1, 0, -1):\n        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])\n\n    # to calculate area under PR curve, look for points\n    # where X axis (recall) changes value\n    i = np.where(mrec[1:] != mrec[:-1])[0]\n\n    # and sum (\\Delta recall) * prec\n    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])\n    return ap\n\n\ndef mean_average_precision_for_boxes(ann, pred, iou_threshold=0.4, exclude_not_in_annotations=False, verbose=True):\n    \"\"\"\n    :param ann: path to CSV-file with annotations or numpy array of shape (N, 6)\n    :param pred: path to CSV-file with predictions (detections) or numpy array of shape (N, 7)\n    :param iou_threshold: IoU between boxes which count as 'match'. Default: 0.5\n    :param exclude_not_in_annotations: exclude image IDs which are not exist in annotations. Default: False\n    :param verbose: print detailed run info. Default: True\n    :return: tuple, where first value is mAP and second values is dict with AP for each class.\n    \"\"\"\n\n    if isinstance(ann, str):\n        valid = pd.read_csv(ann)\n    else:\n        valid = pd.DataFrame(ann, columns=['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax'])\n\n    if isinstance(pred, str):\n        preds = pd.read_csv(pred)\n    else:\n        preds = pd.DataFrame(pred, columns=['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax'])\n\n    ann_unique = valid['ImageID'].unique()\n    preds_unique = preds['ImageID'].unique()\n\n    if verbose:\n        print('Number of files in annotations: {}'.format(len(ann_unique)))\n        print('Number of files in predictions: {}'.format(len(preds_unique)))\n\n    # Exclude files not in annotations!\n    if exclude_not_in_annotations:\n        preds = preds[preds['ImageID'].isin(ann_unique)]\n        preds_unique = preds['ImageID'].unique()\n        if verbose:\n            print('Number of files in detection after reduction: {}'.format(len(preds_unique)))\n\n    unique_classes = valid['LabelName'].unique().astype(np.str)\n    if verbose:\n        print('Unique classes: {}'.format(len(unique_classes)))\n\n    all_detections = get_detections(preds)\n    all_annotations = get_real_annotations(valid)\n    if verbose:\n        print('Detections length: {}'.format(len(all_detections)))\n        print('Annotations length: {}'.format(len(all_annotations)))\n\n    average_precisions = {}\n    for zz, label in enumerate(sorted(unique_classes)):\n\n        # Negative class\n        if str(label) == 'nan':\n            continue\n\n        false_positives = []\n        true_positives = []\n        scores = []\n        num_annotations = 0.0\n\n        for i in range(len(ann_unique)):\n            detections = []\n            annotations = []\n            id = ann_unique[i]\n            if id in all_detections:\n                if label in all_detections[id]:\n                    detections = all_detections[id][label]\n            if id in all_annotations:\n                if label in all_annotations[id]:\n                    annotations = all_annotations[id][label]\n\n            if len(detections) == 0 and len(annotations) == 0:\n                continue\n\n            num_annotations += len(annotations)\n            detected_annotations = []\n\n            annotations = np.array(annotations, dtype=np.float64)\n            for d in detections:\n                scores.append(d[4])\n\n                if len(annotations) == 0:\n                    false_positives.append(1)\n                    true_positives.append(0)\n                    continue\n\n                overlaps = compute_overlap(np.expand_dims(np.array(d, dtype=np.float64), axis=0), annotations)\n                assigned_annotation = np.argmax(overlaps, axis=1)\n                max_overlap = overlaps[0, assigned_annotation]\n\n                if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:\n                    false_positives.append(0)\n                    true_positives.append(1)\n                    detected_annotations.append(assigned_annotation)\n                else:\n                    false_positives.append(1)\n                    true_positives.append(0)\n\n        if num_annotations == 0:\n            average_precisions[label] = 0, 0\n            continue\n\n        false_positives = np.array(false_positives)\n        true_positives = np.array(true_positives)\n        scores = np.array(scores)\n\n        # sort by score\n        indices = np.argsort(-scores)\n        false_positives = false_positives[indices]\n        true_positives = true_positives[indices]\n\n        # compute false positives and true positives\n        false_positives = np.cumsum(false_positives)\n        true_positives = np.cumsum(true_positives)\n\n        # compute recall and precision\n        recall = true_positives / num_annotations\n        precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)\n\n        # compute average precision\n        average_precision = _compute_ap(recall, precision)\n        average_precisions[label] = average_precision, num_annotations, precision, recall\n        if verbose:\n            s1 = \"{:30s} | {:.6f} | {:7d}\".format(label, average_precision, int(num_annotations))\n            print(s1)\n\n    present_classes = 0\n    precision = 0\n    for label, (average_precision, num_annotations, _, _) in average_precisions.items():\n        if num_annotations > 0:\n            present_classes += 1\n            precision += average_precision\n    mean_ap = precision / present_classes\n    if verbose:\n        print('mAP: {:.6f}'.format(mean_ap))\n    return mean_ap, average_precisions\n\n\ndef compute_overlap(boxes, query_boxes):\n    \"\"\"\n    Args\n        a: (N, 4) ndarray of float\n        b: (K, 4) ndarray of float\n    Returns\n        overlaps: (N, K) ndarray of overlap between boxes and query_boxes\n    \"\"\"\n    N = boxes.shape[0]\n    K = query_boxes.shape[0]\n    overlaps = np.zeros((N, K), dtype=np.float64)\n    for k in range(K):\n        box_area = (\n            (query_boxes[k, 2] - query_boxes[k, 0]) *\n            (query_boxes[k, 3] - query_boxes[k, 1])\n        )\n        for n in range(N):\n            iw = (\n                min(boxes[n, 2], query_boxes[k, 2]) -\n                max(boxes[n, 0], query_boxes[k, 0])\n            )\n            if iw > 0:\n                ih = (\n                    min(boxes[n, 3], query_boxes[k, 3]) -\n                    max(boxes[n, 1], query_boxes[k, 1])\n                )\n                if ih > 0:\n                    ua = np.float64(\n                        (boxes[n, 2] - boxes[n, 0]) *\n                        (boxes[n, 3] - boxes[n, 1]) +\n                        box_area - iw * ih\n                    )\n                    overlaps[n, k] = iw * ih / ua\n    return overlaps","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pred_thre001 = df_pred[df_pred.conf > 0.15]\ndf_pred_thre05 = df_pred[df_pred.conf > 0.5]\npred_thre001 = df_pred_thre001[['image_id', 'class_id', 'conf','x_min','x_max','y_min','y_max']].values\npred_thre05 = df_pred_thre05[['image_id', 'class_id', 'conf','x_min','x_max','y_min','y_max']].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"anno = val_df[['image_id', 'class_id','x_min','x_max','y_min','y_max']].values\nmean_ap_thre001, average_precisions_thre001 = mean_average_precision_for_boxes(anno, pred_thre001, verbose=False)\nmean_ap_thre05, average_precisions_thre05 = mean_average_precision_for_boxes(anno, pred_thre05, verbose=False)\nprint('mAP with threshold 0.01', round(mean_ap_thre001,2))\nprint('mAP with threshold 0.5 ', round(mean_ap_thre05,2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Please read it.\n\n* As I mentioned on the top of notebook, Its validation set has no normal data. So be careful when you compare CV with LB.\n\n* **Please upvote if this notebook was helpful to you. Thank you:)**\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}