{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U ensemble-boxes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom ensemble_boxes import *\nfrom glob import glob\nimport copy\nfrom tqdm import tqdm\nimport shutil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"height_dict = pd.read_csv('../input/vinbigdata-original-image-dataset/vinbigdata/test.csv').to_dict('records')\nfnl_dict ={}\nfor ix,i in enumerate(height_dict):\n    fnl_dict[i['image_id']] = [i['width'],i['height'],i['width'],i['height']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subs = [\n#     pd.read_csv('../input/submission-vbg/ds_tst_F2_noTH_noClean.csv'),\n#         pd.read_csv('../input/vbg-yolo-submission/Fold 1.csv'),\n#         pd.read_csv('../input/detectron2-resnet50/results/20210110_train_all_500k_512/submission_det.csv'),\n#         pd.read_csv('../input/submission-vbg/ds_tst_F1_noTH_noClean.csv'),\n#         pd.read_csv('../input/vbg-yolo-submission/Fold 4.csv'),\n#         pd.read_csv('../input/vbg-yolo-submission/submission (2).csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission (6).csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission_v25_PS_kaggle_nbv5.csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission (13).csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission (4).csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission (8).csv'),\n#         pd.read_csv('../input/modified-effnet-classification/submission (19).csv')\n#         pd.read_csv('../input/cv-yolo/submission (31).csv')\n    \n    pd.read_csv('../input/yolo-vbd-lots-of-decimals/Fold_1.csv'),\n    pd.read_csv('../input/yolo-vbd-lots-of-decimals/Fold_2.csv'),\n    pd.read_csv('../input/yolo-vbd-lots-of-decimals/Fold_3.csv'),\n    pd.read_csv('../input/yolo-vbd-lots-of-decimals/Fold_4.csv'),\n    pd.read_csv('../input/yolo-vbd-lots-of-decimals/Fold_5.csv'),\n    pd.read_csv('../input/vinbigdata-cxr-ad-yolov5-14-class-infer-184dd1/submission.csv')\n       ]\n\npred_2cls = pd.read_csv('../input/vinbigdata-2class-prediction/2-cls test pred.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submission_decoder(df:pd.DataFrame) -> np.ndarray:\n    info = df.values\n    df_lst = []\n    for i in info:\n        pre_lst = i[1].split(' ')\n        for j in range(0,len(pre_lst),6):\n            df_lst.append([i[0],int(pre_lst[j]),float(pre_lst[j+1]),int(pre_lst[j+2]),int(pre_lst[j+3]),\\\n                           int(pre_lst[j+4]),int(pre_lst[j+5]),fnl_dict.get(i[0])[0],fnl_dict.get(i[0])[1]])\n    return pd.DataFrame(df_lst,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max','width','height'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subs = [submission_decoder(subs[i]) for i in range(len(subs))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxes_dict = {}\nscores_dict = {}\nlabels_dict = {}\nwhwh_dict = {}\n\nfor i in tqdm(subs[0].image_id.unique()):\n    if not i in boxes_dict.keys():\n        boxes_dict[i] = []\n        scores_dict[i] = []\n        labels_dict[i] = []\n        whwh_dict[i] = []\n\n    size_ratio = fnl_dict.get(i)\n    whwh_dict[i].append(size_ratio) \n    tmp_df = [subs[x][subs[x]['image_id']==i] for x in range(len(subs))]\n    \n    for x in range(len(tmp_df)):\n        boxes_dict[i].append(((tmp_df[x][['x_min','y_min','x_max','y_max']].values)/size_ratio).tolist())\n        scores_dict[i].append(tmp_df[x]['score'].values.tolist())\n        labels_dict[i].append(tmp_df[x]['class_id'].values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = [1]*5\nweights += [3]\nweights1 = [3,2,4,5]\niou_thr = 0.25\nskip_box_thr = 0.0\nsigma = 0.1\n\nfnl = {}\n\nfor i in tqdm(boxes_dict.keys()):\n    \n    \n    boxes, scores, labels = weighted_boxes_fusion(boxes_dict[i], scores_dict[i], labels_dict[i],\\\n                                                  weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    \n    \n    \n    \n#     boxes1, scores1, labels1 = nms(boxes_dict[i], scores_dict[i], labels_dict[i], weights=weights, iou_thr=iou_thr)\n\n    \n    \n    \n    \n    \n#     boxes0, scores0, labels0 = soft_nms(boxes_dict[i], scores_dict[i], labels_dict[i], weights=weights,\\\n#                                      iou_thr=iou_thr, sigma=sigma, thresh=skip_box_thr)\n    \n#     boxes2, scores2, labels2 = non_maximum_weighted(boxes_dict[i], scores_dict[i], labels_dict[i], weights=weights,\n#                                                                 skip_box_thr=skip_box_thr)\n\n    \n    \n    \n#     boxes, scores, labels = weighted_boxes_fusion([boxes0,boxes1,boxes2,boxes3],\\\n#                                                   [scores0,scores1,scores2,scores3],\\\n#                                                   [labels0,labels1,labels2,labels3],\\\n#                                                   weights=weights1, iou_thr=iou_thr, skip_box_thr=skip_box_thr)\n    \n    if not i in fnl.keys():\n        fnl[i] = {'boxes':[],'scores':[],'labels':[]}\n        \n    fnl[i]['boxes'] = boxes*whwh_dict[i]\n    fnl[i]['scores'] = scores\n    fnl[i]['labels'] = labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_form = []\nfor i in fnl.keys():\n    b = fnl[i]\n    for j in range(len(b['boxes'])):\n        pd_form.append([i,int(b['labels'][j]),round(b['scores'][j],2),\\\n                        int(b['boxes'][j][0]),int(b['boxes'][j][1]),\\\n                        int(b['boxes'][j][2]),int(b['boxes'][j][3])])\n        \nfinal_df = pd.DataFrame(pd_form,columns = ['image_id','class_id','score','x_min','y_min','x_max','y_max'])\nfinal_df = final_df.drop_duplicates(keep = 'first')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submission_encoder(df:pd.DataFrame) -> np.ndarray:\n    dct = {}\n    for i in tqdm(df['image_id'].unique()):\n        if not i in dct.keys():\n            dct[i] = []\n        tmp = df[df['image_id'] == i].values\n        for j in tmp:\n            dct[i].append(int(j[1]))\n            dct[i].append(float(j[2]))\n            dct[i].append(int(j[3]))\n            dct[i].append(int(j[4]))\n            dct[i].append(int(j[5]))\n            dct[i].append(int(j[6]))\n        \n        dct[i] = map(str,dct[i])\n        dct[i] = ' '.join(dct[i])\n    dct = [[k, v] for k, v in dct.items()]\n    return pd.DataFrame(dct,columns = ['image_id','PredictionString']).reset_index(drop = True)\n\ndf = submission_encoder(final_df)\ndf.to_csv('Fold5Yolo.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NORMAL = \"14 1 0 0 1 1\"\nlow_threshold = 0.00\nhigh_threshold = 0.99\npred_det_df = df  # You can load from another submission.csv here too.\nn_normal_before = len(pred_det_df.query(\"PredictionString == @NORMAL\"))\nmerged_df = pd.merge(pred_det_df, pred_2cls, on=\"image_id\", how=\"left\")\n\nif \"target\" in merged_df.columns:\n    merged_df[\"class0\"] = 1 - merged_df[\"target\"]\n\nc0, c1, c2 = 0, 0, 0\nfor i in range(len(merged_df)):\n    p0 = merged_df.loc[i, \"class0\"]\n    if p0 < low_threshold:\n        # Keep, do nothing.\n        c0 += 1\n    elif low_threshold <= p0 and p0 < high_threshold:\n        # Add, keep \"det\" preds and add normal pred.\n        if ' 14 ' not in merged_df.loc[i, \"PredictionString\"]:\n            merged_df.loc[i, \"PredictionString\"] += f\" 14 {p0} 0 0 1 1\"\n            \n        c1 += 1\n    else:\n        # Replace, remove all \"det\" preds.\n        merged_df.loc[i, \"PredictionString\"] = NORMAL\n        c2 += 1\n\nn_normal_after = len(merged_df.query(\"PredictionString == @NORMAL\"))\nprint(\n    f\"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}\"\n)\nprint(f\"Keep {c0} Add {c1} Replace {c2}\")\nsubmission_filepath = str(\"submission.csv\")\nsubmission_df = merged_df[[\"image_id\", \"PredictionString\"]]\nsubmission_df.to_csv(submission_filepath, index=False)\nprint(f\"Saved to {submission_filepath}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}