{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Part Ivan","metadata":{}},{"cell_type":"code","source":"!rsync -a ../input/mmdetection-v280/mmdetection ../\n!pip install ../input/mmdetection-v280/src/mmdet-2.8.0/mmdet-2.8.0/\n!pip install ../input/mmdetection-v280/src/mmpycocotools-12.0.3/mmpycocotools-12.0.3/\n!pip install ../input/mmdetection-v280/src/addict-2.4.0-py3-none-any.whl\n!pip install ../input/mmdetection-v280/src/yapf-0.30.0-py2.py3-none-any.whl\n!pip install ../input/mmdetection-v280/src/mmcv_full-1.2.6-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ensemble_boxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mkdir -p pkl_preds/cascade_r50_rare/test pkl_preds/cascade_r50_augs_with_empty/test subm_folder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_rare/fold0.pth --out pkl_preds/cascade_r50_rare/test/fold0.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_rare/fold1.pth --out pkl_preds/cascade_r50_rare/test/fold1.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_rare/fold2.pth --out pkl_preds/cascade_r50_rare/test/fold2.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_rare/fold3.pth --out pkl_preds/cascade_r50_rare/test/fold3.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_rare/fold4.pth --out pkl_preds/cascade_r50_rare/test/fold4.pkl\n\n!python ../input/vinbigdata-data-scripts/pkl_to_subs.py --model-name cascade_r50_rare --resolution 1024\n!python ../input/vinbigdata-data-scripts/blend_subs.py --model-name cascade_r50_rare --resolution 1024\n!python ../input/vinbigdata-data-scripts/postprocess_v2.py --model-name cascade_r50_rare --resolution 1024\n!cp subs/cascade_r50_rare_1024/cascade_r50_rare_5folds_1024_postprocess.csv subm_folder/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_augs_with_empty/fold0.pth --out pkl_preds/cascade_r50_augs_with_empty/test/fold0.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_augs_with_empty/fold1.pth --out pkl_preds/cascade_r50_augs_with_empty/test/fold1.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_augs_with_empty/fold2.pth --out pkl_preds/cascade_r50_augs_with_empty/test/fold2.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_augs_with_empty/fold3.pth --out pkl_preds/cascade_r50_augs_with_empty/test/fold3.pkl\n!python ../mmdetection/tools/test.py ../input/vinbigdataconfigs/cascade_rcnn_r50_rfp_1x_xray_test.py ../input/vinbigdata-weights/cascade_r50_augs_with_empty/fold4.pth --out pkl_preds/cascade_r50_augs_with_empty/test/fold4.pkl\n\n!python ../input/vinbigdata-data-scripts/pkl_to_subs.py --model-name cascade_r50_augs_with_empty --resolution 1024\n!python ../input/vinbigdata-data-scripts/blend_subs.py --model-name cascade_r50_augs_with_empty --resolution 1024\n!python ../input/vinbigdata-data-scripts/postprocess_v2.py --model-name cascade_r50_augs_with_empty --resolution 1024\n!cp subs/cascade_r50_augs_with_empty_1024/cascade_r50_augs_with_empty_5folds_1024_postprocess.csv subm_folder/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part Sergey","metadata":{}},{"cell_type":"code","source":"!pip install ensemble_boxes\n!mkdir -p subm_folder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 ../input/vinbigdatagit/part_sergey/yolo_inf_kaggle.py --stage 2 --images ../input/vinbigdata-data/data --weights ../input/vinbigdata-yolo-weights --sub subm_folder/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part ZFTurbo","metadata":{}},{"cell_type":"markdown","source":"**Yolo part**","metadata":{}},{"cell_type":"code","source":"!pip install pycocotools\n!pip install thop\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_0_mAP_0.383_0.184.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold0/ --name test_iou_0.25_0.01\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_1_mAP_0.395_0.182.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold1/ --name test_iou_0.25_0.01\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_2_mAP_0.415_0.196.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold2/ --name test_iou_0.25_0.01\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_3_mAP_0.382_0.183.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold3/ --name test_iou_0.25_0.01\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_4_mAP_0.409_0.189.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold4/ --name test_iou_0.25_0.01\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect_mirror.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_0_mAP_0.383_0.184.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold0/ --name test_iou_0.25_0.01_mirror\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect_mirror.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_1_mAP_0.395_0.182.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold1/ --name test_iou_0.25_0.01_mirror\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect_mirror.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_2_mAP_0.415_0.196.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold2/ --name test_iou_0.25_0.01_mirror\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect_mirror.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_3_mAP_0.382_0.183.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold3/ --name test_iou_0.25_0.01_mirror\n\n!cd /kaggle/input/yolo5-zfturbo-xray-code/;python3 /kaggle/input/yolo5-zfturbo-xray-code/detect_mirror.py --weights /kaggle/input/chest-xray-abnormalities-detection-models/yolo_best/best_fold_4_mAP_0.409_0.189.pt --device 0 --img-size 640 --save-txt --save-conf --conf-thres 0.01 --iou-thres 0.25 --source /kaggle/input/chest-xray-test-diff-sizes/test_png_div_4/ --project /kaggle/working/yolov5_fold4/ --name test_iou_0.25_0.01_mirror","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob \nfor i in range(5):\n    files_orig = glob.glob('yolo*_fold{}/test_iou_0.25_0.01/labels/*'.format(i))\n    print(i, len(files_orig))\n    files_mirror = glob.glob('yolo*_fold{}/test_iou_0.25_0.01_mirror/labels/*'.format(i))\n    print(i, len(files_mirror))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# coding: utf-8\n__author__ = 'ZFTurbo: https://kaggle.com/zfturbo'\n\n\nimport numpy as np\nimport gzip\nimport pickle\nimport os\nimport glob\nimport time\nimport cv2\nimport datetime\nimport pandas as pd\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.model_selection import KFold, train_test_split\nfrom collections import Counter, defaultdict\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss\nimport random\nimport shutil\nimport operator\nfrom PIL import Image\nimport platform\nimport json\nimport base64\nimport typing as t\nimport zlib\nimport pydicom\nimport re\nfrom tqdm import tqdm\n\n\nINPUT_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nOUTPUT_PATH = './'\nSUBM_PATH = './'\n\n\ndef get_train_test_image_sizes():\n    sizes = dict()\n    sizes_train = pd.read_csv('../input/meta-xray/image_width_height_train.csv')\n    sizes_test = pd.read_csv('../input/meta-xray/image_width_height_test.csv')\n    sizes_df = pd.concat((sizes_train, sizes_test), axis=0)\n    for index, row in sizes_df.iterrows():\n        sizes[row['image_id']] = (row['height'], row['width'])\n    return sizes\n\n\ndef convert_preds_test(input_dir, out_file):\n    sizes = get_train_test_image_sizes()\n    s = pd.read_csv(INPUT_PATH + 'sample_submission.csv')\n    files = glob.glob(input_dir + '*.txt')\n    print(len(files))\n    part = s\n    print(len(part))\n    valid_ids = set(part['image_id'].values)\n    out = open(out_file, 'w')\n    out.write('image_id,PredictionString\\n')\n    for f in files:\n        image_id = os.path.basename(f)[:-4]\n        in1 = open(f, 'r')\n        lines = in1.readlines()\n        in1.close()\n        valid_ids.remove(image_id)\n        out.write('{},'.format(image_id))\n        pred_str = ''\n        for line in lines:\n            arr = line.strip().split(' ')\n            class_id = arr[0]\n            x = float(arr[1])\n            y = float(arr[2])\n            w = float(arr[3])\n            h = float(arr[4])\n            x1 = x - (w / 2)\n            x2 = x + (w / 2)\n            y1 = y - (h / 2)\n            y2 = y + (h / 2)\n            conf = arr[5]\n\n            x1 = int(round(x1 * sizes[image_id][1]))\n            y1 = int(round(y1 * sizes[image_id][0]))\n            x2 = int(round(x2 * sizes[image_id][1]))\n            y2 = int(round(y2 * sizes[image_id][0]))\n\n            pred_str += '{} {} {} {} {} {} '.format(class_id, conf, x1, y1, x2, y2)\n        out.write('{}\\n'.format(pred_str))\n\n    print(len(valid_ids))\n\n    # Output empty IDs\n    for image_id in list(valid_ids):\n        out.write('{},14 1 0 0 1 1\\n'.format(image_id))\n\n    out.close()\n\n\nif __name__ == '__main__':\n    for fold_num in range(5):\n        input_dir = OUTPUT_PATH + 'yolov5_fold{}/test_iou_0.25_0.01/labels/'.format(fold_num)\n        out_file = SUBM_PATH + 'yolov5_fold{}_iou_0.25_thr_0.01_test.csv'.format(fold_num)\n        convert_preds_test(input_dir, out_file)\n\n        input_dir = OUTPUT_PATH + 'yolov5_fold{}/test_iou_0.25_0.01_mirror/labels/'.format(fold_num)\n        out_file = SUBM_PATH + 'yolov5_fold{}_iou_0.25_thr_0.01_mirror_test.csv'.format(fold_num)\n        convert_preds_test(input_dir, out_file)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ensemble-boxes","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import weighted_boxes_fusion\n\n\ndef ensemble(\n    subm_list,\n    iou_same=0.5,\n    out_path=None,\n    skip_box_thr=0.00000001,\n):\n    sizes = get_train_test_image_sizes()\n    preds = []\n    weights = []\n    checker = None\n    for path, weight in subm_list:\n        s = pd.read_csv(path)\n        s.sort_values('image_id', inplace=True)\n        s.reset_index(drop=True, inplace=True)\n        ids = s['image_id']\n        if checker:\n            if tuple(ids) != checker:\n                print(set(checker) - set(ids))\n                print('Different IDS!', len(tuple(ids)), path)\n                exit()\n        else:\n            checker = tuple(ids)\n        preds.append(s['PredictionString'].values)\n        weights.append(weight)\n\n    if out_path is None:\n        out_path = SUBM_PATH + 'ensemble_iou_{}.csv'.format(iou_same)\n    out = open(out_path, 'w')\n    out.write('image_id,PredictionString\\n')\n    for j, id in enumerate(list(checker)):\n        # print(id)\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        empty = True\n        for i in range(len(preds)):\n            boxes = []\n            scores = []\n            labels = []\n            p1 = preds[i][j]\n            if str(p1) != 'nan':\n                arr = p1.strip().split(' ')\n                for k in range(0, len(arr), 6):\n                    cls = int(arr[k])\n                    prob = float(arr[k + 1])\n                    x1 = float(arr[k + 2]) / sizes[id][1]\n                    y1 = float(arr[k + 3]) / sizes[id][0]\n                    x2 = float(arr[k + 4]) / sizes[id][1]\n                    y2 = float(arr[k + 5]) / sizes[id][0]\n                    boxes.append([x1, y1, x2, y2])\n                    scores.append(prob)\n                    labels.append(cls)\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n\n        boxes, scores, labels = weighted_boxes_fusion(\n            boxes_list,\n            scores_list,\n            labels_list,\n            iou_thr=iou_same,\n            skip_box_thr=skip_box_thr,\n            weights=weights,\n            allows_overflow=True\n        )\n        # print(len(boxes), len(labels), len(scores))\n        if len(boxes) == 0:\n            out.write('{},14 1 0 0 1 1\\n'.format(id, ))\n        else:\n            final_str = ''\n            for i in range(len(boxes)):\n                cls = int(labels[i])\n                prob = scores[i]\n                x1 = int(boxes[i][0] * sizes[id][1])\n                y1 = int(boxes[i][1] * sizes[id][0])\n                x2 = int(boxes[i][2] * sizes[id][1])\n                y2 = int(boxes[i][3] * sizes[id][0])\n                if cls == 14:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, 0, 0, 1, 1)\n                else:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, x1, y1, x2, y2)\n            out.write('{},{}\\n'.format(id, final_str.strip()))\n\n    out.close()\n    return out_path\n\n\ndef get_test_from_subm_list(subm_list):\n    out = []\n    for s, w in subm_list:\n        s1 = s.replace('_train', '_test')\n        out.append((s1, w))\n    return out\n\n\ndef ensemble_experiment_v4_yolo():\n\n    sp = SUBM_PATH\n    subm_list = [\n        (sp + 'yolov5_fold0_iou_0.25_thr_0.01_train.csv', 1),\n        (sp + 'yolov5_fold1_iou_0.25_thr_0.01_train.csv', 1),\n        (sp + 'yolov5_fold2_iou_0.25_thr_0.01_train.csv', 1),\n        (sp + 'yolov5_fold3_iou_0.25_thr_0.01_train.csv', 1),\n        (sp + 'yolov5_fold4_iou_0.25_thr_0.01_train.csv', 1),\n    ]\n    subm_list_test = get_test_from_subm_list(subm_list)\n\n    best_iou = 0.3\n    out_path = SUBM_PATH + 'ensemble_yolo_standard.csv'.format(len(subm_list_test), best_iou)\n    predictions = ensemble(subm_list_test, best_iou, out_path)\n\n\ndef ensemble_experiment_v12_yolo_mirror():\n    sp = SUBM_PATH\n    subm_list_test = [\n        (sp + 'yolov5_fold0_iou_0.25_thr_0.01_mirror_test.csv', 1),\n        (sp + 'yolov5_fold1_iou_0.25_thr_0.01_mirror_test.csv', 1),\n        (sp + 'yolov5_fold2_iou_0.25_thr_0.01_mirror_test.csv', 1),\n        (sp + 'yolov5_fold3_iou_0.25_thr_0.01_mirror_test.csv', 1),\n        (sp + 'yolov5_fold4_iou_0.25_thr_0.01_mirror_test.csv', 1),\n    ]\n\n    best_iou = 0.3\n    best_map = -1\n    out_path = SUBM_PATH + 'ensemble_yolo_mirror.csv'.format(len(subm_list_test), best_iou, best_map)\n    predictions = ensemble(subm_list_test, best_iou, out_path)\n\n\ndef ensemble_experiment_v13_ensemble_yolo():\n    iou = 0.4\n    mean_ap = -1\n    subm_list = [\n        (SUBM_PATH + 'ensemble_yolo_standard.csv', 1),\n        (SUBM_PATH + 'ensemble_yolo_mirror.csv', 1),\n    ]\n    out_path = SUBM_PATH + 'ensemble_yolo_final.csv'.format(len(subm_list), iou, mean_ap)\n    predictions = ensemble(subm_list, iou, out_path)\n\n    \nif __name__ == '__main__':\n    ensemble_experiment_v4_yolo()\n    ensemble_experiment_v12_yolo_mirror()\n    ensemble_experiment_v13_ensemble_yolo()\n    print('Finished!')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean unneeded folders and move submission\n!rm -rf ./yolov5_fold0\n!rm -rf ./yolov5_fold1\n!rm -rf ./yolov5_fold2\n!rm -rf ./yolov5_fold3\n!rm -rf ./yolov5_fold4\n!mkdir -p ./subm_folder\n!mv ./ensemble_yolo_final.csv ./subm_folder","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**RetinaNet Part**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport gzip\nimport pickle\nimport os\nimport glob\nimport time\nimport cv2\nimport datetime\nimport pandas as pd\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.model_selection import KFold, train_test_split\nfrom collections import Counter, defaultdict\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss\nimport random\nimport shutil\nimport operator\nfrom PIL import Image\nimport platform\nimport json\nimport base64\nimport typing as t\nimport zlib\nimport pydicom\nimport re\nfrom tqdm import tqdm\n\nROOT_PATH = './'\nINPUT_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nSUBM_PATH = './'\n\n\ndef bb_intersection_over_union(boxA, boxB):\n    # determine the (x, y)-coordinates of the intersection rectangle\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n\n    # compute the area of intersection rectangle\n    interArea = max(0, xB - xA) * max(0, yB - yA)\n\n    if interArea == 0:\n        return 0.0\n\n    # compute the area of both the prediction and ground-truth\n    # rectangles\n    boxAArea = (boxA[2] - boxA[0]) * (boxA[3] - boxA[1])\n    boxBArea = (boxB[2] - boxB[0]) * (boxB[3] - boxB[1])\n\n    # compute the intersection over union by taking the intersection\n    # area and dividing it by the sum of prediction + ground-truth\n    # areas - the interesection area\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n\n    # return the intersection over union value\n    return iou\n\n\ndef find_matching_box(boxes_list, new_box, match_iou=0.7):\n    best_iou = match_iou\n    best_index = -1\n    for i in range(len(boxes_list)):\n        box = boxes_list[i]\n        if box[0] != new_box[0]:\n            continue\n        iou = bb_intersection_over_union(box[2:], new_box[2:])\n        if iou > best_iou:\n            best_index = i\n            best_iou = iou\n\n    return best_index, best_iou\n\n\ndef merge_boxes_v2(box1, box2, w1, w2, type):\n    box = [-1, -1, -1, -1, -1, -1]\n    box[0] = box1[0]\n    if type == 'avg':\n        box[1] = ((w1 * box1[1]) + (w2 * box2[1])) / (w1 + w2)\n    elif type == 'max':\n        box[1] = max(box1[1], box2[1])\n    elif type == 'mul':\n        box[1] = np.sqrt(box1[1]*box2[1])\n    else:\n        exit()\n    box[2] = (w1*box1[2] + w2*box2[2]) / (w1 + w2)\n    box[3] = (w1*box1[3] + w2*box2[3]) / (w1 + w2)\n    box[4] = (w1*box1[4] + w2*box2[4]) / (w1 + w2)\n    box[5] = (w1*box1[5] + w2*box2[5]) / (w1 + w2)\n    return box\n\n\ndef merge_all_boxes_for_image(boxes, intersection_thr=0.5, type='avg'):\n\n    new_boxes = boxes[0].copy()\n    init_weight = 1/len(boxes)\n    weights = [init_weight] * len(new_boxes)\n\n    for j in range(1, len(boxes)):\n        for k in range(len(boxes[j])):\n            index, best_iou = find_matching_box(new_boxes, boxes[j][k], intersection_thr)\n            if index != -1:\n                new_boxes[index] = merge_boxes_v2(new_boxes[index], boxes[j][k], weights[index], init_weight, type)\n                weights[index] += init_weight\n            else:\n                new_boxes.append(boxes[j][k])\n                weights.append(init_weight)\n\n    for i in range(len(new_boxes)):\n        new_boxes[i][1] *= weights[i]\n    return np.array(new_boxes)\n\n\ndef filter_boxes(boxes, scores, labels, thr):\n    new_boxes = []\n    for i in range(boxes.shape[0]):\n        box = []\n        for j in range(boxes.shape[1]):\n            label = labels[i, j].astype(np.int64)\n            score = scores[i, j]\n            if score < thr:\n                break\n            # Mirror fix !!!\n            if i % 2 == 0:\n                b = [int(label), float(score), float(boxes[i, j, 0]), float(boxes[i, j, 1]), float(boxes[i, j, 2]), float(boxes[i, j, 3])]\n            else:\n                b = [int(label), float(score), 1 - float(boxes[i, j, 2]), float(boxes[i, j, 1]), 1 - float(boxes[i, j, 0]), float(boxes[i, j, 3])]\n            box.append(b)\n        new_boxes.append(box)\n    return new_boxes\n\n\ndef get_train_test_image_sizes():\n    sizes = dict()\n    sizes_train = pd.read_csv('../input/meta-xray/image_width_height_test.csv')\n    sizes_test = pd.read_csv('../input/meta-xray/image_width_height_test.csv')\n    sizes_df = pd.concat((sizes_train, sizes_test), axis=0)\n    for index, row in sizes_df.iterrows():\n        sizes[row['image_id']] = (row['height'], row['width'])\n    return sizes\n\n\ndef save_in_file_fast(arr, file_name):\n    pickle.dump(arr, open(file_name, 'wb'), protocol=4)\n\n\ndef load_from_file_fast(file_name):\n    return pickle.load(open(file_name, 'rb'))\n\n\ndef create_csv_for_retinanet_predictions(\n        input_data,\n        out_file,\n        skip_box_thr=0.05,\n        intersection_thr=0.5,\n        limit_boxes=300,\n        type='avg'\n):\n    verbose = False\n    sizes = get_train_test_image_sizes()\n    out = open(out_file, 'w')\n    out.write('image_id,PredictionString\\n')\n    input_data = load_from_file_fast(input_data)\n    entries = list(input_data.keys())\n    for id in entries:\n        boxes, scores, labels = input_data[id]\n        filtered_boxes = filter_boxes(boxes, scores, labels, skip_box_thr)\n        # print(len(filtered_boxes[0]), len(filtered_boxes[1]))\n        # print(filtered_boxes[0], filtered_boxes[1])\n        merged_boxes = merge_all_boxes_for_image(filtered_boxes, intersection_thr, type)\n        # reduced_boxes = reduce_similar(merged_boxes)\n        if verbose:\n            print(id, len(filtered_boxes[0]), len(filtered_boxes[1]), len(merged_boxes))\n        if len(merged_boxes) > limit_boxes:\n            # sort by score\n            merged_boxes = np.array(merged_boxes)\n            merged_boxes = merged_boxes[merged_boxes[:, 1].argsort()[::-1]][:limit_boxes]\n\n        out.write(\"{},\".format(id))\n        if len(merged_boxes) > 0:\n            for i in range(len(merged_boxes)):\n                label = int(merged_boxes[i][0])\n                score = merged_boxes[i][1]\n                b = merged_boxes[i][2:]\n\n                xmin = b[0]\n                if xmin < 0:\n                    xmin = 0\n                if xmin > 1:\n                    xmin = 1\n\n                xmax = b[2]\n                if xmax < 0:\n                    xmax = 0\n                if xmax > 1:\n                    xmax = 1\n\n                ymin = b[1]\n                if ymin < 0:\n                    ymin = 0\n                if ymin > 1:\n                    ymin = 1\n\n                ymax = b[3]\n                if ymax < 0:\n                    ymax = 0\n                if ymax > 1:\n                    ymax = 1\n\n                if (xmax < xmin):\n                    print('X min value larger than max value {}: {} {}'.format('label', xmin, xmax))\n                    continue\n\n                if (ymax < ymin):\n                    print('Y min value larger than max value {}: {} {}'.format('label', ymin, ymax))\n                    continue\n\n                if abs(xmax - xmin) < 1e-5:\n                    print('Too small diff for {}: {} and {}'.format('label', xmin, xmax))\n                    continue\n\n                if abs(ymax - ymin) < 1e-5:\n                    print('Too small diff for {}: {} and {}'.format('label', ymin, ymax))\n                    continue\n\n                xmin = int(round(xmin * sizes[id][1]))\n                xmax = int(round(xmax * sizes[id][1]))\n                ymin = int(round(ymin * sizes[id][0]))\n                ymax = int(round(ymax * sizes[id][0]))\n                str1 = \"{} {:.6f} {} {} {} {} \".format(label, score, xmin, ymin, xmax, ymax)\n                out.write(str1)\n        else:\n            str1 = \"14 1 0 0 1 1\"\n            out.write(str1)\n        out.write('\\n')\n        \n\nif __name__ == '__main__':\n    limit_boxes = 100\n    type = 'avg'\n    \n    out_folders = [\n        '../input/inference-zfturbo-retina-1-fold-0/modified_data_folder/resnet101_fold_0_0.3573_26_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-1/modified_data_folder/resnet101_fold_1_0.3481_35_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-2/modified_data_folder/resnet101_fold_2_0.3804_24_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-3/modified_data_folder/resnet101_fold_3_0.3584_24_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-4/modified_data_folder/resnet101_fold_4_0.3514_12_iou_0.3_test_result.pkl',\n    ]\n    best_params_list = [\n        (0.05, 0.35),\n        (0.01, 0.45),\n        (0.03, 0.45),\n        (0.05, 0.45),\n        (0.03, 0.45),\n    ]\n    for i, o in enumerate(out_folders):\n        print('Go {} Params: {}'.format(os.path.basename(o), best_params_list[i]))\n        skip_box_thr = best_params_list[i][0]\n        intersection_thr = best_params_list[i][1]\n        out_file = SUBM_PATH + 'retina_' + os.path.basename(o)[:-4] + '_thr_{}_iou_{}_test.csv'.format(skip_box_thr, intersection_thr)\n        create_csv_for_retinanet_predictions(\n            o,\n            out_file,\n            skip_box_thr,\n            intersection_thr,\n            limit_boxes,\n            type=type\n        )\n        print('Write: {}'.format(out_file))\n    print('Complete retina 1!')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    limit_boxes = 100\n    type = 'avg'\n    \n    out_folders = [\n        '../input/inference-zfturbo-retina-2-fold-0/modified_data_folder/resnet101_fold_0_0.1817_05_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-1/modified_data_folder/resnet101_fold_1_0.2072_19_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-2/modified_data_folder/resnet101_fold_2_0.1938_03_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-3/modified_data_folder/resnet101_fold_3_0.1884_07_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-4/modified_data_folder/resnet101_fold_4_0.2227_05_iou_0.3_test_result.pkl',\n    ]\n    best_params_list = [\n        (0.03, 0.45),\n        (0.01, 0.45),\n        (0.01, 0.40),\n        (0.01, 0.45),\n        (0.01, 0.40),\n    ]\n    for i, o in enumerate(out_folders):\n        print('Go {} Params: {}'.format(os.path.basename(o), best_params_list[i]))\n        skip_box_thr = best_params_list[i][0]\n        intersection_thr = best_params_list[i][1]\n        out_file = SUBM_PATH + 'retina_' + os.path.basename(o)[:-4] + '_thr_{}_iou_{}_test.csv'.format(skip_box_thr, intersection_thr)\n        create_csv_for_retinanet_predictions(\n            o,\n            out_file,\n            skip_box_thr,\n            intersection_thr,\n            limit_boxes,\n            type=type\n        )\n        print('Write: {}'.format(out_file))\n    print('Complete retina 2!')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ensemble-boxes","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import weighted_boxes_fusion\n\n\ndef ensemble(\n    subm_list,\n    iou_same=0.5,\n    out_path=None,\n    skip_box_thr=0.00000001,\n):\n    sizes = get_train_test_image_sizes()\n    preds = []\n    weights = []\n    checker = None\n    for path, weight in subm_list:\n        s = pd.read_csv(path)\n        s.sort_values('image_id', inplace=True)\n        s.reset_index(drop=True, inplace=True)\n        ids = s['image_id']\n        if checker:\n            if tuple(ids) != checker:\n                print(set(checker) - set(ids))\n                print('Different IDS!', len(tuple(ids)), path)\n                exit()\n        else:\n            checker = tuple(ids)\n        preds.append(s['PredictionString'].values)\n        weights.append(weight)\n\n    if out_path is None:\n        out_path = SUBM_PATH + 'ensemble_iou_{}.csv'.format(iou_same)\n    out = open(out_path, 'w')\n    out.write('image_id,PredictionString\\n')\n    for j, id in enumerate(list(checker)):\n        # print(id)\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        empty = True\n        for i in range(len(preds)):\n            boxes = []\n            scores = []\n            labels = []\n            p1 = preds[i][j]\n            if str(p1) != 'nan':\n                arr = p1.strip().split(' ')\n                for k in range(0, len(arr), 6):\n                    cls = int(arr[k])\n                    prob = float(arr[k + 1])\n                    x1 = float(arr[k + 2]) / sizes[id][1]\n                    y1 = float(arr[k + 3]) / sizes[id][0]\n                    x2 = float(arr[k + 4]) / sizes[id][1]\n                    y2 = float(arr[k + 5]) / sizes[id][0]\n                    boxes.append([x1, y1, x2, y2])\n                    scores.append(prob)\n                    labels.append(cls)\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n\n        boxes, scores, labels = weighted_boxes_fusion(\n            boxes_list,\n            scores_list,\n            labels_list,\n            iou_thr=iou_same,\n            skip_box_thr=skip_box_thr,\n            weights=weights,\n            allows_overflow=True\n        )\n        # print(len(boxes), len(labels), len(scores))\n        if len(boxes) == 0:\n            out.write('{},14 1 0 0 1 1\\n'.format(id, ))\n        else:\n            final_str = ''\n            for i in range(len(boxes)):\n                cls = int(labels[i])\n                prob = scores[i]\n                x1 = int(boxes[i][0] * sizes[id][1])\n                y1 = int(boxes[i][1] * sizes[id][0])\n                x2 = int(boxes[i][2] * sizes[id][1])\n                y2 = int(boxes[i][3] * sizes[id][0])\n                if cls == 14:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, 0, 0, 1, 1)\n                else:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, x1, y1, x2, y2)\n            out.write('{},{}\\n'.format(id, final_str.strip()))\n\n    out.close()\n    return out_path\n\n\ndef get_test_from_subm_list(subm_list):\n    out = []\n    for s, w in subm_list:\n        s1 = s.replace('_train', '_test')\n        out.append((s1, w))\n    return out\n\n\ndef ensemble_experiment_v17_retinanet_resnet101_sqr():\n\n    sp = SUBM_PATH\n    subm_list = [\n        (sp + 'retina_resnet101_fold_0_0.3573_26_iou_0.3_test_result_thr_0.05_iou_0.35_test.csv', 1),\n        (sp + 'retina_resnet101_fold_1_0.3481_35_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_2_0.3804_24_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_3_0.3584_24_iou_0.3_test_result_thr_0.05_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_4_0.3514_12_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n    ]\n    subm_list_test = get_test_from_subm_list(subm_list)\n\n    best_iou = 0.4\n    skip_box_thr = 0.01\n    out_path = SUBM_PATH + 'ensemble_retinanet_resnet101_sqr.csv'\n    predictions = ensemble(subm_list_test, best_iou, out_path, skip_box_thr)\n    print('Prediction saved: {}'.format(out_path))\n\n\ndef ensemble_experiment_v24_retinanet_resnet101_sqr_removed_radiologists():\n\n    sp = SUBM_PATH\n    subm_list = [\n        (sp + 'retina_resnet101_fold_0_0.1817_05_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_1_0.2072_19_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_2_0.1938_03_iou_0.3_test_result_thr_0.01_iou_0.4_test.csv', 1),\n        (sp + 'retina_resnet101_fold_3_0.1884_07_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_4_0.2227_05_iou_0.3_test_result_thr_0.01_iou_0.4_test.csv', 1),\n    ]\n    subm_list_test = get_test_from_subm_list(subm_list)\n\n    best_iou = 0.4\n    skip_box_thr = 0.01\n    out_path = SUBM_PATH + 'ensemble_retinanet_resnet101_removed_rad.csv'\n    predictions = ensemble(subm_list_test, best_iou, out_path, skip_box_thr)\n    print('Prediction saved: {}'.format(out_path))\n\n\nif __name__ == '__main__':\n    ensemble_experiment_v17_retinanet_resnet101_sqr()\n    ensemble_experiment_v24_retinanet_resnet101_sqr_removed_radiologists()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move submissions\n!mkdir -p ./subm_folder\n!mv ./ensemble_retinanet_resnet101_sqr.csv ./subm_folder\n!mv ./ensemble_retinanet_resnet101_removed_rad.csv ./subm_folder","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble all","metadata":{}},{"cell_type":"code","source":"!pip install ensemble_boxes\n!mkdir -p subm_folder\n!cp \"../input/vinbigdatasubs/2-cls test pred.csv\" subm_folder/\n# Uncomment to debug ensemble\n\"\"\"\n!cp \"../input/vinbigdatasubs/cascade_r50_rare_5folds_1024_postprocess.csv\" subm_folder/\n!cp \"../input/vinbigdatasubs/cascade_r50_augs_with_empty_5folds_1024_postprocess.csv\" subm_folder/\n!cp \"../input/vinbigdatasubs/ensemble_retinanet_resnet101_removed_rad.csv\" subm_folder/\n!cp \"../input/vinbigdatasubs/ensemble_retinanet_resnet101_sqr.csv\" subm_folder/\n!cp \"../input/vinbigdatasubs/ensemble_yolo_final.csv\" subm_folder/\n!cp \"../input/vinbigdatasubs/yolo_stage2_all_folds.csv\" subm_folder/\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 ../input/vinbigdatagit/ensemble_models.py","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}