{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport gzip\nimport pickle\nimport os\nimport glob\nimport time\nimport cv2\nimport datetime\nimport pandas as pd\nfrom sklearn.metrics import fbeta_score\nfrom sklearn.model_selection import KFold, train_test_split\nfrom collections import Counter, defaultdict\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss\nimport random\nimport shutil\nimport operator\nfrom PIL import Image\nimport platform\nimport json\nimport base64\nimport typing as t\nimport zlib\nimport pydicom\nimport re\nfrom tqdm import tqdm\n\nROOT_PATH = './'\nINPUT_PATH = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nSUBM_PATH = './'\n\n\ndef bb_intersection_over_union(boxA, boxB):\n    # determine the (x, y)-coordinates of the intersection rectangle\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n\n    # compute the area of intersection rectangle\n    interArea = max(0, xB - xA) * max(0, yB - yA)\n\n    if interArea == 0:\n        return 0.0\n\n    # compute the area of both the prediction and ground-truth\n    # rectangles\n    boxAArea = (boxA[2] - boxA[0]) * (boxA[3] - boxA[1])\n    boxBArea = (boxB[2] - boxB[0]) * (boxB[3] - boxB[1])\n\n    # compute the intersection over union by taking the intersection\n    # area and dividing it by the sum of prediction + ground-truth\n    # areas - the interesection area\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n\n    # return the intersection over union value\n    return iou\n\n\ndef find_matching_box(boxes_list, new_box, match_iou=0.7):\n    best_iou = match_iou\n    best_index = -1\n    for i in range(len(boxes_list)):\n        box = boxes_list[i]\n        if box[0] != new_box[0]:\n            continue\n        iou = bb_intersection_over_union(box[2:], new_box[2:])\n        if iou > best_iou:\n            best_index = i\n            best_iou = iou\n\n    return best_index, best_iou\n\n\ndef merge_boxes_v2(box1, box2, w1, w2, type):\n    box = [-1, -1, -1, -1, -1, -1]\n    box[0] = box1[0]\n    if type == 'avg':\n        box[1] = ((w1 * box1[1]) + (w2 * box2[1])) / (w1 + w2)\n    elif type == 'max':\n        box[1] = max(box1[1], box2[1])\n    elif type == 'mul':\n        box[1] = np.sqrt(box1[1]*box2[1])\n    else:\n        exit()\n    box[2] = (w1*box1[2] + w2*box2[2]) / (w1 + w2)\n    box[3] = (w1*box1[3] + w2*box2[3]) / (w1 + w2)\n    box[4] = (w1*box1[4] + w2*box2[4]) / (w1 + w2)\n    box[5] = (w1*box1[5] + w2*box2[5]) / (w1 + w2)\n    return box\n\n\ndef merge_all_boxes_for_image(boxes, intersection_thr=0.5, type='avg'):\n\n    new_boxes = boxes[0].copy()\n    init_weight = 1/len(boxes)\n    weights = [init_weight] * len(new_boxes)\n\n    for j in range(1, len(boxes)):\n        for k in range(len(boxes[j])):\n            index, best_iou = find_matching_box(new_boxes, boxes[j][k], intersection_thr)\n            if index != -1:\n                new_boxes[index] = merge_boxes_v2(new_boxes[index], boxes[j][k], weights[index], init_weight, type)\n                weights[index] += init_weight\n            else:\n                new_boxes.append(boxes[j][k])\n                weights.append(init_weight)\n\n    for i in range(len(new_boxes)):\n        new_boxes[i][1] *= weights[i]\n    return np.array(new_boxes)\n\n\ndef filter_boxes(boxes, scores, labels, thr):\n    new_boxes = []\n    for i in range(boxes.shape[0]):\n        box = []\n        for j in range(boxes.shape[1]):\n            label = labels[i, j].astype(np.int64)\n            score = scores[i, j]\n            if score < thr:\n                break\n            # Mirror fix !!!\n            if i % 2 == 0:\n                b = [int(label), float(score), float(boxes[i, j, 0]), float(boxes[i, j, 1]), float(boxes[i, j, 2]), float(boxes[i, j, 3])]\n            else:\n                b = [int(label), float(score), 1 - float(boxes[i, j, 2]), float(boxes[i, j, 1]), 1 - float(boxes[i, j, 0]), float(boxes[i, j, 3])]\n            box.append(b)\n        new_boxes.append(box)\n    return new_boxes\n\n\ndef get_train_test_image_sizes():\n    sizes = dict()\n    sizes_train = pd.read_csv('../input/meta-xray/image_width_height_test.csv')\n    sizes_test = pd.read_csv('../input/meta-xray/image_width_height_test.csv')\n    sizes_df = pd.concat((sizes_train, sizes_test), axis=0)\n    for index, row in sizes_df.iterrows():\n        sizes[row['image_id']] = (row['height'], row['width'])\n    return sizes\n\n\ndef save_in_file_fast(arr, file_name):\n    pickle.dump(arr, open(file_name, 'wb'), protocol=4)\n\n\ndef load_from_file_fast(file_name):\n    return pickle.load(open(file_name, 'rb'))\n\n\ndef create_csv_for_retinanet_predictions(\n        input_data,\n        out_file,\n        skip_box_thr=0.05,\n        intersection_thr=0.5,\n        limit_boxes=300,\n        type='avg'\n):\n    verbose = False\n    sizes = get_train_test_image_sizes()\n    out = open(out_file, 'w')\n    out.write('image_id,PredictionString\\n')\n    input_data = load_from_file_fast(input_data)\n    entries = list(input_data.keys())\n    for id in entries:\n        boxes, scores, labels = input_data[id]\n        filtered_boxes = filter_boxes(boxes, scores, labels, skip_box_thr)\n        # print(len(filtered_boxes[0]), len(filtered_boxes[1]))\n        # print(filtered_boxes[0], filtered_boxes[1])\n        merged_boxes = merge_all_boxes_for_image(filtered_boxes, intersection_thr, type)\n        # reduced_boxes = reduce_similar(merged_boxes)\n        if verbose:\n            print(id, len(filtered_boxes[0]), len(filtered_boxes[1]), len(merged_boxes))\n        if len(merged_boxes) > limit_boxes:\n            # sort by score\n            merged_boxes = np.array(merged_boxes)\n            merged_boxes = merged_boxes[merged_boxes[:, 1].argsort()[::-1]][:limit_boxes]\n\n        out.write(\"{},\".format(id))\n        if len(merged_boxes) > 0:\n            for i in range(len(merged_boxes)):\n                label = int(merged_boxes[i][0])\n                score = merged_boxes[i][1]\n                b = merged_boxes[i][2:]\n\n                xmin = b[0]\n                if xmin < 0:\n                    xmin = 0\n                if xmin > 1:\n                    xmin = 1\n\n                xmax = b[2]\n                if xmax < 0:\n                    xmax = 0\n                if xmax > 1:\n                    xmax = 1\n\n                ymin = b[1]\n                if ymin < 0:\n                    ymin = 0\n                if ymin > 1:\n                    ymin = 1\n\n                ymax = b[3]\n                if ymax < 0:\n                    ymax = 0\n                if ymax > 1:\n                    ymax = 1\n\n                if (xmax < xmin):\n                    print('X min value larger than max value {}: {} {}'.format('label', xmin, xmax))\n                    continue\n\n                if (ymax < ymin):\n                    print('Y min value larger than max value {}: {} {}'.format('label', ymin, ymax))\n                    continue\n\n                if abs(xmax - xmin) < 1e-5:\n                    print('Too small diff for {}: {} and {}'.format('label', xmin, xmax))\n                    continue\n\n                if abs(ymax - ymin) < 1e-5:\n                    print('Too small diff for {}: {} and {}'.format('label', ymin, ymax))\n                    continue\n\n                xmin = int(round(xmin * sizes[id][1]))\n                xmax = int(round(xmax * sizes[id][1]))\n                ymin = int(round(ymin * sizes[id][0]))\n                ymax = int(round(ymax * sizes[id][0]))\n                str1 = \"{} {:.6f} {} {} {} {} \".format(label, score, xmin, ymin, xmax, ymax)\n                out.write(str1)\n        else:\n            str1 = \"14 1 0 0 1 1\"\n            out.write(str1)\n        out.write('\\n')\n        \n\nif __name__ == '__main__':\n    limit_boxes = 100\n    type = 'avg'\n    \n    out_folders = [\n        '../input/inference-zfturbo-retina-1-fold-0/modified_data_folder/resnet101_fold_0_0.3573_26_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-1/modified_data_folder/resnet101_fold_1_0.3481_35_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-2/modified_data_folder/resnet101_fold_2_0.3804_24_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-3/modified_data_folder/resnet101_fold_3_0.3584_24_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-1-fold-4/modified_data_folder/resnet101_fold_4_0.3514_12_iou_0.3_test_result.pkl',\n    ]\n    best_params_list = [\n        (0.05, 0.35),\n        (0.01, 0.45),\n        (0.03, 0.45),\n        (0.05, 0.45),\n        (0.03, 0.45),\n    ]\n    for i, o in enumerate(out_folders):\n        print('Go {} Params: {}'.format(os.path.basename(o), best_params_list[i]))\n        skip_box_thr = best_params_list[i][0]\n        intersection_thr = best_params_list[i][1]\n        out_file = SUBM_PATH + 'retina_' + os.path.basename(o)[:-4] + '_thr_{}_iou_{}_test.csv'.format(skip_box_thr, intersection_thr)\n        create_csv_for_retinanet_predictions(\n            o,\n            out_file,\n            skip_box_thr,\n            intersection_thr,\n            limit_boxes,\n            type=type\n        )\n        print('Write: {}'.format(out_file))\n    print('Complete retina 1!')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    limit_boxes = 100\n    type = 'avg'\n    \n    out_folders = [\n        '../input/inference-zfturbo-retina-2-fold-0/modified_data_folder/resnet101_fold_0_0.1817_05_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-1/modified_data_folder/resnet101_fold_1_0.2072_19_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-2/modified_data_folder/resnet101_fold_2_0.1938_03_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-3/modified_data_folder/resnet101_fold_3_0.1884_07_iou_0.3_test_result.pkl',\n        '../input/inference-zfturbo-retina-2-fold-4/modified_data_folder/resnet101_fold_4_0.2227_05_iou_0.3_test_result.pkl',\n    ]\n    best_params_list = [\n        (0.03, 0.45),\n        (0.01, 0.45),\n        (0.01, 0.40),\n        (0.01, 0.45),\n        (0.01, 0.40),\n    ]\n    for i, o in enumerate(out_folders):\n        print('Go {} Params: {}'.format(os.path.basename(o), best_params_list[i]))\n        skip_box_thr = best_params_list[i][0]\n        intersection_thr = best_params_list[i][1]\n        out_file = SUBM_PATH + 'retina_' + os.path.basename(o)[:-4] + '_thr_{}_iou_{}_test.csv'.format(skip_box_thr, intersection_thr)\n        create_csv_for_retinanet_predictions(\n            o,\n            out_file,\n            skip_box_thr,\n            intersection_thr,\n            limit_boxes,\n            type=type\n        )\n        print('Write: {}'.format(out_file))\n    print('Complete retina 2!')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ensemble-boxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import weighted_boxes_fusion\n\n\ndef ensemble(\n    subm_list,\n    iou_same=0.5,\n    out_path=None,\n    skip_box_thr=0.00000001,\n):\n    sizes = get_train_test_image_sizes()\n    preds = []\n    weights = []\n    checker = None\n    for path, weight in subm_list:\n        s = pd.read_csv(path)\n        s.sort_values('image_id', inplace=True)\n        s.reset_index(drop=True, inplace=True)\n        ids = s['image_id']\n        if checker:\n            if tuple(ids) != checker:\n                print(set(checker) - set(ids))\n                print('Different IDS!', len(tuple(ids)), path)\n                exit()\n        else:\n            checker = tuple(ids)\n        preds.append(s['PredictionString'].values)\n        weights.append(weight)\n\n    if out_path is None:\n        out_path = SUBM_PATH + 'ensemble_iou_{}.csv'.format(iou_same)\n    out = open(out_path, 'w')\n    out.write('image_id,PredictionString\\n')\n    for j, id in enumerate(list(checker)):\n        # print(id)\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n        empty = True\n        for i in range(len(preds)):\n            boxes = []\n            scores = []\n            labels = []\n            p1 = preds[i][j]\n            if str(p1) != 'nan':\n                arr = p1.strip().split(' ')\n                for k in range(0, len(arr), 6):\n                    cls = int(arr[k])\n                    prob = float(arr[k + 1])\n                    x1 = float(arr[k + 2]) / sizes[id][1]\n                    y1 = float(arr[k + 3]) / sizes[id][0]\n                    x2 = float(arr[k + 4]) / sizes[id][1]\n                    y2 = float(arr[k + 5]) / sizes[id][0]\n                    boxes.append([x1, y1, x2, y2])\n                    scores.append(prob)\n                    labels.append(cls)\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n\n        boxes, scores, labels = weighted_boxes_fusion(\n            boxes_list,\n            scores_list,\n            labels_list,\n            iou_thr=iou_same,\n            skip_box_thr=skip_box_thr,\n            weights=weights,\n            allows_overflow=True\n        )\n        # print(len(boxes), len(labels), len(scores))\n        if len(boxes) == 0:\n            out.write('{},14 1 0 0 1 1\\n'.format(id, ))\n        else:\n            final_str = ''\n            for i in range(len(boxes)):\n                cls = int(labels[i])\n                prob = scores[i]\n                x1 = int(boxes[i][0] * sizes[id][1])\n                y1 = int(boxes[i][1] * sizes[id][0])\n                x2 = int(boxes[i][2] * sizes[id][1])\n                y2 = int(boxes[i][3] * sizes[id][0])\n                if cls == 14:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, 0, 0, 1, 1)\n                else:\n                    final_str += '{} {} {} {} {} {} '.format(cls, prob, x1, y1, x2, y2)\n            out.write('{},{}\\n'.format(id, final_str.strip()))\n\n    out.close()\n    return out_path\n\n\ndef get_test_from_subm_list(subm_list):\n    out = []\n    for s, w in subm_list:\n        s1 = s.replace('_train', '_test')\n        out.append((s1, w))\n    return out\n\n\ndef ensemble_experiment_v17_retinanet_resnet101_sqr():\n\n    sp = SUBM_PATH\n    subm_list = [\n        (sp + 'retina_resnet101_fold_0_0.3573_26_iou_0.3_test_result_thr_0.05_iou_0.35_test.csv', 1),\n        (sp + 'retina_resnet101_fold_1_0.3481_35_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_2_0.3804_24_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_3_0.3584_24_iou_0.3_test_result_thr_0.05_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_4_0.3514_12_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n    ]\n    subm_list_test = get_test_from_subm_list(subm_list)\n\n    best_iou = 0.4\n    skip_box_thr = 0.01\n    out_path = SUBM_PATH + 'ensemble_retinanet_resnet101_sqr.csv'\n    predictions = ensemble(subm_list_test, best_iou, out_path, skip_box_thr)\n    print('Prediction saved: {}'.format(out_path))\n\n\ndef ensemble_experiment_v24_retinanet_resnet101_sqr_removed_radiologists():\n\n    sp = SUBM_PATH\n    subm_list = [\n        (sp + 'retina_resnet101_fold_0_0.1817_05_iou_0.3_test_result_thr_0.03_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_1_0.2072_19_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_2_0.1938_03_iou_0.3_test_result_thr_0.01_iou_0.4_test.csv', 1),\n        (sp + 'retina_resnet101_fold_3_0.1884_07_iou_0.3_test_result_thr_0.01_iou_0.45_test.csv', 1),\n        (sp + 'retina_resnet101_fold_4_0.2227_05_iou_0.3_test_result_thr_0.01_iou_0.4_test.csv', 1),\n    ]\n    subm_list_test = get_test_from_subm_list(subm_list)\n\n    best_iou = 0.4\n    skip_box_thr = 0.01\n    out_path = SUBM_PATH + 'ensemble_retinanet_resnet101_removed_rad.csv'\n    predictions = ensemble(subm_list_test, best_iou, out_path, skip_box_thr)\n    print('Prediction saved: {}'.format(out_path))\n\n\nif __name__ == '__main__':\n    ensemble_experiment_v17_retinanet_resnet101_sqr()\n    ensemble_experiment_v24_retinanet_resnet101_sqr_removed_radiologists()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}