{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' -q\n!pip install '../input/mean-average-precision-for-boxes/map_boxes-1.0.5-py3-none-any.whl' -q\n!pip install '../input/pytorchlightning/pytorch_lightning-1.2.4-py3-none-any.whl' -q","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nsys.path.insert(0, \"../input/omegaconf/omegaconf-master\")\nsys.path.insert(0, \"../input/efficientdetpytorch/efficientdet-pytorch-master\")\nsys.path.insert(0, \"../input/vbd-chest-xray-script\")\nsys.path.insert(0, \"../input/weightedboxesfusion\")\nsys.path.insert(0, \"../input/timm-pytorch-image-models/pytorch-image-models-master\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport pytorch_lightning as pl\n\nfrom tqdm import tqdm\n\nfrom models import XrayClassifier, XrayDetector\nfrom datamodule import XrayTestDataModule, XrayTestEnsembleDataModule\nfrom ensemble_boxes import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_clf_preds(model, image_size, test_loader, device, debug=False):\n    image_key = \"image_\" + str(image_size)\n\n    image_ids = []\n    preds = []\n\n    for index, (sample, image_id, _, _) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n        image = sample[image_key]\n        if debug: print(image_key, image.shape)\n            \n        pred = model(image.to(device))\n        image_ids.extend(image_id)\n        preds.extend(pred.detach().cpu().numpy().squeeze())\n        \n    return image_ids, preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_clf_preds_ensemble(model, image_size_list, test_loader, device, debug=False):\n    image_key_list = [\"image_\" + str(x) for x in image_size_list]\n    \n    image_ids = []\n    preds = []\n    \n    for index, (sample, image_id, _, _) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n            \n        pred_list = []\n        for key, m in zip(image_key_list, model):\n            image = sample[key]\n            if debug: print(image.shape)\n            pred = m(image.to(device))\n            pred_list.append(pred)\n            \n        pred_concat = torch.cat(pred_list, dim=1)\n        pred_mean = torch.mean(pred_concat, dim=1)\n        \n        image_ids.extend(image_id)\n        preds.extend(pred_mean.detach().cpu().numpy().squeeze())\n        \n    return image_ids, preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_clf_preds_df(model, image_size, test_loader, device, debug=False):\n    if isinstance(model, list):\n        if len(model) > 1:\n            image_ids, preds = make_clf_preds_ensemble(model, image_size, test_loader, device, debug)\n        else:\n            image_ids, preds = make_clf_preds(model[0], image_size[0], test_loader, device, debug)\n    else:\n        image_ids, preds = make_clf_preds(model, image_size, test_loader, device, debug)\n        \n    df = pd.DataFrame(data=list(zip(image_ids, preds)), columns=[\"image_id_dicom\", \"preds\"])\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_batch_pred(prediction, height, width, resize_height, resize_width):\n\n    boxes = prediction[:, :, :4].detach().cpu().numpy()\n    scores = prediction[:, :, 4].detach().cpu().numpy()\n    labels = prediction[:, :, 5].detach().cpu().numpy().astype(np.int32)\n\n    # 1-index to 0-index\n    labels -= 1        \n\n    height = height.detach().cpu().numpy()\n    height = np.expand_dims(height, axis=1)\n\n    width = width.detach().cpu().numpy()\n    width = np.expand_dims(width, axis=1)\n\n    boxes[:, :, 0] = boxes[:, :, 0] * width / resize_width\n    boxes[:, :, 1] = boxes[:, :, 1] * height / resize_height\n    boxes[:, :, 2] = boxes[:, :, 2] * width / resize_width\n    boxes[:, :, 3] = boxes[:, :, 3] * height / resize_height\n\n    boxes = boxes.astype(np.int32)\n\n    boxes[:, :, 0] = boxes[:, :, 0].clip(min=0, max=width - 1)\n    boxes[:, :, 1] = boxes[:, :, 1].clip(min=0, max=height - 1)\n    boxes[:, :, 2] = boxes[:, :, 2].clip(min=0, max=width - 1)\n    boxes[:, :, 3] = boxes[:, :, 3].clip(min=0, max=height - 1)\n    \n    return boxes, scores, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_det_preds(model, image_size, test_loader, device, debug=False, downscale_factor = 1):\n    image_key = \"image_\" + str(image_size)\n    \n    image_ids = []\n    boxes_preds = []\n    scores_preds = []\n    labels_preds = []\n\n    for index, (sample, image_id, height_raw, width_raw) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n        image = sample[image_key]\n        if debug: print(image_key, image.shape)\n        pred = model(image.to(device))\n                \n        boxes, scores, labels = convert_batch_pred(\n            pred,\n            height=height_raw,\n            width=width_raw,\n            resize_height=image_size,\n            resize_width=image_size,\n        )\n\n        image_ids.extend(image_id)\n        boxes_preds.extend(boxes)\n        scores_preds.extend(scores)\n        labels_preds.extend(labels)\n\n    return image_ids, boxes_preds, scores_preds, labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_det_preds_ensemble(model, image_size_list, test_loader, device, debug=False, downscale_factor=1, max_det_per_image=None, method=\"nms\"):\n    image_key_list = [\"image_\" + str(x) for x in image_size_list]\n    \n    image_ids = []\n    boxes_preds = []\n    scores_preds = []\n    labels_preds = []\n    \n    for index, (sample, image_id, height_raw, width_raw) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n\n        height_raw_np = height_raw.detach().cpu().numpy()\n        height_raw_np = np.expand_dims(height_raw_np, axis=1)\n\n        width_raw_np = width_raw.detach().cpu().numpy()\n        width_raw_np = np.expand_dims(width_raw_np, axis=1)\n\n        boxes_model = None\n        scores_model = None\n        labels_model = None\n        \n        for key, m in zip(image_key_list, model):\n            image = sample[key]\n            image_height = image.shape[2]\n            image_width = image.shape[3]\n\n            if debug: print(image.shape)\n            pred = m(image.to(device))\n\n            boxes, scores, labels = convert_batch_pred(\n                pred,\n                height=height_raw,\n                width=width_raw,\n                resize_height=image_height,\n                resize_width=image_width,\n            )\n\n            # normalize boxes in range 0 to 1\n            boxes = boxes.astype(np.float32)\n\n            boxes[:, :, 0] = boxes[:, :, 0] / width_raw_np\n            boxes[:, :, 1] = boxes[:, :, 1] / height_raw_np\n            boxes[:, :, 2] = boxes[:, :, 2] / width_raw_np\n            boxes[:, :, 3] = boxes[:, :, 3] / height_raw_np\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n                \n        boxes_model = np.stack(boxes_list, axis=0)\n        scores_model = np.stack(scores_list, axis=0)\n        labels_model = np.stack(labels_list, axis=0)\n        \n        batch_size = labels_model.shape[1]\n\n        iou_thr = 0.5\n        skip_box_thr = 0.0001\n\n        for index in range(batch_size):\n            if method == \"nms\":\n                boxes, scores, labels = nms(\n                    boxes_model[:, index, :, :],\n                    scores_model[:, index, :],\n                    labels_model[:, index, :],\n                    weights=None,\n                    iou_thr=iou_thr\n                )\n            elif method == \"wbf\":\n                boxes, scores, labels = weighted_boxes_fusion(\n                    boxes_model[:, index, :, :],\n                    scores_model[:, index, :],\n                    labels_model[:, index, :],\n                    weights=None,\n                    iou_thr=iou_thr,\n                    skip_box_thr=skip_box_thr,\n                )\n            else:\n                raise Exception(\"method should be 'nms' or 'wbf'\")\n\n            if max_det_per_image is not None:\n                if boxes.shape[0] > max_det_per_image:\n                    ind = np.argsort(scores)[::-1][:max_det_per_image]\n                    boxes = boxes[ind, :]\n                    scores = scores[ind]\n                    labels = labels[ind]\n           \n            labels = labels.astype(np.int64)\n\n            # transform boxes to target size\n            boxes[:, 0] = boxes[:, 0] * width_raw_np[index]\n            boxes[:, 1] = boxes[:, 1] * height_raw_np[index]\n            boxes[:, 2] = boxes[:, 2] * width_raw_np[index]\n            boxes[:, 3] = boxes[:, 3] * height_raw_np[index]\n\n            image_ids.extend([image_id[index]])\n            boxes_preds.extend([boxes])\n            scores_preds.extend([scores])\n            labels_preds.extend([labels])\n\n    return image_ids, boxes_preds, scores_preds, labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def format_pred(labels: np.ndarray, boxes: np.ndarray, scores: np.ndarray) -> str:\n    pred_strings = []\n    for label, score, bbox in zip(labels, scores, boxes):\n        xmin, ymin, xmax, ymax = bbox.astype(np.int64)\n        pred_strings.append(f\"{label} {score} {xmin} {ymin} {xmax} {ymax}\")\n    return \" \".join(pred_strings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_det_preds_df(model, image_size, test_loader, device, debug=False, method=\"nms\", max_det_per_image=None):\n    if isinstance(model, list):\n        if len(model) > 1:\n            image_ids, boxes_preds, scores_preds, labels_preds = make_det_preds_ensemble(\n                model, image_size, test_loader, device, debug, method, max_det_per_image\n            )\n        else:\n            image_ids, boxes_preds, scores_preds, labels_preds = make_det_preds(\n                model[0], image_size[0], test_loader, device, debug\n            )\n    else:\n        image_ids, boxes_preds, scores_preds, labels_preds = make_det_preds(\n            model, image_size, test_loader, device, debug\n        )\n\n    ids = []\n    pred_string_list = []\n\n\n    # class, confidence, xmin, ymin, xmax, ymax\n    for image_id, boxes, scores, labels in zip(\n        image_ids, boxes_preds, scores_preds, labels_preds\n    ):\n        image_id = image_id.split(\".\")[0]\n        ids.append(image_id)\n        \n        pred_string = format_pred(labels, boxes, scores)\n        pred_string_list.append(pred_string)\n\n    df = pd.DataFrame(\n        data=(zip(ids, pred_string_list)), columns=[\"image_id\", \"PredictionString\"]\n    )\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_combined_df(det_df, finding_df):\n    def filter_combined(row):\n        prob = 1 - row[\"preds\"]\n        row[\"PredictionString\"] += f\" 14 {prob} 0 0 1 1\"\n#         row[\"PredictionString\"] += f\" 14 1 0 0 1 1\"\n        return row\n\n    temp_df = finding_df.copy()\n    temp_df[\"image_id\"] = temp_df[\"image_id_dicom\"].str.split(\".\").str[0]\n    det_merged = pd.merge(det_df, temp_df, on=\"image_id\", how=\"left\")\n    \n    result = det_merged.apply(filter_combined, axis=1)\n    return result    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_normal_df(image_ids):\n    ids = []\n    prediction_strings = []\n\n    for image_id in image_ids:\n        image_id = image_id.split(\".\")[0]\n        ids.append(image_id)\n        pred_string = \"14 1 0 0 1 1\"\n        prediction_strings.append(pred_string)\n\n    df = pd.DataFrame(\n        data=(zip(ids, prediction_strings)), columns=[\"image_id\", \"PredictionString\"]\n    )\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# First Models: 2-Stages / CLF + DET","metadata":{}},{"cell_type":"code","source":"# ----------\n# debug mode\n# ----------\nDEBUG = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ----------\n# settings\n# ----------\npl.seed_everything(0)\n\nbatch_size = 16 if not DEBUG else 2\nnum_workers = 2\n\ndataset_dir = \"../input/vinbigdata-chest-xray-abnormalities-detection\"\n\n# No finding quantile threshole\n# 0.65 means 65% of outputs are \"No finding\"\nPRED_THR = 0.65","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# default image size\n# b0: 224, b1: 240, b2: 260, b3: 300\n# b4: 380, b5: 456, b6: 528, b7: 600, b8: 672\n\n# d0: 512, d1: 640, d2: 768, d3: 896\n# d4: 1024, d5: 1280, d6: 1280, d7: 1536\n\n# ----------\n# checkpoint\n# ----------\nclf_checkpoint = []\nclf_image_size = [\n    456,\n    1024,\n    600, 600, 600, 600, 600,\n    528, 528, 528, 528, 528,\n    600, 600, 600,\n]\n\n# b5-456\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-456-timm-bn-5folds-0_VIN-384_checkpoints_xray-classifier-epoch034-val_loss0.5986.ckpt\")\n\n# b5-1024\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-1024-timm-bn-5folds-0_VIN-410_checkpoints_xray-classifier-epoch041-val_loss0.5980.ckpt\")\n\n# b5-600\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-600-timm-bn-5folds-0_VIN-397_checkpoints_xray-classifier-epoch043-val_loss0.5975.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-600-timm-bn-5folds-1_VIN-427_checkpoints_xray-classifier-epoch036-val_loss0.5989.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-600-timm-bn-5folds-2_VIN-433_checkpoints_xray-classifier-epoch038-val_loss0.5980.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-600-timm-bn-5folds-3_VIN-435_checkpoints_xray-classifier-epoch027-val_loss0.5967.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b5-600-timm-bn-5folds-4_VIN-439_checkpoints_xray-classifier-epoch049-val_loss0.5983.ckpt\")\n\n# b6-528\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b6-528-timm-bn-5folds-0_VIN-349_checkpoints_xray-classifier-epoch042-val_loss0.5979.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b6-528-timm-bn-5folds-1_VIN-351_checkpoints_xray-classifier-epoch039-val_loss0.6011.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b6-528-timm-bn-5folds-2_VIN-352_checkpoints_xray-classifier-epoch034-val_loss0.5997.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b6-528-timm-bn-5folds-3_VIN-354_checkpoints_xray-classifier-epoch041-val_loss0.5975.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/b6-528-timm-bn-5folds-4_VIN-355_checkpoints_xray-classifier-epoch042-val_loss0.5996.ckpt\")\n\n# resnet200d-600\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/resnet200d-600-timm-bn-5folds-0_VIN-424_checkpoints_xray-classifier-epoch043-val_loss0.5986.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/resnet200d-600-timm-bn-5folds-1_VIN-440_checkpoints_xray-classifier-epoch039-val_loss0.6002.ckpt\")\nclf_checkpoint.append(\"../input/vbd-final-checkpoint/resnet200d-600-timm-bn-5folds-3_VIN-442_checkpoints_xray-classifier-epoch049-val_loss0.5969.ckpt\")\n\n\ndet_checkpoint = []\ndet_image_size = [\n    1024,\n    1024,\n    896,\n    896, 896, 896, 896, 896,\n    1024,\n    768, 768, 768, 768, 768,\n    896, 896, 896, 896,\n]\n\n# d3-1024\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d3-1024-fin-aug-bn-nms-v2-5folds-0_VIN-403_checkpoints_xray-detector-epoch042-val_loss0.7453.ckpt\")\n\n# d4-1024\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-1024-aug-bn-nms-v2-5folds-0_VIN-325_checkpoints_xray-detector-epoch040-val_loss0.7330.ckpt\")\n\n# d4-896 best LB\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-aug-nms-v2-5folds-0_VIN-269_checkpoints_xray-detector-epoch047-val_loss0.7300.ckpt\")\n\n# d4-896\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-fin-aug-bn-nms-v2-5folds-0_VIN-377_checkpoints_xray-detector-epoch049-val_loss0.7309.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-fin-aug-bn-nms-v2-5folds-1_VIN-379_checkpoints_xray-detector-epoch039-val_loss0.7414.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-fin-aug-bn-nms-v2-5folds-2_VIN-382_checkpoints_xray-detector-epoch039-val_loss0.7460.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-fin-aug-bn-nms-v2-5folds-3_VIN-383_checkpoints_xray-detector-epoch043-val_loss0.7534.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d4-896-fin-aug-bn-nms-v2-5folds-4_VIN-386_checkpoints_xray-detector-epoch046-val_loss0.7629.ckpt\")\n\n# d5-1024\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-1024-fin-aug-bn-nms-v2-5folds-0_VIN-414_checkpoints_xray-detector-epoch038-val_loss0.7259.ckpt\")\n\n# d5-768\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-768-aug-bn-nms-v2-5folds-0_VIN-328_checkpoints_xray-detector-epoch047-val_loss0.7264.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-768-aug-bn-nms-v2-5folds-1_VIN-420_checkpoints_xray-detector-epoch040-val_loss0.7390.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-768-aug-bn-nms-v2-5folds-2_VIN-421_checkpoints_xray-detector-epoch039-val_loss0.7381.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-768-aug-bn-nms-v2-5folds-3_VIN-422_checkpoints_xray-detector-epoch049-val_loss0.7481.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-768-aug-bn-nms-v2-5folds-4_VIN-423_checkpoints_xray-detector-epoch040-val_loss0.7526.ckpt\")\n\n# d5-896\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-896-aug-bn-nms-v2-5folds-0_VIN-368_checkpoints_xray-detector-epoch038-val_loss0.7235.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-896-aug-bn-nms-v2-5folds-1_VIN-436_checkpoints_xray-detector-epoch034-val_loss0.7374.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-896-aug-bn-nms-v2-5folds-2_VIN-438_checkpoints_xray-detector-epoch049-val_loss0.7312.ckpt\")\ndet_checkpoint.append(\"../input/vbd-final-checkpoint/d5-896-aug-bn-nms-v2-5folds-3_VIN-441_checkpoints_xray-detector-epoch038-val_loss0.7518.ckpt\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ----------\n# device\n# ----------\ndevice = (\n    torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n)\nprint(f\"device {device}\")\n\ntorch.set_grad_enabled(False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# -------------------------\n# Stage 1: Classification\n# -------------------------\nprint(\"Stage 1: Classification - finding vs no-finding(normal)\")\ndm_clf = XrayTestEnsembleDataModule(\n    dataset_dir=dataset_dir,\n    batch_size=batch_size,\n    num_workers=num_workers,\n    image_size_list=clf_image_size,\n)\n\ndm_clf.prepare_data()\ndm_clf.setup()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_list = []\n\nfor ckpt in clf_checkpoint:\n    clf_list.append(XrayClassifier.load_from_checkpoint(ckpt, pretrained=False))\n\nfor clf in clf_list:\n    clf.to(device)\n    clf.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_df = make_clf_preds_df(\n    clf_list, clf_image_size, dm_clf.test_dataloader(), device, debug=DEBUG\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_thres = clf_df.preds.quantile(PRED_THR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Finding    prob thr: {pred_thres}\")\nprint(f\"No finding prob thr: {1 - pred_thres}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf_df[clf_df.preds<=pred_thres].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"finding_df = clf_df[clf_df.preds > pred_thres]\nno_finding_df = clf_df[clf_df.preds <= pred_thres]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --------------------\n# Stage 2: Detection\n# --------------------\n\nimage_ids = finding_df[\"image_id_dicom\"].tolist()\n\nprint(\"Stage 2: Detection\")\ndm_det = XrayTestEnsembleDataModule(\n    dataset_dir=dataset_dir,\n    image_ids=image_ids,\n    batch_size=batch_size,\n    num_workers=num_workers,\n    image_size_list=det_image_size,\n)\n\ndm_det.prepare_data()\ndm_det.setup()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"det_list = []\n\nfor ckpt, image_size in zip(det_checkpoint, det_image_size):\n    det_list.append(\n        XrayDetector.load_from_checkpoint(\n            ckpt,\n            pretrained=False,\n            pretrained_backbone=False,\n            image_size=image_size\n        )\n    )\n\nfor det in det_list:\n    det.to(device)\n    det.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(det_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"det_df = make_det_preds_df(\n    det_list, det_image_size, dm_det.test_dataloader(), device, debug=DEBUG, method=\"nms\", max_det_per_image=None,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"det_combined_df = make_combined_df(det_df, finding_df)\ndet_combined_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"det_combined_df.iloc[0, 1][-110:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normal_df = make_normal_df(no_finding_df.image_id_dicom.tolist())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1_df = pd.concat([det_combined_df[[\"image_id\", \"PredictionString\"]], normal_df])\nsub1_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Second Models: 1-Stage / DET for all classes","metadata":{}},{"cell_type":"code","source":"def make_preds(model, image_size, test_loader, device, debug=False, downscale_factor = 1):\n    image_key = \"image_\" + str(image_size)\n    \n    image_ids = []\n    boxes_preds = []\n    scores_preds = []\n    labels_preds = []\n\n    for index, (sample, image_id, height_raw, width_raw) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n        image = sample[image_key]\n        if debug: print(image_key, image.shape)\n        pred = model(image.to(device))\n                \n        boxes, scores, labels = convert_batch_pred(\n            pred,\n            height=height_raw,\n            width=width_raw,\n            resize_height=image_size,\n            resize_width=image_size,\n        )\n\n        image_ids.extend(image_id)\n        boxes_preds.extend(boxes)\n        scores_preds.extend(scores)\n        labels_preds.extend(labels)\n\n    return image_ids, boxes_preds, scores_preds, labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_preds_ensemble(model, image_size_list, test_loader, device, debug=False, downscale_factor=1, max_det_per_image=None, method=\"nms\"):\n    image_key_list = [\"image_\" + str(x) for x in image_size_list]\n    \n    image_ids = []\n    boxes_preds = []\n    scores_preds = []\n    labels_preds = []\n    \n    for index, (sample, image_id, height_raw, width_raw) in enumerate(tqdm(test_loader)):\n        if debug and index > 2:\n            break\n\n        boxes_list = []\n        scores_list = []\n        labels_list = []\n\n        height_raw_np = height_raw.detach().cpu().numpy()\n        height_raw_np = np.expand_dims(height_raw_np, axis=1)\n\n        width_raw_np = width_raw.detach().cpu().numpy()\n        width_raw_np = np.expand_dims(width_raw_np, axis=1)\n\n        boxes_model = None\n        scores_model = None\n        labels_model = None\n        \n        for key, m in zip(image_key_list, model):\n            image = sample[key]\n            image_height = image.shape[2]\n            image_width = image.shape[3]\n            \n            if debug: print(image.shape)\n            pred = m(image.to(device))\n\n            boxes, scores, labels = convert_batch_pred(\n                pred,\n                height=height_raw,\n                width=width_raw,\n                resize_height=image_height,\n                resize_width=image_width,\n            )\n\n            # normalize boxes in range 0 to 1\n            boxes = boxes.astype(np.float32)\n\n            boxes[:, :, 0] = boxes[:, :, 0] / width_raw_np\n            boxes[:, :, 1] = boxes[:, :, 1] / height_raw_np\n            boxes[:, :, 2] = boxes[:, :, 2] / width_raw_np\n            boxes[:, :, 3] = boxes[:, :, 3] / height_raw_np\n\n            boxes_list.append(boxes)\n            scores_list.append(scores)\n            labels_list.append(labels)\n                \n        boxes_model = np.stack(boxes_list, axis=0)\n        scores_model = np.stack(scores_list, axis=0)\n        labels_model = np.stack(labels_list, axis=0)\n        \n        batch_size = labels_model.shape[1]\n\n        iou_thr = 0.5\n        skip_box_thr = 0.0001\n\n        for index in range(batch_size):\n            if method == \"nms\":\n                boxes, scores, labels = nms(\n                    boxes_model[:, index, :, :],\n                    scores_model[:, index, :],\n                    labels_model[:, index, :],\n                    weights=None,\n                    iou_thr=iou_thr\n                )\n            elif method == \"wbf\":\n                boxes, scores, labels = weighted_boxes_fusion(\n                    boxes_model[:, index, :, :],\n                    scores_model[:, index, :],\n                    labels_model[:, index, :],\n                    weights=None,\n                    iou_thr=iou_thr,\n                    skip_box_thr=skip_box_thr,\n                )\n            else:\n                raise Exception(\"method should be 'nms' or 'wbf'\")\n\n            if max_det_per_image is not None:\n                if boxes.shape[0] > max_det_per_image:\n                    ind = np.argsort(scores)[::-1][:max_det_per_image]\n                    boxes = boxes[ind, :]\n                    scores = scores[ind]\n                    labels = labels[ind]\n           \n            labels = labels.astype(np.int64)\n\n            # transform boxes to target size\n            boxes[:, 0] = boxes[:, 0] * width_raw_np[index]\n            boxes[:, 1] = boxes[:, 1] * height_raw_np[index]\n            boxes[:, 2] = boxes[:, 2] * width_raw_np[index]\n            boxes[:, 3] = boxes[:, 3] * height_raw_np[index]\n\n            image_ids.extend([image_id[index]])\n            boxes_preds.extend([boxes])\n            scores_preds.extend([scores])\n            labels_preds.extend([labels])\n\n    return image_ids, boxes_preds, scores_preds, labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_duplicate_nofinding(image_ids, boxes_preds, scores_preds, labels_preds):\n    filtered_boxes_preds = []\n    filtered_scores_preds = []\n    filtered_labels_preds = []\n    \n    for boxes, scores, labels in zip(boxes_preds, scores_preds, labels_preds):\n        # Find indices of \"No finding\".\n        ind = np.argwhere(labels==14)\n\n        if ind.size != 0:\n            # To leave unique one \"No finding\" of highest confidence score,\n            # drop index with maximum confidence score.\n            scores_nofinding = scores[ind]\n            ind = np.delete(ind, np.argmax(scores_nofinding))\n\n            # Remove duplicate of \"No finding\"\n            boxes = np.delete(boxes, ind, axis=0)\n            scores = np.delete(scores, ind, axis=0)\n            labels = np.delete(labels, ind, axis=0)\n        \n        filtered_boxes_preds.append(boxes)\n        filtered_scores_preds.append(scores)\n        filtered_labels_preds.append(labels)\n        \n    return image_ids, filtered_boxes_preds, filtered_scores_preds, filtered_labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_nofinding_probs(labels_preds, scores_preds):\n    probs = []\n    for labels, scores in zip(labels_preds, scores_preds):\n        ind = np.argwhere(labels==14)\n        prob = scores[ind] if ind.size != 0 else np.array([0.0])\n        probs.append(prob)\n        \n    probs = np.asarray(probs)\n    return probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_quantiles(data):\n    range_list = [x * .1 for x in range(10)]\n    quantiles = np.quantile(data, range_list)\n\n    print(\"--- quantiles ---\")\n    for r, q in zip(range_list, quantiles):\n        print(f\"{r:.2f}: {q:.6f}\")\n    print(\"-----------------\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clear_nofinding_det(image_ids, boxes_preds, scores_preds, labels_preds, prob_thr=1.0):\n    filtered_boxes_preds = []\n    filtered_scores_preds = []\n    filtered_labels_preds = []\n    \n    for boxes, scores, labels in zip(boxes_preds, scores_preds, labels_preds):\n        # Find indices of \"No finding\".\n        ind = np.argwhere(labels==14)\n\n        if ind.size != 0 and scores[ind] > prob_thr:\n            try:\n                ind = ind.squeeze(axis=0)\n            except:\n                raise ValueError(\"Size of ind should be 0 or 1\")\n\n            # Delete all \"finding\" detection of \"No finding\" with prob > prob_thr\n            boxes = boxes[ind]\n#             scores = scores[ind]\n            scores = np.array([1])\n            labels = labels[ind]\n        \n        filtered_boxes_preds.append(boxes)\n        filtered_scores_preds.append(scores)\n        filtered_labels_preds.append(labels)\n        \n    return image_ids, filtered_boxes_preds, filtered_scores_preds, filtered_labels_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_nofinding_box(labels: np.ndarray, boxes: np.ndarray) -> np.ndarray:\n    bbox_list = []\n    for label, bbox in zip(labels, boxes):\n        if label == 14:\n            bbox = np.array([0, 0, 1, 1])\n        bbox_list.append(bbox)\n    \n    boxes_np = np.asarray(bbox_list)\n    return boxes_np","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_preds_df(model, image_size, test_loader, device, debug=False, method=\"nms\", max_det_per_image=None, quantile_thr=0.4):\n    if isinstance(model, list):\n        if len(model) > 1:\n            image_ids, boxes_preds, scores_preds, labels_preds = make_preds_ensemble(\n                model, image_size, test_loader, device, debug, method, max_det_per_image\n            )\n        else:\n            image_ids, boxes_preds, scores_preds, labels_preds = make_preds(\n                model[0], image_size[0], test_loader, device, debug\n            )\n    else:\n        image_ids, boxes_preds, scores_preds, labels_preds = make_preds(\n            model, image_size, test_loader, device, debug\n        )\n\n    image_ids, boxes_preds, scores_preds, labels_preds = remove_duplicate_nofinding(\n        image_ids, boxes_preds, scores_preds, labels_preds\n    )\n        \n\n    ids = []\n    pred_string_list = []\n\n    \n    nofinding_probs = get_nofinding_probs(labels_preds, scores_preds)\n    print_quantiles(nofinding_probs)\n\n    prob_thr = np.quantile(nofinding_probs, quantile_thr)\n    print(f\"prob_thr: {prob_thr}\")\n\n    image_ids, boxes_preds, scores_preds, labels_preds = clear_nofinding_det(\n        image_ids, boxes_preds, scores_preds, labels_preds, prob_thr\n    )\n\n    # class, confidence, xmin, ymin, xmax, ymax\n    for image_id, boxes, scores, labels in zip(\n        image_ids, boxes_preds, scores_preds, labels_preds\n    ):\n        image_id = image_id.split(\".\")[0]\n        ids.append(image_id)\n\n        boxes = convert_nofinding_box(labels, boxes)\n        \n        pred_string = format_pred(labels, boxes, scores)\n        pred_string_list.append(pred_string)\n\n    df = pd.DataFrame(\n        data=(zip(ids, pred_string_list)), columns=[\"image_id\", \"PredictionString\"]\n    )\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ----------\n# settings\n# ----------\n# Thresholds for filtering \"No finding\"'s detection\nQUANTILE_THR = 0.70\n\n# default image size\n# d0: 512, d1: 640, d2: 768, d3: 896\n# d4: 1024, d5: 1280, d6: 1280, d7: 1536\n\n# ----------\n# checkpoint\n# ----------\ncheckpoint = []\nimage_size_list = [896, 896]\ncheckpoint.append(\"../input/vbd-final-checkpoint/d4-896-all-aug-bn-nms-v2-5folds-0_VIN-412_checkpoints_xray-detector-epoch046-val_loss0.3329.ckpt\")\ncheckpoint.append(\"../input/vbd-final-checkpoint/d4-896-all-aug-bn-nms-v2-5folds-1_VIN-431_checkpoints_xray-detector-epoch046-val_loss0.3230.ckpt\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --------------------\n# Prediction\n# --------------------\nprint(\"Prediction\")\ndm = XrayTestEnsembleDataModule(\n    dataset_dir=dataset_dir,\n    batch_size=batch_size,\n    num_workers=num_workers,\n    image_size_list=image_size_list,\n)\n\ndm.prepare_data()\ndm.setup()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\n\nfor ckpt, image_size in zip(checkpoint, image_size_list):\n    models.append(\n        XrayDetector.load_from_checkpoint(\n            ckpt,\n            pretrained=False,\n            pretrained_backbone=False,\n            image_size=image_size\n        )\n    )\n\nfor m in models:\n    m.to(device)\n    m.eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(models)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub2_df = make_preds_df(\n    models,\n    image_size_list,\n    dm.test_dataloader(),\n    device,\n    debug=DEBUG,\n    method=\"nms\",\n    max_det_per_image=None,\n    quantile_thr=QUANTILE_THR,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"def get_pred_dict(df):\n    df_dict = {}\n\n    for img_id, pred_str in df.itertuples(index=False):\n        preds = pred_str.split()\n        max_len = len(preds)\n\n        labels = []\n        for k in range(0, max_len, 6):\n            labels.append(preds[k])\n\n        scores = []\n        for k in range(1, max_len, 6):\n            scores.append(preds[k])\n\n        boxes = []\n        for k in range(2, max_len, 6):\n            boxes.append(preds[k:k+4])\n\n        df_dict[img_id] = dict(\n            labels=np.asarray(labels, dtype=np.float),\n            scores=np.asarray(scores, dtype=np.float),\n            boxes=np.asarray(boxes, dtype=np.float),\n        )\n    return df_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_ensemble_pred_dict(*args, iou_thr=0.5, weights=None, norm_factor=10_000):\n    pred_dict = {}\n\n    for img_id in args[0].keys():\n        labels_list = []\n        scores_list = []\n        boxes_list = []\n\n        for df_dict in args:\n            labels_list.append(df_dict[img_id][\"labels\"])\n            scores_list.append(df_dict[img_id][\"scores\"])\n            boxes_list.append(df_dict[img_id][\"boxes\"] / norm_factor)\n\n        boxes, scores, labels = nms(boxes_list, scores_list, labels_list, weights=weights, iou_thr=iou_thr)\n        boxes *= norm_factor\n        boxes = boxes.astype(np.int)\n        labels = labels.astype(np.int)\n\n        pred_dict[img_id] = dict(\n            labels=labels,\n            scores=scores,\n            boxes=boxes,\n        )\n    return pred_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_submission_df(pred_dict):\n    sub_df = pd.DataFrame(pred_dict.keys(), columns=[\"image_id\"])\n    sub_df[\"PredictionString\"] = \"\"\n\n    for img_id in pred_dict.keys():\n        pred_str = ''\n\n        labels = pred_dict[img_id][\"labels\"]\n        scores = pred_dict[img_id][\"scores\"]\n        boxes = pred_dict[img_id][\"boxes\"]\n\n        for label, score, box in zip(labels, scores, boxes):\n            pred_str += str(label) + \" \" + str(score) + \" \" + ' '.join(map(str, box)) + \" \"\n\n        pred_str = pred_str.strip()\n        sub_df[\"PredictionString\"][sub_df.image_id == img_id] = pred_str\n    return sub_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1_dict = get_pred_dict(sub1_df)\nsub2_dict = get_pred_dict(sub2_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_pred_dict = make_ensemble_pred_dict(sub1_dict, sub2_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = make_submission_df(ensemble_pred_dict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}