{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DETR Object Detection for localization and classification of thoracic abnormalities \nAuthors: Haris Mpournas & Elena Stamatelou\n","metadata":{}},{"cell_type":"markdown","source":"## 1. Select mode\nmode='train' for the training, mode='predict' for the predictions\nuncomment the one you want to enable and comment the one you want to enable ","metadata":{}},{"cell_type":"code","source":"mode = 'train'\n#mode = 'predict'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### For **mode = 'train**',\n1) Add data --> Competitions Data --> Search for \"VinBigData Chest X-ray Abnormalities Detection\" </br>\n2) Add data --> Datasets --> Search for \"vinbigdata-chest-xray-original-png\"</br>\n3) Enable the GPU in the Settings --> Accelarator --> GPU</br>\nThe output of the mode is \"detr_model.pth\"</br>\n\n#### For **mode = 'predict'**, \n1) Go to the outputs of the previous mode (train mode \"detr_model.pth\"), select \"Add new version\" and keep the created URL</br>\n2) Go back to Kaggle's notebook --> Add data --> Datasets --> Search by URL with the saved URL from the last step</br>\n3) Εnable the CPU in the Settings --> Accelarator --> CPU</br>\n ","metadata":{}},{"cell_type":"markdown","source":"## 2. Import libraries","metadata":{}},{"cell_type":"code","source":"# clone github repo of detr\n!git clone https://github.com/facebookresearch/detr.git   \n\n# general libraries\nimport os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm.autonotebook import tqdm\nimport re\nimport pydicom\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\n\n# torch.\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\nfrom torch.utils.data import DataLoader, Dataset\n\n# sklearn\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import model_selection\n\n# CV\nimport cv2\n\n# DETR FUCNTIONS FOR LOSS\nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n\n# albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n# Glob\nfrom glob import glob\n\n# ensembling \n!pip install ensemble_boxes\nfrom tqdm import tqdm\nfrom ensemble_boxes import *\n\n# mAP\n!pip install map_boxes \nfrom map_boxes import mean_average_precision_for_boxes","metadata":{"papermill":{"duration":12.706766,"end_time":"2021-03-08T07:30:05.276357","exception":false,"start_time":"2021-03-08T07:29:52.569591","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# thoracic abnormalities (classes)\nCLASSES = [\n    'Aortic enlargement', 'Atelectasis', 'Calcification', 'Cardiomegaly', 'Consolidation',\n    'ILD', 'Infiltration', 'Lung Opacity', 'Nodule/Mass', 'Other lesion', \n    'Pleural effusion', 'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis', 'No Finding'\n]\n\n# colors for visualization\nCOLORS = [[0.000, 0.447, 0.741], [0.850, 0.325, 0.098], [0.929, 0.694, 0.125],\n          [0.494, 0.184, 0.556], [0.466, 0.674, 0.188], [0.301, 0.745, 0.933]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Preprocessing image metadata","metadata":{}},{"cell_type":"code","source":"def read_images():\n    # read the images with size 512x512 \n    # add the dataset in the data section if it is not added yet\n    train_df = pd.read_csv('../input/vinbigdata-512-image-dataset/vinbigdata/train.csv')\n    train_df.fillna(0, inplace=True)\n    return train_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.1 Scale images","metadata":{}},{"cell_type":"code","source":"def scale_images(train_df):\n    # scale the coordinates of the bounding boxes from their initial values to fit the 512x512 images\n    # set to the images with no object (class 14), bounding box with coordinates [xmin=0 ymin=0 xmax=1 ymax=1]\n    train_df.loc[train_df[\"class_id\"] == 14, ['x_max', 'y_max']] = 1.0\n    train_df.loc[train_df[\"class_id\"] == 14, ['x_min', 'y_min']] = 0\n\n    # scale the input image coordinates to fit 512x512 image\n    IMG_SIZE = 512\n    train_df['xmin'] = (train_df['x_min']/train_df['width'])*IMG_SIZE\n    train_df['ymin'] = (train_df['y_min']/train_df['height'])*IMG_SIZE\n    train_df['xmax'] = (train_df['x_max']/train_df['width'])*IMG_SIZE\n    train_df['ymax'] = (train_df['y_max']/train_df['height'])*IMG_SIZE\n\n    # set to the images with no object (class 14), bounding box with coordinates [xmin=0 ymin=0 xmax=1 ymax=1]\n    train_df.loc[train_df[\"class_id\"] == 14, ['xmax', 'ymax']] = 1.0\n    train_df.loc[train_df[\"class_id\"] == 14, ['xmin', 'ymin']] = 0\n    return train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Define folds","metadata":{}},{"cell_type":"code","source":"def define_folds(train_df):\n    unique_images = train_df[\"image_id\"].unique()\n    df_split = pd.DataFrame(unique_images, columns = ['unique_images']) \n\n    # create one column with the number of fold (for the k-fold cross validation)\n    df_split[\"kfold\"] = -1\n    df_split = df_split.sample(frac=1).reset_index(drop=True)\n    y = df_split.unique_images.values\n    kf = model_selection.GroupKFold(n_splits=5)\n    for f, (t_, v_) in enumerate(kf.split(X=df_split, y=y, groups=df_split.unique_images.values)):\n        df_split.loc[v_, \"kfold\"] = f\n\n    # annotated boxes from same \"image id\" (image) should be in the same fold [during training each image with its boxes is as one input]\n    train_df[\"kfold\"] = -1\n    for ind in train_df.index: \n         train_df[\"kfold\"][ind] = df_split.loc[ df_split[\"unique_images\"] ==  train_df[\"image_id\"][ind]][\"kfold\"]\n\n    train_df.set_index('image_id', inplace=True)\n    return train_df","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":233.474768,"end_time":"2021-03-08T07:34:09.033007","exception":false,"start_time":"2021-03-08T07:30:15.558239","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Weight boxes fusion ","metadata":{}},{"cell_type":"code","source":"def boxes_fusion(df):\n    # apply weighted boxes fusion for ensemling overlapping annotated boxes\n    # Default WBF config \n    iou_thr = 0.75\n    skip_box_thr = 0.0001\n    sigma = 0.1\n    results = []\n    image_ids = df.index.unique()\n   \n    for image_id in tqdm(image_ids, total=len(image_ids)):\n        # All annotations for the current image.\n        data = df[df.index == image_id]\n        kfold = data['kfold'].unique()[0]\n        data = data.reset_index(drop=True)\n        \n        # WBF expects the coordinates in 0-1 range.\n        max_value = data.iloc[:, 4:].values.max()\n        data.loc[:, [\"xmin\", \"ymin\", \"xmax\", \"ymax\"]] = data.iloc[:, 4:] / max_value\n        #print(\"data\",data)\n        if data.class_id.unique()[0] !=14:\n            annotations = {}\n            weights = []\n            # Loop through all of the annotations\n            for idx, row in data.iterrows():\n                rad_id = row[\"rad_id\"]\n                if rad_id not in annotations:\n                    annotations[rad_id] = {\n                        \"boxes_list\": [],\n                        \"scores_list\": [],\n                        \"labels_list\": [],\n                    }\n                    # We consider all of the radiologists as equal.\n                    weights.append(1.0)\n                annotations[rad_id][\"boxes_list\"].append([row[\"xmin\"], row[\"ymin\"], row[\"xmax\"], row[\"ymax\"]])\n                annotations[rad_id][\"scores_list\"].append(1.0)\n                annotations[rad_id][\"labels_list\"].append(row[\"class_id\"])\n\n            boxes_list = []\n            scores_list = []\n            labels_list = []\n\n            for annotator in annotations.keys():\n                boxes_list.append(annotations[annotator][\"boxes_list\"])\n                scores_list.append(annotations[annotator][\"scores_list\"])\n                labels_list.append(annotations[annotator][\"labels_list\"])\n\n            # Calculate WBF\n            boxes, scores, labels = weighted_boxes_fusion(boxes_list,\n                scores_list,\n                labels_list,\n                weights=weights,\n                iou_thr=iou_thr,\n                skip_box_thr=skip_box_thr\n            )\n            for idx, box in enumerate(boxes):\n                results.append({\n                    \"image_id\": image_id,\n                    \"class_id\": int(labels[idx]),\n                    \"rad_id\": \"wbf\",\n                    \"xmin\": box[0]* max_value,\n                    \"ymin\": box[1]* max_value,\n                    \"xmax\": box[2]* max_value,\n                    \"ymax\": box[3]* max_value,\n                    \"kfold\":kfold,\n                })\n        # if class is nothing then have it once (instead of 3 times in the same image)\n        if data.class_id.unique()[0] ==14:\n            for idx, box in enumerate([0]):\n                results.append({\n                    \"image_id\": image_id,\n                    \"class_id\": data.class_id[0],\n                    \"rad_id\": \"wbf\",\n                    \"xmin\": 0,\n                    \"ymin\": 0,\n                    \"xmax\": 1,\n                    \"ymax\": 1,\n                    \"kfold\":kfold,\n                })\n            \n    results = pd.DataFrame(results)\n    return results","metadata":{"papermill":{"duration":0.108117,"end_time":"2021-03-08T07:34:09.946739","exception":false,"start_time":"2021-03-08T07:34:09.838622","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Pascal to coco","metadata":{}},{"cell_type":"code","source":"def pascal_to_coco(train_df):\n    # Good exlanation of coco, pascal etc \n    # https://albumentations.ai/docs/getting_started/bounding_boxes_augmentation/\n    train_df['coco_x'] = train_df['xmin'] + (train_df['xmax'] - train_df['xmin'] )/2\n    train_df['coco_y'] = train_df['ymin'] + (train_df['ymax'] - train_df['ymin'] )/2\n    train_df['coco_w'] = train_df['xmax'] - train_df['xmin'] \n    train_df['coco_h'] = train_df['ymax'] - train_df['ymin'] \n\n    train_df.loc[train_df['class_id'] == 14, 'coco_x'] = 1\n    train_df.loc[train_df['class_id'] == 14, 'coco_y'] = 1\n    train_df.loc[train_df['class_id'] == 14, 'coco_w'] = 0.5\n    train_df.loc[train_df['class_id'] == 14, 'coco_h'] = 0.5\n    \n    return train_df","metadata":{"papermill":{"duration":0.756067,"end_time":"2021-03-08T07:38:12.723577","exception":false,"start_time":"2021-03-08T07:38:11.96751","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Main preprocessing function","metadata":{}},{"cell_type":"code","source":"def preprocessing():\n    train_df = read_images()\n    train_df = scale_images(train_df)\n    train_df = define_folds(train_df)\n    train_df = boxes_fusion(train_df)\n    train_df.set_index('image_id', inplace=True)\n    train_df = pascal_to_coco(train_df)\n    return train_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Creating Image Dataset class\n","metadata":{"papermill":{"duration":0.722017,"end_time":"2021-03-08T07:38:18.663681","exception":false,"start_time":"2021-03-08T07:38:17.941664","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_train_transforms():\n    # image augmentations for the training set\n    return A.Compose([A.ToGray(p=0.01),\n                      A.Cutout(num_holes=10, max_h_size=32, max_w_size=32, fill_value=0, p=0.5),\n                      ToTensorV2(p=1.0)],\n                      p=1.0,\n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    # image augmentations for the validation set\n    return A.Compose([ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )","metadata":{"papermill":{"duration":0.73256,"end_time":"2021-03-08T07:38:21.603348","exception":false,"start_time":"2021-03-08T07:38:20.870788","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIR_TRAIN = '../input/vinbigdata-chest-xray-abnormalities-detection/train'\nDIR_TRAIN_PNG = '../input/vinbigdata-512-image-dataset/vinbigdata/train'\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nclass VinDataset(Dataset):\n    def __init__(self,image_ids,dataframe,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n        \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df.loc[image_id]\n        labels = records['class_id']\n        \n        image = cv2.imread(f'{DIR_TRAIN_PNG}/{image_id}.png', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        \n        # DETR takes in data in coco format    \n        boxes = records[['coco_x', 'coco_y', 'coco_w', 'coco_h']].values\n     \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.array(labels)\n    \n        if boxes.ndim == 1 : \n            boxes = np.expand_dims(boxes, axis=0)\n            labels = np.expand_dims(labels, axis=0)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.array(labels)\n\n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n\n        sample = self.transforms(**sample)\n        image = sample['image']\n        boxes = sample['bboxes']\n        labels = sample['labels']\n        \n        # Normalizing BBOXES\n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n\n        return image/255, target, image_id    ","metadata":{"papermill":{"duration":0.739748,"end_time":"2021-03-08T07:38:23.0673","exception":false,"start_time":"2021-03-08T07:38:22.327552","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. DETR model initialization","metadata":{}},{"cell_type":"code","source":"import torch.nn.functional as F\nclass DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        \n        for param in self.model.parameters():\n            param.requires_grad = True\n\n\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes+1)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"papermill":{"duration":0.740885,"end_time":"2021-03-08T07:38:33.747833","exception":false,"start_time":"2021-03-08T07:38:33.006948","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Modeling functions","metadata":{}},{"cell_type":"markdown","source":"### 6.1 Average meter","metadata":{}},{"cell_type":"code","source":"# AverageMeter - class for averaging loss,metric,etc over epochs\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"papermill":{"duration":0.736016,"end_time":"2021-03-08T07:38:39.578678","exception":false,"start_time":"2021-03-08T07:38:38.842662","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.2 Training Function\n\nTraining of DETR is unique and different from FasteRRcnn  and EfficientDET , as we train the criterion as well , the training function can be viewed here : https://github.com/facebookresearch/detr/blob/master/engine.py","metadata":{"papermill":{"duration":0.72206,"end_time":"2021-03-08T07:38:38.117095","exception":false,"start_time":"2021-03-08T07:38:37.395035","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    summary_loss = AverageMeter()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    check_repeats = []\n    for step, (images, targets, image_ids) in enumerate(tk0):\n            if image_ids in check_repeats:\n                continue\n            else:\n                check_repeats.append(image_ids)\n\n                images = list(image.to(device) for image in images)\n                targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n                #print(\"images : {}\".format(images))\n\n                output = model(images)\n\n                loss_dict = criterion(output, targets)\n                weight_dict = criterion.weight_dict\n\n                losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n\n                optimizer.zero_grad()\n\n                losses.backward()\n                optimizer.step()\n                if scheduler is not None:\n                    scheduler.step()\n\n                summary_loss.update(losses.item(),BATCH_SIZE)\n                tk0.set_postfix(loss=summary_loss.avg)\n\n    return summary_loss","metadata":{"papermill":{"duration":0.746273,"end_time":"2021-03-08T07:38:41.093304","exception":false,"start_time":"2021-03-08T07:38:40.347031","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.3 Evaluation Function","metadata":{"papermill":{"duration":0.750813,"end_time":"2021-03-08T07:38:42.567597","exception":false,"start_time":"2021-03-08T07:38:41.816784","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# for output bounding box post-processing\ndef box_cxcywh_to_xyxy(x):\n    x_c, y_c, w, h = x.unbind(1)\n    b = [(x_c - 0.5 * w), (y_c - 0.5 * h),\n         (x_c + 0.5 * w), (y_c + 0.5 * h)]\n    return torch.stack(b, dim=1)\n\ndef rescale_bboxes(out_bbox, size):\n    img_w, img_h = size\n    b = box_cxcywh_to_xyxy(out_bbox)\n    b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32).to(device='cuda')\n    return b","metadata":{"papermill":{"duration":0.735305,"end_time":"2021-03-08T07:38:44.029862","exception":false,"start_time":"2021-03-08T07:38:43.294557","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    summary_loss = AverageMeter()\n    map_df = pd.DataFrame()\n    map_df_target = pd.DataFrame()\n    \n    with torch.no_grad():\n        check_repeats_val = []\n        tk0 = tqdm(data_loader, total=len(data_loader))\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            if image_ids in check_repeats_val:\n                continue\n            else:\n                check_repeats_val.append(image_ids)\n\n                images = list(image.to(device) for image in images)\n                targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n                outputs = model(images)\n\n                # MAP targets\n                for count, label in enumerate(targets[0]['labels']):\n                    text = f'{CLASSES[label]}' \n                    xmin = targets[0]['boxes'][count][0] - (targets[0]['boxes'][count][2])/2\n                    xmax = targets[0]['boxes'][count][0] + (targets[0]['boxes'][count][2])/2  \n                    ymin = targets[0]['boxes'][count][1] - (targets[0]['boxes'][count][3])/2\n                    ymax = targets[0]['boxes'][count][1] + (targets[0]['boxes'][count][3])/2\n\n                    data = pd.DataFrame({\"ImageID\": [image_ids[0]],\"LabelName\": [text],\n                    \"XMin\": [xmin.item()], \"XMax\": [xmax.item()], \"YMin\": [ymin.item()], \"YMax\": [ymax.item()]})\n                    map_df_target = map_df_target.append(data)                \n\n                probas = outputs['pred_logits'].softmax(-1)[0, :, :-1]\n                keep = probas.max(-1).values > 0.08\n                boxes = rescale_bboxes(outputs['pred_boxes'][0, keep], (512,512))\n                prob = probas[keep]\n\n                colors = COLORS * 100\n                for p, (xmin, ymin, xmax, ymax), c in zip(prob, boxes.tolist(), colors):\n\n                    cl = p.argmax()\n                    text = f'{CLASSES[cl]}' \n                    \n                    # Dataframe for MAP\n                    data = pd.DataFrame({\"ImageID\": [image_ids[0]],\"LabelName\": [text], \"Conf\": [p[cl].item()], \"XMin\": [xmin/512], \"XMax\": [xmax/512], \"YMin\": [ymin/512], \"YMax\": [ymax/512]})\n                    map_df = map_df.append(data)          \n\n                loss_dict = criterion(outputs, targets)\n                weight_dict = criterion.weight_dict\n\n                losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n\n                summary_loss.update(losses.item(),BATCH_SIZE)\n                tk0.set_postfix(loss=summary_loss.avg)\n        \n        ann = map_df_target[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values\n        det = map_df[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values\n        mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det, iou_threshold=0.4)\n\n        print(\"mean_ap : {}\".format(mean_ap))\n        print(\"average_precisions : {}\".format(average_precisions))\n        \n    return summary_loss, mean_ap","metadata":{"papermill":{"duration":0.77267,"end_time":"2021-03-08T07:38:53.119469","exception":false,"start_time":"2021-03-08T07:38:52.346799","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.4 Run DETR","metadata":{"papermill":{"duration":0.754881,"end_time":"2021-03-08T07:38:45.546458","exception":false,"start_time":"2021-03-08T07:38:44.791577","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"papermill":{"duration":0.870487,"end_time":"2021-03-08T07:38:54.714487","exception":false,"start_time":"2021-03-08T07:38:53.844","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(train_df, fold):\n            \n    df_train = train_df[train_df['kfold'] != fold]\n    df_valid = train_df[train_df['kfold'] == fold]\n\n    train_dataset = VinDataset(\n    image_ids=df_train.index.values,\n    dataframe=df_train,\n    transforms=get_train_transforms()\n    )\n\n    valid_dataset = VinDataset(\n    image_ids=df_valid.index.values,\n    dataframe=df_valid,\n    transforms=get_valid_transforms()\n    )\n    \n    train_data_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n\n    valid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=1,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n    \n    # Bipartite Matching Loss\n    matcher = HungarianMatcher()\n    weight_dict = weight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n    losses = ['labels', 'boxes', 'cardinality']\n\n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes,num_queries=num_queries)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\n    criterion = criterion.to(device)\n    \n    LR = 3e-5\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n   \n    best_loss = 0\n    val_loss_track_switch = 0\n    all_train_losses = []\n    all_valid_losses = []\n    all_mean_ap = []\n    columns = ['train_losses', 'valid_losses', 'mean_ap']\n    df_losses = pd.DataFrame(columns = columns )\n    df_losses.to_csv(\"all_losses.csv\",mode='a', index=False)\n    for epoch in range(EPOCHS):\n        optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n        train_loss = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        if val_loss_track_switch % 2 == 0: \n            LR = LR/1.12        \n            valid_loss, map_validation = eval_fn(valid_data_loader, model,criterion, device)\n        val_loss_track_switch = val_loss_track_switch + 1\n        \n        df_losses = df_losses.append({'train_losses': train_loss.avg,'valid_losses': valid_loss.avg,'mean_ap': map_validation}, ignore_index=True)\n        df_losses.to_csv(\"all_losses.csv\",index=False, header=False,mode='a')\n        df_losses.drop(df_losses.tail(1).index,inplace=True)\n        \n        print('|EPOCH {}| TRAIN_LOSS {}| VALID_LOSS {}|'.format(epoch+1,train_loss.avg,valid_loss.avg))\n        \n        if map_validation > best_loss:\n            best_loss = map_validation\n            print('Best model found for Fold {} in Epoch {}........Saving Model'.format(fold,epoch+1))\n            torch.save(model.state_dict(), f'detr_model.pth')\n    return model","metadata":{"papermill":{"duration":0.9243,"end_time":"2021-03-08T07:38:50.151473","exception":false,"start_time":"2021-03-08T07:38:49.227173","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Main training function ","metadata":{}},{"cell_type":"code","source":"n_folds = 5\nseed = 42\nnum_classes = 15\nnum_queries = 2\nnull_class_coef = 0.2\nBATCH_SIZE = 32\nEPOCHS = 16","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_training():\n    train_df = preprocessing()\n    import torch, gc\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    # run this function for training the model\n    model = run(train_df, fold=0)\n    return","metadata":{"execution":{"iopub.status.busy":"2021-05-21T12:41:37.076283Z","iopub.execute_input":"2021-05-21T12:41:37.077322Z","iopub.status.idle":"2021-05-21T12:41:37.091513Z","shell.execute_reply.started":"2021-05-21T12:41:37.077213Z","shell.execute_reply":"2021-05-21T12:41:37.090469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == 'train':\n    model_training()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Prediction","metadata":{}},{"cell_type":"markdown","source":"### 8.1 Load model\n\nAfter training the model, comment the line \"model = run(fold=0)\", and load the model to run the predictions in the test set ","metadata":{}},{"cell_type":"code","source":"def load_model():\n    ## Loading a model\n    num_classes = 15\n    num_queries = 2\n    model = DETRModel(num_classes=num_classes,num_queries=num_queries)\n    model.load_state_dict(torch.load(\"../input/detr-model/detr_model.pth\", map_location=torch.device('cpu')))\n    return model","metadata":{"papermill":{"duration":9.024725,"end_time":"2021-03-08T07:39:07.431171","exception":false,"start_time":"2021-03-08T07:38:58.406446","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.2 Plotting results\nPlotting expected and predicted boxes with labels\n","metadata":{"papermill":{"duration":0.73016,"end_time":"2021-03-08T07:39:23.88173","exception":false,"start_time":"2021-03-08T07:39:23.15157","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# COCO classes\nCLASSES = [\n    'Aortic enlargement', 'Atelectasis', 'Calcification', 'Cardiomegaly', 'Consolidation',\n    'ILD', 'Infiltration', 'Lung Opacity', 'Nodule/Mass', 'Other lesion', \n    'Pleural effusion', 'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis', 'No Finding'\n]\n\n# colors for visualization\nCOLORS = [[0.000, 0.447, 0.741], [0.850, 0.325, 0.098], [0.929, 0.694, 0.125],\n          [0.494, 0.184, 0.556], [0.466, 0.674, 0.188], [0.301, 0.745, 0.933]]","metadata":{"papermill":{"duration":0.792281,"end_time":"2021-03-08T07:39:25.406916","exception":false,"start_time":"2021-03-08T07:39:24.614635","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for output bounding box post-processing\ndef box_cxcywh_to_xyxy(x):\n    x_c, y_c, w, h = x.unbind(1)\n    b = [(x_c - 0.5 * w), (y_c - 0.5 * h),\n         (x_c + 0.5 * w), (y_c + 0.5 * h)]\n    return torch.stack(b, dim=1)\n\ndef rescale_bboxes(out_bbox, size):\n    img_w, img_h = size\n    b = box_cxcywh_to_xyxy(out_bbox)\n    b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32)\n    return b","metadata":{"papermill":{"duration":0.867847,"end_time":"2021-03-08T07:39:27.053203","exception":false,"start_time":"2021-03-08T07:39:26.185356","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_sample_check_MAP(df_valid,model,device, index_to_show):\n    map_df = pd.DataFrame()\n    map_df_target = pd.DataFrame()    \n    \n    valid_dataset = VinDataset(\n    image_ids=df_valid.index.values,\n    dataframe=df_valid,\n    transforms=get_valid_transforms()\n    )\n    \n    valid_data_loader = DataLoader(valid_dataset,\n                                   batch_size=164,\n                                   shuffle=False,\n                                   num_workers=4,\n                                   collate_fn=collate_fn)\n    \n    images, targets, image_ids = next(iter(valid_data_loader))\n    #print(\"targets[index_to_show] : {}\".format(targets[index_to_show]))\n    _,h,w = images[index_to_show].shape # for de normalizing images\n    print(\"h,w  : {}\".format(h,w))\n    print(\"targets[index_to_show]['labels']  : {}\".format(targets[index_to_show]['labels']))\n    images = list(img.to(device) for img in images)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n    annotated_boxes = targets[index_to_show]['boxes'].cpu().numpy()\n    print(\"Anottated boxes.shape AFTER picking \"\"index_to_show .shape\"\"  : {}\".format(annotated_boxes.shape))\n    print(\"Anottated boxes[0] AFTER picking \"\"index_to_show\"\"  : {}\".format(annotated_boxes[0]))\n    annotated_boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(annotated_boxes,h,w)]\n    print(\"denormalize_bboxes Anottated boxes[0] AFTER picking \"\"index_to_show\"\" (in coco) : {}\".format(annotated_boxes[0]))    \n    #annotated_boxes = rescale_bboxes(annotated_boxes[index_to_show], (512,512))         \n    \n    # MAP targets\n    for count, label in enumerate(targets[index_to_show]['labels']):\n        print(\"label : {}\".format(label))\n        text = f'{CLASSES[label]}' \n        print(\"text : {}\".format(text))\n        xmin = targets[index_to_show]['boxes'][count][0] - (targets[index_to_show]['boxes'][count][2])/2\n        xmax = targets[index_to_show]['boxes'][count][0] + (targets[index_to_show]['boxes'][count][2])/2  \n        ymin = targets[index_to_show]['boxes'][count][1] - (targets[index_to_show]['boxes'][count][3])/2\n        ymax = targets[index_to_show]['boxes'][count][1] + (targets[index_to_show]['boxes'][count][3])/2\n\n        data = pd.DataFrame({\"ImageID\": [image_ids[0]],\"LabelName\": [text],\n        \"XMin\": [xmin.item()], \"XMax\": [xmax.item()], \"YMin\": [ymin.item()], \"YMax\": [ymax.item()]})\n        map_df_target = map_df_target.append(data)     \n    \n    model.eval()\n    model.to(device)\n    cpu_device = torch.device(\"cpu\")\n    \n    with torch.no_grad():\n        outputs = model(images)   \n\n    # keep only predictions with 0.7+ confidence\n    print(\"outputs['pred_logits'].shape : {}\".format(outputs['pred_logits'].shape))\n    print(\"outputs['pred_logits'].softmax(-1).shape : {}\".format(outputs['pred_logits'].softmax(-1).shape))\n    print(\"outputs['pred_logits'].softmax(-1)[0, :, :-1].shape : {}\".format(outputs['pred_logits'].softmax(-1)[0, :, :-1].shape))\n    probas = outputs['pred_logits'].softmax(-1)[index_to_show, :, :-1]\n    print(\"probas.shape : {}\".format(probas.shape))\n    keep = probas.max(-1).values > 0.08\n    print(\"keep : {}\".format(keep))\n    # convert boxes from [0; 1] to image scales\n    \n    print(\"outputs['pred_boxes'].shape : {}\".format(outputs['pred_boxes'].shape))\n    #print(\"outputs['pred_boxes'][index_to_show]: {}\".format(outputs['pred_boxes'][index_to_show]))\n    \n    boxes = rescale_bboxes(outputs['pred_boxes'][index_to_show, keep], (512,512))\n    print(\"Predicted boxes.shape AFTER picking \"\"index_to_show\"\"  : {}\".format(boxes.shape))\n    #print(\"Predicted boxes[0] AFTER picking \"\"index_to_show\"\" (in pascal) : {}\".format(boxes[0]))\n    prob = probas[keep]\n    #return probas[keep],bboxes_scaled,image_ids\n\n    string_from_image = f\"../input/vinbigdata-512-image-dataset/vinbigdata/train/{image_ids[index_to_show]}.png\"\n    \n    im = Image.open(string_from_image)\n    pil_img = im.convert('RGB')\n    pil_img.save('colors.jpg')\n    plt.figure(figsize=(16,10))\n    plt.imshow(pil_img)\n    ax = plt.gca()\n    colors = COLORS * 100\n    #print(\"prob : {}\".format(prob))\n    for p, (xmin, ymin, xmax, ymax), c in zip(prob, boxes.tolist(), colors):\n        ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,\n                                   fill=False, color=c, linewidth=3))\n        print(\"xmin : {}\".format(xmin))\n        print(\"ymin : {}\".format(ymin))\n        print(\"xmax : {}\".format(xmax))\n        print(\"ymax : {}\".format(ymax))\n        \n        cl = p.argmax()\n        print(\"cl : {}\".format(cl))\n        text = f'{CLASSES[cl]}: {p[cl]:0.2f}'\n        text_df = f'{CLASSES[cl]}'\n        \n        # Dataframe for MAP\n        data = pd.DataFrame({\"ImageID\": [image_ids[0]],\"LabelName\": [text_df], \"Conf\": [p[cl].item()], \"XMin\": [xmin/512], \"XMax\": [xmax/512], \"YMin\": [ymin/512], \"YMax\": [ymax/512]})\n        map_df = map_df.append(data)\n        ax.text(xmin, ymin, text, fontsize=15,\n                bbox=dict(facecolor='yellow', alpha=0.5))\n    plt.axis('off')\n    plt.show()\n\n# Plot the onces annotated by the doctors\n    plt.figure(figsize=(16,10))\n    plt.imshow(pil_img)\n    ax = plt.gca()\n    colors = COLORS * 100\n    #print(\"prob : {}\".format(prob))\n    for p, (x0, x1, x2, x3), c in zip(targets[index_to_show]['labels'], annotated_boxes, colors):\n        ax.add_patch(plt.Rectangle((x0-x2/2, x1-x3/2), x2, x3,\n                                   fill=False, color=c, linewidth=3))\n        #print(\"x0 : {}\".format(x0))\n        #print(\"x1 : {}\".format(x1))\n        #print(\"x2 : {}\".format(x2))\n        #print(\"x3 : {}\".format(x3))\n        #print(\"annotated_boxes : {}\".format(annotated_boxes))\n        cl = p\n        print(\"cl : {}\".format(cl))\n        text = f'{CLASSES[cl]}'\n        ax.text(x0, x1, text, fontsize=15,\n                bbox=dict(facecolor='yellow', alpha=0.5))\n    plt.axis('off')\n    plt.show()\n    \n    print(\"map_df_target : {}\".format(map_df_target))\n    print(\"map_df : {}\".format(map_df))\n    \n    ann = map_df_target[['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax']].values\n    det = map_df[['ImageID', 'LabelName', 'Conf', 'XMin', 'XMax', 'YMin', 'YMax']].values\n    mean_ap, average_precisions = mean_average_precision_for_boxes(ann, det)\n\n    print(\"mean_ap : {}\".format(mean_ap))\n    print(\"average_precisions : {}\".format(average_precisions))\n\n    return","metadata":{"papermill":{"duration":0.789403,"end_time":"2021-03-08T07:39:38.858436","exception":false,"start_time":"2021-03-08T07:39:38.069033","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_results():\n    train_df = preprocessing()\n    model = load_model()\n    view_sample_check_MAP(train_df[train_df['kfold'] == 0],model=model,device=torch.device('cpu'),index_to_show=10)\n    return\n# uncomment if you want to visualize the training results \n#visualize_results()","metadata":{"papermill":{"duration":0.757945,"end_time":"2021-03-08T07:39:40.408858","exception":false,"start_time":"2021-03-08T07:39:39.650913","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.3 Submission\n","metadata":{"papermill":{"duration":0.745233,"end_time":"2021-03-08T07:40:21.659637","exception":false,"start_time":"2021-03-08T07:40:20.914404","status":"completed"},"tags":[]}},{"cell_type":"code","source":"DIR_TEST = f'../input/vinbigdata-512-image-dataset/vinbigdata/test'\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nclass VinDataset_for_test(Dataset):\n    def __init__(self,image_ids,dataframe,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n    \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df.loc[image_id]\n        labels = records['class_id']\n        image = cv2.imread(f'{DIR_TEST}/{image_id}.png', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n \n        # DETR takes in data in coco format    \n        boxes = records[['coco_x', 'coco_y', 'coco_w', 'coco_h']].values\n\n        #print(\"boxes : {}\".format(boxes))\n     \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.array(labels)\n    \n        if boxes.ndim == 1 : \n            boxes = np.expand_dims(boxes, axis=0)\n            labels = np.expand_dims(labels, axis=0)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.array(labels)\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }      \n        \n        sample = self.transforms(**sample)\n        image = sample['image']\n        boxes = sample['bboxes']\n        labels = sample['labels']\n   \n        # Normalizing BBOXES\n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        #print(\"boxes after normalization : {}\".format(boxes))  \n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        #print(\"image_id : {}\".format(image_id))\n        \n        return image/255, target, image_id","metadata":{"papermill":{"duration":0.74082,"end_time":"2021-03-08T07:40:26.117264","exception":false,"start_time":"2021-03-08T07:40:25.376444","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    data = np.stack([data]*3).transpose(1,2,0)\n    return data","metadata":{"papermill":{"duration":0.723101,"end_time":"2021-03-08T07:41:00.091504","exception":false,"start_time":"2021-03-08T07:40:59.368403","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predictions(model):\n    DIR_INPUT = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection'\n    test_df = pd.read_csv(f'{DIR_INPUT}/sample_submission.csv')\n    test_df['coco_x']=[1 for i in range(3000)]\n    test_df['coco_y']=[1 for i in range(3000)]\n    test_df['coco_w']=[1 for i in range(3000)]\n    test_df['coco_h']=[1 for i in range(3000)]\n    test_df['class_id']=[1 for i in range(3000)]\n    test_df['kflod']=[88 for i in range(3000)]\n    test_df['class_confidence_box'] = [[] for i in range(3000)]\n    test_df.set_index('image_id', inplace=True)\n    valid_dataset = VinDataset_for_test(image_ids=test_df.index.values,\n                                        dataframe=test_df,\n                                        transforms=get_valid_transforms()\n                                        )\n\n    valid_data_loader = DataLoader(valid_dataset,\n                                   batch_size=1,\n                                   shuffle=False,\n                                   num_workers=4,\n                                   collate_fn=collate_fn)\n\n    dataloader_iterator = iter(valid_data_loader)\n\n    for i in range(test_df.shape[0]):\n        images, targets, image_ids = next(dataloader_iterator)\n        _,h,w = images[0].shape # for de normalizing images\n       \n        #print(\"image_ids[0] : {}\".format(image_ids[0]))\n        device=torch.device('cpu')\n        images = list(img.to(device) for img in images)\n        model.eval()\n        model.to(device)\n        cpu_device = torch.device(\"cpu\")\n\n        with torch.no_grad():\n            outputs = model(images) \n        #print(\"outputs : {}\".format(outputs))\n        # keep only predictions with 0.7+ confidence\n        #print(\"outputs['pred_logits'].shape : {}\".format(outputs['pred_logits'].shape))\n        #print(\"outputs['pred_logits'].softmax(-1) : {}\".format(outputs['pred_logits'].softmax(-1)))\n        #print(\"outputs['pred_logits'].softmax(-1)[0, :, :-1].shape : {}\".format(outputs['pred_logits'].softmax(-1)[0, :, :-1].shape))\n        probas = outputs['pred_logits'].softmax(-1)[0, :, :-1]\n        keep = probas.max(-1).values > 0.08\n        #print(\"keep : {}\".format(keep))\n        # convert boxes from [0; 1] to image scales\n\n        #print(\"outputs['pred_boxes'].shape : {}\".format(outputs['pred_boxes'].shape))\n\n        boxes = rescale_bboxes(outputs['pred_boxes'][0, keep], (512,512))\n        ####print(\"boxes : {}\".format(boxes))\n        #print(\"Predicted boxes.shape AFTER picking \"\"0\"\"  : {}\".format(boxes.shape))\n        #print(\"Predicted boxes shape (in pascal) : {}\".format(boxes.shape))\n        #print(\"Predicted boxes[0] \"\"0\"\" (in pascal) : {}\".format(boxes[0]))\n        prob = probas[keep]\n        for i in range(prob.shape[0]) :\n            class_pred = prob[i].argmax()\n            #print(\"prob[i].max() : {}\".format(prob[i].max()))\n            #print(\"class_pred : {}\".format(class_pred))\n\n        # Read the dicom with id \"image_ids[0]\" to get the actual size\n        image_dicom = read_xray(f'../input/vinbigdata-chest-xray-abnormalities-detection/test/{image_ids[0]}.dicom')\n        dicom_y = image_dicom.shape[0]\n        dicom_x = image_dicom.shape[1]\n        ####print(\"image_dicom.shape : {}\".format(image_dicom.shape))        \n        ####print(\"dicom_x : {}\".format(dicom_x))   \n        ####print(\"dicom_y : {}\".format(dicom_y))   \n\n        #print(\"prob : {}\".format(prob))\n        for i in range(prob.shape[0]) :\n            class_pred = prob[i].argmax()\n            box_list = boxes[i].tolist()\n            ####print(\"box_list : {}\".format(box_list))\n\n            # Rescale the box based on the actual size\n            box_list[0] = (dicom_x/512) *  box_list[0]\n            box_list[2] = (dicom_x/512) *  box_list[2]\n\n            box_list[1] = (dicom_y/512) *  box_list[1]\n            box_list[3] = (dicom_y/512) *  box_list[3]\n            if class_pred != 14:\n                boxz_string= ' '.join(str(e) for e in box_list)\n                test_df.loc[image_ids]['class_confidence_box'].append([str(class_pred.numpy()), str(prob[i].max().numpy()),boxz_string])\n            else:\n                boxz_string= '0 0 1 1'\n                test_df.loc[image_ids]['class_confidence_box'].append([str(class_pred.numpy()), str(prob[i].max().numpy()),boxz_string])\n\n        list_of_results = test_df.loc[image_ids]['class_confidence_box']\n        list_of_results = [' '.join(im) for im in list_of_results]\n    #    print(\"list_of_results : {}\".format(list_of_results))    \n\n        #Assign the final string to \"PredictionString\"\n        if not list_of_results:\n            pass\n        else:\n            test_df.loc[image_ids,'PredictionString'] =' '.join(list_of_results)\n\n    #print(\"test_df.loc[image_ids]['class_confidence_box'] : {}\".format(test_df.loc[image_ids]['class_confidence_box']))\n    test_df.drop(['coco_x', 'coco_y','coco_w', 'coco_h','class_id', 'kflod', 'class_confidence_box'], axis=1, inplace=True)\n    test_df['image_id'] = test_df.index\n    test_df = test_df[['image_id','PredictionString']]\n    test_df.to_csv('submission.csv', index=False)\n    return ","metadata":{"papermill":{"duration":5817.153839,"end_time":"2021-03-08T09:18:00.956641","exception":false,"start_time":"2021-03-08T07:41:03.802802","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_model():\n    print(\"to load model\")\n    model = load_model()\n    print(\"model loaded\")\n    predictions(model)\n    return","metadata":{"papermill":{"duration":0.843546,"end_time":"2021-03-08T09:18:02.612809","exception":false,"start_time":"2021-03-08T09:18:01.769263","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if mode == 'predict':\n    prediction_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}