{"cells":[{"metadata":{},"cell_type":"markdown","source":"Original notebook (full credit): https://www.kaggle.com/tanulsingh077/end-to-end-object-detection-with-transformers-detr\n\nAdapted  the notebook from the one above.\nBBox predictions work fine.\nCouldn't figure out the classifications, for some reason the predicted label is always 13 (probably because the input labels are wrong or sth). If you can figure this out let me know. I really wanted to test DETR but, running out of time"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git   #cloning github repo of detr to import its unique loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm.autonotebook import tqdm\n\n\n#Torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold\n\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion # the rule for stopping the algorithm you're using\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob\nfrom sklearn.model_selection import GroupKFold, train_test_split\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils\n\n* AverageMeter - class for averaging loss,metric,etc over epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration\n\nBasic configuration for this model"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_folds = 5\nseed = 42\nnum_classes = 14\nnum_queries = 14\nnull_class_coef = 0.5 # what  is this?\nBATCH_SIZE = 8\nLR = 2e-5\nEPOCHS = 1\nsize = 512\nDIR_TRAIN = '../input/vinbigdata-512-image-dataset/vinbigdata/train'\nDIR_TEST = '../input/vinbigdata-512-image-dataset/vinbigdata/test'\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Seed Everything\n\nSeeding everything for reproducible results"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing the Data\n\n* For preparation of data I use code from Alex's awesome kernel [here](https://www.kaggle.com/shonenkov/training-efficientdet)\n* The data can be split into any number of folds as you want , split is stratified based on number of boxes and source"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/vinbigdata-512-image-dataset/vinbigdata/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['x_min'] = df.apply(lambda row: (row.x_min)/row.width, axis =1)\ndf['y_min'] = df.apply(lambda row: (row.y_min)/row.height, axis =1)\n\ndf['x_max'] = df.apply(lambda row: (row.x_max)/row.width, axis =1)\ndf['y_max'] = df.apply(lambda row: (row.y_max)/row.height, axis =1)\n\ndf['x_mid'] = df.apply(lambda row: (row.x_max+row.x_min)/2, axis =1)\ndf['y_mid'] = df.apply(lambda row: (row.y_max+row.y_min)/2, axis =1)\n\ndf['w'] = df.apply(lambda row: (row.x_max-row.x_min), axis =1)\ndf['h'] = df.apply(lambda row: (row.y_max-row.y_min), axis =1)\n\ndf['area'] = df['w']*df['h']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating Folds\nskf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)\n\ndf['fold'] = -1\ngroup_kfold  = GroupKFold(n_splits = 3)\nfor fold, (train_index, val_index) in enumerate(group_kfold.split(df, groups=df.image_id.tolist())):\n    df.loc[val_index, 'fold'] = fold\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.drop(columns=['rad_id','width', 'height'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations\n\n* As suggested by aleksendra in her kernel ,augentations will play a major role and hence took her up advice and use awesome augmentations , cut-mix and other will be included in future versions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([A.OneOf([A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),\n                               \n                      A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.9)],p=0.9),\n                      \n                      A.ToGray(p=0.01),\n                      \n                      A.HorizontalFlip(p=0.5),\n                      \n                      A.VerticalFlip(p=0.5),\n                      \n                      A.Resize(height=512, width=512, p=1),\n                      \n                      A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n                      \n                      ToTensorV2(p=1.0)],\n                      \n                      p=1.0,\n                     \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_valid_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels'])\n                      )\n\ndef get_test_transforms():\n    return A.Compose([A.Resize(height=512, width=512, p=1.0),\n                      ToTensorV2(p=1.0)], \n                      p=1.0, \n                      )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Dataset\n\n* I hope you have the video by now , DETR accepts data in coco format which is (x,y,w,h)(for those who do not know there are two formats coco and pascal(smin,ymin,xmax,ymax) which are widely used) . So now we need to prepare data in that format"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(Dataset):\n    def __init__(self,image_ids,class_ids,dataframe,DIR,transforms=None):\n        self.image_ids = image_ids\n        self.class_ids = class_ids\n        self.df = dataframe\n        self.transforms = transforms\n        self.DIR = DIR\n        \n    def __len__(self) -> int:\n        return self.df.shape[0]\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index] # what is this?\n        class_id = self.class_ids[index] \n        records = self.df[self.df['image_id'] == image_id]\n#         print(class_id)\n\n        image = cv2.imread(f'{self.DIR}/{image_id}.png', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # DETR takes in data in coco format \n        boxes = records[['x_min', 'y_min', 'w', 'h']].values\n        \n        #Area of bb\n        area = boxes[:,2]*boxes[:,3]\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n#         labels =  np.zeros(len(boxes), dtype=np.int32)\n#         labels =  np.full(1,class_id)\n        labels = np.asarray([class_id])\n#         print(labels)\n#         print(np.full(len(boxes),class_id))\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = boxes # boxes.join?\n            labels = sample['labels']\n            \n            \n        #Normalizing BBOXES\n            \n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        \n        return image, target, image_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model\n\n* Initial DETR model is trained on coco dataset , which has 91 classes + 1 background class , hence we need to modify it to take our own number of classes\n* Also DETR model takes in 100 queries ie ,it outputs total of 100 bboxes for every image , we can very well change that too"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Matcher and Bipartite Matching Loss\n\nNow we make use of the unique loss that the model uses and for that we need to define the matcher. DETR calcuates three individual losses :\n* Classification Loss for labels(its weight can be set by loss_ce)\n* Bbox Loss (its weight can be set by loss_bbox)\n* Loss for Background class"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\ncode taken from github repo detr , 'code present in engine.py'\n'''\n\nmatcher = HungarianMatcher()\n\nweight_dict = weight_dict = {'loss_ce': 1, 'loss_bbox': 1 , 'loss_giou': 1}\n\nlosses = ['labels', 'boxes', 'cardinality']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Function\n\nTraining of DETR is unique and different from FasteRRcnn  and EfficientDET , as we train the criterion as well , the training function can be viewed here : https://github.com/facebookresearch/detr/blob/master/engine.py"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    summary_loss = AverageMeter()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    for (images, targets, image_ids) in tqdm(data_loader):\n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n\n        output = model(images)\n        \n        loss_dict = criterion(output, targets)\n        weight_dict = criterion.weight_dict\n        \n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n        optimizer.zero_grad()\n\n        losses.backward()\n        optimizer.step()\n        if scheduler is not None:\n            scheduler.step()\n        \n        summary_loss.update(losses.item(),BATCH_SIZE)\n        tk0.set_postfix(loss=summary_loss.avg)\n        \n    return summary_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Eval Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    summary_loss = AverageMeter()\n    \n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n        \n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            \n            summary_loss.update(losses.item(),BATCH_SIZE)\n            tk0.set_postfix(loss=summary_loss.avg)\n    \n    return summary_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Engine"},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(fold):\n    \n    df_train = df[df['fold'] != fold]\n    df_valid = df[df['fold'] == fold]\n    \n    train_dataset = Dataset(\n    image_ids=df_train['image_id'].values,\n    class_ids=df_train['class_id'].values,\n    dataframe=df_train,\n    DIR=DIR_TRAIN,\n    transforms=get_train_transforms()\n    )\n\n    valid_dataset = Dataset(\n    image_ids=df_valid['image_id'].values,\n    class_ids=df_valid['class_id'].values,\n    dataframe=df_valid,\n    DIR=DIR_TRAIN,\n    transforms=get_valid_transforms()\n    )\n    \n    train_data_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n\n    valid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n    )\n    \n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes,num_queries=num_queries)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\n    criterion = criterion.to(device)\n    \n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n    \n    best_loss = 10**5\n    for epoch in range(EPOCHS):\n        train_loss = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_loss = eval_fn(valid_data_loader, model,criterion, device)\n        \n        print('|EPOCH {}| TRAIN_LOSS {}| VALID_LOSS {}|'.format(epoch+1,train_loss.avg,valid_loss.avg))\n        \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n            print('Best model found for Fold {} in Epoch {}........Saving Model'.format(fold,epoch+1))\n            torch.save(model.state_dict(), f'detr_best_{fold}.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run(fold=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_prediction_string(out):\n    PredictionStrings = []\n    \n    for j, (bboxes, logits) in enumerate(zip(out['pred_boxes'], out['pred_logits'])):\n\n        oboxes = bboxes.detach().cpu().numpy()\n        oboxes = np.array([\n            np.array(box).astype(np.int32) \n            for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,512,512)\n        ])\n        prob   = logits.softmax(1).detach().cpu().numpy()[0, :]\n        # scale boxes \n        oboxes = (oboxes*2).astype(np.int32).clip(min=0, max=1023)\n\n        PredictionString = ' '.join(\n            str(np.argmax(prob)) \n            + ' ' +\n            str(round(confidence,4)) \n            + ' '\n            + ' '.join(str(int(round(float(x)))) for x in box) \n            for box, confidence in zip(oboxes, prob)\n            if confidence > confidence_thrsh\n            )\n        PredictionStrings.append(PredictionString)\n        \n    return PredictionStrings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = [ 'Aortic enlargement',\n            'Atelectasis',\n            'Calcification',\n            'Cardiomegaly',\n            'Consolidation',\n            'ILD',\n            'Infiltration',\n            'Lung Opacity',\n            'Nodule/Mass',\n            'Other lesion',\n            'Pleural effusion',\n            'Pleural thickening',\n            'Pneumothorax',\n            'Pulmonary fibrosis']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image, ImageDraw, ImageFont","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def view_sample(df_valid,model,device):\n    '''\n    Code taken from Peter's Kernel \n    https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train\n    '''\n    valid_dataset = Dataset(image_ids=df_valid['image_id'].values,\n                                 class_ids=df_valid['class_id'].values,\n                                 dataframe=df_valid,\n                                 transforms=get_valid_transforms(),\n                                 DIR=DIR_TRAIN\n                                )\n     \n    valid_data_loader = DataLoader(valid_dataset,\n                                    batch_size=15,\n                                    shuffle=False,\n                                   num_workers=4,\n                                   collate_fn=collate_fn)\n    \n    images, targets, image_ids = next(iter(valid_data_loader))\n    images, targets, image_ids = next(iter(valid_data_loader))\n    _,h,w = images[0].shape # for de normalizing images\n    \n    images = list(img.to(device) for img in images)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n    \n    boxes = targets[0]['boxes'].cpu().numpy()\n    boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(boxes,h,w)]\n    \n    model.eval()\n    model.to(device)\n    cpu_device = torch.device(\"cpu\")\n    with torch.no_grad():\n        outputs = model(images)\n        \n    outputs = [{k: v.to(cpu_device) for k, v in outputs.items()}]\n    \n    #Drawing\n#     sample = images[0].permute(1,2,0).cpu().numpy()\n#     fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n#     for box in boxes:\n#         cv2.rectangle(sample,\n#                   (box[0], box[1]),\n#                   (box[2]+box[0], box[3]+box[1]),\n#                   (220, 0, 0), 1)\n        \n\n#     oboxes = outputs[0]['pred_boxes'][0].detach().cpu().numpy()\n#     oboxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,h,w)]\n#     prob   = outputs[0]['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n#     print(prob)\n#     for box,p in zip(oboxes,prob):\n# #         print(p)\n#         if p >0.5:\n#             print(box)\n#             color = (0,0,220) #if p>0.5 else (0,0,0)\n#             cv2.rectangle(sample,\n#                   (box[0], box[1]),\n#                   (box[2]+box[0], box[3]+box[1]),\n#                   color, 1)\n    \n#     ax.set_axis_off()\n#     ax.imshow(sample)\n    output = outputs[0]\n    \n    pred_logits=output['pred_logits'][0][:, :len(CLASSES)]\n    pred_boxes=output['pred_boxes'][0]\n\n    max_output = pred_logits.softmax(-1).max(-1)\n    topk = max_output.values.topk(15)\n\n    pred_logits = pred_logits[topk.indices]\n    pred_boxes = pred_boxes[topk.indices]\n    pred_logits.shape\n\n\n    for logits, box in zip(pred_logits, pred_boxes):\n        print(pred_boxes)\n        cls = logits.argmax()\n        if cls >= len(CLASSES):\n            continue\n        label = CLASSES[cls]\n#         print(label)\n        box = box.cpu() * torch.Tensor([800, 600, 800, 600])\n        x, y, w, h = box\n        x0, x1 = x-w//2, x+w//2\n        y0, y1 = y-h//2, y+h//2\n#         drw = ImageDraw.Draw(images[0])\n#         drw.rectangle([x0, y0, x1, y1], outline='red', width=5)\n#         drw.text((x, y), label, fill='white')\n\n    return outputs[0]\n \n    \nmodel = DETRModel(num_classes=num_classes,num_queries=num_queries)\nmodel.load_state_dict(torch.load(\"./detr_best_1.pth\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outputs = view_sample(df[df['fold'] == 1],model=model,device=torch.device('cuda'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Datasettest(Dataset):\n    def __init__(self,image_ids,dataframe,DIR,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n        self.DIR = DIR\n        \n    def __len__(self) -> int:\n        return self.df.shape[0]\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index] # what is this?\n        records = self.df[self.df['image_id'] == image_id]\n        \n        image = cv2.imread(f'{self.DIR}/{image_id}.png', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n        \n        return image, image_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('../input/vinbigdata-512-image-dataset/vinbigdata/test.csv')\ntest_dataset = Datasettest(image_ids=df_test['image_id'].values,\n                                 dataframe=df_test,\n                                 transforms=get_test_transforms(),\n                                 DIR=DIR_TEST)\n\ntest_data_loader = DataLoader(test_dataset,\n                              batch_size=8,\n                              shuffle=False,\n                              num_workers=4,\n                              collate_fn=collate_fn)\n\ndevice = torch.device('cuda')\nconfidence_thrsh = 0.5\nfinal_predictionString = []\n\nfor images, image_ids in tqdm(test_data_loader, total=len(test_data_loader)):\n    _,h,w = images[0].shape # for de normalizing images\n\n    images = list(img.to(device) for img in images)\n\n    model.eval()\n    model.to(device)\n    cpu_device = torch.device(\"cuda\")\n\n    with torch.no_grad():\n        outputs = model(images)\n\n    outputs = [{k: v.to(cpu_device) for k, v in outputs.items()}]\n    out = outputs[0]\n    PredictionString = generate_prediction_string(out)\n    print(PredictionString)\n    \n    for pred in PredictionString:\n        final_predictionString.append(pred)\n    \n\npred_df = pd.DataFrame({'image_id':df_test[\"image_id\"].values, 'PredictionString':final_predictionString})\nsub_df = pd.merge(df_test, pred_df, on = 'image_id', how = 'left').fillna(\"14 1 0 0 1 1\")\nsub_df = sub_df[['image_id', 'PredictionString']]\nsub_df.to_csv('/kaggle/working/submission.csv',index = False)\nsub_df.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for value in sub_df[\"PredictionString\"].values:\n    print(value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}