{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport time\nfrom tqdm.notebook import tqdm\nimport torch\nimport torchvision\nimport torchvision.transforms as T\nfrom collections import defaultdict, deque\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\nfrom torchvision.models.detection.rpn import AnchorGenerator\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\n\nimport cv2\nimport os,sys,matplotlib,re\nfrom PIL import Image\nfrom skimage import exposure\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport matplotlib.image as immg\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/vinbigdata-resized-image-512/'\ndf = pd.read_csv('../input/vinbigdata-weighted-bbox-fusion/weighted_box_fused_train_vinBigData.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# replace NaN by 0\ndf = df.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dim = pd.read_csv('../input/vinbigdata-resized-image-512/train_meta.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.merge(img_dim,on='image_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['x_min'] = df['x_min']*512/df['dim1']\ndf['x_max'] = np.ceil(df['x_max']*512/df['dim1'])\ndf['y_min'] = df['y_min']*512/df['dim0']\ndf['y_max'] = np.ceil(df['y_max']*512/df['dim0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df[df['class_id']!=14]\ndf['class_id'] = df['class_id']+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_grp = df.groupby(['image_id'])\nb_fea = ['x_min', 'y_min', 'x_max', 'y_max']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Sample Image"},{"metadata":{"trusted":true},"cell_type":"code","source":"name = df.image_id.tolist()[85]\nloc = '../input/vinbigdata-resized-image-512/train/'+name+'.png'\naaa = df_grp.get_group(name)\nbbx = aaa.loc[:,b_fea]\nimg = immg.imread(loc)\nfig,ax = plt.subplots(figsize=(18,10))\nax.imshow(img,cmap='binary')\nfor i in range(len(bbx)):\n    box = bbx.iloc[i].values\n    x,y,w,h = box[0], box[1], box[2]-box[0], box[3]-box[1]\n    rect = matplotlib.patches.Rectangle((x,y),w,h,linewidth=1,edgecolor='r',facecolor='none',)\n    ax.text(*box[:2], aaa['class_id'].iloc[i], verticalalignment='top', color='white', fontsize=15, weight='bold')\n    ax.add_patch(rect)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir = \"../input/vinbigdata-resized-image-512/train/\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class XrayDataset(object):\n    def __init__(self, df, IMG_DIR, transforms=None):\n        # select only those classes that have boxes\n        \n        self.df = df\n        self.img_dir = IMG_DIR\n        self.transforms = transforms\n        self.image_ids = self.df['image_id'].unique().tolist()\n        \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, idx):\n        \n        image_id = self.image_ids[idx]\n        records = self.df[self.df['image_id'] == image_id]\n        image = cv2.imread(self.img_dir+image_id+\".png\",cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        boxes = records[['x_min', 'y_min', 'x_max', 'y_max']].to_numpy()\n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        labels = records['class_id'].tolist()\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = torch.tensor(labels)\n        target['image_id'] = torch.tensor([idx])\n        target['area'] = torch.as_tensor(area, dtype=torch.float32)\n        target['iscrowd'] = torch.zeros((records.shape[0],), dtype=torch.int64)\n    \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            \n            target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n\n        \n        return torch.tensor(image), target, image_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    #img = cv2.resize(data,(512,512)).astype(np.float32)\n    \n    return np.dstack((data,data,data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ts_img_dir1 = '../input/vinbigdata-chest-xray-abnormalities-detection/test/'\nts_img_dir2 = '../input/vinbigdata-resized-image-512/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TestDataset(object):\n    def __init__(self, df, IMG_DIR, transforms=None):\n        # select only those classes that have boxes\n        \n        self.df = df\n        self.img_dir = IMG_DIR\n        self.transforms = transforms\n        self.image_ids = self.df['image_id'].tolist()\n        \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, idx):\n        \n        image_id = self.image_ids[idx]\n        #image = read_xray(ts_img_dir+image_id+\".dicom\")\n        image = cv2.imread(self.img_dir+image_id+\".png\",cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        #image /= 255.0\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        return image, image_id,image.shape[1],image.shape[2]\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\ndef get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_test_transform(IMG_SIZE=(512,512)):\n    return A.Compose([\n         A.Normalize(mean=(0, 0, 0), std=(1, 1, 1), max_pixel_value=255.0, p=1.0),\n        A.Resize(*IMG_SIZE),\n        ToTensorV2(p=1.0)\n    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CDS = XrayDataset(df, img_dir ,get_train_transform())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset Sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimg, tar,_ = CDS[random.randint(0,1000)]\nbbox = tar['boxes'].numpy()\nlabels = tar['labels'].numpy()\nfig,ax = plt.subplots(figsize=(18,10))\nax.imshow(img.permute(1,2,0).cpu().numpy())\nfor i in range(len(bbox)):\n    box = bbox[i]\n    x,y,w,h = box[0], box[1], box[2]-box[0], box[3]-box[1]\n    rect = matplotlib.patches.Rectangle((x,y),w,h,linewidth=2,edgecolor='r',facecolor='none',)\n    ax.text(*box[:2], labels[0], verticalalignment='top', color='red', fontsize=13, weight='bold')\n    ax.add_patch(rect)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_ids = df['image_id'].unique()\ntrain_ids = image_ids[:3075]\nvalid_ids = image_ids[3075:]\ntrain_df = df[df['image_id'].isin(train_ids)]\nvalid_df = df[df['image_id'].isin(valid_ids)]\ntrain_df.shape,valid_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_dataset = XrayDataset(train_df,img_dir , get_train_transform())\nvalid_dataset = XrayDataset(valid_df,img_dir, get_valid_transform())\n\n\n# split the dataset in train and test set\nindices = torch.randperm(len(train_dataset)).tolist()\n\ntrain_data_loader = DataLoader(\n    train_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=4,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Faster RCNN pretrained "},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = len(df['class_id'].unique())+1  # 1 class (wheat) + background\n# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\n#optimiser\n#optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\noptimizer = torch.optim.Adam(params, lr=3e-4)\n#lr_scheduler\n#lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\nlr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n                    optimizer, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1\n                        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Averager:\n    def __init__(self):\n        self.current_total = 0.0\n        self.iterations = 0.0\n\n    def send(self, value):\n        self.current_total += value\n        self.iterations += 1\n\n    @property\n    def value(self):\n        if self.iterations == 0:\n            return 0\n        else:\n            return 1.0 * self.current_total / self.iterations\n\n    def reset(self):\n        self.current_total = 0.0\n        self.iterations = 0.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"num_epochs = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_hist = Averager()\nbest_epoch = 0\nmin_loss = sys.maxsize\nfor epoch in range(num_epochs):\n    loss_hist.reset()\n    tk = tqdm(train_data_loader)\n    for images, targets, image_ids in tk:\n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n        loss_dict = model(images, targets)\n\n        losses = sum(loss for loss in loss_dict.values())\n        loss_value = losses.item()\n\n        loss_hist.send(loss_value)\n\n        optimizer.zero_grad()\n        losses.backward()\n        optimizer.step()\n        \n        tk.set_postfix(train_loss=loss_value)\n    tk.close()\n    \n    # update the learning rate\n    if lr_scheduler is not None:\n        lr_scheduler.step()\n\n    print(f\"Epoch #{epoch} loss: {loss_hist.value}\") \n    \n    if loss_hist.value<min_loss:\n        print(\"Better model found at epoch {0} with {1:0.5f} loss value\".format(epoch,loss_hist.value))\n        torch.save(model.state_dict(), f\"model_state_epoch_{epoch}.pth\")\n        min_loss = loss_hist.value\n        best_epoch = epoch\n        \nmodel.load_state_dict(torch.load(f\"./model_state_epoch_{best_epoch}.pth\"));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2 align=center style=\"color:blue; border:1px dotted blue\">Validation and Prediction</h2>"},{"metadata":{},"cell_type":"markdown","source":"#### Below apply_nms function\n\n**Performs non-maximum suppression (NMS) on the boxes according to their intersection-over-union (IoU).**\n\n**NMS iteratively removes lower scoring boxes which have an IoU greater than iou_threshold with another (higher scoring) box.**\n\n**If multiple boxes have the exact same score and satisfy the IoU criterion with respect to a reference box, the selected box is not guaranteed to be the same between CPU and GPU. This is similar to the behavior of argsort in PyTorch when repeated values are present.**\n\nSource : https://pytorch.org/vision/stable/ops.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_state_dict(torch.load('../input/vinbigdata-abnormalities-detection-imgx512/model_state_epoch_12.pth'));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# the function takes the original prediction and the iou threshold.\ndef apply_nms(orig_prediction, iou_thresh=0.3):\n    \n    # torchvision returns the indices of the bboxes to keep\n    keep = torchvision.ops.nms(orig_prediction['boxes'], orig_prediction['scores'], iou_thresh)\n    \n    final_prediction = orig_prediction\n    final_prediction['boxes'] = final_prediction['boxes'][keep]\n    final_prediction['scores'] = final_prediction['scores'][keep]\n    final_prediction['labels'] = final_prediction['labels'][keep]\n    \n    return final_prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Let's take a sample from valid dataset and make Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ts_ids = submission.image_id.unique().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TCDS = TestDataset(submission,ts_img_dir2,get_test_transform())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data_loader = DataLoader(\n    TCDS,\n    batch_size=8,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction on Test Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\nmodel.eval()\nwith torch.no_grad():\n\n    for images, image_ids,_,_ in tqdm(test_data_loader):\n\n        images = list(image.to(device) for image in images)\n        outputs = model(images)\n        results.append(outputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img,tar,_ = train_dataset[6]\nbbox = tar['boxes'].numpy()\nlabels = tar['labels'].numpy()\nfig,ax = plt.subplots(figsize=(8,8))\nax.imshow(img.permute(1,2,0).cpu().numpy())\nfor i in range(len(bbox)):\n    box = bbox[i]\n    x,y,w,h = box[0], box[1], box[2]-box[0], box[3]-box[1]\n    rect = matplotlib.patches.Rectangle((x,y),w,h,linewidth=2,edgecolor='r',facecolor='none',)\n    ax.text(*box[:2], labels[0], verticalalignment='top', color='red', fontsize=13, weight='bold')\n    ax.add_patch(rect)\nplt.show()\n# put the model in evaluation mode\nmodel.eval()\nwith torch.no_grad():\n    prediction = model([img.to(device)])[0]\n    \nprint('predicted #boxes: ', len(prediction['labels']))\nprint('real #boxes: ', len(tar['labels']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction['boxes'].shape,apply_nms(prediction)['boxes'].shape,","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plotting Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_valid(img,prediction,nms=True,detect_thresh=0.5,iou_thresh=0.2):\n    '''\n    img = val_image\n    nms = use non maximum-supression \n    prediction dict\n    detection threshold\n    intersection over union threshold for non-maximum suppression (NMS)\n    '''\n    fig,ax = plt.subplots(figsize=(8,8))\n    val_img = img.permute(1,2,0).cpu().detach().numpy()\n    ax.imshow(val_img)\n    nms_prediction = apply_nms(prediction, iou_thresh=iou_thresh) if nms else prediction\n    val_scores = nms_prediction['scores'].cpu().detach().numpy()\n    bbox = nms_prediction['boxes'].cpu().detach().numpy()\n    lbls = nms_prediction['labels'].cpu().detach().numpy()\n    for i in range(len(bbox)):\n        if val_scores[i]>=detect_thresh:\n            box = bbox[i]\n            x,y,w,h = box[0], box[1], box[2]-box[0], box[3]-box[1]\n            rect = matplotlib.patches.Rectangle((x,y),w,h,linewidth=2,edgecolor='r',facecolor='none',)\n            ax.text(*box[:2], \"Class {0} | {1:.3f}\".format(lbls[i],val_scores[i]), verticalalignment='top', color='white', fontsize=12, weight='bold')\n            ax.add_patch(rect)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_valid(img,prediction,False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}