{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Chest X-ray Abnormalities Detection","metadata":{}},{"cell_type":"markdown","source":"![intro](https://www.ucsfhealth.org/-/media/project/ucsf/ucsf-health/medical-tests/hero/chest-x-ray-2x.jpg)","metadata":{}},{"cell_type":"markdown","source":"## Intro\n\nThis notebook is based on the competition - [VinBigData Chest X-ray Abnormalities Detection](https://www.kaggle.com/c/vinbigdata-chest-xray-abnormalities-detection).\n\nThe objective of this competition is to automatically localize and classify 14 types of thoracic abnormalities from chest radiographs.","metadata":{}},{"cell_type":"code","source":"!pip install bbox-visualizer","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import model_selection\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport bbox_visualizer as bbv\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom glob import glob\nfrom skimage import exposure\n\nimport torch\n# Neural networks can be constructed using the torch.nn package.\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nimport torchvision\n#from torchvision.models.detection import retinanet_resnet50_fpn\n#from torchvision.models.detection.retinanet import RetinaNet\n\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.models.detection import FasterRCNN\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nimport warnings\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '../input/vinbigdata-chest-xray-abnormalities-detection/'\nWEIGHTS_FILE = './'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(os.path.join(BASE_DIR, \"train.csv\"))\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\n\nsns.countplot(train.class_name)\nplt.xticks(fontsize=14,rotation=90)\nplt.yticks(fontsize=14)\nplt.title(\"Distribution of labels in Annotation dataframe\", fontsize=16);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[train.class_name!='No finding'].reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 8))\n\nsns.countplot(train.class_name)\nplt.xticks(fontsize=14,rotation=90)\nplt.yticks(fontsize=14)\nplt.title(\"Distribution of labels in Annotation dataframe\", fontsize=16);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of unique images in the Data-frame is {}\".format(train['image_id'].nunique()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, valid_df = train_test_split(train, test_size=0.33, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Designing the Data-set Loader","metadata":{}},{"cell_type":"code","source":"class LungsAnnotationDataset(Dataset):\n    def __init__(self,dataframe, image_dir, transforms=None):\n        super().__init__()\n        \n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        #self.labels = torch.nn.functional.one_hot(torch.tensor(dataframe.class_id))\n        self.transforms = transforms\n    \n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id]\n        \n        dcm_data = pydicom.read_file(f'{self.image_dir}/{image_id}.dicom')\n        image = apply_voi_lut(dcm_data.pixel_array, dcm_data)\n        # depending on this value, X-ray may look inverted - fix that:\n        if dcm_data.PhotometricInterpretation == \"MONOCHROME1\":\n            image = np.amax(image) - image\n            \n        #intercept = dcm_data.RescaleIntercept if \"RescaleIntercept\" in dcm_data else 0.0\n        #slope = dcm_data.RescaleSlope if \"RescaleSlope\" in dcm_data else 1.0\n        \n        #if slope != 1:\n        #    image = slope * image.astype(np.float64)\n        #    image = image.astype(np.int16)\n            \n        #image += np.int16(intercept)\n        image = np.stack([image, image, image])\n        image = image - np.min(image)\n\n        image = image / image.max()\n        #image = image * 255.0\n        #image = image.astype('float32')\n        image = exposure.equalize_hist(image) #Normalization of X-ray images\n        image = image.astype('float32')\n\n        image = image.transpose(1,2,0)\n        \n        boxes = records[['x_min','y_min','x_max','y_max']].values\n        #boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        #boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        \n        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        labels = records.class_id.values + 1\n        class_name = records.class_name.values\n        # suppose all instances are not crowd\n        iscrowd = torch.zeros((records.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = torch.tensor(boxes)\n        target['labels'] = torch.tensor(labels)\n        #target['class'] = class_name\n        #target['image_id'] = torch.tensor([index])\n        target['area'] = torch.tensor(area)\n        target['iscrowd'] = torch.tensor(iscrowd)\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': target['boxes'],\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            \n            target['boxes'] = torch.tensor(sample['bboxes'])\n            \n\n        return image, target\n    \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Albumentations\ndef get_train_transform():\n    return A.Compose([\n        ToTensorV2(),\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(),\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))\n\nDIR_TRAIN = os.path.join(BASE_DIR, \"train\")\ntrain_dataset = LungsAnnotationDataset(train_df, DIR_TRAIN,get_train_transform())\nvalid_dataset = LungsAnnotationDataset(valid_df, DIR_TRAIN,get_valid_transform())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_loader = DataLoader(\n    train_dataset,\n    batch_size=6,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)\n\nvalid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=6,\n    shuffle=False,\n    num_workers=4,\n    collate_fn=collate_fn\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, targets = next(iter(train_data_loader))\nimages = list(image.to(device) for image in images)\n\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_plot(idx,images,targets):\n    class_types = {\n        0: 'Aortic enlargement',\n        1: 'Atelectasis',\n        2: 'Calcification',\n        3: 'Cardiomegaly',\n        4: 'Consolidation',\n        5: 'ILD',\n        6: 'Infiltration',\n        7: 'Lung Opacity',\n        8: 'Nodule/Mass',\n        9: 'Other lesion',\n        10: 'Pleural effusion',\n        11: 'Pleural thickening',\n        12: 'Pneumothorax',\n        13: 'Pulmonary fibrosis'\n        }\n    boxes = targets[idx]['boxes'].cpu().numpy().astype(np.int32)\n    labels = targets[idx]['labels']-1\n    sample = images[idx].permute(1,2,0).cpu().numpy()\n    \n    img = sample.copy()\n    plt.figure(figsize=(16, 16))\n    for box,label in zip(boxes,labels):\n        bbv.add_label(img, class_types[label.item()], box, \n                      draw_bg=True,\n                      text_bg_color=(255,0,0),\n                      text_color=(0,0,0),\n                      )\n        cv2.rectangle(img ,\n                      (box[0], box[1]),\n                      (box[2], box[3]),\n                      (255,0,0), 3)\n\n\n    plt.imshow(img)    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing Chest X-ray Abnormalities","metadata":{}},{"cell_type":"code","source":"visualize_plot(2,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_plot(3,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_plot(5,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_plot(0,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_plot(1,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_plot(4,images,targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Building\n\nI tried implementing the model with both retina-net and Fast RCNN, but due to some error while implementing retina-net with torchvision, I moved on with the Fast RCNN model.","metadata":{}},{"cell_type":"code","source":"# replace the classifier with a new one, that has\n# num_classes which is user-defined\nnum_classes = 15  # 14 class (anomalies) + background\n\n# get number of input features for the classifier\n#backbone = torchvision.models.mobilenet_v2(pretrained=True).features\n#backbone.out_channels = 1280\n#model = RetinaNet(backbone,\n#                 num_classes=num_classes)\n\n# load a model; pre-trained on COCO\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)\n# get number of input features for the classifier\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n\n# replace the pre-trained head with a new one\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.0005, momentum=0.9, weight_decay=0.0005)\n#torch.optim.Adam(params, lr = 1e-3)\n#lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.1)\nlr_scheduler = None\n\nnum_epochs = 4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Averager:\n    def __init__(self):\n        self.current_total = 0.0\n        self.iterations = 0.0\n\n    def send(self, value):\n        self.current_total += value\n        self.iterations += 1\n\n    @property\n    def value(self):\n        if self.iterations == 0:\n            return 0\n        else:\n            return 1.0 * self.current_total / self.iterations\n\n    def reset(self):\n        self.current_total = 0.0\n        self.iterations = 0.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_validation(idx, images, targets, predicted):\n    class_types = {\n        0: 'Aortic enlargement',\n        1: 'Atelectasis',\n        2: 'Calcification',\n        3: 'Cardiomegaly',\n        4: 'Consolidation',\n        5: 'ILD',\n        6: 'Infiltration',\n        7: 'Lung Opacity',\n        8: 'Nodule/Mass',\n        9: 'Other lesion',\n        10: 'Pleural effusion',\n        11: 'Pleural thickening',\n        12: 'Pneumothorax',\n        13: 'Pulmonary fibrosis'\n        }\n    boxes = targets[idx]['boxes'].cpu().numpy().astype(np.int32)\n    labels = targets[idx]['labels']-1\n    \n    boxes_pred = predicted[idx]['boxes']#.detach().numpy().astype(np.int32)\n    labels_pred = predicted[idx]['labels']-1\n    scores = predicted[idx]['scores'].data.cpu().numpy()\n    # Threshold\n    boxes_pred = boxes_pred[scores >= 0.2]\n    \n    sample = images[idx].permute(1,2,0).cpu().numpy()\n    \n    img = sample.copy()\n    plt.figure(figsize=(16, 16))\n    \n    for box,label in zip(boxes,labels):\n        bbv.add_label(img, class_types[label.item()], box, \n                      draw_bg=True,\n                      text_bg_color=(255,0,0),\n                      text_color=(0,0,0),\n                      )\n        cv2.rectangle(img ,\n                      (box[0], box[1]),\n                      (box[2], box[3]),\n                      (255,0,0), 3)\n    for box,label in zip(boxes_pred,labels_pred):\n        bbv.add_label(img, class_types[label.item()], box, \n                      draw_bg=True,\n                      text_bg_color=(255, 255, 0),\n                      text_color=(0,0,0),\n                      )\n        cv2.rectangle(img ,\n                      (box[0], box[1]),\n                      (box[2], box[3]),\n                      (255, 255, 0), 3)\n\n    plt.imshow(img)\n    plt.title('Ground Truth - Red, Predicted box - Yellow',fontsize=14)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(train_dataset, fold):\n    train_data_loader = DataLoader(\n        train_dataset,\n        batch_size=2,\n        shuffle=False,\n        num_workers=4,\n        collate_fn=collate_fn\n    )\n    loss_hist = Averager()\n    itr = 1\n    for epoch in range(num_epochs):\n        loss_hist.reset()\n        model.train()\n        for images, targets in train_data_loader:\n            optimizer.zero_grad()\n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            loss_dict = model(images, targets)\n            #losses = sum(loss for loss in loss_dict.values())\n            loss_classifier, loss_box_reg, loss_objectness, loss_rpn_box_reg = loss_dict.values()\n            losses = sum([loss_objectness, \n                         10*loss_classifier, \n                         10*loss_rpn_box_reg, \n                         (0.5*loss_box_reg**2)\n                         ]\n                        )\n            loss_value = losses.item()\n            loss_hist.send(loss_value)\n\n            losses.backward()\n            optimizer.step()\n            if itr%100==0:\n                print(f\"Fold #{fold} Epoch #{epoch+1} Iteration #{itr} loss: {loss_hist.value}\")\n            \n            itr += 1\n            del loss_dict, loss_classifier, loss_box_reg, loss_objectness, loss_rpn_box_reg,loss_value\n        itr=1    \n        # update the learning rate\n        if lr_scheduler is not None:\n            lr_scheduler.step()\n\n        print(f\"Fold #{fold} Epoch #{epoch+1} loss: {loss_hist.value}\")\n        print(\"Saving Epoch's state...\")\n        torch.save(model.state_dict(), f\"model_state.pth\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training the model","metadata":{}},{"cell_type":"code","source":"k=1\ndf = train.sample(frac=1).reset_index(drop=True)\ny = train.class_id.values\n## GroupK-Fold Splitting\nkfold = model_selection.GroupKFold(n_splits=5)\n\n\nfor train_index, val_index in kfold.split(df, y,groups=df.image_id.values):\n    \n    train_dataset = LungsAnnotationDataset(df.loc[val_index], DIR_TRAIN,get_train_transform())\n    train_model(train_dataset, k)\n    if k==5:\n        valid_dataset = LungsAnnotationDataset(df.loc[train_index], DIR_TRAIN,get_train_transform())\n        val_data_loader = DataLoader(\n                                valid_dataset,\n                                batch_size=6,\n                                shuffle=False,\n                                num_workers=4,\n                                collate_fn=collate_fn\n                            )\n        \n    k+=1","metadata":{"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_ = iter(val_data_loader)\nimages_val0, targets_val0 = next(iter_)\nimages_val1, targets_val1 = next(iter_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading model weights","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load(WEIGHTS_FILE+'model_state.pth', map_location=device))\nmodel.eval()\n\nmodel.to(device)\nmodel.eval()\ndevice = torch.device(\"cuda\")\nimages_0 = list(image.to(device) for image in images_val0)\noutputs0 = model(images_0)\noutputs0 = [{k: v.to(device) for k, v in t.items()} for t in outputs0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_1 = list(image_.to(device) for image_ in images_val1)\noutputs1 = model(images_1)\noutputs1 = [{k: v.to(device) for k, v in t.items()} for t in outputs1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing the model performance over validation images","metadata":{}},{"cell_type":"code","source":"plot_validation(0, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(0, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(1, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(1, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(2, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(2, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(3, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(3, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(4, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(4, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(5, images_val0, targets_val0, outputs0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_validation(5, images_val1, targets_val1, outputs1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Credits\n\nPlease go and visit these below mentioned notebooks and support their works. Without the help of the below mentioned notebooks, it would have been much difficult for me to approach the solution.\n1. [VinBigData FasterRCNN PyTorch - Inference](https://www.kaggle.com/pestipeti/vinbigdata-fasterrcnn-pytorch-inference)\n2. [VinBigData FasterRCNN Pytorch - Train](https://www.kaggle.com/pestipeti/vinbigdata-fasterrcnn-pytorch-train)\n3. [chest generate training folds](https://www.kaggle.com/abhishek/chest-generate-training-folds)\n4. [Popular X-ray image normalization techniques](https://www.kaggle.com/raddar/popular-x-ray-image-normalization-techniques)","metadata":{}}]}