{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pydicom\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport matplotlib\nfrom skimage import exposure\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/vinbigdata-chest-xray-abnormalities-detection'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_count = len(list(glob(f'{data_dir}/train/*.dicom')))\nprint('Image count is : ' + str(image_count))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Time to read the csv file\n%time\ntrain = pd.read_csv(data_dir+'/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info(memory_usage=\"deep\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndicom_paths = glob(f'{data_dir}/train/*.dicom')\nimgs = [dicom2array(path) for path in dicom_paths[:12]]\n\n#Without Plot\n#12.7 seconds for 12 images\n\n#Time doesn't seem to be increasing exponentially","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndicom_paths = glob(f'{data_dir}/train/*.dicom')\nimgs = [dicom2array(path) for path in dicom_paths[:12]]\nplot_imgs(imgs)\n\n#With Plot\n#5 seconds for 2 images\n#11 seconds for 5 images\n#18 seconds for 12 images\n\n#Time doesn't seem to be increasing exponentially","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### Performing Histogram Equalization on these images","metadata":{}},{"cell_type":"code","source":"imgs = [exposure.equalize_hist(img) for img in imgs]\nplot_imgs(imgs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(data_dir+'/train.csv')\ntrain_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 4))\nx = train_data['class_name'].value_counts().keys()\ny = train_data['class_name'].value_counts().values\nax.bar(x, y)\nax.set_xticklabels(x, rotation=90)\nax.set_title('Distribution of the labels')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_data[train_data['class_name']!='No finding']\nf, ax = plt.subplots(1,1, figsize=(10,8))\ntotal = float(len(train))\nax.set_xticklabels(x, rotation=90)\nsns.countplot(train['class_name'],order = train['class_name'].value_counts().index, palette='Set3')\nfor p in ax.patches:\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n            height + 3,\n            '{:1.2f}%'.format(100*height/total),\n            ha=\"center\") \n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = train.groupby(['image_id','class_name'])['image_id'].count()\ndf = pd.DataFrame(data={'Exams': tmp.values}, index=tmp.index).reset_index()\ntmp = df.groupby(['Exams','class_name']).count()\ndf2 = pd.DataFrame(data=tmp.values, index=tmp.index).reset_index()\ndf2.columns = ['Exams', 'class_name', 'Entries']\ndf2\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nfig, ax = plt.subplots(nrows=1,figsize=(12,6))\nax.set_xticklabels(x, rotation=90)\nsns.barplot(ax=ax,x = 'class_name', y='Entries', hue='Exams',data=df2, palette='Set2')\nplt.title(\"Chest examinations class and class_name\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target1 = train[train['class_name']=='Aortic enlargement']\nsns.set_style('whitegrid')\nplt.figure()\nfig, ax = plt.subplots(2,2,figsize=(12,12))\nsns.distplot(target1['x_max'],kde=True,bins=50, color=\"red\", ax=ax[0,0])\nsns.distplot(target1['y_max'],kde=True,bins=50, color=\"blue\", ax=ax[0,1])\nsns.distplot(target1['x_min'],kde=True,bins=50, color=\"green\", ax=ax[1,0])\nsns.distplot(target1['y_min'],kde=True,bins=50, color=\"magenta\", ax=ax[1,1])\nlocs, labels = plt.xticks()\nplt.tick_params(axis='both', which='major', labelsize=12)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom as dicom\n\ndef plot_example(idx_list):\n    fig, axs = plt.subplots(1, 3, figsize=(15, 10))\n    fig.subplots_adjust(hspace = .1, wspace=.1)\n    axs = axs.ravel()\n    for i in range(3):\n        image_id = train_data.loc[idx_list[i], 'image_id']\n        data_file = dicom.dcmread(data_dir+'/train/'+image_id+'.dicom')\n        img = data_file.pixel_array\n        axs[i].imshow(img, cmap='gray')\n        axs[i].set_title(train_data.loc[idx_list[i], 'class_name'])\n        axs[i].set_xticklabels([])\n        axs[i].set_yticklabels([])\n        if train_data.loc[idx_list[i], 'class_name'] != 'No finding':\n            bbox = [train_data.loc[idx_list[i], 'x_min'],\n                    train_data.loc[idx_list[i], 'y_min'],\n                    train_data.loc[idx_list[i], 'x_max'],\n                    train_data.loc[idx_list[i], 'y_max']]\n            p = matplotlib.patches.Rectangle((bbox[0], bbox[1]),\n                                             bbox[2]-bbox[0],\n                                             bbox[3]-bbox[1],\n                                             ec='r', fc='none', lw=2.)\n            axs[i].add_patch(p)\n            \nfor num in range(15):\n    idx_list = train_data[train_data['class_id']==num][0:3].index.values\n    plot_example(idx_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport glob\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nimport torchvision.transforms as transforms\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train Data Size : {}\".format(train_data.shape[0]))\ntrain_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels = train_data.iloc[:,[0,2]]\nclass_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_labels.iloc[:,0:2]\nclass_labels = class_labels.drop_duplicates(subset=[\"image_id\"])\nclass_labels.head()\nclass_labels.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_on_gpu = False\n\nif torch.cuda.is_available():\n    train_on_gpu = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        \n        # convolutional layer (sees 512x512x3 image tensor)\n        self.conv1 = nn.Conv2d(1, 4, 3, padding=1)\n\n        # convolutional layer (sees 256x256x4 tensor)\n        self.conv2 = nn.Conv2d(4, 8, 3, padding=1)\n        \n        # max pooling layer\n        self.pool = nn.MaxPool2d(2, 2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}