{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport pydicom\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom skimage import exposure\nimport cv2\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom bokeh.plotting import figure as bokeh_figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\nfrom PIL import Image\nfrom sklearn import preprocessing\nimport random\nfrom random import randint\n\n# ---- #\nfrom sklearn.utils import shuffle\nfrom IPython.core.display import display, HTML, Javascript\nfrom string import Template\nimport json, random\nimport IPython.display\nfrom plotly.offline import init_notebook_mode, iplot\nfrom plotly import subplots\nimport plotly.figure_factory as ff\nimport plotly as py\nimport plotly.graph_objects as go\ninit_notebook_mode(connected=True)\n\n\n# Defining all our palette colors\nprimary_blue = \"#496595\"\nprimary_blue2 = \"#85a1c1\"\nprimary_blue3 = \"#3f4d63\"\nprimary_grey = \"#c6ccd8\"\nprimary_black = \"#202022\"\nprimary_bgcolor = \"#f4f0ea\"\n\n# \"coffee\" pallette turqoise-gold.\nf1 = \"#a2885e\"\nf2 = \"#e9cf87\"\nf3 = \"#f1efd9\"\nf4 = \"#8eb3aa\"\nf5 = \"#235f83\"\nf6 = \"#b4cde3\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"html_contents =\"\"\"\n<!DOCTYPE html>\n<html lang=\"en\">\n    <head>\n    <style>\n    .toc h2{\n        color: white;\n        background: #3f4d63;\n        font-weight: 600;\n        font-family: Helvetica;\n        font-size: 23px;\n        padding: 6px 12px;\n        margin-bottom: 2px;\n    }\n    \n    .toc ol li{\n        list-style:none;\n        line-height:normal;\n        }\n     \n    .toc li{\n        background: #235f83;\n        color: white;\n        font-weight: 600;\n        font-family: Helvetica;\n        font-size: 18px;\n        margin-bottom: 2px;\n        padding: 6px 12px;\n    }\n\n    .toc ol ol li{\n        background: #fff;\n        color: #4d4d4d;\n        font-weight: 400;\n        font-size: 15px;\n        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;\n        margin-top: 0px;\n        margin-bottom: 0px;\n        padding: 3px 12px;\n    } \n    \n    .section_title{\n        background-color: #3f4d63;\n        color: white;\n        font-family: Helvetica;\n        font-size: 25px;\n        padding: 6px 12px;\n        margin-bottom: 5px;\n    }\n    .subsection_title{\n        background: #235f83;\n        color: white;\n        font-family: Helvetica;\n        font-size: 21px;\n        padding: 6px 12px;\n        margin-bottom: 0px;\n    }\n    .sidenote{\n        font-size: 13px;\n        border: 1px solid #d7d7d7;\n        padding: 1px 10px 2px;\n        box-shadow: 1px 1px 2px 1px rgba(0,0,0,0.3);\n        margin-bottom: 3px;\n    }\n    </style>\n    </head>\n    <body>\n        <div class=\"toc\">\n        \n        <ol> \n        <h2> Table of Contents </h2>\n        <li>1. Introduction </li> \n        <li>2. Dicom to Numpy array</li>\n        <li>3. EDA csv</li>\n        <ol> \n            <li>3.1 Plot Bounding Box </li>\n            <li>3.2 Plot Histogram </li> \n        </ol>\n        <li>4. References </li>\n        </ol>\n        </div>\n    </body>\n</html>\n\"\"\"\n\nHTML(html_contents)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Introduction\n\nIn this competition, we will classify 14 types of thoracic abnormalities from chest radiographs. We will work with 18K scans, annotated by experienced radiologists. We will train the model with 15K independently labeled images and then will evaluate on a test of 3K images. These annotations have been collected vua VinBigData's web-based platform, VinLab. Details on building the dataset can be found in our recent paper “VinDr-CXR: An open dataset of chest X-rays with radiologist's annotations”.\n\n"},{"metadata":{},"cell_type":"markdown","source":"# 2. Dicom to Numpy array\n\nAll the images are in Dicom format, so we will convert them into numpy array.\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Loading Data\n\ndataset_dir = '../input/vinbigdata-chest-xray-abnormalities-detection'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dicom2array(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n        \n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    return data\n\ndef plot_img(img, size = (12,12), is_rgb = True, title = \"\", cmap = 'gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap = cmap)\n    plt.suptitle(title)\n    plt.show()\n    \ndef plot_imgs(imgs, cols = 3, size = 10, is_rgb=True, title = \"\", cmap = 'gray', img_size = (300, 300)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n            \n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = dicom2array('../input/vinbigdata-chest-xray-abnormalities-detection/train/0108949daa13dc94634a7d650a05c0bb.dicom')\nplt.figure(figsize= (15,15))\nplt.imshow(img, 'gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Image without fixing MONOCHROME issue:"},{"metadata":{"trusted":true},"cell_type":"code","source":"img = dicom2array('../input/vinbigdata-chest-xray-abnormalities-detection/train/0108949daa13dc94634a7d650a05c0bb.dicom', fix_monochrome = False)\nplt.figure(figsize = (15,15))\nplt.imshow(img, 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom_paths = glob(f'{dataset_dir}/train/*.dicom')\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's try preprocess like equalize histograms to see the difference between before and after\n\nimgs = [exposure.equalize_hist(img) for img in imgs]\nplot_imgs(imgs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. EDA csv\n\nLet's perform EDA to find out important features in this dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox_area(row):\n    return (row['x_max']-row['x_min'])*(row['y_max']-row['y_min'])\n\ntrain_df = pd.read_csv(f'{dataset_dir}/train.csv')\nle = preprocessing.LabelEncoder()\ntrain_df['rad_label'] = le.fit_transform(train_df['rad_id'])\n\nfinding_df = train_df[train_df['class_name'] != 'No finding']\nfinding_df['bbox_area'] = finding_df.apply(get_bbox_area, axis = 1)\nfinding_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3.1 Plot Bounding Box"},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nimg_ids = finding_df['image_id'].values\nclass_ids = finding_df['class_id'].unique()\n\n# map label_id to specify color\nlabel2color = {class_id:[randint(0,255) for i in range(3)] for class_id in class_ids}\nthickness = 3\nscale = 6\n\nfor i in range(9):\n    img_id = random.choice(img_ids)\n    img_path = f'{dataset_dir}/train/{img_id}.dicom'\n    img = dicom2array(path=img_path)\n    img = cv2.resize(img, None, fx=1/scale, fy = 1/scale)\n    img = np.stack([img, img, img], axis = -1)\n    \n    boxes = finding_df.loc[finding_df['image_id'] == img_id, ['x_min', 'y_min', 'x_max', 'y_max']].values/scale\n    labels = finding_df.loc[finding_df['image_id'] == img_id, ['class_id']].values.squeeze()\n    \n    \n    for label_id, box in zip(labels, boxes):\n        color = label2color[label_id]\n        img = cv2.rectangle(img,\n                            (int(box[0]), int(box[1])),\n                            (int(box[2]), int(box[3])),\n                            color, thickness\n                           )\n    img = cv2.resize(img, (600, 600))\n    imgs.append(img)\n    \nplot_imgs(imgs, cmap=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3.2 Plot Histogram"},{"metadata":{"trusted":true},"cell_type":"code","source":"def hist_hover(dataframe, column, color = ['#94c8d8', '#ea5e51'], bins=30, title=\"\", value_range=None):\n    \"\"\"\n    Plotting Histogram\n    \"\"\"\n    hist, edges = np.histogram(dataframe[column], bins = bins, range=value_range)\n    hist_frame = pd.DataFrame({\n        column: hist,\n        \"left\": edges[:-1],\n        \"right\": edges[1:]\n    })\n    hist_frame[\"interval\"] = [\"%d to %d\" %\n                             (left, right) for left, right in zip(edges[:-1], edges[1:])]\n    src = ColumnDataSource(hist_frame)\n    plot = bokeh_figure(\n    plot_height=400, plot_width=600,\n        title=title, x_axis_label=column,\n        y_axis_label = 'Count'\n    )\n    \n    plot.quad(\n    bottom = 0, top = column, left = \"left\", right = \"right\",\n        source = src, fill_color = color[0], line_color = '#35838d',\n        fill_alpha=0.7, hover_fill_alpha=0.7,\n        hover_fill_color = color[1]\n    )\n    \n    hover = HoverTool(\n    tooltips= [(\"Interval\", \"@interval\"), (\"Count\", str(f\"@{column}\"))]\n    )\n    plot.add_tools(hover)\n    output_notebook()\n    show(plot)\n    \nhist_hover(finding_df, column = 'class_id')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_hover(finding_df, column = 'rad_label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_hover(finding_df, column = 'bbox_area')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. References\n\nhttps://www.kaggle.com/trungthanhnguyen0502/eda-vinbigdata-chest-x-ray-abnormalities\n\nhttps://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}