{"cells":[{"metadata":{},"cell_type":"markdown","source":"# VinBigData Chest X-ray Abnormalities Detection\n### Automatically localize and classify thoracic abnormalities from chest radiographs\n\n![image](https://storage.googleapis.com/kaggle-competitions/kaggle/24800/logos/header.png)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"ROOT = \"../input/vinbigdata-chest-xray-abnormalities-detection/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls {ROOT}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(ROOT+'train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(ROOT+'sample_submission.csv')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir(ROOT+'train/')[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir(ROOT+'train/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir(ROOT+'test/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train.image_id.unique()), len(sub.image_id.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nfig = plt.figure(figsize=(6,6))\nsns.countplot(y='class_name', data=train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(6,6))\nsns.countplot(y ='class_name', data=train[train['class_name']!=\"No finding\"]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.image_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['image_id'] == '7a1d72be9ef473df66d225c53e61f77e']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train.rad_id.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.countplot(x='rad_id', data=train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isna().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['class_name']!=\"No finding\"].isna().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train = train[train['class_name']!=\"No finding\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(new_train['x_min']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(new_train['x_max']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(new_train['y_min']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(new_train['y_max']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\n\npath = ROOT + 'train/4d390e07733ba06e5ff07412f09c0a92.dicom'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom = pydicom.dcmread(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dicom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir(dicom)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom.Rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom.Columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom.PatientSex","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nrows, columns, sex = [], [], []\nids = new_train['image_id'].unique()\nfor i in ids:\n    path = ROOT+ 'train/' + i + '.dicom'\n    dicom = pydicom.dcmread(path, stop_before_pixels=True)\n    rows.append(dicom.Rows)\n    columns.append(dicom.Columns)\n    sex.append(dicom.PatientSex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"info = pd.DataFrame({'image_id':ids, 'rows':rows, 'columns':columns, 'sex':sex})\ninfo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.countplot(info['sex']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(info['rows']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(info['columns']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nax = sns.scatterplot(x='rows', y='columns', data=info, alpha=0.3)\nplt.title(\"row(x) column(x) scatter plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nax = sns.scatterplot(x='x_min', y='y_min', data=new_train, alpha=0.3)\nplt.title(\"min coordinate scatter plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nax = sns.scatterplot(x='x_max', y='y_max', data=new_train, alpha=0.3)\nplt.title(\"max coordinate scatter plot\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.merge(train, info)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert (train['x_min'] < train['columns']).all()\nassert (train['x_min'] < train['x_max']).all()\nassert (train['y_min'] < train['y_max']).all()\nassert (train['x_max'] <= train['columns']).all()\nassert (train['y_min'] < train['rows']).all()\nassert (train['y_max'] <= train['rows']).all()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(train['rows']*train['columns']);\nplt.title(\"total pixels in images\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(train['x_max'] - train['x_min']);\nplt.title(\"width of bounding box\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize=(16, 40), sharex=True)\nfig.suptitle(\"width of bounding box for different categories\", fontsize=16)\nfor j, i in enumerate(train.class_name.unique()):\n    ttrain = train[train['class_name'] == i]\n    sns.distplot(ttrain['x_max'] - ttrain['x_min'], ax=axes[j%7, j//7]);\n    axes[j%7, j//7].title.set_text(i);\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot(train['y_max'] - train['y_min']);\nplt.title(\"height of bounding box\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize=(16, 40), sharex=True)\nfig.suptitle(\"height of bounding box for different categories\", fontsize=16)\nfor j, i in enumerate(train.class_name.unique()):\n    ttrain = train[train['class_name'] == i]\n    sns.distplot(ttrain['y_max'] - ttrain['y_min'], ax=axes[j%7, j//7]);\n    axes[j%7, j//7].title.set_text(i);\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(12,6))\nsns.distplot((train['y_max'] - train['y_min']) * (train['x_max'] - train['x_min']));\nplt.title(\"area of bounding box\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize=(16, 40), sharex=True)\nfig.suptitle(\"area of bounding box for different categories\", fontsize=16)\nfor j, i in enumerate(train.class_name.unique()):\n    ttrain = train[train['class_name'] == i]\n    sns.distplot((ttrain['y_max'] - ttrain['y_min']) * (ttrain['x_max'] - ttrain['x_min']), ax=axes[j%7, j//7]);\n    axes[j%7, j//7].title.set_text(i);\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from here https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n\n    data = data - np.min(data)\n    data = data / np.max(data)\n    return (data * 255).astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_ = plt.figure(figsize=(10, 10))\nplt.imshow(read_xray(path), cmap='gray');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport matplotlib.patches as patches\n\n_, axes = plt.subplots(4,4, figsize=(20, 20))\nfor i in range(4):\n    for j in range(4):\n        path = ROOT + 'train/' + train.iloc[random.randint(0, len(train))]['image_id'] + '.dicom'\n        axes[i][j].imshow(read_xray(path), cmap='gray');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['class_name'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(name):\n    ttrain = train[train['class_name'] == name]\n    fig, axes = plt.subplots(4,4, figsize=(20, 20))\n    fig.suptitle(name+\" examples\", fontsize=16)\n    for i in range(4):\n        for j in range(4):\n            row = ttrain.iloc[random.randint(0, len(ttrain))]\n            path = ROOT + 'train/' + row['image_id'] + '.dicom'\n            axes[i][j].imshow(read_xray(path), cmap='gray')\n            axes[i][j].add_patch(patches.Rectangle(\n                (row['x_min'], row['y_min']), \n                row['x_max'] - row['x_min'], \n                row['y_max'] - row['y_min'], \n                edgecolor='blue', \n                fill=False)\n            )\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(\"Cardiomegaly\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(\"Pleural effusion\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot(\"Pleural thickening\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"the notebook is still WIP but\n### do upvote if it helped :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}