{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"!# Download models\n!git clone --depth 1 https://github.com/tensorflow/models\n\n!# Compile proto files \n! # sudo apt install -y protobuf-compiler # Already present\n%cd models/research\n!protoc object_detection/protos/*.proto --python_out=.\n%cd ..\n%cd ..\n\n!# Install cocoapi\n!pip install cython \n!git clone https://github.com/cocodataset/cocoapi.git\n%cd cocoapi/PythonAPI\n!make\n%cd ..\n%cd ..\n!cp -r cocoapi/PythonAPI/pycocotools models/research/\n\n!# Install object detection api\n%cd models/research\n!cp object_detection/packages/tf2/setup.py .\n!python -m pip install .\n%cd ..\n%cd ..","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ensemble-boxes\n!pip install tensorflow_io","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<div align='center'><font size=\"5\" color='#353B47'>Chest-X-ray</font></div>\n<div align='center'><font size=\"4\" color=\"#353B47\">Exploratory Data Analysis</font>\n<br>\n<hr>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pydicom\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom skimage import exposure\nimport cv2\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dir = '../input/vinbigdata-chest-xray-abnormalities-detection'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **dicom2array**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n    \ndef draw_bboxes(img, boxes, thickness=10, color=(255, 0, 0), img_size=(500,500)):\n    img_copy = img.copy()\n    if len(img_copy.shape) == 2:\n        img_copy = np.stack([img_copy, img_copy, img_copy], axis=-1)\n    for box in boxes:\n        img_copy = cv2.rectangle(\n            img_copy,\n            (int(box[0]), int(box[1])),\n            (int(box[2]), int(box[3])),\n            color, thickness)\n    if img_size is not None:\n        img_copy = cv2.resize(img_copy, img_size)\n    return img_copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom_paths = glob(f'{dataset_dir}/train/*.dicom')\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = [exposure.equalize_hist(img) for img in imgs]\nplot_imgs(imgs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **EDA**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from bokeh.plotting import figure as bokeh_figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn import preprocessing\nimport random\nfrom random import randint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_bbox_area(row):\n    return (row['x_max']-row['x_min'])*(row['y_max']-row['y_min'])\n\ntrain_df = pd.read_csv(f'{dataset_dir}/train.csv')\nle = preprocessing.LabelEncoder()  # encode rad_id\ntrain_df['rad_label'] = le.fit_transform(train_df['rad_id'])\n\nfinding_df = train_df[train_df['class_name'] != 'No finding']\nfinding_df['bbox_area'] = finding_df.apply(get_bbox_area, axis=1)\nfinding_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(finding_df['class_name'] == 'No finding').unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1. **Plot Bounding Box**"},{"metadata":{"trusted":true},"cell_type":"code","source":"imgs = []\nimg_ids = finding_df['image_id'].values\nclass_ids = finding_df['class_id'].unique()\n\n# map label_id to specify color\nlabel2color = {class_id:[randint(0,255) for i in range(3)] for class_id in class_ids}\nthickness = 3\nscale = 5\n\n\nfor i in range(8):\n    img_id = random.choice(img_ids)\n    img_path = f'{dataset_dir}/train/{img_id}.dicom'\n    img = dicom2array(path=img_path)\n    img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n    img = np.stack([img, img, img], axis=-1)\n    \n    boxes = finding_df.loc[finding_df['image_id'] == img_id, ['x_min', 'y_min', 'x_max', 'y_max']].values/scale\n    labels = finding_df.loc[finding_df['image_id'] == img_id, ['class_id']].values.squeeze()\n    \n    for label_id, box in zip(labels, boxes):\n        color = label2color[label_id]\n        img = cv2.rectangle(\n            img,\n            (int(box[0]), int(box[1])),\n            (int(box[2]), int(box[3])),\n            color, thickness\n    )\n    img = cv2.resize(img, (500,500))\n    imgs.append(img)\n    \nplot_imgs(imgs, cmap=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From the images it can be seen that the boxes with the same color tend to overlap. This indicates that the boxes were made form multiple radiologist. There are also some overlap with boxes of different color."},{"metadata":{},"cell_type":"markdown","source":"## **2. Histogram**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def hist_hover(dataframe, column, color=[\"#94c8d8\", \"#ea5e51\"], bins=30, title=\"\", value_range=None):\n    \"\"\"\n    Plot histogram\n    \"\"\"\n    hist, edges = np.histogram(dataframe[column], bins=bins, range=value_range)\n    hist_frame = pd.DataFrame({\n        column: hist,\n        \"left\": edges[:-1],\n        \"right\": edges[1:]\n    })\n    hist_frame[\"interval\"] = [\"%d to %d\" %\n                              (left, right) for left, right in zip(edges[:-1], edges[1:])]\n    src = ColumnDataSource(hist_frame)\n    plot = bokeh_figure(\n        plot_height=400, plot_width=600,\n        title=title, x_axis_label=column,\n        y_axis_label=\"Count\"\n    )\n    plot.quad(\n        bottom=0, top=column, left=\"left\", right=\"right\",\n        source=src, fill_color=color[0], line_color=\"#35838d\",\n        fill_alpha=0.7, hover_fill_alpha=0.7,\n        hover_fill_color=color[1]\n    )\n    hover = HoverTool(\n        tooltips=[(\"Interval\", \"@interval\"), (\"Count\", str(f\"@{column}\"))]\n    )\n    plot.add_tools(hover)\n    output_notebook()\n    show(plot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.graph_objects as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_distribution_classes(x_values, y_values):\n    \n    colors = ['rgb(26, 118, 255)',] * 15\n    colors[0] = 'lightslategray'\n\n    fig = go.Figure(data=[go.Bar(\n        x=x_values, \n        y=y_values,\n        text=y_values,\n        marker_color=colors\n    )])\n\n    fig.update_layout(height=600, width=900, title_text=\"Distribution of radiographic observations\")\n    fig.update_xaxes(type=\"category\")\n\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indexes = train_df.class_name.unique()\ncounts = train_df.class_name.value_counts()\n\nsorted_dict = dict(zip(indexes, counts))\nsorted_dict = {k: v for k, v in sorted(sorted_dict.items(), key=lambda item: item[1], reverse = True)}\n\nx = list(sorted_dict.keys())\ny = list(sorted_dict.values())\n\nplot_distribution_classes(x, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are some imbalance in the data. Class_id 14 corresponds to 'No Finding'"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_distribution_radiologist_obs(x_values, y_values):\n    \n    colors = ['lightslategray',] * 17\n    colors[0] = 'crimson'\n    colors[1] = 'crimson'\n    colors[2] = 'crimson'\n    \n    fig = go.Figure(data=[go.Bar(\n        x=x_values, \n        y=y_values,\n        text=y_values,\n        marker_color=colors\n    )])\n\n    fig.update_layout(height=600, width=900, title_text=\"Distribution of radiologist observations\")\n    fig.update_xaxes(type=\"category\")\n\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show distribution of count of radio\nindexes = train_df[['rad_id', 'image_id']].groupby(['rad_id']).agg(['count']).index\ncounts = train_df[['rad_id', 'image_id']].groupby(['rad_id']).agg(['count']).values.ravel()\n\nsorted_dict = dict(zip(indexes, counts))\nsorted_dict = {k: v for k, v in sorted(sorted_dict.items(), key=lambda item: item[1], reverse = True)}\n\nx = list(sorted_dict.keys())\ny = list(sorted_dict.values())\n\nplot_distribution_radiologist_obs(x, y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Radiologist 'R9' 'R10' and 'R8' contributes to the observations significantly more than other radiologist"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.rad_id","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **3. Dicom Embedded Files**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nfrom matplotlib.patches import Rectangle\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\nimport random\nimport matplotlib.pyplot as plt\nimport plotly\nimport plotly.graph_objects as go\nimport plotly.express as px\nfrom pydicom import dcmread\nfrom tqdm import tqdm\nimport multiprocessing as mp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = \"../input/vinbigdata-chest-xray-abnormalities-detection\"\nds = dcmread(os.path.join(PATH, 'train', '000434271f63a053c4128a0ba6352c7f.dicom'))\nds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def infer_one_observation(file_path):\n    \n    ds = dcmread(file_path)\n    image_id = os.path.basename(file_path)\n        \n    observation_dict = {}\n    observation_dict['image_id'] = image_id.split(sep=\".\")[0]\n    \n    file_meta_keys = list(ds.file_meta._dict.keys())\n    remaining_meta_keys = list(ds._dict.keys())\n    \n    for key in file_meta_keys:\n        observation_dict[str(key)] = str(ds.file_meta[key].value)\n        \n    # Not taking into account pixel value\n    for key in remaining_meta_keys:\n        if key != (0x7fe0, 0x0010):\n            observation_dict[str(key)] = str(ds[key].value)\n        \n    return observation_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mapper_dict = {'image_id':'image_id',\n               '(0002, 0000)':\"File Meta Information Group Length\",\n               '(0002, 0001)':\"File Meta Information Version\",\n               '(0002, 0002)':\"Media Storage SOP Class UID\",\n               '(0002, 0003)':\"Media Storage SOP Instance UID\",\n               '(0002, 0010)':\"Transfer Syntax UID\",\n               '(0002, 0012)':\"Implementation Class UID\",\n               '(0002, 0013)':\"Implementation Version Name\",\n               '(0002, 0016)':\"Source Application Entity Title\",\n               '(0010, 0040)':\"Patient's Sex\",\n               '(0010, 1010)':\"Patient's Age\",\n               '(0010, 1020)':\"Patient's Size\",\n               '(0010, 1030)':\"Patient's Weight\",\n               '(0028, 0002)':\"Samples per Pixel\",\n               '(0028, 0004)':\"Photometric Interpretation\",\n               '(0028, 0008)':\"Number of Frames\",\n               '(0028, 0010)':\"Rows\",\n               '(0028, 0011)':\"Columns\",\n               '(0028, 0030)':\"Pixel Spacing\",\n               '(0028, 0034)':\"Pixel Aspect Ratio\",\n               '(0028, 0100)':\"Bits Allocated\",\n               '(0028, 0101)':\"Bits Stored\",\n               '(0028, 0102)':\"High Bit\",\n               '(0028, 0103)':\"Pixel Representation\",\n               '(0028, 0106)':\"Smallest Image Pixel Value\",\n               '(0028, 0107)':\"Largest Image Pixel Value\",\n               '(0028, 1050)':\"Window Center\",\n               '(0028, 1051)':\"Window Width\",\n               '(0028, 1052)':\"Rescale Intercept\",\n               '(0028, 1053)':\"Rescale Slope\",\n               '(0028, 2110)':\"Lossy Image Compression\",\n               '(0028, 2112)':\"Lossy Image Compression Ratio\",\n               '(0028, 2114)':\"Lossy Image Compression Method\"\n              }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_meta_information(folder):\n    \n    folder_filenames = os.listdir(os.path.join(PATH, folder))\n    one_obs = infer_one_observation(os.path.join(PATH, folder, folder_filenames[0]))\n    metadata = pd.DataFrame(columns = one_obs.keys())\n\n    for filename in tqdm(folder_filenames):\n        one_obs = infer_one_observation(os.path.join(PATH, folder, filename))\n        metadata = metadata.append(one_obs, ignore_index=True)\n        \n    metadata.columns = metadata.columns.map(mapper_dict)\n    metadata.to_csv(f\"{folder}_dicom_metadata.csv\", index=False)\n    \n    return metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filenames = os.listdir(os.path.join(PATH, \"train\"))\nBATCH_SIZES = list(map(lambda x:int(x/100), [0, 4000, 8000, 12000, 15000]))\n\none_obs_train = infer_one_observation(os.path.join(PATH, 'train', train_filenames[0]))\n\ndef extract_train_meta(BATCH):\n    \n    print(\"----- Train metadata extraction starting -----\")\n    \n    index_loop = 0\n    metadata = pd.DataFrame(columns = one_obs_train.keys())\n    \n    for filename in train_filenames[BATCH_SIZES[BATCH]:BATCH_SIZES[BATCH+1]]:\n        one_obs = infer_one_observation(os.path.join(PATH, 'train', filename))\n        metadata = metadata.append(one_obs_train, ignore_index=True)\n\n        if index_loop%10==0:\n            print(f\"{index_loop} train DICOM metadata successfully extracted\")\n        index_loop+=1\n    \n    metadata.columns = metadata.columns.map(mapper_dict)\n    metadata.to_csv(f\"batch_{BATCH}_dicom_train_metadata.csv\", index=False)\n    \n    print(\"----- Train metadata extraction fully completed -----\")\n    \n    return metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filenames = os.listdir(os.path.join(PATH, \"test\"))\nBATCH_SIZES = list(map(lambda x:int(x/10), [0, 750, 1500, 2250, 3000]))\n\none_obs_test = infer_one_observation(os.path.join(PATH, 'test', test_filenames[0]))\n\ndef extract_test_meta(BATCH):\n    print(\"----- Test metadata extraction starting -----\")\n    index_loop = 0\n    metadata = pd.DataFrame(columns = one_obs_test.keys())\n    \n    for filename in test_filenames[BATCH_SIZES[BATCH]:BATCH_SIZES[BATCH+1]]:\n        one_obs = infer_one_observation(os.path.join(PATH, 'test', filename))\n        metadata = metadata.append(one_obs_test, ignore_index=True)\n\n        if index_loop%10==0:\n            print(f\"{index_loop} test DICOM metadata successfully extracted\")\n        index_loop+=1\n    \n    metadata.columns = metadata.columns.map(mapper_dict)\n    metadata.to_csv(f\"batch_{BATCH}_dicom_test_metadata.csv\", index=False)\n    \n    print(\"----- Test metadata extraction fully completed -----\")\n    \n    return metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this function only select small batch sizes for time execution purpose. If you want to rerun the whole extraction, please change these lines:\n\n# BATCH_SIZES = list(map(lambda x:int(x/100), [0, 4000, 8000, 12000, 15000]))\n\n# and\n\n# if index_loop%10==0:\n\n# to\n\n# BATCH_SIZES = [0, 4000, 8000, 12000, 15000]\n\n# and\n\n# if index_loop%1000==0:\n\n# for train. And for the test, change these following lines:\n\n# BATCH_SIZES = list(map(lambda x:int(x/10), [0, 750, 1500, 2250, 3000]))\n\n# and\n\n# if index_loop%10==0:\n\n# to\n\n# BATCH_SIZES = [0, 750, 1500, 2250, 3000]\n\n# and\n\n# if index_loop%100==0:","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pool = mp.Pool(mp.cpu_count())\npool.map(extract_train_meta, [i for i in range(4)])\npool.map(extract_test_meta, [i for i in range(4)])\npool.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **4. Preprocessing DICOM Features**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\n\nROOT = '../input/vinbigdata-chestxray-metadata'\ntrain = pd.read_csv('../input/vinbigdata-chestxray-metadata/train_dicom_metadata.csv')\ntrain_cleaned = pd.read_csv('../input/vinbigdata-chestxray-metadata/train_dicom_metadata_cleaned.csv')\ntest = pd.read_csv('../input/vinbigdata-chestxray-metadata/test_dicom_metadata.csv')\ntest_cleaned = pd.read_csv('../input/vinbigdata-chestxray-metadata/test_dicom_metadata_cleaned.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Keep some columns only\ntrain_metadata_filtered = train[[\"Patient's Sex\", \"Patient's Age\", \"Patient's Size\", \"Patient's Weight\"]]\ntest_metadata_filtered = test[[\"Patient's Sex\", \"Patient's Age\", \"Patient's Size\", \"Patient's Weight\"]]\ntrain_metadata_filtered_cleaned = train_cleaned[[\"Patient's Sex\", \"Patient's Age\", \"Patient's Size\", \"Patient's Weight\"]]\ntest_metadata_filtered_cleaned = test_cleaned[[\"Patient's Sex\", \"Patient's Age\", \"Patient's Size\", \"Patient's Weight\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_filtered.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_filtered_cleaned.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_metadata_filtered[\"Patient's Size\"].value_counts())\nprint(train_metadata_filtered[\"Patient's Weight\"].value_counts())\n\nprint(test_metadata_filtered[\"Patient's Size\"].value_counts())\nprint(test_metadata_filtered[\"Patient's Weight\"].value_counts())\n\ntrain_metadata_filtered = train_metadata_filtered.drop([\"Patient's Size\", \"Patient's Weight\"], axis=1)\ntest_metadata_filtered = test_metadata_filtered.drop([\"Patient's Size\", \"Patient's Weight\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_first_el(row):\n    resu = 'NA'\n    if len(str(row))>5:\n        resu = re.search(r\"(?<=\\[)(.*?)(?=\\,)\", row).group()\n    return resu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_filtered[\"Patient's Sex\"] = train_metadata_filtered[\"Patient's Sex\"].fillna(\"NA\")\ntrain_metadata_filtered.loc[train_metadata_filtered[\"Patient's Sex\"]==\"O\"] = np.nan\n\ntrain_metadata_filtered[\"Patient's Age\"] = train_metadata_filtered[\"Patient's Age\"].fillna(\"0\")\ntrain_metadata_filtered[\"Patient's Age\"] = train_metadata_filtered[\"Patient's Age\"].apply(lambda x:re.search(r\"\\d*\", str(x)).group())\ntrain_metadata_filtered.loc[train_metadata_filtered[\"Patient's Age\"]== '']= np.nan\ntrain_metadata_filtered[\"Patient's Age\"] = train_metadata_filtered[\"Patient's Age\"].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_metadata_filtered[\"Patient's Sex\"] = test_metadata_filtered[\"Patient's Sex\"].fillna(\"NA\")\ntest_metadata_filtered.loc[test_metadata_filtered[\"Patient's Sex\"]==\"O\"] = np.nan\n\ntest_metadata_filtered[\"Patient's Age\"] = test_metadata_filtered[\"Patient's Age\"].fillna(\"0\")\ntest_metadata_filtered[\"Patient's Age\"] = test_metadata_filtered[\"Patient's Age\"].apply(lambda x:re.search(r\"\\d*\", str(x)).group())\ntest_metadata_filtered.loc[test_metadata_filtered[\"Patient's Age\"]== '']= np.nan\ntest_metadata_filtered[\"Patient's Age\"] = test_metadata_filtered[\"Patient's Age\"].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_filtered.to_csv(\"train_dicom_metadata_filtered.csv\", index=False)\ntest_metadata_filtered.to_csv(\"test_dicom_metadata_filtered.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_age = train_metadata_filtered.loc[(train_metadata_filtered[\"Patient's Age\"] > 0) & \n                                                 (train_metadata_filtered[\"Patient's Age\"] < 100), :]\n\nfig = px.histogram(train_metadata_age, x=\"Patient's Age\",\n                   marginal=\"box\",\n                   hover_data=train_metadata_age.columns)\n\nfig.update_layout(\n    title=\"Age distribution (train)\")\n\nfig.show()\n\ndel(train_metadata_age)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_distribution_age_imageid(x_values, y_values):\n    \n    colors = ['lightslategray',] * 17\n    colors[0] = 'crimson'\n    colors[1] = 'crimson'\n    colors[2] = 'crimson'\n    \n    fig = go.Figure(data=[go.Bar(\n        x=x_values, \n        y=y_values,\n        text=y_values,\n        marker_color=colors\n    )])\n\n    fig.update_layout(height=600, width=900, title_text=\"Distribution of age observations\")\n    fig.update_xaxes(type=\"category\")\n\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show distribution of count of image_id by age\n# data = train_metadata_filtered.loc[(train_metadata_filtered[\"Patient's Age\"] > 0) &\n#                                    (train_metadata_filtered[\"Patient's Age\"] < 100), :]\n\n# indexes = data[[\"Patient's Age\", 'image_id']].groupby([\"Patient's Age\"]).agg(['count']).index\n# counts = data[[\"Patient's Age\", 'image_id']].groupby([\"Patient's Age\"]).agg(['count']).values.ravel()\n\n# sorted_dict = dict(zip(indexes, counts))\n# sorted_dict = {k: v for k, v in sorted(sorted_dict.items(), key=lambda item: item[1], reverse = True)}\n\n# x = list(sorted_dict.keys())\n# y = list(sorted_dict.values())\n\n# plot_distribution_age_imageid(x, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_counts = list(train_metadata_filtered.loc[train_metadata_filtered[\"Patient's Sex\"] != \"NA\", \"Patient's Sex\"].value_counts())\ntrain_metadata_labels = list(train_metadata_filtered.loc[train_metadata_filtered[\"Patient's Sex\"] != \"NA\", \"Patient's Sex\"].value_counts().index)\n\nfig = go.Figure(data=[go.Pie(labels=train_metadata_labels, \n                             values=train_metadata_counts, \n                             hole=.3,\n                             title_text=\"Sex distribution (train)\")])\nfig.show()\n\ndel(train_metadata_counts, train_metadata_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_filtered[\"Patient's Age\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train_metadata_filtered.loc[(train_metadata_filtered[\"Patient's Sex\"] != \"NA\") &\n                                   (train_metadata_filtered[\"Patient's Age\"] > 0) &\n                                   (train_metadata_filtered[\"Patient's Age\"] < 100), :]\n\nfig = px.histogram(data, \n                   x=\"Patient's Age\", \n                   color=\"Patient's Sex\", \n                   marginal=\"box\",\n                   hover_data=data.columns,\n                   histnorm = \"probability\")\n\nfig.update_layout(\n    title=\"Age distribution by sex (train)\")\n\nfig.show()\n\ndel(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filtered_w_imageid = train_metadata_filtered\ntrain_filtered_w_imageid['image_id'] = train.image_id\ntrain_filtered_w_imageid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_filtered_w_imageid.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **5. Radiocardiographic Representation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_rectangle_parameter(dataframe, index):\n    \n    \"Adapt coordinates of bounding box for patch.Rectangle function\"\n    \n    x_min = dataframe.loc[index, 'x_min']\n    y_min = dataframe.loc[index, 'y_min']\n    x_max = dataframe.loc[index, 'x_max']\n    y_max = dataframe.loc[index, 'y_max']\n    \n    anchor_point = (x_min, y_min)\n    height = y_max - y_min\n    width = x_max - x_min\n    \n    return anchor_point, height, width","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def select_9_from_each(dataframe):\n    \n    \"For each class, returns 9 indexes and image paths\"\n    \n    # Initialize dictionaries\n    class_id_index_examples, class_id_image_examples = {}, {}\n    # Loop over different classes\n    for class_id in range(14):\n        # Infer indexes\n        class_id_index_examples[str(class_id)] = dataframe[dataframe.class_id == class_id].sample(9).index\n        # Infer image paths\n        class_id_image_examples[str(class_id)] = dataframe.loc[class_id_index_examples[str(class_id)],'image_id'].tolist()\n        \n    return class_id_index_examples, class_id_image_examples\n\nclass_id_index_examples, class_id_image_examples = select_9_from_each(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_images(class_id, graph_indexes = np.arange(9)):\n    \n    # Get files\n    files_index = class_id_index_examples[str(class_id)]\n    files_list = class_id_image_examples[str(class_id)]\n    \n    # define subplot\n    fig, axs = plt.subplots(3,3, figsize=(12,12))\n    for graph_index in graph_indexes:\n        \n        full_filename = files_list[graph_index]+'.dicom'\n        ds = dcmread(os.path.join(PATH, \n                                  'train',\n                                  full_filename))\n        \n\n#         axs[graph_index%3, (graph_index)//3].set_title('Label: %s \\n'%class_id,\n#                   fontsize=18)\n        axs[graph_index%3, (graph_index)//3].imshow(ds.pixel_array, cmap=plt.get_cmap('gray'))\n                  \n        if str(class_id) != '14':\n            \n            # Add rectangle\n            anchor_point, height, width = get_rectangle_parameter(train_dataframe, \n                                                                  files_index[graph_index])\n            rect = Rectangle(anchor_point, \n                                     height, \n                                     width, \n                                     edgecolor='r', \n                                     facecolor=\"none\")\n            axs[graph_index%3, (graph_index)//3].add_patch(rect)\n                     \n    # the bottom of the subplots of the figure\n    plt.subplots_adjust(bottom = 0.001)\n    plt.subplots_adjust(top = 0.99)\n    \n    # show the figure\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **6. X-Ray Visualization**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom ensemble_boxes import *\nfrom tqdm.notebook import tqdm\n\nimport pydicom\nfrom pydicom.tag import Tag\n\nimport tensorflow as tf\nimport tensorflow_io as tfio\n\nfrom object_detection.protos.string_int_label_map_pb2 import StringIntLabelMap, StringIntLabelMapItem\nfrom object_detection.dataset_tools import tf_record_creation_util\nfrom object_detection.utils import dataset_util\nimport contextlib2\n\nfrom google.protobuf import text_format","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_dicom(path, max_dim):\n    image_bytes = tf.io.read_file(path)\n    image = tfio.image.decode_dicom_image(\n        image_bytes, \n        dtype = tf.uint16\n    )\n    \n    image = tf.squeeze(image, axis = 0)\n    \n    h, w, _ = image.shape\n    \n    image = tf.image.resize(\n        image, \n        (max_dim, max_dim), \n        preserve_aspect_ratio = True\n    )\n    \n    image = image - tf.reduce_min(image)\n    image = image / tf.reduce_max(image)\n    image = tf.cast(image * 255, tf.uint8)\n    \n    return image, h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/vinbigdata-chest-xray-abnormalities-detection\"\ndf = pd.read_csv(os.path.join(path, \"train.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nmax_dim = 500\ndemo_image = \"6d5acf3f8a973a26844d617fffe72998.dicom\"\nimage, h, w = read_dicom(os.path.join(path, \"train\", demo_image), max_dim)\n\nplt.figure(figsize = (5, 5))\nplt.imshow(tf.squeeze(image), 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Image Processing technique CLAHE (Contrast Limited Adaptive Histogram Equalization).\n# This image pre-processing redistributes the lightness values of the image making patterns more apparent \ndef CLAHE(image):\n    clahe = cv2.createCLAHE(\n        clipLimit = 2., \n        tileGridSize = (10, 10)\n    )\n    \n    image = clahe.apply(image.numpy()) \n    image = tf.expand_dims(image, axis = 2)\n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nfig = plt.figure(figsize = (8, 8))\n\naxes = fig.add_subplot(1, 2, 1)\nplt.imshow(tf.squeeze(image), cmap = \"gray\")\naxes.set_title(\"Original\")\n\naxes = fig.add_subplot(1, 2, 2)\nimage = CLAHE(image)\nplt.imshow(tf.squeeze(image), cmap = \"gray\")\naxes.set_title(\"Post CLAHE\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The API requires the classes to be from 1 to n and outputs 0 when no class is found. Since our labels start with 0, we make unit increment to the class_id and use the new label-map."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating LabelMap\ndf[\"class_id\"] = df[\"class_id\"] + 1 # Incrementing by 1\nLabelMap = df.loc[df[\"class_name\"] != \"No finding\", [\"class_name\", \"class_id\"]] # Removing the examples with no finding\nLabelMap = LabelMap.drop_duplicates().reset_index(drop = True)\nLabelMap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using 14 unique colors to annotate the abnormalities.\nLABEL_COLORS = [\n    (230, 25, 75), (60, 180, 75), (255, 225, 25), (0, 130, 200), (245, 130, 48), (145, 30, 180), (70, 240, 240), \n    (240, 50, 230), (210, 245, 60), (250, 190, 212), (0, 128, 128), (220, 190, 255), (170, 110, 40), (255, 250, 200), \n]\nLabelMap[\"colors\"] = LABEL_COLORS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save mappings as .pbtxt\ndef save_mapping(LabelMap):\n    msg = StringIntLabelMap()\n    \n    for i, row in LabelMap.iterrows():\n        msg.item.append(StringIntLabelMapItem(id = row[\"class_id\"], name = row[\"class_name\"]))\n    \n    text = str(text_format.MessageToBytes(msg, as_utf8 = True), 'utf-8')\n    \n    f = open(\"LabelMap.pbtxt\", \"w\")\n    f.write(text)\n    f.close()\n    \nsave_mapping(LabelMap)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove examples with no findings (won't be used for training)\ndf = df.dropna().reset_index(drop = True)\n\n# Change data types\ndf = df.astype({\n    \"x_min\": int, \n    \"y_min\": int, \n    \"x_max\": int, \n    \"y_max\": int,\n    \"class_id\": str\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_boxes(image, data, title):    \n    img = cv2.cvtColor(image.numpy(), cv2.COLOR_GRAY2RGB)\n    \n    for i, row in data.iterrows():\n    \n        x1, y1 = row[\"x_min\"], row[\"y_min\"]\n        x2, y2 = row[\"x_max\"], row[\"y_max\"]\n    \n        cv2.rectangle(\n            img,\n            pt1 = (x1, y1),\n            pt2 = (x2, y2),\n            color = row[\"colors\"],\n            thickness = 2\n        )\n    \n        cv2.putText(\n            img, \n            row[\"class_name\"], \n            (x1, y1-5), \n            cv2.FONT_HERSHEY_SIMPLEX, \n            0.5, \n            row[\"colors\"], \n            1\n        )\n\n    plt.figure(figsize = (8, 8))\n    plt.imshow(img) \n    plt.title(title)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Selecting a particular radiologist\ndemo_rad = \"R9\"\n\n# Preprocessing metadata to suit needs\ndata = df.loc[\n    (df[\"image_id\"] == demo_image[:-6]) & (df[\"rad_id\"] == demo_rad),\n    [\"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n]\n\nH, W, _ = image.shape\ndata[[\"x_min\", \"x_max\"]] = (data[[\"x_min\", \"x_max\"]]* W/w).astype(int)\ndata[[\"y_min\", \"y_max\"]] = (data[[\"y_min\", \"y_max\"]]* H/h).astype(int)\n\ndata = pd.merge(data, LabelMap)\n\n# Plotting annotation by radiologist\nplot_boxes(image, data, \"Labels for \" + demo_image + \" by \" + demo_rad)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Explore annotations from other radiologist for this x-ray"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing metadata to suit needs\ndata = df.loc[\n    (df[\"image_id\"] == demo_image[:-6]),\n    [\"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n]\n\nH, W, _ = image.shape\ndata[[\"x_min\", \"x_max\"]] = (data[[\"x_min\", \"x_max\"]]* W/w).astype(int)\ndata[[\"y_min\", \"y_max\"]] = (data[[\"y_min\", \"y_max\"]]* H/h).astype(int)\n\ndata = pd.merge(data, LabelMap)\n\n# Plotting annotation by all radiologists\nplot_boxes(image, data, \"Labels for \" + demo_image + \" by all radiologists\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **7. Image Data Preprocessing**"},{"metadata":{},"cell_type":"markdown","source":"We shall use a technique called Weighted Boxes Fusion (WBF) to provide us with the best annotation. This will definitely reduce the metadata size by a lot."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing as needed for WBF\ndata = df.loc[\n    (df[\"image_id\"] == demo_image[:-6]),\n    [\"class_name\", \"x_min\", \"y_min\", \"x_max\", \"y_max\"]\n]\n\ndata[[\"x_min\", \"x_max\"]] = data[[\"x_min\", \"x_max\"]]/w\ndata[[\"y_min\", \"y_max\"]] = data[[\"y_min\", \"y_max\"]]/h\n\ndata = pd.merge(data, LabelMap)\n\nboxes_list = data[[\"x_min\", \"y_min\", \"x_max\", \"y_max\"]].values.tolist()\nscores_list = [1]*len(boxes_list)\nlabels_list = list(data[\"class_id\"])\n\n# Applying WBF\nboxes, _, labels = weighted_boxes_fusion(\n    boxes_list = [boxes_list],\n    scores_list = [scores_list],\n    labels_list = [labels_list],\n    weights = None, \n    iou_thr = 0.3, \n    skip_box_thr = 0.0001\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Postprocessing after applying WBF \ndata = pd.DataFrame(boxes, columns = [\"x_min\", \"y_min\", \"x_max\", \"y_max\"])\n\nH, W, _ = image.shape\ndata[[\"x_min\", \"x_max\"]] = (data[[\"x_min\", \"x_max\"]]* W).astype(int)\ndata[[\"y_min\", \"y_max\"]] = (data[[\"y_min\", \"y_max\"]]* H).astype(int)\n\ndata[\"class_id\"] = labels.astype(int)\n\ndata = pd.merge(data, LabelMap)\n\n# Plotting annotation by all radiologists\nplot_boxes(image, data, \"Labels for \" + demo_image + \" post WBF\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The TFRecord format is a simple format for storing a sequence of binary records. This format is efficient in terms of storage and retrieval. It is the desired input format for the API. But before creating TFRecords, we must first apply WBF to the metadata. To apply WBF we must normalize the coordinates. Reading each image to extract dimensions can be time consuming. Using PyDICOM we can obtain x-ray metadata from which dimensions can be quickly extracted."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dropping rad_id as it is not required for training\ndf = df.drop(columns = [\"rad_id\"])\n\n# Obtaining set of x-rays with at least one finding\nxrays = set(df[\"image_id\"]) # Only 4394 x-rays, not 15000. Roughly 30% of the x-rays remain.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dimensions = []\nfor i, xray in tqdm(enumerate(xrays)):\n    ds = pydicom.dcmread(\n        os.path.join(path, \"train\", xray + \".dicom\"), \n        specific_tags = [\n            Tag(\"0028\", \"0010\"), # Tag for Rows (Height)\n            Tag(\"0028\", \"0011\")  # Tag for Columns (Width)\n        ]\n    )\n    \n    dimensions.append([xray, ds.Rows, ds.Columns])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dimensions = pd.DataFrame(dimensions, columns = [\"image_id\", \"height\", \"width\"])\ndf = pd.merge(dimensions, df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalize coordinates\ndf[\"x_min\"], df[\"x_max\"] = df[\"x_min\"]/df[\"width\"], df[\"x_max\"]/df[\"width\"]\ndf[\"y_min\"], df[\"y_max\"] = df[\"y_min\"]/df[\"height\"], df[\"y_max\"]/df[\"height\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Before applying WBF we had 36096 rows\ndf_list = []\nfor i, xray in tqdm(enumerate(xrays)):\n    data = df[df[\"image_id\"] == xray]\n\n    boxes_list = data[[\"x_min\", \"y_min\", \"x_max\", \"y_max\"]].values.tolist()\n    scores_list = [1]*len(boxes_list)\n    labels_list = list(data[\"class_id\"])\n\n    # Applying WBF\n    boxes, _, labels = weighted_boxes_fusion(\n        boxes_list = [boxes_list],\n        scores_list = [scores_list],\n        labels_list = [labels_list],\n        weights = None, \n        iou_thr = 0.3, \n        skip_box_thr = 0.0001\n    )\n    \n    data = pd.DataFrame(boxes, columns = [\"x_min\", \"y_min\", \"x_max\", \"y_max\"]) \n    # Leaving the coordinates normalized since the API expects them to be so. \n    \n    data[\"class_id\"] = labels.astype(int)\n    \n    data[\"image_id\"] = xray \n    \n    df_list.append(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.concat(df_list) # After applying WBF we have 21836 rows\ndf = pd.merge(df, LabelMap)\ndf = df.drop(columns = [\"colors\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since we have more than a few thousand examples, it is beneficial to shard the dataset into multiple files:\n\n* Parallel reading improves throughput.\n* Easy shuffling improves performance.\n\nSharding is cool but you know what's cooler? Stratified K-Fold Sharding. Basically we break down our dataset into multiple (\"K\") TFRecords (each is a shard) in such a way that:\n\n* The distribution of abnormalities remains the same in each shard.\n* Each x-ray is part of exactly one shard (to avoid information leak).\n\nWe can conveniently use these shards for training, validation and testing."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stratified K-Fold Sharding\n\nnum_shards = 25\n\nskf = StratifiedKFold(\n    n_splits = num_shards, \n    shuffle = True, \n    random_state = 0\n)\n\ndf_folds = df[['image_id']].copy()\n\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()   # Number of bounding boxes in the image\ndf_folds.loc[:, 'object_count'] = df.groupby('image_id')['class_id'].nunique() # Number of classes in the image\n\n# Preparing stratify groups\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['object_count'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\n\n# Determining which fold the x-ray will fall in\ndf_folds.loc[:, 'fold'] = 0\nskf_split = skf.split(\n    X = df_folds.index, \n    y = df_folds['stratify_group']\n)\n\nfor fold_number, (train_index, val_index) in enumerate(skf_split):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\n    \ndf_folds.reset_index(inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.merge(df, df_folds)\n\ntemp = df.groupby([\"fold\", \"class_name\"]).agg(\n    count = pd.NamedAgg(\"class_name\", \"count\")\n).reset_index()\n\ntemp = temp.pivot_table(\n    index = \"class_name\",\n    columns = \"fold\",\n    values = \"count\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (20, 10))\nsns.heatmap(\n    temp,\n    annot = True,\n    cmap = \"YlGnBu\",\n    fmt = \"g\"\n)\nplt.title(\"Heatmap of class distribution\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Notice how color is similar along a row. The color distribution indicates the similar class disturbution across all folds (shards).\n\nOnce sharding is done, it is important to create TFRecords after applying CLAHE to each x-ray. We must remember to apply the same transformations to the x-rays we intend to make predictions for."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_tf_record(img_path, max_dim, img_df):\n    \n    filename = img_path.split(\"/\")[-1].encode()\n    source_id = img_path.encode()\n    \n    # Preprocess image \n    img, _, _ = read_dicom(img_path, max_dim)\n    height, width, _ = img.shape\n    img = CLAHE(img)\n    \n    # Encode as JPEG (Lossy compression)\n    img = tf.io.encode_jpeg(\n        img, \n        quality = 100, \n        format = 'grayscale'\n    )\n    \n    img_bytes = img.numpy()\n    \n    img_format = b'jpeg'\n\n    xmin_list = list(img_df[\"x_min\"])\n    xmax_list = list(img_df[\"x_max\"])\n    ymin_list = list(img_df[\"y_min\"])\n    ymax_list = list(img_df[\"y_max\"])\n    \n    class_name_list = list(img_df[\"class_name\"])\n    class_name_list = [c.encode() for c in class_name_list]\n    \n    class_id_list = list(img_df[\"class_id\"])\n    \n    # Creating TFRecord\n    tf_record = tf.train.Example(\n        features = tf.train.Features(\n            feature = {\n                'image/height': dataset_util.int64_feature(height),\n                'image/width': dataset_util.int64_feature(width),\n                'image/filename': dataset_util.bytes_feature(filename),\n                'image/source_id': dataset_util.bytes_feature(source_id),\n                'image/encoded': dataset_util.bytes_feature(img_bytes),\n                'image/format': dataset_util.bytes_feature(img_format),\n                'image/object/bbox/xmin': dataset_util.float_list_feature(xmin_list),\n                'image/object/bbox/xmax': dataset_util.float_list_feature(xmax_list),\n                'image/object/bbox/ymin': dataset_util.float_list_feature(ymin_list),\n                'image/object/bbox/ymax': dataset_util.float_list_feature(ymax_list),\n                'image/object/class/text': dataset_util.bytes_list_feature(class_name_list),\n                'image/object/class/label': dataset_util.int64_list_feature(class_id_list),\n            }\n        )\n    )\n    \n    return tf_record","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"annot_path = \"workspace/annotations\" \nos.makedirs(annot_path, exist_ok = True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_cnt = np.zeros(num_shards, dtype = int)\n\nwith contextlib2.ExitStack() as tf_record_close_stack:\n    output_tfrecords = tf_record_creation_util.open_sharded_output_tfrecords(\n        tf_record_close_stack, \n        annot_path, \n        num_shards\n    )\n    \n    for i in tqdm(range(num_shards)):\n        df_shard = df[df[\"fold\"] == i]\n        xrays = set(df_shard[\"image_id\"])\n        \n        for xray in xrays:\n            df_image = df_shard[df_shard[\"image_id\"] == xray]\n            \n            img_path = os.path.join(path, \"train\", xray + \".dicom\")\n            tf_record = create_tf_record(img_path, max_dim, df_image)\n            output_tfrecords[i].write(tf_record.SerializeToString())\n            \n            img_cnt[i] += 1\n\nprint(\"Converted {} images\".format(np.sum(img_cnt)))\nprint(\"Images per shard: {}\".format(img_cnt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save dataframe\ndf.to_csv(\"data.csv\", index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Credits\n\n* https://www.kaggle.com/bhallaakshit/dicom-wrangling-and-enhancement\n* https://www.kaggle.com/bryanb/vinbigdata-chest-x-ray-eda-fusing-boxes"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}