{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\n!pip install ensemble-boxes\nfrom ensemble_boxes import *\nfrom tqdm import tqdm\n\n#Read in the labels from the text file in csv format#\ntrain_df = pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# ===============================\n# Default WBF config (you can change these)\niou_thr = 0.5\nskip_box_thr = 0.0001\nsigma = 0.1\n# ===============================\n\n# Loading the train DF\ndf = pd.read_csv(\"../input/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ndf.fillna(0, inplace=True)\ndf.loc[df[\"class_id\"] == 14, ['x_max', 'y_max']] = 1.0\n\nresults = []\nimage_ids = df[\"image_id\"].unique()\n\nfor image_id in tqdm(image_ids, total=len(image_ids)):\n\n    # All annotations for the current image.\n    data = df[df[\"image_id\"] == image_id]\n    data = data.reset_index(drop=True)\n\n    annotations = {}\n    weights = []\n\n    # WBF expects the coordinates in 0-1 range.\n    max_value = data.iloc[:, 4:].values.max()\n    data.loc[:, [\"x_min\", \"y_min\", \"x_max\", \"y_max\"]] = data.iloc[:, 4:] / max_value\n\n    # Loop through all of the annotations\n    for idx, row in data.iterrows():\n\n        rad_id = row[\"rad_id\"]\n\n        if rad_id not in annotations:\n            annotations[rad_id] = {\n                \"boxes_list\": [],\n                \"scores_list\": [],\n                \"labels_list\": [],\n            }\n\n            # We consider all of the radiologists as equal.\n            weights.append(1.0)\n\n        annotations[rad_id][\"boxes_list\"].append([row[\"x_min\"], row[\"y_min\"], row[\"x_max\"], row[\"y_max\"]])\n        annotations[rad_id][\"scores_list\"].append(1.0)\n        annotations[rad_id][\"labels_list\"].append(row[\"class_id\"])\n\n    boxes_list = []\n    scores_list = []\n    labels_list = []\n\n    for annotator in annotations.keys():\n        boxes_list.append(annotations[annotator][\"boxes_list\"])\n        scores_list.append(annotations[annotator][\"scores_list\"])\n        labels_list.append(annotations[annotator][\"labels_list\"])\n\n    # Calculate WBF\n    boxes, scores, labels = weighted_boxes_fusion(\n        boxes_list,\n        scores_list,\n        labels_list,\n        weights=weights,\n        iou_thr=iou_thr,\n        skip_box_thr=skip_box_thr\n    )\n\n    for idx, box in enumerate(boxes):\n        results.append({\n            \"image_id\": image_id,\n            \"class_id\": int(labels[idx]),\n            \"rad_id\": \"wbf\",\n            \"x_min\": box[0] * max_value,\n            \"y_min\": box[1] * max_value,\n            \"x_max\": box[2] * max_value,\n            \"y_max\": box[3] * max_value,\n        })\n\nresults = pd.DataFrame(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dic = dict(zip(range(15),[results[results['class_id']==x].count()[0] for x in range(15)]))\ndic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nfrom glob import glob\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom skimage import exposure\nimport cv2\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title=\"\", cmap='gray', img_size=(500,500)):\n    rows = len(imgs)//cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n    \ndef get_bbox_area(row):\n    return (row['x_max']-row['x_min'])*(row['y_max']-row['y_min'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dir = '../input/vinbigdata-chest-xray-abnormalities-detection'\ndicom_paths = glob(f'{dataset_dir}/train/*.dicom')\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from bokeh.plotting import figure as bokeh_figure\nfrom bokeh.io import output_notebook, show, output_file\nfrom bokeh.models import ColumnDataSource, HoverTool, Panel\nfrom bokeh.models.widgets import Tabs\nimport pandas as pd\nfrom PIL import Image\nfrom sklearn import preprocessing\nimport random\nfrom random import randint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = results\nle = preprocessing.LabelEncoder()  # encode rad_id\ntrain_df['rad_label'] = le.fit_transform(train_df['rad_id'])\nfinding_df = train_df[train_df['class_id'] == 13 ]\nfinding_df['bbox_area'] = finding_df.apply(get_bbox_area, axis=1)\nfinding_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resize\nimgs = []\nimg_ids = finding_df['image_id'].values\nclass_ids = finding_df['class_id'].unique()\n\n# map label_id to specify color\nlabel2color = {class_id:[randint(0,255) for i in range(3)] for class_id in class_ids}\nthickness = 3\nscale = 5\n\n\nfor i in range(8):\n    img_id = random.choice(img_ids)\n    img_path = f'{dataset_dir}/train/{img_id}.dicom'\n    img = dicom2array(path=img_path)\n    img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n    img = np.stack([img, img, img], axis=-1)\n    \n    boxes = finding_df.loc[finding_df['image_id'] == img_id, ['x_min', 'y_min', 'x_max', 'y_max']].values/scale\n    labels = finding_df.loc[finding_df['image_id'] == img_id, ['class_id']].values.squeeze()\n    \n    try:\n        for label_id, box in zip(labels, boxes):\n            color = label2color[label_id]\n            img = cv2.rectangle(\n                img,\n                (int(box[0]), int(box[1])),\n                (int(box[2]), int(box[3])),\n                color, thickness\n            )\n        img = cv2.resize(img, (500,500))\n        imgs.append(img)\n    except:\n        label_id = int(labels)\n        box = boxes[0]\n        color = label2color[label_id]\n        img = cv2.resize(img, (500,500))\n        img = cv2.rectangle(\n                img,\n                (int(box[0]), int(box[1])),\n                (int(box[2]), int(box[3])),\n                color, thickness\n            )\n        imgs.append(img)\n    \nplot_imgs(imgs, cmap=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.mkdir('tmp')\nos.mkdir('tmp/train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nimgs=[]\nimg_ids = train_df['image_id'].unique()\ncount = 0\nfor img_id in tqdm(img_ids):\n    img_path = f'{dataset_dir}/train/{img_id}.dicom'\n    img = dicom2array(path=img_path)\n    #img = np.stack([img, img, img], axis=-1)\n    \n    try:\n        img = cv2.resize(img, (512,512))\n    except:\n        print('err')\n    \n#    count+=1\n#    if count>10:\n#        break\n    imgs.append(img)\n#    img = Image.fromarray(img)\n#    img.save('tmp/train/' + img_id + '.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import joblib\njoblib.dump(imgs, \"imgs.txt\",compress=5)\nimg_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(\"./imgs.txt\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(data=img_ids).to_csv('ids.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.rmtree('tmp/train/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im = np.array(Image.open('./tmp/train/47ed17dcb2cbeec15182ed335a8b5a9e.jpg'))\nplt.imshow(im)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}