{"cells":[{"metadata":{"papermill":{"duration":0.019161,"end_time":"2021-01-11T18:10:47.544368","exception":false,"start_time":"2021-01-11T18:10:47.525207","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# TFRecords\nThis notebook will create TFRecords of the VinBigData Chest X-rays. The TFRecords can then be used as training data with the TensorFlow Object Detection API. The labels in this dataset are very noisy, so the objects are filtered using [Weighted Boxes Fusion](https://github.com/ZFTurbo/Weighted-Boxes-Fusion). Images are resized to 1024 while preserving aspect ratio (change to any desired value of IMAGE_SIZE)."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2021-01-11T18:10:47.595062Z","iopub.status.busy":"2021-01-11T18:10:47.594365Z","iopub.status.idle":"2021-01-11T18:10:54.529664Z","shell.execute_reply":"2021-01-11T18:10:54.529056Z"},"papermill":{"duration":6.965819,"end_time":"2021-01-11T18:10:54.529792","exception":false,"start_time":"2021-01-11T18:10:47.563973","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/weightedboxesfusion\")\nfrom ensemble_boxes import *\nimport pydicom\nfrom pydicom import dcmread\nfrom pydicom.pixel_data_handlers.util import *\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport hashlib\nimport os\nfrom io import BytesIO\nfrom PIL import Image, ImageFont, ImageDraw\nimport cv2\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.019179,"end_time":"2021-01-11T18:10:54.569089","exception":false,"start_time":"2021-01-11T18:10:54.54991","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Start by reading training metadata. "},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:10:54.617259Z","iopub.status.busy":"2021-01-11T18:10:54.616328Z","iopub.status.idle":"2021-01-11T18:10:54.843447Z","shell.execute_reply":"2021-01-11T18:10:54.844044Z"},"papermill":{"duration":0.255418,"end_time":"2021-01-11T18:10:54.844188","exception":false,"start_time":"2021-01-11T18:10:54.58877","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"raw_df = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\nraw_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.021062,"end_time":"2021-01-11T18:10:54.886848","exception":false,"start_time":"2021-01-11T18:10:54.865786","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Helper functions\nThe images are digitized in 12-14bit resolution - converting this to JPEG will cause quite a bit of information to be lost. To preserve all image information, the images could be saved as 16bit PNG. But here we use Contrast Limiting Adaptive Histogram Equalization (CLAHE). This image pre-processing step must then also be used during inference time. "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\n#for masking\nfrom skimage.measure import label,regionprops\nfrom sklearn.cluster import KMeans\nfrom skimage.segmentation import clear_border\n\nimport scipy.ndimage as ndimage\nfrom scipy.ndimage.interpolation import zoom\nfrom skimage import measure, morphology, segmentation\nfrom skimage.transform import resize\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom as dcm\n\ndef get_first_of_dicom_field_as_int(x):\n    #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n    if type(x) == dcm.multival.MultiValue: return int(x[0])\n    else: return int(x)\n\n    \ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value] #window_width\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.pi/np.e","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:10:54.946075Z","iopub.status.busy":"2021-01-11T18:10:54.945093Z","iopub.status.idle":"2021-01-11T18:10:54.947736Z","shell.execute_reply":"2021-01-11T18:10:54.948237Z"},"papermill":{"duration":0.04067,"end_time":"2021-01-11T18:10:54.94837","exception":false,"start_time":"2021-01-11T18:10:54.9077","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 1024 # change this to desired value\nCLIP_LIMIT = 8\nGRID_SIZE = (16,16)\n\n\ndef window_image(data):\n    #window_center, window_width = get_windowing(data)\n    data = data.pixel_array\n    #data = (data*slope +intercept) #for translation adjustments given in the dicom file. \n    data_min = data.min()\n    data_max = data.max()\n    dmean = (data_max-data_min)//2\n    d1 = int(dmean*(0.05))\n    #data[data<dmean] += d1 #set data_min for all HU levels less than minimum HU level\n    data[data>dmean] -= d1 #set data_max for all HU levels higher than maximum HU level\n\n    return data\n\n\ndef read_image(fname, target_size=IMAGE_SIZE, use_clahe=True):\n    ds = dcmread(fname)\n    ds_ = window_image(ds)\n    data = apply_voi_lut(ds_, ds)\n    #data = (data*data)/4-data\n    data = data - np.min(data)\n    data = 255. * data / np.max(data)\n    if ds.PhotometricInterpretation == \"MONOCHROME1\": # check for inverted image\n        data = 255. - data\n    #data[data<5] = 0\n    #data[data>250] = 0\n    if use_clahe:\n        clahe = cv2.createCLAHE(clipLimit=CLIP_LIMIT, tileGridSize=GRID_SIZE)\n        climg = clahe.apply(data.astype('uint8'))\n        #climg = (climg-data)\n        img = Image.fromarray(climg.astype('uint8'), 'L')\n    else:\n        img = Image.fromarray(data.astype('uint8'), 'L')\n    org_size = img.size\n    if max(img.size) > target_size:\n        img.thumbnail((target_size, target_size), Image.ANTIALIAS)\n    \n    return img, org_size","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020781,"end_time":"2021-01-11T18:10:54.989718","exception":false,"start_time":"2021-01-11T18:10:54.968937","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Let's see what CLAHE does:"},{"metadata":{"trusted":true},"cell_type":"code","source":"fname = '../input/vinbigdata-chest-xray-abnormalities-detection/train/0005e8e3701dfb1dd93d53e2ff537b6e.dicom'\nfig = plt.figure(figsize=(20,20))\naxes = fig.add_subplot(1, 2, 1)\nimg, size = read_image(fname, use_clahe=False)\naxes.set_title('Original')\nplt.imshow(img, cmap='gray')\naxes = fig.add_subplot(1, 2, 2)\nimg, size = read_image(fname, use_clahe=True)\naxes.set_title('CLAHE')\nplt.imshow(img, cmap='gray');","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.036805,"end_time":"2021-01-11T18:10:57.029957","exception":false,"start_time":"2021-01-11T18:10:56.993152","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Weighted Boxes Fusion\nThe first task is to run WBF on the raw data to filter out overlapping objects. Images with no findings will also be removed."},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:10:57.10754Z","iopub.status.busy":"2021-01-11T18:10:57.106508Z","iopub.status.idle":"2021-01-11T18:10:57.127654Z","shell.execute_reply":"2021-01-11T18:10:57.128276Z"},"papermill":{"duration":0.061906,"end_time":"2021-01-11T18:10:57.12843","exception":false,"start_time":"2021-01-11T18:10:57.066524","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def plot_boxes(img, boxes, labels, thickness=5):\n    for i in range(len(boxes)):\n        box = boxes[i].astype(int)\n        cv2.rectangle(img, (box[0], box[1]), (box[2],  box[3]), LABEL_COLORS[labels[i].astype(int)], thickness)\n    return img\n\ndef plot_two(fname, idf):\n    image, size = read_image(fname)\n    image = cv2.cvtColor(np.array(image), cv2.COLOR_GRAY2RGB)\n    image2 = image.copy()\n    fig = plt.figure(figsize=(20,20))\n    fig.tight_layout()\n    axes = fig.add_subplot(1, 2, 1)\n    plt.setp(axes, xticks=[], yticks=[])\n    axes.set_title('Original')\n    boxes = idf[['x_min', 'y_min', 'x_max', 'y_max']].values * IMAGE_SIZE / max(size)\n    labels = idf.class_id.values\n    image = plot_boxes(image, boxes, labels)\n    plt.imshow(image, cmap='gray')\n    # wbf\n    axes = fig.add_subplot(1, 2, 2)\n    plt.setp(axes, xticks=[], yticks=[])\n    axes.set_title('After WBF')\n    boxes_list = boxes / 1024.\n    boxes_list = boxes_list.tolist()\n    boxes1, _, labels1 = weighted_boxes_fusion([boxes_list], [np.ones(len(labels)).tolist()], [labels.tolist()], \n                                               weights=None, iou_thr=0.42, skip_box_thr=0.0001)\n    boxes1 *= 1024\n    image2 = plot_boxes(image2, boxes1, labels1)\n    plt.imshow(image2, cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:10:57.209952Z","iopub.status.busy":"2021-01-11T18:10:57.209251Z","iopub.status.idle":"2021-01-11T18:10:57.26253Z","shell.execute_reply":"2021-01-11T18:10:57.261951Z"},"papermill":{"duration":0.096932,"end_time":"2021-01-11T18:10:57.262657","exception":false,"start_time":"2021-01-11T18:10:57.165725","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"NUM_CLASSES = 14\nLABEL_COLORS = [(230, 25, 75), (60, 180, 75), (255, 225, 25), (0, 130, 200), \n                (245, 130, 48), (145, 30, 180), (70, 240, 240), (240, 50, 230), \n                (210, 245, 60), (250, 190, 212), (0, 128, 128), (220, 190, 255), \n                (170, 110, 40), (255, 250, 200), (128, 0, 0), (170, 255, 195), \n                (128, 128, 0), (255, 215, 180), (0, 0, 128), (128, 128, 128), \n                (255, 255, 255), (0, 0, 0)]\n\nfindings = raw_df[raw_df.class_id != 14]\nxrays = findings.image_id.unique()\nclass_names = []\nfor i in range(NUM_CLASSES):\n    class_names.append(findings[findings.class_id == i].class_name.iloc[0])\nclass_names","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:10:57.355377Z","iopub.status.busy":"2021-01-11T18:10:57.354654Z","iopub.status.idle":"2021-01-11T18:11:00.830667Z","shell.execute_reply":"2021-01-11T18:11:00.831191Z"},"papermill":{"duration":3.531442,"end_time":"2021-01-11T18:11:00.831326","exception":false,"start_time":"2021-01-11T18:10:57.299884","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"i = 105\nidf = raw_df[raw_df.image_id == xrays[i]]\nfname = '../input/vinbigdata-chest-xray-abnormalities-detection/train/'+xrays[i]+'.dicom'\nplot_two(fname, idf)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.048016,"end_time":"2021-01-11T18:11:00.926186","exception":false,"start_time":"2021-01-11T18:11:00.87817","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Run WBF on entire dataset"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:11:01.020878Z","iopub.status.busy":"2021-01-11T18:11:01.020204Z","iopub.status.idle":"2021-01-11T18:11:59.80224Z","shell.execute_reply":"2021-01-11T18:11:59.801307Z"},"papermill":{"duration":58.83063,"end_time":"2021-01-11T18:11:59.80237","exception":false,"start_time":"2021-01-11T18:11:00.97174","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"wbf = []\n\nfor i in range(len(xrays)):\n    idf = raw_df[raw_df.image_id == xrays[i]]\n    boxes = idf[['x_min', 'y_min', 'x_max', 'y_max']].values\n    max_pos = np.max(boxes)\n    boxes /= max_pos\n    boxes_list = boxes.tolist()\n    labels = idf.class_id.values\n    boxes1, _, labels1 = weighted_boxes_fusion([boxes_list], [np.ones(len(labels)).tolist()], [labels.tolist()], \n                                               weights=None, iou_thr=0.42, skip_box_thr=0.0001)\n    boxes1 *= max_pos\n    boxes1 = np.floor(boxes1)\n    for j in range(len(boxes1)):\n        wbf.append([xrays[i], class_names[labels1[j].astype(int)], labels1[j].astype(int), boxes1[j][0], boxes1[j][1], boxes1[j][2], boxes1[j][3]])\n\nwbf_df = pd.DataFrame (wbf, columns=['image_id', 'class_name', 'class_id', 'x_min', 'y_min', 'x_max', 'y_max'])\nwbf_df.to_csv('wbf_objects.csv')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:11:59.897533Z","iopub.status.busy":"2021-01-11T18:11:59.896926Z","iopub.status.idle":"2021-01-11T18:11:59.911794Z","shell.execute_reply":"2021-01-11T18:11:59.911279Z"},"papermill":{"duration":0.063253,"end_time":"2021-01-11T18:11:59.911909","exception":false,"start_time":"2021-01-11T18:11:59.848656","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"wbf_df.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:00.008705Z","iopub.status.busy":"2021-01-11T18:12:00.008027Z","iopub.status.idle":"2021-01-11T18:12:00.014843Z","shell.execute_reply":"2021-01-11T18:12:00.015332Z"},"papermill":{"duration":0.05706,"end_time":"2021-01-11T18:12:00.015465","exception":false,"start_time":"2021-01-11T18:11:59.958405","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"wbf_df.class_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:00.111973Z","iopub.status.busy":"2021-01-11T18:12:00.111351Z","iopub.status.idle":"2021-01-11T18:12:00.118162Z","shell.execute_reply":"2021-01-11T18:12:00.118698Z"},"papermill":{"duration":0.056984,"end_time":"2021-01-11T18:12:00.118832","exception":false,"start_time":"2021-01-11T18:12:00.061848","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"findings.class_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.047451,"end_time":"2021-01-11T18:12:00.213522","exception":false,"start_time":"2021-01-11T18:12:00.166071","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Stratified K-Folds\nTo make sure that each shard has about the same class distribution, we use statified K-Folds on the data set. Modified code from [this notebook](https://www.kaggle.com/backtracking/smart-data-split-train-eval-for-object-detection/comments)."},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:00.311656Z","iopub.status.busy":"2021-01-11T18:12:00.310984Z","iopub.status.idle":"2021-01-11T18:12:01.360936Z","shell.execute_reply":"2021-01-11T18:12:01.36025Z"},"papermill":{"duration":1.100087,"end_time":"2021-01-11T18:12:01.361051","exception":false,"start_time":"2021-01-11T18:12:00.260964","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nNUM_SHARDS = 20\n\nskf = StratifiedKFold(n_splits=NUM_SHARDS, shuffle=True, random_state=42)\ndf_folds = wbf_df[['image_id']].copy()\n\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds.loc[:, 'object_count'] = wbf_df.groupby('image_id')['class_id'].nunique()\n\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['object_count'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str))\n\ndf_folds.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\ndf_folds.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.046826,"end_time":"2021-01-11T18:12:01.461664","exception":false,"start_time":"2021-01-11T18:12:01.414838","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Check the distribution of objects between the shards:"},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:01.559486Z","iopub.status.busy":"2021-01-11T18:12:01.558883Z","iopub.status.idle":"2021-01-11T18:12:01.796554Z","shell.execute_reply":"2021-01-11T18:12:01.796015Z"},"papermill":{"duration":0.287812,"end_time":"2021-01-11T18:12:01.796689","exception":false,"start_time":"2021-01-11T18:12:01.508877","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df_shard = pd.merge(wbf_df, df_folds[df_folds['fold'] == 0], on='image_id')\ndfs = df_shard.class_name.value_counts().to_frame('S0').sort_index()\nfor i in range(1,20):\n    df_shard = pd.merge(wbf_df, df_folds[df_folds['fold'] == i], on='image_id')\n    dfs['S'+str(i)] = df_shard.class_name.value_counts().to_frame().sort_index()\ndfs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.050344,"end_time":"2021-01-11T18:12:01.897694","exception":false,"start_time":"2021-01-11T18:12:01.84735","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Create TFRecords\nThe records will be compatible with TensorFlow Object Detection API. We only add images with objects. "},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:02.03125Z","iopub.status.busy":"2021-01-11T18:12:02.030384Z","iopub.status.idle":"2021-01-11T18:12:02.033377Z","shell.execute_reply":"2021-01-11T18:12:02.033897Z"},"papermill":{"duration":0.086055,"end_time":"2021-01-11T18:12:02.034046","exception":false,"start_time":"2021-01-11T18:12:01.947991","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TPATH = '../input/vinbigdata-chest-xray-abnormalities-detection/train/'\n\n# Create example for TensorFlow Object Detection API\ndef create_tf_example(imagedf, longest_edge=IMAGE_SIZE):  \n    fname = TPATH+imagedf.image_id.iloc[0]+'.dicom'\n    filename=fname.split('/')[-1] # exclude path    \n    img, org_size = read_image(fname, target_size=IMAGE_SIZE, use_clahe=True)\n    height = img.size[1] # Image height\n    width = img.size[0] # Image width\n    buf= BytesIO()\n    img.save(buf, format= 'JPEG') # encode to jpeg in memory\n    encoded_image_data= buf.getvalue()\n    image_format = b'jpeg'\n    source_id = imagedf.image_id.iloc[0]\n    # A hash of the image is used in some frameworks\n    key = hashlib.sha256(encoded_image_data).hexdigest()   \n    # object bounding boxes \n    xmins = imagedf.x_min.values/org_size[0] # List of normalized left x coordinates in bounding box \n    xmaxs = imagedf.x_max.values/org_size[0] # List of normalized right x coordinates in bounding box\n    ymins = imagedf.y_min.values/org_size[1] # List of normalized top y coordinates in bounding box \n    ymaxs = imagedf.y_max.values/org_size[1] # List of normalized bottom y coordinates in bounding box\n    # List of string class name & id of bounding box (1 per box)\n    object_cnt = len(imagedf)\n    classes_text = []\n    classes = []\n    for i in range(object_cnt):\n        classes_text.append(imagedf.class_name.iloc[i].encode())\n        classes.append(1+imagedf.class_id.iloc[i]) # 0 is not a valid class\n        \n    # unused features from Open Image \n    depiction = np.zeros(object_cnt, dtype=int)\n    group_of = np.zeros(object_cnt, dtype=int)\n    occluded = np.zeros(object_cnt, dtype=int) #also Pascal VOC\n    truncated = np.zeros(object_cnt, dtype=int) # also Pascal VOC\n    # Pascal VOC\n    view_text = []\n    for i in range(object_cnt):\n        view_text.append('frontal'.encode())\n    difficult = np.zeros(object_cnt, dtype=int)\n\n    tf_record = tf.train.Example(features=tf.train.Features(feature={\n        'image/height': tf.train.Feature(int64_list=tf.train.Int64List(value=[height])),\n        'image/width': tf.train.Feature(int64_list=tf.train.Int64List(value=[width])),\n        'image/filename': tf.train.Feature(bytes_list=tf.train.BytesList(value=[filename.encode()])),\n        'image/source_id': tf.train.Feature(bytes_list=tf.train.BytesList(value=[source_id.encode()])),\n        'image/encoded': tf.train.Feature(bytes_list=tf.train.BytesList(value=[encoded_image_data])),\n        'image/key/sha256': tf.train.Feature(bytes_list=tf.train.BytesList(value=[key.encode()])),\n        'image/format': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_format])),\n        'image/object/bbox/xmin': tf.train.Feature(float_list=tf.train.FloatList(value=xmins)),\n        'image/object/bbox/xmax': tf.train.Feature(float_list=tf.train.FloatList(value=xmaxs)),\n        'image/object/bbox/ymin': tf.train.Feature(float_list=tf.train.FloatList(value=ymins)),\n        'image/object/bbox/ymax': tf.train.Feature(float_list=tf.train.FloatList(value=ymaxs)),\n        'image/object/class/text': tf.train.Feature(bytes_list=tf.train.BytesList(value=classes_text)),\n        'image/object/class/label': tf.train.Feature(int64_list=tf.train.Int64List(value=classes)),\n        'image/object/depiction': tf.train.Feature(int64_list=tf.train.Int64List(value=depiction)),\n        'image/object/group_of': tf.train.Feature(int64_list=tf.train.Int64List(value=group_of)),\n        'image/object/occluded': tf.train.Feature(int64_list=tf.train.Int64List(value=occluded)),\n        'image/object/truncated': tf.train.Feature(int64_list=tf.train.Int64List(value=truncated)),\n        'image/object/difficult': tf.train.Feature(int64_list=tf.train.Int64List(value=difficult)),\n        'image/object/view': tf.train.Feature(bytes_list=tf.train.BytesList(value=view_text))\n    }))\n    return tf_record","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.047993,"end_time":"2021-01-11T18:12:02.130658","exception":false,"start_time":"2021-01-11T18:12:02.082665","status":"completed"},"tags":[]},"cell_type":"markdown","source":"We use sharding to create 20 TFRecords. This gives us a 5% resolution when creating train/validation split."},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T18:12:02.231704Z","iopub.status.busy":"2021-01-11T18:12:02.230649Z","iopub.status.idle":"2021-01-11T20:05:27.492539Z","shell.execute_reply":"2021-01-11T20:05:27.493224Z"},"papermill":{"duration":6805.314485,"end_time":"2021-01-11T20:05:27.493381","exception":false,"start_time":"2021-01-11T18:12:02.178896","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport contextlib2\n\ndef open_sharded_tfrecords(exit_stack, base_path, num_shards):\n    tf_record_output_filenames = [\n        '{}-{:03d}-of-{:03}.tfrecord'.format(base_path, idx, num_shards)\n        for idx in range(num_shards)\n        ]\n    tfrecords = [\n        exit_stack.enter_context(tf.io.TFRecordWriter(file_name))\n        for file_name in tf_record_output_filenames\n    ]\n    return tfrecords\n\noutput_filebase='./VinBig'\n\nimg_cnt = np.zeros(NUM_SHARDS, dtype=int)\nwith contextlib2.ExitStack() as tf_record_close_stack:\n    output_tfrecords = open_sharded_tfrecords(tf_record_close_stack, output_filebase, NUM_SHARDS)\n    for i in range(NUM_SHARDS):\n        df_shard = pd.merge(wbf_df, df_folds[df_folds['fold'] == i], on='image_id')\n        ids = df_shard.image_id.unique()\n        for j in range (len(ids)):\n            imagedf = df_shard[df_shard.image_id == ids[j]]\n            tf_record = create_tf_example(imagedf, longest_edge=IMAGE_SIZE)            \n            output_tfrecords[i].write(tf_record.SerializeToString())\n            img_cnt[i] += 1\nprint(\"Converted {} images\".format(np.sum(img_cnt)))\nprint(\"Images per shard: {}\".format(img_cnt))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.050236,"end_time":"2021-01-11T20:05:27.595222","exception":false,"start_time":"2021-01-11T20:05:27.544986","status":"completed"},"tags":[]},"cell_type":"markdown","source":"Parameters used in the TFRecord creation are saved in a .json file for use in training and inference notebooks."},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T20:05:27.704218Z","iopub.status.busy":"2021-01-11T20:05:27.702509Z","iopub.status.idle":"2021-01-11T20:05:27.707564Z","shell.execute_reply":"2021-01-11T20:05:27.707001Z"},"papermill":{"duration":0.062085,"end_time":"2021-01-11T20:05:27.707697","exception":false,"start_time":"2021-01-11T20:05:27.645612","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import json\n\ndparams = {\n    \"IMAGE_SIZE\": IMAGE_SIZE,\n    \"CLIP_LIMIT\": CLIP_LIMIT,\n    \"GRID_SIZE\": GRID_SIZE}\nwith open(\"dparams.json\", \"w\") as json_file:\n    json_file.write(json.dumps(dparams, indent = 4))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.054878,"end_time":"2021-01-11T20:05:27.816633","exception":false,"start_time":"2021-01-11T20:05:27.761755","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Label data\nWe also need to create a label data file. Note that the TF Object Detection API expects the first class to be \"1\" and not \"0\". "},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T20:05:27.943816Z","iopub.status.busy":"2021-01-11T20:05:27.942421Z","iopub.status.idle":"2021-01-11T20:05:27.945774Z","shell.execute_reply":"2021-01-11T20:05:27.945088Z"},"papermill":{"duration":0.066646,"end_time":"2021-01-11T20:05:27.94589","exception":false,"start_time":"2021-01-11T20:05:27.879244","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"labels = ['Aortic enlargement', 'Atelectasis', 'Calcification', 'Cardiomegaly', 'Consolidation',\n          'ILD', 'Infiltration', 'Lung Opacity', 'Nodule/Mass', 'Other lesion', 'Pleural effusion',\n          'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis']\n\nwith open('./VinBig.pbtxt', 'w') as f:\n    for i in range (len(labels)): \n        f.write('item {{\\n id: {}\\n name:\\'{}\\'\\n}}\\n\\n'.format(i+1, labels[i])) ","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.050335,"end_time":"2021-01-11T20:05:28.048272","exception":false,"start_time":"2021-01-11T20:05:27.997937","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Check TFRecords\nVerify the result by reading and plotting a few X-rays."},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T20:05:28.170175Z","iopub.status.busy":"2021-01-11T20:05:28.169469Z","iopub.status.idle":"2021-01-11T20:05:28.205125Z","shell.execute_reply":"2021-01-11T20:05:28.204486Z"},"papermill":{"duration":0.106577,"end_time":"2021-01-11T20:05:28.20524","exception":false,"start_time":"2021-01-11T20:05:28.098663","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Some helper functions to draw image with object boundary boxes\nfontname = '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf'\nfont = ImageFont.truetype(fontname, 40) if os.path.isfile(fontname) else ImageFont.load_default()\n\ndef bbox(img, xmin, ymin, xmax, ymax, color, width, label, score):\n    draw = ImageDraw.Draw(img)\n    xres, yres = img.size[0], img.size[1]\n    box = np.multiply([xmin, ymin, xmax, ymax], [xres, yres, xres, yres]).astype(int).tolist()\n    txt = \" {}: {}%\" if score >= 0. else \" {}\"\n    txt = txt.format(label, round(score, 1))\n    ts = draw.textsize(txt, font=font)\n    draw.rectangle(box, outline=color, width=width)\n    if len(label) > 0:\n        if box[1] >= ts[1]+3:\n            xsmin, ysmin = box[0], box[1]-ts[1]-3\n            xsmax, ysmax = box[0]+ts[0]+2, box[1]\n        else:\n            xsmin, ysmin = box[0], box[3]\n            xsmax, ysmax = box[0]+ts[0]+2, box[3]+ts[1]+1\n        draw.rectangle([xsmin, ysmin, xsmax, ysmax], fill=color)\n        draw.text((xsmin, ysmin), txt, font=font, fill='white')\n\ndef plot_img(img, axes, xmin, ymin, xmax, ymax, classes, class_label, by):\n    img = img.convert(\"RGB\")\n    for i in range(len(xmin)):\n        color = LABEL_COLORS[class_label[i]]\n        bbox(img, xmin[i], ymin[i], xmax[i], ymax[i], color, 5, classes[i].decode(), -1)\n    plt.setp(axes, xticks=[], yticks=[])\n    axes.set_title(by)\n    plt.imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-11T20:05:28.333686Z","iopub.status.busy":"2021-01-11T20:05:28.3328Z","iopub.status.idle":"2021-01-11T20:05:30.357708Z","shell.execute_reply":"2021-01-11T20:05:30.358243Z"},"papermill":{"duration":2.102059,"end_time":"2021-01-11T20:05:30.358384","exception":false,"start_time":"2021-01-11T20:05:28.256325","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"fname='./VinBig-000-of-020.tfrecord' \ndataset3 = tf.data.TFRecordDataset(fname)\nfig = plt.figure(figsize=(20,30))\nidx=1\nfor raw_record in dataset3.take(6):\n    axes = fig.add_subplot(3, 2, idx)\n    example = tf.train.Example()\n    example.ParseFromString(raw_record.numpy())\n    xmin=example.features.feature['image/object/bbox/xmin'].float_list.value[:]\n    xmax=example.features.feature['image/object/bbox/xmax'].float_list.value[:]\n    ymin=example.features.feature['image/object/bbox/ymin'].float_list.value[:]\n    ymax=example.features.feature['image/object/bbox/ymax'].float_list.value[:]\n    classes=example.features.feature['image/object/class/text'].bytes_list.value[:]\n    class_label=example.features.feature['image/object/class/label'].int64_list.value[:]\n    img_encoded=example.features.feature['image/encoded'].bytes_list.value[0]\n    img = Image.open(BytesIO(img_encoded))\n    plot_img(img, axes, xmin, ymin, xmax, ymax, classes, class_label, \"\")\n    idx=idx+1","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}