{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Mayo Clinic Create Parsed Image Dataset\n\nMuch of this code was adapted from: https://www.kaggle.com/code/simsonimus/hubmap-training\n\n- Split into parts based on file size","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom skimage import io\nimport glob, os, json, cv2, gc, shutil\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import backend as K\nfrom tqdm.notebook import tqdm\n\nIMG_SIZE = 1024\nINPUT_PATH = \"../input/mayo-clinic-strip-ai\"\nPRINT_PLOTS = False\nFULL_RUN = True\n\ndef create_folder(folder):\n    if not os.path.exists(folder):\n        os.makedirs(folder)\n        \n# create_folder(\"./plots\")\ncreate_folder(\"./train\")\ncreate_folder(\"./test\")\n\ndf_train = pd.read_csv(os.path.join(INPUT_PATH, 'train.csv'))\ndisplay(df_train)\n\ndf_test = pd.read_csv(os.path.join(INPUT_PATH, 'test.csv'))\ndisplay(df_test)\n\nGROUP_TO_RUN = 1\nTOTAL_GROUPS = 10\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T16:19:41.170588Z","iopub.execute_input":"2022-07-07T16:19:41.171401Z","iopub.status.idle":"2022-07-07T16:19:41.209351Z","shell.execute_reply.started":"2022-07-07T16:19:41.171363Z","shell.execute_reply":"2022-07-07T16:19:41.208167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Filesizes\n- Split into equally sized groups based on these file sizes","metadata":{}},{"cell_type":"code","source":"for i, d in tqdm(df_train.iterrows()):\n\n    image_id = d['image_id']\n    file_size = os.path.getsize(f'../input/mayo-clinic-strip-ai/train/{image_id}.tif')\n    df_train.loc[i, 'file_size'] = file_size\ntarget_group_size = df_train['file_size'].sum() / (TOTAL_GROUPS - 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:19:59.571902Z","iopub.execute_input":"2022-07-07T16:19:59.573219Z","iopub.status.idle":"2022-07-07T16:20:00.302513Z","shell.execute_reply.started":"2022-07-07T16:19:59.573155Z","shell.execute_reply":"2022-07-07T16:20:00.301274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['file_size'].plot(kind='hist', bins=20)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:20:01.265004Z","iopub.execute_input":"2022-07-07T16:20:01.26613Z","iopub.status.idle":"2022-07-07T16:20:01.494109Z","shell.execute_reply.started":"2022-07-07T16:20:01.266052Z","shell.execute_reply":"2022-07-07T16:20:01.49281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create 10 equal sized groups by file size\ngroup = 1\ngroup_size = 0\nfor i, d in tqdm(df_train.iterrows()):\n    image_id = d['image_id']\n    file_size = os.path.getsize(f'../input/mayo-clinic-strip-ai/train/{image_id}.tif')\n    if group_size + file_size > target_group_size:\n        group += 1\n        group_size = file_size\n    else:\n        group_size += file_size\n    df_train.loc[i, 'file_size'] = file_size\n    df_train.loc[i, 'group'] = group\n    \ndf_train['group'] = df_train['group'].astype('int')\ndf_train.to_csv('train_with_groups.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:20:02.347646Z","iopub.execute_input":"2022-07-07T16:20:02.348754Z","iopub.status.idle":"2022-07-07T16:20:02.812776Z","shell.execute_reply.started":"2022-07-07T16:20:02.348707Z","shell.execute_reply":"2022-07-07T16:20:02.811505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby('group')['file_size'].sum().plot(kind='barh', figsize=(10, 5))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:20:03.371575Z","iopub.execute_input":"2022-07-07T16:20:03.372817Z","iopub.status.idle":"2022-07-07T16:20:03.585994Z","shell.execute_reply.started":"2022-07-07T16:20:03.372742Z","shell.execute_reply":"2022-07-07T16:20:03.584843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.query('group == @GROUP_TO_RUN')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:19:11.817506Z","iopub.execute_input":"2022-07-07T16:19:11.818342Z","iopub.status.idle":"2022-07-07T16:19:11.829269Z","shell.execute_reply.started":"2022-07-07T16:19:11.818304Z","shell.execute_reply":"2022-07-07T16:19:11.82803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def read_tiff(image_path):\n    image = io.imread(image_path)\n    image = np.squeeze(image) # some images have unnecessary axes with shape 1 --> remove\n    if image.shape[0] == 3: # some images have color as first axis -> swap axes\n        image = image.swapaxes(0,1)\n        image = image.swapaxes(1,2)\n    return image\n\ndef read_mask(image, encoded_mask):\n    mask = rle_decode(encoded_mask, (image.shape[1], image.shape[0])) # with inverted axes\n    mask = mask.swapaxes(0,1) # swap back axes\n    mask = np.expand_dims(mask, -1) # add one axis to have same shape as images\n    return mask\n\ndef delete_directory_contents(dir):\n    for file in os.scandir(dir):\n        os.remove(file.path)\n        \ndef plot_masked_image(image, mask, name):\n    plt.imshow(image, interpolation='none')\n    plt.imshow(mask, cmap='jet', alpha=0.3, interpolation='none')\n    \n    plt.savefig(f\"./plots/{name}.png\", dpi = 1000)\n    plt.show()\n    \ndef slice_images(image_id, image, mask=[], folder=\"\"):\n    print('Slicing Image ' + image_id + ' ...')\n\n    possible_slices_x = image.shape[0] // IMG_SIZE\n    possible_slices_y = image.shape[1] // IMG_SIZE\n\n    for x in range(possible_slices_x):\n        for y in range(possible_slices_y):\n            image_slice = image[x * IMG_SIZE : (x+1) * IMG_SIZE, y * IMG_SIZE : (y+1) * IMG_SIZE]\n            \n            #if np.any(image_slice) and not (image_slice > 200).all(): # only process non-black and non-gray images --> no background images\n\n            if not len(mask) == 0:\n                mask_slice = mask[x * IMG_SIZE : (x+1) * IMG_SIZE, y * IMG_SIZE : (y+1) * IMG_SIZE] * 255\n                if 255 in mask_slice:\n                    cv2.imwrite(f\"./{folder}/{image_id}-imgslice.{x}.{y}.jpg\", image_slice)\n                    cv2.imwrite(f\"./{folder}/{image_id}-maskslice.{x}.{y}.png\", mask_slice.astype(int))\n            else:\n                cv2.imwrite(f\"./{folder}/{image_id}-imgslice.{x}.{y}.jpg\", image_slice)\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_decode(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)\n\n## ref.: https://www.kaggle.com/bguberfain/memory-aware-rle-encoding\ndef rle_encode_less_memory(img):\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:19:15.421114Z","iopub.execute_input":"2022-07-07T16:19:15.422536Z","iopub.status.idle":"2022-07-07T16:19:15.443482Z","shell.execute_reply.started":"2022-07-07T16:19:15.422492Z","shell.execute_reply":"2022-07-07T16:19:15.442598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Slice Training Images","metadata":{}},{"cell_type":"code","source":"def slice_training_images(df_train):\n    if not FULL_RUN:\n        df_train = df_train.iloc[0:1, :]  # only use one training image for quicker debug runs\n    else:\n        df_train = df_train.iloc[1:, :]\n    for index, train_sample in tqdm(df_train.iterrows(), total=len(df_train)):\n        image_id = train_sample['image_id']\n\n        image_path = os.path.join(INPUT_PATH, f\"train/{image_id}.tif\")\n        image = read_tiff(image_path)\n\n        slice_images(image_id, image, [], \"train\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T15:23:17.650028Z","iopub.execute_input":"2022-07-07T15:23:17.650455Z","iopub.status.idle":"2022-07-07T15:23:17.658918Z","shell.execute_reply.started":"2022-07-07T15:23:17.650423Z","shell.execute_reply":"2022-07-07T15:23:17.657508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slice_training_images(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T15:23:18.654189Z","iopub.execute_input":"2022-07-07T15:23:18.655065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Slice Test Images","metadata":{}},{"cell_type":"code","source":"def slice_test_images(df_test):\n    if not FULL_RUN:\n        df_test = df_test.iloc[0:1, :]  # only use one training image for quicker debug runs\n    else:\n        df_test = df_test.iloc[1:, :]\n    for index, test_sample in tqdm(df_test.iterrows(), total=len(df_test)):\n        image_id = test_sample['image_id']\n\n        image_path = os.path.join(INPUT_PATH, f\"test/{image_id}.tif\")\n        image = read_tiff(image_path)\n\n        slice_images(image_id, image, [], \"test\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# slice_test_images(df_test)","metadata":{},"execution_count":null,"outputs":[]}]}