{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import the modules","metadata":{}},{"cell_type":"markdown","source":"### Basic imports","metadata":{}},{"cell_type":"code","source":"import os\nfrom tqdm.notebook import tqdm\n\nimport numpy as np \nimport pandas as pd \nfrom collections import Counter\n\nimport matplotlib\nfrom matplotlib import pyplot as plt\n\nimport PIL\nfrom PIL import Image\nfrom IPython.display import display\n\nimport openslide\nimport skimage.io\n\nfrom sklearn.model_selection import train_test_split\n\nimport cv2\nimport zipfile","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:52:23.014531Z","iopub.execute_input":"2021-09-07T12:52:23.014926Z","iopub.status.idle":"2021-09-07T12:52:24.402647Z","shell.execute_reply.started":"2021-09-07T12:52:23.014831Z","shell.execute_reply":"2021-09-07T12:52:24.401803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model based imports","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras import Model\nfrom keras.layers import Input, Dropout, concatenate, Dense\nfrom keras.layers.convolutional import Conv2D, MaxPooling2D, UpSampling2D\n\nimport keras.backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom keras import models\nfrom keras import layers\nfrom keras import optimizers","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:52:24.931078Z","iopub.execute_input":"2021-09-07T12:52:24.931405Z","iopub.status.idle":"2021-09-07T12:52:29.047353Z","shell.execute_reply.started":"2021-09-07T12:52:24.931377Z","shell.execute_reply":"2021-09-07T12:52:29.046456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data folders","metadata":{}},{"cell_type":"code","source":"BASE_DIR = '../input/prostate-cancer-grade-assessment/'\nSAVE_DIR = '/kaggle/working/'\n\nOVERLAY_IMG_DIR = '/kaggle/working/overlay/'\nTILING_IMG_DIR = '/kaggle/working/tiled_images/'\n\nIMG_SIZE = 224","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:14:09.488453Z","iopub.execute_input":"2021-09-07T13:14:09.488769Z","iopub.status.idle":"2021-09-07T13:14:09.493894Z","shell.execute_reply.started":"2021-09-07T13:14:09.488741Z","shell.execute_reply":"2021-09-07T13:14:09.492985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The Dataset","metadata":{}},{"cell_type":"code","source":"base_data = pd.read_csv(BASE_DIR+'train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:52:29.059484Z","iopub.execute_input":"2021-09-07T12:52:29.059779Z","iopub.status.idle":"2021-09-07T12:52:29.09264Z","shell.execute_reply.started":"2021-09-07T12:52:29.059748Z","shell.execute_reply":"2021-09-07T12:52:29.091878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filtering non-mask images","metadata":{}},{"cell_type":"code","source":"images_without_mask = []\nfor image in base_data['image_id']:\n    if not os.path.exists(BASE_DIR+'train_label_masks/'+image+'_mask.tiff'):\n        images_without_mask.append(image)\n\ndata_without_mask = base_data[base_data['image_id'].isin(images_without_mask)]\n\nbase_data = base_data[~base_data['image_id'].isin(images_without_mask)]","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:52:29.094214Z","iopub.execute_input":"2021-09-07T12:52:29.094562Z","iopub.status.idle":"2021-09-07T12:52:49.490068Z","shell.execute_reply.started":"2021-09-07T12:52:29.094528Z","shell.execute_reply":"2021-09-07T12:52:49.489218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Separate the data sources","metadata":{}},{"cell_type":"code","source":"radboud_train_set = base_data[base_data['data_provider']=='radboud']\nkarolinska_train_set = base_data[base_data['data_provider']=='karolinska']","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:52:49.49184Z","iopub.execute_input":"2021-09-07T12:52:49.492379Z","iopub.status.idle":"2021-09-07T12:52:49.502817Z","shell.execute_reply.started":"2021-09-07T12:52:49.492337Z","shell.execute_reply":"2021-09-07T12:52:49.501999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 1: Segmentation Model","metadata":{}},{"cell_type":"markdown","source":"### Split and load the data for the segmentation model","metadata":{}},{"cell_type":"code","source":"def load_images(image, with_mask=True):\n    \n    slide = openslide.OpenSlide(os.path.join(BASE_DIR+\"train_images\", f'{image}.tiff'))\n    \n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    img = slide.get_thumbnail(size=(IMG_SIZE,IMG_SIZE))\n    \n    \n    img = Image.fromarray(np.array(img))\n    img = img.resize((IMG_SIZE, IMG_SIZE))\n    img = np.array(img)\n\n    if with_mask:\n        mask =  openslide.OpenSlide(os.path.join(BASE_DIR+'train_label_masks', f'{image}_mask.tiff'))\n        mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n\n        mask_data = Image.fromarray(np.array(mask_data))\n        mask_data = mask_data.resize((IMG_SIZE, IMG_SIZE))\n        mask_data = np.array(mask_data)\n        mask_data = mask_data/5\n    \n        return img, mask_data[:,:,0]\n\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:53:32.602866Z","iopub.execute_input":"2021-09-07T12:53:32.603323Z","iopub.status.idle":"2021-09-07T12:53:32.617788Z","shell.execute_reply.started":"2021-09-07T12:53:32.603286Z","shell.execute_reply":"2021-09-07T12:53:32.616926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unet_train_ids, unet_test_ids, unet_train_labels, unet_test_labels = train_test_split(radboud_train_set['image_id'], radboud_train_set['isup_grade'], train_size=0.85)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:53:35.658545Z","iopub.execute_input":"2021-09-07T12:53:35.658852Z","iopub.status.idle":"2021-09-07T12:53:35.666039Z","shell.execute_reply.started":"2021-09-07T12:53:35.658824Z","shell.execute_reply":"2021-09-07T12:53:35.664972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loss metric: Dice coefficient","metadata":{}},{"cell_type":"code","source":"def Dice_coeff(y_true, y_pred):\n    smooth = 1\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:32:18.972662Z","iopub.execute_input":"2021-09-07T12:32:18.973029Z","iopub.status.idle":"2021-09-07T12:32:18.980484Z","shell.execute_reply.started":"2021-09-07T12:32:18.972993Z","shell.execute_reply":"2021-09-07T12:32:18.979706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### UNet model","metadata":{}},{"cell_type":"code","source":"def unet_model():\n    in1 = Input(shape=(IMG_SIZE, IMG_SIZE, 3 ))\n\n    conv1 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(in1)\n    conv1 = Dropout(0.3)(conv1)\n    conv1 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv1)\n    pool1 = MaxPooling2D((2, 2))(conv1)\n\n    conv2 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(pool1)\n    conv2 = Dropout(0.3)(conv2)\n    conv2 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv2)\n    pool2 = MaxPooling2D((2, 2))(conv2)\n\n    conv3 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(pool2)\n    conv3 = Dropout(0.3)(conv3)\n    conv3 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv3)\n    pool3 = MaxPooling2D((2, 2))(conv3)\n\n    conv4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(pool3)\n    conv4 = Dropout(0.3)(conv4)\n    conv4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv4)\n\n    up1 = concatenate([UpSampling2D((2, 2))(conv4), conv3], axis=-1)\n    conv5 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(up1)\n    conv5 = Dropout(0.3)(conv5)\n    conv5 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv5)\n    \n    up2 = concatenate([UpSampling2D((2, 2))(conv5), conv2], axis=-1)\n    conv6 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(up2)\n    conv6 = Dropout(0.3)(conv6)\n    conv6 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv6)\n\n    up2 = concatenate([UpSampling2D((2, 2))(conv6), conv1], axis=-1)\n    conv7 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(up2)\n    conv7 = Dropout(0.3)(conv7)\n    conv7 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(conv7)\n    segmentation = Conv2D(1, (1, 1), activation='sigmoid', name='seg')(conv7)\n\n    model = Model(inputs=[in1], outputs=[segmentation])\n    print(model.summary())\n    \n    optimizer=keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=0.001, decay=0.0, amsgrad=True)\n    \n    model.compile(optimizer='adam', loss = 'binary_crossentropy', metrics = ['acc', Dice_coeff])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:32:18.982225Z","iopub.execute_input":"2021-09-07T12:32:18.98263Z","iopub.status.idle":"2021-09-07T12:32:19.002346Z","shell.execute_reply.started":"2021-09-07T12:32:18.982593Z","shell.execute_reply":"2021-09-07T12:32:19.00096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_model = unet_model()","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:32:19.003567Z","iopub.execute_input":"2021-09-07T12:32:19.004076Z","iopub.status.idle":"2021-09-07T12:32:21.396383Z","shell.execute_reply.started":"2021-09-07T12:32:19.00399Z","shell.execute_reply":"2021-09-07T12:32:21.39559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model checkpoint and Early Stopping","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping,ModelCheckpoint\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)\nmc = ModelCheckpoint(SAVE_DIR+'Segmentor_checkpoint.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:32:21.39844Z","iopub.execute_input":"2021-09-07T12:32:21.398803Z","iopub.status.idle":"2021-09-07T12:32:21.403371Z","shell.execute_reply.started":"2021-09-07T12:32:21.398767Z","shell.execute_reply":"2021-09-07T12:32:21.402523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unet_train_X = []\nunet_train_Y = []\n\nfor img_id in tqdm(unet_train_ids):\n    train_img, msk_img = load_images(img_id)\n    unet_train_X.append(train_img)\n    unet_train_Y.append(msk_img)\nunet_train_X = np.array(unet_train_X)\nunet_train_Y = np.array(unet_train_Y)\nprint('Training done')","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:32:21.4282Z","iopub.execute_input":"2021-09-07T12:32:21.428622Z","iopub.status.idle":"2021-09-07T12:42:01.101027Z","shell.execute_reply.started":"2021-09-07T12:32:21.428583Z","shell.execute_reply":"2021-09-07T12:42:01.100109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the model","metadata":{}},{"cell_type":"code","source":"num_epoch = 60\nbatch_size = 30\nn_points = len(unet_train_X)\n\nhistory = seq_model.fit(x=unet_train_X, y=unet_train_Y, \n                validation_split=0.15,\n                epochs=num_epoch,steps_per_epoch = np.ceil(n_points / batch_size), callbacks =[es,mc],  shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:42:01.102699Z","iopub.execute_input":"2021-09-07T12:42:01.103223Z","iopub.status.idle":"2021-09-07T12:49:03.835211Z","shell.execute_reply.started":"2021-09-07T12:42:01.103185Z","shell.execute_reply":"2021-09-07T12:49:03.833714Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_model.predict(unet_train_X[:2])","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:49:27.284449Z","iopub.execute_input":"2021-09-07T12:49:27.284823Z","iopub.status.idle":"2021-09-07T12:49:27.714314Z","shell.execute_reply.started":"2021-09-07T12:49:27.284791Z","shell.execute_reply":"2021-09-07T12:49:27.713579Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 2: Overlay the mask on the images","metadata":{}},{"cell_type":"code","source":"def overlay_mask_on_slide(image_id, alpha=0.8, max_size=(IMG_SIZE, IMG_SIZE)):\n    \n    slide = openslide.OpenSlide(os.path.join(BASE_DIR+\"train_images\", f'{image_id}.tiff'))\n    mask = openslide.OpenSlide(os.path.join(BASE_DIR+\"train_label_masks\", f'{image_id}_mask.tiff'))\n#     mask = seq_model.predict(x=[slide])\n\n    slide_data = slide.read_region((0,0), slide.level_count - 1, slide.level_dimensions[-1])\n    mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n    mask_data = mask_data.split()[0]\n\n    alpha_int = int(round(255*alpha))\n    alpha_content = np.less(mask_data.split()[0], 2).astype('uint8') * alpha_int + (255 - alpha_int)\n\n    alpha_content = PIL.Image.fromarray(alpha_content)\n    preview_palette = np.zeros(shape=768, dtype=int)\n\n    preview_palette[0:18] = (np.array([0, 0, 0, 0.5, 0.5, 0.5, 0, 1, 0, 1, 1, 0.7, 1, 0.5, 0, 1, 0, 0]) * 255).astype(int)\n    \n    mask_data.putpalette(data=preview_palette.tolist())\n    mask_rgb = mask_data.convert(mode='RGB')\n    overlayed_image = PIL.Image.composite(image1=slide_data, image2=mask_rgb, mask=alpha_content)\n    overlayed_image.thumbnail(size=max_size, resample=0)\n\n    overlayed_image = overlayed_image.resize(max_size)\n\n    return overlayed_image","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:54:09.948352Z","iopub.execute_input":"2021-09-07T12:54:09.948673Z","iopub.status.idle":"2021-09-07T12:54:09.957837Z","shell.execute_reply.started":"2021-09-07T12:54:09.948644Z","shell.execute_reply":"2021-09-07T12:54:09.957059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating directories for the images\noverlay_train_img_folder = OVERLAY_IMG_DIR+'train/'\noverlay_test_img_folder = OVERLAY_IMG_DIR+'test/'\n\nif not os.path.exists(overlay_train_img_folder):\n    os.makedirs(overlay_train_img_folder)\n\nif not os.path.exists(overlay_test_img_folder):\n    os.makedirs(overlay_test_img_folder)\n\n#A directory for each label\nlabels = unet_train_labels.unique()\nfor lbl in labels:\n    if not os.path.exists(overlay_train_img_folder+str(lbl)):\n        os.makedirs(overlay_train_img_folder+str(lbl))\n    \n    if not os.path.exists(overlay_test_img_folder+str(lbl)):\n        os.makedirs(overlay_test_img_folder+str(lbl))","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:53:46.202268Z","iopub.execute_input":"2021-09-07T12:53:46.202596Z","iopub.status.idle":"2021-09-07T12:53:46.211749Z","shell.execute_reply.started":"2021-09-07T12:53:46.202567Z","shell.execute_reply":"2021-09-07T12:53:46.210681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, img_id in tqdm(enumerate(unet_train_ids)):\n    overlay_image = overlay_mask_on_slide(image_id=img_id)\n    file_name = overlay_train_img_folder+'/'+str(unet_train_labels.iloc[i])+'/'+img_id+'.jpeg'\n    overlay_image.save(file_name)\n\nfor i, img_id in tqdm(enumerate(unet_test_ids)):\n    overlay_image = overlay_mask_on_slide(image_id=img_id)\n    file_name = overlay_test_img_folder+'/'+str(unet_test_labels.iloc[i])+'/'+img_id+'.jpeg'\n    overlay_image.save(file_name)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T12:54:13.906784Z","iopub.execute_input":"2021-09-07T12:54:13.90716Z","iopub.status.idle":"2021-09-07T13:04:33.354959Z","shell.execute_reply.started":"2021-09-07T12:54:13.90713Z","shell.execute_reply":"2021-09-07T13:04:33.353866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 3: Tiling of the overlay images","metadata":{}},{"cell_type":"code","source":"tiling_train_img_data = TILING_IMG_DIR+'train/'\ntiling_test_img_data = TILING_IMG_DIR+'test/'\n\nif not os.path.exists(tiling_train_img_data):\n    os.makedirs(tiling_train_img_data)\n\nif not os.path.exists(tiling_test_img_data):\n    os.makedirs(tiling_test_img_data)\n    \nlabels = unet_train_labels.unique()\nfor lbl in labels:\n    if not os.path.exists(tiling_train_img_data+str(lbl)):\n        os.makedirs(tiling_train_img_data+str(lbl))\n    \n    if not os.path.exists(tiling_test_img_data+str(lbl)):\n        os.makedirs(tiling_test_img_data+str(lbl))\n\n# Parameters for cropping images\ncropPx= 56\ncropN = 16\nassert np.sqrt(cropN) == round(np.sqrt(cropN))","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:14:15.043033Z","iopub.execute_input":"2021-09-07T13:14:15.043361Z","iopub.status.idle":"2021-09-07T13:14:15.052098Z","shell.execute_reply.started":"2021-09-07T13:14:15.043332Z","shell.execute_reply":"2021-09-07T13:14:15.051102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile(img):\n    result = []\n    shape = img.shape\n    pad0,pad1 = (cropPx - shape[0]%cropPx)%cropPx, (cropPx - shape[1]%cropPx)%cropPx\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    \n    img = img.reshape(img.shape[0]//cropPx,cropPx,img.shape[1]//cropPx,cropPx,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,cropPx,cropPx,3)\n    \n    if len(img) < cropN:\n        img = np.pad(img,[[0,cropN-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:cropN]\n    img = img[idxs]\n    for i in range(len(img)):\n        result.append({'img':img[i], 'idx':i})\n    return result","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:14:20.417078Z","iopub.execute_input":"2021-09-07T13:14:20.417399Z","iopub.status.idle":"2021-09-07T13:14:20.42689Z","shell.execute_reply.started":"2021-09-07T13:14:20.41737Z","shell.execute_reply":"2021-09-07T13:14:20.425656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir(overlay_test_img_folder+str(label)))","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:19:38.75586Z","iopub.execute_input":"2021-09-07T13:19:38.756217Z","iopub.status.idle":"2021-09-07T13:19:38.762777Z","shell.execute_reply.started":"2021-09-07T13:19:38.756187Z","shell.execute_reply":"2021-09-07T13:19:38.761934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nbCol = int(np.sqrt(cropN))\n\nlabels = os.listdir(overlay_train_img_folder)\nfor label in tqdm(labels):\n    for name in tqdm(os.listdir(overlay_train_img_folder+str(label))):\n        img = skimage.io.MultiImage(os.path.join(overlay_train_img_folder+str(label)+'/',name))[-1]\n        tiles = tile(img)\n        stackImg = np.vstack([np.hstack([tiles[nbCol*col + row]['img'] for row in range(nbCol)])\n                   for col in range(nbCol)])\n        cv2.imwrite(tiling_train_img_data+str(label)+'/'+name, stackImg)\n\n\nnames = os.listdir(overlay_test_img_folder)\nfor label in tqdm(labels):\n    for name in tqdm(os.listdir(overlay_test_img_folder+str(label))):\n        img = skimage.io.MultiImage(os.path.join(overlay_test_img_folder+str(label)+'/',name))[-1]\n        tiles = tile(img)\n        stackImg = np.vstack([np.hstack([tiles[nbCol*col + row]['img'] for row in range(nbCol)])\n                   for col in range(nbCol)])\n        cv2.imwrite(tiling_test_img_data+str(label)+'/'+name, stackImg)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:20:04.593416Z","iopub.execute_input":"2021-09-07T13:20:04.593761Z","iopub.status.idle":"2021-09-07T13:20:06.674759Z","shell.execute_reply.started":"2021-09-07T13:20:04.593731Z","shell.execute_reply":"2021-09-07T13:20:06.673692Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"overlay_test_img_folder+'0/'+os.listdir(overlay_test_img_folder+'0')[0]","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:29:32.544156Z","iopub.execute_input":"2021-09-07T13:29:32.544483Z","iopub.status.idle":"2021-09-07T13:29:32.551147Z","shell.execute_reply.started":"2021-09-07T13:29:32.544455Z","shell.execute_reply":"2021-09-07T13:29:32.550173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(overlay_test_img_folder)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:30:42.994035Z","iopub.execute_input":"2021-09-07T13:30:42.994362Z","iopub.status.idle":"2021-09-07T13:30:43.001025Z","shell.execute_reply.started":"2021-09-07T13:30:42.994333Z","shell.execute_reply":"2021-09-07T13:30:43.000013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open(overlay_test_img_folder+'0/'+os.listdir(overlay_test_img_folder+'0')[0])\nimg = img.resize((IMG_SIZE, IMG_SIZE))\nimg = np.array(img)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:32:15.018927Z","iopub.execute_input":"2021-09-07T13:32:15.01925Z","iopub.status.idle":"2021-09-07T13:32:15.026544Z","shell.execute_reply.started":"2021-09-07T13:32:15.019221Z","shell.execute_reply":"2021-09-07T13:32:15.02558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:32:24.185897Z","iopub.execute_input":"2021-09-07T13:32:24.186272Z","iopub.status.idle":"2021-09-07T13:32:24.399737Z","shell.execute_reply.started":"2021-09-07T13:32:24.186241Z","shell.execute_reply":"2021-09-07T13:32:24.398824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 4: Classifier","metadata":{}},{"cell_type":"markdown","source":"### Image data Generator for the model","metadata":{}},{"cell_type":"code","source":" image_gen = ImageDataGenerator(\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    rescale=1/255,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode=\"nearest\"\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:34:07.183527Z","iopub.execute_input":"2021-09-07T13:34:07.183851Z","iopub.status.idle":"2021-09-07T13:34:07.18845Z","shell.execute_reply.started":"2021-09-07T13:34:07.183823Z","shell.execute_reply":"2021-09-07T13:34:07.187588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\ntrain_image_gen = image_gen.flow_from_directory(tiling_train_img_data,\n                                                target_size=(IMG_SIZE, IMG_SIZE),\n                                                batch_size=batch_size,\n                                                class_mode=\"categorical\")","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:34:09.828388Z","iopub.execute_input":"2021-09-07T13:34:09.82873Z","iopub.status.idle":"2021-09-07T13:34:09.943551Z","shell.execute_reply.started":"2021-09-07T13:34:09.828699Z","shell.execute_reply":"2021-09-07T13:34:09.942654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_gen = image_gen.flow_from_directory(tiling_test_img_data,\n                                                target_size=(IMG_SIZE, IMG_SIZE),\n                                                batch_size=batch_size,\n                                                class_mode=\"categorical\")","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:34:11.07317Z","iopub.execute_input":"2021-09-07T13:34:11.0735Z","iopub.status.idle":"2021-09-07T13:34:11.184797Z","shell.execute_reply.started":"2021-09-07T13:34:11.073472Z","shell.execute_reply":"2021-09-07T13:34:11.183844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The classifier model - Xception","metadata":{}},{"cell_type":"code","source":"model_builder = keras.applications.xception.Xception\n\nmodel = models.Sequential()\nmodel.add(model_builder(include_top=False, input_shape=(IMG_SIZE, IMG_SIZE, 3), pooling='avg'))\nmodel.add(Dense(6, activation='softmax'))\n\n# optimizer=keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=0.001, decay=0.0, amsgrad=True)\n\nmodel.compile(\n    loss=\"categorical_crossentropy\",\n    optimizer=optimizers.RMSprop(lr=3e-4),\n    metrics=[\"acc\"],\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:34:13.798488Z","iopub.execute_input":"2021-09-07T13:34:13.798822Z","iopub.status.idle":"2021-09-07T13:34:18.033565Z","shell.execute_reply.started":"2021-09-07T13:34:13.798792Z","shell.execute_reply":"2021-09-07T13:34:18.032745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping,ModelCheckpoint\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)\nmc = ModelCheckpoint(SAVE_DIR+'Classifier_checkpoint.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:34:40.676578Z","iopub.execute_input":"2021-09-07T13:34:40.676893Z","iopub.status.idle":"2021-09-07T13:34:40.683452Z","shell.execute_reply.started":"2021-09-07T13:34:40.676865Z","shell.execute_reply":"2021-09-07T13:34:40.682493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMBER_OF_TRAINING_IMAGES = unet_train_ids.shape[0]\nNUMBER_OF_TESTING_IMAGES = unet_test_ids.shape[0]\nresults = model.fit(\n    train_image_gen,\n    steps_per_epoch=NUMBER_OF_TRAINING_IMAGES // batch_size,\n    epochs=50,\n    validation_data=test_image_gen,\n    validation_steps=NUMBER_OF_TESTING_IMAGES // batch_size,\n    verbose=1,\n    use_multiprocessing=True,\n    callbacks =[es,mc],\n    workers=4,\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T13:35:18.76862Z","iopub.execute_input":"2021-09-07T13:35:18.768956Z","iopub.status.idle":"2021-09-07T13:56:34.012819Z","shell.execute_reply.started":"2021-09-07T13:35:18.768924Z","shell.execute_reply":"2021-09-07T13:56:34.01177Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:14:46.315212Z","iopub.execute_input":"2021-09-07T14:14:46.315539Z","iopub.status.idle":"2021-09-07T14:14:46.339992Z","shell.execute_reply.started":"2021-09-07T14:14:46.31551Z","shell.execute_reply":"2021-09-07T14:14:46.338228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.open(tiling_test_img_data+'2/'+os.listdir(overlay_test_img_folder+'2')[1])\nimg = img.resize((IMG_SIZE, IMG_SIZE))\nimg = np.array(img)\n\nimg = np.expand_dims(img, axis=0)\nmodel.predict(img)","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:14:27.347641Z","iopub.execute_input":"2021-09-07T14:14:27.347995Z","iopub.status.idle":"2021-09-07T14:14:27.401558Z","shell.execute_reply.started":"2021-09-07T14:14:27.347963Z","shell.execute_reply":"2021-09-07T14:14:27.400784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:11:02.098203Z","iopub.execute_input":"2021-09-07T14:11:02.09853Z","iopub.status.idle":"2021-09-07T14:11:02.104013Z","shell.execute_reply.started":"2021-09-07T14:11:02.0985Z","shell.execute_reply":"2021-09-07T14:11:02.103106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:10:55.566312Z","iopub.execute_input":"2021-09-07T14:10:55.566646Z","iopub.status.idle":"2021-09-07T14:10:55.58577Z","shell.execute_reply.started":"2021-09-07T14:10:55.566617Z","shell.execute_reply":"2021-09-07T14:10:55.584832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-09-07T14:12:18.643149Z","iopub.execute_input":"2021-09-07T14:12:18.643534Z","iopub.status.idle":"2021-09-07T14:12:19.492635Z","shell.execute_reply.started":"2021-09-07T14:12:18.643501Z","shell.execute_reply":"2021-09-07T14:12:19.491839Z"},"trusted":true},"execution_count":null,"outputs":[]}]}