{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Analysing Effects of Pretraining On Similar Dataset\n\nIn this notebook I will try to do the following:\n* Pretrain the model on a previous competiton similar dataset\n* Finetune the pretrained model on the current competition dataset\n* Generate pseudo labels for the previous dataset from fine tuned dataset\n* Retrain the model on the generated pseudo labels and observe the model performance on the current competition dataset.","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:36:36.276531Z","iopub.execute_input":"2021-07-08T06:36:36.276896Z","iopub.status.idle":"2021-07-08T06:36:36.977424Z","shell.execute_reply.started":"2021-07-08T06:36:36.276867Z","shell.execute_reply":"2021-07-08T06:36:36.976367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet -q","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:36:40.101463Z","iopub.execute_input":"2021-07-08T06:36:40.10191Z","iopub.status.idle":"2021-07-08T06:36:45.72424Z","shell.execute_reply.started":"2021-07-08T06:36:40.101866Z","shell.execute_reply":"2021-07-08T06:36:45.723152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\n\nfrom sklearn.model_selection import GroupKFold, train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:36:45.726035Z","iopub.execute_input":"2021-07-08T06:36:45.726381Z","iopub.status.idle":"2021-07-08T06:36:47.901768Z","shell.execute_reply.started":"2021-07-08T06:36:45.726342Z","shell.execute_reply":"2021-07-08T06:36:47.900913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Coniguration\n","metadata":{}},{"cell_type":"code","source":"class Config:\n    IMAGES = '../input/vinbigdata-chest-xray-resized-png-1024x1024/train'\n    DATA = '../input/vinbigdata-chest-xray-abnormalities-detection/train.csv'\n    \n    AUTOTUNE = tf.data.experimental.AUTOTUNE\n    \n    PRETRAINING_IMAGE_SIZE = 256 # I am using smaller image size for faster training\n    PRETRAINING_BATCH_SIZE = 8\n    PRETRAINING_NUM_CLASSES = 15\n    PRETRAINING_LR = 0.001","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:04:31.537588Z","iopub.execute_input":"2021-07-08T04:04:31.538059Z","iopub.status.idle":"2021-07-08T04:04:31.543554Z","shell.execute_reply.started":"2021-07-08T04:04:31.538022Z","shell.execute_reply":"2021-07-08T04:04:31.542296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:04:31.545762Z","iopub.execute_input":"2021-07-08T04:04:31.546268Z","iopub.status.idle":"2021-07-08T04:04:31.554035Z","shell.execute_reply.started":"2021-07-08T04:04:31.54623Z","shell.execute_reply":"2021-07-08T04:04:31.553199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the pretraining data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(config.DATA)\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:04:31.55732Z","iopub.execute_input":"2021-07-08T04:04:31.557591Z","iopub.status.idle":"2021-07-08T04:04:31.656437Z","shell.execute_reply.started":"2021-07-08T04:04:31.55756Z","shell.execute_reply":"2021-07-08T04:04:31.655392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SELECTING ONLY REQUIRED COLUMNS\ndf['image_path'] = df['image_id'].map(lambda x: f'{config.IMAGES}/{x}.png')\ndf = df[['image_path', 'class_id']]\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:04:34.860899Z","iopub.execute_input":"2021-07-08T04:04:34.863443Z","iopub.status.idle":"2021-07-08T04:04:34.93685Z","shell.execute_reply.started":"2021-07-08T04:04:34.8634Z","shell.execute_reply":"2021-07-08T04:04:34.936065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.class_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:04:59.390233Z","iopub.execute_input":"2021-07-08T04:04:59.390571Z","iopub.status.idle":"2021-07-08T04:04:59.396902Z","shell.execute_reply.started":"2021-07-08T04:04:59.390539Z","shell.execute_reply":"2021-07-08T04:04:59.395595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the pretraining dataset","metadata":{}},{"cell_type":"code","source":"df_train, df_valid = train_test_split(df, test_size=0.1, random_state=1234)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:18.681288Z","iopub.execute_input":"2021-07-08T04:05:18.681639Z","iopub.status.idle":"2021-07-08T04:05:18.696394Z","shell.execute_reply.started":"2021-07-08T04:05:18.681606Z","shell.execute_reply":"2021-07-08T04:05:18.695483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aug_func(image_path, label):\n    file_bytes = tf.io.read_file(image_path)\n    img = tf.image.decode_png(file_bytes, channels=3)\n    img = tf.image.resize(img, [config.PRETRAINING_IMAGE_SIZE, config.PRETRAINING_IMAGE_SIZE])\n    img = img/255.\n    return img, label","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:18.909556Z","iopub.execute_input":"2021-07-08T04:05:18.909921Z","iopub.status.idle":"2021-07-08T04:05:18.916351Z","shell.execute_reply.started":"2021-07-08T04:05:18.909887Z","shell.execute_reply":"2021-07-08T04:05:18.915119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((df_train['image_path'].values, df_train['class_id'].values))\ntrain_dataset = train_dataset.map(aug_func, num_parallel_calls=config.AUTOTUNE)\ntrain_dataset = train_dataset.repeat()\ntrain_dataset = train_dataset.batch(config.PRETRAINING_BATCH_SIZE)\ntrain_dataset = train_dataset.prefetch(config.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:19.074668Z","iopub.execute_input":"2021-07-08T04:05:19.075085Z","iopub.status.idle":"2021-07-08T04:05:19.809328Z","shell.execute_reply.started":"2021-07-08T04:05:19.075048Z","shell.execute_reply":"2021-07-08T04:05:19.808476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dataset = tf.data.Dataset.from_tensor_slices((df_valid['image_path'].values, df_valid['class_id'].values))\nvalid_dataset = valid_dataset.map(aug_func, num_parallel_calls=config.AUTOTUNE)\nvalid_dataset = valid_dataset.batch(config.PRETRAINING_BATCH_SIZE)\nvalid_dataset = valid_dataset.prefetch(config.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:19.810774Z","iopub.execute_input":"2021-07-08T04:05:19.811127Z","iopub.status.idle":"2021-07-08T04:05:19.82967Z","shell.execute_reply.started":"2021-07-08T04:05:19.81109Z","shell.execute_reply":"2021-07-08T04:05:19.828944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, j in zip(train_dataset, valid_dataset):\n    print(i[0].shape, i[1].shape, j[0].shape, j[1].shape)\n    plt.figure(figsize=(20,10))\n    plt.subplot(1,2,1)\n    plt.imshow(i[0][0])\n    plt.subplot(1,2,2)\n    plt.imshow(j[0][0])\n    break","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:19.831255Z","iopub.execute_input":"2021-07-08T04:05:19.831612Z","iopub.status.idle":"2021-07-08T04:05:21.107397Z","shell.execute_reply.started":"2021-07-08T04:05:19.831568Z","shell.execute_reply":"2021-07-08T04:05:21.106585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the model","metadata":{}},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    efn.EfficientNetB2(\n        input_shape=(config.PRETRAINING_IMAGE_SIZE, config.PRETRAINING_IMAGE_SIZE, 3),\n        weights='imagenet',\n        include_top=False),\n    tf.keras.layers.GlobalAveragePooling2D(),\n    tf.keras.layers.Dense(config.PRETRAINING_NUM_CLASSES, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:05:22.865933Z","iopub.execute_input":"2021-07-08T04:05:22.866297Z","iopub.status.idle":"2021-07-08T04:05:25.831135Z","shell.execute_reply.started":"2021-07-08T04:05:22.866265Z","shell.execute_reply":"2021-07-08T04:05:25.83026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training only the classifier layer for 1 epoch\nmodel.layers[0].trainable = False\n\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=config.PRETRAINING_LR),\n    loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n    metrics=tf.keras.metrics.SparseCategoricalAccuracy())\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:06:25.529278Z","iopub.execute_input":"2021-07-08T04:06:25.529631Z","iopub.status.idle":"2021-07-08T04:06:25.583672Z","shell.execute_reply.started":"2021-07-08T04:06:25.529597Z","shell.execute_reply":"2021-07-08T04:06:25.582548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEPS = len(df_train) // config.PRETRAINING_BATCH_SIZE\n\nmodel_checkpoint = tf.keras.callbacks.ModelCheckpoint('./efficientNet_Pretraining', save_best_only=True)\nlr_schedular = tf.keras.callbacks.ReduceLROnPlateau(patience=1, min_delta=0.01)\nearly_stopping = tf.keras.callbacks.EarlyStopping(min_delta=0.001, patience=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:06:29.757428Z","iopub.execute_input":"2021-07-08T04:06:29.757794Z","iopub.status.idle":"2021-07-08T04:06:29.762591Z","shell.execute_reply.started":"2021-07-08T04:06:29.757758Z","shell.execute_reply":"2021-07-08T04:06:29.761778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretraining the model","metadata":{}},{"cell_type":"code","source":"# Only last layer\n\nmodel.fit(x = train_dataset,\n         epochs = 1,\n         steps_per_epoch = STEPS,\n         validation_data = valid_dataset,\n         callbacks = [model_checkpoint, lr_schedular, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:06:55.016522Z","iopub.execute_input":"2021-07-08T04:06:55.016917Z","iopub.status.idle":"2021-07-08T04:22:48.950526Z","shell.execute_reply.started":"2021-07-08T04:06:55.016885Z","shell.execute_reply":"2021-07-08T04:22:48.949681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training whole model with a small learning rate for 1 epoch\n\nmodel.layers[0].trainable = True\n\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate = 0.0001),\n    loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n    metrics=tf.keras.metrics.SparseCategoricalAccuracy())\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:23:40.672561Z","iopub.execute_input":"2021-07-08T04:23:40.672906Z","iopub.status.idle":"2021-07-08T04:23:40.727463Z","shell.execute_reply.started":"2021-07-08T04:23:40.672875Z","shell.execute_reply":"2021-07-08T04:23:40.726658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Whole model with hlaf steps so that weights don't distort much\n\nmodel.fit(x = train_dataset,\n         epochs = 3,\n         steps_per_epoch = STEPS,\n         validation_data = valid_dataset,\n         callbacks = [model_checkpoint, lr_schedular, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:25:04.685068Z","iopub.execute_input":"2021-07-08T04:25:04.68542Z","iopub.status.idle":"2021-07-08T04:36:11.019049Z","shell.execute_reply.started":"2021-07-08T04:25:04.68539Z","shell.execute_reply":"2021-07-08T04:36:11.01819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I tried with a little larger learning rate but it didn't seem to be working well so I moved back to the smaller one.","metadata":{}},{"cell_type":"code","source":"# Reloading the saved model and also saving the model in the output dir\nmodel = tf.keras.models.load_model('./efficientNet_Pretraining')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:42:37.602841Z","iopub.execute_input":"2021-07-08T04:42:37.603254Z","iopub.status.idle":"2021-07-08T04:42:56.898053Z","shell.execute_reply.started":"2021-07-08T04:42:37.603216Z","shell.execute_reply":"2021-07-08T04:42:56.897088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cross checking the model performance\nmodel.evaluate(valid_dataset)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:43:06.473018Z","iopub.execute_input":"2021-07-08T04:43:06.473336Z","iopub.status.idle":"2021-07-08T04:44:35.737355Z","shell.execute_reply.started":"2021-07-08T04:43:06.473306Z","shell.execute_reply":"2021-07-08T04:44:35.736563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training whole model \n\nmodel.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate = 0.0001),\n    loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n    metrics=tf.keras.metrics.SparseCategoricalAccuracy())","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:45:12.648652Z","iopub.execute_input":"2021-07-08T04:45:12.649Z","iopub.status.idle":"2021-07-08T04:45:12.684714Z","shell.execute_reply.started":"2021-07-08T04:45:12.648967Z","shell.execute_reply":"2021-07-08T04:45:12.683938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Whole model\n\nmodel.fit(x = train_dataset,\n         epochs = 3,\n         steps_per_epoch = STEPS,\n         validation_data = valid_dataset,\n         callbacks = [model_checkpoint, lr_schedular, early_stopping])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T04:45:33.781878Z","iopub.execute_input":"2021-07-08T04:45:33.782208Z","iopub.status.idle":"2021-07-08T05:59:49.435142Z","shell.execute_reply.started":"2021-07-08T04:45:33.782178Z","shell.execute_reply":"2021-07-08T05:59:49.433258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This much pre-training is enough so let's use this model as feature extractor for our current competition data","metadata":{}},{"cell_type":"markdown","source":"-----------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# Starting work on current Dataset","metadata":{}},{"cell_type":"markdown","source":"___________________________________________________","metadata":{}},{"cell_type":"markdown","source":"# Configuration Current","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:01:15.389415Z","iopub.execute_input":"2021-07-08T06:01:15.389772Z","iopub.status.idle":"2021-07-08T06:10:53.534829Z","shell.execute_reply.started":"2021-07-08T06:01:15.389739Z","shell.execute_reply":"2021-07-08T06:10:53.533545Z"}}},{"cell_type":"code","source":"class CurrentConfig:\n    DATA = '../input/siim-covid19-detection/train_study_level.csv'\n    IMAGE_FOLDER = '../input/siimfisabiorsna-covid19-image-size-1024/study'\n    \n    AUTOTUNE = tf.data.experimental.AUTOTUNE\n    \n    IMAGE_SIZE = 256 # I am using smaller image size for faster training\n    BATCH_SIZE = 8\n    NUM_CLASSES = 15\n    LR = 0.001","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:12.682921Z","iopub.execute_input":"2021-07-08T06:39:12.683279Z","iopub.status.idle":"2021-07-08T06:39:12.687934Z","shell.execute_reply.started":"2021-07-08T06:39:12.683249Z","shell.execute_reply":"2021-07-08T06:39:12.686806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"current_config = CurrentConfig()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:12.82151Z","iopub.execute_input":"2021-07-08T06:39:12.821823Z","iopub.status.idle":"2021-07-08T06:39:12.825727Z","shell.execute_reply.started":"2021-07-08T06:39:12.821796Z","shell.execute_reply":"2021-07-08T06:39:12.824515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the dataset","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(current_config.DATA)\n\nimage_files = glob.glob(current_config.IMAGE_FOLDER + '/*')\ntemp_df = pd.DataFrame()\ntemp_df['paths'] = image_files\ntemp_df['id'] = temp_df['paths'].map(lambda x: x.split('/')[-1].split('.')[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:13.990176Z","iopub.execute_input":"2021-07-08T06:39:13.990494Z","iopub.status.idle":"2021-07-08T06:39:14.038724Z","shell.execute_reply.started":"2021-07-08T06:39:13.990464Z","shell.execute_reply":"2021-07-08T06:39:14.037942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(df, temp_df, on='id')\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:16.703139Z","iopub.execute_input":"2021-07-08T06:39:16.703473Z","iopub.status.idle":"2021-07-08T06:39:16.727654Z","shell.execute_reply.started":"2021-07-08T06:39:16.703442Z","shell.execute_reply":"2021-07-08T06:39:16.72662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(df.isna())","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:21.803239Z","iopub.execute_input":"2021-07-08T06:39:21.803557Z","iopub.status.idle":"2021-07-08T06:39:21.815744Z","shell.execute_reply.started":"2021-07-08T06:39:21.803527Z","shell.execute_reply":"2021-07-08T06:39:21.814831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(df.paths):\n    if not os.path.exists(i):\n        print('image not found')\n        break","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:26.159378Z","iopub.execute_input":"2021-07-08T06:39:26.1597Z","iopub.status.idle":"2021-07-08T06:39:27.411501Z","shell.execute_reply.started":"2021-07-08T06:39:26.15967Z","shell.execute_reply":"2021-07-08T06:39:27.410487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spliting the dataset\nFor our purpose we will split the dataset in 3 parts. train, valid and test. I will not be doing k-fold cross validation as it takes too much time.","metadata":{}},{"cell_type":"code","source":"def mapper(row):\n    if row[0] == 1: return 1\n    if row[1] == 1: return 2\n    if row[2] == 1: return 3\n    if row[3] == 1: return 4\n\nlabels = []    \n\nfor row in df[df.columns[1:-1]].values:\n    labels.append(mapper(row))\n    \ndf['class'] = labels\ndf.columns = ['id', 'N', 'T', 'I', 'A', 'paths', 'class']\n\ndf.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:30.887442Z","iopub.execute_input":"2021-07-08T06:39:30.887785Z","iopub.status.idle":"2021-07-08T06:39:30.919344Z","shell.execute_reply.started":"2021-07-08T06:39:30.887752Z","shell.execute_reply":"2021-07-08T06:39:30.918597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_temp = train_test_split(df, test_size=0.4, random_state=1234, stratify=df['class'])\n\ndf_valid, df_test = train_test_split(df_temp, test_size=0.5, random_state=1234, stratify=df_temp['class'])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:31.788516Z","iopub.execute_input":"2021-07-08T06:39:31.78889Z","iopub.status.idle":"2021-07-08T06:39:31.806011Z","shell.execute_reply.started":"2021-07-08T06:39:31.788859Z","shell.execute_reply":"2021-07-08T06:39:31.805208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clearing GPU Memory","metadata":{}},{"cell_type":"code","source":"# Clearing GPU memory\nfrom numba import cuda\ncuda.get_current_device().reset()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:39:33.061756Z","iopub.execute_input":"2021-07-08T06:39:33.062093Z","iopub.status.idle":"2021-07-08T06:39:33.597297Z","shell.execute_reply.started":"2021-07-08T06:39:33.062057Z","shell.execute_reply":"2021-07-08T06:39:33.596412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the dataset\n\nFor some reason my session crashed so I have restarted from this point.","metadata":{}},{"cell_type":"code","source":"def aug_func(image_path, label):\n    file_bytes = tf.io.read_file(image_path)\n    img = tf.image.decode_png(file_bytes, channels=3)\n    img = tf.image.resize(img, [current_config.IMAGE_SIZE, current_config.IMAGE_SIZE])\n    img = img/255.\n    return img, label","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:41:49.311146Z","iopub.execute_input":"2021-07-08T06:41:49.311511Z","iopub.status.idle":"2021-07-08T06:41:49.317815Z","shell.execute_reply.started":"2021-07-08T06:41:49.311475Z","shell.execute_reply":"2021-07-08T06:41:49.316823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((df_train['paths'].values, (df_train['N'].values, \n                                                                               df_train['T'].values,\n                                                                               df_train['I'].values,\n                                                                               df_train['A'].values)))\n\ntrain_dataset = train_dataset.map(aug_func, num_parallel_calls=current_config.AUTOTUNE)\ntrain_dataset = train_dataset.repeat()\ntrain_dataset = train_dataset.batch(current_config.BATCH_SIZE)\ntrain_dataset = train_dataset.prefetch(current_config.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:42:00.243093Z","iopub.execute_input":"2021-07-08T06:42:00.243482Z","iopub.status.idle":"2021-07-08T06:42:00.26817Z","shell.execute_reply.started":"2021-07-08T06:42:00.243451Z","shell.execute_reply":"2021-07-08T06:42:00.267381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dataset = tf.data.Dataset.from_tensor_slices((df_valid['paths'].values, (df_valid['N'].values, \n                                                                               df_valid['T'].values,\n                                                                               df_valid['I'].values,\n                                                                               df_valid['A'].values)))\n\nvalid_dataset = valid_dataset.map(aug_func, num_parallel_calls=current_config.AUTOTUNE)\nvalid_dataset = valid_dataset.batch(current_config.BATCH_SIZE)\nvalid_dataset = valid_dataset.prefetch(current_config.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:45:04.236984Z","iopub.execute_input":"2021-07-08T06:45:04.237341Z","iopub.status.idle":"2021-07-08T06:45:04.259969Z","shell.execute_reply.started":"2021-07-08T06:45:04.237306Z","shell.execute_reply":"2021-07-08T06:45:04.259166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = tf.data.Dataset.from_tensor_slices((df_test['paths'].values, (df_test['N'].values, \n                                                                             df_test['T'].values,\n                                                                             df_test['I'].values,\n                                                                             df_test['A'].values)))\n\ntest_dataset = test_dataset.map(aug_func, num_parallel_calls=current_config.AUTOTUNE)\ntest_dataset = test_dataset.batch(current_config.BATCH_SIZE)\ntest_dataset = test_dataset.prefetch(current_config.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:45:29.170812Z","iopub.execute_input":"2021-07-08T06:45:29.171151Z","iopub.status.idle":"2021-07-08T06:45:29.193759Z","shell.execute_reply.started":"2021-07-08T06:45:29.171121Z","shell.execute_reply":"2021-07-08T06:45:29.193008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, j, k in zip(train_dataset, valid_dataset, test_dataset):\n    print(i[0].shape, i[1][0].shape, i[1][1].shape, i[1][2].shape, i[1][3].shape)\n    plt.figure(figsize=(20,10))\n    plt.subplot(1,3,1)\n    plt.imshow(i[0][0])\n    plt.subplot(1,3,2)\n    plt.imshow(j[0][0])\n    plt.subplot(1,3,3)\n    plt.imshow(k[0][0])\n    break","metadata":{"execution":{"iopub.status.busy":"2021-07-08T06:47:56.873066Z","iopub.execute_input":"2021-07-08T06:47:56.8734Z","iopub.status.idle":"2021-07-08T06:47:58.006455Z","shell.execute_reply.started":"2021-07-08T06:47:56.873368Z","shell.execute_reply":"2021-07-08T06:47:58.005653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How would a not-so pretrained model perform","metadata":{}},{"cell_type":"code","source":"base_model = model = tf.keras.Sequential([\n    efn.EfficientNetB2(\n        input_shape=(current_config.IMAGE_SIZE, current_config.IMAGE_SIZE, 3),\n        weights='imagenet',\n        include_top=False),\n    tf.keras.layers.GlobalAveragePooling2D()\n])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:24:14.66101Z","iopub.execute_input":"2021-07-08T07:24:14.661338Z","iopub.status.idle":"2021-07-08T07:24:17.499557Z","shell.execute_reply.started":"2021-07-08T07:24:14.661308Z","shell.execute_reply":"2021-07-08T07:24:17.498742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(base_model):\n    \n    inputs = tf.keras.layers.Input(shape=(current_config.IMAGE_SIZE, current_config.IMAGE_SIZE, 3))\n    \n    classifier_one   = tf.keras.layers.Dense(1, activation='softmax', name='out1')\n    classifier_two   = tf.keras.layers.Dense(1, activation='softmax', name='out2')\n    classifier_three = tf.keras.layers.Dense(1, activation='softmax', name='out3')    \n    classifier_four  = tf.keras.layers.Dense(1, activation='softmax', name='out4')    \n    \n    x = base_model(inputs)\n    \n    out1 = classifier_one(x)\n    out2 = classifier_two(x)\n    out3 = classifier_three(x)\n    out4 = classifier_four(x)\n    \n    return tf.keras.models.Model(inputs=[inputs], outputs=[out1, out2, out3, out4])","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:24:51.877621Z","iopub.execute_input":"2021-07-08T07:24:51.878158Z","iopub.status.idle":"2021-07-08T07:24:51.885253Z","shell.execute_reply.started":"2021-07-08T07:24:51.878121Z","shell.execute_reply":"2021-07-08T07:24:51.88423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model(base_model)\nmodel.layers[0].trainable = False\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:28:38.432014Z","iopub.execute_input":"2021-07-08T07:28:38.432436Z","iopub.status.idle":"2021-07-08T07:28:39.275688Z","shell.execute_reply.started":"2021-07-08T07:28:38.432398Z","shell.execute_reply":"2021-07-08T07:28:39.274598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:28:45.650164Z","iopub.execute_input":"2021-07-08T07:28:45.65049Z","iopub.status.idle":"2021-07-08T07:28:45.946477Z","shell.execute_reply.started":"2021-07-08T07:28:45.65046Z","shell.execute_reply":"2021-07-08T07:28:45.945577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=current_config.LR),\n              \n              loss=[tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.BinaryCrossentropy(),\n                   tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.BinaryCrossentropy()],\n             \n              metrics={'out1': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'), \n                      'out2': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'),\n                      'out3': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'),\n                      'out4': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR')})","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:28:47.201915Z","iopub.execute_input":"2021-07-08T07:28:47.202268Z","iopub.status.idle":"2021-07-08T07:28:47.251108Z","shell.execute_reply.started":"2021-07-08T07:28:47.202235Z","shell.execute_reply":"2021-07-08T07:28:47.250159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the Model\nI will not finetune the models and their hyperparameter as that will be very tidious and the notebook is already long enough. My main objective to see which pretrained weights models performs better.","metadata":{}},{"cell_type":"code","source":"model_checkpoint = tf.keras.callbacks.ModelCheckpoint('./efficientNet_before_pretraining', save_best_only=True)\nlr_schedular = tf.keras.callbacks.ReduceLROnPlateau(patience=1, min_delta=0.01)\nearly_stopping = tf.keras.callbacks.EarlyStopping(min_delta=0.001, patience=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:28:49.446325Z","iopub.execute_input":"2021-07-08T07:28:49.446657Z","iopub.status.idle":"2021-07-08T07:28:49.453665Z","shell.execute_reply.started":"2021-07-08T07:28:49.446626Z","shell.execute_reply":"2021-07-08T07:28:49.45288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training the last layers\n\nSTEPS = len(df_train)//current_config.BATCH_SIZE\n\n# Before using Pretrained weights\nBP_history = model.fit(x = train_dataset, \n                  validation_data = valid_dataset,\n                  steps_per_epoch = STEPS,\n                  epochs = 2,\n                  callbacks=[model_checkpoint, lr_schedular, early_stopping]\n                 )","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:28:50.772513Z","iopub.execute_input":"2021-07-08T07:28:50.772846Z","iopub.status.idle":"2021-07-08T07:33:35.572293Z","shell.execute_reply.started":"2021-07-08T07:28:50.772814Z","shell.execute_reply":"2021-07-08T07:33:35.571436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the model with the pretrained base_model","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.models.load_model('./efficientNet_Pretraining')\nbase_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:00.514804Z","iopub.execute_input":"2021-07-08T07:36:00.515136Z","iopub.status.idle":"2021-07-08T07:36:20.338196Z","shell.execute_reply.started":"2021-07-08T07:36:00.515105Z","shell.execute_reply":"2021-07-08T07:36:20.336359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing the top layer\nbase_model.pop()\nbase_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:20.339869Z","iopub.execute_input":"2021-07-08T07:36:20.340232Z","iopub.status.idle":"2021-07-08T07:36:20.382655Z","shell.execute_reply.started":"2021-07-08T07:36:20.340194Z","shell.execute_reply":"2021-07-08T07:36:20.381878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_model(base_model)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:20.384267Z","iopub.execute_input":"2021-07-08T07:36:20.384593Z","iopub.status.idle":"2021-07-08T07:36:21.175358Z","shell.execute_reply.started":"2021-07-08T07:36:20.384559Z","shell.execute_reply":"2021-07-08T07:36:21.17433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:21.177009Z","iopub.execute_input":"2021-07-08T07:36:21.177366Z","iopub.status.idle":"2021-07-08T07:36:21.492007Z","shell.execute_reply.started":"2021-07-08T07:36:21.177327Z","shell.execute_reply":"2021-07-08T07:36:21.491078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=current_config.LR),\n              \n              loss=[tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.BinaryCrossentropy(),\n                   tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.BinaryCrossentropy()],\n             \n              metrics={'out1': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'), \n                      'out2': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'),\n                      'out3': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR'),\n                      'out4': tf.keras.metrics.AUC(name='auc_precision_recall', curve='PR')})","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:21.493746Z","iopub.execute_input":"2021-07-08T07:36:21.494128Z","iopub.status.idle":"2021-07-08T07:36:21.562912Z","shell.execute_reply.started":"2021-07-08T07:36:21.49409Z","shell.execute_reply":"2021-07-08T07:36:21.56215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"model_checkpoint = tf.keras.callbacks.ModelCheckpoint('./efficientNet_after_pretraining', save_best_only=True)\nlr_schedular = tf.keras.callbacks.ReduceLROnPlateau(patience=1, min_delta=0.01)\nearly_stopping = tf.keras.callbacks.EarlyStopping(min_delta=0.001, patience=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:21.564255Z","iopub.execute_input":"2021-07-08T07:36:21.564772Z","iopub.status.idle":"2021-07-08T07:36:21.571823Z","shell.execute_reply.started":"2021-07-08T07:36:21.564735Z","shell.execute_reply":"2021-07-08T07:36:21.571026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training the last layers\n\nSTEPS = len(df_train)//current_config.BATCH_SIZE\n\nAP_history = model.fit(x = train_dataset, \n                  validation_data = valid_dataset,\n                  steps_per_epoch = STEPS,\n                  epochs = 2,\n                  callbacks=[model_checkpoint, lr_schedular, early_stopping]\n                 )","metadata":{"execution":{"iopub.status.busy":"2021-07-08T07:36:25.330494Z","iopub.execute_input":"2021-07-08T07:36:25.33086Z","iopub.status.idle":"2021-07-08T07:39:22.552624Z","shell.execute_reply.started":"2021-07-08T07:36:25.330827Z","shell.execute_reply":"2021-07-08T07:39:22.549872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion (Work in Progress)\nFor now as you can see I am getting this error (if anyone knows why this error is coming then pls do tell in the comments) but one more thing to note is that even in the first epochs there is considerable difference. Now you have to decide weather you want to pretrain your models on similar dataset or not. In my opinion the difference will only increase with bigger models with larger Image size.","metadata":{}},{"cell_type":"markdown","source":"# Kindly Upvote 😊","metadata":{}}]}