{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**(Version 6)**\n- Training `DenseNet201`\n\n**(Version 7)**\n- Try for 60 epochs\n\n**(Version 11)**\n- Try `ResNext101`\n- Change Bone window to soft\n- Add RadomContrast (Wasn't a good idea, maybe resulted in adding noise to the windowing operation)\n- Two useful notebooks were: [Pytorch ResNext101](https://www.kaggle.com/braquino/pytorch-resnext-32x8d-centercrop), [Keras ResNext101](https://www.kaggle.com/afsan123/keras-resnext50-holdout-split)\n- ++ Other changes which can be explored in version comparing\n- Removing the additional layers\n\n**(Version 12)**\n- Bringing back the additional layers\n\n**(Version 14)**\n- Return to bone windowing (adding soft windowing seems to have some issues)\n\n**(Version 15)**\n- Use soft window\n- Use Densenet201 and make all layers trainable\n- Remove some layers from the augmentation, they may have resulted in noise\n- Modify the last layers of the model (Removed Dense(100) and Dropout)\n- n_samples is 15000\n\n**(Version 16)**\n- Increase the Dropout to 0.5 in the last layer\n- n_samples is 20000\n\n**(Version 17)**\n- Increase the Dropout to 0.5 in the last layer\n- n_samples is 20000\n\n**(Version 18)**\n- Increase the Dropout to 0.8 in the last layer\n- Make the whole model trainable, except the last layer (This worked better surprisingly!) (LR = 0.000125)\n\n\n**(Version 19)**\n- Try `InceptionResnetV2`\n\n**(Version 20)**\n- Freeze the first 15 layers of the model and make the rest trainable\n\n**(Version 23)**\n- The whole model is trainable\n- Return to bone windowing\n- Return to the very first data augmentation layers\n- Try `DenseNet201` again\n\n--------------------------------------------------------------------------------------------------------------\n**(Version 24)**\n- Try `DenseNet121` again\n\n--------------------------------------------------------------------------------------------------------------\n- Try `DenseNet169` again\n--------------------------------------------------------------------------------------------------------------\n- `MobileNetV2` sample size 107933\n\n- MobileNetV1","metadata":{}},{"cell_type":"code","source":"# !pip install image-classifiers\n# # !pip install iterative-stratification","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:02.653341Z","iopub.execute_input":"2021-11-14T13:52:02.654037Z","iopub.status.idle":"2021-11-14T13:52:02.675573Z","shell.execute_reply.started":"2021-11-14T13:52:02.653919Z","shell.execute_reply":"2021-11-14T13:52:02.674928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport pydicom\nimport math\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport os\nimport seaborn as sns\nfrom sklearn.metrics import multilabel_confusion_matrix\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #disable \nimport matplotlib\nmatplotlib.rc('xtick', labelsize=15) \nmatplotlib.rc('ytick', labelsize=15) \nsns.set_style(\"darkgrid\")\nsns.set_context(\"notebook\", font_scale=1.5, rc={\"lines.linewidth\": 4})","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-14T13:52:02.678754Z","iopub.execute_input":"2021-11-14T13:52:02.679866Z","iopub.status.idle":"2021-11-14T13:52:07.34686Z","shell.execute_reply.started":"2021-11-14T13:52:02.679829Z","shell.execute_reply":"2021-11-14T13:52:07.345908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/afsan123/keras-resnext50-holdout-split#kln-220","metadata":{}},{"cell_type":"code","source":"# from classification_models.tfkeras import Classifiers","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:07.349042Z","iopub.execute_input":"2021-11-14T13:52:07.349306Z","iopub.status.idle":"2021-11-14T13:52:07.354321Z","shell.execute_reply.started":"2021-11-14T13:52:07.349271Z","shell.execute_reply":"2021-11-14T13:52:07.353269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nSUBCLASSES = ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural']\nSAMPLE_SUBCLASS = 107933\nHU_MIN = 0\nHU_MAX = 100\nIMAGE_SIZE = (224,224)\nBATCH_SIZE = 32\nNUM_CLASSES = 5\nEPOCHS =10 \nMETRICS = [tf.keras.metrics.BinaryAccuracy(), \n           tf.keras.metrics.Precision(),\n           tf.keras.metrics.Recall(),\n           tf.keras.metrics.AUC(),\n           tf.keras.metrics.SpecificityAtSensitivity(0.5),\n           tf.keras.metrics.SensitivityAtSpecificity(0.5)\n          ]","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:07.355766Z","iopub.execute_input":"2021-11-14T13:52:07.35637Z","iopub.status.idle":"2021-11-14T13:52:10.635504Z","shell.execute_reply.started":"2021-11-14T13:52:07.356333Z","shell.execute_reply":"2021-11-14T13:52:10.634784Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_learning_curves(history, metrics_to_plot = ['loss','binary_accuracy', 'precision', 'recall', 'auc']):\n  ncols = 2\n  nrows = math.ceil(len(metrics_to_plot) / 2)\n  if len(metrics_to_plot) <= 2:\n        fig, axes = plt.subplots(nrows,ncols, figsize=(20,10))\n        for i in range(2):\n            axes[i].plot(history.history[metrics_to_plot[i]], label=metrics_to_plot[i] +' (training data)')\n            axes[i].plot(history.history['val_'+metrics_to_plot[i]], label=metrics_to_plot[i] + ' (val data)')\n            axes[i].set_ylabel('Value', fontsize = 20)\n            axes[i].set_xlabel('No. epoch', fontsize = 20)\n            axes[i].legend(prop={'size': 20})\n            axes[i].set_title(metrics_to_plot[i], size = 22)\n  else:        \n      fig, axes = plt.subplots(nrows,ncols, figsize=(15,20))\n\n      for i in range(ncols):\n        for j in range(nrows):\n          metric_idx = j * ncols + i\n          if metric_idx >= len(metrics_to_plot):\n                break\n          axes[j,i].plot(history.history[metrics_to_plot[metric_idx]], label=metrics_to_plot[metric_idx] +' (training data)')\n          axes[j,i].plot(history.history['val_'+metrics_to_plot[metric_idx]], label=metrics_to_plot[metric_idx] + ' (val data)')\n          axes[j,i].set_ylabel('Value', fontsize = 20)\n          axes[j,i].set_xlabel('No. epoch', fontsize = 20)\n          axes[j,i].legend(prop={'size': 20})\n          axes[j,i].set_title(metrics_to_plot[metric_idx], size = 22)\n  plt.tight_layout()\n     ","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:10.637606Z","iopub.execute_input":"2021-11-14T13:52:10.637882Z","iopub.status.idle":"2021-11-14T13:52:10.654113Z","shell.execute_reply.started":"2021-11-14T13:52:10.637846Z","shell.execute_reply":"2021-11-14T13:52:10.652343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'\ntrain_df = pd.read_csv('../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:10.656584Z","iopub.execute_input":"2021-11-14T13:52:10.656976Z","iopub.status.idle":"2021-11-14T13:52:14.404382Z","shell.execute_reply.started":"2021-11-14T13:52:10.656942Z","shell.execute_reply":"2021-11-14T13:52:14.403593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = train_df.Label\ntrain_df = train_df.ID.str.rsplit('_', n=1, expand = True)\ntrain_df['label'] = label\ntrain_df.rename({0:'id', 1: 'subtype'}, axis =1, inplace=True)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:14.405694Z","iopub.execute_input":"2021-11-14T13:52:14.405953Z","iopub.status.idle":"2021-11-14T13:52:24.192922Z","shell.execute_reply.started":"2021-11-14T13:52:14.405927Z","shell.execute_reply":"2021-11-14T13:52:24.192205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.pivot_table(train_df, index='id', columns='subtype', values = 'label')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:24.194193Z","iopub.execute_input":"2021-11-14T13:52:24.19459Z","iopub.status.idle":"2021-11-14T13:52:32.002292Z","shell.execute_reply.started":"2021-11-14T13:52:24.194554Z","shell.execute_reply":"2021-11-14T13:52:32.001602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.index = train_df.index.astype(str) + '.dcm'\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:32.003485Z","iopub.execute_input":"2021-11-14T13:52:32.003809Z","iopub.status.idle":"2021-11-14T13:52:32.734064Z","shell.execute_reply.started":"2021-11-14T13:52:32.003773Z","shell.execute_reply":"2021-11-14T13:52:32.733256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"abnormal_df = train_df[train_df['any'] == 1]\nabnormal_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:32.735306Z","iopub.execute_input":"2021-11-14T13:52:32.735627Z","iopub.status.idle":"2021-11-14T13:52:32.760501Z","shell.execute_reply.started":"2021-11-14T13:52:32.735591Z","shell.execute_reply":"2021-11-14T13:52:32.759752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"abnormal_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:32.763648Z","iopub.execute_input":"2021-11-14T13:52:32.763865Z","iopub.status.idle":"2021-11-14T13:52:32.768953Z","shell.execute_reply.started":"2021-11-14T13:52:32.763834Z","shell.execute_reply":"2021-11-14T13:52:32.768109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sums = abnormal_df.drop(columns=['any']).sum()\nsns.barplot(data=sums, x = sums.index, y = sums.values)\nplt.xticks(ticks = range(5), labels=sums.sort_values().index, rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:32.770883Z","iopub.execute_input":"2021-11-14T13:52:32.771167Z","iopub.status.idle":"2021-11-14T13:52:33.152381Z","shell.execute_reply.started":"2021-11-14T13:52:32.771133Z","shell.execute_reply":"2021-11-14T13:52:33.151582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = abnormal_df.sample(SAMPLE_SUBCLASS, replace = False, random_state = SEED)\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.153761Z","iopub.execute_input":"2021-11-14T13:52:33.154005Z","iopub.status.idle":"2021-11-14T13:52:33.173639Z","shell.execute_reply.started":"2021-11-14T13:52:33.153971Z","shell.execute_reply":"2021-11-14T13:52:33.172994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.drop(columns = ['any'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.174811Z","iopub.execute_input":"2021-11-14T13:52:33.175214Z","iopub.status.idle":"2021-11-14T13:52:33.180019Z","shell.execute_reply.started":"2021-11-14T13:52:33.175164Z","shell.execute_reply":"2021-11-14T13:52:33.179191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sums = sample.sum()\nsns.barplot(data=sums, x = sums.index, y = sums.values)\nplt.xticks(ticks = range(5), labels=sums.sort_values().index, rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.181424Z","iopub.execute_input":"2021-11-14T13:52:33.181727Z","iopub.status.idle":"2021-11-14T13:52:33.391441Z","shell.execute_reply.started":"2021-11-14T13:52:33.181692Z","shell.execute_reply":"2021-11-14T13:52:33.390793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(sample.index, sample, test_size = 0.3, random_state = SEED)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.39247Z","iopub.execute_input":"2021-11-14T13:52:33.393013Z","iopub.status.idle":"2021-11-14T13:52:33.4036Z","shell.execute_reply.started":"2021-11-14T13:52:33.392975Z","shell.execute_reply":"2021-11-14T13:52:33.402887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(y_train.index, y_train, test_size = 0.3, random_state = SEED)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.404889Z","iopub.execute_input":"2021-11-14T13:52:33.405306Z","iopub.status.idle":"2021-11-14T13:52:33.415292Z","shell.execute_reply.started":"2021-11-14T13:52:33.40527Z","shell.execute_reply":"2021-11-14T13:52:33.414454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = y_train\nval_df = y_val\ntest_df = y_test","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.416516Z","iopub.execute_input":"2021-11-14T13:52:33.417369Z","iopub.status.idle":"2021-11-14T13:52:33.421531Z","shell.execute_reply.started":"2021-11-14T13:52:33.417334Z","shell.execute_reply":"2021-11-14T13:52:33.420893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weights = (len(sample) / (len(SUBCLASSES) * sample.sum())).values\nclass_weights","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.422913Z","iopub.execute_input":"2021-11-14T13:52:33.423473Z","iopub.status.idle":"2021-11-14T13:52:33.434013Z","shell.execute_reply.started":"2021-11-14T13:52:33.423309Z","shell.execute_reply":"2021-11-14T13:52:33.43308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Custom loss\n#### Multilabel Loss","metadata":{}},{"cell_type":"code","source":"def np_multilabel_loss(class_weights=None):\n    def single_class_crossentropy(y_true, y_pred):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.cast(y_pred, tf.float32)\n        \n        y_pred = tf.where(y_pred > 1-(1e-07), 1-1e-07, y_pred)\n        y_pred = tf.where(y_pred < 1e-07, 1e-07, y_pred)\n        single_class_cross_entropies = - tf.reduce_mean(y_true * tf.math.log(y_pred) + (1-y_true) * tf.math.log(1-y_pred), axis=0)\n\n        if class_weights is None:\n            loss = tf.reduce_mean(single_class_cross_entropies)\n        else:\n            loss = tf.reduce_sum(class_weights*single_class_cross_entropies)\n        return loss\n    return single_class_crossentropy","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.435513Z","iopub.execute_input":"2021-11-14T13:52:33.436185Z","iopub.status.idle":"2021-11-14T13:52:33.44506Z","shell.execute_reply.started":"2021-11-14T13:52:33.436159Z","shell.execute_reply":"2021-11-14T13:52:33.444195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"METRICS = METRICS + [np_multilabel_loss()]\nMETRICS_NAMES = []\nfor metric in METRICS:\n    if hasattr(metric, 'name'):\n        METRICS_NAMES.append(metric.name)\n    else:\n        METRICS_NAMES.append(metric.__name__)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.446518Z","iopub.execute_input":"2021-11-14T13:52:33.446918Z","iopub.status.idle":"2021-11-14T13:52:33.454491Z","shell.execute_reply.started":"2021-11-14T13:52:33.44688Z","shell.execute_reply":"2021-11-14T13:52:33.453565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_dcm(dcm):\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n    \ndef get_first_of_dicom_field_as_int(x):\n    if type(x) == pydicom.multival.MultiValue:\n        return int(x[0])\n    return int(x)\n    \ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, # window center\n                    data[('0028','1051')].value, # window width\n                    data[('0028','1052')].value, # intercept\n                    data[('0028','1053')].value, # slope\n                   ]\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n    \n\ndef get_min_max_of_window_value(window_center, window_width):\n    mini = window_center - (window_width // 2)\n    maxi = window_center + (window_width // 2) \n    return mini, maxi\n\ndef window_image(img, window_center, window_width):\n    try:\n        _,_, intercept, slope = get_windowing(img)\n        img = img.pixel_array * slope + intercept\n        img_min, img_max = get_min_max_of_window_value(window_center, window_width)\n        img[img < img_min] = img_min\n        img[img > img_max] = img_max\n#         img = (img - np.min(img)) / (np.max(img) - np.min(img)) #normalize\n    except:\n        img = img_min * np.ones(IMAGE_SIZE)\n        \n    return img\n\ndef normalize(channel, wc_ww: tuple, norm_type = 'none'):\n    if norm_type.lower() == 'none':\n        return channel\n    if norm_type.lower() == 'min_max':\n        mini, maxi = get_min_max_of_window_value(wc_ww[0], wc_ww[1])\n        resulted_channel = (channel - mini) / (maxi - mini)\n        return resulted_channel\n    \n\ndef bsb_window(img):\n    bsb_config = {'brain': (40,80),\n             'subdural': (80,200),\n             'soft': (50, 350)}\n    brain_img = window_image(img, *bsb_config['brain'])\n    subdural_img = window_image(img,*bsb_config['subdural'])\n    soft_img = window_image(img, *bsb_config['soft'])\n    \n    brain_img = normalize(brain_img, bsb_config['brain'], 'min_max')\n    subdural_img = normalize(subdural_img, bsb_config['subdural'], 'min_max')\n    soft_img = normalize(soft_img, bsb_config['soft'], 'min_max')\n    \n#         print(np.min(soft_img))\n#     brain_img = (brain_img - 0) / 80\n#     subdural_img = (subdural_img - (-20)) / 200\n#     soft_img = (soft_img - (-150)) / 380 # (-150 = 40 - 380 / 2)\n#         print(np.min(soft_img))\n    bsb_img = np.zeros((brain_img.shape[0], brain_img.shape[1],3))\n    bsb_img[:, :, 0] = brain_img\n    bsb_img[:, :, 1] = subdural_img\n    bsb_img[:, :, 2] = soft_img\n    \n    if (np.any(np.isnan(bsb_img))):\n        bsb_img = np.ones((*IMAGE_SIZE,3))\n        \n    return bsb_img\n\n    \nclass ImageGenerator(tf.keras.utils.Sequence):\n    def __init__(self, dataframe,batch_size,shuffle,num_classes = NUM_CLASSES):\n        self.dataframe = dataframe\n        self.num_classes = num_classes\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        \n    def __len__(self):\n        return math.ceil(len(self.dataframe) / self.batch_size)\n    \n    def __getitem__(self, index):\n        batch_df = self.dataframe.iloc[index * self.batch_size: (index+1) * self.batch_size]\n        paths = TRAIN_PATH + batch_df.index.astype(str)\n        X = np.empty((len(batch_df), *IMAGE_SIZE, 3))\n        y = np.empty((len(batch_df), self.num_classes))\n        for i, path in enumerate(paths):\n            dcm = pydicom.dcmread(path)\n            # correct dcm\n            if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n                correct_dcm(dcm)\n#             rescaled_img = rescale_pixelarray(dcm)\n#             windowed_img = set_manual_window(rescaled_img, HU_MIN, HU_MAX)\n#             img = tf.convert_to_tensor(windowed_img, dtype=tf.float32)\n            img = bsb_window(dcm)\n            img = tf.convert_to_tensor(img, dtype=tf.float64)\n#             assert tf.reduce_min(img) >= -1 and tf.reduce_max(img) <= 1, 'Check these values img in (-1,1)'\n            X[i] = tf.image.resize(img, IMAGE_SIZE)\n            y[i] = batch_df.iloc[i].values\n#             assert tf.reduce_min(y[i]) >= 0 and tf.reduce_max(y[i]) <= 1, 'Check target values in (0,1)'\n            \n        return X, y\n    def on_epoch_end(self):\n        if self.shuffle:\n            self.dataframe = self.dataframe.sample(len(self.dataframe), replace = False, random_state = SEED)\n        self.current_epoch += 1","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.456632Z","iopub.execute_input":"2021-11-14T13:52:33.456969Z","iopub.status.idle":"2021-11-14T13:52:33.485389Z","shell.execute_reply.started":"2021-11-14T13:52:33.456927Z","shell.execute_reply":"2021-11-14T13:52:33.484553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_generator_train = ImageGenerator(train_df, BATCH_SIZE, shuffle=True)\nimg_generator_val = ImageGenerator(val_df, BATCH_SIZE, shuffle = True)\nimg_generator_test = ImageGenerator(test_df, BATCH_SIZE, shuffle = False)\ntrain_data = tf.data.Dataset.from_generator(lambda: map(tuple, img_generator_train), \n                                            output_types=(tf.float64, tf.uint8),\n                                            output_shapes = (\n                                                    tf.TensorShape((None, *IMAGE_SIZE,3)),\n                                                    tf.TensorShape((None, NUM_CLASSES))\n                                            ))\nval_data = tf.data.Dataset.from_generator(lambda: map(tuple, img_generator_val), \n                                          output_types=(tf.float64, tf.uint8),\n                                          output_shapes = (\n                                                    tf.TensorShape((None, *IMAGE_SIZE,3)),\n                                                    tf.TensorShape((None, NUM_CLASSES))\n                                            ))\ntest_data = tf.data.Dataset.from_generator(lambda: map(tuple, img_generator_test), \n                                          output_types=(tf.float64, tf.uint8),\n                                          output_shapes = (\n                                                    tf.TensorShape((None, *IMAGE_SIZE,3)),\n                                                    tf.TensorShape((None, NUM_CLASSES))\n                                            ))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.487832Z","iopub.execute_input":"2021-11-14T13:52:33.488459Z","iopub.status.idle":"2021-11-14T13:52:33.568067Z","shell.execute_reply.started":"2021-11-14T13:52:33.488266Z","shell.execute_reply":"2021-11-14T13:52:33.567384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n   tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n   tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n   tf.keras.layers.experimental.preprocessing.RandomZoom(0.2),\n   tf.keras.layers.experimental.preprocessing.RandomHeight(0.2),\n   tf.keras.layers.experimental.preprocessing.RandomWidth(0.2),  \n])\n\n# ResNext101, preprocess_input = Classifiers.get('resnext101')\n# base_model = ResNext101(IMAGE_SIZE + (3,), weights = 'imagenet', include_top = False)\nbase_model = tf.keras.applications.MobileNet(include_top = False)\nfor layer in base_model.layers:\n    layer.trainable = True\n    \n\ninputs = tf.keras.layers.Input(shape = IMAGE_SIZE + (3,), name = \"input_layer\")\n# x = preprocess_input(inputs)\nx = data_augmentation(inputs)\n\nx = base_model(x)\n\nx = tf.keras.layers.GlobalAveragePooling2D()(x)\nx = tf.keras.layers.Dropout(0.8)(x)\n# x = tf.keras.layers.Dense(100)(x)\n# x = tf.keras.layers.Dropout(0.5)(x)\n\noutputs = tf.keras.layers.Dense(NUM_CLASSES, activation='sigmoid')(x)\nmodel = tf.keras.Model(inputs, outputs)\n\nmodel.compile(loss=np_multilabel_loss(class_weights),\n             optimizer = tf.keras.optimizers.Adam(learning_rate=0.000125),\n             metrics = METRICS)\nprint(model.summary())\nhistory = model.fit(train_data, \n                   epochs = EPOCHS,\n                   validation_data = val_data,\n                   callbacks = [tf.keras.callbacks.ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True, mode='min', save_freq='epoch'),\n                               tf.keras.callbacks.EarlyStopping(restore_best_weights=True, patience=5),\n                               tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',\n                                           factor=0.5,\n                                           patience=2,\n                                           min_lr=1e-8,\n                                           mode=\"min\")\n                               ]\n)\n\neval_res = model.evaluate(val_data)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:52:33.569404Z","iopub.execute_input":"2021-11-14T13:52:33.569674Z","iopub.status.idle":"2021-11-14T14:01:00.644663Z","shell.execute_reply.started":"2021-11-14T13:52:33.569639Z","shell.execute_reply":"2021-11-14T14:01:00.642571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.645993Z","iopub.status.idle":"2021-11-14T14:01:00.646654Z","shell.execute_reply.started":"2021-11-14T14:01:00.646394Z","shell.execute_reply":"2021-11-14T14:01:00.64642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.647962Z","iopub.status.idle":"2021-11-14T14:01:00.648596Z","shell.execute_reply.started":"2021-11-14T14:01:00.648343Z","shell.execute_reply":"2021-11-14T14:01:00.648367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_prob = model.predict(test_data)\npred_prob","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.649803Z","iopub.status.idle":"2021-11-14T14:01:00.650439Z","shell.execute_reply.started":"2021-11-14T14:01:00.650199Z","shell.execute_reply":"2021-11-14T14:01:00.650224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_prob = pd.DataFrame(pred_prob, columns = SUBCLASSES, index = test_df.index)\npred_prob","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.65166Z","iopub.status.idle":"2021-11-14T14:01:00.652293Z","shell.execute_reply.started":"2021-11-14T14:01:00.652053Z","shell.execute_reply":"2021-11-14T14:01:00.652077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_values = pd.DataFrame((pred_prob > 0.5).astype(int), columns = SUBCLASSES, index = test_df.index)\npred_values","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.653797Z","iopub.status.idle":"2021-11-14T14:01:00.654428Z","shell.execute_reply.started":"2021-11-14T14:01:00.654189Z","shell.execute_reply":"2021-11-14T14:01:00.654213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_values.sum(axis = 1).value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.655642Z","iopub.status.idle":"2021-11-14T14:01:00.656291Z","shell.execute_reply.started":"2021-11-14T14:01:00.656054Z","shell.execute_reply":"2021-11-14T14:01:00.656079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.657505Z","iopub.status.idle":"2021-11-14T14:01:00.65813Z","shell.execute_reply.started":"2021-11-14T14:01:00.657892Z","shell.execute_reply":"2021-11-14T14:01:00.657916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_values","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.659336Z","iopub.status.idle":"2021-11-14T14:01:00.659989Z","shell.execute_reply.started":"2021-11-14T14:01:00.659751Z","shell.execute_reply":"2021-11-14T14:01:00.659774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_values.sum(axis = 1) == 2","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.661187Z","iopub.status.idle":"2021-11-14T14:01:00.661856Z","shell.execute_reply.started":"2021-11-14T14:01:00.661596Z","shell.execute_reply":"2021-11-14T14:01:00.661621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_class_names(row):\n    trues = row == 1\n    return np.array(trues[trues].index)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.663078Z","iopub.status.idle":"2021-11-14T14:01:00.663698Z","shell.execute_reply.started":"2021-11-14T14:01:00.663458Z","shell.execute_reply":"2021-11-14T14:01:00.663482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pred_values.apply(get_class_names, axis = 1)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.664915Z","iopub.status.idle":"2021-11-14T14:01:00.665571Z","shell.execute_reply.started":"2021-11-14T14:01:00.665321Z","shell.execute_reply":"2021-11-14T14:01:00.665344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = test_df.apply(get_class_names, axis = 1)\ny_true","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.666797Z","iopub.status.idle":"2021-11-14T14:01:00.667415Z","shell.execute_reply.started":"2021-11-14T14:01:00.667169Z","shell.execute_reply":"2021-11-14T14:01:00.667193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_values.to_csv('pred_values.csv')\npred_prob.to_csv('pred_prob.csv')\ntest_df.to_csv('test_df.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.668621Z","iopub.status.idle":"2021-11-14T14:01:00.669264Z","shell.execute_reply.started":"2021-11-14T14:01:00.669022Z","shell.execute_reply":"2021-11-14T14:01:00.669046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history[\"lr\"], 'o-')\nplt.title('Learning Rate')\nplt.xlabel('Epochs')\nplt.ylabel('LR')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.670529Z","iopub.status.idle":"2021-11-14T14:01:00.671192Z","shell.execute_reply.started":"2021-11-14T14:01:00.670956Z","shell.execute_reply":"2021-11-14T14:01:00.670981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, class_name,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n\n    # Plot confusion matrix in a beautiful manner\n    fig = plt.figure(figsize=(12, 10))\n    ax= plt.subplot()\n    sns.heatmap(cm, annot=True, ax = ax, fmt = 'g', cmap = cmap); #annot=True to annotate cells\n    # labels, title and ticks\n    ax.set_xlabel('Predicted', fontsize=20)\n    ax.xaxis.set_label_position('bottom')\n    plt.xticks(rotation=0)\n    ax.xaxis.tick_bottom()\n\n    ax.set_ylabel('True', fontsize=20)\n    plt.yticks(rotation=0)\n\n    plt.title(class_name, fontsize=20)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.672402Z","iopub.status.idle":"2021-11-14T14:01:00.67316Z","shell.execute_reply.started":"2021-11-14T14:01:00.672883Z","shell.execute_reply":"2021-11-14T14:01:00.67291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for subclass_name, matrix in zip(SUBCLASSES, multilabel_confusion_matrix(test_df, pred_values)):\n#     print(matrix)\n    plot_confusion_matrix(matrix, subclass_name)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.674479Z","iopub.status.idle":"2021-11-14T14:01:00.675121Z","shell.execute_reply.started":"2021-11-14T14:01:00.674884Z","shell.execute_reply":"2021-11-14T14:01:00.674908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(test_df, pred_values, target_names = SUBCLASSES))","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.6764Z","iopub.status.idle":"2021-11-14T14:01:00.677037Z","shell.execute_reply.started":"2021-11-14T14:01:00.676798Z","shell.execute_reply":"2021-11-14T14:01:00.676822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_curves(history, metrics_to_plot=['loss'] + METRICS_NAMES)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T14:01:00.678233Z","iopub.status.idle":"2021-11-14T14:01:00.678891Z","shell.execute_reply.started":"2021-11-14T14:01:00.678631Z","shell.execute_reply":"2021-11-14T14:01:00.678655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}