{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Description\n\nHi gyus, from the public kernels, @lafoss shared some great notebooks about training tiles images on fast.ai/pytorch. <br />\nIn most of the previous competitions, the Pytorch/fast.ai become more and more popular than tensorflow. <br />\nWe can noticed that there are very few public notebooks were implemented in tensorflow. <br />\nSo I implemented the same idea as @lafoss's in tensorflow framework and shared it in this kernel. <br />\n\nSince it seems impossible to change the batch_size during training in tensorflow. (at least from I known) <br />\nSo I splited the original model to two separated models and used customized training loop to get the same effect as @lafoss shared. <br />\nBut I believe there must be some better ways to implement this idea in tensorflow. <br />\nAnyway, if you are also a tensorflow user, feel free to use this kernel as your baseline kernel. <br />\nI believe with some fine tunes, this kernel can get 0.7+ LB quite easy. <br />\nGood luck!\n\nAnd thanks @lafoss again for the great ideas!\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/qubvel/classification_models.git","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport PIL\nimport time\nimport math\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\nfrom classification_models.tfkeras import Classifiers\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\n\nSEED = 2020\nwarnings.filterwarnings('ignore')\nprint('Tensorflow version : {}'.format(tf.__version__))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAIN_DIR = '../input/prostate-cancer-grade-assessment'\nTRAIN_IMG_DIR = '../input/panda-tiles/train'\nTRAIN_MASKS_DIR = '../input/panda-tiles/masks'\ntrain_csv = pd.read_csv(os.path.join(MAIN_DIR, 'train.csv'))\nnoise_csv = pd.read_csv('../input/noisy-csv/suspicious_test_cases.csv')\n\nfor image_id in noise_csv['image_id'].values:\n    train_csv = train_csv[train_csv['image_id'] != image_id]\n\nradboud_csv = train_csv[train_csv['data_provider'] == 'radboud']\nkarolinska_csv = train_csv[train_csv['data_provider'] != 'radboud']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True)\nsplits = list(splits.split(radboud_csv, radboud_csv.isup_grade))\nfold_splits = np.zeros(len(radboud_csv)).astype(np.int)\nfor i in range(5): \n    fold_splits[splits[i][1]]=i\nradboud_csv['fold'] = fold_splits\nradboud_csv.head(5)\n\nsplits = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True)\nsplits = list(splits.split(karolinska_csv, karolinska_csv.isup_grade))\nfold_splits = np.zeros(len(karolinska_csv)).astype(np.int)\nfor i in range(5): \n    fold_splits[splits[i][1]]=i\nkarolinska_csv['fold'] = fold_splits\nkarolinska_csv.head(5)\n\ntrain_csv = pd.concat([radboud_csv, karolinska_csv])\ntrain_csv.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FLAG=False\nTRAIN_FOLD = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_csv[train_csv['fold'] != TRAIN_FOLD]\nvalid_df = train_csv[train_csv['fold'] == TRAIN_FOLD]\n\nprint(train_df.shape)\nprint(valid_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_DIM = (128, 128)\nCLASSES_NUM = 6\nBATCH_SIZE = 32\nEPOCHS = 15\nN=12\n\nLEARNING_RATE = 1e-4\nFOLDED_NUM_TRAIN_IMAGES = train_df.shape[0]\nFOLDED_NUM_VALID_IMAGES = valid_df.shape[0]\nSTEPS_PER_EPOCH = FOLDED_NUM_TRAIN_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = FOLDED_NUM_VALID_IMAGES // BATCH_SIZE\nPRETRAIN_PATH1 = '../input/tiles-pretrain/stage1.h5'\nPRETRAIN_PATH2 = '../input/tiles-pretrain/stage2.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('*'*20)\nprint('Notebook info')\nprint('Training data : {}'.format(FOLDED_NUM_TRAIN_IMAGES))\nprint('Validing data : {}'.format(FOLDED_NUM_VALID_IMAGES))\nprint('Categorical classes : {}'.format(CLASSES_NUM))\nprint('Training image size : {}'.format(IMG_DIM))\nprint('Training epochs : {}'.format(EPOCHS))\nprint('*'*20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n    \n    def __init__(self,\n                 image_shape,\n                 batch_size, \n                 df,\n                 img_dir,\n                 mask_dir,\n                 augmentation,\n                 is_training=True\n                 ):\n        \n        self.image_shape = image_shape\n        self.batch_size = batch_size\n        self.df = df\n        self.img_dir = img_dir\n        self.mask_dir = mask_dir\n        self.aug = augmentation\n        self.is_training = is_training\n        self.indices = range(df.shape[0])\n        \n    def __len__(self):\n        return self.df.shape[0] // self.batch_size\n    \n    def on_epoch_start(self):\n        if self.is_training:\n            np.random.shuffle(self.indices)\n    \n    def __getitem__(self, index):\n        batch_indices = self.indices[index * self.batch_size : (index+1) * self.batch_size]\n        image_ids = self.df['image_id'].iloc[batch_indices].values\n        batch_images = [self.__getimages__(image_id) for image_id in image_ids]\n        batch_labels = [self.df[self.df['image_id'] == image_id]['isup_grade'].values[0] for image_id in image_ids]\n        \n        return np.stack(batch_images).reshape(-1,128, 128, 3), np.stack(batch_labels)\n        \n        \n    def __getimages__(self, image_id):\n        fnames = [image_id+'_'+str(i)+'.png' for i in range(N)]\n        images = []\n        for fn in fnames:\n            img = np.array(PIL.Image.open(os.path.join(self.img_dir, fn)).convert('RGB'))[:, :, ::-1]\n            if self.aug:\n                images.append(self.aug(image=img)['image'])\n            else:\n                images.append(img)     \n        return np.stack(images) / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_augumentation = albu.Compose([\n                            albu.OneOf([\n                                albu.RandomBrightness(limit=0.15),\n                                albu.RandomContrast(limit=0.3),\n                                albu.RandomGamma(),\n                            ], p=0.25),\n                            albu.HorizontalFlip(p=0.4),\n                            albu.VerticalFlip(p=0.4),\n                            albu.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=20, p=0.3)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = DataGenerator(image_shape=IMG_DIM,\n                                batch_size=BATCH_SIZE,\n                                df=train_df,\n                                img_dir=TRAIN_IMG_DIR,\n                                mask_dir=TRAIN_MASKS_DIR,\n                                augmentation=train_augumentation)\n\nvalid_generator = DataGenerator(image_shape=IMG_DIM,\n                                batch_size=BATCH_SIZE,\n                                df=valid_df,\n                                img_dir=TRAIN_IMG_DIR,\n                                mask_dir=TRAIN_MASKS_DIR,\n                                augmentation=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    resnet18, _ = Classifiers.get('resnet18')\n    stage1_model = resnet18(input_shape=(*IMG_DIM, 3), weights='imagenet', include_top=False)\n    \n    input_layer = tf.keras.layers.Input(shape=(4, 4, stage1_model.output_shape[-1]))\n    x = tf.keras.layers.GlobalAveragePooling2D()(input_layer)\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dense(units=512, kernel_initializer='he_normal', activation='relu')(x)\n    cls_head = tf.keras.layers.Dense(units=CLASSES_NUM, activation='softmax')(x)\n    \n    stage2_model = tf.keras.models.Model(inputs=[input_layer], outputs=[cls_head])\n    return stage1_model, stage2_model, stage1_model.output_shape[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stage1_model, stage2_model, stage1_channels = build_model()\noptimizer = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)\nloss_fn = lambda a,b: tf.nn.compute_average_loss(tf.keras.losses.sparse_categorical_crossentropy(a,b), global_batch_size=BATCH_SIZE)\ntrain_loss = tf.keras.metrics.Sum()\nvalid_loss = tf.keras.metrics.Sum()\ntrain_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\nvalid_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if PRETRAIN_PATH1:\n    print('load stage 1 model pretrain weights..')\n    stage1_model.load_weights(PRETRAIN_PATH1)\n    \nif PRETRAIN_PATH2:\n    print('load stage 2 model pretrain weights..')\n    stage2_model.load_weights(PRETRAIN_PATH2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef train_step(images, labels):\n    with tf.GradientTape() as stage1_tape, tf.GradientTape() as stage2_tape:\n        stage1_output = stage1_model(images, training=True)\n        stage1_output = tf.reshape(stage1_output, (-1, N, 4, 4, stage1_channels))\n        stage1_output = tf.transpose(stage1_output, (0, 2, 1, 3, 4))\n        stage1_output = tf.reshape(stage1_output, (-1, 4, 4*N, stage1_channels))\n        stage2_output = stage2_model(stage1_output, training=True)\n        loss = loss_fn(labels, stage2_output)\n        \n    stage1_grads = stage1_tape.gradient(loss, stage1_model.trainable_variables)\n    stage2_grads = stage2_tape.gradient(loss, stage2_model.trainable_variables)\n    \n    optimizer.apply_gradients(zip(stage1_grads, stage1_model.trainable_variables))\n    optimizer.apply_gradients(zip(stage2_grads, stage2_model.trainable_variables))\n    \n    train_loss.update_state(loss)\n    train_accuracy.update_state(labels, stage2_output)\n    \ndef valid_step(images, labels):\n    stage1_output = stage1_model(images, training=False)\n    stage1_output = tf.reshape(stage1_output, (-1, N, 4, 4, stage1_channels))\n    stage1_output = tf.transpose(stage1_output, (0, 2, 1, 3, 4))\n    stage1_output = tf.reshape(stage1_output, (-1, 4, 4*N, stage1_channels))\n    stage2_output = stage2_model(stage1_output, training=False)\n    \n    loss = loss_fn(labels, stage2_output)\n    valid_loss.update_state(loss)\n    valid_accuracy.update_state(labels, stage2_output)\n    \n    return stage2_output.numpy()\n\ndef inference_step(images):\n    stage1_output = stage1_model(images, training=False)\n    stage1_output = tf.reshape(stage1_output, (-1, N, 4, 4, stage1_channels))\n    stage1_output = tf.transpose(stage1_output, (0, 2, 1, 3, 4))\n    stage1_output = tf.reshape(stage1_output, (-1, 4, 4*N, stage1_channels))\n    stage2_output = stage2_model(stage1_output, training=False)\n    \n    return stage2_output.numpy()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_valid_qwk = 0\nhistory = {\n    'train_loss' : [],\n    'valid_loss' : [],\n    'train_accuracy' : [],\n    'valid_accuracy' : [],\n    'qwk' : []\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN_FLAG:\n\n    print(\"Steps per epoch:\", STEPS_PER_EPOCH, \"Valid steps per epoch:\", VALIDATION_STEPS)\n    epoch = 0\n    for epoch in range(EPOCHS):\n        start_time = time.time()\n        #model training\n        for step in range(train_generator.__len__()):\n            images, labels = train_generator.__getitem__(step)\n            train_step(images, labels)\n            if step % 10 == 0:\n                print('=', end='', flush=True)\n\n        print('|', end='', flush=True)\n        #model validation\n        predictions = []\n        groundtruths = []\n        for v_step in range(valid_generator.__len__()):\n            images, labels = valid_generator.__getitem__(v_step)\n            valid_preds = valid_step(images, labels)\n            valid_preds = np.argmax(valid_preds, axis=-1)\n            groundtruths += list(labels)\n            predictions += list(valid_preds)\n            if v_step % 10 == 0:\n                print('=', end='', flush=True)\n\n        qwk = cohen_kappa_score(groundtruths, predictions, labels=None, weights= 'quadratic', sample_weight=None)\n\n        history['train_loss'].append(train_loss.result().numpy() / STEPS_PER_EPOCH)\n        history['valid_loss'].append(valid_loss.result().numpy() / VALIDATION_STEPS)\n        history['train_accuracy'].append(train_accuracy.result().numpy())\n        history['valid_accuracy'].append(valid_accuracy.result().numpy())\n        history['qwk'].append(qwk)\n\n        print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n        print('loss: {:0.4f}'.format(history['train_loss'][-1]),'val_loss: {:0.4f}'.format(history['valid_loss'][-1]))\n        print('accuracy : {}'.format(history['train_accuracy'][-1]), 'val_accuracy : {}'.format(history['valid_accuracy'][-1]))\n        print('validation qwk : {}'.format(qwk))\n\n        # set up next epoch\n        valid_loss.reset_states()\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        valid_accuracy.reset_states()\n\n\n        if history['qwk'][-1] > best_valid_qwk:\n            print('Validation qwk improve from {} to {}, save model checkpoint'.format(best_valid_qwk, history['qwk'][-1]))\n            stage1_model.save('stage1.h5')\n            stage2_model.save('stage2.h5')\n            best_valid_qwk = history['qwk'][-1]\n\n        print('Spending time : {}...'.format(time.time()-start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_training_history(history):\n    \n    plt.figure(figsize=(12,12))\n    plt.plot(np.arange(0,len(history['train_loss']),1), history['train_loss'], label='train_loss', color='g')\n    plt.plot(np.arange(0,len(history['valid_loss']),1), history['valid_loss'], label='validation_loss', color='r')\n    plt.title('Training loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()\n    \nif TRAIN_FLAG:\n    plot_training_history(history)\n    stage1_model.save('stage1_finalcheckpoint.h5')\n    stage2_model.save('stage2_finalcheckpoint.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference():\n    \n    ground_truth = []\n    prediction = []\n    \n    for step in range(valid_generator.__len__()):\n        images, labels = valid_generator.__getitem__(step)\n        preds = inference_step(images)\n        if step % 10 == 0:\n            print('=', end='', flush=True)\n            \n        preds = np.argmax(preds, axis=-1)\n        \n        for y_true, y_pred in zip(labels, preds):\n            ground_truth.append(y_true)\n            prediction.append(y_pred)\n        \n    qwk = cohen_kappa_score(ground_truth, prediction, labels=None, weights= 'quadratic', sample_weight=None)\n    print('\\nValid QWK : {}'.format(qwk))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN_FLAG:\n    stage1_model.load_weights('stage1.h5')\n    stage2_model.load_weights('stage2.h5')\ninference()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}