{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nEffn-B4\n1536 image size\n8 epochs\n3 Folds\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow-addons","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport math\n\nfrom matplotlib import pyplot as plt\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedKFold\n\nimport efficientnet.tfkeras as efn\n\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nGCS_PATH = KaggleDatasets().get_gcs_path('panda-dataset-medium-36-256-256')\n\n# Configuration\nEPOCHS = 12\nBATCH_SIZE = 4 * strategy.num_replicas_in_sync\nimg_size = 1536","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train['isup_grade'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images_path = '../input/panda-dataset-medium-36-256-256/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = plt.imread(train_images_path + train.loc[1]['image_id'] + '.png')\nplt.imshow(img)\nprint(img.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"image_path\"] = train[\"image_id\"].apply(lambda x: GCS_PATH + '/' + x + '.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splits = 3\nskf = StratifiedKFold(n_splits = 3)\n\nresult = []   \nfor train_idx, val_idx in skf.split(train['image_path'], train[\"isup_grade\"]):\n    train_fold = train.iloc[train_idx]\n    val_fold = train.iloc[val_idx]\n    result.append((train_fold, val_fold))\n    \ntrain_fold_1, val_fold_1 = result[0][0],result[0][1]\ntrain_fold_2, val_fold_2 = result[1][0],result[1][1]\ntrain_fold_3, val_fold_3 = result[2][0],result[2][1]\n# train_fold_4, val_fold_4 = result[3][0],result[3][1]\n# train_fold_5, val_fold_5 = result[4][0],result[4][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold_1 = pd.read_csv('../input/pandakaranfoldsfinetuning/train_fold_1.csv')\nval_fold_1   = pd.read_csv('../input/pandakaranfoldsfinetuning/val_fold_1.csv')\n\ntrain_fold_2 = pd.read_csv('../input/pandakaranfoldsfinetuning/train_fold_2.csv')\nval_fold_2   = pd.read_csv('../input/pandakaranfoldsfinetuning/val_fold_2.csv')\n\ntrain_fold_3 = pd.read_csv('../input/pandakaranfoldsfinetuning/train_fold_3.csv') \nval_fold_3   = pd.read_csv('../input/pandakaranfoldsfinetuning/val_fold_3.csv')\n# train_fold_4.to_csv('train_fold_4.csv',index=False)\n# val_fold_4.to_csv('val_fold_4.csv',index=False)\n# train_fold_5.to_csv('train_fold_5.csv',index=False)\n# val_fold_5.to_csv('val_fold_5.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xtrain, xval, ytrain, yval = train_test_split(train[\"image_path\"], train[\"isup_grade\"], test_size = 0.15, stratify = train[\"isup_grade\"])\n\n# df_train = pd.DataFrame({\"image_path\":xtrain, \"isup_grade\":ytrain})\n# df_val = pd.DataFrame({\"image_path\":xval, \"isup_grade\":yval})\n\n# df_train[\"isup_grade\"] = df_train[\"isup_grade\"].astype('int')\n# df_val[\"isup_grade\"] = df_val[\"isup_grade\"].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_labels = pd.get_dummies(df_train['isup_grade']).astype('int32').values\n# valid_labels = pd.get_dummies(df_val['isup_grade']).astype('int32').values\n\n# print(train_labels.shape) \n# print(valid_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None, image_size=(img_size, img_size)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, size = image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef data_augment(image, label=None):\n    image = tf.image.rot90(image)\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\ndef int_div_round_up(a, b):\n    return (a + b - 1) // b\n\n# NUM_TRAINING_IMAGES = df_train.shape[0]\n# NUM_VALIDATION_IMAGES = df_val.shape[0]\n# STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n# VALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\n# print('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((df_train['image_path'], train_labels))\n#     .map(decode_image, num_parallel_calls=AUTO)\n#     .map(data_augment, num_parallel_calls=AUTO)\n#     .repeat()\n#     .shuffle(512)\n#     .batch(BATCH_SIZE)\n#     .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((df_val['image_path'], valid_labels))\n#     .map(decode_image, num_parallel_calls=AUTO)\n#     .batch(BATCH_SIZE)\n#     .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.0001\nLR_MAX = 0.0001 * 1\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 0\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .7\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # checkpoint\n# filepath=\"effnB4-{epoch:02d}-{val_accuracy:.2f}.hdf5\"\n# checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Definition","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            efn.EfficientNetB4(\n                input_shape=(img_size, img_size, 3),\n                weights='imagenet',\n                include_top=False\n            ),\n            L.GlobalAveragePooling2D(),\n            L.Dense(1024, activation = 'relu',kernel_regularizer = tf.keras.regularizers.l2()), \n            L.Dropout(0.3),\n            L.Dense(512, activation= 'relu'),\n            L.Dense(6, activation='softmax')\n        ])\n\n    model.compile(\n        optimizer='adam',\n        loss = 'categorical_crossentropy',\n        metrics=['accuracy']\n    )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FOLD 1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths_fold_1 = train_fold_1.image_path.values\nval_paths_fold_1 = val_fold_1.image_path.values\n\ntrain_labels_fold_1 = pd.get_dummies(train_fold_1['isup_grade']).astype('int32').values\nval_labels_fold_1 = pd.get_dummies(val_fold_1['isup_grade']).astype('int32').values\n\ntrain_dataset_fold_1 = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths_fold_1, train_labels_fold_1))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nvalid_dataset_fold_1 = (\n    tf.data.Dataset\n    .from_tensor_slices((val_paths_fold_1, val_labels_fold_1))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nNUM_TRAINING_IMAGES = train_fold_1.shape[0]\nNUM_VALIDATION_IMAGES = val_fold_1.shape[0]\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# checkpoint\nfilepath=\"eb4-finetune-fold1-{epoch:02d}-{val_accuracy:.2f}.hdf5\"\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=False, mode='max')\ncallbacks_list = [checkpoint]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fold_1 = get_model()\nhistory1 = model_fold_1.fit(\n    train_dataset_fold_1, \n    epochs = EPOCHS, \n    callbacks = [lr_callback, checkpoint],\n    steps_per_epoch = STEPS_PER_EPOCH,\n    validation_data = valid_dataset_fold_1,\n    validation_steps = VALIDATION_STEPS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history1.history['accuracy']\nval_acc = history1.history['val_accuracy']\n\nloss = history1.history['loss']\nval_loss = history1.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model_fold_1.save('effnB4_tpu_png_1536_fold_1.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fold 2","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths_fold_2 = train_fold_2.image_path.values\nval_paths_fold_2 = val_fold_2.image_path.values\n\ntrain_labels_fold_2 = pd.get_dummies(train_fold_2['isup_grade']).astype('int32').values\nval_labels_fold_2 = pd.get_dummies(val_fold_2['isup_grade']).astype('int32').values\n\ntrain_dataset_fold_2 = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths_fold_2, train_labels_fold_2))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nvalid_dataset_fold_2 = (\n    tf.data.Dataset\n    .from_tensor_slices((val_paths_fold_2, val_labels_fold_2))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nNUM_TRAINING_IMAGES = train_fold_2.shape[0]\nNUM_VALIDATION_IMAGES = val_fold_2.shape[0]\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model_fold_2 = get_model()\nhistory2 = model_fold_2.fit(\n    train_dataset_fold_2, \n    epochs = EPOCHS, \n    callbacks = [lr_callback],\n    steps_per_epoch = STEPS_PER_EPOCH,\n    validation_data = valid_dataset_fold_2,\n    validation_steps = VALIDATION_STEPS\n)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history2.history['accuracy']\nval_acc = history2.history['val_accuracy']\n\nloss = history2.history['loss']\nval_loss = history2.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fold_2.save('effnB4_tpu_png_1536_fold_2.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fold 3","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths_fold_3 = train_fold_3.image_path.values\nval_paths_fold_3 = val_fold_3.image_path.values\n\ntrain_labels_fold_3 = pd.get_dummies(train_fold_3['isup_grade']).astype('int32').values\nval_labels_fold_3 = pd.get_dummies(val_fold_3['isup_grade']).astype('int32').values\n\ntrain_dataset_fold_3 = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths_fold_3, train_labels_fold_3))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nvalid_dataset_fold_3 = (\n    tf.data.Dataset\n    .from_tensor_slices((val_paths_fold_3, val_labels_fold_3))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nNUM_TRAINING_IMAGES = train_fold_3.shape[0]\nNUM_VALIDATION_IMAGES = val_fold_3.shape[0]\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model_fold_3 = get_model()\nhistory3 = model_fold_3.fit(\n    train_dataset_fold_3, \n    epochs = EPOCHS,\n    callbacks = [lr_callback],\n    steps_per_epoch = STEPS_PER_EPOCH,\n    validation_data = valid_dataset_fold_3,\n    validation_steps = VALIDATION_STEPS\n)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history3.history['accuracy']\nval_acc = history3.history['val_accuracy']\n\nloss = history3.history['loss']\nval_loss = history3.history['val_loss']\n\nepochs = range(len(acc))\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fold_3.save('effnB4_tpu_png_1536_fold_3.h5')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}