{"cells":[{"metadata":{},"cell_type":"markdown","source":"## RSNA_using_TPU_train\nRSNA STR Pulmonary Embolism Detection Using TPU\n\nReferences:\n[CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n[Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n[Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n\n[Flower Classification focal loss [+0.98]](https://www.kaggle.com/afshiin/flower-classification-focal-loss-0-98)\n\n[tensorflow](https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/python/keras/applications/efficientnet.py)\n\nUsing datasets:\n\n[STARTER DATASET: Train JPEGs (256x256)](https://www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/discussion/182930)"},{"metadata":{},"cell_type":"markdown","source":"### Initialize Environment"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport time\nfrom IPython.display import clear_output\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.callbacks import ModelCheckpoint as MC\nfrom tensorflow.keras import backend as K\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf,re,math\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Conv2D\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold,StratifiedKFold\nimport tensorflow_addons as tfa\nroot = '/kaggle/input/rsna-str-pulmonary-embolism-detection'\nfor item in os.listdir(root):\n    path = os.path.join(root, item)\n    if os.path.isfile(path):\n        print(path)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEVICE = \"TPU\" # \"TPU\" or \"GPU\"\nFOLDS = 5\nBATCH_SIZE = 8\nEPOCHS = [10]*FOLDS\ndebug = 1\nAUG_BATCH = BATCH_SIZE\nTRAIN_TYPE = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n\nif REPLICAS == 8:\n    BATCH_SIZE = 1024\n\nprint(BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS\nfor i in range(FOLDS):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('rsnav4')\n\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/*.tfrec')))\nlen(files_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset and Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 3.0\nSHR_ = 0.05\nHZOOM_ = 100\nWZOOM_ = 100\nHSHIFT_ = 3\nWSHIFT_ = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform_mat(image, DIM=256):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [256, 256, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"StudyInstanceUID\": tf.io.FixedLenFeature([], tf.string),\n        \"SeriesInstanceUID\": tf.io.FixedLenFeature([], tf.string),\n        \"SOPInstanceUID\": tf.io.FixedLenFeature([], tf.string),\n        \"negative_exam_for_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"rv_lv_ratio_gte_1\": tf.io.FixedLenFeature([], tf.float32),  # shape [] means single element\n        \"rv_lv_ratio_lt_1\": tf.io.FixedLenFeature([], tf.float32),\n        \"leftsided_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"chronic_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"rightsided_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"acute_and_chronic_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"central_pe\": tf.io.FixedLenFeature([], tf.float32),\n        \"indeterminate\": tf.io.FixedLenFeature([], tf.float32), \n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label1 = tf.cast(example['negative_exam_for_pe'], tf.float32)\n    label2 = tf.cast(example['rv_lv_ratio_gte_1'], tf.float32)\n    label3 = tf.cast(example['rv_lv_ratio_lt_1'], tf.float32)\n    label4 = tf.cast(example['leftsided_pe'], tf.float32)\n    label5 = tf.cast(example['chronic_pe'], tf.float32)\n    label6 = tf.cast(example['rightsided_pe'], tf.float32)\n    label7 = tf.cast(example['acute_and_chronic_pe'], tf.float32)\n    label8 = tf.cast(example['central_pe'], tf.float32)\n    label9 = tf.cast(example['indeterminate'], tf.float32)\n    return image, {'negative_exam_for_pe':label1,\n                      'rv_lv_ratio_gte_1':label2,\n                      'rv_lv_ratio_lt_1':label3,\n                      'leftsided_pe':label4,\n                      'chronic_pe':label5,\n                      'rightsided_pe':label6,\n                      'acute_and_chronic_pe':label7,\n                      'central_pe':label8,\n                      'indeterminate':label9} # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    #image = tf.image.random_flip_left_right(image)\n    return image, label\n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=False):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    #print(n)\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int( count_data_items(files_train) * (FOLDS-1.)/FOLDS )\nNUM_VALIDATION_IMAGES = int( count_data_items(files_train) * (1./FOLDS) )\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images'.format(NUM_TRAINING_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def horizontal_mask(image, DIM=[256,256], PROBABILITY = 0.75, CT = 3, SZ = 0.02):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,3]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def vertical_mask(image, DIM=[256,256], PROBABILITY = 0.75, CT = 3, SZ = 0.02):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): return image\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,3]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],3])\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augmentation(img):\n    img += tf.random.normal(tf.shape(img), 0, 1e-3)\n    img = transform_mat(img)\n    img = horizontal_mask(img, PROBABILITY = 0.5)\n    img = vertical_mask(img, PROBABILITY = 0.5)\n    img = horizontal_mask(img, PROBABILITY = 0.5)\n    img = vertical_mask(img, PROBABILITY = 0.5)\n    img = horizontal_mask(img, PROBABILITY = 0.5)\n    img = vertical_mask(img, PROBABILITY = 0.5)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(image,label):\n    imgs = []\n    for j in range(AUG_BATCH):\n        imgs.append(augmentation(image[j,]))\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    images = tf.reshape(tf.stack(imgs),(AUG_BATCH,256,256,3))\n    return images,label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if debug:\n    row = 2; col = 2;\n    row = min(row,AUG_BATCH//col)\n    all_elements = get_training_dataset(load_dataset(files_train),do_aug=False).unbatch()\n    augmented_element = all_elements.repeat().batch(AUG_BATCH).map(transform)\n\n    for (img, label) in augmented_element:\n        print(label)\n        plt.figure(figsize=(15,int(15*row/col)))\n        for j in range(row*col):\n            plt.subplot(row,col,j+1)\n            plt.axis('off')\n            plt.imshow(img[j,])\n        plt.show()\n        break\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import regularizers\nREG = 1e-4\nDO = 0\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Copyright 2019 The TensorFlow Authors. All Rights Reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ==============================================================================\n# pylint: disable=invalid-name\n\"\"\"EfficientNet models for Keras.\n\nReference paper:\n  - [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks]\n    (https://arxiv.org/abs/1905.11946) (ICML 2019)\n\"\"\"\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport copy\nimport math\nimport os\n\nfrom tensorflow.python.keras import backend\nfrom tensorflow.python.keras import layers\nfrom tensorflow.python.keras.applications import imagenet_utils\nfrom tensorflow.python.keras.engine import training\nfrom tensorflow.python.keras.utils import data_utils\nfrom tensorflow.python.keras.utils import layer_utils\nfrom tensorflow.python.util.tf_export import keras_export\n\n\nBASE_WEIGHTS_PATH = 'https://storage.googleapis.com/keras-applications/'\n\nWEIGHTS_HASHES = {\n    'b0': ('902e53a9f72be733fc0bcb005b3ebbac',\n           '50bc09e76180e00e4465e1a485ddc09d'),\n    'b1': ('1d254153d4ab51201f1646940f018540',\n           '74c4e6b3e1f6a1eea24c589628592432'),\n    'b2': ('b15cce36ff4dcbd00b6dd88e7857a6ad',\n           '111f8e2ac8aa800a7a99e3239f7bfb39'),\n    'b3': ('ffd1fdc53d0ce67064dc6a9c7960ede0',\n           'af6d107764bb5b1abb91932881670226'),\n    'b4': ('18c95ad55216b8f92d7e70b3a046e2fc',\n           'ebc24e6d6c33eaebbd558eafbeedf1ba'),\n    'b5': ('ace28f2a6363774853a83a0b21b9421a',\n           '38879255a25d3c92d5e44e04ae6cec6f'),\n    'b6': ('165f6e37dce68623721b423839de8be5',\n           '9ecce42647a20130c1f39a5d4cb75743'),\n    'b7': ('8c03f828fec3ef71311cd463b6759d99',\n           'cbcfe4450ddf6f3ad90b1b398090fe4a'),\n}\n\nDEFAULT_BLOCKS_ARGS = [{\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 32,\n    'filters_out': 16,\n    'expand_ratio': 1,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 2,\n    'filters_in': 16,\n    'filters_out': 24,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 2,\n    'filters_in': 24,\n    'filters_out': 40,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 3,\n    'filters_in': 40,\n    'filters_out': 80,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 3,\n    'filters_in': 80,\n    'filters_out': 112,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 5,\n    'repeats': 4,\n    'filters_in': 112,\n    'filters_out': 192,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 2,\n    'se_ratio': 0.25\n}, {\n    'kernel_size': 3,\n    'repeats': 1,\n    'filters_in': 192,\n    'filters_out': 320,\n    'expand_ratio': 6,\n    'id_skip': True,\n    'strides': 1,\n    'se_ratio': 0.25\n}]\n\nCONV_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 2.0,\n        'mode': 'fan_out',\n        'distribution': 'truncated_normal'\n    }\n}\n\nDENSE_KERNEL_INITIALIZER = {\n    'class_name': 'VarianceScaling',\n    'config': {\n        'scale': 1. / 3.,\n        'mode': 'fan_out',\n        'distribution': 'uniform'\n    }\n}\n\n\ndef EfficientNet(\n    width_coefficient,\n    depth_coefficient,\n    default_size,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n    depth_divisor=8,\n    activation='swish',\n    blocks_args='default',\n    model_name='efficientnet',\n    include_top=True,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=None,\n    pooling=None,\n    classes=1000,\n    classifier_activation='softmax',\n):\n  \"\"\"Instantiates the EfficientNet architecture using given scaling coefficients.\n\n  Optionally loads weights pre-trained on ImageNet.\n  Note that the data format convention used by the model is\n  the one specified in your Keras config at `~/.keras/keras.json`.\n\n  Arguments:\n    width_coefficient: float, scaling coefficient for network width.\n    depth_coefficient: float, scaling coefficient for network depth.\n    default_size: integer, default input image size.\n    dropout_rate: float, dropout rate before final classifier layer.\n    drop_connect_rate: float, dropout rate at skip connections.\n    depth_divisor: integer, a unit of network width.\n    activation: activation function.\n    blocks_args: list of dicts, parameters to construct block modules.\n    model_name: string, model name.\n    include_top: whether to include the fully-connected\n        layer at the top of the network.\n    weights: one of `None` (random initialization),\n          'imagenet' (pre-training on ImageNet),\n          or the path to the weights file to be loaded.\n    input_tensor: optional Keras tensor\n        (i.e. output of `layers.Input()`)\n        to use as image input for the model.\n    input_shape: optional shape tuple, only to be specified\n        if `include_top` is False.\n        It should have exactly 3 inputs channels.\n    pooling: optional pooling mode for feature extraction\n        when `include_top` is `False`.\n        - `None` means that the output of the model will be\n            the 4D tensor output of the\n            last convolutional layer.\n        - `avg` means that global average pooling\n            will be applied to the output of the\n            last convolutional layer, and thus\n            the output of the model will be a 2D tensor.\n        - `max` means that global max pooling will\n            be applied.\n    classes: optional number of classes to classify images\n        into, only to be specified if `include_top` is True, and\n        if no `weights` argument is specified.\n    classifier_activation: A `str` or callable. The activation function to use\n        on the \"top\" layer. Ignored unless `include_top=True`. Set\n        `classifier_activation=None` to return the logits of the \"top\" layer.\n\n  Returns:\n    A `keras.Model` instance.\n\n  Raises:\n    ValueError: in case of invalid argument for `weights`,\n      or invalid input shape.\n    ValueError: if `classifier_activation` is not `softmax` or `None` when\n      using a pretrained top layer.\n  \"\"\"\n  if blocks_args == 'default':\n    blocks_args = DEFAULT_BLOCKS_ARGS\n\n  if not (weights in {'imagenet', None} or os.path.exists(weights)):\n    raise ValueError('The `weights` argument should be either '\n                     '`None` (random initialization), `imagenet` '\n                     '(pre-training on ImageNet), '\n                     'or the path to the weights file to be loaded.')\n\n  if weights == 'imagenet' and include_top and classes != 1000:\n    raise ValueError('If using `weights` as `\"imagenet\"` with `include_top`'\n                     ' as true, `classes` should be 1000')\n\n  # Determine proper input shape\n  input_shape = imagenet_utils.obtain_input_shape(\n      input_shape,\n      default_size=default_size,\n      min_size=32,\n      data_format=backend.image_data_format(),\n      require_flatten=include_top,\n      weights=weights)\n\n  if input_tensor is None:\n    img_input = layers.Input(shape=input_shape)\n  else:\n    if not backend.is_keras_tensor(input_tensor):\n      img_input = layers.Input(tensor=input_tensor, shape=input_shape)\n    else:\n      img_input = input_tensor\n\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  def round_filters(filters, divisor=depth_divisor):\n    \"\"\"Round number of filters based on depth multiplier.\"\"\"\n    filters *= width_coefficient\n    new_filters = max(divisor, int(filters + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_filters < 0.9 * filters:\n      new_filters += divisor\n    return int(new_filters)\n\n  def round_repeats(repeats):\n    \"\"\"Round number of repeats based on depth multiplier.\"\"\"\n    return int(math.ceil(depth_coefficient * repeats))\n\n  # Build stem\n  x = img_input\n  #x = layers.Rescaling(1. / 255.)(x)\n  x = layers.Normalization(axis=bn_axis)(x)\n\n  x = layers.ZeroPadding2D(\n      padding=imagenet_utils.correct_pad(x, 3),\n      name='stem_conv_pad')(x)\n  x = layers.Conv2D(\n      round_filters(32),\n      3,\n      strides=2,\n      padding='valid',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='stem_conv',\n      kernel_regularizer=regularizers.l2(REG))(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='stem_bn')(x)\n  x = layers.Activation(activation, name='stem_activation')(x)\n  #x = layers.Dropout(0.01, name='top_dropout0')(x)\n\n  # Build blocks\n  blocks_args = copy.deepcopy(blocks_args)\n\n  b = 0\n  blocks = float(sum(args['repeats'] for args in blocks_args))\n  for (i, args) in enumerate(blocks_args):\n    assert args['repeats'] > 0\n    # Update block input and output filters based on depth multiplier.\n    args['filters_in'] = round_filters(args['filters_in'])\n    args['filters_out'] = round_filters(args['filters_out'])\n\n    for j in range(round_repeats(args.pop('repeats'))):\n      # The first block needs to take care of stride and filter size increase.\n      if j > 0:\n        args['strides'] = 1\n        args['filters_in'] = args['filters_out']\n      x = block(\n          x,\n          activation,\n          drop_connect_rate * b / blocks,\n          name='block{}{}_'.format(i + 1, chr(j + 97)),\n          **args)\n      b += 1\n\n  # Build top\n  x = layers.Conv2D(\n      round_filters(1280),\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name='top_conv',\n      kernel_regularizer=regularizers.l2(REG))(x)\n  x = layers.BatchNormalization(axis=bn_axis, name='top_bn')(x)\n  x = layers.Activation(activation, name='top_activation')(x)\n  if include_top:\n    x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    if dropout_rate > 0:\n      x = layers.Dropout(dropout_rate, name='top_dropout1')(x)\n    imagenet_utils.validate_activation(classifier_activation, weights)\n    x = layers.Dense(\n        classes,\n        activation=classifier_activation,\n        kernel_initializer=DENSE_KERNEL_INITIALIZER,\n        name='predictions',\n          kernel_regularizer=regularizers.l2(REG))(x)\n  else:\n    if pooling == 'avg':\n      x = layers.GlobalAveragePooling2D(name='avg_pool')(x)\n    elif pooling == 'max':\n      x = layers.GlobalMaxPooling2D(name='max_pool')(x)\n\n  # Ensure that the model takes into account\n  # any potential predecessors of `input_tensor`.\n  if input_tensor is not None:\n    inputs = layer_utils.get_source_inputs(input_tensor)\n  else:\n    inputs = img_input\n\n  # Create model.\n  model = training.Model(inputs, x, name=model_name)\n\n  # Load weights.\n  if weights == 'imagenet':\n    if include_top:\n      file_suffix = '.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][0]\n    else:\n      file_suffix = '_notop.h5'\n      file_hash = WEIGHTS_HASHES[model_name[-2:]][1]\n    file_name = model_name + file_suffix\n    weights_path = data_utils.get_file(\n        file_name,\n        BASE_WEIGHTS_PATH + file_name,\n        cache_subdir='models',\n        file_hash=file_hash)\n    model.load_weights(weights_path)\n  elif weights is not None:\n    model.load_weights(weights)\n  return model\n\n\ndef block(inputs,\n          activation='swish',\n          drop_rate=0.,\n          name='',\n          filters_in=32,\n          filters_out=16,\n          kernel_size=3,\n          strides=1,\n          expand_ratio=1,\n          se_ratio=0.,\n          id_skip=True):\n  \"\"\"An inverted residual block.\n\n  Arguments:\n      inputs: input tensor.\n      activation: activation function.\n      drop_rate: float between 0 and 1, fraction of the input units to drop.\n      name: string, block label.\n      filters_in: integer, the number of input filters.\n      filters_out: integer, the number of output filters.\n      kernel_size: integer, the dimension of the convolution window.\n      strides: integer, the stride of the convolution.\n      expand_ratio: integer, scaling coefficient for the input filters.\n      se_ratio: float between 0 and 1, fraction to squeeze the input filters.\n      id_skip: boolean.\n\n  Returns:\n      output tensor for the block.\n  \"\"\"\n  bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1\n\n  # Expansion phase\n  filters = filters_in * expand_ratio\n  if expand_ratio != 1:\n    x = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        use_bias=False,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'expand_conv',\n          kernel_regularizer=regularizers.l2(REG))(\n            inputs)\n    x = layers.BatchNormalization(axis=bn_axis, name=name + 'expand_bn')(x)\n    x = layers.Activation(activation, name=name + 'expand_activation')(x)\n    #x = layers.Dropout(0.01, name=name+'top_dropout3')(x)\n  else:\n    x = inputs\n\n  # Depthwise Convolution\n  if strides == 2:\n    x = layers.ZeroPadding2D(\n        padding=imagenet_utils.correct_pad(x, kernel_size),\n        name=name + 'dwconv_pad')(x)\n    conv_pad = 'valid'\n  else:\n    conv_pad = 'same'\n  x = layers.DepthwiseConv2D(\n      kernel_size,\n      strides=strides,\n      padding=conv_pad,\n      use_bias=False,\n      depthwise_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'dwconv')(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'bn')(x)\n  x = layers.Activation(activation, name=name + 'activation')(x)\n  #x = layers.Dropout(0.01, name=name+'top_dropout')(x)\n\n  # Squeeze and Excitation phase\n  if 0 < se_ratio <= 1:\n    filters_se = max(1, int(filters_in * se_ratio))\n    se = layers.GlobalAveragePooling2D(name=name + 'se_squeeze')(x)\n    se = layers.Reshape((1, 1, filters), name=name + 'se_reshape')(se)\n    se = layers.Conv2D(\n        filters_se,\n        1,\n        padding='same',\n        activation=activation,\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_reduce',\n          kernel_regularizer=regularizers.l2(REG))(\n            se)\n    #se = layers.Dropout(0.1, name=name+'top_dropout5')(se)\n    se = layers.Conv2D(\n        filters,\n        1,\n        padding='same',\n        activation='sigmoid',\n        kernel_initializer=CONV_KERNEL_INITIALIZER,\n        name=name + 'se_expand',\n          kernel_regularizer=regularizers.l2(REG))(se)\n    #se = layers.Dropout(0.1, name=name+'top_dropout6')(se)\n    x = layers.multiply([x, se], name=name + 'se_excite')\n\n  # Output phase\n  x = layers.Conv2D(\n      filters_out,\n      1,\n      padding='same',\n      use_bias=False,\n      kernel_initializer=CONV_KERNEL_INITIALIZER,\n      name=name + 'project_conv',\n      kernel_regularizer=regularizers.l2(REG))(x)\n  x = layers.BatchNormalization(axis=bn_axis, name=name + 'project_bn')(x)\n  if id_skip and strides == 1 and filters_in == filters_out:\n    if drop_rate > 0:\n      x = layers.Dropout(\n          drop_rate, noise_shape=(None, 1, 1, 1), name=name + 'drop')(x)\n    x = layers.add([x, inputs], name=name + 'add')\n  return x\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB0',\n              'keras.applications.EfficientNetB0')\ndef EfficientNetB0(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.0,\n      224,\n      0.5,\n      model_name='efficientnetb0',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB1',\n              'keras.applications.EfficientNetB1')\ndef EfficientNetB1(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.0,\n      1.1,\n      240,\n      0.2,\n      model_name='efficientnetb1',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB2',\n              'keras.applications.EfficientNetB2')\ndef EfficientNetB2(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.1,\n      1.2,\n      260,\n      0.3,\n      model_name='efficientnetb2',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB3',\n              'keras.applications.EfficientNetB3')\ndef EfficientNetB3(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.2,\n      1.4,\n      300,\n      0.3,\n      model_name='efficientnetb3',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB4',\n              'keras.applications.EfficientNetB4')\ndef EfficientNetB4(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.4,\n      1.8,\n      380,\n      0.4,\n      model_name='efficientnetb4',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB5',\n              'keras.applications.EfficientNetB5')\ndef EfficientNetB5(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.6,\n      2.2,\n      456,\n      0.4,\n      model_name='efficientnetb5',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB6',\n              'keras.applications.EfficientNetB6')\ndef EfficientNetB6(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      1.8,\n      2.6,\n      528,\n      0.5,\n      model_name='efficientnetb6',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.EfficientNetB7',\n              'keras.applications.EfficientNetB7')\ndef EfficientNetB7(include_top=True,\n                   weights='imagenet',\n                   input_tensor=None,\n                   input_shape=None,\n                   pooling=None,\n                   classes=1000,\n                   **kwargs):\n  return EfficientNet(\n      2.0,\n      3.1,\n      600,\n      0.5,\n      model_name='efficientnetb7',\n      include_top=include_top,\n      weights=weights,\n      input_tensor=input_tensor,\n      input_shape=input_shape,\n      pooling=pooling,\n      classes=classes,\n      **kwargs)\n\n\n@keras_export('keras.applications.efficientnet.preprocess_input')\ndef preprocess_input(x, data_format=None):  # pylint: disable=unused-argument\n  return x\n\n\n@keras_export('keras.applications.efficientnet.decode_predictions')\ndef decode_predictions(preds, top=5):\n  \"\"\"Decodes the prediction result from the model.\n\n  Arguments\n    preds: Numpy tensor encoding a batch of predictions.\n    top: Integer, how many top-guesses to return.\n\n  Returns\n    A list of lists of top class prediction tuples\n    `(class_name, class_description, score)`.\n    One list of tuples per sample in batch input.\n\n  Raises\n    ValueError: In case of invalid shape of the `preds` array (must be 2D).\n  \"\"\"\n  return imagenet_utils.decode_predictions(preds, top=top)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\nimport dill\n\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed\n\n\ndef categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Sum the losses in mini_batch\n        return K.sum(loss, axis=1)\n\n    return categorical_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(train_type=0):\n    inputs = Input((256, 256, 3))\n    #x = Conv2D(3, (1, 1), activation='relu')(inputs)\n    base_model = EfficientNetB3(\n        include_top=False,\n        weights=\"../input/ckpt2h5/efficientnetb3_notop.h5\",\n        input_shape=[256,256,3]\n    )\n    #print(len(base_model.layers))\n    if train_type==1:\n        base_model.trainable = False\n    \n    if train_type==2:\n        for layer in base_model.layers[-20:]:\n            if not isinstance(layer, layers.BatchNormalization):\n                layer.trainable = True\n\n    outputs = base_model(inputs)#, training=True)\n    outputs = keras.layers.GlobalAveragePooling2D()(outputs)\n    outputs = layers.BatchNormalization()(outputs)\n    outputs = Dropout(0.25)(outputs)\n    nefp = Dense(1, activation='sigmoid', name='negative_exam_for_pe')(outputs)\n    rlrg1 = Dense(1, activation='sigmoid', name='rv_lv_ratio_gte_1')(outputs)\n    rlrl1 = Dense(1, activation='sigmoid', name='rv_lv_ratio_lt_1')(outputs) \n    lspe = Dense(1, activation='sigmoid', name='leftsided_pe')(outputs)\n    cpe = Dense(1, activation='sigmoid', name='chronic_pe')(outputs)\n    rspe = Dense(1, activation='sigmoid', name='rightsided_pe')(outputs)\n    aacpe = Dense(1, activation='sigmoid', name='acute_and_chronic_pe')(outputs)\n    cnpe = Dense(1, activation='sigmoid', name='central_pe')(outputs)\n    indt = Dense(1, activation='sigmoid', name='indeterminate')(outputs)\n\n    model = Model(inputs=inputs, outputs={'negative_exam_for_pe':nefp,\n                                          'rv_lv_ratio_gte_1':rlrg1,\n                                          'rv_lv_ratio_lt_1':rlrl1,\n                                          'leftsided_pe':lspe,\n                                          'chronic_pe':cpe,\n                                          'rightsided_pe':rspe,\n                                          'acute_and_chronic_pe':aacpe,\n                                          'central_pe':cnpe,\n                                          'indeterminate':indt})\n\n    opt = keras.optimizers.Adam(lr=0.001)\n    #loss = binary_focal_loss()\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n    model.compile(optimizer=opt,\n                  loss=loss,\n                  #loss='binary_crossentropy',\n                  metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Schedule"},{"metadata":{"trusted":true},"cell_type":"code","source":"sv = tf.keras.callbacks.ModelCheckpoint(\n        'best_model.h5', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\nes = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto',\n    baseline=None, restore_best_weights=False\n)\n\ndef get_lr_callback(batch_size=8,train_type=0):\n    if train_type==0 or train_type==2:\n        lr_start   = 0.00005\n        #lr_max     = 0.000000125 * REPLICAS * batch_size\n        lr_max     = 0.0001\n        lr_min     = 0.00001\n        lr_ramp_ep = 5\n        lr_sus_ep  = 0\n        lr_decay   = 0.8\n    else:\n        lr_start   = 0.00020\n        #lr_max     = 0.000000125 * REPLICAS * batch_size\n        lr_max     = 0.00025\n        lr_min     = 0.00001\n        lr_ramp_ep = 1\n        lr_sus_ep  = 0\n        lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=12)\n#skf = StratifiedKFold(n_splits=FOLDS,shuffle=True,random_state=12)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(251))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    #print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold,(idxT,idxV) in enumerate(skf.split(np.arange(251))):\n    if fold!=0:\n        continue\n    # REPEAT SAME FOLD OVER AND OVER\n        \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### EXPERIMENT',fold+1)\n    print('Fold',fold,'has TRAIN:',idxT[:5],'VALID:',idxV[:5])\n\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train-%i-*.tfrec'%x for x in idxT])\n    #files_train = files_train[:200]\n    print('#### all trains',len(files_train))\n        \n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train-%i-*.tfrec'%x for x in idxV])\n    files_valid = files_valid[:5] \n    print('#### all valids',len(files_valid))\n    \n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(train_type=TRAIN_TYPE)\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', min_delta=0, patience=10, verbose=1, mode='auto',\n        baseline=None, restore_best_weights=False\n    )\n    # TRAIN\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']), labeled = True, ordered = True)\n    print(model.summary())\n    print('Training...')\n    VERBOSE = 2\n    if debug:\n        VERBOSE = 1\n    history = model.fit(\n            get_training_dataset(train_dataset), \n            steps_per_epoch = STEPS_PER_EPOCH,\n            epochs = EPOCHS[fold],\n            callbacks = [sv, get_lr_callback(BATCH_SIZE,train_type=TRAIN_TYPE)],\n            validation_data = get_validation_dataset(val_dataset),\n            verbose=VERBOSE,\n            validation_freq=1,\n        )\n    model.save_weights('fold-%if.h5'%fold)\n    \n    \n    if 1==1:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_acute_and_chronic_pe_auc'],'-o',label='val_acute_and_chronic_pe AUC',color='#ff0000')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_central_pe_auc_1'],'-o',label='val_central_pe AUC',color='#00ff00')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_chronic_pe_auc_2'],'-o',label='val_chronic_pe AUC',color='#ffff00')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_indeterminate_auc_3'],'-o',label='val_indeterminate AUC',color='#0000ff')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_leftsided_pe_auc_4'],'-o',label='val_leftsided_pe AUC',color='#ff00ff')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_negative_exam_for_pe_auc_5'],'-o',label='val_negative_exam_for_pe AUC',color='#00ffff')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_rightsided_pe_auc_6'],'-o',label='val_rightsided_pe AUC',color='#aaaaaa')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_rv_lv_ratio_gte_1_auc_7'],'-o',label='val_rv_lv_ratio_gte_1 AUC',color='#000000')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_rv_lv_ratio_lt_1_auc_8'],'-o',label='val_rv_lv_ratio_lt_1 AUC',color='#aa0000')\n        x = np.argmax( history.history['val_acute_and_chronic_pe_auc'] ); y = np.max( history.history['val_acute_and_chronic_pe_auc'] )\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n\n        plt.show()  \n        \n    del model; z = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}