{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2N TF-Keras Traing Phase With TPU","metadata":{}},{"cell_type":"markdown","source":"### References, see also them\n\n#### [Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n#### [CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n#### [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n#### [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n\n#### [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu)","metadata":{}},{"cell_type":"markdown","source":"Warning: without augumentation\nthe TTA validations test are irelevant (Test Time Augmentation (TTA))\nTo do Test Time Augmentation, we can reuse the same Data Generator used for training, and apply it to validation images.\nWe can then show the model 10 times (for example) the randomly modified images, get the prediction for each, and take the average:","metadata":{}},{"cell_type":"markdown","source":"## Initialize Environment","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\ntf.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\nFOLDS = 5\nIMG_SIZE = [360,128]\n\nBATCH_SIZE = 32\nEPOCH = 75\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        #policy = mixed_precision.Policy('mixed_bfloat16')\n        #mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What we're looking for is a printout of Number of replicas: 8, corresponding to the 8 cores of a TPU. If Number of replicas = 1 then the TPUs is not enabled.\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE *= strategy.num_replicas_in_sync","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"from path import Path","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('tfrecg2netdataset2')\nGCS_PATH_DasOriginal = KaggleDatasets().get_gcs_path('g2net-tfr-spectrogram-datasets')\nGCS_PATH_STRATIFICATED_modified=KaggleDatasets().get_gcs_path('traintfrecg2netdataset2')\n# GCS_PATH_STRATIFICATED = '/kaggle/input/g2net-tfr-spectrogram-datasets/'\n# GCS_PATH_STRATIFICATED = '/kaggle/input/g2net-tfr-spectrogram-datasets/'\nfiles_train = tf.io.gfile.glob(GCS_PATH_STRATIFICATED_modified + '/train*.tfrec')\nfiles_test  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED_modified + '/test*.tfrec')\n# files_test_compozit  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED_modified + '/**/test*.tfrec')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# files_test_compozit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(files_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('we have a number of',len(files_train),'train file and a number of ',len(files_test),'test files')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"image_augumentation=True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Masking images:","metadata":{}},{"cell_type":"code","source":"def freq_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.33, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    #for image float 64 \n    #two = tf.zeros([yb-ya,xb-xa,2],dtype=tf.dtypes.float64)\n    maximum=tf.reduce_mean(image)\n    two =tf.random.uniform([yb-ya,xb-xa,2],maximum*0.6,maximum*1.1,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freq_mask2(image, DIM=IMG_SIZE, PROBABILITY = 0.33, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2],dtype=tf.dtypes.float64)\n    maximum=tf.reduce_mean(image)\n    two =tf.random.uniform([yb-ya,xb-xa,2],maximum*0.7,maximum,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.33, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2],dtype=tf.dtypes.float64)\n\n#     two =tf.random.uniform([yb-ya,xb-xa,2],1e-25,1e-24,dtype=tf.dtypes.float64)\n\n    maximum=tf.reduce_mean(image)\n    two =tf.random.uniform([yb-ya,xb-xa,2],maximum*0.6,maximum*1.1,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask2(image, DIM=IMG_SIZE, PROBABILITY = 0.33, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2],dtype=tf.dtypes.float64)\n \n    two =tf.random.uniform([yb-ya,xb-xa,2],1e-25,1e-24,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_shuffle(image, DIM=IMG_SIZE, PROBABILITY = 0.3):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0): return image\n    \n\n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x)\n    xb = DIM[1]\n   \n    image = tf.concat([image[:,xa:DIM[1],:],image[:,0:xa,:]],axis=1)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_augment(image, label):\n#     # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n#     # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n#     image = tf.image.random_flip_left_right(image)\n#     return image, label\n# data_augmentation = keras.Sequential(\n#     [\n#         layers.RandomFlip(\"horizontal_and_vertical\"), \n# #         layers.GaussianNoise(stddev=0.2),\n# #         layers.GaussianNoise(stddev=0.1),\n# #         layers.RandomRotation(factor=0.1),\n# #         layers.RandomRotation(factor=0.2),\n# #         layers.RandomRotation(factor=0.3),\n# #         layers.RandomRotation(factor=0.4),\n# #         layers.RandomTranslation(height_factor =0.1,width_factor =0.1,fill_mode =\"nearest\",interpolation = \"bilinear\"),\n# #         #layers.RandomCrop(320, 320, seed=None,[height=360,width=360]),\n#         layers.RandomContrast(factor=0.3),\n# #         #layers.RandomTranslation(height_factor =(-0.1, 0.2),width_factor = (-0.3, 0.3),fill_mode =\"nearest\",interpolation = \"bilinear\"),\n# #         layers.RandomTranslation(height_factor =0.05,width_factor =0.05,fill_mode =\"nearest\",interpolation = \"bilinear\"),\n\n#     ]\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def augmentation(img):\n#     img = freq_mask(img)\n#     img = time_mask(img)\n#     img = tf.image.random_flip_left_right(img) # To be tested\n#     img = freq_mask(img,SZ = 0.02)\n#     img = time_mask(img)\n#     img = freq_mask(img)\n#     img = time_mask(img,SZ = 0.02)\n#     img = freq_mask(img,SZ = 0.01)\n#     img = time_mask(img,SZ = 0.02)\n#     img = freq_mask(img,SZ = 0.02)\n#     img = time_mask(img,SZ = 0.07)\n#     img = time_shuffle(img, PROBABILITY = 0.25)\n#     return img\ndef augmentation(img):\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = tf.image.random_flip_left_right(img) # To be tested\n#     img =tf.image.random_contrast(img, 0.2, 0.5)\n    img = freq_mask2(img,SZ = 0.02)\n    img = time_mask(img, SZ = 0.01)\n    img = freq_mask2(img,SZ = 0.01)\n    img = time_shuffle(img, PROBABILITY = 0.2)\n#     img = time_shuffle(img, PROBABILITY = 0.15)\n    img = time_mask(img,SZ = 0.03)\n    img = freq_mask2(img,SZ = 0.01)\n    img = time_mask(img,SZ = 0.06)\n    img = freq_mask2(img,SZ = 0.005)\n    img = freq_mask(img,SZ = 0.07)\n    img = time_mask2(img,SZ = 0.1)\n    img = freq_mask(img,SZ = 0.1)\n    img = freq_mask2(img,SZ = 0.01)\n    img = time_mask2(img,SZ = 0.02)\n    img = time_mask2(img,SZ = 0.03)\n    img = freq_mask2(img,SZ = 0.06)\n    img = time_mask2(img,SZ = 0.04)\n    img = freq_mask2(img,SZ = 0.05)\n    img = time_mask2(img,SZ = 0.03)\n    img = time_mask2(img,SZ = 0.01)\n    img = time_mask2(img,SZ = 0.02)\n    img = time_mask2(img,SZ = 0.01)\n    img = time_mask2(img,SZ = 0.07)\n    img = freq_mask2(img,SZ = 0.03)\n    img = time_mask2(img,SZ = 0.01)\n    img = time_mask2(img,SZ = 0.02)\n    img = freq_mask2(img,SZ = 0.03)\n    img = time_mask2(img,SZ = 0.06)\n    img = time_mask2(img,SZ = 0.04)\n    img = time_mask2(img,SZ = 0.053)\n    img = freq_mask2(img,SZ = 0.031)\n    img = time_mask2(img,SZ = 0.011)\n    img = freq_mask2(img,SZ = 0.022)\n    img = time_mask2(img,SZ = 0.015)\n    img = time_mask2(img,SZ = 0.025)\n    img = time_mask2(img,SZ = 0.005)\n    img = time_shuffle(img, PROBABILITY = 0.25)\n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read TF records","metadata":{}},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float64)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    #print(example['spectrogram'].shape)\n    return example['spectrogram'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'id'                   : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float64)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    return example['spectrogram'], example['id'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=False):    \n    if augment:\n        img = augmentation(img)\n# for testing withou a dummy function should be created: \n                                            # def augmentation(img):\n                                            #     img = img\n                                            #     return img\n    #The tf.reshape does not change the order of or the total number of elements in the tensor, \n    #and so it can reuse the underlying data buffer. \n    #This makes it a fast operation independent of how big of a tensor it is operating on.\n    img = tf.reshape(img, [*IMG_SIZE, 2])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=32):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8 if tpu else 128)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    #Augumentation here\n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment,), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# files_test_compozit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FNAME = tf.io.gfile.glob([GCS_PATH_STRATIFICATED_modified + '/train*.tfrec'])\nrow = 16; col = 2;\nrow = min(row,16//col)\n\n# all_elements = get_dataset(FNAME, augment=False, batch_size=32).unbatch()\n# augmented_element = all_elements.repeat().batch(32)\nall_elements = get_dataset(FNAME, augment=image_augumentation, batch_size=32)\n# for (img,mask) in augmented_element:\nfor (img,mask) in all_elements:\n    print(img.shape)\n    img = (img-np.min(img))/(np.max(img)-np.min(img)+1e-5)\n    plt.figure(figsize=(15,int(15*row/col)))\n\n    i=0\n    j=1\n    while j<=row*col:\n        plt.subplot(row,col,j)\n        plt.axis('on')\n\n        plt.imshow(img[i,:,:,0])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(img[i,:,:,1])\n        j += 1\n\n        i += 1\n\n    plt.show()\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"#tf.keras.applications.EfficientNetB7","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n    in_conv = tf.keras.layers.Conv2D(3, 3, strides=(1, 1), padding='same')\n    base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     base = tf.keras.applications.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n    x = in_conv(inp)\n    x = base(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dropout(.2)(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n#     opt = tf.keras.optimizers.SGD(learning_rate=0.001,clipvalue=1)\n#     opt = tf.keras.optimizers.RMSprop(learning_rate=0.001,clipvalue=1.)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5) \n    model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 5e-5\n    lr_max     = 7e-4\n    lr_min     = 1e-5\n    lr_ramp_ep = 4\n    lr_sus_ep  = 4\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Learning rate callback function\n# BATCH_SIZE\n# def get_lr_callback():\n#     lr_start   = 0.0001\n#     lr_max     = 0.000015 * BATCH_SIZE\n#     lr_min     = 0.0000001\n#     lr_ramp_ep = 3\n#     lr_sus_ep  = 0\n#     lr_decay   = 0.7\n   \n#     def lrfn(epoch):\n#         if epoch < lr_ramp_ep:\n#             lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n#         elif epoch < lr_ramp_ep + lr_sus_ep:\n#             lr = lr_max    \n#         else:\n#             lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n#         return lr\n\n#     lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = VERBOSE)\n#     return lr_callback","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_lr_callback():\n#     lr_constant     = 5e-4\n   \n#     def lrfn(epoch):\n#         return lr_constant\n\n#     lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n#     return lr_callback","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2 if tpu else 1\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=False)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob(GCS_PATH_STRATIFICATED_modified + '/train*.tfrec')\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED_modified + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED_modified + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED_modified + '/test*.tfrec')\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    \n    es =tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=12,\n        verbose=1,\n        mode='auto',\n        )\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=image_augumentation, shuffle=True, repeat=True,\n                batch_size=BATCH_SIZE), \n        epochs=EPOCH, callbacks = [sv,get_lr_callback()], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False),\n#         verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF ...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_valid = count_data_items(files_valid); STEPS = ct_valid/BATCH_SIZE/4\n    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n   \n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, \n            labeled=True, return_image_names=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False,\n                labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) / FOLDS\n    \n    # REPORT RESULTS\n#     auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n#     print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCH),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCH),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCH),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCH),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i'%(fold+1),size=18)\n        plt.legend(loc=3)\n        plt.show()  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1+1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate OOF AUC","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_tar)\nnames = np.concatenate(oof_names)\nfolds = np.concatenate(oof_folds)\n# auc = roc_auc_score(true,oof)\n# print('Overall OOF AUC with TTA = %.4f'%auc)\n\n# # SAVE OOF TO DISK\n# df_oof = pd.DataFrame(dict(Id = names, target=true, pred = oof, fold=folds))\n# df_oof.to_csv('oof.csv',index=False)\n# df_oof.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selectie=df_oof[df_oof.Id.apply(lambda x: len(str(x))<=10)]\n# selectie['diference']=df_oof.target-df_oof.pred\n# selectie.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_oof.sample(100).head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submit","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = preds[:,0]\nsubmission = submission.sort_values('id') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros((count_data_items(files_test),1))\nlen(preds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}