{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2N TF-Keras Traing Phase With TPU","metadata":{}},{"cell_type":"markdown","source":"### References, see also them\n\n#### [Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n#### [CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n#### [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n#### [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n\n#### [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu)","metadata":{}},{"cell_type":"markdown","source":"Warning: without augumentation\nthe TTA validations test are irelevant (Test Time Augmentation (TTA))\nTo do Test Time Augmentation, we can reuse the same Data Generator used for training, and apply it to validation images.\nWe can then show the model 10 times (for example) the randomly modified images, get the prediction for each, and take the average:","metadata":{}},{"cell_type":"markdown","source":"## Initialize Environment","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:13.231585Z","iopub.execute_input":"2022-12-23T10:50:13.231985Z","iopub.status.idle":"2022-12-23T10:50:21.26936Z","shell.execute_reply.started":"2022-12-23T10:50:13.231949Z","shell.execute_reply":"2022-12-23T10:50:21.268482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\ntf.__version__","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.272144Z","iopub.execute_input":"2022-12-23T10:50:21.272478Z","iopub.status.idle":"2022-12-23T10:50:21.28343Z","shell.execute_reply.started":"2022-12-23T10:50:21.272434Z","shell.execute_reply":"2022-12-23T10:50:21.282618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\nFOLDS = 5\nIMG_SIZE = [360,128]\nChannels=2\nBATCH_SIZE = 32\nEPOCH = 60\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:55:47.276987Z","iopub.execute_input":"2022-12-23T10:55:47.277335Z","iopub.status.idle":"2022-12-23T10:55:47.283681Z","shell.execute_reply.started":"2022-12-23T10:55:47.277298Z","shell.execute_reply":"2022-12-23T10:55:47.282465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        #policy = mixed_precision.Policy('mixed_bfloat16')\n        #mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.296755Z","iopub.execute_input":"2022-12-23T10:50:21.296995Z","iopub.status.idle":"2022-12-23T10:50:21.316444Z","shell.execute_reply.started":"2022-12-23T10:50:21.296966Z","shell.execute_reply":"2022-12-23T10:50:21.31546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What we're looking for is a printout of Number of replicas: 8, corresponding to the 8 cores of a TPU. If Number of replicas = 1 then the TPUs is not enabled.\n","metadata":{}},{"cell_type":"code","source":"from pydantic import BaseModel","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.31896Z","iopub.execute_input":"2022-12-23T10:50:21.320344Z","iopub.status.idle":"2022-12-23T10:50:21.428897Z","shell.execute_reply.started":"2022-12-23T10:50:21.320308Z","shell.execute_reply":"2022-12-23T10:50:21.428056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config(BaseModel):\n    seed = 887\n    model_name = \"enetb1_v1\"\n    model_dir = \"/kaggle/working/model\"\n    # data\n \n    path_submission = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\"\n\n    img_size = (360, 360)\n    channels = 3\n    img_shape = (*img_size, channels)\n    # model\n    base_model_weights = \"imagenet\"\n    dropout = 0.3\n    # training\n    shuffle_size = 128\n    epochs = 150\n#     batch_size = 16 * strategy.num_replicas_in_sync\n    batch_size = 32 * strategy.num_replicas_in_sync\n    test_batch_size = 64\n    lr = 2e-5\n    patience = 12\n    \ncfg = Config()\ncfg.dict()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.430296Z","iopub.execute_input":"2022-12-23T10:50:21.431086Z","iopub.status.idle":"2022-12-23T10:50:21.447629Z","shell.execute_reply.started":"2022-12-23T10:50:21.431052Z","shell.execute_reply":"2022-12-23T10:50:21.44679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE *= strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.449489Z","iopub.execute_input":"2022-12-23T10:50:21.449812Z","iopub.status.idle":"2022-12-23T10:50:21.458399Z","shell.execute_reply.started":"2022-12-23T10:50:21.449771Z","shell.execute_reply":"2022-12-23T10:50:21.457562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"from path import Path","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.459485Z","iopub.execute_input":"2022-12-23T10:50:21.459736Z","iopub.status.idle":"2022-12-23T10:50:21.48131Z","shell.execute_reply.started":"2022-12-23T10:50:21.45971Z","shell.execute_reply":"2022-12-23T10:50:21.480516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('tfrecg2netdataset2')\nGCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('traintfrecg2netdataset1')\n# GCS_PATH_STRATIFICATED = '/kaggle/input/g2net-tfr-spectrogram-datasets/'\n# GCS_PATH_STRATIFICATED = '/kaggle/input/g2net-tfr-spectrogram-datasets/'\nfiles_train = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/train*.tfrec')\nfiles_test  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:21.48547Z","iopub.execute_input":"2022-12-23T10:50:21.485756Z","iopub.status.idle":"2022-12-23T10:50:22.149039Z","shell.execute_reply.started":"2022-12-23T10:50:21.485726Z","shell.execute_reply":"2022-12-23T10:50:22.147088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('we have a number of',len(files_train),'train file and a number of ',len(files_test),'test files')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.150552Z","iopub.execute_input":"2022-12-23T10:50:22.150923Z","iopub.status.idle":"2022-12-23T10:50:22.158692Z","shell.execute_reply.started":"2022-12-23T10:50:22.150884Z","shell.execute_reply":"2022-12-23T10:50:22.157703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"image_augumentation=True","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.160741Z","iopub.execute_input":"2022-12-23T10:50:22.161425Z","iopub.status.idle":"2022-12-23T10:50:22.167717Z","shell.execute_reply.started":"2022-12-23T10:50:22.161377Z","shell.execute_reply":"2022-12-23T10:50:22.166923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Masking images:","metadata":{}},{"cell_type":"code","source":"def freq_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,2]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.170869Z","iopub.execute_input":"2022-12-23T10:50:22.171132Z","iopub.status.idle":"2022-12-23T10:50:22.186056Z","shell.execute_reply.started":"2022-12-23T10:50:22.171086Z","shell.execute_reply":"2022-12-23T10:50:22.185031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,2]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.189683Z","iopub.execute_input":"2022-12-23T10:50:22.189932Z","iopub.status.idle":"2022-12-23T10:50:22.200238Z","shell.execute_reply.started":"2022-12-23T10:50:22.189899Z","shell.execute_reply":"2022-12-23T10:50:22.199268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_shuffle(image, DIM=IMG_SIZE, PROBABILITY = 0.4):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0): return image\n    \n\n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x)\n    xb = DIM[1]\n   \n    image = tf.concat([image[:,xa:DIM[1],:],image[:,0:xa,:]],axis=1)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.202183Z","iopub.execute_input":"2022-12-23T10:50:22.203191Z","iopub.status.idle":"2022-12-23T10:50:22.216572Z","shell.execute_reply.started":"2022-12-23T10:50:22.203149Z","shell.execute_reply":"2022-12-23T10:50:22.21547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_augment(image, label):\n#     # Thanks to the dataset.prefetch(AUTO) statement in the following function this happens essentially for free on TPU. \n#     # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n#     image = tf.image.random_flip_left_right(image)\n#     return image, label\n# data_augmentation = keras.Sequential(\n#     [\n#         layers.RandomFlip(\"horizontal_and_vertical\"), \n# #         layers.GaussianNoise(stddev=0.2),\n# #         layers.GaussianNoise(stddev=0.1),\n# #         layers.RandomRotation(factor=0.1),\n# #         layers.RandomRotation(factor=0.2),\n# #         layers.RandomRotation(factor=0.3),\n# #         layers.RandomRotation(factor=0.4),\n# #         layers.RandomTranslation(height_factor =0.1,width_factor =0.1,fill_mode =\"nearest\",interpolation = \"bilinear\"),\n# #         #layers.RandomCrop(320, 320, seed=None,[height=360,width=360]),\n#         layers.RandomContrast(factor=0.3),\n# #         #layers.RandomTranslation(height_factor =(-0.1, 0.2),width_factor = (-0.3, 0.3),fill_mode =\"nearest\",interpolation = \"bilinear\"),\n# #         layers.RandomTranslation(height_factor =0.05,width_factor =0.05,fill_mode =\"nearest\",interpolation = \"bilinear\"),\n\n#     ]\n# )","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.218129Z","iopub.execute_input":"2022-12-23T10:50:22.218504Z","iopub.status.idle":"2022-12-23T10:50:22.228586Z","shell.execute_reply.started":"2022-12-23T10:50:22.218465Z","shell.execute_reply":"2022-12-23T10:50:22.227488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation(img):\n    img = freq_mask(img, SZ = 0.05)\n    img = time_mask(img, SZ = 0.02)\n    img = tf.image.random_flip_left_right(img) # To be tested\n    img = freq_mask(img, SZ = 0.05)\n    img = time_mask(img, SZ = 0.02)\n    img = freq_mask(img, SZ = 0.07)\n    img = time_mask(img, SZ = 0.07)\n    img = freq_mask(img, SZ = 0.1)\n    img = time_mask(img, SZ = 0.1)\n    img = freq_mask(img, SZ = 0.1)\n    img = time_mask(img, SZ = 0.1)\n    img = time_shuffle(img, PROBABILITY = 0.75)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.230313Z","iopub.execute_input":"2022-12-23T10:50:22.23084Z","iopub.status.idle":"2022-12-23T10:50:22.243864Z","shell.execute_reply.started":"2022-12-23T10:50:22.230796Z","shell.execute_reply":"2022-12-23T10:50:22.24292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read TF records","metadata":{}},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    #print(example['spectrogram'].shape)\n    return example['spectrogram'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'id'                   : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    return example['spectrogram'], example['id'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=False):    \n    if augment:\n        img = augmentation(img)\n# for testing withou a dummy function should be created: \n                                            # def augmentation(img):\n                                            #     img = img\n                                            #     return img\n    #The tf.reshape does not change the order of or the total number of elements in the tensor, \n    #and so it can reuse the underlying data buffer. \n    #This makes it a fast operation independent of how big of a tensor it is operating on.\n    img = tf.reshape(img, [*IMG_SIZE, 2])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.245917Z","iopub.execute_input":"2022-12-23T10:50:22.246493Z","iopub.status.idle":"2022-12-23T10:50:22.260055Z","shell.execute_reply.started":"2022-12-23T10:50:22.246436Z","shell.execute_reply":"2022-12-23T10:50:22.259144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=32):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8 if tpu else 128)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    #Augumentation here\n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment,), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.262702Z","iopub.execute_input":"2022-12-23T10:50:22.263306Z","iopub.status.idle":"2022-12-23T10:50:22.278374Z","shell.execute_reply.started":"2022-12-23T10:50:22.263265Z","shell.execute_reply":"2022-12-23T10:50:22.277315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FNAME = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train*.tfrec'])\nrow = 16; col = 2;\nrow = min(row,16//col)\n\n# all_elements = get_dataset(FNAME, augment=False, batch_size=32).unbatch()\n# augmented_element = all_elements.repeat().batch(32)\nall_elements = get_dataset(FNAME, augment=image_augumentation, batch_size=32)\n# for (img,mask) in augmented_element:\nfor (img,mask) in all_elements:\n\n    img = (img-np.min(img))/(np.max(img)-np.min(img)+1e-5)\n    plt.figure(figsize=(15,int(15*row/col)))\n\n    i=0\n    j=1\n    while j<=row*col:\n        plt.subplot(row,col,j)\n        plt.axis('on')\n\n        plt.imshow(img[i,:,:,0])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(img[i,:,:,1])\n        j += 1\n\n        i += 1\n\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:22.280061Z","iopub.execute_input":"2022-12-23T10:50:22.280674Z","iopub.status.idle":"2022-12-23T10:50:30.86266Z","shell.execute_reply.started":"2022-12-23T10:50:22.280633Z","shell.execute_reply":"2022-12-23T10:50:30.86188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import WaveTf","metadata":{}},{"cell_type":"code","source":"package='/kaggle/input/wavetf/WaveTF-master/* ./'\n#!cp -r ../input/kagglepet/kagglePet/* ./\n!cp -r /kaggle/input/wavetf/WaveTF-master/* ./\n!cp -r /kaggle/input/wavetf/WaveTF-master/wavetf/* ./","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:30.86672Z","iopub.execute_input":"2022-12-23T10:50:30.867669Z","iopub.status.idle":"2022-12-23T10:50:33.074101Z","shell.execute_reply.started":"2022-12-23T10:50:30.867623Z","shell.execute_reply":"2022-12-23T10:50:33.07297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wavetf\nfrom wavetf import WaveTFFactory","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.077101Z","iopub.execute_input":"2022-12-23T10:50:33.07737Z","iopub.status.idle":"2022-12-23T10:50:33.098837Z","shell.execute_reply.started":"2022-12-23T10:50:33.07734Z","shell.execute_reply":"2022-12-23T10:50:33.097991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"IMG_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.100471Z","iopub.execute_input":"2022-12-23T10:50:33.100781Z","iopub.status.idle":"2022-12-23T10:50:33.107523Z","shell.execute_reply.started":"2022-12-23T10:50:33.100742Z","shell.execute_reply":"2022-12-23T10:50:33.106747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras.layers import Input,Flatten,Conv3D,MaxPooling3D, Conv2D, Concatenate, Dense, Lambda, BatchNormalization, GlobalAveragePooling2D, Activation,MaxPooling2D\nfrom tensorflow.keras.models import Model\nfrom wavetf import WaveTFFactory\nfrom tensorflow.keras import layers, Model, Input, losses, metrics, optimizers, callbacks","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.108961Z","iopub.execute_input":"2022-12-23T10:50:33.109398Z","iopub.status.idle":"2022-12-23T10:50:33.124077Z","shell.execute_reply.started":"2022-12-23T10:50:33.109357Z","shell.execute_reply":"2022-12-23T10:50:33.123225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import Xception","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.125668Z","iopub.execute_input":"2022-12-23T10:50:33.125981Z","iopub.status.idle":"2022-12-23T10:50:33.13684Z","shell.execute_reply.started":"2022-12-23T10:50:33.125944Z","shell.execute_reply":"2022-12-23T10:50:33.135935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def wavelet_cnn(input_shape, ks=3, baselev=4, wavelet=True,\n#                 wave_kern='haar', hsv=False, convrep=4, num_classes=2):\n#     inputs = Input(input_shape)\n#     chans = input_shape[2] # number of channels, e.g., 3 if RGB\n#     chans = 2\n#     bl = baselev\n#     wave0 = inputs\n#     wave1 = WaveTFFactory.build(wave_kern)(wave0)\n#     convolution=Conv2D(3, kernel_size=(3, 3), activation='relu', kernel_initializer='he_uniform',input_shape=(180, 64, 8))(wave1)\n#     #in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n# #     base = Xception(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     base = Xception(weights='imagenet', include_top=False)(convolution)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(base)\n#     convolution=Dense(1, activation='sigmoid')(x)\n# # #     x = in_conv(inp)\n# # #     x = base(x)\n# # #     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n# #     convolution=Dense(256, activation='relu', kernel_initializer='he_uniform')(convolution)\n# #     convolution=Dense(1, activation='sigmoid')(convolution)\n#     model = tf.keras.Model(inputs=inputs, outputs=convolution)\n#     return model\n# # #      model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', input_shape=sample_shape))\n# #     model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n# #     model.add(Conv3D(64, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n# #     model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n# #     model.add(Flatten())\n# #     model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n# #     model.add(Dense(no_classes, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.138641Z","iopub.execute_input":"2022-12-23T10:50:33.139006Z","iopub.status.idle":"2022-12-23T10:50:33.148443Z","shell.execute_reply.started":"2022-12-23T10:50:33.138971Z","shell.execute_reply":"2022-12-23T10:50:33.147313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wavelet_cnn(input_shape, ks=3, baselev=4, wavelet=True,\n                wave_kern='haar', hsv=False, convrep=4, num_classes=2):\n    inputs = Input(input_shape)\n    chans = input_shape[2] # number of channels, e.g., 3 if RGB\n    chans = 2\n    bl = baselev\n    wave0 = inputs\n    #Disabled wave convolution\n#     wave1 = WaveTFFactory.build(wave_kern)(wave0)\n    convolution=Conv2D(3, kernel_size=(3, 3), activation='relu', kernel_initializer='he_uniform',input_shape=(180, 64, 8))(wave0)\n    #in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n#     base = Xception(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n    base = Xception(weights='imagenet', include_top=False)(convolution)\n    x = tf.keras.layers.GlobalAveragePooling2D()(base)\n    convolution=Dense(1, activation='sigmoid')(x)\n# #     x = in_conv(inp)\n# #     x = base(x)\n# #     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n#     convolution=Dense(256, activation='relu', kernel_initializer='he_uniform')(convolution)\n#     convolution=Dense(1, activation='sigmoid')(convolution)\n    model = tf.keras.Model(inputs=inputs, outputs=convolution)\n    return model\n# #      model.add(Conv3D(32, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform', input_shape=sample_shape))\n#     model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n#     model.add(Conv3D(64, kernel_size=(3, 3, 3), activation='relu', kernel_initializer='he_uniform'))\n#     model.add(MaxPooling3D(pool_size=(2, 2, 2)))\n#     model.add(Flatten())\n#     model.add(Dense(256, activation='relu', kernel_initializer='he_uniform'))\n#     model.add(Dense(no_classes, activation='softmax'))","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:55:06.187611Z","iopub.execute_input":"2022-12-23T10:55:06.188026Z","iopub.status.idle":"2022-12-23T10:55:06.195555Z","shell.execute_reply.started":"2022-12-23T10:55:06.18799Z","shell.execute_reply":"2022-12-23T10:55:06.19465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def wavelet_cnn(input_shape, ks=3, baselev=4, wavelet=True,\n#                 wave_kern='haar', hsv=False, convrep=4, num_classes=2):\n#     inputs = Input(input_shape)\n#     chans = input_shape[2] # number of channels, e.g., 3 if RGB\n#     chans = 2\n#     bl = baselev\n\n#     # wavelet computation\n#     if (wavelet) :\n#         # convert RGB to HSV?\n#         if (hsv):\n#             wave0 = Lambda(lambda x: tf.image.rgb_to_hsv(x))(inputs)\n#         else:\n#             wave0 = inputs\n#         # compute 4 level of wavelet\n#         wave1 = WaveTFFactory.build(wave_kern)(wave0)\n#         # compute new wavelet features from LL componenents\n#         wave2 = WaveTFFactory.build(wave_kern)(wave1[:,:,:,:chans]) \n#         wave3 = WaveTFFactory.build(wave_kern)(wave2[:,:,:,:chans])\n#         wave4 = WaveTFFactory.build(wave_kern)(wave3[:,:,:,:chans])\n#         # normalize\n#         waves = [wave1, wave2, wave3, wave4]\n#         for l in waves :\n#             l = BatchNormalization()(l)\n#     else :\n#         wave1 = wave2 = wave3 = wave4 = None\n\n#     kinit ='glorot_normal' # 'he_normal'\n\n#     def rep_conv(cnn, scale = 1) :\n#         for i in range(convrep) :\n#             cnn = Conv2D(scale * bl, ks, activation = 'relu', padding = 'same',\n#                          kernel_initializer = kinit)(cnn)\n# #             cnn= MaxPooling2D(3, strides=2, padding=\"same\")(cnn)\n#         return cnn\n\n#     def pool_down(cnn, mul):\n#         cnn = Conv2D(mul * bl, ks, activation = 'relu', padding = 'same',\n#                      kernel_initializer = kinit, strides=(2, 2))(cnn)\n#         return (cnn)\n#     def max_poolingX(cnn, scale = 1):\n#         cnn=cnn, MaxPooling2D(2, strides=2, padding=\"same\")(cnn)\n#         return cnn\n#     cnn = inputs\n#     cnn = rep_conv(cnn, 1)\n# #     cnn = max_poolingX(cnn, 1)\n#     #cnn= MaxPooling2D(3, strides=2, padding=\"same\")(cnn)\n#     for l in range(4) :\n#         cnn = pool_down(cnn, 2**(l+1))\n#         cnn = rep_conv(cnn, 2**(l+1))\n#         if (wavelet):\n#              cnn = Concatenate(axis=3)([cnn, waves[l]])\n#             #cnn = Concatenate(axis=3)([waves[l],cnn ])\n\n#     # output\n#     cnn = Conv2D(2048, ks)(cnn)\n#     cnn = Activation('relu')(cnn)\n#     cnn = GlobalAveragePooling2D()(cnn)\n# #     outputs = Dense(num_classes, activation='softmax')(cnn)\n#     outputs = Dense(1, activation='sigmoid')(cnn)\n# #     outputs = Dense(1, activation='tanh')(cnn)\n#     model = Model(inputs = inputs, outputs = outputs)\n#     return model","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.168457Z","iopub.execute_input":"2022-12-23T10:50:33.168702Z","iopub.status.idle":"2022-12-23T10:50:33.181099Z","shell.execute_reply.started":"2022-12-23T10:50:33.168677Z","shell.execute_reply":"2022-12-23T10:50:33.180229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def wavelet_cnn(input_shape, ks=2, baselev=4, wavelet=True,\n#                 wave_kern='haar', hsv=True, convrep=4, num_classes=2):\n#     inputs = Input(input_shape)\n#     inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n# #     chans = input_shape[2] # number of channels, e.g., 3 if RGB\n#     chans =2\n#     bl = baselev\n\n#     # wavelet computation\n#     if (wavelet) :\n#         # convert RGB to HSV?\n#         if (hsv):\n#             wave0 = Lambda(lambda x: tf.image.rgb_to_hsv(x))(inputs)\n#         else:\n#             wave0 = inputs\n#         # compute 4 level of wavelet\n#         wave1 = WaveTFFactory.build(wave_kern)(wave0)\n#         # compute new wavelet features from LL componenents\n#         wave2 = WaveTFFactory.build(wave_kern)(wave1[:,:,:,:chans]) \n#         wave3 = WaveTFFactory.build(wave_kern)(wave2[:,:,:,:chans])\n#         wave4 = WaveTFFactory.build(wave_kern)(wave3[:,:,:,:chans])\n#         # normalize\n#         waves = [wave1, wave2, wave3, wave4]\n#         for l in waves :\n#             l = BatchNormalization()(l)\n#     else :\n#         wave1 = wave2 = wave3 = wave4 = None\n\n#     kinit ='glorot_normal' # 'he_normal'\n\n#     def rep_conv(cnn, scale = 1) :\n#         for i in range(convrep) :\n#             cnn = Conv2D(scale * bl, ks, activation = 'relu', padding = 'same',\n#                          kernel_initializer = kinit)(cnn)\n# #             cnn= MaxPooling2D(3, strides=2, padding=\"same\")(cnn)\n#         return cnn\n\n#     def pool_down(cnn, mul):\n#         cnn = Conv2D(mul * bl, ks, activation = 'relu', padding = 'same',\n#                      kernel_initializer = kinit, strides=(2, 2))(cnn)\n#         return (cnn)\n#     def max_poolingX(cnn, scale = 1):\n#         cnn=cnn, MaxPooling2D(2, strides=2, padding=\"same\")(cnn)\n#         return cnn\n#     cnn = inputs\n#     cnn = rep_conv(cnn, 1)\n# #     cnn = max_poolingX(cnn, 1)\n#     #cnn= MaxPooling2D(3, strides=2, padding=\"same\")(cnn)\n#     for l in range(4) :\n#         cnn = pool_down(cnn, 2**(l+1))\n#         cnn = rep_conv(cnn, 2**(l+1))\n#         if (wavelet):\n#              cnn = Concatenate(axis=3)([cnn, waves[l]])\n#             #cnn = Concatenate(axis=3)([waves[l],cnn ])\n\n#     # output\n#     cnn = Conv2D(2048, ks)(cnn)\n#     cnn = Activation('relu')(cnn)\n#     cnn = GlobalAveragePooling2D()(cnn)\n# #     outputs = Dense(num_classes, activation='softmax')(cnn)\n#     in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n#     base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     x = in_conv(inp)\n#     x = base(x)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     final=Concatenate(axis=3)([cnn, x])\n    \n    \n#     final = tf.keras.layers.Dense(1,activation='sigmoid')(final)\n    \n    \n#     model = tf.keras.Model(inputs=inp, outputs=final)\n\n\n\n\n\n# #     outputs = Dense(1, activation='sigmoid')(cnn)\n# #     outputs = Dense(1, activation='tanh')(cnn)\n# #     model = Model(inputs = inputs, outputs = outputs)\n#     return model\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:33.182679Z","iopub.execute_input":"2022-12-23T10:50:33.182927Z","iopub.status.idle":"2022-12-23T10:50:33.199411Z","shell.execute_reply.started":"2022-12-23T10:50:33.182895Z","shell.execute_reply":"2022-12-23T10:50:33.198633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\nwith strategy.scope():\n     model=wavelet_cnn([360,128,2], wave_kern='haar')\n#    model1=wavelet_cnn([360,360,3])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:55:12.733112Z","iopub.execute_input":"2022-12-23T10:55:12.733427Z","iopub.status.idle":"2022-12-23T10:55:14.3965Z","shell.execute_reply.started":"2022-12-23T10:55:12.73339Z","shell.execute_reply":"2022-12-23T10:55:14.395698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, to_file=\"my_model.png\", show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:55:14.451423Z","iopub.execute_input":"2022-12-23T10:55:14.452091Z","iopub.status.idle":"2022-12-23T10:55:14.794515Z","shell.execute_reply.started":"2022-12-23T10:55:14.452049Z","shell.execute_reply":"2022-12-23T10:55:14.793547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n        optimizer=optimizers.Adam(learning_rate=cfg.lr),\n#         loss=losses.SquaredHinge(), \n        loss=losses.BinaryCrossentropy(), \n#         loss=losses.CategoricalCrossentropy(),     \n        metrics=['auc'])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.431081Z","iopub.execute_input":"2022-12-23T10:50:37.431427Z","iopub.status.idle":"2022-12-23T10:50:37.4574Z","shell.execute_reply.started":"2022-12-23T10:50:37.43138Z","shell.execute_reply":"2022-12-23T10:50:37.456614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VERBOSE = 2 if tpu else 1\n# DISPLAY_PLOT = True\n# # SAVE BEST MODEL EACH FOLD\n# sv = tf.keras.callbacks.ModelCheckpoint(\n#     'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n#     save_weights_only=True, mode='min', save_freq='epoch')\n\n# es =tf.keras.callbacks.EarlyStopping(\n#     monitor='val_loss',\n#     min_delta=0,\n#     patience=5,\n#     verbose=1,\n#     mode='auto',\n#     )\n\n\n# history = model.fit(\n#         get_dataset(files_train, augment=image_augumentation, shuffle=True, repeat=True,\n#                 batch_size=BATCH_SIZE), \n#         epochs=EPOCH, callbacks = [sv,get_lr_callback()], \n#         steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n#         validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n#                 repeat=False),\n#         verbose=VERBOSE\n#     )\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.458944Z","iopub.execute_input":"2022-12-23T10:50:37.459572Z","iopub.status.idle":"2022-12-23T10:50:37.464628Z","shell.execute_reply.started":"2022-12-23T10:50:37.459511Z","shell.execute_reply":"2022-12-23T10:50:37.46341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# history = model.fit(\n#     train_ds, \n#     epochs=cfg.epochs,\n#     validation_data=val_ds,\n#     callbacks=get_callbacks(),\n#     verbose=2\n# )","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.466442Z","iopub.execute_input":"2022-12-23T10:50:37.466812Z","iopub.status.idle":"2022-12-23T10:50:37.479589Z","shell.execute_reply.started":"2022-12-23T10:50:37.466773Z","shell.execute_reply":"2022-12-23T10:50:37.478744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def build_model():\n#     inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n#     in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n#     base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     x = in_conv(inp)\n#     x = base(x)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n#     model = tf.keras.Model(inputs=inp, outputs=x)\n#     opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n#     loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5) \n#     model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n#     return model","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.481473Z","iopub.execute_input":"2022-12-23T10:50:37.481839Z","iopub.status.idle":"2022-12-23T10:50:37.490921Z","shell.execute_reply.started":"2022-12-23T10:50:37.481777Z","shell.execute_reply":"2022-12-23T10:50:37.489995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 5e-5\n    lr_max     = 5e-4\n    lr_min     = 1e-5\n    lr_ramp_ep = 4\n    lr_sus_ep  = 4\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.492121Z","iopub.execute_input":"2022-12-23T10:50:37.494438Z","iopub.status.idle":"2022-12-23T10:50:37.50241Z","shell.execute_reply.started":"2022-12-23T10:50:37.494399Z","shell.execute_reply":"2022-12-23T10:50:37.501511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2 if tpu else 1\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=False)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model.compile(\n        optimizer=optimizers.Adam(learning_rate=cfg.lr),\n#         loss=losses.SquaredHinge(), \n        loss=losses.BinaryCrossentropy(), \n#         loss=losses.CategoricalCrossentropy(),     \n        metrics=['AUC'])\n#         model = build_model()\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    \n    es =tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=5,\n        verbose=1,\n        mode='auto',\n        )\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=image_augumentation, shuffle=True, repeat=True,\n                batch_size=BATCH_SIZE), \n        epochs=EPOCH, callbacks = [sv,get_lr_callback()], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False),\n        verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF ...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_valid = count_data_items(files_valid); STEPS = ct_valid/BATCH_SIZE/4\n    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n   \n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, \n            labeled=True, return_image_names=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False,\n                labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) / FOLDS\n    \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCH),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCH),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCH),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCH),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i'%(fold+1),size=18)\n        plt.legend(loc=3)\n        plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:50:37.508973Z","iopub.execute_input":"2022-12-23T10:50:37.509373Z","iopub.status.idle":"2022-12-23T10:53:37.05002Z","shell.execute_reply.started":"2022-12-23T10:50:37.50934Z","shell.execute_reply":"2022-12-23T10:53:37.048703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate OOF AUC","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_tar)\nnames = np.concatenate(oof_names)\nfolds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.4f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(Id = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:53:37.051558Z","iopub.status.idle":"2022-12-23T10:53:37.052229Z","shell.execute_reply.started":"2022-12-23T10:53:37.051955Z","shell.execute_reply":"2022-12-23T10:53:37.051984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selectie=df_oof[df_oof.Id.apply(lambda x: len(str(x))<=10)]\nselectie['diference']=df_oof.target-df_oof.pred\nselectie.describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T10:53:37.053541Z","iopub.status.idle":"2022-12-23T10:53:37.054222Z","shell.execute_reply.started":"2022-12-23T10:53:37.053954Z","shell.execute_reply":"2022-12-23T10:53:37.053984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_oof.sample(100).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T17:53:50.157373Z","iopub.status.idle":"2022-12-22T17:53:50.157732Z","shell.execute_reply.started":"2022-12-22T17:53:50.157551Z","shell.execute_reply":"2022-12-22T17:53:50.157567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submit","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T17:53:50.15859Z","iopub.status.idle":"2022-12-22T17:53:50.158913Z","shell.execute_reply.started":"2022-12-22T17:53:50.158755Z","shell.execute_reply":"2022-12-22T17:53:50.15877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = preds[:,0]\nsubmission = submission.sort_values('id') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T17:53:50.159585Z","iopub.status.idle":"2022-12-22T17:53:50.159935Z","shell.execute_reply.started":"2022-12-22T17:53:50.159768Z","shell.execute_reply":"2022-12-22T17:53:50.159785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T17:53:50.161668Z","iopub.status.idle":"2022-12-22T17:53:50.162277Z","shell.execute_reply.started":"2022-12-22T17:53:50.162108Z","shell.execute_reply":"2022-12-22T17:53:50.162126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.zeros((count_data_items(files_test),1))\nlen(preds)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T17:53:50.163171Z","iopub.status.idle":"2022-12-22T17:53:50.163787Z","shell.execute_reply.started":"2022-12-22T17:53:50.163569Z","shell.execute_reply":"2022-12-22T17:53:50.163586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}