{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2N TF-Keras Traing Phase With TPU","metadata":{}},{"cell_type":"markdown","source":"## References, see also them\n\n## [Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n## [CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n## [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n## [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)","metadata":{}},{"cell_type":"markdown","source":"## Initialize Environment","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:25:50.912426Z","iopub.execute_input":"2022-12-30T23:25:50.912989Z","iopub.status.idle":"2022-12-30T23:26:01.799879Z","shell.execute_reply.started":"2022-12-30T23:25:50.912889Z","shell.execute_reply":"2022-12-30T23:26:01.79858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\ntf.__version__","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:01.803114Z","iopub.execute_input":"2022-12-30T23:26:01.803801Z","iopub.status.idle":"2022-12-30T23:26:07.671021Z","shell.execute_reply.started":"2022-12-30T23:26:01.803755Z","shell.execute_reply":"2022-12-30T23:26:07.670015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\nDEVICE = \"GPU\" #or \"GPU\"\n\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\nFOLDS = 5\nIMG_SIZE = [360,128]\n\nBATCH_SIZE = 8\nEPOCH = 6\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:07.672664Z","iopub.execute_input":"2022-12-30T23:26:07.673345Z","iopub.status.idle":"2022-12-30T23:26:07.679622Z","shell.execute_reply.started":"2022-12-30T23:26:07.673301Z","shell.execute_reply":"2022-12-30T23:26:07.677695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        #policy = mixed_precision.Policy('mixed_bfloat16')\n        #mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:07.682346Z","iopub.execute_input":"2022-12-30T23:26:07.683053Z","iopub.status.idle":"2022-12-30T23:26:07.700711Z","shell.execute_reply.started":"2022-12-30T23:26:07.683015Z","shell.execute_reply":"2022-12-30T23:26:07.699651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE *= strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:07.702402Z","iopub.execute_input":"2022-12-30T23:26:07.703034Z","iopub.status.idle":"2022-12-30T23:26:07.708275Z","shell.execute_reply.started":"2022-12-30T23:26:07.702996Z","shell.execute_reply":"2022-12-30T23:26:07.70695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"GCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('traintfrecg2netdataset1')\nfiles_train = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/train*.tfrec')\nfiles_test  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:07.71012Z","iopub.execute_input":"2022-12-30T23:26:07.710929Z","iopub.status.idle":"2022-12-30T23:26:08.927862Z","shell.execute_reply.started":"2022-12-30T23:26:07.710887Z","shell.execute_reply":"2022-12-30T23:26:08.926829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_train","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:08.931363Z","iopub.execute_input":"2022-12-30T23:26:08.931657Z","iopub.status.idle":"2022-12-30T23:26:08.938201Z","shell.execute_reply.started":"2022-12-30T23:26:08.931629Z","shell.execute_reply":"2022-12-30T23:26:08.937061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"def freq_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2])\n    maximum=tf.reduce_mean(image)\n    two =tf.random.uniform([yb-ya,xb-xa,2],maximum*0.6,maximum*1.1,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:08.939914Z","iopub.execute_input":"2022-12-30T23:26:08.940672Z","iopub.status.idle":"2022-12-30T23:26:08.952805Z","shell.execute_reply.started":"2022-12-30T23:26:08.940634Z","shell.execute_reply":"2022-12-30T23:26:08.951801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2]) \n    maximum=tf.reduce_mean(image)\n    two =tf.random.uniform([yb-ya,xb-xa,2],maximum*0.6,maximum*1.1,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:08.954244Z","iopub.execute_input":"2022-12-30T23:26:08.954716Z","iopub.status.idle":"2022-12-30T23:26:08.967747Z","shell.execute_reply.started":"2022-12-30T23:26:08.954678Z","shell.execute_reply":"2022-12-30T23:26:08.966746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask2(image, DIM=IMG_SIZE, PROBABILITY = 0.33, SZ = 0.05):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n#     two = tf.zeros([yb-ya,xb-xa,2],dtype=tf.dtypes.float64)\n \n    two =tf.random.uniform([yb-ya,xb-xa,2],1e-25,1e-24,dtype=tf.dtypes.float64)\n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:08.97227Z","iopub.execute_input":"2022-12-30T23:26:08.972609Z","iopub.status.idle":"2022-12-30T23:26:08.984003Z","shell.execute_reply.started":"2022-12-30T23:26:08.972566Z","shell.execute_reply":"2022-12-30T23:26:08.982717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_mask(image, DIM=IMG_SIZE, X_MNSZ = 0.03, Y_MNSZ = 0.1, X_MXSZ = 0.05, Y_MXSZ = 0.2, MIN_MASK = 6, MAX_MASK = 10):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    mask_num = np.random.randint(MIN_MASK, high = (MAX_MASK + 1))\n    \n    for _ in range(mask_num):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([], 0, DIM[0]),tf.int32)\n        y = tf.cast( tf.random.uniform([], 0, DIM[1]),tf.int32)\n        # CHOOSE RANDOM RECTANGLE SIZE\n        dx = tf.cast( tf.random.uniform([], X_MNSZ, X_MXSZ) * DIM[0],tf.int32)\n        dy = tf.cast( tf.random.uniform([], Y_MNSZ, Y_MXSZ) * DIM[1],tf.int32)\n        # COMPUTE RECTANGLE\n        xb = tf.math.minimum(DIM[0],x + dx)\n        yb = tf.math.minimum(DIM[1],y + dy)\n        # DROPOUT IMAGE\n        one = image[x:xb,0:y,:]\n#         two = tf.zeros([xb-x,yb-y,2]) \n        maximum=tf.reduce_mean(image)\n#         two =tf.random.uniform([xb-x,yb-y,2],maximum*0.6,maximum*1.1,dtype=tf.dtypes.float64)\n        two =tf.zeros([xb-x,yb-y,2],dtype=tf.dtypes.float64)\n        three = image[x:xb,yb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:x,:,:],middle,image[xb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:08.985582Z","iopub.execute_input":"2022-12-30T23:26:08.986642Z","iopub.status.idle":"2022-12-30T23:26:08.999021Z","shell.execute_reply.started":"2022-12-30T23:26:08.986606Z","shell.execute_reply":"2022-12-30T23:26:08.997981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_shuffle(image, DIM=IMG_SIZE, PROBABILITY = 0.5):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0): return image\n    \n\n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x)\n    xb = DIM[1]\n   \n    image = tf.concat([image[:,xa:DIM[1],:],image[:,0:xa,:]],axis=1)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.00043Z","iopub.execute_input":"2022-12-30T23:26:09.000969Z","iopub.status.idle":"2022-12-30T23:26:09.012715Z","shell.execute_reply.started":"2022-12-30T23:26:09.000933Z","shell.execute_reply":"2022-12-30T23:26:09.011743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation(img):\n#     img=img\n    \n    img = freq_mask(img)\n    img = time_mask(img)\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = time_mask2(img,SZ = 0.01)\n    img = time_mask2(img,SZ = 0.02)\n    img = time_mask2(img,SZ = 0.06)\n    img = time_mask2(img,SZ = 0.04)\n    img = time_mask2(img,SZ = 0.053)\n    img = time_mask2(img,SZ = 0.011)\n    img = time_mask2(img,SZ = 0.015)\n    img = time_mask2(img,SZ = 0.025)\n    img = time_mask2(img,SZ = 0.005)\n    img = time_shuffle(img, PROBABILITY = 0.75)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.014028Z","iopub.execute_input":"2022-12-30T23:26:09.014592Z","iopub.status.idle":"2022-12-30T23:26:09.025742Z","shell.execute_reply.started":"2022-12-30T23:26:09.014556Z","shell.execute_reply":"2022-12-30T23:26:09.024703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float64)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    #print(example['spectrogram'].shape)\n    hr, lr = tf.split(example['spectrogram'], 2, axis=-1)\n    X = {'x_h_r': hr, 'x_l_r': lr}\n    return X, example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'id'                   : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float64)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    hr, lr = tf.split(example['spectrogram'], 2, axis=-1)\n    X = {'x_h_r': hr, 'x_l_r': lr}\n    return X, example['id'] if return_image_name else 0\n\ndef read_pretrain_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float64)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    hr, lr = tf.split(random_mask(example['spectrogram']), 2, axis=-1)\n    X = {'x_h_r': hr, 'x_l_r': lr}\n    y = example['spectrogram']\n    return X, y\n\ndef prepare_image(img, augment=True):    \n    if augment:\n        img = augmentation(img)\n        \n    img = tf.reshape(img, [*IMG_SIZE, 2])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.027467Z","iopub.execute_input":"2022-12-30T23:26:09.027894Z","iopub.status.idle":"2022-12-30T23:26:09.044348Z","shell.execute_reply.started":"2022-12-30T23:26:09.027859Z","shell.execute_reply":"2022-12-30T23:26:09.043407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, pretrain = False, batch_size=16):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8 if tpu else 128)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if pretrain:\n        ds = ds.map(read_pretrain_tfrecord, num_parallel_calls=AUTO)\n    else:\n        if labeled: \n            ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)   \n        else:\n            ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                        num_parallel_calls=AUTO)      \n\n#         ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment), imgname_or_label), \n#                                                    num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.045749Z","iopub.execute_input":"2022-12-30T23:26:09.046173Z","iopub.status.idle":"2022-12-30T23:26:09.056057Z","shell.execute_reply.started":"2022-12-30T23:26:09.046116Z","shell.execute_reply":"2022-12-30T23:26:09.055079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!for a in /sys/bus/pci/devices/*; do echo 0 | sudo tee -a $a/numa_node; done","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.057361Z","iopub.execute_input":"2022-12-30T23:26:09.058019Z","iopub.status.idle":"2022-12-30T23:26:09.068145Z","shell.execute_reply.started":"2022-12-30T23:26:09.057979Z","shell.execute_reply":"2022-12-30T23:26:09.067113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FNAME = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train*.tfrec'])\nrow = 16; col = 2;\nrow = min(row,16//col)\n\nall_elements = get_dataset(FNAME, pretrain = True, batch_size=32).unbatch()\naugmented_element = all_elements.repeat().batch(32)\n\nfor (img,mask) in augmented_element:\n    img = tf.concat([img['x_h_r'], img['x_l_r']], axis = -1)\n    img = (img-np.min(img))/(np.max(img)-np.min(img)+1e-5)\n    plt.figure(figsize=(15,int(15*row/col)))\n\n    i=0\n    j=1\n    while j<=row*col:\n        plt.subplot(row,col,j)\n        plt.axis('on')\n\n        plt.imshow(img[i,:,:,0])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(img[i,:,:,1])\n        j += 1\n\n        i += 1\n\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:09.069631Z","iopub.execute_input":"2022-12-30T23:26:09.070035Z","iopub.status.idle":"2022-12-30T23:26:18.138269Z","shell.execute_reply.started":"2022-12-30T23:26:09.069996Z","shell.execute_reply":"2022-12-30T23:26:18.137191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Patches(tf.keras.layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n\n    def call(self, images):\n        batch_size = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images=images,\n            sizes=[1, self.patch_size, 1, 1],\n            strides=[1, self.patch_size, 1, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        return patches\n\nclass PatchEncoder(tf.keras.layers.Layer):\n    def __init__(self, num_patches, projection_dim):\n        super(PatchEncoder, self).__init__()\n        self.num_patches = num_patches\n        self.projection = tf.keras.layers.Dense(units=projection_dim)\n        self.position_embedding = tf.keras.layers.Embedding(\n            input_dim=num_patches, output_dim=projection_dim\n        )\n\n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.num_patches, delta=1)\n        encoded = self.projection(patch) + self.position_embedding(positions)\n        return encoded","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:26:18.141737Z","iopub.execute_input":"2022-12-30T23:26:18.142198Z","iopub.status.idle":"2022-12-30T23:26:18.152088Z","shell.execute_reply.started":"2022-12-30T23:26:18.142151Z","shell.execute_reply":"2022-12-30T23:26:18.150865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_heads = 4\nprojection_dim = IMG_SIZE[0]\ntransformer_layers = 8\ntransformer_units = [\n    projection_dim,\n    projection_dim,\n]\n\ndef mlp(x, vitname, i, hidden_units, dropout_rate):\n    for index, units in enumerate(hidden_units):\n        x = tf.keras.layers.Dense(units, activation=tf.nn.gelu, name = vitname + '_MLP_' + str(i) + '_' + str(index))(x)\n        x = tf.keras.layers.Dropout(dropout_rate)(x)\n    return x\n\ndef create_vit_classifier(x_input, vitname):\n#     inputs = tf.tile(x_input, [1,1,1,3])\n#     print('ouo', inputs.shape)\n    # Augment data. \n    # Create patches.\n    patches = Patches(IMG_SIZE[0])(x_input)\n    # Encode patches.\n    encoded_patches = PatchEncoder(IMG_SIZE[1], projection_dim)(patches)\n\n    # Create multiple layers of the Transformer block.\n    for i in range(transformer_layers):\n        # Layer normalization 1.\n        x1 = tf.keras.layers.LayerNormalization(epsilon=1e-6, name = vitname + '_LN_1_' + str(i))(encoded_patches)\n        # Create a multi-head attention layer.\n        attention_output = tf.keras.layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection_dim, dropout=0.5, name = vitname + '_MHA_' + str(i)\n        )(x1, x1)\n        # Skip connection 1.\n        x2 = tf.keras.layers.Add()([attention_output, encoded_patches])\n        # Layer normalization 2.\n        x3 = tf.keras.layers.LayerNormalization(epsilon=1e-6, name = vitname + '_LN_2_' + str(i))(x2)\n        # MLP.\n        x3 = mlp(x3, vitname, i, hidden_units=transformer_units, dropout_rate=0.5)\n        # Skip connection 2.\n        encoded_patches = tf.keras.layers.Add()([x3, x2])\n\n    # Create a [batch_size, projection_dim] tensor.\n    representation = tf.keras.layers.LayerNormalization(epsilon=1e-6, name = vitname + '_LN_3')(encoded_patches)\n    representation = tf.keras.layers.Permute((1, 3, 2), input_shape=(1, IMG_SIZE[1], IMG_SIZE[0]))(representation)\n    representation = tf.keras.layers.Reshape((IMG_SIZE[0], IMG_SIZE[1]), input_shape=(1, IMG_SIZE[0], IMG_SIZE[1]))(representation)\n\n    return representation\n\nlayers_name_list = [ 'hr' + '_LN_1_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'hr' + '_MHA_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'hr' + '_LN_2_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'hr' + '_MLP_' + str(i) + '_' + str(projection_dim) for i in range(transformer_layers)] + \\\n                   [ 'hr' + '_MLP_' + str(i) + '_' + str(projection_dim * 2) for i in range(transformer_layers)] + \\\n                   [ 'hr' + '_LN_3'] + \\\n                   [ 'lr' + '_LN_1_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'lr' + '_MHA_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'lr' + '_LN_2_' + str(i) for i in range(transformer_layers)] + \\\n                   [ 'lr' + '_MLP_' + str(i) + '_' + str(projection_dim) for i in range(transformer_layers)] + \\\n                   [ 'lr' + '_MLP_' + str(i) + '_' + str(projection_dim * 2) for i in range(transformer_layers)] + \\\n                   [ 'lr' + '_LN_3']","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:18.139776Z","iopub.execute_input":"2022-12-30T23:49:18.14033Z","iopub.status.idle":"2022-12-30T23:49:18.164462Z","shell.execute_reply.started":"2022-12-30T23:49:18.140279Z","shell.execute_reply":"2022-12-30T23:49:18.163284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"# def build_model():\n#     inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n#     in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n#     base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     x = in_conv(inp)\n#     x = base(x)\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x)\n#     x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n#     model = tf.keras.Model(inputs=inp, outputs=x)\n#     opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n#     loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5) \n#     model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n#     return model\ndef build_pretrain_model():\n    # 1) Hanford Real\n    h_r_input = tf.keras.layers.Input(shape=(*IMG_SIZE,1), dtype=tf.float64, name='x_h_r')\n    # 2) Livingston Real\n    l_r_input = tf.keras.layers.Input(shape=(*IMG_SIZE,1), dtype=tf.float64, name='x_l_r')\n    \n    # Get embedding from vit\n    h_r_embed = create_vit_classifier(h_r_input, vitname = 'hr')\n    l_r_embed = create_vit_classifier(l_r_input, vitname = 'lr')\n    h_r_embed = tf.expand_dims(h_r_embed, axis = -1)\n    l_r_embed = tf.expand_dims(l_r_embed, axis = -1)\n    \n    output = tf.keras.layers.Concatenate()([h_r_embed, l_r_embed])\n    \n    # Model\n    inputs = [h_r_input, l_r_input]\n    model = tf.keras.models.Model(inputs=inputs, outputs=output)\n    \n    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n\n    model.compile(\n        optimizer=optimizer,\n        loss=tf.keras.losses.MeanSquaredError(),\n    )\n    return model\n\ndef build_model():\n    # 1) Hanford Real\n    h_r_input = tf.keras.layers.Input(shape=(*IMG_SIZE,1), dtype=tf.float64, name='x_h_r')\n    # 2) Livingston Real\n    l_r_input = tf.keras.layers.Input(shape=(*IMG_SIZE,1), dtype=tf.float64, name='x_l_r')\n    \n    # Get embedding from vit\n    h_r_embed = create_vit_classifier(h_r_input, vitname = 'hr')\n    l_r_embed = create_vit_classifier(l_r_input, vitname = 'lr')\n    h_r_embed = tf.expand_dims(h_r_embed, axis = -1)\n    l_r_embed = tf.expand_dims(l_r_embed, axis = -1)\n    \n    # Concatenate embeddings\n    inp = tf.keras.layers.Concatenate()([h_r_embed, l_r_embed])\n    in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n    base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n#     base = tf.keras.applications.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet',\n#     classes=2, include_top=False)\n    x = in_conv(inp)\n    x = base(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    # Target prediction in range [0,1] with sigmoid activation\n    output = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n    \n    # Model\n    inputs = [h_r_input, l_r_input]\n    model = tf.keras.models.Model(inputs=inputs, outputs=output)\n    \n    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n\n    model.compile(\n        optimizer=optimizer,\n        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5),\n        metrics = [\n            tf.keras.metrics.AUC(),\n        ]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:20.503393Z","iopub.execute_input":"2022-12-30T23:49:20.504305Z","iopub.status.idle":"2022-12-30T23:49:20.519821Z","shell.execute_reply.started":"2022-12-30T23:49:20.504269Z","shell.execute_reply":"2022-12-30T23:49:20.518693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 5e-5\n    lr_max     = 5e-4\n    lr_min     = 1e-5\n    lr_ramp_ep = 4\n    lr_sus_ep  = 4\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:22.833978Z","iopub.execute_input":"2022-12-30T23:49:22.834707Z","iopub.status.idle":"2022-12-30T23:49:22.842594Z","shell.execute_reply.started":"2022-12-30T23:49:22.834666Z","shell.execute_reply":"2022-12-30T23:49:22.841103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#     # BUILD MODEL\n#     K.clear_session()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:24.17167Z","iopub.execute_input":"2022-12-30T23:49:24.172063Z","iopub.status.idle":"2022-12-30T23:49:24.176835Z","shell.execute_reply.started":"2022-12-30T23:49:24.172029Z","shell.execute_reply":"2022-12-30T23:49:24.175708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"# BUILD MODEL\n#K.clear_session()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:25.296455Z","iopub.execute_input":"2022-12-30T23:49:25.296835Z","iopub.status.idle":"2022-12-30T23:49:25.301322Z","shell.execute_reply.started":"2022-12-30T23:49:25.296803Z","shell.execute_reply":"2022-12-30T23:49:25.300266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2 if tpu else 1\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=False)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\n#pretrain\nwith strategy.scope():\n    pretrain_model = build_pretrain_model()\nfiles_pretrain = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train*.tfrec'])\npsv = tf.keras.callbacks.ModelCheckpoint(\n        'pretrain', monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\nes =tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=5,\n        verbose=1,\n        mode='auto',\n        )","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:27.161021Z","iopub.execute_input":"2022-12-30T23:49:27.161779Z","iopub.status.idle":"2022-12-30T23:49:30.186427Z","shell.execute_reply.started":"2022-12-30T23:49:27.161739Z","shell.execute_reply":"2022-12-30T23:49:30.185229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCH=175","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:33.832799Z","iopub.execute_input":"2022-12-30T23:49:33.837428Z","iopub.status.idle":"2022-12-30T23:49:33.843603Z","shell.execute_reply.started":"2022-12-30T23:49:33.837379Z","shell.execute_reply":"2022-12-30T23:49:33.842593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Pretraining...')\npretrain_history = pretrain_model.fit(\n    get_dataset(files_pretrain, augment=False, shuffle=False, repeat=True,\n                pretrain = True, labeled=False),\n    epochs=int(EPOCH), callbacks = [psv,get_lr_callback()],\n    steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n    verbose = VERBOSE,\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-30T23:49:35.786454Z","iopub.execute_input":"2022-12-30T23:49:35.786835Z","iopub.status.idle":"2022-12-31T00:02:52.995622Z","shell.execute_reply.started":"2022-12-30T23:49:35.786804Z","shell.execute_reply":"2022-12-31T00:02:52.994497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Show a case that image went through unsupervised pretrained block\n\n#pretrain_model.load_weights('pretrain.h5')\nFNAME = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train*.tfrec'])\nrow = 16; col = 2;\nrow = min(row,16//col)\n\nall_elements = get_dataset(FNAME, pretrain = True, batch_size=32).unbatch()\naugmented_element = all_elements.repeat().batch(32)\n\nfor (img,mask) in augmented_element:\n    res = pretrain_model.predict(img)\n    img = tf.concat([img['x_h_r'], img['x_l_r']], axis = -1)\n    img = (img-np.min(img))/(np.max(img)-np.min(img)+1e-5)\n    plt.figure(figsize=(15,int(15*row/col)))\n\n    i=0\n    j=1\n    while j<=row*col:\n        plt.subplot(row,col,j)\n        plt.axis('on')\n\n        plt.imshow(img[i,:,:,0])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(img[i,:,:,1])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(res[i,:,:,0])\n        j += 1\n\n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(res[i,:,:,1])\n        j += 1\n\n        i += 1\n\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2022-12-31T00:02:59.96159Z","iopub.execute_input":"2022-12-31T00:02:59.961986Z","iopub.status.idle":"2022-12-31T00:03:09.754524Z","shell.execute_reply.started":"2022-12-31T00:02:59.961951Z","shell.execute_reply":"2022-12-31T00:03:09.753326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrain_model.save_weights('pretrain.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:45:31.150874Z","iopub.execute_input":"2022-12-30T22:45:31.151188Z","iopub.status.idle":"2022-12-30T22:45:31.577693Z","shell.execute_reply.started":"2022-12-30T22:45:31.151159Z","shell.execute_reply":"2022-12-30T22:45:31.576668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\n# SEED = 42\n\n# FOLDS = 5\n# IMG_SIZE = [360,128]\n\n# BATCH_SIZE = 8\n# EPOCH = 2\n\n# # TEST TIME AUGMENTATION STEPS\n# TTA = 1","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:17:56.569645Z","iopub.execute_input":"2022-12-30T22:17:56.570074Z","iopub.status.idle":"2022-12-30T22:17:56.576714Z","shell.execute_reply.started":"2022-12-30T22:17:56.570007Z","shell.execute_reply":"2022-12-30T22:17:56.575337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #K-Fold training\n# for fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    \n#     # DISPLAY FOLD INFO\n#     if DEVICE=='TPU':\n#         if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n#     print('#'*25); print('#### FOLD',fold+1)\n#     a`","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:17:56.582312Z","iopub.execute_input":"2022-12-30T22:17:56.583183Z","iopub.status.idle":"2022-12-30T22:17:56.681386Z","shell.execute_reply.started":"2022-12-30T22:17:56.583141Z","shell.execute_reply":"2022-12-30T22:17:56.679854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#    #K-Fold training\n# for fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    \n#     # DISPLAY FOLD INFO\n#     if DEVICE=='TPU':\n#         if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n#     print('#'*25); print('#### FOLD',fold+1)\n    \n#     # CREATE TRAIN AND VALIDATION SUBSETS\n#     files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n#     files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n#     files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\n    \n#     # BUILD MODEL\n#     K.clear_session()\n    \n#     with strategy.scope():\n#         model = build_model()\n        \n#     # SAVE BEST MODEL EACH FOLD\n#     sv = tf.keras.callbacks.ModelCheckpoint(\n#         'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n#         save_weights_only=True, mode='min', save_freq='epoch')\n    \n#     es =tf.keras.callbacks.EarlyStopping(\n#         monitor='val_loss',\n#         min_delta=0,\n#         patience=5,\n#         verbose=1,\n#         mode='auto',\n#         )\n   \n#     # TRAIN \n#     print('Training...')\n#     model.load_weights('pretrain.h5', by_name = True, skip_mismatch = True)\n#     history = model.fit(\n#         get_dataset(files_train, augment=False, shuffle=False, repeat=True,\n#                 batch_size=BATCH_SIZE), \n#         epochs=EPOCH, callbacks = [sv,get_lr_callback()], \n#         steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n#         validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n#                 repeat=False),\n#         verbose=VERBOSE\n#     )\n    \n#     print('Loading best model...')\n#     model.load_weights('fold-%i.h5'%fold)\n    \n#     # PREDICT OOF USING TTA\n#     print('Predicting OOF ...')\n#     ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n#     ct_valid = count_data_items(files_valid); STEPS = ct_valid/BATCH_SIZE/4\n#     pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:ct_valid,] \n#     oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n   \n    \n#     # GET OOF TARGETS AND NAMES\n#     ds_valid = get_dataset(files_valid, augment=False, repeat=False, \n#             labeled=True, return_image_names=True)\n#     oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n#     oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n#     ds = get_dataset(files_valid, augment=False, repeat=False,\n#                 labeled=False, return_image_names=True)\n#     oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n#     # PREDICT TEST USING TTA\n#     print('Predicting Test')\n#     ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n#             repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n#     ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n#     pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n#     preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) / FOLDS\n    \n#     # REPORT RESULTS\n#     auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n#     oof_val.append(np.max( history.history['val_auc'] ))\n#     print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n#     # PLOT TRAINING\n#     if DISPLAY_PLOT:\n#         plt.figure(figsize=(15,5))\n#         plt.plot(np.arange(EPOCH),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n#         plt.plot(np.arange(EPOCH),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n#         x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n#         xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n#         plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n#         plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n#         plt.legend(loc=2)\n#         plt2 = plt.gca().twinx()\n#         plt2.plot(np.arange(EPOCH),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n#         plt2.plot(np.arange(EPOCH),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n#         x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n#         ydist = plt.ylim()[1] - plt.ylim()[0]\n#         plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n#         plt.ylabel('Loss',size=14)\n#         plt.title('FOLD %i'%(fold+1),size=18)\n#         plt.legend(loc=3)\n#         plt.show()  \n#         plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:17:56.683545Z","iopub.execute_input":"2022-12-30T22:17:56.683872Z","iopub.status.idle":"2022-12-30T22:31:26.011151Z","shell.execute_reply.started":"2022-12-30T22:17:56.683844Z","shell.execute_reply":"2022-12-30T22:31:26.007703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate OOF AUC","metadata":{}},{"cell_type":"code","source":"# # COMPUTE OVERALL OOF AUC\n# oof = np.concatenate(oof_pred)\n# true = np.concatenate(oof_tar)\n# names = np.concatenate(oof_names)\n# folds = np.concatenate(oof_folds)\n# auc = roc_auc_score(true,oof)\n# print('Overall OOF AUC with TTA = %.4f'%auc)\n\n# # SAVE OOF TO DISK\n# df_oof = pd.DataFrame(dict(Id = names, target=true, pred = oof, fold=folds))\n# df_oof.to_csv('oof.csv',index=False)\n# df_oof.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:31:26.012683Z","iopub.status.idle":"2022-12-30T22:31:26.013213Z","shell.execute_reply.started":"2022-12-30T22:31:26.012935Z","shell.execute_reply":"2022-12-30T22:31:26.01296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submit","metadata":{}},{"cell_type":"code","source":"# submission = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\n# submission['target'] = preds[:,0]\n# submission = submission.sort_values('id') \n# submission.to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:31:26.015067Z","iopub.status.idle":"2022-12-30T22:31:26.015576Z","shell.execute_reply.started":"2022-12-30T22:31:26.01531Z","shell.execute_reply":"2022-12-30T22:31:26.015335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.hist(submission.target,bins=100)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-30T22:31:26.017306Z","iopub.status.idle":"2022-12-30T22:31:26.017841Z","shell.execute_reply.started":"2022-12-30T22:31:26.017544Z","shell.execute_reply":"2022-12-30T22:31:26.017577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}