{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2N TF-Keras Traing Phase With TPU","metadata":{}},{"cell_type":"markdown","source":"## References, see also them\n\n## [Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n## [CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n## [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n## [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)","metadata":{}},{"cell_type":"markdown","source":"## Initialize Environment","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:03:45.820232Z","iopub.execute_input":"2022-10-28T12:03:45.821019Z","iopub.status.idle":"2022-10-28T12:04:01.791099Z","shell.execute_reply.started":"2022-10-28T12:03:45.82092Z","shell.execute_reply":"2022-10-28T12:04:01.789191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\n\ntf.__version__","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:01.793877Z","iopub.execute_input":"2022-10-28T12:04:01.794281Z","iopub.status.idle":"2022-10-28T12:04:10.637073Z","shell.execute_reply.started":"2022-10-28T12:04:01.794244Z","shell.execute_reply":"2022-10-28T12:04:10.63558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\nFOLDS = 5\nIMG_SIZE = [384,512]\n\nBATCH_SIZE = 30\nEPOCH = 75\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 1","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:10.638417Z","iopub.execute_input":"2022-10-28T12:04:10.639077Z","iopub.status.idle":"2022-10-28T12:04:10.645655Z","shell.execute_reply.started":"2022-10-28T12:04:10.639035Z","shell.execute_reply":"2022-10-28T12:04:10.643811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to get hardware strategy\ndef get_hardware_strategy():\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        #policy = mixed_precision.Policy('mixed_bfloat16')\n        #mixed_precision.set_global_policy(policy)\n        tf.config.optimizer.set_jit(True)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return tpu, strategy\n\ntpu, strategy = get_hardware_strategy()","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:10.649163Z","iopub.execute_input":"2022-10-28T12:04:10.650011Z","iopub.status.idle":"2022-10-28T12:04:10.683785Z","shell.execute_reply.started":"2022-10-28T12:04:10.649961Z","shell.execute_reply":"2022-10-28T12:04:10.682443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE *= strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:10.685688Z","iopub.execute_input":"2022-10-28T12:04:10.686191Z","iopub.status.idle":"2022-10-28T12:04:10.691854Z","shell.execute_reply.started":"2022-10-28T12:04:10.686121Z","shell.execute_reply":"2022-10-28T12:04:10.690638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"GCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('g2net-tfr-spectrogram-datasets')\nfiles_train = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/train*.tfrec')\nfiles_test  = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:10.693574Z","iopub.execute_input":"2022-10-28T12:04:10.693986Z","iopub.status.idle":"2022-10-28T12:04:13.681497Z","shell.execute_reply.started":"2022-10-28T12:04:10.693952Z","shell.execute_reply":"2022-10-28T12:04:13.680231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation","metadata":{}},{"cell_type":"code","source":"def freq_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[0],tf.int32) * P\n    ya = tf.math.maximum(0,y-WIDTH//2)\n    yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n    xa = 0\n    xb = DIM[1]\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,2]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0],DIM[1],2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:01.630068Z","iopub.execute_input":"2022-10-28T12:05:01.630521Z","iopub.status.idle":"2022-10-28T12:05:01.643511Z","shell.execute_reply.started":"2022-10-28T12:05:01.630488Z","shell.execute_reply":"2022-10-28T12:05:01.642042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask(image, DIM=IMG_SIZE, PROBABILITY = 0.66, SZ = 0.2):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(SZ==0): return image\n    \n    SZ = SZ * tf.random.uniform([],minval=0.25, maxval=1, dtype='float32')\n    \n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32) \n    # COMPUTE SQUARE \n    WIDTH = tf.cast( SZ*DIM[1],tf.int32) * P\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x-WIDTH//2)\n    xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n    # DROPOUT IMAGE\n    one = image[ya:yb,0:xa,:]\n    two = tf.zeros([yb-ya,xb-xa,2]) \n    three = image[ya:yb,xb:DIM[1],:]\n    middle = tf.concat([one,two,three],axis=1)\n    image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.120802Z","iopub.execute_input":"2022-10-28T12:05:02.12124Z","iopub.status.idle":"2022-10-28T12:05:02.133775Z","shell.execute_reply.started":"2022-10-28T12:05:02.121206Z","shell.execute_reply":"2022-10-28T12:05:02.132262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_shuffle(image, DIM=IMG_SIZE, PROBABILITY = 0.5):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0): return image\n    \n\n    # CHOOSE RANDOM LOCATION\n    x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n    y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n\n    ya = 0\n    yb = DIM[0]\n    xa = tf.math.maximum(0,x)\n    xb = DIM[1]\n   \n    image = tf.concat([image[:,xa:DIM[1],:],image[:,0:xa,:]],axis=1)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM[0], DIM[1], 2])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.155999Z","iopub.execute_input":"2022-10-28T12:05:02.156444Z","iopub.status.idle":"2022-10-28T12:05:02.167659Z","shell.execute_reply.started":"2022-10-28T12:05:02.156408Z","shell.execute_reply":"2022-10-28T12:05:02.16631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def augmentation(img):\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = freq_mask(img)\n    img = time_mask(img)\n    img = time_shuffle(img, PROBABILITY = 0.75)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.183149Z","iopub.execute_input":"2022-10-28T12:05:02.183577Z","iopub.status.idle":"2022-10-28T12:05:02.19144Z","shell.execute_reply.started":"2022-10-28T12:05:02.183543Z","shell.execute_reply":"2022-10-28T12:05:02.189535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'target'               : tf.io.FixedLenFeature([], tf.float32),\n        'id'                   : tf.io.FixedLenFeature([], tf.string),\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    #print(example['spectrogram'].shape)\n    return example['spectrogram'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'spectrogram'          : tf.io.FixedLenFeature([], tf.string),\n        'id'                   : tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    example['spectrogram'] = tf.io.parse_tensor(example['spectrogram'],out_type=tf.float32)\n    example['spectrogram'] = tf.reshape(example['spectrogram'], [*IMG_SIZE,2])\n    return example['spectrogram'], example['id'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True):    \n    if augment:\n        img = augmentation(img)\n        \n    img = tf.reshape(img, [*IMG_SIZE, 2])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.217044Z","iopub.execute_input":"2022-10-28T12:05:02.218221Z","iopub.status.idle":"2022-10-28T12:05:02.230968Z","shell.execute_reply.started":"2022-10-28T12:05:02.21818Z","shell.execute_reply":"2022-10-28T12:05:02.22925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=16):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8 if tpu else 128)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment,), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.245601Z","iopub.execute_input":"2022-10-28T12:05:02.246075Z","iopub.status.idle":"2022-10-28T12:05:02.25872Z","shell.execute_reply.started":"2022-10-28T12:05:02.246037Z","shell.execute_reply":"2022-10-28T12:05:02.257298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FNAME = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train*.tfrec'])\nrow = 16; col = 2;\nrow = min(row,16//col)\n\nall_elements = get_dataset(FNAME, augment=True, batch_size=32).unbatch()\naugmented_element = all_elements.repeat().batch(32)\n\nfor (img,mask) in augmented_element:\n\n    img = (img-np.min(img))/(np.max(img)-np.min(img)+1e-5)\n    plt.figure(figsize=(15,int(15*row/col)))\n\n    i=0\n    j=1\n    while j<=row*col:\n        plt.subplot(row,col,j)\n        plt.axis('on')\n\n        plt.imshow(img[i,:,:,0])\n        j += 1\n        \n        plt.subplot(row,col,j)\n        plt.axis('on')\n        plt.imshow(img[i,:,:,1])\n        j += 1\n\n        i += 1\n\n    plt.show()\n    break","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:02.27264Z","iopub.execute_input":"2022-10-28T12:05:02.27352Z","iopub.status.idle":"2022-10-28T12:05:15.566955Z","shell.execute_reply.started":"2022-10-28T12:05:02.273444Z","shell.execute_reply":"2022-10-28T12:05:15.56541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"def build_model():\n    inp = tf.keras.layers.Input(shape=(*IMG_SIZE, 2))\n    in_conv = tf.keras.layers.Conv2D(3, 7, strides=(1, 1), padding='same')\n    base = efn.EfficientNetB7(input_shape=(*IMG_SIZE, 3), weights='imagenet', include_top=False)\n    x = in_conv(inp)\n    x = base(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp, outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-5) \n    model.compile(optimizer=opt, loss=loss, metrics=['AUC'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:15.571778Z","iopub.execute_input":"2022-10-28T12:05:15.572237Z","iopub.status.idle":"2022-10-28T12:05:15.581498Z","shell.execute_reply.started":"2022-10-28T12:05:15.572201Z","shell.execute_reply":"2022-10-28T12:05:15.580519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback():\n    lr_start   = 5e-5\n    lr_max     = 5e-4\n    lr_min     = 1e-5\n    lr_ramp_ep = 4\n    lr_sus_ep  = 4\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:15.582976Z","iopub.execute_input":"2022-10-28T12:05:15.58329Z","iopub.status.idle":"2022-10-28T12:05:15.601043Z","shell.execute_reply.started":"2022-10-28T12:05:15.583262Z","shell.execute_reply":"2022-10-28T12:05:15.600104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2 if tpu else 1\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits=FOLDS,shuffle=False)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(5))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n    files_test = tf.io.gfile.glob(GCS_PATH_STRATIFICATED + '/test*.tfrec')\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    \n    es =tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=5,\n        verbose=1,\n        mode='auto',\n        )\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=True, shuffle=True, repeat=True,\n                batch_size=BATCH_SIZE), \n        epochs=EPOCH, callbacks = [sv,get_lr_callback()], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZE,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False),\n        verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF ...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_valid = count_data_items(files_valid); STEPS = ct_valid/BATCH_SIZE/4\n    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n   \n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, \n            labeled=True, return_image_names=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False,\n                labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=False,\n            repeat=True,shuffle=False,batch_size=BATCH_SIZE*4)\n    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZE/4\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) / FOLDS\n    \n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCH),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCH),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCH),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCH),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i'%(fold+1),size=18)\n        plt.legend(loc=3)\n        plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:05:15.603639Z","iopub.execute_input":"2022-10-28T12:05:15.604465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate OOF AUC","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred)\ntrue = np.concatenate(oof_tar)\nnames = np.concatenate(oof_names)\nfolds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.4f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(Id = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submit","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv')\nsubmission['target'] = preds[:,0]\nsubmission = submission.sort_values('id') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-28T12:04:15.911333Z","iopub.status.idle":"2022-10-28T12:04:15.911707Z","shell.execute_reply.started":"2022-10-28T12:04:15.911519Z","shell.execute_reply":"2022-10-28T12:04:15.911536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}