{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA 2022 TF-Keras TPU + ARCFace","metadata":{"papermill":{"duration":0.017167,"end_time":"2022-09-15T04:10:34.857392","exception":false,"start_time":"2022-09-15T04:10:34.840225","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Set up environment","metadata":{"papermill":{"duration":0.015452,"end_time":"2022-09-15T04:10:34.889322","exception":false,"start_time":"2022-09-15T04:10:34.87387","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:10:34.929414Z","iopub.status.busy":"2022-09-15T04:10:34.927829Z","iopub.status.idle":"2022-09-15T04:10:34.932035Z","shell.execute_reply":"2022-09-15T04:10:34.931239Z","shell.execute_reply.started":"2022-09-15T03:25:48.884814Z"},"papermill":{"duration":0.026715,"end_time":"2022-09-15T04:10:34.932209","exception":false,"start_time":"2022-09-15T04:10:34.905494","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U keras-cv-attention-models\n!pip install -q efficientnet\n!pip install tensorflow_addons","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:10:34.967554Z","iopub.status.busy":"2022-09-15T04:10:34.966812Z","iopub.status.idle":"2022-09-15T04:11:02.684457Z","shell.execute_reply":"2022-09-15T04:11:02.683369Z","shell.execute_reply.started":"2022-09-15T03:25:48.910596Z"},"papermill":{"duration":27.736338,"end_time":"2022-09-15T04:11:02.684642","exception":false,"start_time":"2022-09-15T04:10:34.948304","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math, re, os, gc\nfrom keras_cv_attention_models import nfnets, efficientnet,coatnet,davit\nfrom tensorflow.keras import layers\nimport efficientnet.tfkeras as efn\nimport tensorflow as tf\nimport albumentations\nimport tensorflow.keras.backend as K\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:02.735937Z","iopub.status.busy":"2022-09-15T04:11:02.735186Z","iopub.status.idle":"2022-09-15T04:11:11.720378Z","shell.execute_reply":"2022-09-15T04:11:11.719632Z","shell.execute_reply.started":"2022-09-15T03:26:15.498047Z"},"papermill":{"duration":9.013977,"end_time":"2022-09-15T04:11:11.720578","exception":false,"start_time":"2022-09-15T04:11:02.706601","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detect TPU","metadata":{"papermill":{"duration":0.021819,"end_time":"2022-09-15T04:11:11.765268","exception":false,"start_time":"2022-09-15T04:11:11.743449","status":"completed"},"tags":[]}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:11.836903Z","iopub.status.busy":"2022-09-15T04:11:11.836205Z","iopub.status.idle":"2022-09-15T04:11:17.566307Z","shell.execute_reply":"2022-09-15T04:11:17.566817Z","shell.execute_reply.started":"2022-09-15T03:26:25.375322Z"},"papermill":{"duration":5.778209,"end_time":"2022-09-15T04:11:17.567016","exception":false,"start_time":"2022-09-15T04:11:11.788807","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up variables","metadata":{"papermill":{"duration":0.02224,"end_time":"2022-09-15T04:11:17.611913","exception":false,"start_time":"2022-09-15T04:11:17.589673","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class config:\n    \n    bnneck = True\n    gem = True\n    MIX_UP = False\n    PSEUDO = True\n    TEST = True\n    EVALUATE = True\n    \n    SEED = 42\n    FOLD_TO_RUN = 1\n    FOLDS = 10\n    DEBUG = False\n    RESUME = False\n    RESUME_EPOCH = None\n    \n    \n    ### Dataset\n    BATCH_SIZE =16*strategy.num_replicas_in_sync\n    IMAGE_SIZE = 512\n    # IMAGE_SIZE = 32\n    N_CLASSES = 8\n    \n    ### Model\n    model_type = 'effnetv2'  # nfnet effnetv1 effnetv2\n    EFF_NET = 7\n    # EFF_NET = 0\n    \n    EFF_NETV2 = 's-21k-ft1k'\n    FREEZE_BATCH_NORM = True\n    head = 'arcface' # arcface  curricular\n    EPOCHS = 35\n    LR = 0.001\n    \n    ### Augmentations\n    CUTOUT = False\n    \n    ### Save-Directory\n    save_dir = '.'\n\n    \ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\n\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:17.661142Z","iopub.status.busy":"2022-09-15T04:11:17.66043Z","iopub.status.idle":"2022-09-15T04:11:17.669769Z","shell.execute_reply":"2022-09-15T04:11:17.670277Z","shell.execute_reply.started":"2022-09-15T03:26:25.394036Z"},"papermill":{"duration":0.03606,"end_time":"2022-09-15T04:11:17.67046","exception":false,"start_time":"2022-09-15T04:11:17.6344","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('rsna2022-split-fold-id-wise-tfr')\nBATCH_SIZE = 16 if strategy.num_replicas_in_sync==8 else 1\nBATCH_SIZE *= strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nIMAGE_SIZE = [512, 512]\nEPOCHS = 20\nFOLDS = 5\nimg_size = (512, 512)\nGCS_PATH_STRATIFICATED","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:17.720317Z","iopub.status.busy":"2022-09-15T04:11:17.719324Z","iopub.status.idle":"2022-09-15T04:11:18.138262Z","shell.execute_reply":"2022-09-15T04:11:18.138759Z","shell.execute_reply.started":"2022-09-15T03:26:25.408522Z"},"papermill":{"duration":0.44526,"end_time":"2022-09-15T04:11:18.138956","exception":false,"start_time":"2022-09-15T04:11:17.693696","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decode the data and Define data loading methods","metadata":{"papermill":{"duration":0.023456,"end_time":"2022-09-15T04:11:18.186419","exception":false,"start_time":"2022-09-15T04:11:18.162963","status":"completed"},"tags":[]}},{"cell_type":"code","source":"l_type = tf.int64","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.237923Z","iopub.status.busy":"2022-09-15T04:11:18.236935Z","iopub.status.idle":"2022-09-15T04:11:18.240148Z","shell.execute_reply":"2022-09-15T04:11:18.239589Z","shell.execute_reply.started":"2022-09-15T03:26:25.804105Z"},"papermill":{"duration":0.030498,"end_time":"2022-09-15T04:11:18.240306","exception":false,"start_time":"2022-09-15T04:11:18.209808","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(posting_id, image, label, matches):\n    return posting_id, image,tf.one_hot(label, config.N_CLASSES), matches\n\n\ndef arcface_format(posting_id, image, label_group, matches):\n    return posting_id, {'inp1': image, 'inp2': label_group}, label_group, matches\n\ndef arcface_inference_format(posting_id, image, label_group, matches):\n    return image,posting_id\n\ndef arcface_eval_format(posting_id, image, label_group, matches):\n    return image,label_group\n\n# Data augmentation function\ndef data_augment(posting_id, image, label_group, matches):\n\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    image = tf.image.random_brightness(image, 0.10)\n    return posting_id, image, label_group, matches\n\ndef data_augment_test(posting_id, image, label_group, matches):\n\n    return posting_id, image, label_group, matches\n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.per_image_standardization(image)\n    image = tf.image.resize(image, [config.IMAGE_SIZE,config.IMAGE_SIZE])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"patient_overall\": tf.io.FixedLenFeature([], l_type),\n        \"C1\": tf.io.FixedLenFeature([], l_type),\n        \"C2\": tf.io.FixedLenFeature([], l_type),  # shape [] means single element\n        \"C3\": tf.io.FixedLenFeature([], l_type),\n        \"C4\": tf.io.FixedLenFeature([], l_type),\n        \"C5\": tf.io.FixedLenFeature([], l_type),\n        \"C6\": tf.io.FixedLenFeature([], l_type),\n        \"C7\": tf.io.FixedLenFeature([], l_type),\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    posting_id = example['image']\n    patient_overall = tf.cast(example['patient_overall'], tf.float32)\n    image = decode_image(example['image'])\n    c1 = tf.cast(example['C1'], tf.float32)\n    c2 = tf.cast(example['C2'], tf.float32)\n    c3 = tf.cast(example['C3'], tf.float32)\n    c4 = tf.cast(example['C4'], tf.float32)\n    c5 = tf.cast(example['C5'], tf.float32)\n    c6 = tf.cast(example['C6'], tf.float32)\n    c7 = tf.cast(example['C7'], tf.float32)\n    matches = 1\n    return posting_id, image, [patient_overall, c1, c2, c3, c4, c5, c6, c7], matches\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n#     dataset = dataset.cache()\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training tensors\ndef get_training_dataset(dataset):\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our training tensors\ndef get_val_dataset(dataset):\n    dataset = dataset.map(data_augment_test, num_parallel_calls = AUTO)\n    dataset = dataset.map(arcface_format, num_parallel_calls = AUTO)\n    dataset = dataset.map(lambda posting_id, image, label_group, matches: (image, label_group))\n    dataset = dataset.batch(config.BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.30925Z","iopub.status.busy":"2022-09-15T04:11:18.305327Z","iopub.status.idle":"2022-09-15T04:11:18.312268Z","shell.execute_reply":"2022-09-15T04:11:18.31169Z","shell.execute_reply.started":"2022-09-15T03:26:25.810295Z"},"papermill":{"duration":0.049101,"end_time":"2022-09-15T04:11:18.312421","exception":false,"start_time":"2022-09-15T04:11:18.26332","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building our model","metadata":{"papermill":{"duration":0.023518,"end_time":"2022-09-15T04:11:18.360271","exception":false,"start_time":"2022-09-15T04:11:18.336753","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def sigmoid_focal_crossentropy(y_true, y_pred, alpha=0.25, gamma=2.0):\n    \n    def smooth(y, smooth_factor):\n        assert len(y.shape) == 2\n        y *= 1 - smooth_factor\n        y += smooth_factor / y.shape[1]\n        return y\n    \n    #label smoothing factor\n    FACTOR = 0.1\n    alpha_factor = 1.0\n    modulating_factor = 1.0\n\n    y_pred = tf.convert_to_tensor(y_pred)\n    y_true = tf.convert_to_tensor(smooth(y_true, FACTOR), dtype=y_pred.dtype)\n\n    ce = K.binary_crossentropy(y_true, y_pred, from_logits=False)\n\n    p_t = (y_true * y_pred) + ((1 - y_true) * (1 - y_pred))\n    \n    alpha = tf.convert_to_tensor(alpha, dtype=K.floatx())\n    alpha_factor = y_true * alpha + (1 - y_true) * (1 - alpha)\n\n    gamma = tf.convert_to_tensor(gamma, dtype=K.floatx())\n    modulating_factor = tf.pow((1.0 - p_t), gamma)\n\n    return tf.reduce_sum(alpha_factor * modulating_factor * ce, axis=-1)","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.416601Z","iopub.status.busy":"2022-09-15T04:11:18.415491Z","iopub.status.idle":"2022-09-15T04:11:18.418414Z","shell.execute_reply":"2022-09-15T04:11:18.417741Z","shell.execute_reply.started":"2022-09-15T03:26:25.835343Z"},"papermill":{"duration":0.035031,"end_time":"2022-09-15T04:11:18.418563","exception":false,"start_time":"2022-09-15T04:11:18.383532","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass CurricularFace(tf.keras.layers.Layer):\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n        super(CurricularFace, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n        self._USE_V2_BEHAVIOR = True\n\n    def _assign_new_value(self, variable, value):\n        with K.name_scope('AssignNewValue') as scope:\n          if tf.compat.v1.executing_eagerly_outside_functions():\n            return variable.assign(value, name=scope)\n          else:\n            with tf.compat.v1.colocate_with(variable):  # pylint: disable=protected-access\n              return tf.compat.v1.assign(variable, value, name=scope)\n\n\n    def _get_training_value(self, training=None):\n        if training is None:\n          training = K.learning_phase()\n        if self._USE_V2_BEHAVIOR:\n          if isinstance(training, int):\n            training = bool(training)\n          if not self.trainable:\n            # When the layer is not trainable, it overrides the value passed from\n            # model.\n            training = False\n        return training\n\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(CurricularFace, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n        \n        self.t = self.add_weight(\n            name='t',\n            shape=(1),\n            initializer=tf.zeros_initializer(),\n            dtype='float32',\n            trainable=False,\n            regularizer=None,\n            aggregation=tf.VariableAggregation.MEAN,\n            experimental_autocast=False,\n            synchronization=tf.VariableSynchronization.ON_READ)\n        \n    def call(self, inputs, training=None):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n\n        do_training = self._get_training_value(training)\n\n        if do_training:\n            cosine = tf.matmul(\n                tf.math.l2_normalize(X, axis=1),\n                tf.math.l2_normalize(self.W, axis=0)\n            )\n            sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n            phi = cosine * self.cos_m - sine * self.sin_m\n\n            # target_logit = tf.reduce_sum(cosine * tf.cast(tf.one_hot(y, depth=self.n_classes),dtype=cosine.dtype), axis=-1)\n            target_logit = tf.reduce_sum(cosine * tf.cast(y, dtype=cosine.dtype), axis=-1)\n            sin_theta = tf.math.sqrt(1.0 - tf.math.pow(target_logit, 2))\n            cos_theta_m = target_logit * self.cos_m - sin_theta * self.sin_m\n\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#             one_hot = tf.cast(\n#                 tf.one_hot(y, depth=self.n_classes),\n#                 dtype=cosine.dtype\n#             )\n            one_hot = tf.cast(\n            y,\n            dtype=cosine.dtype\n        ) \n            t = tf.reduce_mean(target_logit) * 0.01 + (1 - 0.01) * self.t\n            self._assign_new_value(self.t, t)\n            cosine = tf.where(cosine > tf.expand_dims(cos_theta_m, axis=-1), cosine*(self.t+cosine), cosine)\n\n            if self.ls_eps > 0:\n                one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n            output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n            output *= self.s\n\n        else:\n            output = tf.matmul(\n                tf.math.l2_normalize(X, axis=1),\n                tf.math.l2_normalize(self.W, axis=0)\n            )\n\n        return output\n    \n# Arcmarginproduct class keras layer\nclass ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n#         one_hot = tf.cast(\n#             tf.one_hot(y, depth=self.n_classes),\n#             dtype=cosine.dtype\n#         )\n        one_hot = tf.cast(\n            y,\n            dtype=cosine.dtype\n        ) \n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n    \n    \nclass GeMPoolingLayer(tf.keras.layers.Layer):\n    def __init__(self, p=1., train_p=False):\n        super().__init__()\n        if train_p:\n            self.p = tf.Variable(p, dtype=tf.float32)\n        else:\n            self.p = p\n        self.eps = 1e-6\n\n    def call(self, inputs: tf.Tensor, **kwargs):\n        inputs = tf.clip_by_value(inputs, clip_value_min=1e-6, clip_value_max=tf.reduce_max(inputs))\n        inputs = tf.pow(inputs, self.p)\n        inputs = tf.reduce_mean(inputs, axis=[1, 2], keepdims=False)\n        inputs = tf.pow(inputs, 1./self.p)\n        return inputs","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.469687Z","iopub.status.busy":"2022-09-15T04:11:18.468604Z","iopub.status.idle":"2022-09-15T04:11:18.503858Z","shell.execute_reply":"2022-09-15T04:11:18.504436Z","shell.execute_reply.started":"2022-09-15T03:26:25.85582Z"},"papermill":{"duration":0.062595,"end_time":"2022-09-15T04:11:18.504638","exception":false,"start_time":"2022-09-15T04:11:18.442043","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n# Function to create our EfficientNetB3 model\ndef get_model():\n\n   \n    head1 = ArcMarginProduct\n    head2 = CurricularFace\n    \n    with strategy.scope():\n        margin1 = head1(\n            n_classes = config.N_CLASSES, \n            s = 30, \n            m = 0.3, \n            name=f'head/arcface', \n            dtype='float32'\n            )\n\n        margin2 = head2(\n            n_classes = config.N_CLASSES, \n            s = 30, \n            m = 0.3, \n            name=f'head/curricular', \n            dtype='float32'\n            ) \n        \n        \n\n        inp = tf.keras.layers.Input(shape = [config.IMAGE_SIZE, config.IMAGE_SIZE, 3], name = 'inp1')\n        label = tf.keras.layers.Input(shape = (), name = 'inp2')\n        M_D=[]\n        x = efficientnet.EfficientNetV2M(input_shape=[config.IMAGE_SIZE, config.IMAGE_SIZE, 3], pretrained=\"imagenet\", num_classes=0)(inp)\n        \n        embed = GeMPoolingLayer(train_p=False)(x)\n        for p in np.linspace(0.1,0.5,5):\n            D=tf.keras.layers.Dropout(p)(embed)\n            M_D.append(D)\n        embed=tf.keras.layers.Average()(M_D)\n        embed = tf.keras.layers.Dense(512)(embed)\n        x = margin1([embed, label])\n        \n        output = tf.keras.layers.Dense(8, activation='sigmoid')(x)\n        \n        model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])\n        \n        \n        opt = tf.keras.optimizers.Adam(learning_rate = config.LR)\n        # opt = tfa.optimizers.AdamW(learning_rate = config.LR, weight_decay=0.01)\n        if config.FREEZE_BATCH_NORM:\n            freeze_BN(model)\n\n        model.compile(\n            optimizer = opt,\n            loss = [sigmoid_focal_crossentropy],\n            metrics = [tf.keras.metrics.AUC(multi_label=True)]\n        )\n        return model","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.555772Z","iopub.status.busy":"2022-09-15T04:11:18.554744Z","iopub.status.idle":"2022-09-15T04:11:18.567902Z","shell.execute_reply":"2022-09-15T04:11:18.568479Z","shell.execute_reply.started":"2022-09-15T03:26:25.900097Z"},"papermill":{"duration":0.040734,"end_time":"2022-09-15T04:11:18.568662","exception":false,"start_time":"2022-09-15T04:11:18.527928","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(get_model(), show_layer_names=False, show_shapes=True)","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:18.619944Z","iopub.status.busy":"2022-09-15T04:11:18.618965Z","iopub.status.idle":"2022-09-15T04:11:59.633813Z","shell.execute_reply":"2022-09-15T04:11:59.634349Z","shell.execute_reply.started":"2022-09-15T03:26:25.919811Z"},"papermill":{"duration":41.042453,"end_time":"2022-09-15T04:11:59.634555","exception":false,"start_time":"2022-09-15T04:11:18.592102","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lrfn(epoch):\n    lr = [1e-3]*5 + [1e-4]*2 + [1e-5]*1\n    return lr[epoch]\nLR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:59.71475Z","iopub.status.busy":"2022-09-15T04:11:59.714099Z","iopub.status.idle":"2022-09-15T04:11:59.718807Z","shell.execute_reply":"2022-09-15T04:11:59.71938Z","shell.execute_reply.started":"2022-09-15T03:26:33.909614Z"},"papermill":{"duration":0.047065,"end_time":"2022-09-15T04:11:59.71958","exception":false,"start_time":"2022-09-15T04:11:59.672515","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = KFold(n_splits=5,shuffle=False)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:59.800405Z","iopub.status.busy":"2022-09-15T04:11:59.799664Z","iopub.status.idle":"2022-09-15T04:11:59.812969Z","shell.execute_reply":"2022-09-15T04:11:59.813549Z","shell.execute_reply.started":"2022-09-15T03:26:33.91671Z"},"papermill":{"duration":0.055743,"end_time":"2022-09-15T04:11:59.81373","exception":false,"start_time":"2022-09-15T04:11:59.757987","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train loop","metadata":{"papermill":{"duration":0.037923,"end_time":"2022-09-15T04:11:59.89008","exception":false,"start_time":"2022-09-15T04:11:59.852157","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    print(); print('#'*25)\n    print('### FOLD',fold+1)\n    print('#'*25)\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n\n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']))\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']))\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'model-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,save_weights_only=True,  mode='min', save_freq='epoch')\n    lrs = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-5, factor=0.5, verbose=1)\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=20,\n        verbose=1,\n        mode='auto',\n        )\n\n    model = get_model()\n    history = model.fit(\n        get_training_dataset(train_dataset), \n        steps_per_epoch = STEPS_PER_EPOCH,\n        epochs = 100,\n        callbacks = [sv, lrs, es],\n        validation_data = get_val_dataset(val_dataset),\n        verbose=1\n    )\n    del model\n    gc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-09-15T04:11:59.979125Z","iopub.status.busy":"2022-09-15T04:11:59.97839Z","iopub.status.idle":"2022-09-15T05:38:36.786656Z","shell.execute_reply":"2022-09-15T05:38:36.787588Z"},"papermill":{"duration":5196.859856,"end_time":"2022-09-15T05:38:36.788187","exception":false,"start_time":"2022-09-15T04:11:59.928331","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}