{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About Notebook\n\nIt is basic starter notebook for **breast cancer detection**, written in `keras` with `tensorflow 2` backend. Here the following component are demonstrated with high-level intuition. This notebook can be run on single or multi-gpu devices.\n\n- Dataloader \n- Hyper-parameters\n- Modeling\n- Inference\n\n\nHere, for dataloder, `tf.data` API is used. In hyper-parameter section, the `loss`, `metric`, `learning-rate modifier`, `optimizer` are discussed in the relevant section. For modeling, see the backbone model list down below. **Note**, the backbone model is also further extended to support custom operations, so that <u>some desired computation</u> can be saved using `callback` API. Lastly, inference code is also included.\n\n---\n\n> **N.B**. This might be an intermediate-level notebook. It emphasizes more on the modeling APIs, rather than aiming for achieving higher leaderboard scores. So, if you don't get about any component, feel free to leave a message.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nimport os\nimport glob\nimport cv2\nimport warnings\nfrom packaging.version import parse\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import GroupKFold, StratifiedGroupKFold\n\nwarnings.simplefilter(action=\"ignore\")\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow_hub as hub\nimport tensorflow_addons as tfa\nfrom tensorflow.python.client import device_lib\nfrom tensorflow.experimental import numpy as tnp","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-28T03:09:54.167087Z","iopub.execute_input":"2023-06-28T03:09:54.16753Z","iopub.status.idle":"2023-06-28T03:10:12.119521Z","shell.execute_reply.started":"2023-06-28T03:09:54.167495Z","shell.execute_reply":"2023-06-28T03:10:12.118442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Device Setup**\n\nTPU-VM is not supported yet, soon!","metadata":{}},{"cell_type":"code","source":"def set_cpu_gpus(mixed_precision=True, set_jit=False):\n    try: \n        # printed out the detected devices\n        list_ld = device_lib.list_local_devices()\n        for dev in list_ld: \n            print(dev.name,dev.memory_limit)\n        \n        # get the lisf of physical devices\n        physical_devices = tf.config.list_physical_devices(\n            'GPU' if len(list_ld) - 1 else 'CPU'\n        )\n        \n        # For GPU devices, configure related stuff\n        if 'GPU' in physical_devices[-1]:\n            tf.config.optimizer.set_jit(set_jit)\n            \n            if mixed_precision:\n                keras.mixed_precision.set_global_policy(\n                    \"mixed_float16\"\n                )\n            else:\n                keras.mixed_precision.set_global_policy(\n                    keras.backend.floatx()\n                )\n                \n            for pd in physical_devices:\n                tf.config.experimental.set_memory_growth(\n                    pd, True\n                )\n                \n        strategy = tf.distribute.MirroredStrategy()\n        return (strategy, physical_devices)\n    except: \n        raise ValueError('No Device Detected!')","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:12.121333Z","iopub.execute_input":"2023-06-28T03:10:12.12248Z","iopub.status.idle":"2023-06-28T03:10:12.132362Z","shell.execute_reply.started":"2023-06-28T03:10:12.122443Z","shell.execute_reply":"2023-06-28T03:10:12.131474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mxp = True\njit = True\n\nstrategy, physical_devices = set_cpu_gpus(mixed_precision=mxp, set_jit=jit)\nphysical_devices, tf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:12.1339Z","iopub.execute_input":"2023-06-28T03:10:12.134722Z","iopub.status.idle":"2023-06-28T03:10:17.973141Z","shell.execute_reply.started":"2023-06-28T03:10:12.134686Z","shell.execute_reply":"2023-06-28T03:10:17.972135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Utils**","metadata":{}},{"cell_type":"code","source":"def make_plot(tfdata, figsize=(20, 20)):\n    \n    plt.figure(figsize=figsize)\n    xy = int(np.ceil(tfdata.shape[0] * 0.5))\n\n    for i in range(tfdata.shape[0]):\n        plt.subplot(xy, xy, i + 1)\n        plt.imshow(tf.cast(tfdata[i], dtype=tf.uint8))\n        plt.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:17.976069Z","iopub.execute_input":"2023-06-28T03:10:17.976941Z","iopub.status.idle":"2023-06-28T03:10:17.984409Z","shell.execute_reply.started":"2023-06-28T03:10:17.976914Z","shell.execute_reply":"2023-06-28T03:10:17.98323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# True for 'Inference' (turn off the internet)\n# False for 'Training' (turn on the internet)\nSUBMIT = True\n\n# General\n# Supported: [convnext, efficientnet-v2, resnet-rs, densenet]\nBACKBONE_MODEL = ['efficientnet-v2', 'convnext', 'resnet-rs', 'densenet'][0] \nINP_SIZE = 1024\nSEED = 101\nSPLITS = 4\nValidationFold = 0 # < SPLITS\n\n# SetAutoTune\nEPOCHS = 5\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nAUTOTUNE = tf.data.AUTOTUNE\nBATCHES_PER_STEPS = 10 # 10 BATCH_SIZE # Be aware (keras/issues/16573)\nkeras.utils.set_random_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:17.985906Z","iopub.execute_input":"2023-06-28T03:10:17.986436Z","iopub.status.idle":"2023-06-28T03:10:18.002581Z","shell.execute_reply.started":"2023-06-28T03:10:17.986402Z","shell.execute_reply":"2023-06-28T03:10:18.001696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Set","metadata":{}},{"cell_type":"code","source":"DF_PATH = '/kaggle/input/rsna-breast-cancer-detection'\nIMG_PATH = f'/kaggle/input/rsna-breast-cancer-{INP_SIZE}-pngs/output'\n\ndf = pd.read_csv(f\"{DF_PATH}/train.csv\")\ndf['img_path'] = df.apply(\n    lambda i: os.path.join(\n        f\"{IMG_PATH}\", str(i['patient_id']) + \"_\" + str(i['image_id']) + '.png'\n    ), axis=1\n)\n\ndisplay(df.head())\nprint(df.shape)\ndf.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:18.003738Z","iopub.execute_input":"2023-06-28T03:10:18.004008Z","iopub.status.idle":"2023-06-28T03:10:19.203727Z","shell.execute_reply.started":"2023-06-28T03:10:18.003986Z","shell.execute_reply":"2023-06-28T03:10:19.202728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:19.2053Z","iopub.execute_input":"2023-06-28T03:10:19.205915Z","iopub.status.idle":"2023-06-28T03:10:19.28883Z","shell.execute_reply.started":"2023-06-28T03:10:19.205881Z","shell.execute_reply":"2023-06-28T03:10:19.287805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_missing_data(data):\n    total = data.isnull().sum().sort_values(ascending = False)\n    percentage = (\n        data.isnull().sum()/data.isnull().count()\n    ).sort_values(\n        ascending = False\n    )\n    return pd.concat([total,percentage] , axis = 1 , keys = ['Total' , 'Percent'])\nfind_missing_data(df)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:19.290333Z","iopub.execute_input":"2023-06-28T03:10:19.29065Z","iopub.status.idle":"2023-06-28T03:10:19.485396Z","shell.execute_reply.started":"2023-06-28T03:10:19.29062Z","shell.execute_reply":"2023-06-28T03:10:19.484189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgkf = StratifiedGroupKFold(\n    n_splits=SPLITS, shuffle=True, random_state=SEED\n)\ndf['fold'] = -1\n\nfor fold, (_, test_index) in enumerate(\n    sgkf.split(df, df.cancer, df.patient_id)\n):\n    df.loc[test_index, 'fold'] = fold\n    \ndisplay(df.groupby(['fold', \"cancer\"]).size())\ndf.to_csv(f'df_folds_{ValidationFold}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:19.487048Z","iopub.execute_input":"2023-06-28T03:10:19.487554Z","iopub.status.idle":"2023-06-28T03:10:24.422818Z","shell.execute_reply.started":"2023-06-28T03:10:19.487523Z","shell.execute_reply":"2023-06-28T03:10:24.421817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f'df_folds_{ValidationFold}.csv')\ntrain_df = df.query(f'fold != {ValidationFold}')\nvalid_df = df.query(f'fold == {ValidationFold}')\nprint(train_df.shape, valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.427552Z","iopub.execute_input":"2023-06-28T03:10:24.427837Z","iopub.status.idle":"2023-06-28T03:10:24.574462Z","shell.execute_reply.started":"2023-06-28T03:10:24.427812Z","shell.execute_reply":"2023-06-28T03:10:24.573242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.patient_id.nunique())\nprint(train_df.cancer.value_counts(normalize=True))\n\nprint(valid_df.patient_id.nunique())\nprint(valid_df.cancer.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.576559Z","iopub.execute_input":"2023-06-28T03:10:24.577067Z","iopub.status.idle":"2023-06-28T03:10:24.591917Z","shell.execute_reply.started":"2023-06-28T03:10:24.577028Z","shell.execute_reply":"2023-06-28T03:10:24.590856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader","metadata":{}},{"cell_type":"code","source":"def image_decoder(with_labels):\n\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        img = tf.image.decode_jpeg(file_bytes, channels = 3)\n        img = tf.reshape(img, [*[INP_SIZE]*2, 3])\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), tf.cast(label, dtype=tf.float32)\n    \n    return decode_with_labels if with_labels else decode\n\ndef create_dataset(\n    df, \n    batch_size  = 32, \n    with_labels = False,  \n    shuffle     = False,\n    repeat      = True\n):\n    # Image file decoder\n    decode_fn = image_decoder(with_labels)\n\n    # Create Dataset\n    if with_labels:\n        dataset = tf.data.Dataset.from_tensor_slices(\n            (df['img_path'].values, df['cancer'].values)\n        )\n    else:\n        dataset = tf.data.Dataset.from_tensor_slices(\n            (df['img_path'].values)\n        )\n        \n    dataset = dataset.map(decode_fn, num_parallel_calls = AUTOTUNE)\n    dataset = dataset.shuffle(\n        8 * BATCH_SIZE, reshuffle_each_iteration = False\n    ) if shuffle else dataset\n    dataset = dataset.batch(batch_size, drop_remainder=shuffle)\n    dataset = dataset.repeat() if repeat else dataset\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.595053Z","iopub.execute_input":"2023-06-28T03:10:24.596441Z","iopub.status.idle":"2023-06-28T03:10:24.608314Z","shell.execute_reply.started":"2023-06-28T03:10:24.5964Z","shell.execute_reply":"2023-06-28T03:10:24.607187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_dataset = create_dataset(\n    train_df,\n    batch_size = BATCH_SIZE, \n    with_labels = True, \n    shuffle = True,\n    repeat  = True\n)\n\nvalid_dataset = create_dataset(\n    valid_df,\n    batch_size = BATCH_SIZE, \n    with_labels = True, \n    shuffle = False,\n    repeat = False\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.611346Z","iopub.execute_input":"2023-06-28T03:10:24.612469Z","iopub.status.idle":"2023-06-28T03:10:24.822491Z","shell.execute_reply.started":"2023-06-28T03:10:24.61243Z","shell.execute_reply":"2023-06-28T03:10:24.821481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter Settings \n\n- Loss Functions\n- Metrics\n- Learning Rate Schedular\n- Optimizer","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow.keras import losses\nfrom tensorflow.keras import metrics\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import applications","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.824044Z","iopub.execute_input":"2023-06-28T03:10:24.824438Z","iopub.status.idle":"2023-06-28T03:10:24.830793Z","shell.execute_reply.started":"2023-06-28T03:10:24.824405Z","shell.execute_reply":"2023-06-28T03:10:24.8292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Learning Rate Schedular**","metadata":{}},{"cell_type":"code","source":"lr_start   = 0.000005\nlr_max     = 0.00000125 * BATCH_SIZE\nlr_min     = 0.000001\nlr_ramp_ep = 5\nlr_sus_ep  = 0\nlr_decay   = 0.8\nwd_decay   = lr_start * 0.04\n\n\ndef get_lr_callback(batch_size=8):\n    \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = callbacks.LearningRateScheduler(lrfn, verbose=True)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.835446Z","iopub.execute_input":"2023-06-28T03:10:24.835761Z","iopub.status.idle":"2023-06-28T03:10:24.859677Z","shell.execute_reply.started":"2023-06-28T03:10:24.835736Z","shell.execute_reply":"2023-06-28T03:10:24.858386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Metrics**","metadata":{}},{"cell_type":"markdown","source":"[**Competition Metrics**](https://www.kaggle.com/code/sohier/probabilistic-f-score) - **stateless**\n\nIt will be used inside the callback API, typically after batch end or epoch end, depending on target.","metadata":{}},{"cell_type":"code","source":"def tf_pfbeta(from_logits=True, beta=1.0, epsilon=1e-07):\n    \n    def pfbeta(y_true, y_pred):\n        y_pred = tf.cond(\n            tf.cast(from_logits, dtype=tf.bool),\n            lambda: tf.nn.sigmoid(y_pred),\n            lambda: y_pred,\n        )\n        y_true = tf.reshape(y_true, [-1])\n        y_pred = tf.reshape(y_pred, [-1])\n\n        ctp = tf.reduce_sum(y_true * y_pred, axis=-1)\n        cfp = tf.reduce_sum(y_pred, axis=-1) - ctp\n\n        c_precision = ctp / (ctp + cfp)\n        c_recall = ctp / tf.reduce_sum(y_true)\n        \n        def compute_fractions():\n            numerator = c_precision * c_recall\n            denominator = beta**2 * c_precision + c_recall + epsilon\n            return (1 + beta**2) * tf.math.divide_no_nan(numerator, denominator)\n        \n        return tf.cond(\n            tf.logical_and(\n                tf.greater(c_precision, 0.), tf.greater(c_recall, 0.)\n            ),\n            compute_fractions,\n            lambda: tf.constant(0, dtype=tf.float32)\n        )\n    \n    return pfbeta","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.863283Z","iopub.execute_input":"2023-06-28T03:10:24.863584Z","iopub.status.idle":"2023-06-28T03:10:24.874005Z","shell.execute_reply.started":"2023-06-28T03:10:24.86356Z","shell.execute_reply":"2023-06-28T03:10:24.872999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[**Competition Metrics**](https://www.kaggle.com/code/sohier/probabilistic-f-score) - **stateful**\n\nIt will be used in training time `(model.compile)`.","metadata":{}},{"cell_type":"code","source":"class pFBeta(keras.metrics.Metric):\n    def __init__(\n        self, \n        from_logits=True, \n        beta=1.0, \n        threshold=None, \n        epsilon=1e-07, \n        name='pFBeta', \n        **kwargs\n    ):\n        super().__init__(name=name, **kwargs)\n        self.beta = beta\n        self.epsilon = epsilon\n        self.threshold = threshold\n        self.from_logits = from_logits\n        self.true_positives = self.add_weight(name='tp', initializer='zeros')\n        self.false_positives = self.add_weight(name='fp', initializer='zeros')\n        self.false_negatives = self.add_weight(name='fn', initializer='zeros')\n\n    @tf.function\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_pred = tf.cond(\n            tf.cast(self.from_logits, dtype=tf.bool),\n            lambda: tf.nn.sigmoid(y_pred),\n            lambda: y_pred,\n        )\n        if self.threshold is not None:\n            y_pred = y_pred > self.threshold\n            \n        y_true = tf.reshape(tf.cast(y_true, dtype=tf.float32), [-1])\n        y_pred = tf.reshape(tf.cast(y_pred, dtype=tf.float32), [-1])\n        \n        self.true_positives.assign_add(tf.reduce_sum(y_true * y_pred, axis=-1))\n        self.false_positives.assign_add(\n            tf.reduce_sum(y_pred * (1 - y_true))\n        )\n        self.false_negatives.assign_add(\n            tf.reduce_sum((1 - y_pred) * y_true)\n        )\n\n    @tf.function\n    def result(self):\n        precision = tf.math.divide_no_nan(\n            self.true_positives, self.true_positives + self.false_positives\n        )\n        recall = tf.math.divide_no_nan(\n            self.true_positives, self.true_positives + self.false_negatives\n        )\n        numerator = precision * recall\n        denominator = self.beta**2 * precision + recall + self.epsilon\n        fscore = (1 + self.beta**2) * tf.math.divide_no_nan(numerator, denominator)\n        return fscore\n    \n    def reset_state(self):\n        for v in self.variables:\n            v.assign(0)\n        \n    def get_config(self):\n        config = {\n            \"from_logits\": self.from_logits,\n            \"beta\": self.beta,\n            \"epsilon\": self.epsilon,\n            \"threshold\": self.threshold,\n        }\n        base_config = super().get_config()\n        return {**base_config, **config}","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.875927Z","iopub.execute_input":"2023-06-28T03:10:24.876688Z","iopub.status.idle":"2023-06-28T03:10:24.89339Z","shell.execute_reply.started":"2023-06-28T03:10:24.876652Z","shell.execute_reply":"2023-06-28T03:10:24.892223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_auc(from_logits=True):\n    auc_fn = metrics.AUC()\n    \n    def auc(y_true, y_pred):\n        y_pred = tf.cond(\n            tf.cast(from_logits, dtype=tf.bool),\n            lambda: tf.nn.sigmoid(y_pred),\n            lambda: y_pred,\n        )\n        return auc_fn(y_true, y_pred)\n    \n    return auc","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.895751Z","iopub.execute_input":"2023-06-28T03:10:24.896479Z","iopub.status.idle":"2023-06-28T03:10:24.90773Z","shell.execute_reply.started":"2023-06-28T03:10:24.896446Z","shell.execute_reply":"2023-06-28T03:10:24.90694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loss Functions**","metadata":{}},{"cell_type":"markdown","source":"**Weighted Binary Loss**\n\n> A value `pos_weight > 1` decreases the false negative count, hence increasing the recall. Conversely setting `pos_weight < 1` decreases the false positive count and increases the precision. This can be seen from the fact that `pos_weight` is introduced as a multiplicative coefficient for the positive labels term in the loss expression:","metadata":{}},{"cell_type":"code","source":"def weighted_binary_loss(\n    apply_positive_weight, from_logits=True, reduction=\"mean\"\n):\n    def inverse_sigmoid(sigmoidal):\n        return - tf.math.log(1. / sigmoidal - 1.)\n\n    def weighted_loss(labels, predictions):\n        predictions = tf.convert_to_tensor(predictions)\n        labels = tf.cast(labels, predictions.dtype)\n        num_samples = tf.cast(tf.shape(labels)[-1], dtype=labels.dtype)\n\n        logits = tf.cond(\n            tf.cast(from_logits, dtype=tf.bool),\n            lambda: predictions,\n            lambda: inverse_sigmoid(sigmoidal=predictions),\n        )\n        loss = tf.nn.weighted_cross_entropy_with_logits(\n            labels, \n            logits, \n            pos_weight=apply_positive_weight\n        )\n        \n        if reduction.lower() == \"mean\":\n            return tf.reduce_mean(loss)\n        elif reduction.lower() == \"sum\":\n            return tf.reduce_sum(loss) / num_samples\n        elif reduction.lower() == \"none\":\n            return loss\n        else:\n            raise ValueError(\n                'Reduction type is should be `mean` or `sum` or `none`. ',\n                f'But, received {reduction}'\n            )\n    return weighted_loss","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.909655Z","iopub.execute_input":"2023-06-28T03:10:24.910596Z","iopub.status.idle":"2023-06-28T03:10:24.92045Z","shell.execute_reply.started":"2023-06-28T03:10:24.910563Z","shell.execute_reply":"2023-06-28T03:10:24.919463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Binary Focal Crossentropy**\n\n> `focal_factor = (1 - output) ** gamma` for class 1 `focal_factor = output ** gamma` for class 0 where `gamma` is a focusing parameter. When `gamma=0`, this function is equivalent to the binary crossentropy loss.","metadata":{}},{"cell_type":"code","source":"# ref...tf/keras/losses/BinaryFocalCrossentropy (available from tf 2.9)\ndef binary_focal_loss(\n    alpha=0.25, \n    gamma=2.0, \n    label_smoothing=0, \n    from_logits=False,\n    apply_class_balancing=False,\n    apply_positive_weight=1,\n    reduction=\"mean\"\n):\n    '''\n    alpha: A weight balancing factor for class 1, default is 0.25. \n        The weight for class 0 is 1.0 - alpha.\n    \n    gamma: A focusing parameter used to compute the focal factor, default is 2.0\n    \n    apply_class_balancing: A bool, whether to apply weight balancing on the binary \n        classes 0 and 1.\n    '''\n    \n    def smooth_labels(labels):\n        return labels * (1.0 - label_smoothing) + 0.5 * label_smoothing\n    \n    def compute_loss(labels, logits):\n        logits = tf.convert_to_tensor(logits)\n        labels = tf.cast(labels, logits.dtype)\n        labels = tf.cond(\n            tf.cast(label_smoothing, dtype=tf.bool),\n            lambda: smooth_labels(labels),\n            lambda: labels,\n        )\n        num_samples = tf.cast(tf.shape(labels)[-1], dtype=labels.dtype)\n        cross_entropy = weighted_binary_loss(\n            apply_positive_weight, from_logits, reduction='none'\n        )(labels, logits)\n        \n        sigmoidal = tf.cond(\n            tf.cast(from_logits, dtype=tf.bool),\n            lambda: tf.nn.sigmoid(logits),\n            lambda: logits,\n        )\n        pt = labels * sigmoidal + (1.0 - labels) * (1.0 - sigmoidal)\n        focal_factor = tf.pow(1.0 - pt, gamma)\n        focal_bce = focal_factor * cross_entropy\n        \n        if apply_class_balancing:\n            weight = labels * alpha + (1 - labels) * (1 - alpha)\n            focal_bce = weight * focal_bce\n\n        if reduction == 'mean':\n            return tf.reduce_mean(focal_bce)\n        elif reduction == 'sum':\n            return tf.reduce_sum(focal_bce) / num_samples\n        else:\n            raise ValueError(\n                'Reduction type should be `mean` or `sum` ',\n                f'But, received {reduction}'\n            )\n    return compute_loss","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.92194Z","iopub.execute_input":"2023-06-28T03:10:24.922474Z","iopub.status.idle":"2023-06-28T03:10:24.93649Z","shell.execute_reply.started":"2023-06-28T03:10:24.922442Z","shell.execute_reply":"2023-06-28T03:10:24.935407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Optimizer**","metadata":{}},{"cell_type":"markdown","source":"**Symbolic Discovery of Optimization Algorithms**\n\nA new optimizer from google (2023). [Code.](https://github.com/google/automl/tree/master/lion)","metadata":{}},{"cell_type":"code","source":"class Lion(keras.optimizers.legacy.Optimizer):\n    def __init__(\n        self,\n        learning_rate=0.0001,\n        beta_1=0.9,\n        beta_2=0.99,\n        wd=0,\n        name='lion', \n        **kwargs\n    ):\n        super().__init__(name, **kwargs)\n        self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))\n        self._set_hyper('beta_1', beta_1)\n        self._set_hyper('beta_2', beta_2)\n        self._set_hyper('wd', wd)\n    \n    def _create_slots(self, var_list):\n        # Create slots for the first and second moments.\n        # Separate for-loops to respect the ordering of slot variables from v1.\n        for var in var_list:\n            self.add_slot(var, 'm')\n    \n    def _prepare_local(self, var_device, var_dtype, apply_state):\n        super(Lion, self)._prepare_local(var_device, var_dtype, apply_state)\n        beta_1_t = tf.identity(self._get_hyper('beta_1', var_dtype))\n        beta_2_t = tf.identity(self._get_hyper('beta_2', var_dtype))\n        wd_t = tf.identity(self._get_hyper('wd', var_dtype))\n        lr = apply_state[(var_device, var_dtype)]['lr_t']\n        apply_state[(var_device, var_dtype)].update(\n            dict(\n                lr=lr,\n                beta_1_t=beta_1_t,\n                one_minus_beta_1_t=1 - beta_1_t,\n                beta_2_t=beta_2_t,\n                one_minus_beta_2_t=1 - beta_2_t,\n                wd_t=wd_t\n            )\n        ) \n    \n    @tf.function(jit_compile=True)\n    def _resource_apply_dense(self, grad, var, apply_state=None):\n        var_device, var_dtype = var.device, var.dtype.base_dtype\n        coefficients = (\n            (apply_state or {}).get(\n                (\n                    var_device, var_dtype\n                )\n            ) or self._fallback_apply_state(var_device, var_dtype)\n        ) \n        \n        m = self.get_slot(var, 'm')\n        var_t = var.assign_sub(\n            coefficients['lr_t'] * (\n                tf.math.sign(\n                    m * coefficients['beta_1_t'] + \n                    grad * coefficients['one_minus_beta_1_t']\n                ) + var * coefficients['wd_t'])\n        )\n        \n        with tf.control_dependencies([var_t]):\n            m.assign(\n                m * coefficients['beta_2_t'] + grad * coefficients['one_minus_beta_2_t']\n            )\n    \n    @tf.function(jit_compile=True)\n    def _resource_apply_sparse(self, grad, var, indices, apply_state=None):\n        var_device, var_dtype = var.device, var.dtype.base_dtype\n        coefficients = (\n            (apply_state or {}).get(\n                (\n                    var_device, var_dtype\n                )\n            ) or self._fallback_apply_state(var_device, var_dtype)\n        )\n\n        m = self.get_slot(var, 'm')\n        m_t = m.assign(m * coefficients['beta_1_t'])\n        m_scaled_g_values = grad * coefficients['one_minus_beta_1_t']\n        m_t = m_t.scatter_add(tf.IndexedSlices(m_scaled_g_values, indices))\n        var_t = var.assign_sub(\n            coefficients['lr'] * (\n                tf.math.sign(m_t) + var * coefficients['wd_t'])\n        )\n\n        with tf.control_dependencies([var_t]):\n            m_t = m_t.scatter_add(tf.IndexedSlices(-m_scaled_g_values, indices))\n            m_t = m_t.assign(\n                m_t * coefficients['beta_2_t'] / coefficients['beta_1_t']\n            )\n            m_scaled_g_values = grad * coefficients['one_minus_beta_2_t']\n            m_t.scatter_add(tf.IndexedSlices(m_scaled_g_values, indices))\n    \n    def get_config(self):\n        config = super(Lion, self).get_config()\n        config.update({\n            'learning_rate': self._serialize_hyperparameter('learning_rate'),\n            'beta_1': self._serialize_hyperparameter('beta_1'),\n            'beta_2': self._serialize_hyperparameter('beta_2'),\n            'wd': self._serialize_hyperparameter('wd'),\n        })\n        return config","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.939284Z","iopub.execute_input":"2023-06-28T03:10:24.939623Z","iopub.status.idle":"2023-06-28T03:10:24.960437Z","shell.execute_reply.started":"2023-06-28T03:10:24.939593Z","shell.execute_reply":"2023-06-28T03:10:24.959582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_optimizer(mode='adamw'):\n    \n    if mode.lower() == 'adamw':\n        opt = tfa.optimizers.AdamW(\n            learning_rate=0.003, weight_decay=wd_decay\n        )\n    elif mode.lower() == 'lion':\n        opt = Lion(\n            learning_rate=0.003, wd=wd_decay\n        )\n    else:\n        opt = keras.optimizers.Adam(\n            learning_rate=0.003, weight_decay=wd_decay\n        )\n        \n    return opt","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.961534Z","iopub.execute_input":"2023-06-28T03:10:24.961899Z","iopub.status.idle":"2023-06-28T03:10:24.977097Z","shell.execute_reply.started":"2023-06-28T03:10:24.961868Z","shell.execute_reply":"2023-06-28T03:10:24.975998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_loss_fn(mode='weighted_binary'):\n    \n    if mode.lower() == 'weighted_binary':\n        loss_fn = weighted_binary_loss(\n            apply_positive_weight=5, \n            from_logits=True, \n            reduction=\"mean\"\n        )\n    elif mode.lower() == 'focal':\n        loss_fn = binary_focal_loss(\n            alpha=0.25, \n            gamma=2.0, \n            label_smoothing=0.05, \n            from_logits=True,\n            apply_class_balancing=True,\n            apply_positive_weight=1,\n            reduction=\"mean\"\n        )\n    else:\n        loss_fn = keras.losses.BinaryCrossentropy(\n            from_logits=True,\n            label_smoothing=0.01\n        )\n    \n    return loss_fn","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:24.978593Z","iopub.execute_input":"2023-06-28T03:10:24.979098Z","iopub.status.idle":"2023-06-28T03:10:24.999739Z","shell.execute_reply.started":"2023-06-28T03:10:24.978993Z","shell.execute_reply":"2023-06-28T03:10:24.998803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_metrics():\n    metrics_list = [\n        pFBeta(beta=1.0, from_logits=True), \n        tf_auc(from_logits=True)\n    ]\n    return metrics_list","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.00127Z","iopub.execute_input":"2023-06-28T03:10:25.001676Z","iopub.status.idle":"2023-06-28T03:10:25.011158Z","shell.execute_reply.started":"2023-06-28T03:10:25.001646Z","shell.execute_reply":"2023-06-28T03:10:25.010394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Augmentation [On GPU]\n\nWe like to insert data augmentation inside the model to get leverage the GPU/TPU speed. Note, the augmentation layers are active during the training time but not testing or inference time. That makes sense but you may want to consider **Test-Time-Augmentation**, see more [details](https://github.com/keras-team/keras/issues/17385). ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import Resizing\nfrom tensorflow.keras.layers import Rescaling\nfrom tensorflow.keras.layers import RandomCrop\nfrom tensorflow.keras.layers import RandomFlip\nfrom tensorflow.keras.layers import RandomZoom\nfrom tensorflow.keras.layers import RandomRotation\nfrom tensorflow.keras.layers import RandomBrightness\nfrom tensorflow.keras.layers import RandomContrast","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.011936Z","iopub.execute_input":"2023-06-28T03:10:25.012216Z","iopub.status.idle":"2023-06-28T03:10:25.021863Z","shell.execute_reply.started":"2023-06-28T03:10:25.012149Z","shell.execute_reply":"2023-06-28T03:10:25.020811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocessing\ndata_preprocessing = keras.Sequential(\n    [\n        Resizing(\n            *[INP_SIZE] * 2, \n            interpolation=\"bilinear\"\n        ),\n    ], \n    name='PreprocessingLayers'\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.023214Z","iopub.execute_input":"2023-06-28T03:10:25.023752Z","iopub.status.idle":"2023-06-28T03:10:25.042727Z","shell.execute_reply.started":"2023-06-28T03:10:25.02372Z","shell.execute_reply":"2023-06-28T03:10:25.041781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# why this RandomApply? ans. https://stackoverflow.com/a/72558994/9215780\nclass RandomApply(layers.Layer):\n    \"\"\"RandomApply will randomly apply the transformation layer\n    based on the given probability.\n    \n    Ref. https://stackoverflow.com/a/72558994/9215780\n    \"\"\"\n\n    def __init__(self, layer, probability, **kwargs):\n        super().__init__(**kwargs)\n        self.layer = layer\n        self.probability = probability\n\n    def call(self, inputs, training=True):\n        apply_layer = tf.random.uniform([]) < self.probability\n        outputs = tf.cond(\n            pred=tf.logical_and(apply_layer, training),\n            true_fn=lambda: self.layer(inputs),\n            false_fn=lambda: inputs,\n        )\n        return outputs\n\n    def get_config(self):\n        config = super().get_config()\n        config.update(\n            {\n                \"layer\": layers.serialize(self.layer),\n                \"probability\": self.probability,\n            }\n        )\n        return config\n\n# Augmentation\ndata_augmentation = keras.Sequential(\n    [\n        RandomApply(\n            RandomFlip(\"horizontal\"), probability=0.7\n        ),\n        RandomApply(\n            RandomContrast(factor=0.4), probability=0.2\n        ),\n        RandomApply(\n            RandomBrightness(\n                factor=0.3, value_range=(0, 255)\n            ), probability=0.2\n        ),\n    ],\n    name=\"augment\",\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.046456Z","iopub.execute_input":"2023-06-28T03:10:25.046716Z","iopub.status.idle":"2023-06-28T03:10:25.069336Z","shell.execute_reply.started":"2023-06-28T03:10:25.046694Z","shell.execute_reply":"2023-06-28T03:10:25.068215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMIT:\n    temp_ds = create_dataset(\n        valid_df.sample(20),\n        batch_size=10, \n        with_labels=True, \n        shuffle=False\n    )\n    x, y = next(iter(temp_ds))\n    x = data_preprocessing(x)\n    aug_x = data_augmentation(x, training=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.07801Z","iopub.execute_input":"2023-06-28T03:10:25.078416Z","iopub.status.idle":"2023-06-28T03:10:25.08349Z","shell.execute_reply.started":"2023-06-28T03:10:25.07839Z","shell.execute_reply":"2023-06-28T03:10:25.082484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMIT:\n    make_plot(x)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.084724Z","iopub.execute_input":"2023-06-28T03:10:25.085792Z","iopub.status.idle":"2023-06-28T03:10:25.09234Z","shell.execute_reply.started":"2023-06-28T03:10:25.085755Z","shell.execute_reply":"2023-06-28T03:10:25.091208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMIT:\n    make_plot(aug_x) ","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.093632Z","iopub.execute_input":"2023-06-28T03:10:25.094245Z","iopub.status.idle":"2023-06-28T03:10:25.102398Z","shell.execute_reply.started":"2023-06-28T03:10:25.094207Z","shell.execute_reply":"2023-06-28T03:10:25.101782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\n---\n\n**Might be useful**\n\n- [All Weights variation of official EfficientNet V2](https://www.kaggle.com/competitions/petfinder-pawpularity-score/discussion/285720)\n- [Latest EfficientNets-B0-B7 checkpoints](https://www.kaggle.com/competitions/petfinder-pawpularity-score/discussion/275221)\n- [Hybrid EfficientNet Swin-Transformer](https://www.kaggle.com/code/ipythonx/tf-hybrid-efficientnet-swin-transformer-gradcam)","metadata":{}},{"cell_type":"code","source":"def get_backbone_mode(model_name):\n    \n    if model_name == 'convnext':\n        backbone = applications.ConvNeXtTiny(\n                include_top=False, \n                pooling='avg', \n                include_preprocessing=True,\n                weights='imagenet' if not SUBMIT else None\n            )\n    elif model_name == 'efficientnet-v2':\n        backbone = applications.EfficientNetV2B0(\n            include_top=False, \n            pooling='avg', \n            include_preprocessing=True,\n            weights='imagenet' if not SUBMIT else None\n        )\n    elif model_name == 'resnet-rs':\n        backbone = applications.ResNetRS50(\n            include_top=False, \n            pooling='avg', \n            include_preprocessing=True,\n            weights='imagenet' if not SUBMIT else None\n        )\n    elif model_name == 'densenet':\n        backbone = keras.Sequential(\n            [\n                layers.Rescaling(scale=1/255., offset=0.0),\n                applications.DenseNet121(\n                    include_top=False, \n                    pooling='avg', \n                    weights='imagenet' if not SUBMIT else None\n                )\n            ], name=model_name\n        )\n    else:\n        raise ValueError(\n            f'Supported Models are [convnext, efficientnet-v2, resnet-rs, densenet] ',\n            f'But got {model_name}'\n        )\n    \n    return backbone","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.103415Z","iopub.execute_input":"2023-06-28T03:10:25.10436Z","iopub.status.idle":"2023-06-28T03:10:25.114011Z","shell.execute_reply.started":"2023-06-28T03:10:25.10431Z","shell.execute_reply":"2023-06-28T03:10:25.113451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def BreastCancerDetect(backbone_model_name='efficientnet-v2'):\n    \n    backbone_model = get_backbone_mode(\n        model_name=backbone_model_name.lower()\n    )\n    \n    model = keras.Sequential(\n        [\n            keras.layers.InputLayer(input_shape=(INP_SIZE, INP_SIZE, 3)),\n            data_preprocessing,\n            data_augmentation,\n            backbone_model,\n            keras.layers.Dense(1, activation=None, dtype='float32')\n        ], name='CancerDetect'\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.115041Z","iopub.execute_input":"2023-06-28T03:10:25.116162Z","iopub.status.idle":"2023-06-28T03:10:25.130248Z","shell.execute_reply.started":"2023-06-28T03:10:25.116132Z","shell.execute_reply":"2023-06-28T03:10:25.12939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with tf.device('/CPU:0'):\n    val_gt = tf.Variable(\n        tnp.empty((0), dtype=tf.float32), shape=[None], trainable=False\n    )\n    val_pred = tf.Variable(\n        tnp.empty((0), dtype=tf.float32), shape=[None], trainable=False\n    )\n    \n\nclass ExtendedModel(keras.Model):\n    def __init__(self, model, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        # Actual model\n        self.model = model\n    \n    def test_step(self, data):\n        x, y = data\n        y_pred = self.model(x, training=False)\n        self.compiled_loss(y, y_pred, regularization_losses=self.losses)\n        self.compiled_metrics.update_state(y, y_pred)\n\n        val_gt.assign(\n            tf.concat([val_gt, y], axis=0)\n        )\n        val_pred.assign(\n            tf.concat([val_pred, tf.squeeze(y_pred)], axis=0)\n        )\n        return {m.name: m.result() for m in self.metrics}\n    \n    def call(self, inputs):\n        return self.model(inputs)\n    \n    def save_weights(\n        self, filepath, overwrite=True, save_format=None, options=None\n    ):\n        # Overriding this method will allow us to use the `ModelCheckpoint`\n        self.model.save_weights(\n            filepath=filepath,\n            overwrite=overwrite,\n            save_format=save_format,\n            options=options,\n        )\n        \n    def save(\n        self, filepath, overwrite=True, include_optimizer=True, \n        save_format=None, signatures=None, options=None\n    ):\n        # Overriding this method will allow us to use the `ModelCheckpoint`\n        self.model.save(\n            filepath=filepath,\n            overwrite=overwrite,\n            save_format=save_format,\n            options=options,\n            include_optimizer=include_optimizer,\n            signatures=signatures\n        )","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.131602Z","iopub.execute_input":"2023-06-28T03:10:25.132069Z","iopub.status.idle":"2023-06-28T03:10:25.161533Z","shell.execute_reply.started":"2023-06-28T03:10:25.132034Z","shell.execute_reply":"2023-06-28T03:10:25.16072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resets all state generated by Keras, if.\nkeras.backend.clear_session()\n\n# Open a strategy scope.\nwith strategy.scope():\n    # build cancer detecting model\n    model = BreastCancerDetect(\n        backbone_model_name=BACKBONE_MODEL\n    )\n    model = ExtendedModel(\n        model, name=model.name\n    )\n    \n    # Compile\n    model.compile(\n        optimizer=get_optimizer(mode='lion'),\n        loss=get_loss_fn(mode='weighted_binary'),\n        metrics=get_metrics(),\n        steps_per_execution=BATCHES_PER_STEPS,\n    )\n\n_ = model(tf.ones(shape=(1, INP_SIZE, INP_SIZE, 3)))    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:25.162672Z","iopub.execute_input":"2023-06-28T03:10:25.163602Z","iopub.status.idle":"2023-06-28T03:10:41.859833Z","shell.execute_reply.started":"2023-06-28T03:10:25.163558Z","shell.execute_reply":"2023-06-28T03:10:41.858853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers:\n    print(layer.trainable, layer.name)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:41.861414Z","iopub.execute_input":"2023-06-28T03:10:41.862037Z","iopub.status.idle":"2023-06-28T03:10:41.867883Z","shell.execute_reply.started":"2023-06-28T03:10:41.862004Z","shell.execute_reply":"2023-06-28T03:10:41.866759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks\n\nWe will create a custom callback, that will be used to find the optimal value of the target metric (`F1`) with corresponding threshold value. ","metadata":{}},{"cell_type":"code","source":"class OptimalPFBetaWithThresholdCallback(keras.callbacks.Callback):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.cmp_metric = tf_pfbeta(\n            from_logits=False, beta=1.0, epsilon=1e-07\n        )\n    \n    def on_epoch_begin(self, epoch, logs=None):\n        val_gt.assign(\n            tf.Variable(\n                tnp.empty((0), dtype=tf.float32), shape=[None]\n            )\n        )\n        val_pred.assign(\n            tf.Variable(\n                tnp.empty((0), dtype=tf.float32), shape=[None]\n            )\n        )\n\n    def on_epoch_end(self, epoch, logs=None):\n        y_true = val_gt.numpy()\n        y_pred = val_pred.numpy()\n        y_pred = tf.nn.sigmoid(y_pred)\n        max_pfbeta_score, at_threshold = self.tf_pfbeta_opt(y_true, y_pred)\n        logs['val_pFBeta_binarize'] = max_pfbeta_score\n        logs['val_threshold'] = at_threshold\n        \n    def tf_pfbeta_opt(self, y_true, y_pred):\n        thresholds = tf.range(0, 1, 0.05)\n        pfbeta_scores = []\n        \n        for threshold in thresholds:\n            scores = self.cmp_metric(\n                y_true, tf.cast(y_pred > threshold, dtype=tf.float32)\n            )\n            pfbeta_scores.append(scores)\n            \n        max_pfbeta_score = tf.reduce_max(pfbeta_scores)\n        at_threshold = thresholds[tf.argmax(pfbeta_scores)]\n        return max_pfbeta_score.numpy(), at_threshold.numpy()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:41.86951Z","iopub.execute_input":"2023-06-28T03:10:41.870128Z","iopub.status.idle":"2023-06-28T03:10:41.882806Z","shell.execute_reply.started":"2023-06-28T03:10:41.870097Z","shell.execute_reply":"2023-06-28T03:10:41.882211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_callbacks(monitor, ckpt_id, more_callback=None):\n    \n    pfbeta_clbk = OptimalPFBetaWithThresholdCallback()\n    lr_clbk = get_lr_callback(BATCH_SIZE)\n    ckpt_clbk = callbacks.ModelCheckpoint(\n        filepath='model.{epoch:02d}-{val_loss:.4f}-{val_pFBeta_binarize:.3f}-{val_threshold:.2f}.h5',\n        monitor='val_pFBeta_binarize',\n        mode='max',\n        save_best_only=True\n    )\n    csv_clbk = callbacks.CSVLogger(f'history_fold_{ValidationFold}.csv')\n    \n    # order matters\n    list_of_callbacks = [\n        pfbeta_clbk,\n        lr_clbk,\n        ckpt_clbk,\n        csv_clbk,\n    ]\n    \n    return list_of_callbacks ","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:41.884218Z","iopub.execute_input":"2023-06-28T03:10:41.884937Z","iopub.status.idle":"2023-06-28T03:10:41.900034Z","shell.execute_reply.started":"2023-06-28T03:10:41.884903Z","shell.execute_reply":"2023-06-28T03:10:41.899119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"if not SUBMIT:\n    training_callbacks = get_callbacks()\n    model.fit(\n        training_dataset, \n        validation_data=valid_dataset, \n        epochs=EPOCHS,\n        callbacks=training_callbacks,\n        steps_per_epoch=len(train_df) // BATCH_SIZE,\n    )\n    history = pd.read_csv(f'history_fold_{ValidationFold}.csv')\nelse:\n    history = pd.read_csv('/kaggle/input/rsna-breast-cancer/history_fold_0.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:41.902418Z","iopub.execute_input":"2023-06-28T03:10:41.903043Z","iopub.status.idle":"2023-06-28T03:10:41.926663Z","shell.execute_reply.started":"2023-06-28T03:10:41.903013Z","shell.execute_reply":"2023-06-28T03:10:41.925693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(\n    history.style.highlight_max(\n        axis=0, props='background-color:lightblue;', \n        subset=['val_auc','val_pFBeta', 'val_pFBeta_binarize']\n    ).highlight_min(\n        axis=0, props='background-color:lightgreen;', \n        subset=['loss', 'val_loss']\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:41.928316Z","iopub.execute_input":"2023-06-28T03:10:41.928651Z","iopub.status.idle":"2023-06-28T03:10:42.00196Z","shell.execute_reply.started":"2023-06-28T03:10:41.92862Z","shell.execute_reply":"2023-06-28T03:10:42.000803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load Best Model**","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:42.003812Z","iopub.execute_input":"2023-06-28T03:10:42.004193Z","iopub.status.idle":"2023-06-28T03:10:43.096612Z","shell.execute_reply.started":"2023-06-28T03:10:42.004143Z","shell.execute_reply":"2023-06-28T03:10:43.095379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_best_weight(weight_list):\n    max_pfbeta_bin = round(\n        history.val_pFBeta_binarize.max(), 3\n    )\n    for wg in weight_list:\n        if str(max_pfbeta_bin) in str(wg):\n            return wg\n        \ntrained_weight_files = get_best_weight(glob.glob('/kaggle/working/*.h5'))\ntrained_weight_files = trained_weight_files or glob.glob('/kaggle/input/rsna-breast-cancer/*.h5')[0]\ntrained_weight_files ","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:43.099339Z","iopub.execute_input":"2023-06-28T03:10:43.100462Z","iopub.status.idle":"2023-06-28T03:10:43.116225Z","shell.execute_reply.started":"2023-06-28T03:10:43.10042Z","shell.execute_reply":"2023-06-28T03:10:43.11522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = BreastCancerDetect(backbone_model_name=BACKBONE_MODEL)\n    model.load_weights(trained_weight_files)\n    model.compile(steps_per_execution=BATCHES_PER_STEPS, jit_compile=True)\n    model.trainable = False\nmodel.summary(line_length=80)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:43.120372Z","iopub.execute_input":"2023-06-28T03:10:43.120665Z","iopub.status.idle":"2023-06-28T03:10:50.22845Z","shell.execute_reply.started":"2023-06-28T03:10:43.120641Z","shell.execute_reply":"2023-06-28T03:10:50.227397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Build Model for TTA**","metadata":{}},{"cell_type":"code","source":"# Set-up for Test-Time-Augmentation\nclass Flip(keras.layers.Layer):\n    def call(self, inputs):\n        x = tf.image.flip_left_right(inputs)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:50.229729Z","iopub.execute_input":"2023-06-28T03:10:50.230328Z","iopub.status.idle":"2023-06-28T03:10:50.236054Z","shell.execute_reply.started":"2023-06-28T03:10:50.230295Z","shell.execute_reply":"2023-06-28T03:10:50.235209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_layer(model, new_layer):\n    initial_input = model.input\n    flipped_input = new_layer(initial_input)\n    tensor_output = keras.layers.Average(name='tta_avg')(\n        [\n            model(initial_input), model(flipped_input)\n        ]\n    )\n    new_model = keras.Model(\n        inputs=initial_input, outputs=tensor_output, name=model.name\n    )\n    return new_model\n\n\nwith strategy.scope():\n    tta_model = insert_layer(\n        model, Flip(name='InputFlipping')\n    )\n    tta_model.compile(\n        steps_per_execution=BATCHES_PER_STEPS, jit_compile=True\n    )\n    tta_model.trainable = False\ntta_model.summary(line_length=100)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:50.237948Z","iopub.execute_input":"2023-06-28T03:10:50.23876Z","iopub.status.idle":"2023-06-28T03:10:54.247303Z","shell.execute_reply.started":"2023-06-28T03:10:50.238728Z","shell.execute_reply":"2023-06-28T03:10:54.246371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not SUBMIT:\n    val_y_true = val_gt.numpy()\n    val_y_pred = val_pred.numpy()\n    val_y_pred = tf.nn.sigmoid(val_y_pred)\n    val_y_pred_tta = tta_model.predict(valid_dataset)\n    val_y_pred_tta = tf.nn.sigmoid(val_y_pred_tta)\nelse:\n    val_y_true = np.array(\n        list(\n            map(np.float32, valid_df.cancer.tolist())\n        )\n    )\n    val_y_pred = model.predict(valid_dataset)\n    val_y_pred = tf.nn.sigmoid(val_y_pred)\n    val_y_pred_tta = tta_model.predict(valid_dataset)\n    val_y_pred_tta = tf.nn.sigmoid(val_y_pred_tta)\n    val_y_pred_tta = tf.cast(val_y_pred_tta, dtype=tf.float32)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:10:54.248945Z","iopub.execute_input":"2023-06-28T03:10:54.249571Z","iopub.status.idle":"2023-06-28T03:20:10.944265Z","shell.execute_reply.started":"2023-06-28T03:10:54.249536Z","shell.execute_reply":"2023-06-28T03:20:10.943189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# return [max_pfbeta, threshold]\npfbeta_clbk = OptimalPFBetaWithThresholdCallback()\nmax_pfbeta, threshold = pfbeta_clbk.tf_pfbeta_opt(val_y_true, val_y_pred)\nmax_pfbeta_tta, threshold_tta = pfbeta_clbk.tf_pfbeta_opt(val_y_true, val_y_pred_tta)\nprint(\n    f'without tta : pfbeta {max_pfbeta} @ {threshold}'\n)\nprint(\n    f'with tta    : pfbeta {max_pfbeta_tta} @ {threshold_tta}'\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:20:10.946461Z","iopub.execute_input":"2023-06-28T03:20:10.946905Z","iopub.status.idle":"2023-06-28T03:20:11.177039Z","shell.execute_reply.started":"2023-06-28T03:20:10.94687Z","shell.execute_reply":"2023-06-28T03:20:11.175849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"from IPython.display import clear_output\n!pip install --no-deps ../input/for-pydicom/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install --no-deps ../input/for-pydicom/python_gdcm-3.0.22-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install --no-deps ../input/for-pydicom/pylibjpeg_libjpeg-1.3.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install --no-deps ../input/for-pydicom/dicomsdl-0.109.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\nclear_output()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-28T03:20:11.178377Z","iopub.execute_input":"2023-06-28T03:20:11.178784Z","iopub.status.idle":"2023-06-28T03:20:22.494048Z","shell.execute_reply.started":"2023-06-28T03:20:11.178747Z","shell.execute_reply":"2023-06-28T03:20:22.492909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXTENSION = \"png\"\nTEMP_FOLDER = \"/kaggle/tmp/output/\"\nTEST_DICOM = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/test_images/*/*.dcm\")\nos.makedirs(TEMP_FOLDER, exist_ok=True)\n\ntest_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\ntest_df['cancer'] = 0\ntest_df['img_path'] = (\n    TEMP_FOLDER + test_df[\"patient_id\"].astype(str) + \"_\" + test_df[\"image_id\"].astype(str) + \".png\"\n)\ntest_ds = create_dataset(\n    test_df, \n    with_labels = False,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    repeat=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:20:22.495887Z","iopub.execute_input":"2023-06-28T03:20:22.496641Z","iopub.status.idle":"2023-06-28T03:20:22.568089Z","shell.execute_reply.started":"2023-06-28T03:20:22.49656Z","shell.execute_reply":"2023-06-28T03:20:22.567157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following preprocessing cell is taken from [this](https://www.kaggle.com/code/theoviel/rsna-breast-baseline-inference) code example.","metadata":{}},{"cell_type":"code","source":"import dicomsdl\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\ndef read_dicom(dicom_file):\n    dicom = dicomsdl.open(dicom_file)\n    image = dicom.pixelData(storedvalue=False)\n    image = (image - image.min()) / (image.max() - image.min())\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        image = 1 - image\n    return image\n    \ndef saving(file, size=512, save_folder=\"\", extension=\"png\"):\n    image = read_dicom(file)\n    image = cv2.resize(image, (size, size))\n    image = (image * 255).astype(np.uint8)\n    \n    patient_id = file.split('/')[-2]\n    image_name = file.split('/')[-1][:-4]\n    cv2.imwrite(\n        save_folder + f\"{patient_id}_{image_name}.{extension}\", image\n    )\n    \n_ = Parallel(n_jobs=-1)(\n    delayed(saving)(\n        uid, size=INP_SIZE, save_folder=TEMP_FOLDER, extension=EXTENSION\n    ) for uid in tqdm(TEST_DICOM)\n)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-06-28T03:20:22.569557Z","iopub.execute_input":"2023-06-28T03:20:22.569935Z","iopub.status.idle":"2023-06-28T03:20:25.746947Z","shell.execute_reply.started":"2023-06-28T03:20:22.569907Z","shell.execute_reply":"2023-06-28T03:20:25.745528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.70 # from model weight file name\npred = model.predict(test_ds)\npred = tf.nn.sigmoid(pred).numpy()\ntest_df[\"cancer\"] = (pred > threshold).astype(int)\n\ntest_df['prediction_id'] = test_df['patient_id'].astype(str) + \"_\" + test_df['laterality']\nsub = test_df[['prediction_id', 'cancer']].groupby(\"prediction_id\").mean().reset_index()\nsub.to_csv('/kaggle/working/submission.csv', index=False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-28T03:20:25.751087Z","iopub.execute_input":"2023-06-28T03:20:25.751408Z","iopub.status.idle":"2023-06-28T03:20:55.973182Z","shell.execute_reply.started":"2023-06-28T03:20:25.751378Z","shell.execute_reply":"2023-06-28T03:20:55.972109Z"},"trusted":true},"execution_count":null,"outputs":[]}]}