{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA Screening Mammography Breast Cancer Detection\n\nHi, everyone!\n\nThis notebook demonstrates training EfficientNet with TensorFlow/Keras on GPU T4. \n\n1. Model\n\n    It is now conventient to try different models with keras-cv-attention-models. Feel free to change it.\n\n2. Data\n    \n    With tf.data.Dataset pipeline, it is fast and safe to do distributed training on multiple GPUs or TPUs.\n\nOf course, there are a lot of hyperparameters to tune to get better score. \n    \nIf you feel this notebook helpful, don't forget to **Upvote**. Good luck!","metadata":{}},{"cell_type":"markdown","source":"# Installs and imports","metadata":{}},{"cell_type":"code","source":"!pip install -qU tensorflow==2.8\n!pip install -q keras-cv-attention-models\n!apt -y install --allow-change-held-packages libcudnn8=8.1.0.77-1+cuda11.2","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-02T05:05:12.379797Z","iopub.execute_input":"2023-02-02T05:05:12.380308Z","iopub.status.idle":"2023-02-02T05:07:48.486754Z","shell.execute_reply.started":"2023-02-02T05:05:12.380206Z","shell.execute_reply":"2023-02-02T05:07:48.485568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nimport numpy as np\nfrom keras_cv_attention_models import efficientnet\nimport os\nimport matplotlib.pyplot as plt\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport random\nfrom tensorflow.keras import Input\nfrom sklearn.model_selection import StratifiedGroupKFold\nimport seaborn as sns\nimport math\nimport gc\nimport tensorflow_addons as tfa\n# os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"2\"","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:07:48.489157Z","iopub.execute_input":"2023-02-02T05:07:48.489544Z","iopub.status.idle":"2023-02-02T05:07:55.277086Z","shell.execute_reply.started":"2023-02-02T05:07:48.489505Z","shell.execute_reply":"2023-02-02T05:07:55.276064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = tf.distribute.MirroredStrategy()\nprint(f\"Number of devices : {strategy.num_replicas_in_sync}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:07:55.278628Z","iopub.execute_input":"2023-02-02T05:07:55.279249Z","iopub.status.idle":"2023-02-02T05:07:58.280488Z","shell.execute_reply.started":"2023-02-02T05:07:55.27922Z","shell.execute_reply":"2023-02-02T05:07:58.279522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 256\nIMG_DIR = \"/kaggle/input/rsna-breast-cancer-256-pngs\"\nN_FOLD = 5\nBATCH_SIZE = 32*strategy.num_replicas_in_sync\n# BATCH_SIZE = 2\nAUTO = tf.data.AUTOTUNE\nBUFFER_SIZE = 1024\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:50:49.971351Z","iopub.execute_input":"2023-02-02T05:50:49.973094Z","iopub.status.idle":"2023-02-02T05:50:49.979449Z","shell.execute_reply.started":"2023-02-02T05:50:49.973054Z","shell.execute_reply":"2023-02-02T05:50:49.978292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=SEED):\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:50:53.421437Z","iopub.execute_input":"2023-02-02T05:50:53.421832Z","iopub.status.idle":"2023-02-02T05:50:53.610841Z","shell.execute_reply.started":"2023-02-02T05:50:53.421778Z","shell.execute_reply":"2023-02-02T05:50:53.609765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ndf[\"path\"] = IMG_DIR + '/' + df.patient_id.astype(str) +'_' + df.image_id.astype(str) +'.png'","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:50:57.277851Z","iopub.execute_input":"2023-02-02T05:50:57.279088Z","iopub.status.idle":"2023-02-02T05:50:57.41805Z","shell.execute_reply.started":"2023-02-02T05:50:57.279043Z","shell.execute_reply":"2023-02-02T05:50:57.417069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sgkf = StratifiedGroupKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\nfor k, (_, test_idx) in enumerate(sgkf.split(df, df.cancer, groups=df.patient_id)):\n    df.loc[test_idx, \"fold\"] = k\ndf.fold = df.fold.astype(int)\ndf.groupby(\"fold\").cancer.mean()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:50:59.458568Z","iopub.execute_input":"2023-02-02T05:50:59.458947Z","iopub.status.idle":"2023-02-02T05:51:04.284903Z","shell.execute_reply.started":"2023-02-02T05:50:59.458914Z","shell.execute_reply":"2023-02-02T05:51:04.283653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(df):\n    paths = df.path.values\n    labels = df.cancer.values\n    \n    def decode_img(paths, labels):\n        raw = tf.io.read_file(paths)\n        img = tf.io.decode_png(raw, channels=3)\n        img = tf.image.resize(img, (IMG_SIZE, IMG_SIZE))\n#         img = tf.cast(img, tf.float32) / 255.0\n        img = tf.reshape(img, (IMG_SIZE, IMG_SIZE, 3))\n        return img, tf.cast(labels, tf.float32)\n\n    ds = tf.data.Dataset.from_tensor_slices((paths, labels))\n    ds = ds.map(decode_img, num_parallel_calls=AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:51:04.287237Z","iopub.execute_input":"2023-02-02T05:51:04.287971Z","iopub.status.idle":"2023-02-02T05:51:04.297224Z","shell.execute_reply.started":"2023-02-02T05:51:04.287929Z","shell.execute_reply":"2023-02-02T05:51:04.295852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Augment(layers.Layer):\n    def __init__(self, target_size=IMG_SIZE, prob=0.5, seed=SEED):\n        super().__init__()\n        self.target_size = target_size\n        self.prob = prob\n        self.seed = seed\n        \n    def call(self, images, labels):\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_flip_left_right(images, seed=self.seed)\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_flip_up_down(images, seed=self.seed)\n        # Random crop\n#         if tf.random.uniform([]) < self.prob:\n#             crop_size = tf.random.uniform(shape=(), minval=self.target_size//4, maxval=self.target_size, dtype=tf.int32)\n#             images = tf.image.random_crop(images, (crop_size, crop_size, 3), seed=self.seed)\n#             images = tf.image.resize(images, size=(self.target_size, self.target_size), method=\"bilinear\")\n        \n    # Rotate\n        if tf.random.uniform([]) < self.prob:\n            angle = tf.random.uniform([], -45*math.pi/180, 45*math.pi/180, dtype=tf.float32)\n            images = tfa.image.rotate(images, angle, interpolation=\"bilinear\", fill_mode=\"reflect\")\n        # Shear x\n        if tf.random.uniform([]) < self.prob:\n            s = tf.random.uniform([], -0.1, 0.1, dtype=tf.float32)\n            images = tfa.image.shear_x(images, level=s, replace=[0,0,0])\n        # Shear y\n        if tf.random.uniform([]) < self.prob:\n            s = tf.random.uniform([], -0.1, 0.1, dtype=tf.float32)\n            images = tfa.image.shear_y(images, level=s, replace=[0,0,0])\n            \n        # Random zoom\n        if tf.random.uniform([]) < self.prob:\n            w = self.target_size\n            h = self.target_size\n            s = tf.random.uniform([], 0.8, 1.2)\n            scale = (s, s)\n            translate_x =  0.5 * (w - w * scale[0])\n            translate_y = 0.5 * (h - h * scale[1])\n            matrix = [scale[0], 0.0, translate_x, 0.0, scale[1], translate_y, 0.0, 0.0]\n            images = tfa.image.transform(images, matrix, fill_mode=\"constant\")\n        # Shift\n        if tf.random.uniform([]) < self.prob:\n            dx = tf.random.uniform([], -20, 20, dtype=tf.int32)\n            dy = tf.random.uniform([], -20, 20, dtype=tf.int32)\n            images = tfa.image.translate(images, (dx, dy), fill_mode=\"constant\")\n        # Random cutout\n        if tf.random.uniform([]) < self.prob:\n            t = tf.random.uniform([], 10, 15, dtype=tf.int32)\n            for i in range(t):\n                m = tf.random.uniform([], self.target_size//40, self.target_size//30, dtype=tf.int32)*2\n                n = tf.random.uniform([], self.target_size//40, self.target_size//30, dtype=tf.int32)*2\n                images = tf.expand_dims(images, axis=0)\n                images = tfa.image.random_cutout(images, (m, n))\n                images = tf.squeeze(images)\n        \n        # elastic deform\n        if tf.random.uniform([]) < self.prob:\n            size = tf.cast(tf.shape(images), tf.int32)\n            scale = 20\n            strength = 20\n            flow = tf.random.uniform([tf.math.floordiv(size[0], scale),\n                                  tf.math.floordiv(size[1], scale),\n                                  2], -1, 1)\n            flow = tfa.image.gaussian_filter2d(flow, filter_shape=(3, 3), sigma=5) * strength\n            flow = tf.image.resize(flow, size[0:2])\n\n            images = tfa.image.dense_image_warp(tf.expand_dims(images, axis=0), tf.expand_dims(flow, axis=0))[0]\n        \n        # pixel-level transforms\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_brightness(images, 0.2)\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_contrast(images, 0.5, 1.2)\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_hue(images, 0.2)\n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_saturation(images, 0.5, 1.5)\n            \n        images = tf.clip_by_value(images, 0, 255)\n        images /= 255.0\n        \n        if tf.random.uniform([]) < self.prob:\n            images = tf.image.random_jpeg_quality(images, 75, 100)\n            \n        if tf.random.uniform([]) < self.prob:\n            g = tf.random.uniform([], 0.7, 1.5, dtype=tf.float32)\n            images = tf.image.adjust_gamma(images, g)\n            \n#         if tf.random.uniform([]) < self.prob:\n#             images = tfa.image.gaussian_filter2d(images)\n        images = tf.reshape(images, (self.target_size, self.target_size, 3))\n        \n        return images, labels\n            ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:22:12.833198Z","iopub.execute_input":"2023-02-02T07:22:12.83357Z","iopub.status.idle":"2023-02-02T07:22:12.863711Z","shell.execute_reply.started":"2023-02-02T07:22:12.833538Z","shell.execute_reply":"2023-02-02T07:22:12.862833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AugmentVal(keras.layers.Layer):\n    def __init__(self, target_size=IMG_SIZE, seed=SEED):\n        super().__init__()\n        self.target_size = target_size\n    def call(self, images, labels):\n#         images = tf.image.resize(images, size=(self.target_size, self.target_size), method='bilinear')\n#         masks = tf.image.resize(masks, size=(self.target_size, self.target_size), method='nearest')\n#         images /= 255.0\n        images = tf.cast(images, tf.float32) / 255.0\n        return images, labels","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:22:16.647043Z","iopub.execute_input":"2023-02-02T07:22:16.647407Z","iopub.status.idle":"2023-02-02T07:22:16.655467Z","shell.execute_reply.started":"2023-02-02T07:22:16.647377Z","shell.execute_reply":"2023-02-02T07:22:16.654455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(df, shuffle=True, augment=None, repeat=True):\n    ds = load_dataset(df)\n    options = tf.data.Options()\n    options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA\n    ds = ds.with_options(options)\n    if augment is not None:\n        ds = ds.map(augment)\n    if repeat:\n        ds = ds.repeat()\n    if shuffle:\n        ds = ds.shuffle(BUFFER_SIZE)\n    ds = ds.batch(BATCH_SIZE)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:22:18.986179Z","iopub.execute_input":"2023-02-02T07:22:18.986742Z","iopub.status.idle":"2023-02-02T07:22:18.997054Z","shell.execute_reply.started":"2023-02-02T07:22:18.986696Z","shell.execute_reply":"2023-02-02T07:22:18.995746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train dataset sample\nsample_ds = get_dataset(df, shuffle=False, augment=Augment())\nplt.figure(figsize=(12, 8))\nfor ims, labels in sample_ds.take(1):\n    for i in range(8):\n        plt.subplot(2, 4, i+1)\n        plt.imshow(ims[i], cmap='bone')\n        plt.title(str(labels[i].numpy()))\n        plt.axis(\"off\")\n        ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:22:22.105954Z","iopub.execute_input":"2023-02-02T07:22:22.106563Z","iopub.status.idle":"2023-02-02T07:22:29.16866Z","shell.execute_reply.started":"2023-02-02T07:22:22.106507Z","shell.execute_reply":"2023-02-02T07:22:29.167708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation dataset sample (no augment)\nsample_ds = get_dataset(df, shuffle=False, augment=AugmentVal())\nplt.figure(figsize=(12, 8))\nfor ims, labels in sample_ds.take(1):\n    for i in range(8):\n        plt.subplot(2, 4, i+1)\n        plt.imshow(ims[i], cmap='bone')\n        plt.title(str(labels[i].numpy()))\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-02-02T05:45:34.901103Z","iopub.execute_input":"2023-02-02T05:45:34.901507Z","iopub.status.idle":"2023-02-02T05:45:36.120282Z","shell.execute_reply.started":"2023-02-02T05:45:34.901473Z","shell.execute_reply":"2023-02-02T05:45:36.117159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric","metadata":{}},{"cell_type":"code","source":"class pFBeta(keras.metrics.Metric):\n    def __init__(self, beta=1, epsilon=1e-5, name=\"pF1\", **kwargs):\n        super().__init__(name=name, **kwargs)\n        self.beta = beta  # when beta=1, metric is pF1\n        self.epsilon = epsilon\n        self.pos = self.add_weight(name=\"pos\", initializer=\"zeros\")\n        self.ctp = self.add_weight(name=\"ctp\", initializer=\"zeros\")\n        self.cfp = self.add_weight(name='cfp', initializer=\"zeros\")\n        \n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.cast(y_true, tf.float32)\n        y_pred = tf.clip_by_value(y_pred, 0, 1)\n        pos = tf.reduce_sum(y_true)\n        ctp = tf.reduce_sum(y_pred[y_true==1])\n        cfp = tf.reduce_sum(y_pred[y_true==0])\n        self.pos.assign_add(pos)\n        self.ctp.assign_add(ctp)\n        self.cfp.assign_add(cfp)\n    \n    def result(self):\n        beta_squared = self.beta * self.beta\n        c_precision = self.ctp / (self.ctp + self.cfp + self.epsilon)\n        c_recall = self.ctp / (self.pos + self.epsilon)\n        result = (1 + beta_squared) * c_precision * c_recall / (beta_squared * c_precision + c_recall)\n        return tf.cond(c_precision >0 and c_recall >0, lambda: result, lambda: 0.0)\n    \n    def reset_state(self):\n        self.pos.assign(0)\n        self.ctp.assign(0)\n        self.cfp.assign(0)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:12:44.429Z","iopub.execute_input":"2023-02-02T06:12:44.429367Z","iopub.status.idle":"2023-02-02T06:12:44.442277Z","shell.execute_reply.started":"2023-02-02T06:12:44.429337Z","shell.execute_reply":"2023-02-02T06:12:44.44127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def build_model():\n    base_model = efficientnet.EfficientNetV1B0(input_shape=(IMG_SIZE, IMG_SIZE, 3), \n                                               num_classes=0,\n                                               pretrained=\"noisy_student\",\n                                               dropout=0.2,\n                                              )\n    inp = base_model.inputs\n    x = base_model.output\n    x = layers.GlobalAveragePooling2D()(x)\n#     x = layers.Dense(32, activation=\"relu\")(x)\n    x = layers.Dropout(0.2)(x)\n    output = layers.Dense(1, activation=\"sigmoid\")(x)\n    model = keras.Model(inputs=inp, outputs=output)\n    train_num = int(len(df)*(N_FOLD - 1)/N_FOLD)\n    lr_schedule = keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=1e-4,\n                                                           decay_steps=train_num//BATCH_SIZE,\n                                                           decay_rate=0.9)\n    optimizer = keras.optimizers.Adam(learning_rate=lr_schedule)\n    pf1 = pFBeta(name=\"pF1\")\n    auc = keras.metrics.AUC(name=\"auc\")\n    f1 = tfa.metrics.F1Score(num_classes=1, threshold=0.5)\n    model.compile(\n#         loss=tfa.losses.SigmoidFocalCrossEntropy(),\n        loss=keras.losses.BinaryCrossentropy(from_logits=False),\n        optimizer=optimizer,\n        metrics = [\"accuracy\", f1, pf1, auc])\n    return model\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:45:25.631192Z","iopub.execute_input":"2023-02-02T06:45:25.63156Z","iopub.status.idle":"2023-02-02T06:45:25.642776Z","shell.execute_reply.started":"2023-02-02T06:45:25.631523Z","shell.execute_reply":"2023-02-02T06:45:25.641766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"historys = []\nfor fold in range(N_FOLD):\n    tf.keras.backend.clear_session()\n    print(f\"{'*'*12} Fold {fold} {'*'*12}\")\n    train_df = df[df.fold!=fold].reset_index(drop=True)\n    val_df = df[df.fold==fold].reset_index(drop=True)\n    TRAIN_NUM = len(train_df)\n    VAL_NUM = len(val_df)\n    train_ds = get_dataset(train_df, shuffle=True, repeat=True)\n    val_ds = get_dataset(val_df, shuffle=False, repeat=False)\n    with strategy.scope():\n        seed_everything()\n        model = build_model()\n    save_path = f\"rsna_model_fold{fold}.h5\"\n    ck_callback = keras.callbacks.ModelCheckpoint(save_path, \n                                                 monitor=\"val_loss\",\n                                                 save_best_only=True,\n#                                                  initial_value_threshold=0.3,\n                                                 verbose=1)\n#     callbacks = [ck_callback]\n    history = model.fit(train_ds, \n                        steps_per_epoch=TRAIN_NUM // BATCH_SIZE,\n                        validation_data=val_ds,\n#                         callbacks=callbacks,\n                        epochs=10,\n                     )\n    model.save_weights(save_path)\n    historys.append(history)\n    break # Comment this line to run all 5 folds","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:45:32.562981Z","iopub.execute_input":"2023-02-02T06:45:32.563582Z","iopub.status.idle":"2023-02-02T07:16:40.21149Z","shell.execute_reply.started":"2023-02-02T06:45:32.563546Z","shell.execute_reply":"2023-02-02T07:16:40.210456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(len(history.history['val_pF1']))\nplt.plot(epochs, history.history['pF1'], label='train pF1')\nplt.plot(epochs, history.history['val_pF1'], label='val pF1')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:16:40.763497Z","iopub.execute_input":"2023-02-02T07:16:40.764128Z","iopub.status.idle":"2023-02-02T07:16:41.026402Z","shell.execute_reply.started":"2023-02-02T07:16:40.764083Z","shell.execute_reply":"2023-02-02T07:16:41.025523Z"},"trusted":true},"execution_count":null,"outputs":[]}]}