{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n
\n\n

Mayo Clinic - STRIP AI

No Tile Model


\n
CREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n
\n 🛑   WARNING:

THIS IS A WORK IN PROGRESS
\n
\n\n\n
\n 👏   IF YOU FORK THIS OR FIND THIS HELPFUL   👏

PLEASE UPVOTE!

This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. 😅\n
","metadata":{}},{"cell_type":"markdown","source":"

\n\n

\n\n

TABLE OF CONTENTS

\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    SETUP

\n\n---\n\n

2    HELPER FUNCTIONS

\n\n---\n\n

3    CREATE DATASET

\n\n---\n\n

4    MODEL TRAINING

\n\n---","metadata":{}},{"cell_type":"markdown","source":"
\n\n\n\n

0  IMPORTS    ⤒

","metadata":{}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_hub as tfhub; print(f\"\\t\\t– TENSORFLOW HUB VERSION: {tfhub.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\t– SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom scipy.spatial import cKDTree\n\n# # RAPIDS\n# import cudf, cupy, cuml\n# from cuml.neighbors import NearestNeighbors\n# from cuml.manifold import TSNE, UMAP\n# from cuml import PCA\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom zipfile import ZipFile\nfrom glob import glob\nimport openslide\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nfrom matplotlib.patches import Rectangle\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport tifffile as tif\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance; Image.MAX_IMAGE_PIXELS = 5_000_000_000;\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n\nimport plotly.io as pio\nprint(pio.renderers)\n\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\nseed_it_all()\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:44:32.930583Z","iopub.execute_input":"2022-07-13T17:44:32.930997Z","iopub.status.idle":"2022-07-13T17:44:36.468554Z","shell.execute_reply.started":"2022-07-13T17:44:32.930902Z","shell.execute_reply":"2022-07-13T17:44:36.467485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

1  SETUP    ⤒

\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"
\n\n

1.1 ACCELERATOR DETECTION

\n\n---\n\nIn order to use **`TPU`**, we use **`TPUClusterResolver`** for the initialization which is necessary to connect to the remote cluster and initialize cloud TPUs. Let's go over two important points\n\n1. When using TPU on Kaggle, you don't need to specify arguments for **`TPUClusterResolver`**\n2. However, on **G**oogle **C**ompute **E**ngine (**GCE**), you will need to do the following:\n\n
\n\n```python\n# The name you gave to the TPU to use\nTPU_WORKER = 'my-tpu-name'\n\n# or you can also specify the grpc path directly\n# TPU_WORKER = 'grpc://xxx.xxx.xxx.xxx:8470'\n\n# The zone you chose when you created the TPU to use on GCP.\nZONE = 'us-east1-b'\n\n# The name of the GCP project where you created the TPU to use on GCP.\nPROJECT = 'my-tpu-project'\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER, zone=ZONE, project=PROJECT)\n```\n\n
\n 🛑   WARNING:

- Although the Tensorflow documentation says it is the project name that should be provided for the argument `project`, it is actually the Project ID, that you should provide. This can be found on the GCP project dashboard page.
\n
\n\n
\n 📖   REFERENCES:

\n - Guide - Use TPUs
\n - Doc - TPUClusterResolver
\n\n
","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy()\n tf.config.experimental.set_memory_growth(tf.config.list_physical_devices('GPU')[0], True)\n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-13T17:44:36.470288Z","iopub.execute_input":"2022-07-13T17:44:36.470533Z","iopub.status.idle":"2022-07-13T17:44:42.602922Z","shell.execute_reply.started":"2022-07-13T17:44:36.470507Z","shell.execute_reply":"2022-07-13T17:44:42.60201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

1.2 COMPETITION DATA ACCESS

\n\n---\n\nTPUs read data must be read directly from **G**oogle **C**loud **S**torage **(GCS)**. Kaggle provides a utility library – **`KaggleDatasets`** – which has a utility function **`.get_gcs_path`** that will allow us to access the location of our input datasets within **GCS**.

\n\n
\n 📌   TIPS:

- If you have multiple datasets attached to the notebook, you should pass the name of a specific dataset to the `get_gcs_path()` function. In our case, the name of the dataset is the name of the directory the dataset is mounted within.

\n
","metadata":{}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n IMAGE_DIR = KaggleDatasets().get_gcs_path('jpg-images-strip-ai')\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\nelse:\n IMAGE_DIR = \"/kaggle/input/jpg-images-strip-ai\"\n save_locally = None\n load_locally = None\n\nDATA_DIR = \"/kaggle/input/mayo-clinic-strip-ai\"\nprint(f\"... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\\n\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-13T17:44:42.604644Z","iopub.execute_input":"2022-07-13T17:44:42.604954Z","iopub.status.idle":"2022-07-13T17:44:43.090289Z","shell.execute_reply.started":"2022-07-13T17:44:42.604913Z","shell.execute_reply":"2022-07-13T17:44:43.089298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

1.3 LEVERAGING XLA OPTIMIZATIONS

\n\n---\n\n\n**XLA** (Accelerated Linear Algebra) is a domain-specific compiler for linear algebra that can accelerate TensorFlow models with potentially no source code changes. **The results are improvements in speed and memory usage**.\n\n
\n\nWhen a TensorFlow program is run, all of the operations are executed individually by the TensorFlow executor. Each TensorFlow operation has a precompiled GPU/TPU kernel implementation that the executor dispatches to.\n\nXLA provides us with an alternative mode of running models: it compiles the TensorFlow graph into a sequence of computation kernels generated specifically for the given model. Because these kernels are unique to the model, they can exploit model-specific information for optimization.

\n\n
\n 🛑   WARNING:

- XLA can not currently compile functions where dimensions are not inferrable: that is, if it's not possible to infer the dimensions of all tensors without running the entire computation
\n
\n\n
\n 📌   NOTE:

- XLA compilation is only applied to code that is compiled into a graph (in TF2 that's only a code inside tf.function).
- The jit_compile API has must-compile semantics, i.e. either the entire function is compiled with XLA, or an errors.InvalidArgumentError exception is thrown)\n
\n\n
\n 📖   REFERENCE:

- XLA: Optimizing Compiler for Machine Learning
\n
","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-13T17:44:43.092339Z","iopub.execute_input":"2022-07-13T17:44:43.092562Z","iopub.status.idle":"2022-07-13T17:44:43.098179Z","shell.execute_reply.started":"2022-07-13T17:44:43.092536Z","shell.execute_reply":"2022-07-13T17:44:43.097325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

1.4 BASIC DATA DEFINITIONS & INITIALIZATIONS

\n\n---\n\nWe load an evaluation dataset to test our approach (caltech256)","metadata":{}},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\n# From dataset\nINPUT_SHAPE = (512,512,3)\nS2I_LBL_MAP = {\"CE\":0, \"LAA\":1}\nI2S_LBL_MAP = {v:k for k,v in S2I_LBL_MAP.items()}\nN_CLASSES=len(S2I_LBL_MAP)\n\n# Open the training dataframe and display the initial dataframe\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df[\"image_path\"] = train_df[\"image_id\"].apply(lambda x: os.path.join(IMAGE_DIR, \"train\", x+\".jpg\"))\n\nprint(\"\\n... TRAINING DATAFRAME... \\n\")\ndisplay(train_df)\n\n# Open the testing dataframe and display the initial dataframe\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\nTEST_CSV = os.path.join(DATA_DIR, \"test.csv\")\ntest_df = pd.read_csv(TEST_CSV)\ntest_df[\"image_path\"] = test_df[\"image_id\"].apply(lambda x: os.path.join(IMAGE_DIR, \"test\", x+\".jpg\"))\n\nprint(\"\\n... TESTING DATAFRAME... \\n\")\ndisplay(test_df)\n\n# Open the sample submission dataframe\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\n\nprint(\"\\n... SAMPLE SUBMISSION DATAFRAME... \\n\")\ndisplay(ss_df)\n\nprint(\"\\n... BASIC DATA SETUP FINISHED ...\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:44:43.099586Z","iopub.execute_input":"2022-07-13T17:44:43.099827Z","iopub.status.idle":"2022-07-13T17:44:43.164439Z","shell.execute_reply.started":"2022-07-13T17:44:43.099794Z","shell.execute_reply":"2022-07-13T17:44:43.163448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

2  HELPER FUNCTIONS    ⤒

","metadata":{}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\ndef get_train_val_split(df, val_nth=8):\n \"\"\" Get a single stratified fold \"\"\"\n gkf = GroupKFold(n_splits=val_nth) \n for train_idxs, val_idxs in gkf.split(df[\"image_id\"], df[\"label\"]+df[\"center_id\"].astype(str), df[\"patient_id\"]): \n # Get subsets of dataset according to split\n sub_train_df=df.iloc[train_idxs]\n sub_val_df=df.iloc[val_idxs]\n \n # Shuffle datasets\n sub_train_df=sub_train_df.sample(len(sub_train_df)).reset_index(drop=True)\n sub_val_df=sub_val_df.sample(len(sub_val_df)).reset_index(drop=True) \n # Exit and return\n break\n return (sub_train_df, len(sub_train_df)), (sub_val_df, len(sub_val_df))\n\ndef tf_load_image(img_path, img_shape=(512,512,3)):\n \"\"\" Load an image with the correct size and shape \"\"\"\n img = tf.image.decode_image(tf.io.read_file(img_path), channels=img_shape[-1])\n img = tf.reshape(img, img_shape)\n return img\n\ndef augment_batch(img_batch):\n img_batch = tf.image.random_brightness(img_batch, 0.2)\n img_batch = tf.image.random_contrast(img_batch, 0.5, 2.0)\n img_batch = tf.image.random_saturation(img_batch, 0.75, 1.25)\n img_batch = tf.image.random_hue(img_batch, 0.1)\n return img_batch","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:44:43.165724Z","iopub.execute_input":"2022-07-13T17:44:43.165936Z","iopub.status.idle":"2022-07-13T17:44:43.17796Z","shell.execute_reply.started":"2022-07-13T17:44:43.165912Z","shell.execute_reply":"2022-07-13T17:44:43.176829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 3  CREATE DATASET    ⤒\n

\n\n---\n\nCreate **`tf.data.Dataset`** for training.\nIn the future we will use tfrecords but for now we will just use the raw images","metadata":{}},{"cell_type":"code","source":"N_EPOCHS = 12\nREPLICA_BATCH_SIZE = 8\nBATCH_SIZE = N_REPLICAS*REPLICA_BATCH_SIZE\nSHUFFLE_BUFFER = BATCH_SIZE*5\nAUTOTUNE = tf.data.AUTOTUNE\n\n(sub_train_df, N_TRAIN), (sub_val_df, N_VAL) = get_train_val_split(train_df.copy())\ntrain_ds = tf.data.Dataset.from_tensor_slices((\n sub_train_df.image_path.values, \n sub_train_df.label.map(S2I_LBL_MAP).values\n))\ntrain_ds = train_ds.map(lambda x,y: (tf_load_image(x, INPUT_SHAPE), tf.one_hot(y, N_CLASSES, dtype=tf.uint8)), num_parallel_calls=AUTOTUNE)\\\n .shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE, drop_remainder=True)\\\n .map(lambda x,y: (augment_batch(x), y), num_parallel_calls=AUTOTUNE)\\\n .map(lambda x,y: (tf.cast(x, tf.float32), y), num_parallel_calls=AUTOTUNE)\\\n .prefetch(tf.data.AUTOTUNE)\n\nval_ds = tf.data.Dataset.from_tensor_slices((\n sub_val_df.image_path.values, \n sub_val_df.label.map(S2I_LBL_MAP).values\n))\nval_ds = val_ds.map(lambda x,y: (tf_load_image(x, INPUT_SHAPE), tf.one_hot(y, N_CLASSES, dtype=tf.uint8)), num_parallel_calls=AUTOTUNE)\\\n .shuffle(SHUFFLE_BUFFER//3).batch(BATCH_SIZE, drop_remainder=True)\\\n .map(lambda x,y: (tf.cast(x, tf.float32), y), num_parallel_calls=AUTOTUNE)\\\n .prefetch(AUTOTUNE)\n\nprint(f\"\\n\\nTRAIN DATASET:\\n\\t{train_ds}\")\nprint(f\"\\nVALIDATION DATASET:\\n\\t{val_ds}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:44:43.17948Z","iopub.execute_input":"2022-07-13T17:44:43.179757Z","iopub.status.idle":"2022-07-13T17:44:43.551432Z","shell.execute_reply.started":"2022-07-13T17:44:43.179728Z","shell.execute_reply":"2022-07-13T17:44:43.550517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

4  MODEL TRAINING    ⤒

\n\n
\n\n**IMAGE SIZES FOR DIFFERENT EFFICIENTNET MODELS**\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
Base modelresolution
EfficientNetB0224
EfficientNetB1240
EfficientNetB2260
EfficientNetB3300
EfficientNetB4380
EfficientNetB5456
EfficientNetB6528
EfficientNetB7600
\n\n
\n\nTherefore we will use **EFFICIENTNETB6** as it has the closest size to our dataset (528 v. 512)\n\n
","metadata":{}},{"cell_type":"markdown","source":"
\n\n

4.1 LOAD/CREATE MODEL

\n\n---\n\nGet class weights and load model","metadata":{}},{"cell_type":"code","source":"def build_mcsai_notile_model(tf_keras_model_fn, _weights=\"imagenet\", top_dropout=0.5):\n _inputs = tf.keras.layers.Input(shape=INPUT_SHAPE, dtype=tf.float32)\n _bb = tf_keras_model_fn(include_top=False, input_shape=INPUT_SHAPE, weights=_weights, pooling=\"avg\")\n x = tf.keras.layers.Dropout(top_dropout)(_bb(_inputs))\n _outputs = tf.keras.layers.Dense(N_CLASSES, activation=\"softmax\")(x)\n return tf.keras.Model(inputs=_inputs, outputs=_outputs)\n\ndef get_class_weights(df):\n __min_count = train_df.label.value_counts().values.min()\n _class_weights = {S2I_LBL_MAP[_cls]:__min_count/_cnt for _cls, _cnt in train_df.label.value_counts().items()}\n return _class_weights\n \nprint(\"\\n... GET CLASS WEIGHTS FOR TRAINING ...\")\nclass_weights = get_class_weights(train_df)\nfor k,v in class_weights.items(): print(f\"{k} --> {v:.4f}\")\n \nprint(\"\\n\\n\\n... GET MODEL AND PRINT SUMMARY ...\\n\\n\")\n\nwith strategy.scope():\n mcsai_nt_model = build_mcsai_notile_model(tf.keras.applications.EfficientNetB6)\n display(mcsai_nt_model.summary())","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:44:43.552704Z","iopub.execute_input":"2022-07-13T17:44:43.552921Z","iopub.status.idle":"2022-07-13T17:45:28.765318Z","shell.execute_reply.started":"2022-07-13T17:44:43.552895Z","shell.execute_reply":"2022-07-13T17:45:28.764377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

4.2 CREATE LEARNING RATE SCHEDULE

\n\n---\n","metadata":{}},{"cell_type":"code","source":"# REF --> https://freecontent.manning.com/fine-tuning-a-pre-trained-resnet-50/\ndef lr_function(epoch): \n \n # Define terms\n start_lr=tf.constant(0.00001)\n min_lr=tf.constant(0.00001)\n max_lr=tf.constant(0.0001)\n rampup_epochs=tf.constant(4.0)\n sustain_epochs=tf.constant(0.0)\n exp_decay=tf.constant(0.75)\n \n def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n # Rampup phase where the learning rate is increased for rampup_epochs till we reach max_lr\n if epoch < rampup_epochs:\n lr = (((max_lr - start_lr) / rampup_epochs)*epoch + start_lr)\n \n # Sustain phase where we keep learning rate as max_lr for sustain_epochs\n elif epoch < rampup_epochs + sustain_epochs:\n lr = max_lr\n \n # Decay phase where we decay the learning rate by a factor of exp_decay till we reach min_lr\n else:\n lr = ((max_lr - min_lr)*exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr)\n \n return lr\n return lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay)\n \n# Plot for demo purposes\nfig = px.line(x=np.arange(N_EPOCHS),y=[float(lr_function(_epoch)) for _epoch in np.arange(N_EPOCHS)], \n title=f\"Learning Rate Schedule - {N_EPOCHS} Epochs\",\n labels={\"x\":\"Epoch\", \"y\":\"Learning Rate\"}\n )\nfig.update_layout(yaxis=dict(tickformat=\".6f\"))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:46:08.195Z","iopub.execute_input":"2022-07-13T17:46:08.19583Z","iopub.status.idle":"2022-07-13T17:46:08.760849Z","shell.execute_reply.started":"2022-07-13T17:46:08.195784Z","shell.execute_reply":"2022-07-13T17:46:08.759835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

4.3 COMPILE MODEL AND INITIALIZE CALLBACKS

\n\n---\n","metadata":{}},{"cell_type":"code","source":"class GarbageCollectorCallback(tf.keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n gc.collect()\n tf.keras.backend.clear_session()\n\ndef plot_history(_history, fold_num=\"1\", metrics=(\"acc\")):\n \"\"\" TBD \"\"\"\n fig = px.line(_history.history, \n x=range(len(_history.history[\"loss\"])), \n y=[\"loss\", \"val_loss\"],\n labels={\"value\":\"Loss (log-axis)\", \"x\":\"Epoch #\"},\n title=f\"FOLD {fold_num} MODEL - LOSS\", log_y=True\n )\n fig.show()\n\n for _m in metrics:\n fig = px.line(_history.history, \n x=range(len(_history.history[_m])), \n y=[_m, f\"val_{_m}\"],\n labels={\"value\":f\"{_m} (log-axis)\", \"x\":\"Epoch #\"},\n title=f\"FOLD {fold_num} MODEL - {_m}\", log_y=True)\n fig.show() \n \nwith strategy.scope(): \n OPTIMIZER = tf.keras.optimizers.Adam()\n LOSS = \"categorical_crossentropy\"\n METRICS = [\"acc\", tf.keras.metrics.AUC(name=\"auc\")]\n _es_cb = tf.keras.callbacks.EarlyStopping(monitor=\"val_auc\", patience=4, verbose=1, mode=\"max\", restore_best_weights=True)\n _ckpt_cb = tf.keras.callbacks.ModelCheckpoint(f'./efficientnetb6_512_notile', monitor='val_auc', mode=\"max\", save_best_only=True, options=save_locally)\n _gc_cb = GarbageCollectorCallback()\n _lr_cb = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lr_function(epoch))\n CB_LIST = [_es_cb, _ckpt_cb, _lr_cb] # _gc_cb,\n mcsai_nt_model.compile(optimizer=OPTIMIZER, loss=LOSS, metrics=METRICS)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:46:16.566869Z","iopub.execute_input":"2022-07-13T17:46:16.567192Z","iopub.status.idle":"2022-07-13T17:46:16.688067Z","shell.execute_reply.started":"2022-07-13T17:46:16.567157Z","shell.execute_reply":"2022-07-13T17:46:16.687026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n

4.4 TRAIN MODEL

\n\n---\n","metadata":{}},{"cell_type":"code","source":"history = mcsai_nt_model.fit(train_ds, validation_data=val_ds, epochs=N_EPOCHS, callbacks=CB_LIST, class_weight=class_weights)\nplot_history(history, metrics=(\"acc\", \"auc\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T17:46:17.9608Z","iopub.execute_input":"2022-07-13T17:46:17.961129Z","iopub.status.idle":"2022-07-13T18:04:09.212408Z","shell.execute_reply.started":"2022-07-13T17:46:17.961094Z","shell.execute_reply":"2022-07-13T18:04:09.211132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _ in range(5):\n print(mcsai_nt_model.evaluate(val_ds))","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:19:16.057879Z","iopub.execute_input":"2022-07-13T18:19:16.058716Z","iopub.status.idle":"2022-07-13T18:21:08.176925Z","shell.execute_reply.started":"2022-07-13T18:19:16.058679Z","shell.execute_reply":"2022-07-13T18:21:08.176264Z"},"trusted":true},"execution_count":null,"outputs":[]}]}