{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA 2022 TF-Keras Traing Phase With TPU","metadata":{}},{"cell_type":"markdown","source":"## References, see also them\n\n## [Getting Started: TPUs + Cassava Leaf Disease](https://www.kaggle.com/jessemostipak/getting-started-tpus-cassava-leaf-disease)\n\n## [CutMix and MixUp on GPU/TPU](https://www.kaggle.com/cdeotte/cutmix-and-mixup-on-gpu-tpu)\n\n## [Getting started with 100+ flowers on TPU](https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu)\n\n## [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)","metadata":{}},{"cell_type":"markdown","source":"## My Other Works\n### [Create IDwise TF Records](https://www.kaggle.com/code/itsuki9180/create-idwise-tf-records)\n### [RSNA 2022 TF-Keras Traing Phase With TPU](https://www.kaggle.com/code/itsuki9180/rsna-2022-tf-keras-traing-phase-with-tpu) <- Here\n### [RSNA 2022 TF-Keras Inference Phase With GPU](https://www.kaggle.com/itsuki9180/rsna-2022-tf-keras-inference-phase-with-gpu) ","metadata":{}},{"cell_type":"markdown","source":"## Set up environment","metadata":{}},{"cell_type":"code","source":"import math, re, os, gc\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow import keras\nfrom functools import partial\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:39.330894Z","iopub.execute_input":"2023-01-01T21:16:39.331676Z","iopub.status.idle":"2023-01-01T21:16:39.342204Z","shell.execute_reply.started":"2023-01-01T21:16:39.331623Z","shell.execute_reply":"2023-01-01T21:16:39.340274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null\nimport efficientnet.tfkeras as efn","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:39.344307Z","iopub.execute_input":"2023-01-01T21:16:39.344601Z","iopub.status.idle":"2023-01-01T21:16:46.861831Z","shell.execute_reply.started":"2023-01-01T21:16:39.344568Z","shell.execute_reply":"2023-01-01T21:16:46.860658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detect TPU","metadata":{}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:46.863672Z","iopub.execute_input":"2023-01-01T21:16:46.863981Z","iopub.status.idle":"2023-01-01T21:16:53.02328Z","shell.execute_reply.started":"2023-01-01T21:16:46.863939Z","shell.execute_reply":"2023-01-01T21:16:53.022121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up variables","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nGCS_PATH_STRATIFICATED = KaggleDatasets().get_gcs_path('rsna2022-split-fold-id-wise-tfr')\nBATCH_SIZE = 16 if strategy.num_replicas_in_sync==8 else 1\nBATCH_SIZE *= strategy.num_replicas_in_sync\nAUG_BATCH = BATCH_SIZE\nIMAGE_SIZE = [512, 512]\nEPOCHS = 100\nFOLDS = 5\n\nGCS_PATH_STRATIFICATED","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:53.025445Z","iopub.execute_input":"2023-01-01T21:16:53.025746Z","iopub.status.idle":"2023-01-01T21:16:53.361946Z","shell.execute_reply.started":"2023-01-01T21:16:53.02571Z","shell.execute_reply":"2023-01-01T21:16:53.360517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decode the data and Define data loading methods","metadata":{}},{"cell_type":"code","source":"l_type = tf.int64","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:53.364271Z","iopub.execute_input":"2023-01-01T21:16:53.364688Z","iopub.status.idle":"2023-01-01T21:16:53.369681Z","shell.execute_reply.started":"2023-01-01T21:16:53.364642Z","shell.execute_reply":"2023-01-01T21:16:53.368749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"patient_overall\": tf.io.FixedLenFeature([], l_type),\n        \"C1\": tf.io.FixedLenFeature([], l_type),\n        \"C2\": tf.io.FixedLenFeature([], l_type),  # shape [] means single element\n        \"C3\": tf.io.FixedLenFeature([], l_type),\n        \"C4\": tf.io.FixedLenFeature([], l_type),\n        \"C5\": tf.io.FixedLenFeature([], l_type),\n        \"C6\": tf.io.FixedLenFeature([], l_type),\n        \"C7\": tf.io.FixedLenFeature([], l_type),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    patient_overall = tf.cast(example['patient_overall'], tf.float32)\n    c1 = tf.cast(example['C1'], tf.float32)\n    c2 = tf.cast(example['C2'], tf.float32)\n    c3 = tf.cast(example['C3'], tf.float32)\n    c4 = tf.cast(example['C4'], tf.float32)\n    c5 = tf.cast(example['C5'], tf.float32)\n    c6 = tf.cast(example['C6'], tf.float32)\n    c7 = tf.cast(example['C7'], tf.float32)\n    return image, [patient_overall, c1, c2, c3, c4, c5, c6, c7]# returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    return image # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(img, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    img = tf.image.random_flip_left_right(img)\n    #img = tf.image.random_hue(img, 0.01)\n    img = tf.image.random_saturation(img, 0.7, 1.3)\n    img = tf.image.random_contrast(img, 0.8, 1.2)\n    img = tf.image.random_brightness(img, 0.1)\n    return img, label\n\ndef get_training_dataset(dataset, do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.batch(AUG_BATCH)\n    #if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO) # note we put AFTER batching\n    dataset = dataset.unbatch()\n    dataset = dataset.shuffle(2048 if strategy.num_replicas_in_sync==8 else 384)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset, do_onehot=False, shuffle=True):\n    dataset = dataset.batch(BATCH_SIZE)\n    if do_onehot: dataset = dataset.map(onehot, num_parallel_calls=AUTO) # we must use one hot like augmented train data\n    if shuffle: dataset = dataset.shuffle(2048 if strategy.num_replicas_in_sync==8 else 384)\n    #dataset = dataset.repeat()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.cache()\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    #print(n)\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:53.37274Z","iopub.execute_input":"2023-01-01T21:16:53.373041Z","iopub.status.idle":"2023-01-01T21:16:53.400741Z","shell.execute_reply.started":"2023-01-01T21:16:53.373009Z","shell.execute_reply":"2023-01-01T21:16:53.399225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building our model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    with strategy.scope():\n        model = tf.keras.Sequential([\n            efn.EfficientNetB7(input_shape=(*IMAGE_SIZE,3),weights='noisy-student',include_top=False),\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dropout(0.25),\n            tf.keras.layers.Dense(256, activation='swish'),\n            tf.keras.layers.Dropout(0.25),\n            tf.keras.layers.Dense(8, activation='sigmoid')\n        ])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(),\n            loss='binary_crossentropy',\n            metrics=[tf.keras.metrics.AUC(multi_label=True), 'acc'])\n        model.summary()\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-01-01T21:16:53.404263Z","iopub.execute_input":"2023-01-01T21:16:53.404677Z","iopub.status.idle":"2023-01-01T21:16:53.419395Z","shell.execute_reply.started":"2023-01-01T21:16:53.404634Z","shell.execute_reply":"2023-01-01T21:16:53.417628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import VGG16\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import layers as L\ndef get_vgg_model():\n    with strategy.scope():\n        vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=(*IMAGE_SIZE,3))\n\n        model_vgg16 = Sequential()\n        model_vgg16.add(vgg16)\n        model_vgg16.add(L.GlobalAveragePooling2D())\n        model_vgg16.add(L.Dense(8, activation='softmax'))\n        model_vgg16.compile(\n            optimizer=tf.keras.optimizers.Adam(),\n            loss='binary_crossentropy',\n            metrics=[tf.keras.metrics.AUC(multi_label=True), 'acc'])\n        model_vgg16.summary()\n    return model_vgg16","metadata":{"execution":{"iopub.status.busy":"2023-01-01T22:13:12.09562Z","iopub.execute_input":"2023-01-01T22:13:12.096139Z","iopub.status.idle":"2023-01-01T22:13:12.106749Z","shell.execute_reply.started":"2023-01-01T22:13:12.096085Z","shell.execute_reply":"2023-01-01T22:13:12.105677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=False)\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    if fold==(FOLDS-1):\n        idxTT = idxT; idxVV = idxV\n        print('### Using fold',fold,'for experiments')\n    print('Fold',fold,'has TRAIN:',idxT,'VALID:',idxV)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T22:10:11.074047Z","iopub.execute_input":"2023-01-01T22:10:11.074517Z","iopub.status.idle":"2023-01-01T22:10:11.089243Z","shell.execute_reply.started":"2023-01-01T22:10:11.07446Z","shell.execute_reply":"2023-01-01T22:10:11.087746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train loop","metadata":{}},{"cell_type":"code","source":"for fold,(idxT,idxV) in enumerate(skf.split(np.arange(FOLDS))):\n    print(); print('#'*25)\n    print('### FOLD',fold+1)\n    print('#'*25)\n    files_train = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxT])\n    files_valid = tf.io.gfile.glob([GCS_PATH_STRATIFICATED + '/train%.2i*.tfrec'%x for x in idxV])\n\n    NUM_TRAINING_IMAGES = int( count_data_items(files_train))\n    NUM_VALIDATION_IMAGES = int( count_data_items(files_valid) )\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('Dataset: {} training images, {} validation images,'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))\n\n    train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': files_train}).loc[:]['TRAINING_FILENAMES']), labeled = True)\n    val_dataset = load_dataset(list(pd.DataFrame({'VALIDATION_FILENAMES': files_valid}).loc[:]['VALIDATION_FILENAMES']), labeled = True, ordered = True)\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n    lrs = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor=\"val_loss\", patience=3, min_lr=1e-5, factor=0.5, verbose=1)\n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss',\n        min_delta=0,\n        patience=10,\n        verbose=1,\n        mode='auto',\n        )\n\n    model = get_vgg_model()\n    history = model.fit(\n        get_training_dataset(train_dataset), \n        steps_per_epoch = STEPS_PER_EPOCH,\n        epochs = EPOCHS,\n        callbacks = [sv, lrs, es],\n        validation_data = get_validation_dataset(val_dataset, shuffle=False),\n        verbose=2\n    )\n    del model\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T22:13:14.586737Z","iopub.execute_input":"2023-01-01T22:13:14.587166Z","iopub.status.idle":"2023-01-01T22:28:42.596769Z","shell.execute_reply.started":"2023-01-01T22:13:14.587131Z","shell.execute_reply":"2023-01-01T22:28:42.595662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize training \ndef viz_loss(history):\n    train_loss = history[\"loss\"]\n    val_loss = history[\"val_loss\"]\n    iters = [i for i in range(1)]\n    \n    plt.plot(iters, train_loss, label=\"Training Loss\")\n    plt.plot(iters, val_loss, label=\"Validation Loss\")\n    plt.title(\"A plot of Loss against number of iterations\")\n    plt.legend()\n    plt.show()\n    \ndef viz_acc(history):\n    train_loss = history[\"categorical_accuracy\"]\n    val_loss = history[\"val_categorical_accuracy\"]\n    iters = [i for i in range(EPOCHS)]\n    \n    plt.plot(iters, train_loss, label=\"Training Accuracy\")\n    plt.plot(iters, val_loss, label=\"Validation Accuracy\")\n    plt.title(\"A plot of Accuracy against number of iterations\")\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T22:28:42.629892Z","iopub.execute_input":"2023-01-01T22:28:42.630303Z","iopub.status.idle":"2023-01-01T22:28:42.643405Z","shell.execute_reply.started":"2023-01-01T22:28:42.630266Z","shell.execute_reply":"2023-01-01T22:28:42.641115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"viz_loss(history.history)\nviz_acc(history.history)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T22:28:42.650314Z","iopub.execute_input":"2023-01-01T22:28:42.651759Z","iopub.status.idle":"2023-01-01T22:28:42.92003Z","shell.execute_reply.started":"2023-01-01T22:28:42.651695Z","shell.execute_reply":"2023-01-01T22:28:42.918805Z"},"trusted":true},"execution_count":null,"outputs":[]}]}