{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q ../input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install tensorflow_addons","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport argparse\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy\nfrom tensorflow.keras import layers as L\nimport efficientnet.tfkeras as efn\nimport tensorflow.keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fns = ['gs://kds-fdd26a53c479ba2fe0fd09aefd6ba5b5da014d103663a027f2ba8da1/0train_train00-100000.tfrec',\n 'gs://kds-4a274b96dcced2a2d564739ab3265794bbd458d71e6eb5e99ff6a578/0train_train00-100000.tfrec',\n 'gs://kds-839ccc6d5a39024648dbb162ab8c72a9ee59169297f57a257abde0c7/0train_train00-100000.tfrec',\n 'gs://kds-514346ad1ef1306c46b1722bdf0cac13e3b4138d9302eb4b1b820f4f/0train_train00-100000.tfrec',\n 'gs://kds-f6b193f82266500b631303010d4faaffc0d6a18593870dc1b948ef89/0train_train00-100000.tfrec',\n 'gs://kds-094c51c80e99a45ee7b4d9bf3a6bd50a3766e08cdecea2cf34a478a4/0train_train00-100000.tfrec',\n 'gs://kds-7348669340cc33cba51450cdf383dd082456377884297dfe1d1b6328/0train_train00-100000.tfrec',\n 'gs://kds-8b955b9069a2f230c0f031da2a3f7dd122dcb542e2e2f583e400b057/0train_train00-100000.tfrec',\n 'gs://kds-9c24f19b758d727268d0efab4dde21d5073c06828c3e3c7d8c346307/0train_train00-100000.tfrec',\n 'gs://kds-8fcb00abe16292f58c5dc34a3c24dace10e2004ec5848e046d13a2c8/0train_train00-100000.tfrec',\n 'gs://kds-8b562794f0c7cf738927f8df0abb12d6ad4756dc9cab0d0b0f599d47/0train_train00-100000.tfrec',\n 'gs://kds-175a817f8c2bd0b9371a16f8d89f410f1b276a30cd1ad91d80c0c31b/0train_train00-100000.tfrec',\n 'gs://kds-67c4a21d71f589012a90c7555f59f64551f1080920a1ac15a4bbc304/0train_train00-100000.tfrec',\n 'gs://kds-d642de0737aa752538cfdbbde6538f25efd787349d20696fbb06229b/0train_train00-100000.tfrec',\n 'gs://kds-0da7840c65cf60c5c69365e869ec091af5eb8c34cf18611639c2f661/0train_train00-175912.tfrec']\n\nval_fns = ['gs://kds-e6a7a5fbb73e871113c2a6e6506a5d29ce98c7fe70295e8f9bb3f1c8/0val_train00-4558.tfrec']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_strategy():\n  # Detect hardware, return appropriate distribution strategy\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n        print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        strategy = tf.distribute.get_strategy()\n\n    print('REPLICAS: ', strategy.num_replicas_in_sync)\n    return strategy\nstrategy = get_strategy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"backbone_name = 'efficientnet-b0'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class ConvNet(tf.keras.Model):\n\n#     def __init__(self, engine, input_shape, weights):\n#         super(ConvNet, self).__init__()\n        \n#         self.engine = engine(\n#             include_top=False, input_shape=input_shape, weights=weights)\n        \n        \n#         self.flat = tf.keras.layers.GlobalAveragePooling2D()\n# #         self.flat = tf.keras.layers.Flatten()\n#         self.dropout = tf.keras.layers.Dropout(0.3)\n#         self.dropout1 = tf.keras.layers.Dropout(0.3)\n#         self.dense_1 = tf.keras.layers.Dense(1024)\n\n#     @tf.function\n#     def call(self, inputs, **kwargs):\n#         x = self.engine(inputs)\n#         x = self.flat(x)\n#         x = tf.math.l2_normalize(x, axis=1)\n#         x = self.dropout(x, training=kwargs.get('training', False))\n#         x = self.dense_1(x)\n#         x = tf.math.l2_normalize(x, axis=1)\n#         return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(engine, input_shape, weights):\n    \n    inp = tf.keras.layers.Input(shape=input_shape)\n    base = engine(include_top=False, input_shape=input_shape, weights=weights)\n    \n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1))(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(1024)(x)\n    x = tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=1))(x)\n    model = tf.keras.Model(inputs=[inp],outputs=[x])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if backbone_name.startswith('efficientnet'):\n    model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = build_model(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights='imagenet') \n    model.compile(optimizer = tf.keras.optimizers.Adam(lr=0.001), loss=tfa.losses.TripletSemiHardLoss(margin=0.5))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_callback = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss', factor=0.5, patience=3, verbose=1, mode='min',\n    min_delta=0.0001, cooldown=0, min_lr=0.000001)\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(backbone_name+'.h5', monitor='val_loss',\n    verbose=1, save_best_only=True, save_weights_only=True, mode='min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def flip(x: tf.Tensor) -> tf.Tensor:\n    x = tf.image.random_flip_left_right(x)\n    x = tf.image.random_flip_up_down(x)\n    return x\n\ndef decode_image(image_data):\n#     image = tf.io.decode_jpeg(image_data, channels=3, dct_method='INTEGER_ACCURATE')\n    image = tf.io.decode_jpeg(image_data, channels=3)\n    image = tf.image.resize(image, [IMG_SIZE,IMG_SIZE])\n    image = tf.cast(image, tf.float32)\n    image = image / 255.0\n    return image\n\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {}\n\n    LABELED_TFREC_FORMAT['image'] = tf.io.FixedLenFeature([], tf.string)\n    LABELED_TFREC_FORMAT['label'] = tf.io.FixedLenFeature([], tf.int64)\n    \n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    \n    image = decode_image(example['image'])\n    \n    label = tf.cast(example['label'], tf.int32)\n    return image, label\n\n\ndef load_dataset(filenames, labeled=True, ordered=False, training=True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    return dataset\n\ndef get_training_dataset(TRAINING_FILENAMES):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(TEST_FILENAMES, ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=True, ordered=ordered, training=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_train_samples = sum(int(fn.split('-')[-1].split('.')[0]) for fn in train_fns)\nnum_val_samples = sum(int(fn.split('-')[-1].split('.')[0]) for fn in val_fns)\nnum_train_samples, num_val_samples ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 4096\nSTEPS_PER_EPOCH = num_train_samples // BATCH_SIZE\nVAL_STEPS_PER_EPOCH = num_val_samples // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(get_training_dataset(train_fns), steps_per_epoch=STEPS_PER_EPOCH, verbose=1,\n    validation_data = get_validation_dataset(val_fns), validation_steps=VAL_STEPS_PER_EPOCH, epochs=1, callbacks = [lr_callback, checkpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(\"model_iter_1.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}