{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q ../input/kaggle-efficientnet-repo/efficientnet-1.0.0-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport argparse\nimport tensorflow as tf\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.metrics import categorical_accuracy, top_k_categorical_accuracy\nfrom tensorflow.keras import layers as L\nimport efficientnet.tfkeras as efn\nimport tensorflow.keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fns = ['../input/tf-record-256-256-48/0_train00-2104.tfrec', '../input/tf-record-256-256-48/1_train00-2103.tfrec', '../input/tf-record-256-256-48/2_train00-2103.tfrec', '../input/tf-record-256-256-48/3_train00-2103.tfrec']\nval_fns = ['../input/tf-record-256-256-48/4_train00-2103.tfrec']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"backbone_name = 'efficientnet-b6'\nN_TILES = 42\nIMG_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf.keras.mixed_precision.experimental.set_policy('mixed_float16')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvNet(tf.keras.Model):\n\n    def __init__(self, engine, input_shape, weights):\n        super(ConvNet, self).__init__()\n        \n        self.engine = engine(\n            include_top=False, input_shape=input_shape, weights=weights)\n        \n        \n        self.avg_pool2d = tf.keras.layers.GlobalAveragePooling2D()\n        self.dropout = tf.keras.layers.Dropout(0.3)\n        self.dense_1 = tf.keras.layers.Dense(512)\n        self.dense_2 = tf.keras.layers.Dense(128)\n        self.dense_3 = tf.keras.layers.Dense(1)\n\n    @tf.function\n    def call(self, inputs, **kwargs):\n        x = tf.reshape(inputs, (-1, IMG_SIZE, IMG_SIZE, 3))\n        x = self.engine(x)\n        shape = x.shape\n        x = tf.reshape(x, (-1, N_TILES, shape[1], shape[2], shape[3])) \n        x = tf.transpose(x, perm=[0, 2, 1, 3, 4])\n        x = tf.reshape(x, (-1, shape[1], N_TILES*shape[2], shape[3])) \n        x = self.avg_pool2d(x)\n        x = self.dropout(x, training=kwargs.get('training', False))\n        x = self.dense_1(x)\n        x = tf.nn.relu(x)\n        x = self.dropout(x)\n        x = self.dense_2(x)\n        x = tf.nn.relu(x)\n        x = self.dropout(x)\n        return self.dense_3(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if backbone_name.startswith('efficientnet'):\n    model_fn = getattr(efn, f'EfficientNetB{backbone_name[-1]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = ConvNet(engine=model_fn, input_shape=(IMG_SIZE, IMG_SIZE, 3), weights='imagenet') \n    model.compile(optimizer = tf.keras.optimizers.Adam(lr=0.001), loss=\"mean_squared_error\")    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_callback = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss', factor=0.5, patience=2, verbose=1, mode='min',\n    min_delta=0.0001, cooldown=0, min_lr=0.000001)\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(backbone_name+'.h5', monitor='val_loss',\n    verbose=1, save_best_only=True, save_weights_only=True, mode='min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def flip(x: tf.Tensor) -> tf.Tensor:\n    x = tf.image.random_flip_left_right(x)\n    x = tf.image.random_flip_up_down(x)\n    return x\n\ndef decode_image(image_data, rand = True):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.reshape(image, [*(IMG_SIZE, IMG_SIZE), 3]) # explicit size needed for TPU\n    if rand:\n        image = flip(image)\n    image = tf.cast(image, tf.float32)\n    image = image / 255.0\n    return image\n\ndef collage_image(example, rand = False):\n    images = []\n    k = 0\n    while k < N_TILES:\n        images.append(decode_image(example['image_'+str(k)], rand))\n        k += 1\n    images = tf.stack(images)\n\n    return images\n\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {}\n    k = 0\n    while k < N_TILES:\n        LABELED_TFREC_FORMAT['image_'+str(k)] = tf.io.FixedLenFeature([], tf.string)\n        k += 1\n    LABELED_TFREC_FORMAT['label'] = tf.io.FixedLenFeature([], tf.int64)\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = collage_image(example, rand=True)\n    label = tf.cast(example['label'], tf.int32)\n    return image, label\n\ndef read_labeled_tfrecord_noshuffle(example):\n    LABELED_TFREC_FORMAT = {}\n    k = 0\n    while k < N_TILES:\n        LABELED_TFREC_FORMAT['image_'+str(k)] = tf.io.FixedLenFeature([], tf.string)\n        k += 1\n    LABELED_TFREC_FORMAT['label'] = tf.io.FixedLenFeature([], tf.int64)\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = collage_image(example)\n    label = tf.cast(example['label'], tf.int32)\n    return image, label\n\ndef load_dataset(filenames, labeled=True, ordered=False, training=True):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    \n    if training == False:\n        dataset = dataset.map(read_labeled_tfrecord_noshuffle)\n    else:\n        dataset = dataset.map(read_labeled_tfrecord)\n    return dataset\n\ndef get_training_dataset(TRAINING_FILENAMES):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(TEST_FILENAMES, ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=True, ordered=ordered, training=False)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_train_samples = sum(int(fn.split('-')[-1].split('.')[0]) for fn in train_fns)\nnum_val_samples = sum(int(fn.split('-')[-1].split('.')[0]) for fn in val_fns)\nnum_train_samples, num_val_samples ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\nSTEPS_PER_EPOCH = num_train_samples // BATCH_SIZE\nVAL_STEPS_PER_EPOCH = num_val_samples // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(get_training_dataset(train_fns), steps_per_epoch=STEPS_PER_EPOCH, verbose=1,\n    validation_data = get_validation_dataset(val_fns), validation_steps=VAL_STEPS_PER_EPOCH, epochs=40, callbacks = [lr_callback, checkpoint])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}