{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net - TF Baseline\n\nStart with a simple baseline similar to [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification). We effectively use a the time-averaged power spectrum as input images and train a simple EfficientNetB1. \n\nThe TF records have been prepared in a separate kernel [G2Net - Record Generation](https://www.kaggle.com/code/morodertobias/g2net-record-generation).\n\n**Comments welcome!**\n\n## References\n- [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification)\n- [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nimport re\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nprint(tf.__version__)\nfrom pydantic import BaseModel as ConfigBaseModel\nfrom sklearn.model_selection import train_test_split\nfrom kaggle_secrets import UserSecretsClient\nfrom kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:02.193068Z","iopub.execute_input":"2022-11-26T18:28:02.193779Z","iopub.status.idle":"2022-11-26T18:28:07.209682Z","shell.execute_reply.started":"2022-11-26T18:28:02.193696Z","shell.execute_reply":"2022-11-26T18:28:07.208727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"TPU in use!\")\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Strategy:\", strategy)\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:07.211804Z","iopub.execute_input":"2022-11-26T18:28:07.212565Z","iopub.status.idle":"2022-11-26T18:28:07.227084Z","shell.execute_reply.started":"2022-11-26T18:28:07.212525Z","shell.execute_reply":"2022-11-26T18:28:07.225964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted(tf.config.list_logical_devices())","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:07.229327Z","iopub.execute_input":"2022-11-26T18:28:07.229681Z","iopub.status.idle":"2022-11-26T18:28:09.709783Z","shell.execute_reply.started":"2022-11-26T18:28:07.229655Z","shell.execute_reply":"2022-11-26T18:28:09.708727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nuser_credential = user_secrets.get_gcloud_credential()\nuser_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:09.714115Z","iopub.execute_input":"2022-11-26T18:28:09.715674Z","iopub.status.idle":"2022-11-26T18:28:11.657997Z","shell.execute_reply.started":"2022-11-26T18:28:09.715637Z","shell.execute_reply":"2022-11-26T18:28:11.656884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config(ConfigBaseModel):\n    seed = 887\n    model_name = \"enetb1_v7\"\n    path_model = f\"/kaggle/working/{model_name}.h5\"\n    path_model_1 = f\"/kaggle/working/{model_name}_1.h5\"\n    # data\n    gcs = KaggleDatasets().get_gcs_path(\"ds-g2net-record-generation\")\n    path_submission = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\"\n    train_dir = gcs + \"/train/\"\n    test_dir = gcs + \"/test/\"\n    img_size = (360, 128)\n    channels = 2\n    img_shape = (*img_size, channels)\n    # model\n    base_model_weights = \"imagenet\"\n    train_from = \"block7a\"\n    dropout = 0.20\n    # training\n    shuffle_size = 128\n    epochs = 100\n    batch_size = 16 * strategy.num_replicas_in_sync\n    test_batch_size = 64\n    lr = 0.001\n    lr_1 = 3e-6\n    patience = 10\n    \ncfg = Config()\ncfg.dict()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:11.66286Z","iopub.execute_input":"2022-11-26T18:28:11.665334Z","iopub.status.idle":"2022-11-26T18:28:12.174566Z","shell.execute_reply.started":"2022-11-26T18:28:11.665295Z","shell.execute_reply":"2022-11-26T18:28:12.173592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data\n\nPrepare training, validation and test datasets.","metadata":{}},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"size-([0-9]*)\\.\").search(f).group(1)) for f in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:12.179023Z","iopub.execute_input":"2022-11-26T18:28:12.181392Z","iopub.status.idle":"2022-11-26T18:28:12.188892Z","shell.execute_reply.started":"2022-11-26T18:28:12.181356Z","shell.execute_reply":"2022-11-26T18:28:12.187895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = sorted(tf.io.gfile.glob(cfg.train_dir + \"*.tfrec\"))\ntrain_files, valid_files = train_test_split(train_files, random_state=cfg.seed)\ntest_files = sorted(tf.io.gfile.glob(cfg.test_dir + \"*.tfrec\"))\ncount_data_items(train_files), train_files, count_data_items(valid_files), valid_files, count_data_items(test_files), len(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:12.193911Z","iopub.execute_input":"2022-11-26T18:28:12.19671Z","iopub.status.idle":"2022-11-26T18:28:13.801308Z","shell.execute_reply.started":"2022-11-26T18:28:12.196675Z","shell.execute_reply":"2022-11-26T18:28:13.800292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data handling\n\nThe tensorflow records have the stacked image encoded as tensor; so in particular not normalized to fit the default image ranges. A target is present in the training data, an id in the test data.\n","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\n\ndef normalize_as_image(x):\n    t_min = tf.math.reduce_min(x, axis=[0, 1])\n    t_max = tf.math.reduce_max(x, axis=[0, 1])\n    img = (x - t_min) / (t_max - t_min) * 255.\n    return img\n\n\ndef decode_image(img_data):\n    img = tf.io.parse_tensor(img_data, tf.float32)\n    # channel first to last position\n    img = tf.transpose(img, perm=[1, 2, 0])\n    # normalize as float image in range [0. 255.]\n    img = normalize_as_image(img)\n    img = tf.reshape(img, cfg.img_shape)\n    return img\n\n\ndef parse_train_tfrecord(data):\n    features = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    ex = tf.io.parse_single_example(data, features)\n    img = decode_image(ex[\"img\"])\n    return img, ex[\"target\"]\n\n\ndef parse_test_tfrecord(data):\n    features = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'id': tf.io.FixedLenFeature([], tf.string)\n    }\n    ex = tf.io.parse_single_example(data, features)\n    img = decode_image(ex[\"img\"])\n    return img, ex[\"id\"]\n\n\ndef load_dataset(filenames, parse_func, ordered=False):\n    ds = tf.data.TFRecordDataset(filenames=filenames, num_parallel_reads=AUTOTUNE)\n    opt = tf.data.Options()\n    opt.experimental_deterministic = ordered\n    ds = ds.with_options(opt)\n    ds = ds.map(parse_func, num_parallel_calls=AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:13.803016Z","iopub.execute_input":"2022-11-26T18:28:13.803392Z","iopub.status.idle":"2022-11-26T18:28:13.814754Z","shell.execute_reply.started":"2022-11-26T18:28:13.803358Z","shell.execute_reply":"2022-11-26T18:28:13.813572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_augmentation(img, label):\n#     img = tf.image.random_flip_left_right(img)\n#     img = tf.image.random_flip_up_down(img)\n#     return img, label","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:13.816295Z","iopub.execute_input":"2022-11-26T18:28:13.816979Z","iopub.status.idle":"2022-11-26T18:28:13.825143Z","shell.execute_reply.started":"2022-11-26T18:28:13.816942Z","shell.execute_reply":"2022-11-26T18:28:13.824191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_training_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_train_tfrecord, ordered=False)\n    # ds = ds.map(data_augmentation, num_parallel_calls=AUTOTUNE)\n    ds = ds.repeat()\n    ds = ds.shuffle(cfg.shuffle_size)\n    ds = ds.batch(cfg.batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds\n\n\ndef get_validation_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_train_tfrecord, ordered=True)\n    ds = ds.batch(cfg.batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds\n\n\ndef get_test_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_test_tfrecord, ordered=True)\n    ds = ds.batch(cfg.test_batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:13.830533Z","iopub.execute_input":"2022-11-26T18:28:13.830798Z","iopub.status.idle":"2022-11-26T18:28:13.840435Z","shell.execute_reply.started":"2022-11-26T18:28:13.830774Z","shell.execute_reply":"2022-11-26T18:28:13.839472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = get_training_dataset(train_files)\nvalid_ds = get_validation_dataset(valid_files)\ntest_ds = get_test_dataset(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:13.841846Z","iopub.execute_input":"2022-11-26T18:28:13.842641Z","iopub.status.idle":"2022-11-26T18:28:14.275721Z","shell.execute_reply.started":"2022-11-26T18:28:13.842608Z","shell.execute_reply":"2022-11-26T18:28:14.27471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check dataset","metadata":{}},{"cell_type":"code","source":"imgs, labels = next(iter(train_ds))\nimgs.shape, imgs.dtype, labels.shape, labels.dtype","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:14.277358Z","iopub.execute_input":"2022-11-26T18:28:14.277769Z","iopub.status.idle":"2022-11-26T18:28:18.995333Z","shell.execute_reply.started":"2022-11-26T18:28:14.277732Z","shell.execute_reply":"2022-11-26T18:28:18.994301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:18.996784Z","iopub.execute_input":"2022-11-26T18:28:18.997619Z","iopub.status.idle":"2022-11-26T18:28:19.012548Z","shell.execute_reply.started":"2022-11-26T18:28:18.997583Z","shell.execute_reply":"2022-11-26T18:28:19.011564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 0\nfig, axs = plt.subplots(ncols=2, figsize=(8, 4), sharey='all')\nfig.suptitle(f\"Label: {labels[idx].numpy()}\")\nkwargs_imshow = dict(aspect='auto', vmin=0.0, vmax=255.)\naxs[0].imshow(imgs[idx][..., 0], **kwargs_imshow)\naxs[0].set_title(\"h1\")\naxs[1].imshow(imgs[idx][..., 1], **kwargs_imshow)\naxs[1].set_title(\"l1\")\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:19.01399Z","iopub.execute_input":"2022-11-26T18:28:19.014445Z","iopub.status.idle":"2022-11-26T18:28:19.462685Z","shell.execute_reply.started":"2022-11-26T18:28:19.014384Z","shell.execute_reply":"2022-11-26T18:28:19.461235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural network\n\nThe pretrained TF model cannot directly handle 2 input channels (in contrast to ``timm`` as in [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification)). Thus a prio ``1x1`` convolution is used to create a 3 channel input, similar as in [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu).","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications.efficientnet import EfficientNetB1 as BaseModel\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nfrom tensorflow.keras import layers, Model, Input, losses, metrics, optimizers, callbacks","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:19.463723Z","iopub.execute_input":"2022-11-26T18:28:19.464865Z","iopub.status.idle":"2022-11-26T18:28:20.248206Z","shell.execute_reply.started":"2022-11-26T18:28:19.46483Z","shell.execute_reply":"2022-11-26T18:28:20.247274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unfreeze_model(model, train_from=None):\n    trainable = (train_from is None)\n    for layer in model.layers:\n        if not trainable:\n            trainable = (train_from in layer.name)\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = trainable\n        else:\n            layer.trainable = False\n\n\ndef create_model():       \n    inputs = Input(shape=cfg.img_shape, dtype=tf.float32)\n    x = layers.Conv2D(3, 1, name='adjustment')(inputs)\n    x = layers.Lambda(preprocess_input, name=\"preprocess_input\")(x)\n    base_model = BaseModel(include_top=False, weights=cfg.base_model_weights, pooling=\"avg\")\n    for l in base_model.layers:\n        l.trainable = False\n    # base_model.trainable = False\n    # unfreeze_model(base_model, cfg.train_from)\n    x = base_model(x)\n    x = layers.BatchNormalization(name=\"head_bn\")(x)\n    x = layers.Dropout(cfg.dropout, name=\"head_dropout\")(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\", name=\"logits\")(x)\n    model = Model(inputs=inputs, outputs=outputs, name=cfg.model_name)\n    model.compile(\n        optimizer=optimizers.Adam(learning_rate=cfg.lr),\n        loss=losses.BinaryCrossentropy(), \n        metrics=[\"acc\"]\n    )\n    return model\n\n\ndef list_trainable_weights(model):\n    return [x.name for x in model.trainable_weights]","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:20.24976Z","iopub.execute_input":"2022-11-26T18:28:20.250108Z","iopub.status.idle":"2022-11-26T18:28:20.263133Z","shell.execute_reply.started":"2022-11-26T18:28:20.250073Z","shell.execute_reply":"2022-11-26T18:28:20.261294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\nwith strategy.scope():\n    model = create_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:20.26481Z","iopub.execute_input":"2022-11-26T18:28:20.267654Z","iopub.status.idle":"2022-11-26T18:28:25.262485Z","shell.execute_reply.started":"2022-11-26T18:28:20.26762Z","shell.execute_reply":"2022-11-26T18:28:25.261392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.layers","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:25.263894Z","iopub.execute_input":"2022-11-26T18:28:25.264257Z","iopub.status.idle":"2022-11-26T18:28:25.272824Z","shell.execute_reply.started":"2022-11-26T18:28:25.264229Z","shell.execute_reply":"2022-11-26T18:28:25.271804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check model","metadata":{"execution":{"iopub.status.busy":"2022-11-19T09:49:48.999348Z","iopub.execute_input":"2022-11-19T09:49:48.999677Z","iopub.status.idle":"2022-11-19T09:49:52.017972Z","shell.execute_reply.started":"2022-11-19T09:49:48.999641Z","shell.execute_reply":"2022-11-19T09:49:52.016952Z"}}},{"cell_type":"code","source":"model.predict(imgs)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:25.274688Z","iopub.execute_input":"2022-11-26T18:28:25.275499Z","iopub.status.idle":"2022-11-26T18:28:33.173233Z","shell.execute_reply.started":"2022-11-26T18:28:25.275438Z","shell.execute_reply":"2022-11-26T18:28:33.172206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(imgs, labels, return_dict=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:33.17658Z","iopub.execute_input":"2022-11-26T18:28:33.176884Z","iopub.status.idle":"2022-11-26T18:28:36.023799Z","shell.execute_reply.started":"2022-11-26T18:28:33.176855Z","shell.execute_reply":"2022-11-26T18:28:36.022883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def get_callbacks():\n    cbs = [\n        callbacks.EarlyStopping(\n            monitor=\"val_loss\",\n            verbose=1,\n            patience=cfg.patience,\n            restore_best_weights=True,\n        ),\n    ]\n    return cbs\n\n\ndef show_history(history):\n    history_frame = pd.DataFrame(history.history)\n    history_frame.index = pd.RangeIndex(1, len(history_frame) + 1, name=\"epoch\")\n    display(history_frame.style\\\n        .highlight_min(color='lightgreen', subset=['val_loss'])\\\n        .highlight_max(color='lightgreen', subset=['val_acc'])\n    )\n    fig, ax = plt.subplots(1, 2, figsize=(16, 6))\n    history_frame.loc[:, ['loss', 'val_loss']].plot(ax=ax[0], title='loss')\n    history_frame.loc[:, ['acc', 'val_acc']].plot(ax=ax[1], title='acc')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:36.025126Z","iopub.execute_input":"2022-11-26T18:28:36.026032Z","iopub.status.idle":"2022-11-26T18:28:36.035534Z","shell.execute_reply.started":"2022-11-26T18:28:36.025995Z","shell.execute_reply":"2022-11-26T18:28:36.033995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_steps = count_data_items(train_files) // cfg.batch_size\nprint(\"steps per epoch:\", train_steps)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:36.03679Z","iopub.execute_input":"2022-11-26T18:28:36.037219Z","iopub.status.idle":"2022-11-26T18:28:36.051186Z","shell.execute_reply.started":"2022-11-26T18:28:36.037182Z","shell.execute_reply":"2022-11-26T18:28:36.050083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Frozen base model","metadata":{}},{"cell_type":"code","source":"list_trainable_weights(model)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:36.05267Z","iopub.execute_input":"2022-11-26T18:28:36.053121Z","iopub.status.idle":"2022-11-26T18:28:36.063946Z","shell.execute_reply.started":"2022-11-26T18:28:36.053089Z","shell.execute_reply":"2022-11-26T18:28:36.06285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory = model.fit(\n    train_ds, \n    epochs=cfg.epochs,\n    steps_per_epoch=train_steps,\n    validation_data=valid_ds,\n    callbacks=get_callbacks(),\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:28:36.065183Z","iopub.execute_input":"2022-11-26T18:28:36.066135Z","iopub.status.idle":"2022-11-26T18:33:05.347587Z","shell.execute_reply.started":"2022-11-26T18:28:36.066101Z","shell.execute_reply":"2022-11-26T18:33:05.34594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_history(history)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:05.348965Z","iopub.execute_input":"2022-11-26T18:33:05.349319Z","iopub.status.idle":"2022-11-26T18:33:07.582212Z","shell.execute_reply.started":"2022-11-26T18:33:05.349284Z","shell.execute_reply":"2022-11-26T18:33:07.58135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(cfg.path_model)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:07.583855Z","iopub.execute_input":"2022-11-26T18:33:07.584539Z","iopub.status.idle":"2022-11-26T18:33:08.023633Z","shell.execute_reply.started":"2022-11-26T18:33:07.584495Z","shell.execute_reply":"2022-11-26T18:33:08.022651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fine-tune base model part","metadata":{"execution":{"iopub.status.busy":"2022-11-26T17:52:38.228894Z","iopub.execute_input":"2022-11-26T17:52:38.229881Z","iopub.status.idle":"2022-11-26T17:52:38.24787Z","shell.execute_reply.started":"2022-11-26T17:52:38.229816Z","shell.execute_reply":"2022-11-26T17:52:38.246752Z"}}},{"cell_type":"code","source":"unfreeze_model(model.layers[3], train_from=cfg.train_from)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:08.025606Z","iopub.execute_input":"2022-11-26T18:33:08.025991Z","iopub.status.idle":"2022-11-26T18:33:08.040846Z","shell.execute_reply.started":"2022-11-26T18:33:08.025956Z","shell.execute_reply":"2022-11-26T18:33:08.039805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model.compile(\n        optimizer=optimizers.Adam(learning_rate=cfg.lr_1),\n        loss=losses.BinaryCrossentropy(), \n        metrics=[\"acc\"]\n    )\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:08.04711Z","iopub.execute_input":"2022-11-26T18:33:08.047371Z","iopub.status.idle":"2022-11-26T18:33:08.076953Z","shell.execute_reply.started":"2022-11-26T18:33:08.047347Z","shell.execute_reply":"2022-11-26T18:33:08.076078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(valid_ds, return_dict=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:08.078278Z","iopub.execute_input":"2022-11-26T18:33:08.078681Z","iopub.status.idle":"2022-11-26T18:33:12.778341Z","shell.execute_reply.started":"2022-11-26T18:33:08.078647Z","shell.execute_reply":"2022-11-26T18:33:12.777359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory = model.fit(\n    train_ds, \n    epochs=cfg.epochs,\n    steps_per_epoch=train_steps,\n    validation_data=valid_ds,\n    callbacks=get_callbacks(),\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:33:12.779809Z","iopub.execute_input":"2022-11-26T18:33:12.780099Z","iopub.status.idle":"2022-11-26T18:35:33.558144Z","shell.execute_reply.started":"2022-11-26T18:33:12.780074Z","shell.execute_reply":"2022-11-26T18:35:33.556553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_history(history)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:35:33.563926Z","iopub.execute_input":"2022-11-26T18:35:33.564531Z","iopub.status.idle":"2022-11-26T18:35:43.638313Z","shell.execute_reply.started":"2022-11-26T18:35:33.564494Z","shell.execute_reply":"2022-11-26T18:35:43.637372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(cfg.path_model_1)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:35:43.64004Z","iopub.execute_input":"2022-11-26T18:35:43.643256Z","iopub.status.idle":"2022-11-26T18:35:44.111086Z","shell.execute_reply.started":"2022-11-26T18:35:43.643225Z","shell.execute_reply":"2022-11-26T18:35:44.110096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(\"/kaggle/working\")","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:35:44.113094Z","iopub.execute_input":"2022-11-26T18:35:44.113482Z","iopub.status.idle":"2022-11-26T18:35:44.121343Z","shell.execute_reply.started":"2022-11-26T18:35:44.113431Z","shell.execute_reply":"2022-11-26T18:35:44.120091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test\n\nPredict on test dataset.","metadata":{}},{"cell_type":"code","source":"test_ids = np.concatenate([x.numpy() for x in test_ds.map(lambda imgs, ids: ids)])\ntest_ids","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:35:44.123048Z","iopub.execute_input":"2022-11-26T18:35:44.123713Z","iopub.status.idle":"2022-11-26T18:37:06.111367Z","shell.execute_reply.started":"2022-11-26T18:35:44.123679Z","shell.execute_reply":"2022-11-26T18:37:06.110469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_ds.map(lambda imgs, ids: imgs), verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:37:06.112734Z","iopub.execute_input":"2022-11-26T18:37:06.113118Z","iopub.status.idle":"2022-11-26T18:38:27.550369Z","shell.execute_reply.started":"2022-11-26T18:37:06.113083Z","shell.execute_reply":"2022-11-26T18:38:27.549456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_ids,\n    \"target\": preds.ravel()\n})\nsubmission[\"id\"] = submission[\"id\"].str.decode('utf-8')\nsubmission = submission.sort_values(by=\"id\")\nsubmission = submission.reset_index(drop=True)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:38:27.554166Z","iopub.execute_input":"2022-11-26T18:38:27.555591Z","iopub.status.idle":"2022-11-26T18:38:27.58334Z","shell.execute_reply.started":"2022-11-26T18:38:27.555552Z","shell.execute_reply":"2022-11-26T18:38:27.582337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check submission","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:46:13.250228Z","iopub.execute_input":"2022-11-19T12:46:13.250498Z","iopub.status.idle":"2022-11-19T12:46:13.27234Z","shell.execute_reply.started":"2022-11-19T12:46:13.250469Z","shell.execute_reply":"2022-11-19T12:46:13.271668Z"}}},{"cell_type":"code","source":"sample_submission = pd.read_csv(cfg.path_submission)\n(sample_submission[\"id\"] == submission[\"id\"]).all()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:38:27.584775Z","iopub.execute_input":"2022-11-26T18:38:27.58553Z","iopub.status.idle":"2022-11-26T18:38:27.605361Z","shell.execute_reply.started":"2022-11-26T18:38:27.585497Z","shell.execute_reply":"2022-11-26T18:38:27.604276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"target\"].describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:38:27.606687Z","iopub.execute_input":"2022-11-26T18:38:27.607598Z","iopub.status.idle":"2022-11-26T18:38:27.620023Z","shell.execute_reply.started":"2022-11-26T18:38:27.607561Z","shell.execute_reply":"2022-11-26T18:38:27.618826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write submission","metadata":{}},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:38:27.621243Z","iopub.execute_input":"2022-11-26T18:38:27.622201Z","iopub.status.idle":"2022-11-26T18:38:27.641078Z","shell.execute_reply.started":"2022-11-26T18:38:27.622167Z","shell.execute_reply":"2022-11-26T18:38:27.640245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-11-26T18:38:27.642357Z","iopub.execute_input":"2022-11-26T18:38:27.643192Z","iopub.status.idle":"2022-11-26T18:38:28.689628Z","shell.execute_reply.started":"2022-11-26T18:38:27.643156Z","shell.execute_reply":"2022-11-26T18:38:28.688472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}