{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G2Net - TF Baseline -360\n\nStart with a simple baseline similar to [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification). We effectively use a the time-averaged power spectrum as input images and train a simple EfficientNetB1. Image size is 360X360\n\nThe TF records have been prepared in a separate kernel [G2Net - Record Generation](https://www.kaggle.com/code/morodertobias/g2net-record-generation).\n\n## References\n- [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification)\n- [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nimport re\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nprint(tf.__version__)\nfrom pydantic import BaseModel\nfrom sklearn.model_selection import train_test_split\n# from kaggle_secrets import UserSecretsClient\n# from kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:23:47.625384Z","iopub.execute_input":"2022-11-23T11:23:47.626285Z","iopub.status.idle":"2022-11-23T11:23:52.826656Z","shell.execute_reply.started":"2022-11-23T11:23:47.6262Z","shell.execute_reply":"2022-11-23T11:23:52.825602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted(tf.config.list_logical_devices())","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:23:55.568457Z","iopub.execute_input":"2022-11-23T11:23:55.569368Z","iopub.status.idle":"2022-11-23T11:23:57.858923Z","shell.execute_reply.started":"2022-11-23T11:23:55.56933Z","shell.execute_reply":"2022-11-23T11:23:57.85759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print(\"TPU in use!\")\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Strategy:\", strategy)\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:23:59.891868Z","iopub.execute_input":"2022-11-23T11:23:59.892232Z","iopub.status.idle":"2022-11-23T11:23:59.903217Z","shell.execute_reply.started":"2022-11-23T11:23:59.892201Z","shell.execute_reply":"2022-11-23T11:23:59.902099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# user_secrets = UserSecretsClient()\n# user_credential = user_secrets.get_gcloud_credential()\n# user_secrets.set_tensorflow_credential(user_credential)","metadata":{"execution":{"iopub.status.busy":"2022-11-19T22:48:04.474055Z","iopub.execute_input":"2022-11-19T22:48:04.474324Z","iopub.status.idle":"2022-11-19T22:48:05.933561Z","shell.execute_reply.started":"2022-11-19T22:48:04.474295Z","shell.execute_reply":"2022-11-19T22:48:05.932881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:25:12.060503Z","iopub.execute_input":"2022-11-23T11:25:12.061194Z","iopub.status.idle":"2022-11-23T11:25:12.06877Z","shell.execute_reply.started":"2022-11-23T11:25:12.061148Z","shell.execute_reply":"2022-11-23T11:25:12.0677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config(BaseModel):\n    seed = 887\n    model_name = \"enetb1_v1\"\n    model_dir = \"/kaggle/working/model\"\n    # data\n    gcs = KaggleDatasets().get_gcs_path(\"360-g2net-record-generation\")\n    path_submission = \"/kaggle/input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\"\n    train_dir = gcs + \"/train/\"\n    test_dir = gcs + \"/test/\"\n    img_size = (360, 360)\n    channels = 2\n    img_shape = (*img_size, channels)\n    # model\n    base_model_weights = \"imagenet\"\n    dropout = 0.2\n    # training\n    shuffle_size = 128\n    epochs = 200\n    batch_size = 16 * strategy.num_replicas_in_sync\n    test_batch_size = 64\n    lr = 0.001\n    patience = 10\n    \ncfg = Config()\ncfg.dict()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:25:14.642809Z","iopub.execute_input":"2022-11-23T11:25:14.643717Z","iopub.status.idle":"2022-11-23T11:25:26.878254Z","shell.execute_reply.started":"2022-11-23T11:25:14.643669Z","shell.execute_reply":"2022-11-23T11:25:26.877117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data\n\nPrepare training, validation and test datasets.","metadata":{}},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"size-([0-9]*)\\.\").search(f).group(1)) for f in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:25:50.800507Z","iopub.execute_input":"2022-11-23T11:25:50.80124Z","iopub.status.idle":"2022-11-23T11:25:50.806919Z","shell.execute_reply.started":"2022-11-23T11:25:50.8012Z","shell.execute_reply":"2022-11-23T11:25:50.805807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = sorted(tf.io.gfile.glob(cfg.train_dir + \"*.tfrec\"))\ntrain_files, valid_files = train_test_split(train_files, random_state=cfg.seed)\ntest_files = sorted(tf.io.gfile.glob(cfg.test_dir + \"*.tfrec\"))\ncount_data_items(train_files), train_files, count_data_items(valid_files), valid_files, count_data_items(test_files), len(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:25:56.833289Z","iopub.execute_input":"2022-11-23T11:25:56.833647Z","iopub.status.idle":"2022-11-23T11:25:58.986551Z","shell.execute_reply.started":"2022-11-23T11:25:56.833602Z","shell.execute_reply":"2022-11-23T11:25:58.985357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data handling\n\nThe tensorflow records have the stacked image encoded as tensor; so in particular not normalized to fit the default image ranges. A target is present in the training data, an id in the test data.\n","metadata":{}},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\n\ndef normalize_as_image(x):\n    t_min = tf.math.reduce_min(x, axis=[0, 1])\n    t_max = tf.math.reduce_max(x, axis=[0, 1])\n    img = (x - t_min) / (t_max - t_min) * 255.\n    return img\n\n\ndef decode_image(img_data):\n    img = tf.io.parse_tensor(img_data, tf.float32)\n    # channel first to last position\n    img = tf.transpose(img, perm=[1, 2, 0])\n    # normalize as float image in range [0. 255.]\n    img = normalize_as_image(img)\n    return img\n\n\ndef parse_train_tfrecord(data):\n    features = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64)\n    }\n    ex = tf.io.parse_single_example(data, features)\n    img = decode_image(ex[\"img\"])\n    return img, ex[\"target\"]\n\n\ndef parse_test_tfrecord(data):\n    features = {\n        'img': tf.io.FixedLenFeature([], tf.string),\n        'id': tf.io.FixedLenFeature([], tf.string)\n    }\n    ex = tf.io.parse_single_example(data, features)\n    img = decode_image(ex[\"img\"])\n    return img, ex[\"id\"]\n\n\ndef load_dataset(filenames, parse_func, ordered=False):\n    ds = tf.data.TFRecordDataset(filenames=filenames, num_parallel_reads=AUTOTUNE)\n    opt = tf.data.Options()\n    opt.experimental_deterministic = ordered\n    ds = ds.with_options(opt)\n    ds = ds.map(parse_func, num_parallel_calls=AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:05.756199Z","iopub.execute_input":"2022-11-23T11:26:05.756559Z","iopub.status.idle":"2022-11-23T11:26:05.76828Z","shell.execute_reply.started":"2022-11-23T11:26:05.756528Z","shell.execute_reply":"2022-11-23T11:26:05.767138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_training_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_train_tfrecord, ordered=False)\n    ds = ds.repeat()\n    ds = ds.shuffle(cfg.shuffle_size)\n    ds = ds.batch(cfg.batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds\n\n\ndef get_validation_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_train_tfrecord, ordered=True)\n    ds = ds.batch(cfg.batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds\n\n\ndef get_test_dataset(filenames):\n    ds = load_dataset(filenames, parse_func=parse_test_tfrecord, ordered=True)\n    ds = ds.batch(cfg.test_batch_size)\n    ds = ds.prefetch(AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:08.54702Z","iopub.execute_input":"2022-11-23T11:26:08.547372Z","iopub.status.idle":"2022-11-23T11:26:08.557714Z","shell.execute_reply.started":"2022-11-23T11:26:08.547342Z","shell.execute_reply":"2022-11-23T11:26:08.55681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = get_training_dataset(train_files)\nvalid_ds = get_validation_dataset(valid_files)\ntest_ds = get_test_dataset(test_files)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:11.59783Z","iopub.execute_input":"2022-11-23T11:26:11.598503Z","iopub.status.idle":"2022-11-23T11:26:12.365855Z","shell.execute_reply.started":"2022-11-23T11:26:11.598468Z","shell.execute_reply":"2022-11-23T11:26:12.364936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check dataset","metadata":{}},{"cell_type":"code","source":"imgs, labels = next(iter(train_ds))\nimgs.shape, imgs.dtype, labels.shape, labels.dtype","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:14.892585Z","iopub.execute_input":"2022-11-23T11:26:14.893466Z","iopub.status.idle":"2022-11-23T11:26:24.353595Z","shell.execute_reply.started":"2022-11-23T11:26:14.893425Z","shell.execute_reply":"2022-11-23T11:26:24.352574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:31.430605Z","iopub.execute_input":"2022-11-23T11:26:31.431284Z","iopub.status.idle":"2022-11-23T11:26:31.446503Z","shell.execute_reply.started":"2022-11-23T11:26:31.431248Z","shell.execute_reply":"2022-11-23T11:26:31.445505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 0\nfig, axs = plt.subplots(ncols=2, figsize=(8, 4), sharey='all')\nfig.suptitle(f\"Label: {labels[idx].numpy()}\")\nkwargs_imshow = dict(aspect='auto', vmin=0.0, vmax=255.)\naxs[0].imshow(imgs[idx][..., 0], **kwargs_imshow)\naxs[0].set_title(\"h1\")\naxs[1].imshow(imgs[idx][..., 1], **kwargs_imshow)\naxs[1].set_title(\"l1\")\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:34.482424Z","iopub.execute_input":"2022-11-23T11:26:34.482797Z","iopub.status.idle":"2022-11-23T11:26:34.957197Z","shell.execute_reply.started":"2022-11-23T11:26:34.482765Z","shell.execute_reply":"2022-11-23T11:26:34.950807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural network\n\nThe pretrained TF model cannot directly handle 2 input channels (in contrast to ``timm`` as in [Basic spectrogram image classification](https://www.kaggle.com/code/junkoda/basic-spectrogram-image-classification)). Thus a prio ``1x1`` convolution is used to create a 3 channel input, similar as in [G2Net TF-Keras Train-Test With TPU](https://www.kaggle.com/code/itsuki9180/g2net-tf-keras-train-test-with-tpu).","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications.efficientnet import EfficientNetB1 as BaseModel\nfrom tensorflow.keras.applications.efficientnet import preprocess_input\nfrom tensorflow.keras import layers, Model, Input, losses, metrics, optimizers, callbacks","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:44.669219Z","iopub.execute_input":"2022-11-23T11:26:44.669572Z","iopub.status.idle":"2022-11-23T11:26:45.445511Z","shell.execute_reply.started":"2022-11-23T11:26:44.66954Z","shell.execute_reply":"2022-11-23T11:26:45.444352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unfreeze_model(model):\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False\n\n\ndef create_model():       \n    inputs = Input(shape=cfg.img_shape, dtype=tf.float32)\n    x = layers.Conv2D(3, 1, name='adjustment')(inputs)\n    x = layers.Lambda(preprocess_input, name=\"preprocess_input\")(x)\n    base_model = BaseModel(include_top=False, weights=cfg.base_model_weights, pooling=\"avg\")\n    base_model.trainable = False\n    # unfreeze_model(base_model)\n    x = base_model(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(cfg.dropout, name=\"top_dropout\")(x)\n    outputs = layers.Dense(1, activation=\"sigmoid\", name=\"logits\")(x)\n    model = Model(inputs=inputs, outputs=outputs, name=cfg.model_name)\n    model.compile(\n        optimizer=optimizers.Adam(learning_rate=cfg.lr),\n        loss=losses.BinaryCrossentropy(), \n        metrics=['acc']\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:52.532345Z","iopub.execute_input":"2022-11-23T11:26:52.532725Z","iopub.status.idle":"2022-11-23T11:26:52.543348Z","shell.execute_reply.started":"2022-11-23T11:26:52.532692Z","shell.execute_reply":"2022-11-23T11:26:52.542292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\nwith strategy.scope():\n    model = create_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:26:57.683715Z","iopub.execute_input":"2022-11-23T11:26:57.684073Z","iopub.status.idle":"2022-11-23T11:27:02.415452Z","shell.execute_reply.started":"2022-11-23T11:26:57.684042Z","shell.execute_reply":"2022-11-23T11:27:02.414386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check model","metadata":{"execution":{"iopub.status.busy":"2022-11-19T09:49:48.999348Z","iopub.execute_input":"2022-11-19T09:49:48.999677Z","iopub.status.idle":"2022-11-19T09:49:52.017972Z","shell.execute_reply.started":"2022-11-19T09:49:48.999641Z","shell.execute_reply":"2022-11-19T09:49:52.016952Z"}}},{"cell_type":"code","source":"model.predict(imgs)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:27:06.662239Z","iopub.execute_input":"2022-11-23T11:27:06.663114Z","iopub.status.idle":"2022-11-23T11:27:14.904692Z","shell.execute_reply.started":"2022-11-23T11:27:06.663065Z","shell.execute_reply":"2022-11-23T11:27:14.903675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(imgs, labels, return_dict=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:28:03.296816Z","iopub.execute_input":"2022-11-23T11:28:03.297173Z","iopub.status.idle":"2022-11-23T11:28:06.166743Z","shell.execute_reply.started":"2022-11-23T11:28:03.297144Z","shell.execute_reply":"2022-11-23T11:28:06.165699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def get_callbacks():\n    cbs = [\n        callbacks.EarlyStopping(\n            monitor=\"val_loss\",\n            verbose=1,\n            patience=cfg.patience,\n            restore_best_weights=True,\n        ),\n    ]\n    return cbs\n\n\ndef show_history(history):\n    history_frame = pd.DataFrame(history.history)\n    history_frame.index = pd.RangeIndex(1, len(history_frame) + 1, name=\"epoch\")\n    display(history_frame.style\\\n        .highlight_min(color='lightgreen', subset=['val_loss'])\\\n        .highlight_max(color='lightgreen', subset=['val_acc'])\n    )\n    fig, ax = plt.subplots(1, 2, figsize=(16, 6))\n    history_frame.loc[:, ['loss', 'val_loss']].plot(ax=ax[0], title='loss')\n    history_frame.loc[:, ['acc', 'val_acc']].plot(ax=ax[1], title='acc')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:28:24.77687Z","iopub.execute_input":"2022-11-23T11:28:24.777258Z","iopub.status.idle":"2022-11-23T11:28:24.785759Z","shell.execute_reply.started":"2022-11-23T11:28:24.777223Z","shell.execute_reply":"2022-11-23T11:28:24.784542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_steps = count_data_items(train_files) // cfg.batch_size\nprint(\"steps per epoch:\", train_steps)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:28:35.083461Z","iopub.execute_input":"2022-11-23T11:28:35.083851Z","iopub.status.idle":"2022-11-23T11:28:35.089823Z","shell.execute_reply.started":"2022-11-23T11:28:35.083818Z","shell.execute_reply":"2022-11-23T11:28:35.088792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhistory = model.fit(\n    train_ds, \n    epochs=cfg.epochs,\n    steps_per_epoch=train_steps,\n    validation_data=valid_ds,\n    callbacks=get_callbacks(),\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:28:38.826223Z","iopub.execute_input":"2022-11-23T11:28:38.826576Z","iopub.status.idle":"2022-11-23T11:45:58.324883Z","shell.execute_reply.started":"2022-11-23T11:28:38.826545Z","shell.execute_reply":"2022-11-23T11:45:58.323419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_history(history)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:46:30.36693Z","iopub.execute_input":"2022-11-23T11:46:30.367285Z","iopub.status.idle":"2022-11-23T11:47:09.85051Z","shell.execute_reply.started":"2022-11-23T11:46:30.367257Z","shell.execute_reply":"2022-11-23T11:47:09.849557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(cfg.model_dir)\nos.listdir(cfg.model_dir)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:47:25.738407Z","iopub.execute_input":"2022-11-23T11:47:25.738789Z","iopub.status.idle":"2022-11-23T11:48:15.361886Z","shell.execute_reply.started":"2022-11-23T11:47:25.738757Z","shell.execute_reply":"2022-11-23T11:48:15.360877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test\n\nPredict on test dataset.","metadata":{}},{"cell_type":"code","source":"test_ids = np.concatenate([x.numpy() for x in test_ds.map(lambda imgs, ids: ids)])\ntest_ids","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:48:23.687014Z","iopub.execute_input":"2022-11-23T11:48:23.68774Z","iopub.status.idle":"2022-11-23T11:52:32.309584Z","shell.execute_reply.started":"2022-11-23T11:48:23.687702Z","shell.execute_reply":"2022-11-23T11:52:32.308528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_ds, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:52:32.311606Z","iopub.execute_input":"2022-11-23T11:52:32.312262Z","iopub.status.idle":"2022-11-23T11:56:44.055836Z","shell.execute_reply.started":"2022-11-23T11:52:32.312226Z","shell.execute_reply":"2022-11-23T11:56:44.05489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_ids,\n    \"target\": preds.ravel()\n})\nsubmission[\"id\"] = submission[\"id\"].str.decode('utf-8')\nsubmission = submission.sort_values(by=\"id\")\nsubmission = submission.reset_index(drop=True)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:00.508103Z","iopub.execute_input":"2022-11-23T11:57:00.50846Z","iopub.status.idle":"2022-11-23T11:57:00.538224Z","shell.execute_reply.started":"2022-11-23T11:57:00.508431Z","shell.execute_reply":"2022-11-23T11:57:00.53708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check submission","metadata":{"execution":{"iopub.status.busy":"2022-11-19T12:46:13.250228Z","iopub.execute_input":"2022-11-19T12:46:13.250498Z","iopub.status.idle":"2022-11-19T12:46:13.27234Z","shell.execute_reply.started":"2022-11-19T12:46:13.250469Z","shell.execute_reply":"2022-11-19T12:46:13.271668Z"}}},{"cell_type":"code","source":"sample_submission = pd.read_csv(cfg.path_submission)\n(sample_submission[\"id\"] == submission[\"id\"]).all()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:08.504156Z","iopub.execute_input":"2022-11-23T11:57:08.504541Z","iopub.status.idle":"2022-11-23T11:57:08.526442Z","shell.execute_reply.started":"2022-11-23T11:57:08.50451Z","shell.execute_reply":"2022-11-23T11:57:08.525552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:12.128563Z","iopub.execute_input":"2022-11-23T11:57:12.129162Z","iopub.status.idle":"2022-11-23T11:57:12.274037Z","shell.execute_reply.started":"2022-11-23T11:57:12.129129Z","shell.execute_reply":"2022-11-23T11:57:12.273082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"target\"].describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:15.310064Z","iopub.execute_input":"2022-11-23T11:57:15.310431Z","iopub.status.idle":"2022-11-23T11:57:15.334611Z","shell.execute_reply.started":"2022-11-23T11:57:15.3104Z","shell.execute_reply":"2022-11-23T11:57:15.33345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"View distribution","metadata":{}},{"cell_type":"code","source":"sns.displot(submission, x=\"target\", kde=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:24.349974Z","iopub.execute_input":"2022-11-23T11:57:24.350379Z","iopub.status.idle":"2022-11-23T11:57:24.764166Z","shell.execute_reply.started":"2022-11-23T11:57:24.350342Z","shell.execute_reply":"2022-11-23T11:57:24.763021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write submission","metadata":{}},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:19.845727Z","iopub.execute_input":"2022-11-23T11:57:19.846397Z","iopub.status.idle":"2022-11-23T11:57:19.86906Z","shell.execute_reply.started":"2022-11-23T11:57:19.846361Z","shell.execute_reply":"2022-11-23T11:57:19.86814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-11-23T11:57:20.980077Z","iopub.execute_input":"2022-11-23T11:57:20.980435Z","iopub.status.idle":"2022-11-23T11:57:22.043198Z","shell.execute_reply.started":"2022-11-23T11:57:20.980405Z","shell.execute_reply":"2022-11-23T11:57:22.04199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}