{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# imports","metadata":{"papermill":{"duration":0.007978,"end_time":"2023-07-18T09:35:21.917345","exception":false,"start_time":"2023-07-18T09:35:21.909367","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\nimport os\nfrom argparse import Namespace","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:21.935295Z","iopub.status.busy":"2023-07-18T09:35:21.934976Z","iopub.status.idle":"2023-07-18T09:35:30.60864Z","shell.execute_reply":"2023-07-18T09:35:30.60764Z"},"papermill":{"duration":8.685794,"end_time":"2023-07-18T09:35:30.611481","exception":false,"start_time":"2023-07-18T09:35:21.925687","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the GPU device\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:30.632765Z","iopub.status.busy":"2023-07-18T09:35:30.630873Z","iopub.status.idle":"2023-07-18T09:35:30.636953Z","shell.execute_reply":"2023-07-18T09:35:30.636094Z"},"papermill":{"duration":0.017744,"end_time":"2023-07-18T09:35:30.638763","exception":false,"start_time":"2023-07-18T09:35:30.621019","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# config","metadata":{"papermill":{"duration":0.008466,"end_time":"2023-07-18T09:35:30.656449","exception":false,"start_time":"2023-07-18T09:35:30.647983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if os.environ.get(\"KAGGLE_KERNEL_RUN_TYPE\", \"\"):\n    BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\nelse:\n    BASE_DIR =  'data'","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:30.675855Z","iopub.status.busy":"2023-07-18T09:35:30.674378Z","iopub.status.idle":"2023-07-18T09:35:30.679872Z","shell.execute_reply":"2023-07-18T09:35:30.678874Z"},"papermill":{"duration":0.017097,"end_time":"2023-07-18T09:35:30.682169","exception":false,"start_time":"2023-07-18T09:35:30.665072","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs = Namespace(\n        BASE_DIR= BASE_DIR,\n        BATCH_SIZE= 64,\n        LR= 1e-4,\n        EPOCHS= 10,\n        IMG_SIZE= 256,\n        THRESHOLD= 0.4\n)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:30.700442Z","iopub.status.busy":"2023-07-18T09:35:30.70015Z","iopub.status.idle":"2023-07-18T09:35:30.704952Z","shell.execute_reply":"2023-07-18T09:35:30.703901Z"},"papermill":{"duration":0.016625,"end_time":"2023-07-18T09:35:30.707432","exception":false,"start_time":"2023-07-18T09:35:30.690807","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load data","metadata":{"papermill":{"duration":0.008587,"end_time":"2023-07-18T09:35:30.724755","exception":false,"start_time":"2023-07-18T09:35:30.716168","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_train_idx = pd.DataFrame({'idx': os.listdir(f'{configs.BASE_DIR}/train')})\ndf_validation_idx = pd.DataFrame({'idx': os.listdir(f'{configs.BASE_DIR}/validation')})","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:30.74318Z","iopub.status.busy":"2023-07-18T09:35:30.742892Z","iopub.status.idle":"2023-07-18T09:35:31.274386Z","shell.execute_reply":"2023-07-18T09:35:31.273165Z"},"papermill":{"duration":0.543846,"end_time":"2023-07-18T09:35:31.277205","exception":false,"start_time":"2023-07-18T09:35:30.733359","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.listdir(f'{configs.BASE_DIR}/validation')","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:31.298108Z","iopub.status.busy":"2023-07-18T09:35:31.297132Z","iopub.status.idle":"2023-07-18T09:35:31.302186Z","shell.execute_reply":"2023-07-18T09:35:31.301225Z"},"papermill":{"duration":0.017627,"end_time":"2023-07-18T09:35:31.304269","exception":false,"start_time":"2023-07-18T09:35:31.286642","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_validation_idx.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:31.324992Z","iopub.status.busy":"2023-07-18T09:35:31.324035Z","iopub.status.idle":"2023-07-18T09:35:31.341228Z","shell.execute_reply":"2023-07-18T09:35:31.340318Z"},"papermill":{"duration":0.029706,"end_time":"2023-07-18T09:35:31.34334","exception":false,"start_time":"2023-07-18T09:35:31.313634","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_idx.shape","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:31.36375Z","iopub.status.busy":"2023-07-18T09:35:31.362886Z","iopub.status.idle":"2023-07-18T09:35:31.369254Z","shell.execute_reply":"2023-07-18T09:35:31.368284Z"},"papermill":{"duration":0.018155,"end_time":"2023-07-18T09:35:31.37148","exception":false,"start_time":"2023-07-18T09:35:31.353325","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\nN_TIMES_BEFORE = 4\n\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef load_one_record(record_id, BASE_DIR,train=True):\n    band11 = np.load(f\"{BASE_DIR}/{str(record_id)}/band_11.npy\")\n    band14 = np.load(f\"{BASE_DIR}/{str(record_id)}/band_14.npy\")\n    band15 = np.load(f\"{BASE_DIR}/{str(record_id)}/band_15.npy\")\n        \n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    \n    img = false_color[..., N_TIMES_BEFORE]\n\n    if not train:\n        return img\n\n    human_pixel_mask =  np.load(f\"{BASE_DIR}/{str(record_id)}/human_pixel_masks.npy\")\n    \n    return img, human_pixel_mask\n\ndef train_loader_function():\n    for i in range(df_train_idx.shape[0]):\n        row = df_train_idx.iloc[i]\n        record_id = row[\"idx\"]\n        img, mask = load_one_record(record_id, f\"{configs.BASE_DIR}/train\")\n        yield img, mask\n        \ndef validation_loader_function():\n    for i in range(df_validation_idx.shape[0]):\n        row = df_validation_idx.iloc[i]\n        record_id = row[\"idx\"]\n        img, mask = load_one_record(record_id, f\"{configs.BASE_DIR}/validation\")\n        yield img, mask\n\n\n\ntrain_dataloader = tf.data.Dataset.from_generator(train_loader_function, (tf.float32, tf.float32), output_shapes=([configs.IMG_SIZE,configs.IMG_SIZE,3], [configs.IMG_SIZE, configs.IMG_SIZE, 1]))\nvalidation_dataloader = tf.data.Dataset.from_generator(validation_loader_function, (tf.float32, tf.float32), output_shapes=([configs.IMG_SIZE,configs.IMG_SIZE,3], [configs.IMG_SIZE, configs.IMG_SIZE, 1]))\ntrain_dataset = train_dataloader.prefetch(tf.data.AUTOTUNE).batch(configs.BATCH_SIZE).repeat()\nvalidation_dataset = validation_dataloader.prefetch(tf.data.AUTOTUNE).batch(configs.BATCH_SIZE).repeat()","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:31.391505Z","iopub.status.busy":"2023-07-18T09:35:31.390624Z","iopub.status.idle":"2023-07-18T09:35:34.610601Z","shell.execute_reply":"2023-07-18T09:35:34.609487Z"},"papermill":{"duration":3.232803,"end_time":"2023-07-18T09:35:34.613084","exception":false,"start_time":"2023-07-18T09:35:31.380281","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_dataset:\n    print(i[0].shape)\n    print(i[1].shape)\n    break","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:34.63298Z","iopub.status.busy":"2023-07-18T09:35:34.63264Z","iopub.status.idle":"2023-07-18T09:35:41.231243Z","shell.execute_reply":"2023-07-18T09:35:41.229981Z"},"papermill":{"duration":6.611055,"end_time":"2023-07-18T09:35:41.233436","exception":false,"start_time":"2023-07-18T09:35:34.622381","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{"papermill":{"duration":0.009047,"end_time":"2023-07-18T09:35:41.251988","exception":false,"start_time":"2023-07-18T09:35:41.242941","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## U-Net","metadata":{"papermill":{"duration":0.008898,"end_time":"2023-07-18T09:35:41.270139","exception":false,"start_time":"2023-07-18T09:35:41.261241","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# define U-Net model\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, concatenate, Conv2DTranspose, Dropout\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.losses import BinaryCrossentropy\nfrom tensorflow.keras.metrics import MeanIoU\n\ndef create_mask(image, threshold):\n    mask = tf.math.greater(image, threshold)\n    return tf.cast(mask, tf.float32)\n\ndef iou_coef(y_true, y_pred, smooth=1):\n\n    y_pred = create_mask(y_pred, 0.5)\n    intersection = K.sum(K.abs(y_true * y_pred), axis=[1,2,3])\n    union = K.sum(y_true,[1,2,3])+K.sum(y_pred,[1,2,3])-intersection\n    iou = K.mean((intersection + smooth) / (union + smooth), axis=0)\n    return iou\n\ndef iou_coef_loss(y_true, y_pred):\n\n    y_pred = create_mask(y_pred, configs.THRESHOLD)\n\n    batch_size = K.cast(K.shape(y_true)[0], dtype=tf.float32)\n\n    bce_loss = BinaryCrossentropy()(y_true, y_pred)\n    iou_loss = 1 - iou_coef(y_true, y_pred)\n\n\n    return (bce_loss + iou_loss)\n\ndef dice_coef(y_true, y_pred, smooth=1):\n\n    y_pred = create_mask(y_pred, 0.5)\n    intersection = K.sum(y_true * y_pred, axis=[1,2,3])\n    # union = K.sum(y_true, axis=[1,2,3]) + K.sum(y_pred, axis=[1,2,3])\n    union = K.sum(y_true + y_pred, axis=[1,2,3])\n    dice = K.mean((2. * intersection + smooth)/(union + smooth), axis=0)\n    return dice\n\ndef dice_coef_loss(y_true, y_pred):\n\n    y_pred = create_mask(y_pred, 0.5)\n    batch_size = K.cast(K.shape(y_true)[0], dtype=tf.float32)\n    bce_loss = BinaryCrossentropy()(y_true, y_pred)\n    dice_loss = 1 - dice_coef(y_true, y_pred)\n\n    return (bce_loss + dice_loss)\n\n\nclass UNet:\n\n    def __init__(self, input_shape):\n        self.input_shape = input_shape\n\n    def _conv_block(self, input_tensor, n_filters, kernel_size=3):\n        x = Conv2D(filters=n_filters, kernel_size=(kernel_size, kernel_size), padding=\"same\")(input_tensor)\n        x = Conv2D(filters=n_filters, kernel_size=(kernel_size, kernel_size), padding=\"same\")(x)\n        return x\n\n    def resnet_encoder(self, input_tensor, filters):\n        skip= self._conv_block(input_tensor, filters)\n        x = MaxPooling2D((2, 2))(skip)\n\n        return x, skip\n    \n    def resnet_decoder(self, input_tensor, skip_tensor, filters):\n        x = Conv2DTranspose(filters=filters, kernel_size=(3, 3), strides=(2, 2), padding=\"same\")(input_tensor)\n        x = concatenate([x, skip_tensor])\n        x = self._conv_block(x, filters)\n        return x\n    \n    def bottleneck(self, input_tensor, filters):\n        x = self._conv_block(input_tensor, filters)\n        return x\n    \n    def lstm_layer(self, input_tensor, filters):\n        x = tf.keras.layers.ConvLSTM2D(filters=filters, kernel_size=(3, 3), padding=\"same\", return_sequences=False, data_format=\"channels_last\")(input_tensor)\n        return x\n\n\n\n    def build(self):\n        \n        input = Input(shape=self.input_shape)\n\n        x = input\n\n\n        # Encoder\n        x, skip0 = self.resnet_encoder(x, 32)\n\n        x, skip1 = self.resnet_encoder(x, 64)\n        x = tf.keras.layers.BatchNormalization()(x)\n\n        x, skip2 = self.resnet_encoder(x, 128)\n\n        x, skip3 = self.resnet_encoder(x, 256)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = Dropout(0.3)(x)\n\n        x, skip4 = self.resnet_encoder(x, 512)\n\n        # Bottleneck\n        x = self.bottleneck(x, 512)\n\n        # Decoder\n        x = self.resnet_decoder(x, skip4, 512)\n        \n        \n        x = self.resnet_decoder(x, skip3, 256)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = Dropout(0.3)(x)\n        \n        x = self.resnet_decoder(x, skip2, 128)\n\n        x = self.resnet_decoder(x, skip1, 64)\n        x = tf.keras.layers.BatchNormalization()(x)\n        \n        x = self.resnet_decoder(x, skip0, 32)\n\n        # Output\n        output_tensor = Conv2D(filters=1, kernel_size=(1, 1), activation=\"sigmoid\")(x)\n\n        model = Model([input], output_tensor)\n        return model\n\n\nmodel = UNet((256, 256, 3)).build()\n\nmodel.compile(\n    optimizer=Adam(learning_rate=1e-3),\n    loss=tf.keras.losses.BinaryFocalCrossentropy(),\n    metrics=[dice_coef]\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:41.289677Z","iopub.status.busy":"2023-07-18T09:35:41.289379Z","iopub.status.idle":"2023-07-18T09:35:41.862349Z","shell.execute_reply":"2023-07-18T09:35:41.861414Z"},"papermill":{"duration":0.669356,"end_time":"2023-07-18T09:35:41.948515","exception":false,"start_time":"2023-07-18T09:35:41.279159","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = [\n    EarlyStopping(patience=1, verbose=1, restore_best_weights=True, monitor='dice_coef', mode='max'),\n    ReduceLROnPlateau(factor=0.1, patience=3, min_lr=1e-8, verbose=1),\n    ModelCheckpoint('callbacks/model.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]\n\ntrain_steps = df_train_idx.shape[0]//configs.BATCH_SIZE\nvalidation_steps = df_validation_idx.shape[0]//configs.BATCH_SIZE\n\n# train_steps = 20\n# validation_steps = 20\n\n# history = model.fit(train_dataset, epochs=1, verbose=1, validation_data=validation_dataset,  callbacks=callbacks, steps_per_epoch=train_steps, validation_steps=validation_steps)\n\nhistory = model.fit(train_dataset, epochs=5, verbose=1, callbacks=callbacks, steps_per_epoch=train_steps, shuffle=True)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T09:35:41.988967Z","iopub.status.busy":"2023-07-18T09:35:41.988625Z","iopub.status.idle":"2023-07-18T10:33:45.800294Z","shell.execute_reply":"2023-07-18T10:33:45.798351Z"},"papermill":{"duration":3483.839152,"end_time":"2023-07-18T10:33:45.807127","exception":false,"start_time":"2023-07-18T09:35:41.967975","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# save the model","metadata":{"papermill":{"duration":0.10172,"end_time":"2023-07-18T10:33:46.017966","exception":false,"start_time":"2023-07-18T10:33:45.916246","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model.evaluate(validation_dataset, steps=validation_steps)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:33:46.20933Z","iopub.status.busy":"2023-07-18T10:33:46.208286Z","iopub.status.idle":"2023-07-18T10:36:38.004034Z","shell.execute_reply":"2023-07-18T10:36:38.002968Z"},"papermill":{"duration":171.894414,"end_time":"2023-07-18T10:36:38.007666","exception":false,"start_time":"2023-07-18T10:33:46.113252","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model.h5')","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:38.211308Z","iopub.status.busy":"2023-07-18T10:36:38.210036Z","iopub.status.idle":"2023-07-18T10:36:38.976339Z","shell.execute_reply":"2023-07-18T10:36:38.975209Z"},"papermill":{"duration":0.859059,"end_time":"2023-07-18T10:36:38.979082","exception":false,"start_time":"2023-07-18T10:36:38.120023","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_validation_idx.tail()","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:39.129407Z","iopub.status.busy":"2023-07-18T10:36:39.129026Z","iopub.status.idle":"2023-07-18T10:36:39.15586Z","shell.execute_reply":"2023-07-18T10:36:39.154964Z"},"papermill":{"duration":0.106684,"end_time":"2023-07-18T10:36:39.158256","exception":false,"start_time":"2023-07-18T10:36:39.051572","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_validation_image(indx):\n    val_img, val_mask = load_one_record(df_validation_idx.iloc[indx][\"idx\"], f\"{configs.BASE_DIR}/validation\", train=True)\n    plt.figure(figsize=(18, 6))\n    ax = plt.subplot(1, 3, 1)\n    ax.imshow(val_img)\n    ax.set_title('False color image')\n\n    ax = plt.subplot(1, 3, 2)\n    ax.imshow(val_mask, interpolation='none')\n    ax.set_title('Ground truth contrail mask')\n\n\n\n    pred = model.predict(val_img.reshape(1, 256, 256, 3))\n#     pred_mask = create_mask(pred, configs.THRESHOLD)\n    pred_mask = create_mask(pred, 0.4)\n    pred_mask = pred_mask[0, :, :, 0]\n\n    ax = plt.subplot(1, 3, 3)\n    ax.imshow(pred_mask, interpolation='none')\n    ax.set_title('Predicted contrail mask')\n\n\nplot_validation_image(5)\n# plot_validation_image(105)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:39.310368Z","iopub.status.busy":"2023-07-18T10:36:39.309399Z","iopub.status.idle":"2023-07-18T10:36:41.433873Z","shell.execute_reply":"2023-07-18T10:36:41.43287Z"},"papermill":{"duration":2.204704,"end_time":"2023-07-18T10:36:41.436936","exception":false,"start_time":"2023-07-18T10:36:39.232232","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test","metadata":{"papermill":{"duration":0.075771,"end_time":"2023-07-18T10:36:41.588519","exception":false,"start_time":"2023-07-18T10:36:41.512748","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# rle encoding","metadata":{"papermill":{"duration":0.07713,"end_time":"2023-07-18T10:36:41.742971","exception":false,"start_time":"2023-07-18T10:36:41.665841","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#source https://www.kaggle.com/code/inversion/contrails-rle-submission?scriptVersionId=128527711&cellId=4\n\ndef rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s\n","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:41.899105Z","iopub.status.busy":"2023-07-18T10:36:41.898698Z","iopub.status.idle":"2023-07-18T10:36:41.908893Z","shell.execute_reply":"2023-07-18T10:36:41.907952Z"},"papermill":{"duration":0.090547,"end_time":"2023-07-18T10:36:41.911106","exception":false,"start_time":"2023-07-18T10:36:41.820559","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_idx = pd.DataFrame({'idx': os.listdir(f'{configs.BASE_DIR}/test')})\n\ndef test_loader_function():\n    for i in range(df_test_idx.shape[0]):\n        row = df_test_idx.iloc[i]\n        record_id = row[\"idx\"]\n        img = load_one_record(record_id, f\"{configs.BASE_DIR}/test\", train=False)\n        yield img\n\ntest_dataloader = tf.data.Dataset.from_generator(test_loader_function, tf.float32, output_shapes=([configs.IMG_SIZE, configs.IMG_SIZE, 3]))\ntest_dataset = test_dataloader.prefetch(tf.data.AUTOTUNE).batch(configs.BATCH_SIZE)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:42.069636Z","iopub.status.busy":"2023-07-18T10:36:42.069255Z","iopub.status.idle":"2023-07-18T10:36:42.10982Z","shell.execute_reply":"2023-07-18T10:36:42.108769Z"},"papermill":{"duration":0.121609,"end_time":"2023-07-18T10:36:42.112015","exception":false,"start_time":"2023-07-18T10:36:41.990406","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_dataset, verbose=0)","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:42.269966Z","iopub.status.busy":"2023-07-18T10:36:42.269249Z","iopub.status.idle":"2023-07-18T10:36:43.884279Z","shell.execute_reply":"2023-07-18T10:36:43.883187Z"},"papermill":{"duration":1.697368,"end_time":"2023-07-18T10:36:43.887197","exception":false,"start_time":"2023-07-18T10:36:42.189829","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an empty DataFrame for submission\nsubmission_df = pd.DataFrame(columns=['encoded_pixels'])\nsubmission_df.index.name = 'record_id'","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:44.061818Z","iopub.status.busy":"2023-07-18T10:36:44.061221Z","iopub.status.idle":"2023-07-18T10:36:44.071268Z","shell.execute_reply":"2023-07-18T10:36:44.069994Z"},"papermill":{"duration":0.102963,"end_time":"2023-07-18T10:36:44.074007","exception":false,"start_time":"2023-07-18T10:36:43.971044","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_predictions = create_mask(predictions,configs.THRESHOLD)\nmasked_predictions_ = np.array(masked_predictions)\n\n\nfor i in range(predictions.shape[0]):\n    submission_df.loc[df_test_idx.iloc[i]['idx'], \"encoded_pixels\"] = list_to_string(rle_encode(masked_predictions_[i]))","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:44.249383Z","iopub.status.busy":"2023-07-18T10:36:44.247261Z","iopub.status.idle":"2023-07-18T10:36:44.271376Z","shell.execute_reply":"2023-07-18T10:36:44.2702Z"},"papermill":{"duration":0.115471,"end_time":"2023-07-18T10:36:44.274065","exception":false,"start_time":"2023-07-18T10:36:44.158594","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:44.431035Z","iopub.status.busy":"2023-07-18T10:36:44.430618Z","iopub.status.idle":"2023-07-18T10:36:44.440156Z","shell.execute_reply":"2023-07-18T10:36:44.438998Z"},"papermill":{"duration":0.09212,"end_time":"2023-07-18T10:36:44.44268","exception":false,"start_time":"2023-07-18T10:36:44.35056","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.execute_input":"2023-07-18T10:36:44.605792Z","iopub.status.busy":"2023-07-18T10:36:44.604828Z","iopub.status.idle":"2023-07-18T10:36:44.61857Z","shell.execute_reply":"2023-07-18T10:36:44.617582Z"},"papermill":{"duration":0.092687,"end_time":"2023-07-18T10:36:44.620789","exception":false,"start_time":"2023-07-18T10:36:44.528102","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}