{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nfrom path import Path\nimport os\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom IPython import display\nfrom IPython.display import clear_output\nfrom matplotlib import animation\nimport datetime\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.losses import binary_crossentropy\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, Dropout, concatenate\nimport cv2\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n\nprint(f\"tensorflow version: {tf.__version__}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-09T21:28:51.227991Z","iopub.execute_input":"2023-07-09T21:28:51.228417Z","iopub.status.idle":"2023-07-09T21:28:58.328672Z","shell.execute_reply.started":"2023-07-09T21:28:51.228379Z","shell.execute_reply":"2023-07-09T21:28:58.327656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding the Evaluation Metric\n\n## Global Dice Coefficient\n\nThe global dice coefficient, also known as the Sørensen–Dice coefficient, is a similarity metric used to measure the similarity or overlap between two sets. It is defined as twice the size of the intersection of the sets divided by the sum of the sizes of the sets.\n\nMathematically, the global dice coefficient can be expressed as:\n\n![equation](https://latex.codecogs.com/png.latex?%5Cdpi%7B120%7D%20%5Cfrac%7B2%20%7C%20A%20%5Ccap%20B%20%7C%7D%7B%7C%20A%20%7C%20+%20%7C%20B%20%7C%7D)\n\nWhere:\n- |A| represents the size or cardinality of set A.\n- |B| represents the size or cardinality of set B.\n- |A ∩ B| represents the size or cardinality of the intersection of sets A and B.\n\nThe global dice coefficient ranges between 0 and 1, where a value of 1 indicates a perfect overlap or similarity between the sets, and a value of 0 indicates no overlap or dissimilarity.\n","metadata":{}},{"cell_type":"markdown","source":"# Data EDA","metadata":{}},{"cell_type":"code","source":"BASE_DIR = Path(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming\")\n\ntrain_list = os.listdir(BASE_DIR + '/train/')\nval_list = os.listdir(BASE_DIR + '/validation/')\ntest_list = os.listdir(BASE_DIR + '/test/')\n\nprint (f\"Number of training examples: {len(train_list)}\")\nprint(f\"Number of validation examples: {len(val_list)}\")\nprint(f\"Number of test examples: {len(test_list)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:58.330771Z","iopub.execute_input":"2023-07-09T21:28:58.332178Z","iopub.status.idle":"2023-07-09T21:28:58.601603Z","shell.execute_reply.started":"2023-07-09T21:28:58.33214Z","shell.execute_reply":"2023-07-09T21:28:58.600673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json\")\nval_metadata = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json\")\n\ntrain_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:58.602856Z","iopub.execute_input":"2023-07-09T21:28:58.603224Z","iopub.status.idle":"2023-07-09T21:28:58.982092Z","shell.execute_reply.started":"2023-07-09T21:28:58.603185Z","shell.execute_reply":"2023-07-09T21:28:58.981128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:58.985071Z","iopub.execute_input":"2023-07-09T21:28:58.985688Z","iopub.status.idle":"2023-07-09T21:28:58.998802Z","shell.execute_reply.started":"2023-07-09T21:28:58.985644Z","shell.execute_reply":"2023-07-09T21:28:58.997804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Class Distribution\n\nAs we can see there are 11270 negative examples and 9259 positive examples. This shows there are slightly more examples with no contrails however the split seems pretty close to 50/50.","metadata":{}},{"cell_type":"code","source":"#pos_examples = 0\n\n# for example in tqdm(train_list):\n#     im = np.load(f\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/{example}/human_pixel_masks.npy\")\n#     if len(np.unique(im)) > 1:\n#         pos_examples += 1\n\n#print(f\"positive examples: {pos_examples}, neg_examples: {len(train_list) - pos_examples}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:59.00035Z","iopub.execute_input":"2023-07-09T21:28:59.001347Z","iopub.status.idle":"2023-07-09T21:28:59.008554Z","shell.execute_reply.started":"2023-07-09T21:28:59.001307Z","shell.execute_reply":"2023-07-09T21:28:59.007588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos_examples = 9259\nneg_examples = len(train_list) - pos_examples\n\ncategories = ['Positive (contrails)', 'Negative (no contrails)']\ncounts = [pos_examples, neg_examples]\n\nplt.bar(categories, counts)\nplt.xlabel('Category')\nplt.ylabel('Count')\nplt.title('Distribution of Examples in Training Set')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:59.010013Z","iopub.execute_input":"2023-07-09T21:28:59.010502Z","iopub.status.idle":"2023-07-09T21:28:59.231727Z","shell.execute_reply.started":"2023-07-09T21:28:59.010469Z","shell.execute_reply":"2023-07-09T21:28:59.230839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Advanced Baseline Imager(ABI) & Understanding Spectral Bands\n\nThe Advanced Baseline Imager (ABI) is an advanced weather satellite instrument used for Earth observation. It is one of the primary instruments onboard the Geostationary Operational Environmental Satellite (GOES) series, specifically designed for weather monitoring and forecasting. The ABI provides high-resolution, multi-channel imaging capabilities that significantly enhance weather analysis and forecasting capabilities.\n\n**Spectral Bands:** The ABI has 16 spectral bands each designed to capture specific information about the Earth's atmosphere, clouds, land, and water. These bands cover a wide range of wavelengths, including visible, infrared, and near-infrared regions, hence why there are 16 bands in each example. ABI views the Earth with 16 different spectral bands (compared to five on the previous generation of GOES), including two visible channels, four near-infrared channels, and ten infrared channels.\n\n","metadata":{}},{"cell_type":"code","source":"bands = ['08', '09', '10', '11', '12', '13', '14', '15', '16']\ncolors = ['red', 'blue', 'green', 'orange', 'purple', 'cyan', 'magenta', 'yellow', 'black']\n\nfor i, band in enumerate(bands):\n    plot = np.load(f'/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_{band}.npy')\n    plt.plot(plot[0], color=colors[i], label=f'Band {band}')","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:59.233016Z","iopub.execute_input":"2023-07-09T21:28:59.234026Z","iopub.status.idle":"2023-07-09T21:28:59.787309Z","shell.execute_reply.started":"2023-07-09T21:28:59.233977Z","shell.execute_reply":"2023-07-09T21:28:59.786416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Understanding Spectral Bands\n\nIn the dataset, there are 9 bands for each example, and each band represents a series of images captured at different wavelengths of light. For instance, band 08 consists of a series of images highlighting a specific wavelength, while band 16 contains images capturing a different wavelength. It is important to note that each band contains images of the same thing just at different wavelengths so each band will look slightly different and contain more information than others because contrails look different at different wavelengths\n\nThese individual bands contain images taken at 10-minute intervals. There are 8 images in total for each band, taken across 80 minutes with a 10-minute interval between each image. When we examine band 8, for example, we are looking at a sequence of images, each taken 10 minutes after the previous one. This time series of images provides valuable information about the expansion and shape changes of contrails over time.\n\nNow, let's discuss the segmentation masks. There are two files: human_pixel_masks and human_individual_masks. The human_individual_masks represent labels generated by multiple labellers. These labels are then compared and combined to create a final ground truth, which can be found in the human_pixel_masks file. This mask corresponds to the 5th image in the bands. The purpose of this approach is to enhance the accuracy of the ground truth. By involving multiple labellers and aggregating their findings, we can achieve a more precise and reliable ground truth. To draw an analogy, just as a group of doctors can collectively provide a more accurate assessment of a disease, multiple labellers collaborating on the labels can produce a more accurate ground truth.","metadata":{}},{"cell_type":"markdown","source":"Each band contains 8 images of size 256 by 256","metadata":{}},{"cell_type":"code","source":"image = np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_08.npy\")\nprint(f\"band 08 shape {image.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:59.788295Z","iopub.execute_input":"2023-07-09T21:28:59.788643Z","iopub.status.idle":"2023-07-09T21:28:59.796107Z","shell.execute_reply.started":"2023-07-09T21:28:59.788592Z","shell.execute_reply":"2023-07-09T21:28:59.795073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plotting So We Can Visualize Our Understanding\n\nAs we can see in the plot as you go across from the left to the right you are seeing the different spectral bands we are looking at, and as you go down you see the image at different time steps (10 minutes apart). You can see how the image looks slightly different across time and bands. The human pixel mask (ground truth) corresponds to the image at time step 5. Hopefully this image can make it easier to understand what is going on, and please feel free to try this out on other examples too!","metadata":{}},{"cell_type":"code","source":"def plot_example(example_id, split_dir):\n    \"\"\"\n    Args: \n        example_id(str): The id of the example i.e. '1000216489776414077'\n        split_dir(str): The split directoryu i.e. 'test', 'train', 'val'\n    \"\"\"\n    fig, axs = plt.subplots(8, len(bands), figsize=(16, 16)) \n\n    for j, band in enumerate(bands):\n        img = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/band_{band}.npy\")\n        for i in range(8):\n            axs[i, j].imshow(img[..., i]) \n            axs[i, j].set_title(f\"Band {band}\\nTime Step {i+1}\") \n\n    plt.tight_layout()  \n    plt.show()\n    \nplot_example('1000603527582775543', 'train')","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:28:59.797696Z","iopub.execute_input":"2023-07-09T21:28:59.798403Z","iopub.status.idle":"2023-07-09T21:29:08.359226Z","shell.execute_reply.started":"2023-07-09T21:28:59.798369Z","shell.execute_reply":"2023-07-09T21:29:08.358333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking Out The Masks\n\nAs we can see the human individual masks contains 4  (256, 256, 1) masks. These 4 masks corresponding to the masks created by 4 labellers. Although there are 4 in this example other examples can have a varying amount. The human pixel masks are the ground truth and there is only one mask of shape (256, 256, 1). This should be expected because the input images are of shape 256 x 256 so we should expect the output shape to be the same","metadata":{}},{"cell_type":"code","source":"masks = np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_individual_masks.npy\")\nground_truth = np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_pixel_masks.npy\")\n\nprint(f\"The human individual masks has shape {masks.shape}\")\nprint(f\"The human pixel masks has shape {ground_truth.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:08.363496Z","iopub.execute_input":"2023-07-09T21:29:08.364072Z","iopub.status.idle":"2023-07-09T21:29:08.386536Z","shell.execute_reply.started":"2023-07-09T21:29:08.364036Z","shell.execute_reply":"2023-07-09T21:29:08.385773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_masks(example_id, split_dir):\n    \"\"\"\n    Args:\n        example_id(str): The id of the example i.e. '1000216489776414077'\n        split_dir(str): The split directoryu i.e. 'test', 'train', 'val'\n    \"\"\"\n    masks = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/human_individual_masks.npy\")\n    ground_truth = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/human_pixel_masks.npy\")\n\n    fig, axs = plt.subplots(1, len(masks[0,0,0]) + 1, figsize=(2 * (len(masks[0,0,0]) + 1), 16)) \n\n    for i in range(len(masks[0,0,0])):\n        axs[i].imshow(masks[..., i])\n        axs[i].set_title(f\"Labeller {i+1}\")\n\n    axs[i+1].imshow(ground_truth)\n    axs[i+1].set_title(\"Aggregated/Ground Truth\")\n\n    plt.tight_layout() \n    plt.show()\n\nplot_masks('1000603527582775543', 'train')","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:08.387841Z","iopub.execute_input":"2023-07-09T21:29:08.388883Z","iopub.status.idle":"2023-07-09T21:29:09.353813Z","shell.execute_reply.started":"2023-07-09T21:29:08.388849Z","shell.execute_reply":"2023-07-09T21:29:09.352908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## False Color Images\n\nWe know that the image labellers are putting their final annotations on the image at time step 5 (the fifth element) in each band. However, the image that the labellers end up annotating is not found in the spectral bands given to use but rather a false color image. A false color image is a image that can be generated from the spectral bands given to us and it is meant to make contrails appear dark relative to their surroundings making them easier to detect. This false color image is what the labelers ended up using to derive their ground truths. \n\nThe false color images are generated using the **ash color scheme**. Here is a brief overview of the color scheme\n\nThe **ash color scheme** is a false color representation commonly used for visualizing volcanic ash plumes or volcanic cloud observations. It employs a three-channel color scheme (red, green, and blue) to highlight specific features of interest:\n\n- **Red Channel**: Represents the temperature or thermal information of the volcanic plume, with warmer regions shown in red/orange and cooler regions in shades of blue/green.\n\n- **Green Channel**: Indicates the particle size or density of volcanic ash particles in the plume. Darker green indicates denser or larger particles, while lighter green represents finer particles.\n\n- **Blue Channel**: Provides additional information about the plume, such as its height or altitude. Higher blue values suggest greater plume altitude, while darker shades imply lower altitudes.\n\nBy combining these color channels, the ash color scheme enhances the visibility of different properties within the volcanic plume, facilitating analysis and interpretation of volcanic cloud data.\n\nAlthough it is commonly used to study volcanic activity it is also useful for identifying contrails\n","metadata":{}},{"cell_type":"code","source":"def normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef get_false_color(example_id, split_dir):\n    \"\"\"\n    Args:\n        example_id(str): The id of the example i.e. '1000216489776414077'\n        split_dir(str): The split directoryu i.e. 'test', 'train', 'val'\n    \"\"\"\n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n\n    band15 = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/band_15.npy\")\n    band14 = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/band_14.npy\")\n    band11 = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/band_11.npy\")\n\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    \n    return false_color\n\ndef plot_false_color(example_id, split_dir, plot):\n\n    false_color = get_false_color(example_id, split_dir)\n\n    img = false_color[..., 4] # 5th image corresponds to ground truth\n    ground_truth = np.load(BASE_DIR + f\"/{split_dir}/{example_id}/human_pixel_masks.npy\")\n    \n    if plot:\n        fig, axs = plt.subplots(1, 3, figsize=(16, 8))\n\n        axs[0].imshow(img)\n        axs[0].set_title(\"False Color Image\")\n\n        axs[1].imshow(ground_truth)\n        axs[1].set_title(\"Ground Truth\")\n\n        axs[2].imshow(img)\n        axs[2].imshow(ground_truth, cmap='Reds', alpha=.3, interpolation='none')\n        axs[2].set_title('Contrail mask on false color image')\n\n\n        plt.tight_layout() \n        plt.show()\n\n    return img\n    \nimg = plot_false_color('1000603527582775543', 'train', True)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:09.355279Z","iopub.execute_input":"2023-07-09T21:29:09.355969Z","iopub.status.idle":"2023-07-09T21:29:10.312159Z","shell.execute_reply.started":"2023-07-09T21:29:09.355923Z","shell.execute_reply":"2023-07-09T21:29:10.309693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Edge detection Algorithims","metadata":{}},{"cell_type":"code","source":"img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\nimg_blur = cv2.GaussianBlur(img_gray, (3,3), 0) \n\nsobelx = cv2.Sobel(src=img_blur, ddepth=cv2.CV_64F, dx=1, dy=0, ksize=5)\nsobely = cv2.Sobel(src=img_blur, ddepth=cv2.CV_64F, dx=0, dy=1, ksize=5) \nsobelxy = cv2.Sobel(src=img_blur, ddepth=cv2.CV_64F, dx=1, dy=1, ksize=5)\n\nplt.imshow(sobelxy)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:10.313582Z","iopub.execute_input":"2023-07-09T21:29:10.31461Z","iopub.status.idle":"2023-07-09T21:29:10.652345Z","shell.execute_reply.started":"2023-07-09T21:29:10.314574Z","shell.execute_reply":"2023-07-09T21:29:10.651504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Animations!\nThe images are taken across time so we can visualize the false color image changing over time through an animation!  \nAs we can see the contrails are changing over time","metadata":{}},{"cell_type":"code","source":"# Animation\ndef draw_animation(example_id, split_dir):\n    def draw(i):\n        im.set_array(false_color[..., i])\n        return [im]\n    \n    false_color = get_false_color(example_id, split_dir)\n    \n    fig = plt.figure(figsize=(6, 6))\n    im = plt.imshow(false_color[..., 0])\n    \n    anim = animation.FuncAnimation(\n        fig, draw, frames=false_color.shape[-1], interval=500, blit=True\n    )\n    \n    plt.close()\n    return anim \n\nanim = draw_animation('1000603527582775543', 'train')\ndisplay.HTML(anim.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:10.653782Z","iopub.execute_input":"2023-07-09T21:29:10.654771Z","iopub.status.idle":"2023-07-09T21:29:12.652317Z","shell.execute_reply.started":"2023-07-09T21:29:10.654737Z","shell.execute_reply":"2023-07-09T21:29:12.64872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a Input Pipeline Using TensorFlow\n\n# WORK IN PROGRESS","metadata":{}},{"cell_type":"code","source":"ds = tf.data.Dataset.list_files(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/*\", shuffle=False)\nv_ds = tf.data.Dataset.list_files(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation/*\", shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:12.653977Z","iopub.execute_input":"2023-07-09T21:29:12.654651Z","iopub.status.idle":"2023-07-09T21:29:20.238394Z","shell.execute_reply.started":"2023-07-09T21:29:12.6546Z","shell.execute_reply":"2023-07-09T21:29:20.23734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_get_false_color_mask(path):\n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n\n    band15 = np.load(f\"{path}/band_15.npy\")\n    band14 = np.load(f\"{path}/band_14.npy\")\n    band11 = np.load(f\"{path}/band_11.npy\")\n    mask = np.load(f\"{path}/human_pixel_masks.npy\")/255\n\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)[...,4]\n    \n    mask = mask.astype(np.float32)\n    false_color = false_color.astype(np.float32)\n    return false_color, mask\n\ndef map_function(path):\n    def preprocess(path):\n        path = path.decode()\n        false_color, mask = tf_get_false_color_mask(path)\n        return false_color, mask\n    \n    false_color, mask = tf.numpy_function(preprocess, [path], [tf.float32, tf.float32])\n    false_color.set_shape([256,256,3])\n    mask.set_shape([256 ,256,1])\n    return false_color, mask","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:20.240804Z","iopub.execute_input":"2023-07-09T21:29:20.241166Z","iopub.status.idle":"2023-07-09T21:29:20.252183Z","shell.execute_reply.started":"2023-07-09T21:29:20.241137Z","shell.execute_reply":"2023-07-09T21:29:20.250316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display(display_list):\n    plt.figure(figsize=(15, 15))\n\n    title = ['Input Image', 'True Mask', 'Predicted Mask']\n\n    for i in range(len(display_list)):\n        plt.subplot(1, len(display_list), i+1)\n        plt.title(title[i])\n        plt.imshow(tf.keras.utils.array_to_img(display_list[i]))\n        plt.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:20.253831Z","iopub.execute_input":"2023-07-09T21:29:20.25477Z","iopub.status.idle":"2023-07-09T21:29:20.269552Z","shell.execute_reply.started":"2023-07-09T21:29:20.254735Z","shell.execute_reply":"2023-07-09T21:29:20.268546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=4\ntrain_ds = ds.map(map_function, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(BATCH_SIZE).take(1)\nval_ds = v_ds.map(map_function, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:20.273014Z","iopub.execute_input":"2023-07-09T21:29:20.273402Z","iopub.status.idle":"2023-07-09T21:29:20.403479Z","shell.execute_reply.started":"2023-07-09T21:29:20.273368Z","shell.execute_reply":"2023-07-09T21:29:20.402492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, mask in train_ds:\n    display([image[0], mask[0]])\n    print(f'image shape {image.shape}')\n    print(f'mask shape {mask.shape}')\n    break","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:20.405715Z","iopub.execute_input":"2023-07-09T21:29:20.406293Z","iopub.status.idle":"2023-07-09T21:29:23.039203Z","shell.execute_reply.started":"2023-07-09T21:29:20.406255Z","shell.execute_reply":"2023-07-09T21:29:23.038184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, mask in val_ds:\n    display([image[0], mask[0]])\n    break","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:23.040732Z","iopub.execute_input":"2023-07-09T21:29:23.041104Z","iopub.status.idle":"2023-07-09T21:29:25.601714Z","shell.execute_reply.started":"2023-07-09T21:29:23.041069Z","shell.execute_reply":"2023-07-09T21:29:25.600689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the Model\n\n## U - NET","metadata":{}},{"cell_type":"code","source":"def simple_unet_model(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS):\n#Build the model\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))\n    s = inputs\n\n    #Contraction path\n    c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(s)\n    c1 = Dropout(0.1)(c1)\n    c1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c1)\n    p1 = MaxPooling2D((2, 2))(c1)\n    \n    c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p1)\n    c2 = Dropout(0.1)(c2)\n    c2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c2)\n    p2 = MaxPooling2D((2, 2))(c2)\n     \n    c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p2)\n    c3 = Dropout(0.2)(c3)\n    c3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c3)\n    p3 = MaxPooling2D((2, 2))(c3)\n     \n    c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p3)\n    c4 = Dropout(0.2)(c4)\n    c4 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c4)\n    p4 = MaxPooling2D(pool_size=(2, 2))(c4)\n     \n    c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(p4)\n    c5 = Dropout(0.3)(c5)\n    c5 = Conv2D(256, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c5)\n    \n    #Expansive path \n    u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)\n    u6 = concatenate([u6, c4])\n    c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u6)\n    c6 = Dropout(0.2)(c6)\n    c6 = Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c6)\n     \n    u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)\n    u7 = concatenate([u7, c3])\n    c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u7)\n    c7 = Dropout(0.2)(c7)\n    c7 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c7)\n     \n    u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)\n    u8 = concatenate([u8, c2])\n    c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u8)\n    c8 = Dropout(0.1)(c8)\n    c8 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c8)\n     \n    u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)\n    u9 = concatenate([u9, c1], axis=3)\n    c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(u9)\n    c9 = Dropout(0.1)(c9)\n    c9 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same')(c9)\n     \n    outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)\n     \n    model = Model(inputs=[inputs], outputs=[outputs])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:25.604708Z","iopub.execute_input":"2023-07-09T21:29:25.605104Z","iopub.status.idle":"2023-07-09T21:29:25.627837Z","shell.execute_reply.started":"2023-07-09T21:29:25.605066Z","shell.execute_reply":"2023-07-09T21:29:25.626844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convolution_block(\n    block_input,\n    num_filters=256,\n    kernel_size=3,\n    dilation_rate=1,\n    padding=\"same\",\n    use_bias=False,\n):\n    x = layers.Conv2D(\n        num_filters,\n        kernel_size=kernel_size,\n        dilation_rate=dilation_rate,\n        padding=\"same\",\n        use_bias=use_bias,\n        kernel_initializer=keras.initializers.HeNormal(),\n    )(block_input)\n    x = layers.BatchNormalization()(x)\n    return tf.nn.relu(x)\n\n\ndef DilatedSpatialPyramidPooling(dspp_input):\n    dims = dspp_input.shape\n    x = layers.AveragePooling2D(pool_size=(dims[-3], dims[-2]))(dspp_input)\n    x = convolution_block(x, kernel_size=1, use_bias=True)\n    out_pool = layers.UpSampling2D(\n        size=(dims[-3] // x.shape[1], dims[-2] // x.shape[2]), interpolation=\"bilinear\",\n    )(x)\n\n    out_1 = convolution_block(dspp_input, kernel_size=1, dilation_rate=1)\n    out_6 = convolution_block(dspp_input, kernel_size=3, dilation_rate=6)\n    out_12 = convolution_block(dspp_input, kernel_size=3, dilation_rate=12)\n    out_18 = convolution_block(dspp_input, kernel_size=3, dilation_rate=18)\n\n    x = layers.Concatenate(axis=-1)([out_pool, out_1, out_6, out_12, out_18])\n    output = convolution_block(x, kernel_size=1)\n    return output\n\ndef DeeplabV3Plus(image_size, num_classes):\n    model_input = keras.Input(shape=(image_size, image_size, 3))\n    resnet50 = keras.applications.ResNet50(\n        weights=\"/kaggle/input/resweights/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\", include_top=False, input_tensor=model_input\n    )\n    x = resnet50.get_layer(\"conv4_block6_2_relu\").output\n    x = DilatedSpatialPyramidPooling(x)\n\n    input_a = layers.UpSampling2D(\n        size=(image_size // 4 // x.shape[1], image_size // 4 // x.shape[2]),\n        interpolation=\"bilinear\",\n    )(x)\n    input_b = resnet50.get_layer(\"conv2_block3_2_relu\").output\n    input_b = convolution_block(input_b, num_filters=48, kernel_size=1)\n\n    x = layers.Concatenate(axis=-1)([input_a, input_b])\n    x = convolution_block(x)\n    x = convolution_block(x)\n    x = layers.UpSampling2D(\n        size=(image_size // x.shape[1], image_size // x.shape[2]),\n        interpolation=\"bilinear\",\n    )(x)\n    model_output = layers.Conv2D(num_classes, kernel_size=(1, 1), padding=\"same\", activation='sigmoid')(x)\n    return keras.Model(inputs=model_input, outputs=model_output)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:25.631061Z","iopub.execute_input":"2023-07-09T21:29:25.631593Z","iopub.status.idle":"2023-07-09T21:29:25.647124Z","shell.execute_reply.started":"2023-07-09T21:29:25.631566Z","shell.execute_reply":"2023-07-09T21:29:25.646174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss Functions","metadata":{}},{"cell_type":"code","source":"def jaccard_coef(y_true, y_pred):\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    return (intersection + 1.0) / (K.sum(y_true_f) + K.sum(y_pred_f) - intersection + 1.0)\n\ndef jaccard_loss(y_true, y_pred):\n    return -jaccard_coef(y_true, y_pred)\n\ndef FocalTverskyLoss(targets, inputs, alpha=0.4, beta=0.7, gamma=3, smooth=1e-6):\n    \n    inputs = K.flatten(inputs)\n    targets = K.flatten(targets)\n\n    TP = K.sum((inputs * targets))\n    FP = K.sum(((1-targets) * inputs))\n    FN = K.sum((targets * (1-inputs)))\n\n    Tversky = (TP + smooth) / (TP + alpha*FP + beta*FN + smooth)  \n    FocalTversky = K.pow((1 - Tversky), gamma)\n\n    return FocalTversky\n\ndef dice_coefficient(targets, inputs, smooth=1e-6):\n\n    inputs = K.flatten(inputs)\n    targets = K.flatten(targets)\n    \n    intersection = K.sum(targets * inputs)\n    dice = (2*intersection + smooth) / (K.sum(targets) + K.sum(inputs) + smooth)\n    return dice\n\ndef dice_loss(y_true, y_pred):\n    loss = 1-dice_coefficient(y_true, y_pred)\n    return loss\n\ndef loss_fn(y_true, y_pred, sigma=0.5):\n    return (1-sigma) * focal_tversky_loss(y_true, y_pred) + sigma * dice_loss(y_true, y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:25.648795Z","iopub.execute_input":"2023-07-09T21:29:25.649168Z","iopub.status.idle":"2023-07-09T21:29:25.663199Z","shell.execute_reply.started":"2023-07-09T21:29:25.649112Z","shell.execute_reply":"2023-07-09T21:29:25.662133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_path = \"deeplab_model_checkpoint.h5\"\n\ncheckpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath=checkpoint_path,\n    save_weights_only=False,\n    save_best_only=True,\n    monitor='val_loss',\n    mode='min',\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:25.664476Z","iopub.execute_input":"2023-07-09T21:29:25.664921Z","iopub.status.idle":"2023-07-09T21:29:25.676944Z","shell.execute_reply.started":"2023-07-09T21:29:25.664884Z","shell.execute_reply":"2023-07-09T21:29:25.675929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DeeplabV3Plus(256,1)\nopt = tf.keras.optimizers.Adam(.001)\nmodel.compile(loss=[jaccard_loss], optimizer=opt, metrics=['accuracy', dice_coefficient, jaccard_coef])\n#unet_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:25.678333Z","iopub.execute_input":"2023-07-09T21:29:25.678791Z","iopub.status.idle":"2023-07-09T21:29:28.807636Z","shell.execute_reply.started":"2023-07-09T21:29:25.678758Z","shell.execute_reply":"2023-07-09T21:29:28.806649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_image=image\nsample_mask=mask\n\ndef create_mask(predictions):\n    return tf.where(predictions >= 0.5, 1, 0)[0]\n\ndef show_predictions(dataset=None, num=1, model=model):\n    if dataset:\n        for image, mask in dataset.take(num):\n            pred_mask = model.predict(image)\n            display([image, mask, create_mask(pred_mask)])\n    else:\n        display([sample_image[0], sample_mask[0], create_mask(model.predict(sample_image))])\n        \nclass DisplayCallback(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        clear_output(wait=True)\n        show_predictions()\n        print ('\\nSample Prediction after epoch {}\\n'.format(epoch+1))","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:28.809046Z","iopub.execute_input":"2023-07-09T21:29:28.809466Z","iopub.status.idle":"2023-07-09T21:29:28.818356Z","shell.execute_reply.started":"2023-07-09T21:29:28.809429Z","shell.execute_reply":"2023-07-09T21:29:28.816386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_predictions()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:28.819757Z","iopub.execute_input":"2023-07-09T21:29:28.820368Z","iopub.status.idle":"2023-07-09T21:29:34.967387Z","shell.execute_reply.started":"2023-07-09T21:29:28.820334Z","shell.execute_reply":"2023-07-09T21:29:34.966431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_ds)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:34.973746Z","iopub.execute_input":"2023-07-09T21:29:34.974378Z","iopub.status.idle":"2023-07-09T21:29:34.980307Z","shell.execute_reply.started":"2023-07-09T21:29:34.974344Z","shell.execute_reply":"2023-07-09T21:29:34.979119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS=50\n\nhistory = model.fit(train_ds,validation_data=val_ds, epochs=EPOCHS, callbacks=[checkpoint_callback]) ","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:29:34.981779Z","iopub.execute_input":"2023-07-09T21:29:34.982979Z","iopub.status.idle":"2023-07-09T21:59:31.848399Z","shell.execute_reply.started":"2023-07-09T21:29:34.982942Z","shell.execute_reply":"2023-07-09T21:59:31.840694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs = range(1, len(loss) + 1)\nplt.plot(epochs, loss, 'y', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.850953Z","iopub.status.idle":"2023-07-09T21:59:31.851713Z","shell.execute_reply.started":"2023-07-09T21:59:31.851326Z","shell.execute_reply":"2023-07-09T21:59:31.851358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nloss = history.history['dice_coefficient']\nval_loss = history.history['val_dice_coefficient']\nepochs = range(1, len(loss) + 1)\nplt.plot(epochs, loss, 'y', label='Training Dice Coefficient')\nplt.plot(epochs, val_loss, 'r', label='Validation Dice Coefficient')\nplt.title('Training and validation Dice Coefficient')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.855184Z","iopub.status.idle":"2023-07-09T21:59:31.860248Z","shell.execute_reply.started":"2023-07-09T21:59:31.85999Z","shell.execute_reply":"2023-07-09T21:59:31.860015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def error_analysis(model, train_ds, num_samples):\n    fig, axs = plt.subplots(num_samples, 3, figsize=(16, 16))\n    if num_samples == 1:\n        axs = axs[np.newaxis, ...]\n        \n    for i, (image, mask) in enumerate(train_ds.shuffle(buffer_size=100).take(num_samples)):\n        prediction = model.predict(image)\n        axs[i, 0].imshow(image[0].numpy())\n        axs[i, 1].imshow(prediction[0])\n        axs[i, 2].imshow(mask[0].numpy())\n        axs[i, 0].set_title(\"Image\")\n        axs[i, 1].set_title(\"Prediction\")\n        axs[i, 2].set_title(\"Ground Truth\")\n    print(prediction.shape)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.861984Z","iopub.status.idle":"2023-07-09T21:59:31.863037Z","shell.execute_reply.started":"2023-07-09T21:59:31.862271Z","shell.execute_reply":"2023-07-09T21:59:31.862298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model seems to be identifying anything as dark blue as being a contrail, there also seems to be a lot of \"mess\" in the predictions where there are random dots and what not maybe there are some post processing steps I can do to clean that up","metadata":{}},{"cell_type":"code","source":"error_analysis(model, train_ds, 5)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.869762Z","iopub.status.idle":"2023-07-09T21:59:31.870416Z","shell.execute_reply.started":"2023-07-09T21:59:31.870153Z","shell.execute_reply":"2023-07-09T21:59:31.870177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_analysis(model, val_ds, 5)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.872356Z","iopub.status.idle":"2023-07-09T21:59:31.87319Z","shell.execute_reply.started":"2023-07-09T21:59:31.872934Z","shell.execute_reply":"2023-07-09T21:59:31.872958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Finding Ideal Threshold","metadata":{}},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"def get_test_image(filename):\n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n\n    band11 = tf.py_function(lambda path: np.load(path.numpy().decode() + '/band_11.npy'), [filename], tf.float32)\n    band14 = tf.py_function(lambda path: np.load(path.numpy().decode() + '/band_14.npy'), [filename], tf.float32)\n    band15 = tf.py_function(lambda path: np.load(path.numpy().decode() + '/band_15.npy'), [filename], tf.float32)\n    \n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = tf.py_function(lambda r, g, b: np.clip(np.stack([r.numpy(), g.numpy(), b.numpy()], axis=2), 0, 1), [r, g, b], tf.float32)\n\n    false_color = false_color[...,4]\n    \n    false_color.set_shape((256, 256, 3)) \n    \n    false_color = tf.expand_dims(false_color, axis=0)\n    return false_color","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.885729Z","iopub.status.idle":"2023-07-09T21:59:31.88623Z","shell.execute_reply.started":"2023-07-09T21:59:31.885988Z","shell.execute_reply":"2023-07-09T21:59:31.886012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_ds = tf.data.Dataset.list_files(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/test/*\")\ntest_dataset = t_ds.map(get_test_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)\npredictions = model.predict(test_dataset)\nbinary_predictions = tf.squeeze(tf.where(predictions >= 0.5, 1, 0), axis =-1)\n#binary_predictions = tf.squeeze(predictions, axis=-1)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.889883Z","iopub.status.idle":"2023-07-09T21:59:31.893902Z","shell.execute_reply.started":"2023-07-09T21:59:31.893664Z","shell.execute_reply":"2023-07-09T21:59:31.893687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(2,2, figsize=(8,8))\nfor i, image in enumerate(test_dataset):\n    axs[i,0].imshow(image[0])\n    axs[i, 0].set_title('image')\n    \n    axs[i,1].imshow(binary_predictions[i])\n    axs[i,1].set_title('label')","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.895248Z","iopub.status.idle":"2023-07-09T21:59:31.895677Z","shell.execute_reply.started":"2023-07-09T21:59:31.895454Z","shell.execute_reply":"2023-07-09T21:59:31.895474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s\n\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n    '''\n    mask_rle: run-length as string formatted (start length)\n              empty predictions need to be encoded with '-'\n    shape: (height, width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    if mask_rle != '-': \n        s = mask_rle.split()\n        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n        for lo, hi in zip(starts, ends):\n            img[lo:hi] = 1\n    return img.reshape(shape, order='F')  # Needed to align to RLE direction\n\ndef create_submission(binary_predictions):\n    submission = pd.read_csv(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv\", index_col='record_id')\n\n    for i, rec in enumerate(test_list):\n        mask = binary_predictions[i].numpy()\n        submission.loc[int(rec), 'encoded_pixels'] = list_to_string(rle_encode(mask))\n    \n    submission.to_csv('submission.csv')\n    return submission","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.896833Z","iopub.status.idle":"2023-07-09T21:59:31.902985Z","shell.execute_reply.started":"2023-07-09T21:59:31.902735Z","shell.execute_reply":"2023-07-09T21:59:31.90276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_submission(binary_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-07-09T21:59:31.904233Z","iopub.status.idle":"2023-07-09T21:59:31.90616Z","shell.execute_reply.started":"2023-07-09T21:59:31.905832Z","shell.execute_reply":"2023-07-09T21:59:31.905864Z"},"trusted":true},"execution_count":null,"outputs":[]}]}