{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# OpenContrails dataset documentation\n \nThe satellite images are originally obtained from the [GOES-16 Advanced Baseline Imager (ABI)](https://www.goes-r.gov/spacesegment/abi.html), which is publicly available on [Google Cloud Storage](https://console.cloud.google.com/storage/browser/gcp-public-data-goes-16/). The original full-disk images were reprojected using bilinear resampling to generate a local scene image. Because contrails are easier to identify with temporal context, a sequence of images at 10-minute intervals are provided. Each example contains exactly one labeled frame.\n\nLearn more about the dataset from our preprint: [OpenContrails: Benchmarking Contrail Detection on GOES-16 ABI](https://arxiv.org/abs/2304.02122).\n\n## Files\nIn each subdirectory named by `{record_id}`, binary files in numpy `.npy` format that corresponds to a single example are provided:\n\n* **`band_{08-16}.npy`**: array with size of `H x W x T`, where `T = n_times_before + n_times_after + 1`, representing the number of images in the sequence. There are `n_times_before` and `n_times_after` images before and after the labeled frame respectively. In our dataset all examples have  `n_times_before=4` and `n_times_after=3`. Each band represents an infrared channel at different wavelengths and is converted to brightness temperatures based on the calibration parameters. The number in the filename corresponds to the GOES-16 ABI band number. Details of the ABI bands can be found [here](https://www.goes-r.gov/mission/ABI-bands-quick-info.html).\n* **`human_individual_masks.npy`**: array with size of `H x W x 1 x R`. Each example is labeled by `R` individual human labelers. `R` is not the same for all samples. The labeled masks have value either 0 or 1 and correspond to the `(n_times_before+1)`-th image in `band_{08-16}.npy`. They are available only in the training set.\n* **`human_pixel_masks.npy`**: array with size of `H x W x 1` containing the  binary groundtruth. A pixel is regarded as contrail pixel in evaluation if it is labeled as contrail by more than half of the labelers. \n\n**`{train/validation}_metadata.json`**: contains the timestamps and the projection parameters to reproduce the satellite images.\n","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom matplotlib import animation\nimport matplotlib.pyplot as plt\nfrom IPython import display","metadata":{"_uuid":"3ddfdbda-6bea-47d1-ba9d-d4acaa379ad1","_cell_guid":"c7665dd4-f4e4-48e1-93c8-8d162f664b2c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T08:44:58.161524Z","iopub.execute_input":"2023-08-21T08:44:58.162769Z","iopub.status.idle":"2023-08-21T08:44:58.210134Z","shell.execute_reply.started":"2023-08-21T08:44:58.162581Z","shell.execute_reply":"2023-08-21T08:44:58.208855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '1704010292581573769'\n\nwith open(os.path.join(BASE_DIR, record_id, 'band_11.npy'), 'rb') as f:\n    band11 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_14.npy'), 'rb') as f:\n    band14 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_15.npy'), 'rb') as f:\n    band15 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-08-21T08:44:58.212255Z","iopub.execute_input":"2023-08-21T08:44:58.213402Z","iopub.status.idle":"2023-08-21T08:44:58.501853Z","shell.execute_reply.started":"2023-08-21T08:44:58.213354Z","shell.execute_reply":"2023-08-21T08:44:58.500578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Load the numpy files\nband_data = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/band_11.npy')\nhuman_individual_masks = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/human_individual_masks.npy')\nhuman_pixel_masks = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/human_pixel_masks.npy')\n\n# Assuming n_times_before and n_times_after are constants\nn_times_before = 4\nn_times_after = 3\n\n# Extract the labeled frame from the band data\nlabeled_frames = band_data[:, :, n_times_before]\n\n# Create a figure and subplots\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\n# Plot the band data\naxes[0].imshow(labeled_frames)\naxes[0].set_title('Band Data')\naxes[0].axis('off')\n\n# Plot the human individual masks\naxes[1].imshow(human_individual_masks[:, :, 0, 0])  # Plotting the first labeler's mask\naxes[1].set_title('Human Individual Masks')\naxes[1].axis('off')\n\n# Plot the human pixel masks\naxes[2].imshow(human_pixel_masks[:, :, 0])\naxes[2].set_title('Human Pixel Masks')\naxes[2].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:32:06.867096Z","iopub.execute_input":"2023-08-23T10:32:06.868205Z","iopub.status.idle":"2023-08-23T10:32:07.508536Z","shell.execute_reply.started":"2023-08-23T10:32:06.868143Z","shell.execute_reply":"2023-08-23T10:32:07.507582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeled_frames.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:34:00.723787Z","iopub.execute_input":"2023-08-23T10:34:00.724215Z","iopub.status.idle":"2023-08-23T10:34:00.731093Z","shell.execute_reply.started":"2023-08-23T10:34:00.724177Z","shell.execute_reply":"2023-08-23T10:34:00.729974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\n\n\n# Directory containing the band 8 data files\ndata_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769'\n\n# Load the band 8 data\nband_8_data = np.load(os.path.join(data_directory, 'band_16.npy'))\nband_8_data = (band_8_data - np.min(band_8_data)) / (np.max(band_8_data) - np.min(band_8_data))\n\n# Get the number of images in the sequence (T)\nn_times_before = 4\nn_times_after = 3\nT = n_times_before + 1 + n_times_after\n\n# Create a figure and axis\nfig, ax = plt.subplots()\n\n# Define the function to update the plot for each frame\ndef update(frame):\n    ax.clear()\n    ax.imshow(band_8_data[:, :, frame], cmap='binary_r')\n    ax.set_title(f'Frame {frame + 1}/{T}')\n    ax.axis('off')\n\n# Create the animation\nanimation = FuncAnimation(fig, update, frames=T, interval=500)\n\n# Display the animation\ndisplay.HTML(animation.to_jshtml())\n","metadata":{"execution":{"iopub.status.busy":"2023-08-21T09:13:33.883193Z","iopub.execute_input":"2023-08-21T09:13:33.883641Z","iopub.status.idle":"2023-08-21T09:13:35.359449Z","shell.execute_reply.started":"2023-08-21T09:13:33.883581Z","shell.execute_reply":"2023-08-21T09:13:35.358055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Combine bands into a false color image\nIn order to view contrails in GOES, we use the \"ash\" color scheme. This color scheme was originally developed for viewing volcanic ash in the atmosphere but is also useful for viewing thin cirrus, including contrails. In this color scheme, contrails appear in the image as dark blue.\n\nNote that we use a modified version of the ash color scheme here, developed by Kulik et al., which uses slightly different bands and bounds tuned for contrails.\n\nReferences:\n - Ash Color Scheme (page 7): https://eumetrain.org/sites/default/files/2020-05/RGB_recipes.pdf","metadata":{}},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nr = normalize_range(band15 - band14, _TDIFF_BOUNDS)\ng = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(band14, _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:54.46358Z","iopub.execute_input":"2023-08-20T13:57:54.464662Z","iopub.status.idle":"2023-08-20T13:57:54.492299Z","shell.execute_reply.started":"2023-08-20T13:57:54.464601Z","shell.execute_reply":"2023-08-20T13:57:54.491031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize data","metadata":{}},{"cell_type":"code","source":"img = false_color[..., N_TIMES_BEFORE]\n\nplt.figure(figsize=(18, 6))\nax = plt.subplot(1, 3, 1)\nax.imshow(img)\nax.set_title('False color image')\n\nax = plt.subplot(1, 3, 2)\nax.imshow(human_pixel_mask, interpolation='none')\nax.set_title('Ground truth contrail mask')\n\nax = plt.subplot(1, 3, 3)\nax.imshow(img)\nax.imshow(human_pixel_mask, cmap='Reds', alpha=.4, interpolation='none')\nax.set_title('Contrail mask on false color image');","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:54.493747Z","iopub.execute_input":"2023-08-20T13:57:54.496289Z","iopub.status.idle":"2023-08-20T13:57:55.452538Z","shell.execute_reply.started":"2023-08-20T13:57:54.496236Z","shell.execute_reply":"2023-08-20T13:57:55.451376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Individual human masks\nn = human_individual_mask.shape[-1]\nplt.figure(figsize=(16, 4))\nfor i in range(n):\n    plt.subplot(1, n, i+1)\n    plt.imshow(human_individual_mask[..., i], interpolation='none')","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:55.455172Z","iopub.execute_input":"2023-08-20T13:57:55.455888Z","iopub.status.idle":"2023-08-20T13:57:56.207736Z","shell.execute_reply.started":"2023-08-20T13:57:55.455844Z","shell.execute_reply":"2023-08-20T13:57:56.206676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Animation\nfig = plt.figure(figsize=(6, 6))\nim = plt.imshow(false_color[..., 0])\ndef draw(i):\n    im.set_array(false_color[..., i])\n    return [im]\nanim = animation.FuncAnimation(\n    fig, draw, frames=false_color.shape[-1], interval=500, blit=True\n)\nplt.close()\ndisplay.HTML(anim.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:56.209395Z","iopub.execute_input":"2023-08-20T13:57:56.209798Z","iopub.status.idle":"2023-08-20T13:57:58.652131Z","shell.execute_reply.started":"2023-08-20T13:57:56.209756Z","shell.execute_reply":"2023-08-20T13:57:58.650472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \n\ndf = pd.read_csv('/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-22T14:34:43.605884Z","iopub.execute_input":"2023-08-22T14:34:43.606328Z","iopub.status.idle":"2023-08-22T14:34:43.626303Z","shell.execute_reply.started":"2023-08-22T14:34:43.606285Z","shell.execute_reply":"2023-08-22T14:34:43.625041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-08-22T14:34:49.519254Z","iopub.execute_input":"2023-08-22T14:34:49.519613Z","iopub.status.idle":"2023-08-22T14:34:49.546951Z","shell.execute_reply.started":"2023-08-22T14:34:49.519585Z","shell.execute_reply":"2023-08-22T14:34:49.545913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\n\n# Directory containing the training data\ndata_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\n\n# Sample size\nsample_size = 500\n\n# List to store data and masks\ndata_list = []\npixel_mask_list = []\n\n# Loop through directories and load data\nfor dir_name in os.listdir(data_directory):\n    dir_path = os.path.join(data_directory, dir_name)\n    if os.path.isdir(dir_path):\n        data_path = os.path.join(dir_path, 'band_08.npy')\n        pixel_mask_path = os.path.join(dir_path, 'human_pixel_masks.npy')\n        \n        if os.path.exists(data_path) and os.path.exists(pixel_mask_path):\n            data = np.load(data_path)\n            pixel_mask = np.load(pixel_mask_path)\n            \n            # Append data and masks to the lists\n            data_list.append(data)\n            pixel_mask_list.append(pixel_mask)\n            \n            # Stop when sample size is reached\n            if len(data_list) >= sample_size:\n                break\n\n# Convert lists to numpy arrays\ncontrail_time_series = np.array(data_list)\nhuman_pixel_masks = np.array(pixel_mask_list)\n\n# Normalize contrail time series data to [0, 1]\nnormalized_contrail_data = (contrail_time_series - np.min(contrail_time_series)) / (np.max(contrail_time_series) - np.min(contrail_time_series))\n\n# Split data into training and validation sets\ntrain_data, val_data, train_masks, val_masks = train_test_split(normalized_contrail_data, human_pixel_masks, test_size=0.2, random_state=42)\n\n# One-hot encode the binary masks (0 and 1) for categorical cross-entropy loss\ntrain_masks_categorical = to_categorical(train_masks, num_classes=2)\nval_masks_categorical = to_categorical(val_masks, num_classes=2)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:42:02.201902Z","iopub.execute_input":"2023-08-23T09:42:02.202651Z","iopub.status.idle":"2023-08-23T09:42:33.859145Z","shell.execute_reply.started":"2023-08-23T09:42:02.202611Z","shell.execute_reply":"2023-08-23T09:42:33.857904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Dropout, UpSampling2D, concatenate\n\n# Define the U-Net model architecture\ndef create_unet(input_shape, num_classes):\n    inputs = Input(shape=input_shape)\n    \n    # Contracting Path (Downsampling)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(inputs)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n    \n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(pool1)\n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(conv2)\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n    \n    # Bottleneck\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(pool2)\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(conv3)\n    \n    # Expansive Path (Upsampling)\n    up4 = UpSampling2D(size=(2, 2))(conv3)\n    up4 = Conv2D(128, 2, activation='relu', padding='same')(up4)\n    merge4 = concatenate([conv2, up4], axis=3)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(merge4)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(conv4)\n    \n    up5 = UpSampling2D(size=(2, 2))(conv4)\n    up5 = Conv2D(64, 2, activation='relu', padding='same')(up5)\n    merge5 = concatenate([conv1, up5], axis=3)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(merge5)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(conv5)\n    \n    # Output layer\n    outputs = Conv2D(2, 1, activation='softmax')(conv5)\n    \n    model = Model(inputs=inputs, outputs=outputs)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:38:57.189939Z","iopub.execute_input":"2023-08-23T09:38:57.191006Z","iopub.status.idle":"2023-08-23T09:38:57.207455Z","shell.execute_reply.started":"2023-08-23T09:38:57.190964Z","shell.execute_reply":"2023-08-23T09:38:57.206352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the paths to the numpy files\ndata_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077'\nband_08_path = os.path.join(data_directory, 'band_08.npy')\nindividual_masks_path = os.path.join(data_directory, 'human_individual_masks.npy')\npixel_masks_path = os.path.join(data_directory, 'human_pixel_masks.npy')\n\n# Load the numpy arrays\nband_08_data = np.load(band_08_path)\nindividual_masks_data = np.load(individual_masks_path)\npixel_masks_data = np.load(pixel_masks_path)\n\n# Get the shapes of the loaded numpy arrays\ninput_shape = band_08_data.shape  # Add channel dimension\nnum_classes = 2  # Number of classes (contrail and non-contrail)\n\n# Create the U-Net model\nmodel = create_unet(input_shape, num_classes)\n\n# Compile the model\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Print the model summary\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:53:34.722248Z","iopub.execute_input":"2023-08-23T09:53:34.723411Z","iopub.status.idle":"2023-08-23T09:53:34.951843Z","shell.execute_reply.started":"2023-08-23T09:53:34.723363Z","shell.execute_reply":"2023-08-23T09:53:34.950981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nbatch_size = 16\nepochs = 20\n\n# Train the model\nhistory = model.fit(\n    train_data, train_masks_categorical,\n    batch_size=batch_size,\n    epochs=epochs,\n    verbose=1,\n    validation_data=(val_data, val_masks_categorical)\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:53:41.25822Z","iopub.execute_input":"2023-08-23T09:53:41.259294Z","iopub.status.idle":"2023-08-23T09:58:07.081791Z","shell.execute_reply.started":"2023-08-23T09:53:41.259251Z","shell.execute_reply":"2023-08-23T09:58:07.080519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/test'\n\n# List to store test data\ntest_data_list = []\n\n# Loop through test directories and load data\nfor dir_name in os.listdir(test_directory):\n    dir_path = os.path.join(test_directory, dir_name)\n    if os.path.isdir(dir_path):\n        test_band_08_path = os.path.join(dir_path, 'band_08.npy')\n        \n        if os.path.exists(test_band_08_path):\n            test_band_08_data = np.load(test_band_08_path)\n            test_data_list.append(test_band_08_data)\n\n# Convert list to numpy array\ntest_contrail_time_series = np.array(test_data_list)\n\n# Normalize test data\nnormalized_test_contrail_data = (test_contrail_time_series - np.min(test_contrail_time_series)) / (np.max(test_contrail_time_series) - np.min(test_contrail_time_series))\n\n# Evaluate the model on the test data\ntest_predictions = model.predict(normalized_test_contrail_data)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:27:48.778468Z","iopub.execute_input":"2023-08-23T10:27:48.778908Z","iopub.status.idle":"2023-08-23T10:27:49.850276Z","shell.execute_reply.started":"2023-08-23T10:27:48.77887Z","shell.execute_reply":"2023-08-23T10:27:49.849127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:34:30.693411Z","iopub.execute_input":"2023-08-23T10:34:30.694076Z","iopub.status.idle":"2023-08-23T10:34:30.701864Z","shell.execute_reply.started":"2023-08-23T10:34:30.69402Z","shell.execute_reply":"2023-08-23T10:34:30.700562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Prediction\",test_predictions[0].shape)\nprint(\"Band\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_08.npy\").shape)\nprint(\"Human Individual\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_individual_masks.npy\").shape)\nprint(\"Human pixel\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_pixel_masks.npy\").shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:36:50.952813Z","iopub.execute_input":"2023-08-23T10:36:50.953566Z","iopub.status.idle":"2023-08-23T10:36:50.987886Z","shell.execute_reply.started":"2023-08-23T10:36:50.953524Z","shell.execute_reply":"2023-08-23T10:36:50.986658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:32:57.165941Z","iopub.execute_input":"2023-08-23T10:32:57.166735Z","iopub.status.idle":"2023-08-23T10:32:57.173942Z","shell.execute_reply.started":"2023-08-23T10:32:57.166695Z","shell.execute_reply":"2023-08-23T10:32:57.172844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the threshold value\nthreshold = 0.5  # You can adjust this threshold value based on your needs\n\n# Apply thresholding to generate binary predictions\nbinary_test_predictions = (test_predictions > threshold).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:38:44.330072Z","iopub.execute_input":"2023-08-23T10:38:44.331205Z","iopub.status.idle":"2023-08-23T10:38:44.337725Z","shell.execute_reply.started":"2023-08-23T10:38:44.331143Z","shell.execute_reply":"2023-08-23T10:38:44.336431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"binary_test_predictions[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:39:46.755665Z","iopub.execute_input":"2023-08-23T10:39:46.756696Z","iopub.status.idle":"2023-08-23T10:39:46.763878Z","shell.execute_reply.started":"2023-08-23T10:39:46.756653Z","shell.execute_reply":"2023-08-23T10:39:46.762768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(15, 5))\naxes.imshow(binary_test_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:39:17.071829Z","iopub.execute_input":"2023-08-23T10:39:17.072312Z","iopub.status.idle":"2023-08-23T10:39:17.381927Z","shell.execute_reply.started":"2023-08-23T10:39:17.072267Z","shell.execute_reply":"2023-08-23T10:39:17.379574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport random\nfrom tqdm.auto import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:55:15.17738Z","iopub.execute_input":"2023-08-24T15:55:15.178012Z","iopub.status.idle":"2023-08-24T15:55:24.058784Z","shell.execute_reply.started":"2023-08-24T15:55:15.177985Z","shell.execute_reply":"2023-08-24T15:55:24.057795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Directories**","metadata":{}},{"cell_type":"code","source":"train_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nvalidation_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation'","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:55:30.269248Z","iopub.execute_input":"2023-08-24T15:55:30.270262Z","iopub.status.idle":"2023-08-24T15:55:30.274885Z","shell.execute_reply.started":"2023-08-24T15:55:30.270226Z","shell.execute_reply":"2023-08-24T15:55:30.273923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_and_preprocess_data(data_directory, num_samples=None):\n    band_data_list = []\n    mask_data_list = []\n    \n    # Get a list of all record IDs in the directory\n    record_ids = os.listdir(data_directory)\n    \n    # Randomly select num_samples records (if provided)\n    if num_samples is not None:\n        selected_record_ids = np.random.choice(record_ids, size=num_samples, replace=False)\n    else:\n        selected_record_ids = record_ids\n    \n    # Loop through selected records\n    for record_id in tqdm(selected_record_ids):\n        record_directory = os.path.join(data_directory, record_id)\n        \n        # Load contrail image data\n        band_08_path = os.path.join(record_directory, 'band_08.npy')\n        band_08_data = np.load(band_08_path)\n        # Normalize the data\n        normalized_band_08_data = (band_08_data - np.min(band_08_data)) / (np.max(band_08_data) - np.min(band_08_data))\n        \n        # Load human pixel mask\n        human_pixel_mask_path = os.path.join(record_directory, 'human_pixel_masks.npy')\n        human_pixel_mask_data = np.load(human_pixel_mask_path)\n        \n        # Append data to lists\n        band_data_list.append(normalized_band_08_data)\n        mask_data_list.append(human_pixel_mask_data)\n    \n    # Convert lists to numpy arrays\n    band_data = np.array(band_data_list)\n    mask_data = np.array(mask_data_list)\n    \n    return band_data, mask_data\n\n# Load and preprocess 500 training samples and 45 validation samples\ntrain_band_data, train_mask_data = load_and_preprocess_data(train_directory, num_samples=500)\nvalidation_band_data, validation_mask_data = load_and_preprocess_data(validation_directory, num_samples=45)\n\n\n# Print the shapes of the loaded data\nprint(\"Training Band Data Shape:\", train_band_data.shape)\nprint(\"Training Mask Data Shape:\", train_mask_data.shape)\nprint(\"Validation Band Data Shape:\", validation_band_data.shape)\nprint(\"Validation Mask Data Shape:\", validation_mask_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:55:56.272645Z","iopub.execute_input":"2023-08-24T15:55:56.273634Z","iopub.status.idle":"2023-08-24T15:56:26.610399Z","shell.execute_reply.started":"2023-08-24T15:55:56.273595Z","shell.execute_reply":"2023-08-24T15:56:26.609431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_unet(input_shape, num_classes):\n    inputs = Input(input_shape)\n    \n    # Encoder\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(inputs)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n    \n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(pool1)\n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(conv2)\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n    \n    # Middle\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(pool2)\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(conv3)\n    \n    # Decoder\n    up4 = UpSampling2D(size=(2, 2))(conv3)\n    concat4 = concatenate([conv2, up4], axis=-1)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(concat4)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(conv4)\n    \n    up5 = UpSampling2D(size=(2, 2))(conv4)\n    concat5 = concatenate([conv1, up5], axis=-1)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(concat5)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(conv5)\n    \n    # Output layer\n    outputs = Conv2D(num_classes, 1, activation='softmax')(conv5)\n    \n    model = Model(inputs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:56:30.289323Z","iopub.execute_input":"2023-08-24T15:56:30.29003Z","iopub.status.idle":"2023-08-24T15:56:30.300892Z","shell.execute_reply.started":"2023-08-24T15:56:30.289996Z","shell.execute_reply":"2023-08-24T15:56:30.299475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (256, 256, 8)  # Adjust based on your data\nnum_classes = 2  # Binary classification (contrail vs. non-contrail)\n\n# Create the U-Net model\nmodel = create_unet(input_shape, num_classes)\n\n# Compile the model\nmodel.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:56:37.321447Z","iopub.execute_input":"2023-08-24T15:56:37.321799Z","iopub.status.idle":"2023-08-24T15:56:40.465801Z","shell.execute_reply.started":"2023-08-24T15:56:37.321771Z","shell.execute_reply":"2023-08-24T15:56:40.464845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:56:42.897907Z","iopub.execute_input":"2023-08-24T15:56:42.898283Z","iopub.status.idle":"2023-08-24T15:56:42.950119Z","shell.execute_reply.started":"2023-08-24T15:56:42.89825Z","shell.execute_reply":"2023-08-24T15:56:42.949393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the number of epochs and batch size\nepochs = 100\nbatch_size = 32\n\n# Train the model\nhistory = model.fit(train_band_data, train_mask_data, \n                    batch_size=batch_size, \n                    epochs=epochs, \n                    verbose=1, \n                    validation_data=(validation_band_data, validation_mask_data))","metadata":{"execution":{"iopub.status.busy":"2023-08-24T15:56:46.800234Z","iopub.execute_input":"2023-08-24T15:56:46.800695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}