{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":51753,"databundleVersionId":5692552,"sourceType":"competition"}],"dockerImageVersionId":30513,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport os\nfrom tqdm.notebook import tqdm\nfrom pathlib import Path","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/'\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install pillow\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\nimport os\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport os\nfrom PIL import Image\n\ndirectory = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\"\nimport matplotlib.pyplot as plt\n\nimage_count = 0  # Counter for tracking the displayed images\n\n# Adjust the figure size and DPI settings for better clarity\nplt.figure(figsize=(8, 6), dpi=100)\n\n\n\nimage_count = 0  # Counter for tracking the displayed images\n\nfor foldername in os.listdir(directory):\n    folder_path = os.path.join(directory, foldername)\n    if os.path.isdir(folder_path):\n        for filename in os.listdir(folder_path):\n            if filename.endswith(\".npy\"):\n                # Process the .npy file\n                file_path = os.path.join(folder_path, filename)\n                data = np.load(file_path)\n                # Assuming the loaded data represents images\n                images = data  # Retrieve the first four images or adjust the indexing as needed\n                # Display the images\n                for image in images:\n                    plt.imshow(image, aspect='auto')\n                    plt.axis(\"off\")  # Turn off axis labels\n                    plt.show()\n                    image_count += 1\n                    if image_count == 4:  # Displayed four images, exit the loop\n                        break\n                if image_count == 4:\n                    break\n        if image_count == 4:\n            break\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport random\n\n# Step 1: Preprocess the image\ndef preprocess_image(image):\n    # Convert the image data type to uint8 if needed\n    if image.dtype != np.uint8:\n        image = image.astype(np.uint8)\n    \n    # If the image has a single channel, reshape it to have 3 channels\n    if len(image.shape) == 2:\n        image = np.stack((image,) * 3, axis=-1)\n    \n    # Apply any necessary preprocessing steps to the image array\n    # ...\n    \n    return image\n\n# Assuming your image data is stored as a list of NumPy arrays called 'data'\n# Shuffle the data to get a random order of images\nnp.random.shuffle(data)\n\n# Preprocess all the images\npreprocessed_images = []\nfor image_data in data:\n    preprocessed_image = preprocess_image(image_data)\n    preprocessed_images.append(preprocessed_image)\n\n# Select a random image for separate display\nselected_image = random.choice(preprocessed_images)\n\n# Display the selected image separately\nplt.figure(figsize=(8, 6))\nif selected_image.ndim == 2:\n    plt.imshow(selected_image, cmap='gray', aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\nelse:\n    plt.imshow(selected_image, aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\n\nplt.axis('off')\nplt.tight_layout(pad=0)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Apply thresholding and morphological operations to detect contrails\ndef detect_contrails(image):\n    # Convert the image to grayscale\n    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n    \n    # Apply thresholding to obtain a binary image\n    _, binary = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)\n    \n    # Perform morphological operations (e.g., erosion, dilation) to enhance the contrail regions\n    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))\n    opened = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)\n    \n    return opened\n\n# Apply thresholding and morphological operations to each preprocessed image\nprocessed_images = []\nfor image in preprocessed_images:\n    processed_image = detect_contrails(image)\n    processed_images.append(processed_image)\n\n# Select a random image for separate display\nselected_image = random.choice(processed_images)\n\n# Display the selected image separately\nplt.figure(figsize=(8, 6))\nplt.imshow(selected_image, cmap='gray', aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\nplt.axis('off')\nplt.tight_layout(pad=0)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Apply contour detection and feature extraction\ndef detect_contrail_features(image):\n    # Find contours in the image\n    contours, _ = cv2.findContours(image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    \n    # Initialize a list to store the extracted features\n    features = []\n    \n    # Iterate over the contours\n    for contour in contours:\n        # Compute relevant features for each contour\n        perimeter = cv2.arcLength(contour, True)\n        area = cv2.contourArea(contour)\n        # Add the features to the list\n        features.append((perimeter, area))\n    \n    return features\n\n# Apply contour detection and feature extraction to each processed image\nprocessed_features = []\nfor image in processed_images:\n    image_features = detect_contrail_features(image)\n    processed_features.append(image_features)\n\n# Select a random image's features for separate analysis\nselected_features = random.choice(processed_features)\n\n# Print the selected features\nfor feature in selected_features:\n    perimeter, area = feature\n    print(f\"Contrail Perimeter: {perimeter}, Area: {area}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport zlib\n\n# Directory path containing the input images\ndirectory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/'\n\n# Output directory path for images with polygons\noutput_directory = '/kaggle/working/images_with_polygons/'\n\n# Create the output directory if it doesn't exist\nos.makedirs(output_directory, exist_ok=True)\n\n# Output DataFrame\noutput_rows = []\n\n# Apply contrail detection to each processed image\nfor i, image in enumerate(processed_images):\n    # Apply preprocessing on the image (e.g., apply thresholding, noise removal)\n\n    # Apply a threshold to segment the image\n    _, binary_image = cv2.threshold(image, 100, 255, cv2.THRESH_BINARY)\n\n    # Find contours in the binary image\n    contours, _ = cv2.findContours(binary_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n    # Get the record ID\n    record_id = sorted(os.listdir(directory))[i]\n\n    # Create a blank canvas with the original image dimensions\n    canvas = np.zeros_like(image)\n\n    # Iterate over the contours and encode the pixels\n    encoded_pixels = []\n    for contour in contours:\n        if cv2.arcLength(contour, True) > 100:\n            cv2.drawContours(canvas, [contour], -1, (0, 255, 255), 2)\n            mask = np.zeros_like(binary_image)\n            cv2.drawContours(mask, [contour], -1, 255, thickness=cv2.FILLED)\n            masked_pixels = np.where(mask == 255)\n            encoded_pixels.append(\" \".join([f\"{x} {y}\" for x, y in zip(masked_pixels[1], masked_pixels[0])]))\n\n    \n\n    # Add the result to the output DataFrame\n    output_rows.append({'record_id': record_id, 'encoded_pixels': encoded_pixels})\n\n    # Save the image with polygons\n    output_image_path = os.path.join(output_directory, f\"{record_id}_with_polygons.png\")\n    cv2.imwrite(output_image_path, canvas)\n\n# Create the output DataFrame\noutput_df = pd.DataFrame(output_rows)\n\n# Print the output DataFrame\nprint(output_df)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport os\nfrom PIL import Image\n\ndirectory = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation\"\nimport matplotlib.pyplot as plt\n\nimage_count = 0  # Counter for tracking the displayed images\n\n# Adjust the figure size and DPI settings for better clarity\nplt.figure(figsize=(8, 6), dpi=100)\n\n\n\nimage_count = 0  # Counter for tracking the displayed images\n\nfor foldername in os.listdir(directory):\n    folder_path = os.path.join(directory, foldername)\n    if os.path.isdir(folder_path):\n        for filename in os.listdir(folder_path):\n            if filename.endswith(\".npy\"):\n                # Process the .npy file\n                file_path = os.path.join(folder_path, filename)\n                data2 = np.load(file_path)\n                # Assuming the loaded data represents images\n                images = data2 # Retrieve the first four images or adjust the indexing as needed\n                # Display the images\n                for image in images:\n                    plt.imshow(image, aspect='auto')\n                    plt.axis(\"off\")  # Turn off axis labels\n                    plt.show()\n                    image_count += 1\n                    if image_count == 4:  # Displayed four images, exit the loop\n                        break\n                if image_count == 4:\n                    break\n        if image_count == 4:\n            break\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport random\n\n# Step 1: Preprocess the image\ndef preprocess_image(image):\n    # Convert the image data type to uint8 if needed\n    if image.dtype != np.uint8:\n        image = image.astype(np.uint8)\n    \n    # If the image has a single channel, reshape it to have 3 channels\n    if len(image.shape) == 2:\n        image = np.stack((image,) * 3, axis=-1)\n    \n    # Apply any necessary preprocessing steps to the image array\n    # ...\n    \n    return image\n\n# Assuming your image data is stored as a list of NumPy arrays called 'data'\n# Shuffle the data to get a random order of images\nnp.random.shuffle(data2)\n\n# Preprocess all the images\npreprocessed_images_val = []\nfor image_data in data2:\n    preprocessed_image = preprocess_image(image_data)\n    preprocessed_images_val.append(preprocessed_image)\n\n# Select a random image for separate display\nselected_image = random.choice(preprocessed_images_val)\n\n# Display the selected image separately\nplt.figure(figsize=(8, 6))\nif selected_image.ndim == 2:\n    plt.imshow(selected_image, cmap='gray', aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\nelse:\n    plt.imshow(selected_image, aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\n\nplt.axis('off')\nplt.tight_layout(pad=0)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport matplotlib.pyplot as plt\n\n# Apply color normalization to match color distributions\ndef normalize_colors(image):\n    # Convert the image to LAB color space\n    lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)\n    \n    # Split the LAB image into channels\n    l, a, b = cv2.split(lab)\n    \n    # Apply histogram equalization to the L channel\n    l_eq = cv2.equalizeHist(l)\n    \n    # Merge the equalized L channel with the original A and B channels\n    lab_eq = cv2.merge((l_eq, a, b))\n    \n    # Convert the LAB equalized image back to RGB color space\n    normalized_image = cv2.cvtColor(lab_eq, cv2.COLOR_LAB2RGB)\n    \n    return normalized_image\n\n# Apply thresholding and morphological operations to detect contrails\ndef detect_contrails(image):\n    # Normalize colors to match color distributions\n    normalized_image = normalize_colors(image)\n    \n    # Convert the normalized image to grayscale\n    gray = cv2.cvtColor(normalized_image, cv2.COLOR_RGB2GRAY)\n    \n    # Apply thresholding to obtain a binary image\n    _, binary = cv2.threshold(gray, 100, 125, cv2.THRESH_BINARY)\n    \n    # Perform morphological operations (e.g., erosion, dilation) to enhance the contrail regions\n    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))\n    opened = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)\n    \n    return opened\n\n# Apply thresholding and morphological operations to each preprocessed image\nprocessed_images_val = []\nfor image in preprocessed_images_val:\n    processed_image = detect_contrails(image)\n    processed_images_val.append(processed_image)\n\n# Select a random image for separate display\nselected_image = random.choice(processed_images_val)\n\n# Display the selected image separately\nplt.figure(figsize=(8, 6))\nplt.imshow(selected_image, cmap='gray', aspect='auto', extent=(0, selected_image.shape[1], selected_image.shape[0], 0))\nplt.axis('off')\nplt.tight_layout(pad=0)\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport zlib\n\n# Directory path containing the input images\ndirectory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation/'\n\n# Output directory path for images with polygons\noutput_directory = '/kaggle/working/images_with_polygons_val/'\n\n# Create the output directory if it doesn't exist\nos.makedirs(output_directory, exist_ok=True)\n\n# Output DataFrame\noutput_rows = []\n\n# Apply contrail detection to each processed image\nfor i, image in enumerate(processed_images_val):\n    # Apply preprocessing on the image (e.g., apply thresholding, noise removal)\n\n    # Apply a threshold to segment the image\n    _, binary_image = cv2.threshold(image, 100, 255, cv2.THRESH_BINARY)\n\n    # Find contours in the binary image\n    contours, _ = cv2.findContours(binary_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n    # Get the record ID\n    record_id = sorted(os.listdir(directory))[i]\n\n    # Create a blank canvas with the original image dimensions\n    canvas = np.zeros_like(image)\n\n    # Iterate over the contours and encode the pixels\n    encoded_pixels = []\n    for contour in contours:\n        if cv2.arcLength(contour, True) > 100:\n            cv2.drawContours(canvas, [contour], -1, (0, 255, 255), 2)\n            mask = np.zeros_like(binary_image)\n            cv2.drawContours(mask, [contour], -1, 255, thickness=cv2.FILLED)\n            masked_pixels = np.where(mask == 255)\n            encoded_pixels.append(\" \".join([f\"{x} {y}\" for x, y in zip(masked_pixels[1], masked_pixels[0])]))\n\n    \n\n    # Add the result to the output DataFrame\n    output_rows.append({'record_id': record_id, 'encoded_pixels': encoded_pixels})\n\n    # Save the image with polygons\n    output_image_path = os.path.join(output_directory, f\"{record_id}_with_polygons.png\")\n    cv2.imwrite(output_image_path, canvas)\n\n# Create the output DataFrame\noutput_df_val = pd.DataFrame(output_rows)\n\n# Print the output DataFrame\nprint(output_df_val)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Length of processed_images_val:\", len(output_df_val))\nprint(\"Length of output_df:\", len(output_df))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_df = pd.DataFrame(output_df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_df_val = pd.DataFrame(output_df_val)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Load the image from the .npy file\nimage_array = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/human_individual_masks.npy')  # Replace 'path_to_image_file.npy' with your actual .npy file path\n\n# Reshape and normalize the image array\nimage_array = image_array.astype(np.float32)  # Convert to float for normalization\nimage_array /= np.max(image_array)  # Normalize the pixel values between 0 and 1\nimage_array = np.squeeze(image_array)  # Remove any extra dimensions\n\n# Display the image using Matplotlib\nplt.imshow(image_array, cmap='twilight')  # Assuming it's a grayscale image\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Load the image from the .npy file\nimage_array = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/human_pixel_masks.npy')  # Replace 'path_to_image_file.npy' with your actual .npy file path\n\n# Reshape and normalize the image array\nimage_array = image_array.astype(np.float32)  # Convert to float for normalization\nimage_array /= np.max(image_array)  # Normalize the pixel values between 0 and 1\nimage_array = np.squeeze(image_array)  # Remove any extra dimensions\n\n# Display the image using Matplotlib\nplt.imshow(image_array, cmap='twilight')  # Assuming it's a grayscale image\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\n# Load the image from the .npy file\nimage_array = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/band_08.npy')\n\n# Select the first channel\nchannel = image_array[..., 0]\n\n# Rescale the channel values between 0 and 1\nchannel = (channel - np.min(channel)) / (np.max(channel) - np.min(channel))\n\n# Display the channel as grayscale\nplt.imshow(channel, cmap='BuPu')\nplt.axis('off')\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-06-18T21:07:29.783197Z","iopub.execute_input":"2023-06-18T21:07:29.783517Z","iopub.status.idle":"2023-06-18T21:07:31.016784Z","shell.execute_reply.started":"2023-06-18T21:07:29.78349Z","shell.execute_reply":"2023-06-18T21:07:31.015907Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport cv2\nfrom sklearn.ensemble import RandomForestClassifier\n\n# Load training data\ntrain_individual_mask = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_individual_masks.npy')\ntrain_pixel_mask = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_pixel_masks.npy')\n\n# Prepare the training data\ntrain_data = np.concatenate((train_individual_mask, train_pixel_mask), axis=1)\ntrain_labels = np.concatenate((np.ones(len(train_individual_mask)), np.zeros(len(train_pixel_mask))))\n\n# Train the model\nmodel = RandomForestClassifier()\nmodel.fit(train_data, train_labels)\n\n# Apply learning to other images\nother_images = ['/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_08.npy', '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_09.npy']\nfor image_file in other_images:\n    # Load the image\n    image = np.load(image_file)\n\n    # Reshape the image to match the training data format\n    image_data = np.reshape(image, (image.shape[0]*image.shape[1], -1))\n\n    # Predict using the learned model\n    predictions = model.predict(image_data)\n\n    # Reshape the predictions back to the image shape\n    processed_image = np.reshape(predictions, (image.shape[0], image.shape[1]))\n\n    # Save the processed image as PNG\n    output_file = image_file.replace('.npy', '.png')\n    cv2.imwrite(output_file, processed_image)\n\n    print(f\"Processed image saved as {output_file}.\")\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T21:09:06.284772Z","iopub.execute_input":"2023-06-18T21:09:06.285872Z","iopub.status.idle":"2023-06-18T21:09:08.716844Z","shell.execute_reply.started":"2023-06-18T21:09:06.285827Z","shell.execute_reply":"2023-06-18T21:09:08.715719Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom IPython import display\n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '10016536018877742'\n\n# Load band images\nbands = {}\nfor band_num in range(8, 17):\n    filename = f'band_{band_num:02d}.npy'\n    with open(os.path.join(BASE_DIR, record_id, filename), 'rb') as f:\n        bands[band_num] = np.load(f)\n\n# Load ground truth labels\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\n\n# Load individual human masks\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)\n\n# Combine bands into a false color image\ndef normalize_range(data, bounds):\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\nr = normalize_range(bands[15] - bands[14], _TDIFF_BOUNDS)\ng = normalize_range(bands[14] - bands[11], _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(bands[14], _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n\n# Visualize data\nfig, axes = plt.subplots(1, 3, figsize=(18, 6))\nax1, ax2, ax3 = axes\n\nax1.imshow(false_color[..., N_TIMES_BEFORE])\nax1.set_title('False color image')\n\nax2.imshow(human_pixel_mask.squeeze(), interpolation='none')\nax2.set_title('Ground truth contrail mask')\n\nax3.imshow(false_color[..., N_TIMES_BEFORE])\nax3.imshow(human_pixel_mask.squeeze(), cmap='Reds', alpha=0.4, interpolation='none')\nax3.set_title('Contrail mask on false color image')\n\n# Individual human masks\nn = human_individual_mask.shape[-1]\nfig, axes = plt.subplots(1, n, figsize=(16, 4))\nfor i, ax in enumerate(axes):\n    ax.imshow(human_individual_mask[..., i], interpolation='none')\n\n# Animation\nfig = plt.figure(figsize=(6, 6))\nim = plt.imshow(false_color[..., 0])\n\ndef draw_frame(i):\n    im.set_array(false_color[..., i])\n    return [im]\n\nanim = animation.FuncAnimation(fig, draw_frame, frames=false_color.shape[-1], interval=500, blit=True)\nplt.close()\ndisplay.HTML(anim.to_jshtml())","metadata":{"execution":{"iopub.status.busy":"2023-06-18T21:56:21.154458Z","iopub.execute_input":"2023-06-18T21:56:21.155423Z","iopub.status.idle":"2023-06-18T21:56:24.913149Z","shell.execute_reply.started":"2023-06-18T21:56:21.155373Z","shell.execute_reply":"2023-06-18T21:56:24.912336Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom IPython import display\n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '1704010292581573769'\n\n# Load band images\nbands = {}\nfor band_num in range(8, 17):\n    filename = f'band_{band_num:02d}.npy'\n    with open(os.path.join(BASE_DIR, record_id, filename), 'rb') as f:\n        bands[band_num] = np.load(f)\n\n# Load ground truth labels\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\n\n# Load individual human masks\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)\n\n# Combine bands into a false color image\ndef normalize_range(data, bounds):\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\nr = normalize_range(bands[15] - bands[14], _TDIFF_BOUNDS)\ng = normalize_range(bands[14] - bands[11], _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(bands[14], _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n\n# Visualize data\nfig, axes = plt.subplots(1, 3, figsize=(18, 6))\nax1, ax2, ax3 = axes\n\nax1.imshow(false_color[..., N_TIMES_BEFORE])\nax1.set_title('False color image')\n\nax2.imshow(human_pixel_mask.squeeze(), cmap='gray', interpolation='none')\nax2.set_title('Ground truth contrail mask')\n\noverlay = np.zeros_like(false_color)\noverlay[..., 0] = false_color[..., 0]\noverlay[..., 1] = false_color[..., 1]\noverlay[..., 2] = false_color[..., 2]\n\nmask = np.ma.masked_where(human_pixel_mask.squeeze() == 0, human_pixel_mask.squeeze())\nax3.imshow(overlay[..., N_TIMES_BEFORE])\nax3.imshow(mask, cmap='Blues', alpha=0.4, interpolation='none')\nax3.set_title('e')\n\nplt.tight_layout()\nplt.show()\n\n# Individual human masks\nn = human_individual_mask.shape[-1]\nfig, axes = plt.subplots(1, n, figsize=(16, 4))\nfor i, ax in enumerate(axes):\n    ax.imshow(human_individual_mask[..., i], interpolation='none')\n\n# Animation\nfig = plt.figure(figsize=(6, 6))\nim = plt.imshow(false_color[..., 0])\n\ndef draw_frame(i):\n    im.set_array(false_color[..., i])\n    return [im]\n\nanim = animation.FuncAnimation(fig, draw_frame, frames=false_color.shape[-1], interval=500, blit=True)\nplt.close()\ndisplay.HTML(anim.to_jshtml())\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:00:59.970494Z","iopub.execute_input":"2023-06-18T22:00:59.971325Z","iopub.status.idle":"2023-06-18T22:01:03.524249Z","shell.execute_reply.started":"2023-06-18T22:00:59.971292Z","shell.execute_reply":"2023-06-18T22:01:03.523478Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom IPython import display\n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '1704010292581573769'\n\n# Load band images\nbands = {}\nfor band_num in range(8, 17):\n    filename = f'band_{band_num:02d}.npy'\n    with open(os.path.join(BASE_DIR, record_id, filename), 'rb') as f:\n        bands[band_num] = np.load(f)\n\n# Load ground truth labels\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\n\n# Load individual human masks\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)\n\n# Combine bands into a false color image\ndef normalize_range(data, bounds):\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\nr = normalize_range(bands[15] - bands[14], _TDIFF_BOUNDS)\ng = normalize_range(bands[14] - bands[11], _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(bands[14], _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n\n# Visualize data\nfig, axes = plt.subplots(1, 3, figsize=(18, 6))\nax1, ax2, ax3 = axes\n\nax1.imshow(false_color[..., N_TIMES_BEFORE])\nax1.set_title('False color image')\n\nax2.imshow(human_pixel_mask.squeeze(), cmap='gray', interpolation='none')\nax2.set_title('Ground truth contrail mask')\n\noverlay = np.copy(false_color)\noverlay[..., :2] *= 0.5  # Reduce intensity of red and green channels for the overlay\n\nmask = np.ma.masked_where(human_pixel_mask.squeeze() == 0, human_pixel_mask.squeeze())\nax3.imshow(overlay[..., N_TIMES_BEFORE])\nax3.imshow(mask, cmap='Blues', alpha=0.6, vmin=0, vmax=1, interpolation='none')\nax3.set_title('Contrail mask on false color image')\n\nplt.tight_layout()\nplt.show()\n\n# Individual human masks\nn = human_individual_mask.shape[-1]\nfig, axes = plt.subplots(1, n, figsize=(16, 4))\nfor i, ax in enumerate(axes):\n    ax.imshow(human_individual_mask[..., i], cmap='gray', interpolation='none')\n\n# Animation\nfig = plt.figure(figsize=(6, 6))\nim = plt.imshow(false_color[..., 0])\n\ndef draw_frame(i):\n    im.set_array(false_color[..., i])\n    return [im]\n\nanim = animation.FuncAnimation(fig, draw_frame, frames=false_color.shape[-1], interval=500, blit=True)\nplt.close()\ndisplay.HTML(anim.to_jshtml())\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:07:52.370028Z","iopub.execute_input":"2023-06-18T22:07:52.371027Z","iopub.status.idle":"2023-06-18T22:07:55.943191Z","shell.execute_reply.started":"2023-06-18T22:07:52.370979Z","shell.execute_reply":"2023-06-18T22:07:55.94218Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\n\nfrom matplotlib import animation\nfrom IPython import display\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch import Tensor\nfrom torch.utils.data import TensorDataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import DataLoader\n\nfrom torchsummary import summary","metadata":{"execution":{"iopub.status.busy":"2023-06-18T23:00:15.283605Z","iopub.execute_input":"2023-06-18T23:00:15.28445Z","iopub.status.idle":"2023-06-18T23:00:15.290125Z","shell.execute_reply.started":"2023-06-18T23:00:15.284414Z","shell.execute_reply":"2023-06-18T23:00:15.289392Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install /kaggle/input/torchsummary/torchsummary-1.5.1-py3-none-any.whl\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T23:00:08.510456Z","iopub.execute_input":"2023-06-18T23:00:08.510849Z","iopub.status.idle":"2023-06-18T23:00:10.01345Z","shell.execute_reply.started":"2023-06-18T23:00:08.510817Z","shell.execute_reply":"2023-06-18T23:00:10.012278Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir: str = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n","metadata":{"execution":{"iopub.status.busy":"2023-06-18T23:00:32.254311Z","iopub.execute_input":"2023-06-18T23:00:32.255215Z","iopub.status.idle":"2023-06-18T23:00:32.259294Z","shell.execute_reply.started":"2023-06-18T23:00:32.255171Z","shell.execute_reply":"2023-06-18T23:00:32.258389Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_idx = pd.DataFrame({'idx': os.listdir('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train')})\ndf_validation_idx = pd.DataFrame({'idx': os.listdir('/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation')})\ndf_test_idx = pd.DataFrame({'idx': os.listdir('/kaggle/input/google-research-identify-contrails-reduce-global-warming/test')})","metadata":{"execution":{"iopub.status.busy":"2023-06-18T23:03:41.499804Z","iopub.execute_input":"2023-06-18T23:03:41.50027Z","iopub.status.idle":"2023-06-18T23:03:41.517997Z","shell.execute_reply.started":"2023-06-18T23:03:41.500236Z","shell.execute_reply":"2023-06-18T23:03:41.517092Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}