{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom pathlib import Path\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:01.003624Z","iopub.execute_input":"2023-07-13T13:57:01.004059Z","iopub.status.idle":"2023-07-13T13:57:01.009794Z","shell.execute_reply.started":"2023-07-13T13:57:01.004025Z","shell.execute_reply":"2023-07-13T13:57:01.008721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Images collection\n* Create list of the file paths \n* Load all bands for N_TIME image for required index\n* Output 9 bands 1 n_time_image with 256x256 width and height ","metadata":{}},{"cell_type":"code","source":"tarin_path = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\"\ntrain_folder = Path(tarin_path)\ntrain_img_path = [x for x in train_folder.iterdir()] # store the list of image folder","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:15.317347Z","iopub.execute_input":"2023-07-13T13:57:15.317722Z","iopub.status.idle":"2023-07-13T13:57:15.369904Z","shell.execute_reply.started":"2023-07-13T13:57:15.317692Z","shell.execute_reply":"2023-07-13T13:57:15.368957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read images file for all bands_8-16\ndef get_image(filepath, n_image, index):\n    \"\"\"\n    \n    \"\"\"\n    idx_image = []\n    folder = Path(filepath)\n    paths = [x for x in folder.iterdir()]\n    bands = range(8, 17)  # Range from band_08 to band_16\n    for band in bands:\n        band_path = paths[index] / f\"band_{band:02d}.npy\"\n        human_pixel_path = paths[index] / f\"human_pixel_masks.npy\"\n        human_indv_path = paths[index] / f\"human_individual_masks.npy\"\n        human_pixel_mask = np.load(human_pixel_path) # load pixel mask \n        human_indv_mask = np.load(human_indv_path) # load human individual mask\n        band_data = np.load(band_path) # load bands \n        idx_image.append(band_data) # append all bands\n    idx_image = np.array(idx_image)\n    return idx_image, human_pixel_mask, human_indv_mask","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:16.155017Z","iopub.execute_input":"2023-07-13T13:57:16.155389Z","iopub.status.idle":"2023-07-13T13:57:16.163329Z","shell.execute_reply.started":"2023-07-13T13:57:16.155359Z","shell.execute_reply":"2023-07-13T13:57:16.162129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_FILE_PATH = tarin_path\nN_TIMES = 4\nINDEX_NO = 3\n# get all bands and mask for selected index number and N_times image\nimage_train,human_pixel_mask, human_indv_mask = get_image(IMG_FILE_PATH, N_TIMES, INDEX_NO)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:17.117233Z","iopub.execute_input":"2023-07-13T13:57:17.117627Z","iopub.status.idle":"2023-07-13T13:57:17.793424Z","shell.execute_reply.started":"2023-07-13T13:57:17.117598Z","shell.execute_reply":"2023-07-13T13:57:17.792473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:18.468004Z","iopub.execute_input":"2023-07-13T13:57:18.468415Z","iopub.status.idle":"2023-07-13T13:57:18.474722Z","shell.execute_reply.started":"2023-07-13T13:57:18.468381Z","shell.execute_reply":"2023-07-13T13:57:18.47382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"human_pixel_mask.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:19.117258Z","iopub.execute_input":"2023-07-13T13:57:19.117942Z","iopub.status.idle":"2023-07-13T13:57:19.124219Z","shell.execute_reply.started":"2023-07-13T13:57:19.117907Z","shell.execute_reply":"2023-07-13T13:57:19.123336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(image_train[..., 4][8])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:19.468485Z","iopub.execute_input":"2023-07-13T13:57:19.469262Z","iopub.status.idle":"2023-07-13T13:57:19.792256Z","shell.execute_reply.started":"2023-07-13T13:57:19.469222Z","shell.execute_reply":"2023-07-13T13:57:19.791416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize Bands","metadata":{}},{"cell_type":"code","source":"# Plot all bands for n_time(5th) image\ndef plot_all_bands(image):\n    img_nt = image[..., 4]\n\n    # Create subplots for each band\n    fig, axs = plt.subplots(3, 3, figsize=(10, 10))\n    fig.tight_layout()\n\n    bands = range(9)  # Assuming there are 9 bands\n\n    for i, ax in enumerate(axs.flat):\n        band = bands[i]\n        band_data = img_nt[band, :, :]\n        ax.imshow(band_data)\n        ax.set_title(f\"Band {band+8}\")\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:20.736021Z","iopub.execute_input":"2023-07-13T13:57:20.736841Z","iopub.status.idle":"2023-07-13T13:57:20.744822Z","shell.execute_reply.started":"2023-07-13T13:57:20.736796Z","shell.execute_reply":"2023-07-13T13:57:20.743746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_all_bands(image_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:21.747071Z","iopub.execute_input":"2023-07-13T13:57:21.747436Z","iopub.status.idle":"2023-07-13T13:57:23.741734Z","shell.execute_reply.started":"2023-07-13T13:57:21.747405Z","shell.execute_reply":"2023-07-13T13:57:23.74087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Histogram of bands","metadata":{}},{"cell_type":"code","source":"def plot_bands_histograms(image):\n    img_nt = image[..., 4]\n\n    # Create subplots for each band\n    fig, axs = plt.subplots(3, 3, figsize=(10, 10))\n    fig.tight_layout()\n\n    bands = range(9)  # Assuming there are 9 bands\n\n    for i, ax in enumerate(axs.flat):\n        band = bands[i]\n        band_data = img_nt[band, :, :]\n        ax.hist(band_data.ravel(), bins=256, color='blue', alpha=0.6)\n        ax.set_title(f\"Band-{band+8}\")\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:23.743516Z","iopub.execute_input":"2023-07-13T13:57:23.744055Z","iopub.status.idle":"2023-07-13T13:57:23.75352Z","shell.execute_reply.started":"2023-07-13T13:57:23.744023Z","shell.execute_reply":"2023-07-13T13:57:23.752425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_bands_histograms(image_train)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:25.634524Z","iopub.execute_input":"2023-07-13T13:57:25.634891Z","iopub.status.idle":"2023-07-13T13:57:30.580884Z","shell.execute_reply.started":"2023-07-13T13:57:25.634862Z","shell.execute_reply":"2023-07-13T13:57:30.579784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Flase color color composition & Mask","metadata":{}},{"cell_type":"code","source":"# Get rgb band combination\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef get_rgb(img):\n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n    #band_img = image_train[..., 4]\n    band15 = img[15-8]\n    band14 = img[14-8]\n    band11 = img[11-8]\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    return false_color\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:30.583156Z","iopub.execute_input":"2023-07-13T13:57:30.583519Z","iopub.status.idle":"2023-07-13T13:57:30.592554Z","shell.execute_reply.started":"2023-07-13T13:57:30.583487Z","shell.execute_reply":"2023-07-13T13:57:30.590516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot all mask and false color\ndef plot_falsecolor_mask(false_color, human_pixel_mask):\n    plt.figure(figsize=(18, 6))\n    ax = plt.subplot(1, 3, 1)\n    ax.imshow(false_color)\n    ax.set_title('False color image')\n\n    ax = plt.subplot(1, 3, 2)\n    ax.imshow(human_pixel_mask, interpolation='none')\n    ax.set_title('Ground truth contrail mask')\n\n    ax = plt.subplot(1, 3, 3)\n    ax.imshow(false_color)\n    ax.imshow(human_pixel_mask, cmap='Reds', alpha=.4, interpolation='none')\n    ax.set_title('Contrail mask on false color image');","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:30.593753Z","iopub.execute_input":"2023-07-13T13:57:30.59472Z","iopub.status.idle":"2023-07-13T13:57:30.604934Z","shell.execute_reply.started":"2023-07-13T13:57:30.594689Z","shell.execute_reply":"2023-07-13T13:57:30.603958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"false_color = get_rgb(image_train[..., 4])\nplot_falsecolor_mask(false_color, human_pixel_mask)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:30.608019Z","iopub.execute_input":"2023-07-13T13:57:30.610193Z","iopub.status.idle":"2023-07-13T13:57:31.598275Z","shell.execute_reply.started":"2023-07-13T13:57:30.610167Z","shell.execute_reply":"2023-07-13T13:57:31.597155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Prepration\n* Create 3 bands combination image \n* Get human mask for images \n* Split train/test data","metadata":{}},{"cell_type":"code","source":"# Prepare data for model \ndef load_model_image(img_path):\n    image = []\n    # Load and preprocess the image and mask\n    bands = range(8, 17)  # Range from band_08 to band_16\n    for band in bands:\n        band_path = img_path / f\"band_{band:02d}.npy\"\n        img = np.load(band_path)\n        image.append(img)\n    image = np.array(image)\n    fc_image = get_rgb(image[...,4])\n    #image = np.transpose(image[...,4], (1,2,0))\n    mask_path = img_path / \"human_pixel_masks.npy\"\n    mask = np.load(mask_path)\n    \n    # Preprocessing steps on the image and mask\n    # Example:\n    #fc_image = fc_image / 255.0  # Normalize the pixel values to [0, 1]\n    \n    return fc_image, mask","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:31.599972Z","iopub.execute_input":"2023-07-13T13:57:31.600404Z","iopub.status.idle":"2023-07-13T13:57:31.608557Z","shell.execute_reply.started":"2023-07-13T13:57:31.600371Z","shell.execute_reply":"2023-07-13T13:57:31.607562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:31.610294Z","iopub.execute_input":"2023-07-13T13:57:31.611275Z","iopub.status.idle":"2023-07-13T13:57:31.624237Z","shell.execute_reply.started":"2023-07-13T13:57:31.611243Z","shell.execute_reply":"2023-07-13T13:57:31.623177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_paths = np.random.choice(train_img_path, 500, replace=False)\n\n# Split the data into training and test sets\ntrain_img_paths, test_img_paths = train_test_split(sample_paths, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:32.918265Z","iopub.execute_input":"2023-07-13T13:57:32.918654Z","iopub.status.idle":"2023-07-13T13:57:32.966618Z","shell.execute_reply.started":"2023-07-13T13:57:32.91862Z","shell.execute_reply":"2023-07-13T13:57:32.96555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Test traing set image \nt_img, t_mask = load_model_image(train_img_paths[150])\nt_img.shape, t_mask.shape\nplot_falsecolor_mask(t_img, t_mask)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:33.687812Z","iopub.execute_input":"2023-07-13T13:57:33.688181Z","iopub.status.idle":"2023-07-13T13:57:34.919173Z","shell.execute_reply.started":"2023-07-13T13:57:33.688152Z","shell.execute_reply":"2023-07-13T13:57:34.918136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Building","metadata":{}},{"cell_type":"code","source":"%%time\n# Load and preprocess the training images and masks\ntrain_images = []\ntrain_masks = []\nfor img_path in train_img_paths:\n    img, mask = load_model_image(img_path)\n    train_images.append(img)\n    train_masks.append(mask)\n    \n    \ntrain_images = tf.convert_to_tensor(train_images)\ntrain_masks = tf.convert_to_tensor(train_masks)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T13:57:42.667409Z","iopub.execute_input":"2023-07-13T13:57:42.668122Z","iopub.status.idle":"2023-07-13T14:00:45.651591Z","shell.execute_reply.started":"2023-07-13T13:57:42.668086Z","shell.execute_reply":"2023-07-13T14:00:45.65054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Load and preprocess the test images and masks\ntest_images = []\ntest_masks = []\nfor img_path in test_img_paths:\n    img, mask = load_model_image(img_path)\n    test_images.append(img)\n    test_masks.append(mask)\n\ntest_images = tf.convert_to_tensor(test_images)\ntest_masks = tf.convert_to_tensor(test_masks)\n\n# Print the shape of the training and test datasets\nprint(\"Train Images Shape:\", train_images.shape)\nprint(\"Train Masks Shape:\", train_masks.shape)\nprint(\"Test Images Shape:\", test_images.shape)\nprint(\"Test Masks Shape:\", test_masks.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:00:45.655094Z","iopub.execute_input":"2023-07-13T14:00:45.655381Z","iopub.status.idle":"2023-07-13T14:01:32.700703Z","shell.execute_reply.started":"2023-07-13T14:00:45.655356Z","shell.execute_reply":"2023-07-13T14:01:32.699805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport tensorflow.keras.backend as K\n\n# Define the Dice Loss function\ndef dice_loss(y_true, y_pred):\n    smooth = 1e-5\n    y_true = tf.cast(y_true, tf.float32)  # Convert to float32\n    y_pred = tf.cast(y_pred, tf.float32)  # Convert to float32\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    dice_coeff = (2.0 * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n    return 1.0 - dice_coeff\n\n# Define the IoU (Intersection over Union) metric\ndef iou_metric(y_true, y_pred):\n    y_true = tf.cast(y_true, tf.float32)  # Convert to float32\n    y_pred = tf.cast(y_pred, tf.float32)  # Convert to float32\n    y_true_f = K.flatten(y_true)\n    y_pred_f = K.flatten(y_pred)\n    intersection = K.sum(y_true_f * y_pred_f)\n    union = K.sum(K.maximum(y_true_f, y_pred_f))\n    iou = intersection / union\n    return iou\n\n# Define the model architecture\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(256, 256, 3)),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D((2, 2)),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(64, activation='relu'),\n    tf.keras.layers.Dense(256 * 256 * 1, activation='sigmoid'),\n    tf.keras.layers.Reshape((256, 256, 1))\n])\n\n# Compile the model with Adam optimizer, Dice Loss, and IoU metric\nmodel.compile(optimizer=tf.keras.optimizers.Adam(),\n              loss=dice_loss,\n              metrics=[iou_metric])\n\n# Train the model\nmodel.fit(train_images, train_masks, epochs=10, validation_data=(test_images, test_masks))\n\n# Evaluate the model\ntest_loss, test_iou = model.evaluate(test_images, test_masks)\nprint(\"Test Loss:\", test_loss)\nprint(\"Test IoU:\", test_iou)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:01:32.704588Z","iopub.execute_input":"2023-07-13T14:01:32.706706Z","iopub.status.idle":"2023-07-13T14:01:41.552673Z","shell.execute_reply.started":"2023-07-13T14:01:32.706672Z","shell.execute_reply":"2023-07-13T14:01:41.551607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:09:11.23826Z","iopub.execute_input":"2023-07-13T14:09:11.238656Z","iopub.status.idle":"2023-07-13T14:09:11.264131Z","shell.execute_reply.started":"2023-07-13T14:09:11.238622Z","shell.execute_reply":"2023-07-13T14:09:11.263404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions\npredictions = model.predict(test_images)","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:01:41.559425Z","iopub.execute_input":"2023-07-13T14:01:41.559749Z","iopub.status.idle":"2023-07-13T14:01:41.745156Z","shell.execute_reply.started":"2023-07-13T14:01:41.559722Z","shell.execute_reply":"2023-07-13T14:01:41.744177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(predictions[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:01:41.746687Z","iopub.execute_input":"2023-07-13T14:01:41.747059Z","iopub.status.idle":"2023-07-13T14:01:42.047108Z","shell.execute_reply.started":"2023-07-13T14:01:41.747024Z","shell.execute_reply":"2023-07-13T14:01:42.046296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(test_masks[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:01:42.048404Z","iopub.execute_input":"2023-07-13T14:01:42.04931Z","iopub.status.idle":"2023-07-13T14:01:42.315534Z","shell.execute_reply.started":"2023-07-13T14:01:42.049278Z","shell.execute_reply":"2023-07-13T14:01:42.31452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(test_images[0])","metadata":{"execution":{"iopub.status.busy":"2023-07-13T14:10:42.712303Z","iopub.execute_input":"2023-07-13T14:10:42.713034Z","iopub.status.idle":"2023-07-13T14:10:43.076232Z","shell.execute_reply.started":"2023-07-13T14:10:42.712996Z","shell.execute_reply":"2023-07-13T14:10:43.075267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}