{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-15T08:05:17.789685Z","iopub.execute_input":"2023-09-15T08:05:17.790294Z","iopub.status.idle":"2023-09-15T08:05:17.797357Z","shell.execute_reply.started":"2023-09-15T08:05:17.790259Z","shell.execute_reply":"2023-09-15T08:05:17.79628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imports\nimport os\nimport random\nimport csv\nimport datetime\n# Data analysis and manipulation\nimport numpy as np\nimport pandas as pd\n\n# Data visualization\nfrom matplotlib import animation\nimport matplotlib.pyplot as plt\nfrom IPython import display\nimport seaborn as sns\nimport plotly\n\n# ML, DL & Modelling\n# from sklearn import \nimport tensorflow as tf\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2023-09-15T08:05:17.800463Z","iopub.execute_input":"2023-09-15T08:05:17.801322Z","iopub.status.idle":"2023-09-15T08:05:28.135487Z","shell.execute_reply.started":"2023-09-15T08:05:17.801289Z","shell.execute_reply":"2023-09-15T08:05:28.1344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **0 - Loading observations from the dataset to build X_train and y_train**","metadata":{}},{"cell_type":"code","source":"%%time\n# Loading a fraction of the dataset\n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\n\n\n# Building list of record_ids\nrecord_ids = os.listdir(BASE_DIR)\n\n# Randomly selecting observations\nnumber_observations = 1500\nsample_record_ids = random.sample(record_ids, number_observations) #record_ids","metadata":{"execution":{"iopub.status.busy":"2023-09-15T08:05:28.141222Z","iopub.execute_input":"2023-09-15T08:05:28.143964Z","iopub.status.idle":"2023-09-15T08:05:28.447847Z","shell.execute_reply.started":"2023-09-15T08:05:28.14391Z","shell.execute_reply":"2023-09-15T08:05:28.446903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining normalization function\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0,1]\"\"\"\n    normalized_data = (data - bounds[0])/(bounds[1] - bounds[0])\n    return normalized_data","metadata":{"execution":{"iopub.status.busy":"2023-09-15T08:05:28.452343Z","iopub.execute_input":"2023-09-15T08:05:28.454767Z","iopub.status.idle":"2023-09-15T08:05:28.462795Z","shell.execute_reply.started":"2023-09-15T08:05:28.45473Z","shell.execute_reply":"2023-09-15T08:05:28.461705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Looping on the record_id list to select only band_11, with 5th layer and build X_init\n\nband_choice = ['band_11.npy','band_14.npy','band_15.npy']\ntarget_suffix = 'human_pixel_masks.npy'\n\nN_TIMES_BEFORE = 4\n\nrecord_array_list = []\ntarget_array_list = []\nfor sample_id in sample_record_ids:\n   \n    # Building target paths\n    target_path = os.path.join(BASE_DIR, sample_id, target_suffix)\n    target = np.load(open(target_path, 'rb'))\n    \n    # Jumping over observations with no contrails\n    if target.sum()==0:\n        continue\n    else:\n    \n        # Building band paths\n        record_first_band_path = os.path.join(BASE_DIR, sample_id, band_choice[0])\n        record_second_band_path = os.path.join(BASE_DIR, sample_id, band_choice[1])\n        record_third_band_path = os.path.join(BASE_DIR, sample_id, band_choice[2])\n\n        # Loading each band\n        first_band = np.load(open(record_first_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n        second_band = np.load(open(record_second_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n        third_band = np.load(open(record_third_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n        \n        # Normalizing each band with its relevant bounds\n        # a - Defining bounds for each band\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n        \n        # b - Applying normalization functions\n        normalized_r = normalize_range(third_band - second_band, _TDIFF_BOUNDS)\n        normalized_g = normalize_range(second_band - first_band, _CLOUD_TOP_TDIFF_BOUNDS)\n        normalized_b = normalize_range(second_band, _T11_BOUNDS)\n        \n        # Building a single record from all bands\n        record = np.clip(np.stack([normalized_r, normalized_g, normalized_b], axis=2), 0, 1)\n        \n        # Appending target and observations lists\n        target_array_list.append(target)\n        record_array_list.append(record)\n    \nX_init = np.stack(record_array_list, axis=0)\ny_init = np.stack(target_array_list, axis=0).astype(float)\n\nprint(f\"We jumped over {number_observations - X_init.shape[0]} observations that had no contrails in them\")","metadata":{"execution":{"iopub.status.busy":"2023-09-15T08:05:28.469558Z","iopub.execute_input":"2023-09-15T08:05:28.471851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test X_init shape\nprint(X_init.shape)\nprint(y_init.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building a np.array with record ids\nsample_ids_array = np.array(sample_record_ids)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking all shapes\nprint(\"X_init shape is:\", X_init.shape)\nprint(\"y_init shape is:\", y_init.shape)\nprint(\"sample_ids_array shape is:\", sample_ids_array.shape)\nprint(f\"We jumped over {number_observations - X_init.shape[0]} observations that had no contrails in them ({round(100 * X_init.shape[0]/number_observations, 1)}% of observations kept)\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Storing test images\nimage_test = X_init[27]\ntarget_test = y_init[27]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Showing image test\nSHOW_PLOT = False\nif SHOW_PLOT:\n    plt.figure(figsize=(18, 6))\n    ax = plt.subplot(1, 3, 1)\n    ax.imshow(image_test);\n    ax.set_title('False color image')\n\n    ax = plt.subplot(1, 3, 2)\n    ax.imshow(target_test, interpolation='none');\n    ax.set_title('Ground truth contrail mask')\n\n    ax = plt.subplot(1, 3, 3)\n    ax.imshow(image_test)\n    ax.imshow(target_test, cmap='Reds', alpha=.4, interpolation='none')\n    ax.set_title('Contrail mask on false color image');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the number of contrail pixels in test target\n# target_test.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **I - Building the dice metric for the model loss function**","metadata":{}},{"cell_type":"code","source":"# Defining proba_to_pixel to transform predicted probas into categories for the dice metric\n\ndef proba_to_pixel(y):\n    return tf.where(y > 0.5, tf.ones_like(y),tf.zeros_like(y))\n\n# à ajouter après le predict pour pouvoir afficher le imshow","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the dice metric to be used in model compilation\n\ndef dice_metric(y_true, y_pred):\n        #converting y_pred probas to pixels (between 0 and 1)\n        y_pred = proba_to_pixel(y_pred)\n        y_true = proba_to_pixel(y_true)\n\n        # Define epsilon to prevent division by zero\n        smooth = 1e-5 \n\n        # Calculate the sum of y_true and y_pred for each class\n        y_true_sum = tf.reduce_sum(y_true)\n        y_pred_sum = tf.reduce_sum(y_pred)\n\n        # Calculate the intersection and union of y_true and y_pred\n        intersection = tf.reduce_sum(y_true * y_pred)\n        union = y_true_sum + y_pred_sum\n\n        # Calculate the Dice coefficient for each class\n        dice = (2. * intersection + smooth) / (union + smooth)\n\n        return dice","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the dice_closs as loss function used for the model \n\ndef dice_loss(y_true, y_pred):\n        \n    # No need to convert y_pred to pixels for the loss \n    # Define epsilon to prevent division by zero\n    smooth = 1e-5 \n\n    # Calculate the sum of y_true and y_pred for each class\n    y_true_sum = tf.reduce_sum(y_true)\n    y_pred_sum = tf.reduce_sum(y_pred)\n\n    # Calculate the intersection and union of y_true and y_pred\n    intersection = tf.reduce_sum(y_true * y_pred)\n    union = y_true_sum + y_pred_sum\n\n    # Calculate the Dice coefficient for each class\n    dice = (2. * intersection + smooth) / (union + smooth)\n\n    return 1 - dice","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_crossentropy(y_true, y_pred) :\n    return (-1)*tf.math.reduce_sum(y_true * tf.math.log(y_pred + 1e-7)  + (1-y_true)*tf.math.log(1-y_pred + 1e-7))\n\n# def weighted_binary_crossentropy(y_true, y_pred, weight_of_1) :\n#     return (-1)*tf.math.reduce_sum((1-weight_of_1)*y_true * tf.math.log(y_pred + 1e-7)  + weight_of_1*(1-y_true)*tf.math.log(1-y_pred + 1e-7))\n\ndef weighted_binary_crossentropy_func(weight_of_1) :\n    def weighted_binary_crossentropy(y_true, y_pred):\n        return (-1)*tf.math.reduce_sum((1-weight_of_1)*y_true * tf.math.log(y_pred + 1e-7)  + weight_of_1*(1-y_true)*tf.math.log(1-y_pred + 1e-7))\n    return weighted_binary_crossentropy\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing the dice loss function \n\npredictions_test = np.array([0.03, 0.9, 0.8])\ntrue_set = np.array([1.0, 1.0, 0.0])\n\ndice_loss(true_set, predictions_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **II - Building a simple baseline model**","metadata":{}},{"cell_type":"code","source":"# # Identifying the proportion of [1] in the train dataset to refine the baseline model\n\n# # 1-Retrieving the number of [1] in one 'human_pixel_masks.npy)\n# BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\n\n# # Building list of record_ids\n# record_ids = os.listdir(BASE_DIR)\n# # '139041523095796751'\n\n# # Randomly selecting 1 observation\n# sample_record_id = random.sample(record_ids, 1)[0]\n\n# # Loading the observation and counting the number of 1\n# target_suffix = 'human_pixel_masks.npy'\n# target_sample_path = os.path.join(BASE_DIR, '2642394871555025093', target_suffix)\n# # target_sample_path = os.path.join(BASE_DIR, sample_record_id, target_suffix)\n# target_sample = np.load(open(target_sample_path, 'rb'))\n\n# # Computing the proportion of contrail pixels in the picture\n# values, counts = np.unique(target_sample, return_counts=True)\n# proportion_1_pixels = counts[1]/counts.sum()\n# print(f'There are {round(proportion_1_pixels * 100, 2)}% of contrail pixels in the picture')","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:29.239596Z","iopub.execute_input":"2023-09-14T13:21:29.239983Z","iopub.status.idle":"2023-09-14T13:21:29.246356Z","shell.execute_reply.started":"2023-09-14T13:21:29.239954Z","shell.execute_reply":"2023-09-14T13:21:29.244968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Doing this process to get the average for 1K observations\n# # 1-Retrieving the number of [1] in one 'human_pixel_masks.npy)\n# from statistics import mean\n\n# def average_1_pixels(nb_observations):\n#     BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\n\n#     # Building list of record_ids\n#     record_ids = os.listdir(BASE_DIR)\n#     sample_record_ids = random.sample(record_ids, nb_observations)\n#     target_suffix = 'human_pixel_masks.npy'\n\n#     proportion_1_pixels_list = []\n#     for sample_id in sample_record_ids:\n#         target_path = os.path.join(BASE_DIR, sample_id, target_suffix)\n#         target = np.load(open(target_path, 'rb'))\n#         values, counts = np.unique(target, return_counts=True)\n#     #     print(counts)\n#         proportion_0_pixels = counts[0]/counts.sum()\n#         proportion_1_pixels = 1 - proportion_0_pixels\n#     #     print(target.shape) \n#     #     print(target.shape)\n#         proportion_1_pixels_list.append(proportion_1_pixels)\n    \n#     average_1_pixels = mean(proportion_1_pixels_list)\n#     return average_1_pixels\n\n# average_1_pixels(1000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Doing this process to get the average for 1K observations\n# 1-Retrieving the number of [1] in one 'human_pixel_masks.npy)\nfrom statistics import mean\n\ndef proportion_1_pixels(y):\n    \n    values, counts = np.unique(y, return_counts=True)\n    proportion_0_pixels = counts[0]/counts.sum()\n    proportion_1_pixels = 1 - proportion_0_pixels\n    \n    return proportion_1_pixels\n\np = proportion_1_pixels(y_init)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Choosing a model that will classify each pixel as 0 or 1 with a probability of 50%\n# # Shape of the output depends on the shape of the desired target\n\n# def baseline_predict(y):\n    \n#     baseline_pred = np.random.randint(0,2,y.shape)\n#     print('shape of baseline_pred is:', baseline_pred.shape)\n#     return baseline_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list_test = [y_init.shape[i] for i in range(0, len(y_init.shape))]\n# np.prod(list_test)\np","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choosing a model that will classify each pixel as 0 or 1 with a probability of p\n\ndef baseline_predict_flexible(y, p):\n    \n    values = [0,1]\n    probas = [1-p, p]\n    sample_size = np.prod([y.shape[i] for i in range(0, len(y.shape))])\n#     y.shape[0] * y.shape[1]\n    baseline_pred = np.random.choice(values, size=sample_size, p=probas)\n    baseline_pred = baseline_pred.reshape(y.shape)\n    \n    return baseline_pred","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:33.50058Z","iopub.execute_input":"2023-09-14T13:21:33.500962Z","iopub.status.idle":"2023-09-14T13:21:33.508209Z","shell.execute_reply.started":"2023-09-14T13:21:33.50093Z","shell.execute_reply":"2023-09-14T13:21:33.507032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_baseline = baseline_predict_flexible(y_init, p=p).astype(float)\nprint(y_pred_baseline.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:34.34708Z","iopub.execute_input":"2023-09-14T13:21:34.348241Z","iopub.status.idle":"2023-09-14T13:21:37.118755Z","shell.execute_reply.started":"2023-09-14T13:21:34.348193Z","shell.execute_reply":"2023-09-14T13:21:37.117699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values_baseline, counts_baseline = np.unique(y_pred_baseline, return_counts=True)\nprint(unique_values_baseline)\nprint(counts_baseline)\nprint(counts_baseline.sum())","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:37.120522Z","iopub.execute_input":"2023-09-14T13:21:37.120801Z","iopub.status.idle":"2023-09-14T13:21:38.610828Z","shell.execute_reply.started":"2023-09-14T13:21:37.120776Z","shell.execute_reply":"2023-09-14T13:21:38.609561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Computing the dice metric for y_pred_baseline\n#tf.convert_to_tensor\n\ndice_init = dice_loss(y_init, y_pred_baseline)\ndice_init","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:38.612527Z","iopub.execute_input":"2023-09-14T13:21:38.613305Z","iopub.status.idle":"2023-09-14T13:21:41.090367Z","shell.execute_reply.started":"2023-09-14T13:21:38.613261Z","shell.execute_reply":"2023-09-14T13:21:41.089123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **III - Building a first simple CNN model**","metadata":{}},{"cell_type":"markdown","source":"Our first model architecture will have **2 building blocks**: \n* **Downsampling path / Decoder** = convolutional layers extracting features from the image while reducing it size\n* **Upsampling / Decoder** = expanding the size of the image using Transpose convolution to reach an output (the mask) with same size as input image","metadata":{}},{"cell_type":"code","source":"# Imports\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import callbacks","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:41.094533Z","iopub.execute_input":"2023-09-14T13:21:41.094878Z","iopub.status.idle":"2023-09-14T13:21:41.10021Z","shell.execute_reply.started":"2023-09-14T13:21:41.094849Z","shell.execute_reply":"2023-09-14T13:21:41.099042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n\n# Encoder \nmodel.add(layers.Conv2D(16, (3,3), input_shape=(256, 256, 3), padding='same', activation=\"relu\"))\nmodel.add(layers.MaxPool2D(pool_size=(2,2)))\n\nmodel.add(layers.Conv2D(32, (2,2), padding='same', activation=\"relu\"))\nmodel.add(layers.MaxPool2D(pool_size=(2,2)))\n\n# Decoder \nmodel.add(layers.Conv2DTranspose(32, (2,2), padding='same', activation=\"relu\", strides=(2,2)))\nmodel.add(layers.Conv2DTranspose(1, (2,2), padding='same', activation=\"sigmoid\", strides=(2,2)))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:21:41.101809Z","iopub.execute_input":"2023-09-14T13:21:41.102931Z","iopub.status.idle":"2023-09-14T13:21:41.212283Z","shell.execute_reply.started":"2023-09-14T13:21:41.102888Z","shell.execute_reply":"2023-09-14T13:21:41.211473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Defining optimizer in detail\noptimizer = tf.keras.optimizers.Adam(\n    learning_rate=0.005,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=1e-07,\n    amsgrad=False,\n    weight_decay=None,\n    clipnorm=None,\n    clipvalue=None,\n    global_clipnorm=None,\n    use_ema=False,\n    ema_momentum=0.99,\n    ema_overwrite_frequency=None,\n    jit_compile=True,\n    name='Adam'\n)\n\n# 2. Compile\n\n\nmodel.compile(optimizer=optimizer,\n                  loss=weighted_binary_crossentropy_func(0.01183131632324377),\n                  metrics=dice_metric)\n\n# 3. Fit \n\nes = callbacks.EarlyStopping(patience=30)\nhistory_base_model = model.fit(X_init, y_init,\n          batch_size=16, \n          epochs=150,\n          validation_split=0.3,\n          callbacks=[es],\n          verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:23:18.540421Z","iopub.execute_input":"2023-09-14T13:23:18.540793Z","iopub.status.idle":"2023-09-14T13:25:42.437435Z","shell.execute_reply.started":"2023-09-14T13:23:18.540763Z","shell.execute_reply":"2023-09-14T13:25:42.436133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history_base_model.__dict__","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:25:42.439817Z","iopub.execute_input":"2023-09-14T13:25:42.441045Z","iopub.status.idle":"2023-09-14T13:25:42.458613Z","shell.execute_reply.started":"2023-09-14T13:25:42.440987Z","shell.execute_reply":"2023-09-14T13:25:42.457382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_history(history, title='', axs=None, exp_name=\"\"):\n    if axs is not None:\n        ax1, ax2 = axs\n    else:\n        f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\n    \n    if len(exp_name) > 0 and exp_name[0] != '_':\n        exp_name = '_' + exp_name\n    ax1.plot(history.history['loss'], label = 'train' + exp_name)\n    ax1.plot(history.history['val_loss'], label = 'val' + exp_name)\n#     ax1.set_ylim(0., 2.2)\n    ax1.set_title('loss')\n    ax1.legend()\n\n    ax2.plot(history.history['dice_metric'], label='train dice metric'  + exp_name)\n    ax2.plot(history.history['val_dice_metric'], label='val dice metric'  + exp_name)\n#     ax2.set_ylim(0.25, 1.)\n    ax2.set_title('Dice metric')\n    ax2.legend()\n    return (ax1, ax2)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:25:42.740073Z","iopub.execute_input":"2023-09-14T13:25:42.740862Z","iopub.status.idle":"2023-09-14T13:25:42.749457Z","shell.execute_reply.started":"2023-09-14T13:25:42.740824Z","shell.execute_reply":"2023-09-14T13:25:42.748305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history_base_model)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:25:44.56498Z","iopub.execute_input":"2023-09-14T13:25:44.566315Z","iopub.status.idle":"2023-09-14T13:25:45.164104Z","shell.execute_reply.started":"2023-09-14T13:25:44.566265Z","shell.execute_reply":"2023-09-14T13:25:45.162862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **IV - Building an advanced U-net model**","metadata":{}},{"cell_type":"code","source":"# 1. Building the Unet model \n\ndef build_model(input_layer, start_neurons):\n    \n    # Downsampling path / Decoder = convolutional layers extracting features from the image while \n    # reducing it size\n    conv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(input_layer)\n    conv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(conv1)\n    pool1 = layers.MaxPooling2D((2, 2))(conv1)\n    pool1 = layers.Dropout(0.25)(pool1)\n\n    conv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(pool1)\n    conv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(conv2)\n    pool2 = layers.MaxPooling2D((2, 2))(conv2)\n    pool2 = layers.Dropout(0.5)(pool2)\n\n    conv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(pool2)\n    conv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(conv3)\n    pool3 = layers.MaxPooling2D((2, 2))(conv3)\n    pool3 = layers.Dropout(0.5)(pool3)\n\n    conv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(pool3)\n    conv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(conv4)\n    pool4 = layers.MaxPooling2D((2, 2))(conv4)\n    pool4 = layers.Dropout(0.5)(pool4)\n\n    # Middle path / Bottleneck = CNN with large number of layers to extract the image's most important /\n    # complex features\n    convm = layers.Conv2D(start_neurons * 16, (3, 3), activation=\"relu\", padding=\"same\")(pool4)\n    convm = layers.Conv2D(start_neurons * 16, (3, 3), activation=\"relu\", padding=\"same\")(convm)\n    \n    # Upsampling / Decoder using Transpose convolution = expanding the size of the image to reach an output \n    # (the mask) with same size as input image \n    \n    # Skip connections: it helps the model learn both detailed information from the \n    # downsampling / decoder and high-level info from the upsampling / decoder \n    \n    # In practice, after the transposed convolution, the image is upsized from 28x28x1024 → 56x56x512\n    # this image is then concatenated with the corresponding image from the downsampling path \n    # and together makes an image of size 56x56x1024. \n    \n    # upsamppling \n    deconv4 = layers.Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding=\"same\")(convm)\n    # skip-connection\n    uconv4 = layers.concatenate([deconv4, conv4])\n    uconv4 = layers.Dropout(0.5)(uconv4)\n    uconv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(uconv4)\n    uconv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(uconv4)\n\n    deconv3 = layers.Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding=\"same\")(uconv4)\n    uconv3 = layers.concatenate([deconv3, conv3])\n    uconv3 = layers.Dropout(0.5)(uconv3)\n    uconv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(uconv3)\n    uconv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(uconv3)\n\n    deconv2 = layers.Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding=\"same\")(uconv3)\n    uconv2 = layers.concatenate([deconv2, conv2])\n    uconv2 = layers.Dropout(0.5)(uconv2)\n    uconv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(uconv2)\n    uconv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(uconv2)\n\n    deconv1 = layers.Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding=\"same\")(uconv2)\n    uconv1 = layers.concatenate([deconv1, conv1])\n    uconv1 = layers.Dropout(0.5)(uconv1)\n    uconv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(uconv1)\n    uconv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(uconv1)\n    \n    output_layer = layers.Conv2D(1, (1,1), padding=\"same\", activation=\"sigmoid\")(uconv1)\n    \n    return output_layer","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:31:02.590175Z","iopub.execute_input":"2023-09-14T13:31:02.590559Z","iopub.status.idle":"2023-09-14T13:31:02.626896Z","shell.execute_reply.started":"2023-09-14T13:31:02.590527Z","shell.execute_reply":"2023-09-14T13:31:02.625592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining img_size and number of neurons for the CNN layers \n\nimg_size_target = X_init.shape[1]\nnumber_channels_target = X_init.shape[-1]\nstart_neurons = 64\n\ninput_layer = layers.Input((img_size_target, img_size_target, number_channels_target))\noutput_layer = build_model(input_layer, start_neurons)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:07:16.993241Z","iopub.execute_input":"2023-09-14T14:07:16.993692Z","iopub.status.idle":"2023-09-14T14:07:17.320501Z","shell.execute_reply.started":"2023-09-14T14:07:16.993659Z","shell.execute_reply":"2023-09-14T14:07:17.319315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unet model with Keras Functional API\nunet_model = tf.keras.Model(input_layer, output_layer, name=\"U-Net\")","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:07:22.241477Z","iopub.execute_input":"2023-09-14T14:07:22.241904Z","iopub.status.idle":"2023-09-14T14:07:22.255965Z","shell.execute_reply.started":"2023-09-14T14:07:22.24185Z","shell.execute_reply":"2023-09-14T14:07:22.255052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_model_weights = False\nif load_model_weights :\n    unet_model.load_weights('/kaggle/working/tf_checkpoint')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:08:54.369485Z","iopub.execute_input":"2023-09-14T14:08:54.369932Z","iopub.status.idle":"2023-09-14T14:08:54.633119Z","shell.execute_reply.started":"2023-09-14T14:08:54.369899Z","shell.execute_reply":"2023-09-14T14:08:54.632052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"optimizer_2 = tf.keras.optimizers.legacy.Adam(learning_rate=5e-4)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:09:01.503557Z","iopub.execute_input":"2023-09-14T14:09:01.503978Z","iopub.status.idle":"2023-09-14T14:09:01.510161Z","shell.execute_reply.started":"2023-09-14T14:09:01.503945Z","shell.execute_reply":"2023-09-14T14:09:01.509016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n# 2. Compile\nweight_of_1 = 0.01183131632324377 #on taining dataset excluding images without contrails\n\nunet_model.compile(optimizer=optimizer_2,\n                  loss=weighted_binary_crossentropy_func(weight_of_1),\n                  metrics=[dice_metric, dice_loss, weighted_binary_crossentropy_func(weight_of_1)])\n\n# 3. Fit & callbacks\nes = tf.keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True)\nlrp = callbacks.ReduceLROnPlateau(monitor='val_loss',\n                                  factor=0.2,\n                                  patience=5,\n                                  min_lr=0.00001)\nmodel_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath='/kaggle/working/tf_checkpoint',\n    save_weights_only=True,\n    save_best_only=True)\n\n# tensorboard_checkpoint_callback = tf.keras.callbacks.TensorBoard(\n#     log_dir='/kaggle/working/tensorboard_checkpoint',\n#     histogram_freq=0,\n#     write_graph=True,\n#     write_images=False,\n#     write_steps_per_second=False,\n#     update_freq='epoch',\n#     profile_batch=0,\n#     embeddings_freq=0,\n#     embeddings_metadata=None\n# )\n\nhistory_unet_model = unet_model.fit(X_init,\n                                    y_init,\n                                    batch_size=16,\n                                    epochs=200,\n                                    validation_split=0.3,\n                                    callbacks=[es, lrp, model_checkpoint_callback, tensorboard_checkpoint_callback],\n                                    verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:09:04.088932Z","iopub.execute_input":"2023-09-14T14:09:04.089356Z","iopub.status.idle":"2023-09-14T14:24:43.913963Z","shell.execute_reply.started":"2023-09-14T14:09:04.089322Z","shell.execute_reply":"2023-09-14T14:24:43.912755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history_unet_model.__dict__","metadata":{"execution":{"iopub.status.busy":"2023-09-14T13:18:38.958091Z","iopub.execute_input":"2023-09-14T13:18:38.959616Z","iopub.status.idle":"2023-09-14T13:18:38.964424Z","shell.execute_reply.started":"2023-09-14T13:18:38.959575Z","shell.execute_reply":"2023-09-14T13:18:38.963236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history(history_unet_model)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:27:18.62666Z","iopub.execute_input":"2023-09-14T14:27:18.627185Z","iopub.status.idle":"2023-09-14T14:27:40.889158Z","shell.execute_reply.started":"2023-09-14T14:27:18.627138Z","shell.execute_reply":"2023-09-14T14:27:40.888174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_unet_model.history","metadata":{"execution":{"iopub.status.busy":"2023-09-14T14:46:25.628931Z","iopub.execute_input":"2023-09-14T14:46:25.629934Z","iopub.status.idle":"2023-09-14T14:46:25.640954Z","shell.execute_reply.started":"2023-09-14T14:46:25.629886Z","shell.execute_reply":"2023-09-14T14:46:25.639791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nombre_de_batchs = 2\n\ndict_history = {f'batch number {batch_no}' : \"\"  for batch_no in range(nombre_de_batchs)}\n\nfor batch_no in range(nombre_de_batchs) :\n\n    #on telechage les 2000 pts de data\n    #on fait tourner le modele qui fabrique \"history_unet_model\"\n    \n    dict_history[f'batch number {batch_no}'] = history_unet_model.history\n    \n    dict_history_from_current_batch = history_unet_model.history\n    fields = list(dict_history_from_current_batch.keys())\n\n    # Open the CSV file with write permission\n    base_dir = \"/kaggle/working/model_histories/\"\n    \n    with open(f\"{datetime.datetime.now()}output_batch_no_{batch_no}.csv\", \"w\", newline=\"\") as csvfile:\n\n        # Create a CSV writer using the field/column names\n        writer = csv.DictWriter(csvfile, fieldnames=fields)\n\n        # Write the header row (column names)\n        writer.writeheader()\n\n        # Write the data\n        for ligne in range(len(dict_history_from_current_batch[\"loss\"])) :\n            dict_row = { index : dict_history_from_current_batch[index][ligne] for index in fields}\n            writer.writerow(dict_row)\n\n\n    \n    \n    \n    \n    # fini la boucle for\n\n\n\n\n\n# # Sample dictionary to be converted to CSV\n# dict_history\n\n# # Define the fields/columns for the CSV file\n# fields = list(dict_history.keys())\n\n# # Open the CSV file with write permission\n# with open(\"output_final.csv\", \"w\", newline=\"\") as csvfile:\n    \n#     # Create a CSV writer using the field/column names\n#     writer = csv.DictWriter(csvfile, fieldnames=fields)\n\n#     # Write the header row (column names)\n#     writer.writeheader()\n\n#     # Write the data\n#     for row in data:\n#         writer_rows\n#         writer.writerow(row)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-14T15:03:39.608376Z","iopub.execute_input":"2023-09-14T15:03:39.60877Z","iopub.status.idle":"2023-09-14T15:03:39.621865Z","shell.execute_reply.started":"2023-09-14T15:03:39.60874Z","shell.execute_reply":"2023-09-14T15:03:39.620671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}