{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n#\n\n# BASE_DIR = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/\"\n# TRAIN_DIR = BASE_DIR + \"train/\"\n# VALIDATION_DIR = BASE_DIR + \"validation/\"\n# TEST_DIR = BASE_DIR + \"test/\"\n# validation_ids = os.listdir(VALIDATION_DIR)\n# test_ids = os.listdir(TEST_DIR)\n\n# train_id_list = [(TRAIN_DIR, record_id) for record_id in os.listdir(TRAIN_DIR)]\n# validation_id_list = [(VALIDATION_DIR, record_id) for record_id in os.listdir(VALIDATION_DIR)]\n# full_id_list = train_id_list + validation_id_list\n# test_id_list = [(TEST_DIR, record_id) for record_id in os.listdir(TEST_DIR)]\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-26T02:05:51.236644Z","iopub.execute_input":"2023-07-26T02:05:51.23707Z","iopub.status.idle":"2023-07-26T02:05:51.243919Z","shell.execute_reply.started":"2023-07-26T02:05:51.23704Z","shell.execute_reply":"2023-07-26T02:05:51.242579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport pathlib\n\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\n\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import optim\n\nfrom torch.utils.data import DataLoader, random_split\nfrom torch.utils.data import Dataset\n\nimport torchvision\nfrom torchvision import datasets\nimport torchvision.transforms as T\nfrom torchvision.transforms import Compose, ToTensor, Resize\nfrom torchvision.utils import make_grid\n","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:51.983769Z","iopub.execute_input":"2023-07-26T02:05:51.98416Z","iopub.status.idle":"2023-07-26T02:05:51.9927Z","shell.execute_reply.started":"2023-07-26T02:05:51.984129Z","shell.execute_reply":"2023-07-26T02:05:51.991392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.utils.data as data\n# import pandas as pd\n# import numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:52.460429Z","iopub.execute_input":"2023-07-26T02:05:52.46082Z","iopub.status.idle":"2023-07-26T02:05:52.46612Z","shell.execute_reply.started":"2023-07-26T02:05:52.46079Z","shell.execute_reply":"2023-07-26T02:05:52.464933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Log this config file to wandb\nCONFIG = dict(\n    seed=42,\n    DATA_ROOT = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/',\n    BATCH_SIZE = 10,\n    IMG_SIZE = (256,256),\n    NUM_EPOCHS = 1,\n    lr = 0.0003,\n    n_channels = 72)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:53.302533Z","iopub.execute_input":"2023-07-26T02:05:53.302938Z","iopub.status.idle":"2023-07-26T02:05:53.309529Z","shell.execute_reply.started":"2023-07-26T02:05:53.302901Z","shell.execute_reply":"2023-07-26T02:05:53.308283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = torch.device('cpu')\nif torch.cuda.is_available():\n    device = torch.device('cuda')\nelse:\n    device = torch.device('cpu')\n\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:54.006501Z","iopub.execute_input":"2023-07-26T02:05:54.006955Z","iopub.status.idle":"2023-07-26T02:05:54.015371Z","shell.execute_reply.started":"2023-07-26T02:05:54.006923Z","shell.execute_reply":"2023-07-26T02:05:54.014053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = Compose([ToTensor()])\nval_transform = Compose([ToTensor()])\ntest_transform = Compose([ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:57.460494Z","iopub.execute_input":"2023-07-26T02:05:57.460934Z","iopub.status.idle":"2023-07-26T02:05:57.46598Z","shell.execute_reply.started":"2023-07-26T02:05:57.460898Z","shell.execute_reply":"2023-07-26T02:05:57.4652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the full metadata\ndata_types = {'record_id': str, 'projection_wkt': str, 'row_min': float, 'row_size': float,\n              'col_min': float, \"col_size\": float, 'timestamp': pd.Timestamp}\n\ntrain_metadata_df = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json\", dtype=data_types)\njson_train_record_id_list = (train_metadata_df['record_id'].to_list())\nval_metadata_df = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json\", dtype=data_types)\njson_val_record_id_list = (val_metadata_df['record_id'].to_list())\n# Specify the number of samples to test\nnum_samples_to_test = 10\n\n\nrecord_id_list = json_train_record_id_list + json_val_record_id_list ","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:57.912655Z","iopub.execute_input":"2023-07-26T02:05:57.913081Z","iopub.status.idle":"2023-07-26T02:05:58.20749Z","shell.execute_reply.started":"2023-07-26T02:05:57.913046Z","shell.execute_reply":"2023-07-26T02:05:58.20648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class contrailsDataset(Dataset):\n    def __init__(self, base_dir=CONFIG['DATA_ROOT'], mode='test',metadata_df = train_metadata_df, transform=None):\n        super().__init__()\n        \n        assert mode in ['train', 'test', 'validation'], \"Please pass in train, test or validation\"\n        \n        self.base_dir = base_dir\n        self.mode = mode\n        self.transform = transform\n        \n        self.records = os.listdir(self.base_dir+self.mode)\n        \n        if self.mode == 'train': \n            select_cnt = 2 # Change for # samples you want \n            self.records = np.random.choice(self.records, select_cnt, replace=False)\n        \n        if self.mode == 'validation':\n            select_cnt = 1 # Change for # samples you want \n            self.records = np.random.choice(self.records, select_cnt, replace=False)\n            \n    def get_lat_lon(record_id, metadata_df):\n        row = metadata_df.loc[metadata_df['record_id'] == record_id]\n        latitude_of_origin = row['latitude_of_origin'].values[0]\n        central_meridian = row['central_meridian'].values[0]\n        return latitude_of_origin, central_meridian\n     \n    def get_month(record_id, metadata_df):\n        row = metadata_df.loc[metadata_df['record_id'] == record_id]\n        month = row['timestamp'].values[0].month\n        return month\n     \n        \n    def normalize_image(image):\n        \"\"\"Maps data to the range [0, 1].\"\"\"\n        return (image-np.min(image))/(np.max(image)-np.min(image))\n    \n    def get_ash_image(self, idx):\n        record_id = self.records[idx]\n        record_dir = os.path.join(self.base_dir, self.mode, record_id)\n        band_08 = []\n        band_09 = []\n        band_10 = []\n        band_11 = []\n        band_12 = []\n        band_13 = []\n        band_14 = []\n        band_15 = []\n        band_16 = []\n        for i in range(0,8):\n            band_08_i = normalize_image(np.load(os.path.join(record_dir + '/band_08.npy')))[...,i].flatten()\n            band_08.append(band_08_i)\n            band_09_i = normalize_image(np.load(os.path.join(record_dir + '/band_09.npy')))[...,i].flatten()\n            band_09.append(band_09_i)\n            band_10_i = normalize_image(np.load(os.path.join(record_dir + '/band_10.npy')))[...,i].flatten()\n            band_10 = np.asarray(band_10.append(band_10_i)).transpose()\n            band_11_i = normalize_image(np.load(os.path.join(record_dir + '/band_11.npy')))[...,i].flatten()\n            band_11.append(band_11_i)\n            band_12_i = normalize_image(np.load(os.path.join(record_dir + '/band_12.npy')))[...,i].flatten()\n            band_12.append(band_12_i)\n            band_13_i = normalize_image(np.load(os.path.join(record_dir + '/band_13.npy')))[...,i].flatten()\n            band_13.append(band_13_i)\n            band_14_i = normalize_image(np.load(os.path.join(record_dir + '/band_14.npy')))[...,i].flatten()\n            band_14.append(band_14_i)\n            band_15_i = normalize_image(np.load(os.path.join(record_dir + '/band_15.npy')))[...,i].flatten()\n            band_15.append(band_15_i)\n            band_16_i = normalize_image(np.load(os.path.join(record_dir + '/band_16.npy')))[...,i].flatten()\n            band_16.append(band_16_i)\n        band_08 = np.asarray(band_08).transpose()\n        band_09 = np.asarray(band_09).transpose()\n        band_10 = np.asarray(band_10).transpose()\n        band_11 = np.asarray(band_11).transpose()\n        band_12 = np.asarray(band_12).transpose()\n        band_13 = np.asarray(band_13).transpose()\n        band_14 = np.asarray(band_14).transpose()\n        band_15 = np.asarray(band_15).transpose()\n        band_16 = np.asarray(band_16).transpose()\n        image_all = np.clip(np.stack([band_08, band_09, band_10, band_11, band_12, band_13, band_14, band_15, band_16], axis=2), 0, 1)\n        return image_all\n        \n        \n    def __getitem__(self, idx):\n        record_id = self.records[idx]\n        record_dir = os.path.join(self.base_dir, self.mode, record_id)\n        \n        #bands_data = []\n        ash = self.get_ash_image(idx)\n#         for i in range(8, 17):\n#             band_file = os.path.join(record_dir, f'band_{str(i).zfill(2)}.npy')\n#             band_data = np.load(band_file)\n#             bands_data.append(band_data)\n\n#         # Stack band data along the channel axis\n#         bands_data = np.stack(bands_data, axis=-1)\n#         ash = self.get_ash_img(bands_data)\n\n\n        # If the data type is 'train' or 'validation', load the masks\n        if self.mode in ['train', 'validation']:\n            pixel_masks_file = os.path.join(record_dir, 'human_pixel_masks.npy')\n            pixel_masks = np.load(pixel_masks_file).flatten()\n        else:\n            pixel_masks = None  \n\n        if self.transform:\n            ash = self.transform(ash)\n            masks = []\n            lat_lon_values_list = []\n            dynamic_worlds = []\n            lat_of_origin, central_meridian = get_lat_lon(record_id, train_metadata_df)\n            lat_lon_values = torch.tensor([lat_of_origin,central_meridian])\n            lat_lon_values_list.append(lat_lon_values)\n            month = get_month(record_id, metadata_df)\n            month = torch.tensor(month * 65536).long()\n            dynamic_worlds.append((np.ones((65536, 8, 9))*10))\n            masks.append(np.full(( 65536,8,9 ), None))\n            if self.mode != 'test':\n                pixel_masks = self.transform(pixel_masks)\n                sample = { 'arrays': ash, 'masks': torch.tensor(masks), 'dynamic_worlds' : torch_tensor(dynamic_worlds), 'latlons' : torch_tensor(lat_lon_values_list)  , 'month': month, 'y': torch_tensor(pixel_masks)} #'bands': bands_data,\n            else:\n\n                sample = { 'arrays': ash, 'masks': torch.tensor(masks), 'dynamic_worlds' : torch_tensor(dynamic_worlds), 'latlons' : torch_tensor(lat_lon_values_list) 'month': month  } #'bands': bands_data,\n        \n        return sample\n    \n    \n    def __len__(self):\n        return len(self.records)\n    \n    #########################################################\n\n\n\n# # Custom Loss function\n# def DiceLoss(y_true, y_pred, smooth=1e-6):\n    \n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n    \n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     dice = (2*intersection + smooth) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) + smooth)\n#     return dice\n\n# # custom metric for model\n\n# def jaccard_coef(y_true, y_pred):\n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     final_coef_value = (intersection + 1.0) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) - intersection + 1.0)\n#     return final_coef_value\n\n# def dice_coeff(y_true, y_pred):\n#     smooth = 1.0\n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     dice_score = (2.0 * intersection + smooth) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) + smooth)\n#     return dice_score\n\n\n# def dice_loss(y_true, y_pred):\n#     loss = 1 - dice_coeff(y_true, y_pred)\n#     return loss\n\n# def bce_dice_loss(y_true, y_pred):\n#     loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n#     return loss\n\n# # Loss from segmentation models library\n# # TOTAL_LOSS_FACTOR = 5\n# # dice_loss = sm.losses.DiceLoss(class_weights = [0.5,0.5])\n# # focal_loss = sm.losses.CategoricalFocalLoss()\n# # total_loss = dice_loss + (TOTAL_LOSS_FACTOR * focal_loss)\n\n# from tensorflow.python.keras import losses","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:05:58.559592Z","iopub.execute_input":"2023-07-26T02:05:58.559979Z","iopub.status.idle":"2023-07-26T02:05:58.594222Z","shell.execute_reply.started":"2023-07-26T02:05:58.55995Z","shell.execute_reply":"2023-07-26T02:05:58.59299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainData = contrailsDataset(mode='train', transform=train_transform)\nvalData = contrailsDataset(mode='validation', transform=val_transform)\ntestData = contrailsDataset(mode='test', transform=test_transform)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:06:16.94833Z","iopub.execute_input":"2023-07-26T02:06:16.948937Z","iopub.status.idle":"2023-07-26T02:06:16.975279Z","shell.execute_reply.started":"2023-07-26T02:06:16.948903Z","shell.execute_reply":"2023-07-26T02:06:16.974383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(trainData), len(valData), len(testData)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:06:21.750068Z","iopub.execute_input":"2023-07-26T02:06:21.750879Z","iopub.status.idle":"2023-07-26T02:06:21.758224Z","shell.execute_reply.started":"2023-07-26T02:06:21.75084Z","shell.execute_reply":"2023-07-26T02:06:21.756938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(trainData, \n                              batch_size=CONFIG['BATCH_SIZE'], \n                              shuffle=True)\n\nval_dataloader = DataLoader(valData, \n                            batch_size=CONFIG['BATCH_SIZE'],\n                            shuffle=True)\n\ntest_dataloader = DataLoader(testData, \n                             batch_size=CONFIG['BATCH_SIZE'], \n                             shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:06:30.616752Z","iopub.execute_input":"2023-07-26T02:06:30.617156Z","iopub.status.idle":"2023-07-26T02:06:30.624263Z","shell.execute_reply.started":"2023-07-26T02:06:30.617125Z","shell.execute_reply":"2023-07-26T02:06:30.623259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pwd","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:08:00.555351Z","iopub.execute_input":"2023-07-26T02:08:00.555787Z","iopub.status.idle":"2023-07-26T02:08:00.562509Z","shell.execute_reply.started":"2023-07-26T02:08:00.555742Z","shell.execute_reply":"2023-07-26T02:08:00.561716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:09:11.779424Z","iopub.execute_input":"2023-07-26T02:09:11.779873Z","iopub.status.idle":"2023-07-26T02:09:11.787932Z","shell.execute_reply.started":"2023-07-26T02:09:11.779831Z","shell.execute_reply":"2023-07-26T02:09:11.786731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%ls","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:09:52.516671Z","iopub.execute_input":"2023-07-26T02:09:52.517107Z","iopub.status.idle":"2023-07-26T02:09:53.608615Z","shell.execute_reply.started":"2023-07-26T02:09:52.517076Z","shell.execute_reply":"2023-07-26T02:09:53.607522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd presto-main","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:10:00.819253Z","iopub.execute_input":"2023-07-26T02:10:00.819741Z","iopub.status.idle":"2023-07-26T02:10:00.830926Z","shell.execute_reply.started":"2023-07-26T02:10:00.819701Z","shell.execute_reply":"2023-07-26T02:10:00.82964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:10:57.750101Z","iopub.execute_input":"2023-07-26T02:10:57.751024Z","iopub.status.idle":"2023-07-26T02:11:13.712514Z","shell.execute_reply.started":"2023-07-26T02:10:57.750964Z","shell.execute_reply":"2023-07-26T02:11:13.711278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"python -m venv venv\nsource venv/bin/activate\npip install -e ","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:14:40.330748Z","iopub.execute_input":"2023-07-26T02:14:40.331174Z","iopub.status.idle":"2023-07-26T02:14:40.355932Z","shell.execute_reply.started":"2023-07-26T02:14:40.33114Z","shell.execute_reply":"2023-07-26T02:14:40.354171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-07-26T02:07:23.839754Z","iopub.execute_input":"2023-07-26T02:07:23.840144Z","iopub.status.idle":"2023-07-26T02:07:25.381063Z","shell.execute_reply.started":"2023-07-26T02:07:23.840114Z","shell.execute_reply":"2023-07-26T02:07:25.379594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xarray\nfrom pyproj import Transformer\nimport numpy as np\nfrom scipy import stats\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\n\nfrom tqdm import tqdm\n\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\n\nimport presto\n\n# this is to silence the xarray deprecation warning.\n# Our version of xarray is pinned, but we'll need to fix this\n# when we upgrade\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\npretrained_model = presto.Presto.load_pretrained()\npretrained_model.eval()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_list = []\nfor (x, mask, dw, latlons, month) in tqdm(train_dataloader ):\n    with torch.no_grad():\n        encodings = (\n            pretrained_model.encoder(\n                x, dynamic_world=dw, mask=mask, latlons=latlons , month=month\n            )\n            .cpu()\n            .numpy()\n        )\n        features_list.append(encodings)\nfeatures_np = np.concatenate(features_list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import time\n# import pandas as pd\n# import numpy as np\n# from matplotlib import animation\n# import matplotlib.pyplot as plt\n\n# import multiprocessing\n\n# import tensorflow as tf\n# from tensorflow import keras\n# import pickle \n   \n# from keras.models import load_model, Model\n# from keras import losses\n# from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, Conv2DTranspose, concatenate, BatchNormalization, Dropout, Lambda\n\n# from tensorflow.keras.utils import to_categorical\n\n# from keras import backend as K\n# # backend is tensor flow. At tensor level calculation, Keras does not cover it. Keras is only the framework. All backend methods are referred as K\n\n# os.environ[\"SM_FRAMEWORK\"] = \"tf.keras\"\n# #import segmentation_models as sm\n# tf.keras.backend.clear_session()\n\n# # Initialize Google CLoud Storage\n# # Cloud Storage\n# # from google.cloud import storage\n# # storage_client = storage.Client(project='multi-tenancy-dataproc')\n# # bucket = storage_client.get_bucket(\"kaggle-contrails\")\n\n\n\n# #(1000,65536,8,9)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T00:59:59.815841Z","iopub.execute_input":"2023-07-25T00:59:59.816247Z","iopub.status.idle":"2023-07-25T00:59:59.828901Z","shell.execute_reply.started":"2023-07-25T00:59:59.816216Z","shell.execute_reply":"2023-07-25T00:59:59.827946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nBASE_DIR = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/\"\nTRAIN_DIR = BASE_DIR + \"train/\"\nVALIDATION_DIR = BASE_DIR + \"validation/\"\nTEST_DIR = BASE_DIR + \"test/\"\nvalidation_ids = os.listdir(VALIDATION_DIR)\ntest_ids = os.listdir(TEST_DIR)\n\ntrain_id_list = [(TRAIN_DIR, record_id) for record_id in os.listdir(TRAIN_DIR)]\nvalidation_id_list = [(VALIDATION_DIR, record_id) for record_id in os.listdir(VALIDATION_DIR)]\nfull_id_list = train_id_list + validation_id_list\ntest_id_list = [(TEST_DIR, record_id) for record_id in os.listdir(TEST_DIR)]\n","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:16:43.940944Z","iopub.execute_input":"2023-07-26T01:16:43.941326Z","iopub.status.idle":"2023-07-26T01:16:44.266462Z","shell.execute_reply.started":"2023-07-26T01:16:43.941297Z","shell.execute_reply":"2023-07-26T01:16:44.265678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMBER_OF_RECORDS = 1000 # User Enterable","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:16:49.948705Z","iopub.execute_input":"2023-07-26T01:16:49.949092Z","iopub.status.idle":"2023-07-26T01:16:49.953954Z","shell.execute_reply.started":"2023-07-26T01:16:49.949063Z","shell.execute_reply":"2023-07-26T01:16:49.952888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_id_list[0]","metadata":{"execution":{"iopub.status.busy":"2023-07-25T00:56:32.026575Z","iopub.execute_input":"2023-07-25T00:56:32.027031Z","iopub.status.idle":"2023-07-25T00:56:32.037296Z","shell.execute_reply.started":"2023-07-25T00:56:32.027003Z","shell.execute_reply":"2023-07-25T00:56:32.036306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_id_list[0][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-25T00:56:32.038601Z","iopub.execute_input":"2023-07-25T00:56:32.038965Z","iopub.status.idle":"2023-07-25T00:56:32.04928Z","shell.execute_reply.started":"2023-07-25T00:56:32.038938Z","shell.execute_reply":"2023-07-25T00:56:32.04826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_std(spec):\n    return (spec- np.mean(spec))/np.std(spec)\ndef normalize_image(image):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (image-np.min(image))/(np.max(image)-np.min(image))\n\nrecord_id = train_id_list[0]\nband_08 = []\nband_09 = []\nband_10 = []\nband_11 = []\nband_12 = []\nband_13 = []\nband_14 = []\nband_15 = []\nband_16 = []\nfor i in range(0,8):\n    band_08_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_08.npy'))[...,i].flatten()\n    band_08.append(band_08_i)\n    band_09_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_09.npy'))[...,i].flatten()\n    band_09.append(band_09_i)\n    band_10_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_10.npy'))[...,i].flatten()\n    band_10.append(band_10_i)\n    band_11_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_11.npy'))[...,i].flatten()\n    band_11.append(band_11_i)\n    band_12_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_12.npy'))[...,i].flatten()\n    band_12.append(band_12_i)\n    band_13_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_13.npy'))[...,i].flatten()\n    band_13.append(band_13_i)\n    band_14_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_14.npy'))[...,i].flatten()\n    band_14.append(band_14_i)\n    band_15_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_15.npy'))[...,i].flatten()\n    band_15.append(band_15_i)\n    band_16_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_16.npy'))[...,i].flatten()\n    band_16.append(band_16_i)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:16:56.774063Z","iopub.execute_input":"2023-07-26T01:16:56.774464Z","iopub.status.idle":"2023-07-26T01:16:57.208421Z","shell.execute_reply.started":"2023-07-26T01:16:56.774433Z","shell.execute_reply":"2023-07-26T01:16:57.207273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"band_08 = np.asarray(band_08).transpose()\nband_09 = np.asarray(band_09).transpose()\nband_10 = np.asarray(band_10).transpose()\nband_11 = np.asarray(band_11).transpose()\nband_12 = np.asarray(band_12).transpose()\nband_13 = np.asarray(band_13).transpose()\nband_14 = np.asarray(band_14).transpose()\nband_15 = np.asarray(band_15).transpose()\nband_16 = np.asarray(band_16).transpose()\n","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:17:05.198966Z","iopub.execute_input":"2023-07-26T01:17:05.199372Z","iopub.status.idle":"2023-07-26T01:17:05.22005Z","shell.execute_reply.started":"2023-07-26T01:17:05.199339Z","shell.execute_reply":"2023-07-26T01:17:05.219137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# band_08.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-25T00:56:32.638105Z","iopub.execute_input":"2023-07-25T00:56:32.638412Z","iopub.status.idle":"2023-07-25T00:56:32.644364Z","shell.execute_reply.started":"2023-07-25T00:56:32.638386Z","shell.execute_reply":"2023-07-25T00:56:32.643136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_all = np.clip(np.stack([band_08, band_09, band_10, band_11, band_12, band_13, band_14, band_15, band_16], axis=-1), 0, 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:17:44.0642Z","iopub.execute_input":"2023-07-26T01:17:44.064607Z","iopub.status.idle":"2023-07-26T01:17:44.087802Z","shell.execute_reply.started":"2023-07-26T01:17:44.064576Z","shell.execute_reply":"2023-07-26T01:17:44.086876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_all.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-26T01:17:44.434357Z","iopub.execute_input":"2023-07-26T01:17:44.434768Z","iopub.status.idle":"2023-07-26T01:17:44.441722Z","shell.execute_reply.started":"2023-07-26T01:17:44.434735Z","shell.execute_reply":"2023-07-26T01:17:44.44062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# indexes = np.arange(1)","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:38:12.872982Z","iopub.execute_input":"2023-07-22T00:38:12.873448Z","iopub.status.idle":"2023-07-22T00:38:12.880417Z","shell.execute_reply.started":"2023-07-22T00:38:12.873416Z","shell.execute_reply":"2023-07-22T00:38:12.878913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# indexes","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:38:25.207753Z","iopub.execute_input":"2023-07-22T00:38:25.208182Z","iopub.status.idle":"2023-07-22T00:38:25.216127Z","shell.execute_reply.started":"2023-07-22T00:38:25.20815Z","shell.execute_reply":"2023-07-22T00:38:25.214677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#     def __getitem__(self, index):\n#         'Generate one batch of data'\n#         # Generate indexes of the batch\n#         indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n#         # Find list of IDs\n#         list_IDs_temp = [self.list_IDs[k] for k in indexes]\n#         ##stoped here ZILING    \n#         if self.test:\n#             X = []\n#             for record_id in list_IDs_temp:\n#                 X.append(self.get_image(record_id))\n#             return np.asarray(X)\n#         else:\n#             X = []\n#             y = []\n\n#             for record_id in list_IDs_temp:\n#                 X.append(self.get_image(record_id))\n#                 y.append(np.load(record_id[0] + str(record_id[1]) + '/human_pixel_masks.npy'))\n#             return np.asarray(X), keras.utils.to_categorical(np.asarray(y), num_classes=2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# lat_of_origin, central_meridian = get_lat_lon(record_id, train_metadata_df)\n# lat_lon_values = np.array([[lat_of_origin],[central_meridian]])\n# lat_lon_values_list,append(lat_lon_values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_all.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:20.033571Z","iopub.execute_input":"2023-07-22T00:26:20.034024Z","iopub.status.idle":"2023-07-22T00:26:20.042345Z","shell.execute_reply.started":"2023-07-22T00:26:20.033992Z","shell.execute_reply":"2023-07-22T00:26:20.040786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Build Dataset\n# import datetime\n\n# data_types = {'record_id' : str , 'projection_wkt' : str , 'row_min' : float, 'row_size' :  float , 'col_min' : float , \"col_size\" :float, 'timestamp': datetime.datetime}\n# train_metadata_df = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json\" , dtype=data_types)\n\n# json_train_record_id_list = (train_metadata_df['record_id'].to_list())\n# print(len(json_train_record_id_list))\n\n# # Build Dataset\n\n# val_metadata_df = pd.read_json(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json\" , dtype=data_types)\n\n# json_val_record_id_list = (val_metadata_df['record_id'].to_list())\n# print(len(json_val_record_id_list))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-25T01:23:14.066372Z","iopub.execute_input":"2023-07-25T01:23:14.066824Z","iopub.status.idle":"2023-07-25T01:23:14.344389Z","shell.execute_reply.started":"2023-07-25T01:23:14.066789Z","shell.execute_reply":"2023-07-25T01:23:14.343604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_metadata_df","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:20.660294Z","iopub.execute_input":"2023-07-22T00:26:20.661051Z","iopub.status.idle":"2023-07-22T00:26:20.681743Z","shell.execute_reply.started":"2023-07-22T00:26:20.661012Z","shell.execute_reply":"2023-07-22T00:26:20.680522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_metadata_df['projection_wkt'][0]","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:21.000234Z","iopub.execute_input":"2023-07-22T00:26:21.001523Z","iopub.status.idle":"2023-07-22T00:26:21.011881Z","shell.execute_reply.started":"2023-07-22T00:26:21.00147Z","shell.execute_reply":"2023-07-22T00:26:21.009778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# # Assuming 'train_metadata_df' is your DataFrame\n\n# # Function to extract latitude of origin and central meridian values from WKT string\n# def extract_projection_values(wkt_string):\n#     latitude_of_origin = None\n#     central_meridian = None\n    \n#     # Split the WKT string by comma to separate parameters\n#     params = wkt_string.split(',')\n#     for i, param in enumerate(params):\n#         if 'PARAMETER[\"latitude_of_origin\"' in param:\n#             latitude_of_origin = float(params[i+1].rstrip(']'))\n#         elif 'PARAMETER[\"central_meridian\"' in param:\n#             central_meridian = float(params[i+1].rstrip(']'))\n            \n#     return latitude_of_origin, central_meridian\n\n# # Apply the function to 'projection_wkt' column and create new columns\n# train_metadata_df['latitude_of_origin'], train_metadata_df['central_meridian'] = zip(*train_metadata_df['projection_wkt'].apply(extract_projection_values))\n\n# # Display the updated DataFrame\n# print(train_metadata_df)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:21.421947Z","iopub.execute_input":"2023-07-22T00:26:21.422325Z","iopub.status.idle":"2023-07-22T00:26:21.581553Z","shell.execute_reply.started":"2023-07-22T00:26:21.422297Z","shell.execute_reply":"2023-07-22T00:26:21.580404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # Note 1000 is user enterable\n# partial_train_id_list = train_id_list[0:NUMBER_OF_RECORDS]\n# print(len(partial_train_id_list))","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:22.578588Z","iopub.execute_input":"2023-07-22T00:26:22.57919Z","iopub.status.idle":"2023-07-22T00:26:22.584976Z","shell.execute_reply.started":"2023-07-22T00:26:22.579159Z","shell.execute_reply":"2023-07-22T00:26:22.583843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# partial_train_id_list[0] ","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:23.00372Z","iopub.execute_input":"2023-07-22T00:26:23.004145Z","iopub.status.idle":"2023-07-22T00:26:23.012333Z","shell.execute_reply.started":"2023-07-22T00:26:23.004112Z","shell.execute_reply":"2023-07-22T00:26:23.010764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"328369931132336025","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:23.375195Z","iopub.execute_input":"2023-07-22T00:26:23.375669Z","iopub.status.idle":"2023-07-22T00:26:23.384807Z","shell.execute_reply.started":"2023-07-22T00:26:23.375632Z","shell.execute_reply":"2023-07-22T00:26:23.383232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# class DataGenerator(keras.utils.Sequence):\n#     'Generates data for Keras'\n#     def __init__(self, list_IDs, image_channels=3, batch_size=32, shuffle=True, test=False, kaggle=False, metadata_df = train_metadata_df ):\n#         'Initialization'\n# #         self.images_path = images_path\n#         self.indexes = np.arange(len(list_IDs))\n#         self.batch_size = batch_size\n#         self.list_IDs = list_IDs\n#         self.shuffle = shuffle \n#         self.image_channels = image_channels\n#         self.test = test\n#         self.on_epoch_end()\n#         self.kaggle = kaggle\n#         self.X = None\n\n#     def __len__(self):\n#         'Denotes the number of batches per epoch'\n#         return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n#     def get_lat_lon(record_id, metadata_df):\n#         row = metadata_df.loc[metadata_df['record_id'] == record_id]\n#         latitude_of_origin = row['latitude_of_origin'].values[0]\n#         central_meridian = row['central_meridian'].values[0]\n#         return latitude_of_origin, central_meridian\n\n# #     # New function to load human pixel masks from the provided path\n# #     def load_human_pixel_masks(record_id):\n# #         masks = np.load(record_id[0] + str(record_id[1]) + '/human_pixel_masks.npy')\n# #         # One-hot encode the masks (assuming they are binary)\n# #         masks = np.expand_dims(masks, axis=-1)\n# #         masks = keras.utils.to_categorical(masks, num_classes=2)\n# #         return masks\n    \n#     def __getitem__(self, index):\n#         'Generate one batch of data'\n#         # Generate indexes of the batch\n#         indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n#         # Find list of IDs\n#         list_IDs_temp = [self.list_IDs[k] for k in indexes]\n    \n#         if self.test:\n#             X = []\n#             for record_id in list_IDs_temp:\n#                 X.append(self.get_image(record_id))\n#             return np.asarray(X)\n#         else:\n#             X = []\n#             y = []\n#             masks = []\n#             lat_lon_values_list = []\n#             dynamic_worlds = []\n#             for record_id in list_IDs_temp:\n#                 X.append(self.get_image(record_id))\n#                 y.append(np.load(record_id[0] + str(record_id[1]) + '/human_pixel_masks.npy').flatten())\n#                 lat_of_origin, central_meridian = get_lat_lon(record_id, train_metadata_df)\n#                 lat_lon_values = np.array([[lat_of_origin],[central_meridian]])\n#                 lat_lon_values_list,append(lat_lon_values)\n#                 masks.append(np.full(( 65536,8,9 ), None))\n#                 dynamic_worlds.append((np.ones((65536, 8, 9))*10)) \n                \n#             return np.asarray(X), keras.utils.to_categorical(np.asarray(y), num_classes=2)\n        \n    \n# #     def normalize_range(self, data, bounds):\n# #         \"\"\"Maps data to the range [0, 1].\"\"\"\n# #         return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n# #     def normalize_std(self, spec):\n# #         return (spec- np.mean(spec))/np.std(spec)\n    \n#     def normalize_image(image):\n#         \"\"\"Maps data to the range [0, 1].\"\"\"\n#         return (image-np.min(image))/(np.max(image)-np.min(image))\n\n#     def get_image(self, record_id):\n#         # Gets image_id string with the filder prexix included in the name\n#         # Returns numpy array with image and local path\n#         #if self.kaggle:\n#         band_08 = []\n#         band_09 = []\n#         band_10 = []\n#         band_11 = []\n#         band_12 = []\n#         band_13 = []\n#         band_14 = []\n#         band_15 = []\n#         band_16 = []\n#         for i in range(0,8):\n#             band_08_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_08.npy'))[...,i].flatten()\n#             band_08.append(band_08_i)\n#             band_09_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_09.npy'))[...,i].flatten()\n#             band_09.append(band_09_i)\n#             band_10_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_10.npy'))[...,i].flatten()\n#             band_10 = np.asarray(band_10.append(band_10_i)).transpose()\n#             band_11_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_11.npy'))[...,i].flatten()\n#             band_11.append(band_11_i)\n#             band_12_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_12.npy'))[...,i].flatten()\n#             band_12.append(band_12_i)\n#             band_13_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_13.npy'))[...,i].flatten()\n#             band_13.append(band_13_i)\n#             band_14_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_14.npy'))[...,i].flatten()\n#             band_14.append(band_14_i)\n#             band_15_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_15.npy'))[...,i].flatten()\n#             band_15.append(band_15_i)\n#             band_16_i = normalize_image(np.load(record_id[0] + str(record_id[1]) + '/band_16.npy'))[...,i].flatten()\n#             band_16.append(band_16_i)\n#         band_08 = np.asarray(band_08).transpose()\n#         band_09 = np.asarray(band_09).transpose()\n#         band_10 = np.asarray(band_10).transpose()\n#         band_11 = np.asarray(band_11).transpose()\n#         band_12 = np.asarray(band_12).transpose()\n#         band_13 = np.asarray(band_13).transpose()\n#         band_14 = np.asarray(band_14).transpose()\n#         band_15 = np.asarray(band_15).transpose()\n#         band_16 = np.asarray(band_16).transpose()\n#         image_all = np.clip(np.stack([band_08, band_09, band_10, band_11, band_12, band_13, band_14, band_15, band_16], axis=2), 0, 1)\n#         return image_all\n \n#     def on_epoch_end(self):\n#         'Updates indexes after each epoch'\n#         self.indexes = np.arange(len(self.list_IDs))\n#         if self.shuffle == True:\n#             np.random.shuffle(self.indexes)\n            \n\n# # Custom Loss function\n# def DiceLoss(y_true, y_pred, smooth=1e-6):\n    \n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n    \n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     dice = (2*intersection + smooth) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) + smooth)\n#     return dice\n\n# # custom metric for model\n\n# def jaccard_coef(y_true, y_pred):\n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     final_coef_value = (intersection + 1.0) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) - intersection + 1.0)\n#     return final_coef_value\n\n# def dice_coeff(y_true, y_pred):\n#     smooth = 1.0\n#     y_true_flatten = K.flatten(y_true)\n#     y_pred_flatten = K.flatten(y_pred)\n#     intersection = K.sum(y_true_flatten * y_pred_flatten)\n#     dice_score = (2.0 * intersection + smooth) / (K.sum(y_true_flatten) + K.sum(y_pred_flatten) + smooth)\n#     return dice_score\n\n\n# def dice_loss(y_true, y_pred):\n#     loss = 1 - dice_coeff(y_true, y_pred)\n#     return loss\n\n# def bce_dice_loss(y_true, y_pred):\n#     loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n#     return loss\n\n# # Loss from segmentation models library\n# # TOTAL_LOSS_FACTOR = 5\n# # dice_loss = sm.losses.DiceLoss(class_weights = [0.5,0.5])\n# # focal_loss = sm.losses.CategoricalFocalLoss()\n# # total_loss = dice_loss + (TOTAL_LOSS_FACTOR * focal_loss)\n\n# from tensorflow.python.keras import losses","metadata":{"execution":{"iopub.status.busy":"2023-07-25T00:57:15.02369Z","iopub.execute_input":"2023-07-25T00:57:15.024126Z","iopub.status.idle":"2023-07-25T00:57:16.173476Z","shell.execute_reply.started":"2023-07-25T00:57:15.024089Z","shell.execute_reply":"2023-07-25T00:57:16.17161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.utils.data as data\n# import numpy as np","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Data generator\n# KAGGLE = True\n# BATCH_SIZE = 32\n# IMAGE_CHANNELS = 72\n# full_generator = DataGenerator(partial_train_id_list , image_channels=IMAGE_CHANNELS, batch_size=BATCH_SIZE, kaggle=KAGGLE)\n# #test_generator = DataGenerator(test_id_list, image_channels=IMAGE_CHANNELS, batch_size=1, shuffle=False, test=True, kaggle=KAGGLE)\n# # validation_generator = DataGenerator(validation_id_list, image_channels=IMAGE_CHANNELS, batch_size=1, shuffle=False, test=False, kaggle=KAGGLE)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:26:25.93685Z","iopub.execute_input":"2023-07-22T00:26:25.937242Z","iopub.status.idle":"2023-07-22T00:26:25.943689Z","shell.execute_reply.started":"2023-07-22T00:26:25.937212Z","shell.execute_reply":"2023-07-22T00:26:25.942371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_generator","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:27:12.705273Z","iopub.execute_input":"2023-07-22T00:27:12.705661Z","iopub.status.idle":"2023-07-22T00:27:12.712549Z","shell.execute_reply.started":"2023-07-22T00:27:12.705632Z","shell.execute_reply":"2023-07-22T00:27:12.711668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_data = full_generator.X","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:27:45.097104Z","iopub.execute_input":"2023-07-22T00:27:45.097478Z","iopub.status.idle":"2023-07-22T00:27:45.104731Z","shell.execute_reply.started":"2023-07-22T00:27:45.09745Z","shell.execute_reply":"2023-07-22T00:27:45.102977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# index = 0\n# batch_X, batch_y = full_generator.__getitem__(index)\n# # Now you can access the X attribute\n# X_data = full_generator.X","metadata":{"execution":{"iopub.status.busy":"2023-07-22T00:28:59.727735Z","iopub.execute_input":"2023-07-22T00:28:59.728183Z","iopub.status.idle":"2023-07-22T00:29:00.122773Z","shell.execute_reply.started":"2023-07-22T00:28:59.728151Z","shell.execute_reply":"2023-07-22T00:29:00.121115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # the treesat AI data was collected during the summer,\n# # so we estimate the month to be 6 (July)\n# month = torch.tensor([6] * train_data[0].shape[0]).long()\n\n# dl = DataLoader(\n#     TensorDataset(\n#         train_data[0].float(),  # x\n#         train_data[1].bool(),  # mask\n#         train_data[2].long(),  # dynamic world\n#         train_data[3].float(),  # latlons\n#         month\n#     ),\n#     batch_size=batch_size,\n#     shuffle=False,\n# )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features_list = []\n# for (x, mask, dw, latlons, month) in tqdm(dl):\n#     with torch.no_grad():\n#         encodings = (\n#             pretrained_model.encoder(\n#                 x, dynamic_world=dw, mask=mask, latlons=latlons, month=month\n#             )\n#             .cpu()\n#             .numpy()\n#         )\n#         features_list.append(encodings)\n# features_np = np.concatenate(features_list)","metadata":{},"execution_count":null,"outputs":[]}]}