{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About the Data\nI believe that we should focus on these false color images the labelers were provided in that the model is actually learning which pixels are most likely to be contrails in human (labelers') perception rather than doing rigorous scientific analysis.\n\nThus, the aim of this notebook's preprocess is to get the corresponding false color image (and human_pixel_mask of course) of each sample.","metadata":{}},{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"code","source":"import os\n\ndef get_ids(tar_path):\n    ids = []\n    for img_id in os.listdir(tar_path):\n        ids.append(img_id)\n    print(f\"{len(ids)} samples in {tar_path}\")\n    return ids\n\ndef mkdir(tar_path, name, prt=False):\n    dir_path = os.path.join(tar_path, name)\n    if os.path.exists(dir_path):\n        if prt:\n            print(f\"{dir_path} exists!\")\n        return dir_path\n    os.mkdir(dir_path)\n    if prt:\n        print(f\"{dir_path} created\")\n    return dir_path\n\n# create a directory to save the result\ndata_path = mkdir(\"/kaggle/working/\", \"data\", True)\n\n# only training data is processed here\ntar_path = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train\"\n\n# get ids of each sample\nids = get_ids(tar_path)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:27:15.295488Z","iopub.execute_input":"2023-05-21T17:27:15.295889Z","iopub.status.idle":"2023-05-21T17:27:15.436806Z","shell.execute_reply.started":"2023-05-21T17:27:15.29585Z","shell.execute_reply":"2023-05-21T17:27:15.435816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n\ndef false_color(band11, band14, band15):\n    \"\"\"\n    convert bands to rgb that labelers saw\n    \"\"\"\n    def normalize(band, bounds):\n        return (band - bounds[0]) / (bounds[1] - bounds[0])\n    \n    \n    _T11_BOUNDS = (243, 303)\n    _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n    _TDIFF_BOUNDS = (-4, 2)\n\n    r = normalize(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize(band14, _T11_BOUNDS)\n\n    return np.clip(np.stack([r, g, b], axis=2), 0, 1)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:27:15.438739Z","iopub.execute_input":"2023-05-21T17:27:15.439325Z","iopub.status.idle":"2023-05-21T17:27:15.446933Z","shell.execute_reply.started":"2023-05-21T17:27:15.439279Z","shell.execute_reply":"2023-05-21T17:27:15.446121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main Loop","metadata":{}},{"cell_type":"code","source":"import gc\nfrom tqdm.notebook import tqdm\n\nN_TIMES_BEFORE = 4\n\nLENGTH = 100 # 100 for testing, None means processing all the ids\n\nfor i in tqdm(range(len(ids[:LENGTH]))):\n    img_id = ids[i]\n# for img_id in tqdm(ids[:LENGTH]):\n    sample_path = f\"{tar_path}/{img_id}\"\n    band11 = np.load(f\"{sample_path}/band_11.npy\")[..., N_TIMES_BEFORE]\n    band14 = np.load(f\"{sample_path}/band_14.npy\")[..., N_TIMES_BEFORE]\n    band15 = np.load(f\"{sample_path}/band_15.npy\")[..., N_TIMES_BEFORE]\n    human_pixel_mask = np.load(f\"{sample_path}/human_pixel_masks.npy\")\n    mkdir(data_path, img_id)\n    save_path = f\"{data_path}/{img_id}\"\n    image = false_color(band11, band14, band15)\n    np.save(f\"{save_path}/image.npy\", image.astype('float16'))  # use float16 to save disk space\n    np.save(f\"{save_path}/label.npy\", human_pixel_mask.astype('float16'))\n    del band11, band14, band15, human_pixel_mask, image\n    if i % 100 == 0:\n        gc.collect()  # very necessary","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:27:15.448059Z","iopub.execute_input":"2023-05-21T17:27:15.448489Z","iopub.status.idle":"2023-05-21T17:54:04.774035Z","shell.execute_reply.started":"2023-05-21T17:27:15.448464Z","shell.execute_reply":"2023-05-21T17:54:04.77289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimage_paths = [f\"/kaggle/working/data/{img_id}/image.npy\" for img_id in ids]\nlabel_paths = [f\"/kaggle/working/data/{img_id}/label.npy\" for img_id in ids]\ndf = pd.DataFrame({'image': image_paths, 'label': label_paths})\ndf.to_csv(\"data.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:54:56.049613Z","iopub.execute_input":"2023-05-21T17:54:56.050279Z","iopub.status.idle":"2023-05-21T17:54:56.240681Z","shell.execute_reply.started":"2023-05-21T17:54:56.050242Z","shell.execute_reply":"2023-05-21T17:54:56.239683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check the Result","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom IPython import display\n\ntest_idx = 3  # no contrails in 0,1,2 \ntest_path = f\"{data_path}/{ids[test_idx]}\"\nimage = np.load(f\"{test_path}/image.npy\").astype('float32')  # do not use float16 \nlabel = np.load(f\"{test_path}/label.npy\").astype('float32')\n\nplt.figure(figsize=(10, 6))\nax = plt.subplot(1, 2, 1)\nax.imshow(image)\nax.set_title('Image')\n\nax = plt.subplot(1, 2, 2)\nax.imshow(label, interpolation='none')\nax.set_title('Lable')\n\nprint(f\"{ids[test_idx]}\")","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:55:52.947871Z","iopub.execute_input":"2023-05-21T17:55:52.948287Z","iopub.status.idle":"2023-05-21T17:55:53.772031Z","shell.execute_reply.started":"2023-05-21T17:55:52.948254Z","shell.execute_reply":"2023-05-21T17:55:53.771178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download","metadata":{}},{"cell_type":"code","source":"! zip -r data.zip /kaggle/working/data/ -q  # about 10 min","metadata":{"execution":{"iopub.status.busy":"2023-05-21T17:58:46.49574Z","iopub.execute_input":"2023-05-21T17:58:46.496303Z","iopub.status.idle":"2023-05-21T18:11:42.011999Z","shell.execute_reply.started":"2023-05-21T17:58:46.496261Z","shell.execute_reply":"2023-05-21T18:11:42.010296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink, display\ndisplay(FileLink(\"data.zip\"))  # click the link below to download","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:13:21.091722Z","iopub.execute_input":"2023-05-21T18:13:21.092216Z","iopub.status.idle":"2023-05-21T18:13:21.101595Z","shell.execute_reply.started":"2023-05-21T18:13:21.092168Z","shell.execute_reply":"2023-05-21T18:13:21.100484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch Dataset Example","metadata":{}},{"cell_type":"code","source":"class Dataset:\n    def __init__(self, csv_path):\n        df = pd.read_csv(csv_path)\n        self.images = df['image']\n        self.labels = df['label']\n    def __getitem__(self, idx):\n        image = np.load(self.images[idx])\n        label = np.load(self.labels[idx])\n        return image, label\n    def __len__(self):\n        return len(self.images)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T18:13:26.586617Z","iopub.execute_input":"2023-05-21T18:13:26.587619Z","iopub.status.idle":"2023-05-21T18:13:26.593778Z","shell.execute_reply.started":"2023-05-21T18:13:26.587585Z","shell.execute_reply":"2023-05-21T18:13:26.592877Z"},"trusted":true},"execution_count":null,"outputs":[]}]}