{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook introduces an highly optimized dataset for Contrails. [google-research-contrails-datasets](https://www.kaggle.com/datasets/alabibojesomo/google-research-contrails-datasets). This dataset provides the sample in .h5 and was created using [deepdish](https://deepdish.readthedocs.io/en/latest/io.html)\n\nEnjoy !!!","metadata":{}},{"cell_type":"markdown","source":"# let's install deepdish","metadata":{}},{"cell_type":"code","source":"!pip install deepdish","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-15T14:24:58.959688Z","iopub.execute_input":"2023-05-15T14:24:58.960404Z","iopub.status.idle":"2023-05-15T14:25:15.028912Z","shell.execute_reply.started":"2023-05-15T14:24:58.960357Z","shell.execute_reply":"2023-05-15T14:25:15.027252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# importing packages","metadata":{}},{"cell_type":"code","source":"import deepdish\nimport h5py\nimport pandas as pd\nimport numpy as np\nimport glob\nimport os\nfrom torch.utils.data import Dataset","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:29:23.889917Z","iopub.execute_input":"2023-05-15T14:29:23.890348Z","iopub.status.idle":"2023-05-15T14:29:23.896266Z","shell.execute_reply.started":"2023-05-15T14:29:23.890314Z","shell.execute_reply":"2023-05-15T14:29:23.89501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ContrailDataset(Dataset):\n    def __init__(self, data_paths):\n        self.paths = data_paths\n    \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, index):\n        sample_path = self.paths[index]\n        \n        data = deepdish.io.load(sample_path)  # dict (bands, masks)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:37:21.850539Z","iopub.execute_input":"2023-05-15T14:37:21.850944Z","iopub.status.idle":"2023-05-15T14:37:21.858284Z","shell.execute_reply.started":"2023-05-15T14:37:21.850908Z","shell.execute_reply":"2023-05-15T14:37:21.856971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_root = '/kaggle/input/google-research-contrails-datasets/'\n\n# train_paths = np.sort([os.path.basename(x) for x in glob.glob(f\"{data_root}/train/**\")]).tolist()\n# val_paths = np.sort([os.path.basename(x) for x in glob.glob(f\"{data_root}/validation/**\")]).tolist()\n\ntrain_paths = glob.glob(f\"{data_root}/train/**\")\nval_paths = glob.glob(f\"{data_root}/validation/**\")","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:41:06.626277Z","iopub.execute_input":"2023-05-15T14:41:06.62674Z","iopub.status.idle":"2023-05-15T14:41:06.67948Z","shell.execute_reply.started":"2023-05-15T14:41:06.626704Z","shell.execute_reply":"2023-05-15T14:41:06.678285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_paths), len(val_paths)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:41:08.764193Z","iopub.execute_input":"2023-05-15T14:41:08.764581Z","iopub.status.idle":"2023-05-15T14:41:08.775585Z","shell.execute_reply.started":"2023-05-15T14:41:08.76455Z","shell.execute_reply":"2023-05-15T14:41:08.774254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_paths[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:41:11.067468Z","iopub.execute_input":"2023-05-15T14:41:11.067883Z","iopub.status.idle":"2023-05-15T14:41:11.074831Z","shell.execute_reply.started":"2023-05-15T14:41:11.06784Z","shell.execute_reply":"2023-05-15T14:41:11.073672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dst = ContrailDataset(train_paths)\nval_dst = ContrailDataset(val_paths)\n\nlen(train_dst), len(val_dst)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:41:16.223885Z","iopub.execute_input":"2023-05-15T14:41:16.22431Z","iopub.status.idle":"2023-05-15T14:41:16.233646Z","shell.execute_reply.started":"2023-05-15T14:41:16.22427Z","shell.execute_reply":"2023-05-15T14:41:16.232253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# playing with train data","metadata":{}},{"cell_type":"code","source":"sample_tr = train_dst[0]\n\nfor key, value in sample_tr.items():\n    print(key, value.shape, value.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:47:03.355364Z","iopub.execute_input":"2023-05-15T14:47:03.356518Z","iopub.status.idle":"2023-05-15T14:47:03.511745Z","shell.execute_reply.started":"2023-05-15T14:47:03.356477Z","shell.execute_reply":"2023-05-15T14:47:03.50967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masks_individual = sample_tr['masks'][..., :-1]   # the individual annotators mask\nmasks_pixel = sample_tr['masks'][..., -1]   # the pixel annotation (agreement of individual annotators [see actual dataset/competition page])\n\nmasks_individual.shape, masks_pixel.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:47:12.108686Z","iopub.execute_input":"2023-05-15T14:47:12.109105Z","iopub.status.idle":"2023-05-15T14:47:12.117858Z","shell.execute_reply.started":"2023-05-15T14:47:12.10907Z","shell.execute_reply":"2023-05-15T14:47:12.116605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# playing with validation data","metadata":{}},{"cell_type":"code","source":"sample_val = val_dst[0]\n\nfor key, value in sample_val.items():\n    print(key, value.shape, value.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T14:47:25.251122Z","iopub.execute_input":"2023-05-15T14:47:25.251549Z","iopub.status.idle":"2023-05-15T14:47:25.352507Z","shell.execute_reply.started":"2023-05-15T14:47:25.251516Z","shell.execute_reply":"2023-05-15T14:47:25.351308Z"},"trusted":true},"execution_count":null,"outputs":[]}]}