{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SuperFast Data Loading\n\noptimized dataset details - https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-1\n\nThis is a optimized version of [google's contrails dataset](https://www.kaggle.com/competitions/google-research-identify-contrails-reduce-global-warming/data).\n\nFeatures of this dataset:\n- about 55% of the original dataset did not contain contrails which have been removed here.\n- each example target pair is saved in a single npz file, instead of 10 npy files, making it load much faster with a lot less cpu usage, since number of IO operation are 10 times less.\n- Examples stored in float16. This saves a lot of space. The loss of information from using float16 instead of float32 (mean absolute reconstruction error) was less than 0.019%.\n- All Examples have been normalized as per channel mean and std.\n\nDisadvantages:\n- Individual human mask have been omitted.\n- The examples without contrails have been omitted\n- Information lost (less than 0.019%) due to the use of float16 \n\nThere are 5 parts:\n[part_1](https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-1) [part_2](https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-2) [part_3](https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-3) [part_4](https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-4) [part_5](https://www.kaggle.com/datasets/soumyadeepkhandual/google-contrails-normalized-float16-part-5)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport os\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-15T06:27:55.174997Z","iopub.execute_input":"2023-05-15T06:27:55.175372Z","iopub.status.idle":"2023-05-15T06:27:55.181517Z","shell.execute_reply.started":"2023-05-15T06:27:55.175346Z","shell.execute_reply":"2023-05-15T06:27:55.180366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get file path list for train and val","metadata":{}},{"cell_type":"code","source":"base_dir_1 = \"/kaggle/input/google-contrails-normalized-float16-part-1/contrail_f16_part_1\"\nbase_dir_2 = \"/kaggle/input/google-contrails-normalized-float16-part-2/contrail_f16_part_2\"\nbase_dir_3 = \"/kaggle/input/google-contrails-normalized-float16-part-3/contrail_f16_part_3\"\nbase_dir_4 = \"/kaggle/input/google-contrails-normalized-float16-part-4/contrail_f16_part_4\"\nbase_dir_5 = \"/kaggle/input/google-contrails-normalized-float16-part-5/contrail_f16_part_5\"\n\nrecord_id_paths = []\nrecord_ids_only = []\nfor base_dir in [base_dir_1,\n                 base_dir_2,\n                 base_dir_3,\n                 base_dir_4,\n                 base_dir_5\n                ]:\n    record_ids = os.listdir(base_dir)\n    for record_id in record_ids:\n        record_id_paths.append(os.path.join(base_dir,record_id)) \n        record_ids_only.append(str(record_id)[:-4])\n\ntrain_ids_non_empty = np.load(\"/kaggle/input/google-contrails-normalized-float16-part-3/train_ids_non_empty.npy\")\nval_ids_non_empty = np.load(\"/kaggle/input/google-contrails-normalized-float16-part-3/val_ids_non_empty.npy\")\n\nval_ids_non_empty_paths = []\ntrain_ids_non_empty_paths = []\nfor record_id, path in zip(record_ids_only, record_id_paths):\n    if record_id in train_ids_non_empty:\n        train_ids_non_empty_paths.append(path)\n    elif record_id in val_ids_non_empty:\n        val_ids_non_empty_paths.append(path)\n    else:\n        print(\"Whay?! how?!!\")\n\nlen(record_id_paths)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T06:27:58.560061Z","iopub.execute_input":"2023-05-15T06:27:58.560428Z","iopub.status.idle":"2023-05-15T06:27:59.327815Z","shell.execute_reply.started":"2023-05-15T06:27:58.560402Z","shell.execute_reply":"2023-05-15T06:27:59.3267Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dataset return x of shape (8,9,256,256) : (time,channel,H,W)\nclass FastContrails(Dataset):\n    def __init__(self, record_id_paths):\n        self.record_id_paths = record_id_paths\n        self.length = len(self.record_id_paths)\n\n    def __getitem__(self, idx):\n        data = np.load(self.record_id_paths[idx])\n        return {\"image\":data[\"x\"], \"mask\":data[\"y\"]}\n    \n    def __len__(self):\n        return self.length","metadata":{"execution":{"iopub.status.busy":"2023-05-15T06:27:59.531213Z","iopub.execute_input":"2023-05-15T06:27:59.531613Z","iopub.status.idle":"2023-05-15T06:27:59.538049Z","shell.execute_reply.started":"2023-05-15T06:27:59.531579Z","shell.execute_reply":"2023-05-15T06:27:59.536785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = FastContrails(train_ids_non_empty_paths)\nval_dataset = FastContrails(val_ids_non_empty_paths)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=32, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T06:30:08.355158Z","iopub.execute_input":"2023-05-15T06:30:08.355548Z","iopub.status.idle":"2023-05-15T06:30:08.362291Z","shell.execute_reply.started":"2023-05-15T06:30:08.355514Z","shell.execute_reply":"2023-05-15T06:30:08.361201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data in tqdm(train_loader):\n    x = data[\"image\"]\n    y = data[\"mask\"]\n#     print(x.shape, y.shape, x.dtype, y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T06:30:09.42371Z","iopub.execute_input":"2023-05-15T06:30:09.424093Z","iopub.status.idle":"2023-05-15T06:33:55.965754Z","shell.execute_reply.started":"2023-05-15T06:30:09.424064Z","shell.execute_reply":"2023-05-15T06:33:55.96354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data in tqdm(val_loader):\n    x = data[\"image\"]\n    y = data[\"mask\"]\n#     print(x.shape, y.shape, x.dtype, y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-05-15T06:33:56.009208Z","iopub.execute_input":"2023-05-15T06:33:56.009575Z","iopub.status.idle":"2023-05-15T06:34:14.971857Z","shell.execute_reply.started":"2023-05-15T06:33:56.009532Z","shell.execute_reply":"2023-05-15T06:34:14.970697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}