{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# writing a dataset (16 bit) in npz format for fast dataloading\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nfrom math import floor\n# load the data\n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/'\nTRAIN_DIR = BASE_DIR + 'train'\nTEST_DIR = BASE_DIR + 'test'\nVALIDATION_DIR = BASE_DIR + 'validation'\nOUTDIR = '/kaggle/working/train/'\n!mkdir train\nrecord_ids= sorted(glob.glob(os.path.join(TRAIN_DIR, '*')))\nmean_var = [(233.68251494476192, 49.41609113053796),\n    (242.25141679731962, 84.3667759851496),\n    (250.73854949703414, 129.34094451279213),\n    (274.38815456958827, 386.2778745820613),\n    (255.53700834960705, 172.70252117076805),\n    (276.5770302843285, 430.757200225114),\n    (275.3361689617875, 447.36708595674526),\n    (272.5409457075911, 424.7027156746369),\n    (260.4165092867757, 251.74876866159116)]\n\n# number of standard deviations used for normalization\nnstd = 1\n\ndef norm_arr(arr, mean, std, num_std):\n    return (arr - mean) / (std * num_std)\n\ndef get_image(i):\n    bands = []\n    with open(os.path.join(record_ids[i], 'band_08.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[0][0], mean_var[0][1]**.5, nstd))\n    with open(os.path.join( record_ids[i], 'band_09.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[1][0], mean_var[1][1]**.5, nstd))\n    with open(os.path.join( record_ids[i] , 'band_10.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[2][0], mean_var[2][1]**.5, nstd))\n    with open(os.path.join(record_ids[i], 'band_11.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[3][0], mean_var[3][1]**.5, nstd))\n    with open(os.path.join( record_ids[i], 'band_12.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[4][0], mean_var[4][1]**.5, nstd))\n    with open(os.path.join( record_ids[i] , 'band_13.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[5][0], mean_var[5][1]**.5, nstd))\n    with open(os.path.join(record_ids[i], 'band_14.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[6][0], mean_var[6][1]**.5, nstd))\n    with open(os.path.join( record_ids[i], 'band_15.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[7][0], mean_var[7][1]**.5, nstd))\n    with open(os.path.join( record_ids[i] , 'band_16.npy'), 'rb') as f:\n        bands.append(norm_arr(np.load(f).astype(np.float16), mean_var[8][0], mean_var[8][1]**.5, nstd))\n    with open(os.path.join(record_ids[i] , 'human_pixel_masks.npy'), 'rb') as f:\n        human_pixel_mask = np.load(f).astype(bool)\n    with open(os.path.join(record_ids[i] , 'human_individual_masks.npy'), 'rb') as f:\n        human_individual_mask = np.load(f).astype(bool)\n\n    return np.stack([bands])[0], human_pixel_mask[:,:,0], human_individual_mask[:,:,0,:]\n\nimg0, mask0, indmask0 = get_image(0)\nprint(img0.shape)\nprint(mask0.shape)\nprint(indmask0.shape)\n\nbatch_size = 32\n\ndef batch_num(ind):\n    return int(floor(float(ind)/batch_size))\n\n#num_images = len(record_ids)\nnum_images = 2016\nstart = 0\nprint(len(record_ids))\nbatch_data = np.zeros((batch_size, img0.shape[0], img0.shape[1], img0.shape[2], img0.shape[3]), dtype=img0.dtype)\nbatch_record_ids = np.zeros((batch_size,), dtype=np.int64)\nmasks = np.zeros((batch_size, mask0.shape[0], mask0.shape[1]), dtype=bool)\n\n\n# Loop over all indices in record_ids\nfor i in range(start, start+num_images):\n    if i % batch_size == 0 and i != 0:\n        # Write the batch data to an npy file\n        filename = OUTDIR + f\"batch_{batch_num(i-1)}.npz\"\n        data_dict = {\n            'record_id': batch_record_ids,\n            'image_data': batch_data,\n            'mask': masks\n        }\n        np.savez(filename, **data_dict)\n\n    # Get the image data for the current index\n    d, mask, _ = get_image(i)\n    batch_data[i%batch_size] = d\n    folder_name = os.path.basename(record_ids[i])\n\n    batch_record_ids[i%batch_size] = np.int64(folder_name)\n    masks[i%batch_size] = mask\nlast_ind = start + num_images - 1\nfilename = OUTDIR + f\"batch_{batch_num(last_ind)}.npz\"\ndata_dict = {\n    'record_id': batch_record_ids[0:last_ind%batch_size+1],\n    'image_data': batch_data[0:last_ind%batch_size+1],\n    'mask': masks[0:last_ind%batch_size+1]\n}\nnp.savez(filename, **data_dict)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-28T15:20:19.349672Z","iopub.execute_input":"2023-06-28T15:20:19.350061Z","iopub.status.idle":"2023-06-28T15:20:36.021705Z","shell.execute_reply.started":"2023-06-28T15:20:19.350031Z","shell.execute_reply":"2023-06-28T15:20:36.019904Z"},"trusted":true},"execution_count":null,"outputs":[]}]}