{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"},{"sourceId":8704129,"sourceType":"datasetVersion","datasetId":5220718},{"sourceId":8704381,"sourceType":"datasetVersion","datasetId":5220920},{"sourceId":8704475,"sourceType":"datasetVersion","datasetId":5220993},{"sourceId":8704476,"sourceType":"datasetVersion","datasetId":5220994},{"sourceId":8705052,"sourceType":"datasetVersion","datasetId":5221417},{"sourceId":8705070,"sourceType":"datasetVersion","datasetId":5221431},{"sourceId":8705110,"sourceType":"datasetVersion","datasetId":5221457},{"sourceId":8705113,"sourceType":"datasetVersion","datasetId":5221459},{"sourceId":8733926,"sourceType":"datasetVersion","datasetId":5242701},{"sourceId":8733976,"sourceType":"datasetVersion","datasetId":5242737},{"sourceId":8741721,"sourceType":"datasetVersion","datasetId":5248518}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pickle\nimport os","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:01:41.391493Z","iopub.execute_input":"2025-06-10T11:01:41.392018Z","iopub.status.idle":"2025-06-10T11:01:41.438686Z","shell.execute_reply.started":"2025-06-10T11:01:41.391969Z","shell.execute_reply":"2025-06-10T11:01:41.437229Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-06-10T11:01:41.441786Z","iopub.execute_input":"2025-06-10T11:01:41.442177Z","iopub.status.idle":"2025-06-10T11:01:41.453301Z","shell.execute_reply.started":"2025-06-10T11:01:41.442146Z","shell.execute_reply":"2025-06-10T11:01:41.451886Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_indices = np.asarray(range(1, 9))\ncombined_path = [(KaggleDatasets().get_gcs_path(f'leap-tfrecs-combined-{idx}-s-ds')) \n                 for idx in combined_indices]\npickle.dump(combined_path,open('combined_path.p', 'bw'))\ntfrecords_path_lr = combined_path.copy()\ncombined_path","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:01:41.454584Z","iopub.execute_input":"2025-06-10T11:01:41.454937Z","iopub.status.idle":"2025-06-10T11:09:43.377121Z","shell.execute_reply.started":"2025-06-10T11:01:41.454909Z","shell.execute_reply":"2025-06-10T11:09:43.375901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_indices = np.asarray(range(1, 4))\ncombined_path = [(KaggleDatasets().get_gcs_path(f'leap-tfrecs-{idx}-xx')) \n                 for idx in combined_indices]\npickle.dump(combined_path,open('combined_path_2.p', 'bw'))\ntfrecords_path_hr = combined_path.copy()\ncombined_path","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:09:43.37868Z","iopub.execute_input":"2025-06-10T11:09:43.379062Z","iopub.status.idle":"2025-06-10T11:09:44.246323Z","shell.execute_reply.started":"2025-06-10T11:09:43.379031Z","shell.execute_reply":"2025-06-10T11:09:44.244929Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_folder='/kaggle/working/input'\ntry:\n    os.mkdir(input_folder)\nexcept Exception as e:\n    print('exception error:')\n    print(e)\ntry:\n    os.mkdir(f'{input_folder}/lens')\nexcept Exception as e:\n    print('exception error:')\n    print(e)\ntry:\n    os.mkdir(f'{input_folder}/lens')\nexcept Exception as e:\n    print('exception error:')\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:09:44.247726Z","iopub.execute_input":"2025-06-10T11:09:44.248344Z","iopub.status.idle":"2025-06-10T11:09:44.2593Z","shell.execute_reply.started":"2025-06-10T11:09:44.24831Z","shell.execute_reply":"2025-06-10T11:09:44.258065Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n%%capture\n\nall_lens_lr = []\ntffiles_lr = []\nfor curr_tfrecords_path in tfrecords_path_lr:\n    curr_tfrec_bucket = curr_tfrecords_path.split('-')[1]\n    curr_folder_path = f'{input_folder}/lens/{curr_tfrec_bucket}'\n\n    try:\n        os.mkdir(curr_folder_path)\n    except Exception as e:\n        print('exception error:')\n        print(e)\n\n    for i in range(4):\n        lens_path = f'{curr_tfrecords_path}/{i}/lens.p'\n        curr_folder_path_2 = f'{curr_folder_path}/{i}'\n\n        try:\n            os.mkdir(curr_folder_path_2)\n        except Exception as e:\n            print('exception error:')\n            print(e)\n\n        !gsutil -m cp -r $lens_path $curr_folder_path_2\n        curr_lens = pickle.load(open(f'{curr_folder_path_2}/lens.p', 'br'))\n        all_lens_lr.append(curr_lens)\n\n        curr_tffiles = !gsutil ls $curr_tfrecords_path/$i/tfds\n        tffiles_indices = [int(x.split('/')[-1][:-9]) for x in curr_tffiles]\n        curr_tffiles = [x for _, x in sorted(zip(tffiles_indices, curr_tffiles))]\n        tffiles_lr.append(curr_tffiles)","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:09:44.260517Z","iopub.execute_input":"2025-06-10T11:09:44.260833Z","iopub.status.idle":"2025-06-10T11:13:03.061582Z","shell.execute_reply.started":"2025-06-10T11:09:44.26077Z","shell.execute_reply":"2025-06-10T11:13:03.060121Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\ncounter = 0\nall_lens_hr = []\ntffiles_hr = []\n\nfor curr_tfrecords_path in tfrecords_path_hr:\n    folders = !gsutil ls $curr_tfrecords_path\n    folders_num = len(folders)\n    counter+=folders_num\n\n    curr_tfrec_bucket = curr_tfrecords_path.split('-')[1]\n    curr_folder_path = f'{input_folder}/lens/{curr_tfrec_bucket}'\n\n    try:\n        os.mkdir(curr_folder_path)\n    except Exception as e:\n        print('exception error:')\n        print(e)\n\n    for i in range(counter-folders_num, counter):\n        curr_folder_path_2 = f'{curr_folder_path}/{i}'\n        lens_path = f'{curr_tfrecords_path}/{i}/tffiles_len.p'\n        id_path = f'{curr_tfrecords_path}/{i}/tffiles_id.p'\n\n        try:\n            os.mkdir(curr_folder_path_2)\n        except Exception as e:\n            print('exception error:')\n            print(e)\n\n        !gsutil -m cp -r $lens_path $curr_folder_path_2\n        curr_tffiles_len = pickle.load(open(f'{curr_folder_path_2}/tffiles_len.p', 'br'))\n        all_lens_hr.append(curr_tffiles_len)\n\n        !gsutil -m cp -r $id_path $curr_folder_path_2\n        curr_tffiles_id = pickle.load(open(f'{curr_folder_path_2}/tffiles_id.p', 'br'))\n\n        curr_tffiles = [f'{curr_tfrecords_path}/{i}/tfds/{x}' for x in curr_tffiles_id]\n        tffiles_hr.append(curr_tffiles)","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:13:03.066857Z","iopub.execute_input":"2025-06-10T11:13:03.067309Z","iopub.status.idle":"2025-06-10T11:14:56.958507Z","shell.execute_reply.started":"2025-06-10T11:13:03.067274Z","shell.execute_reply":"2025-06-10T11:14:56.955536Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_lens_lr_concat = np.concatenate(all_lens_lr)\nall_tffiles_lr_concat = np.concatenate(tffiles_lr)\nall_lens_hr_concat = np.concatenate(all_lens_hr)\nall_tffiles_hr_concat = np.concatenate(tffiles_hr)","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:14:56.961154Z","iopub.execute_input":"2025-06-10T11:14:56.961656Z","iopub.status.idle":"2025-06-10T11:14:56.995205Z","shell.execute_reply.started":"2025-06-10T11:14:56.961607Z","shell.execute_reply":"2025-06-10T11:14:56.993928Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(all_lens_lr_concat))\nprint(len(all_tffiles_lr_concat))\nprint(len(all_lens_hr_concat))\nprint(len(all_tffiles_hr_concat))","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:14:56.996642Z","iopub.execute_input":"2025-06-10T11:14:56.996992Z","iopub.status.idle":"2025-06-10T11:14:57.005198Z","shell.execute_reply.started":"2025-06-10T11:14:56.996963Z","shell.execute_reply":"2025-06-10T11:14:57.003498Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gcp_data = '/kaggle/working/'\n\npickle.dump(all_lens_lr_concat, open(f'{gcp_data}/all_lens_lr_concat.p', 'bw'))\npickle.dump(all_tffiles_lr_concat, open(f'{gcp_data}/all_tffiles_lr_concat.p', 'bw'))\n\npickle.dump(all_lens_hr_concat, open(f'{gcp_data}/all_lens_hr_concat.p', 'bw'))\npickle.dump(all_tffiles_hr_concat, open(f'{gcp_data}/all_tffiles_hr_concat.p', 'bw'))","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:14:57.00751Z","iopub.execute_input":"2025-06-10T11:14:57.007945Z","iopub.status.idle":"2025-06-10T11:14:57.036351Z","shell.execute_reply.started":"2025-06-10T11:14:57.007913Z","shell.execute_reply":"2025-06-10T11:14:57.035198Z"},"trusted":true},"outputs":[],"execution_count":null}]}