{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"},{"sourceId":8733926,"sourceType":"datasetVersion","datasetId":5242701},{"sourceId":8733976,"sourceType":"datasetVersion","datasetId":5242737},{"sourceId":8741721,"sourceType":"datasetVersion","datasetId":5248518},{"sourceId":8806961,"sourceType":"datasetVersion","datasetId":5296781},{"sourceId":8807039,"sourceType":"datasetVersion","datasetId":5296829},{"sourceId":8807258,"sourceType":"datasetVersion","datasetId":5296968},{"sourceId":8807304,"sourceType":"datasetVersion","datasetId":5296995},{"sourceId":8809531,"sourceType":"datasetVersion","datasetId":5298690},{"sourceId":8810092,"sourceType":"datasetVersion","datasetId":5299125},{"sourceId":8810485,"sourceType":"datasetVersion","datasetId":5299424},{"sourceId":8810544,"sourceType":"datasetVersion","datasetId":5299472},{"sourceId":8815332,"sourceType":"datasetVersion","datasetId":5302775},{"sourceId":8815341,"sourceType":"datasetVersion","datasetId":5302782},{"sourceId":8815501,"sourceType":"datasetVersion","datasetId":5302899},{"sourceId":8816011,"sourceType":"datasetVersion","datasetId":5303253},{"sourceId":8818633,"sourceType":"datasetVersion","datasetId":5305169},{"sourceId":8818655,"sourceType":"datasetVersion","datasetId":5305185},{"sourceId":8818678,"sourceType":"datasetVersion","datasetId":5305202},{"sourceId":8819198,"sourceType":"datasetVersion","datasetId":5305532},{"sourceId":8821400,"sourceType":"datasetVersion","datasetId":5306933},{"sourceId":8821582,"sourceType":"datasetVersion","datasetId":5307037},{"sourceId":8821620,"sourceType":"datasetVersion","datasetId":5307059},{"sourceId":8821842,"sourceType":"datasetVersion","datasetId":5307210},{"sourceId":8821879,"sourceType":"datasetVersion","datasetId":5307228},{"sourceId":8824485,"sourceType":"datasetVersion","datasetId":5309075},{"sourceId":8824508,"sourceType":"datasetVersion","datasetId":5309088},{"sourceId":8825046,"sourceType":"datasetVersion","datasetId":5309455},{"sourceId":8826499,"sourceType":"datasetVersion","datasetId":5310434},{"sourceId":8826804,"sourceType":"datasetVersion","datasetId":5310649},{"sourceId":8826821,"sourceType":"datasetVersion","datasetId":5310659},{"sourceId":8828078,"sourceType":"datasetVersion","datasetId":5311433},{"sourceId":8828372,"sourceType":"datasetVersion","datasetId":5311618},{"sourceId":8828373,"sourceType":"datasetVersion","datasetId":5311619},{"sourceId":8832613,"sourceType":"datasetVersion","datasetId":5314799},{"sourceId":8833036,"sourceType":"datasetVersion","datasetId":5315121},{"sourceId":8833692,"sourceType":"datasetVersion","datasetId":5315601},{"sourceId":8835916,"sourceType":"datasetVersion","datasetId":5317227},{"sourceId":8835944,"sourceType":"datasetVersion","datasetId":5317252},{"sourceId":8836135,"sourceType":"datasetVersion","datasetId":5317373},{"sourceId":8854261,"sourceType":"datasetVersion","datasetId":5329727},{"sourceId":8854696,"sourceType":"datasetVersion","datasetId":5330058}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pickle\nimport os","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T11:01:33.257493Z","iopub.execute_input":"2025-06-10T11:01:33.258478Z","iopub.status.idle":"2025-06-10T11:01:33.340349Z","shell.execute_reply.started":"2025-06-10T11:01:33.258397Z","shell.execute_reply":"2025-06-10T11:01:33.33893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-10T11:01:33.343361Z","iopub.execute_input":"2025-06-10T11:01:33.343863Z","iopub.status.idle":"2025-06-10T11:01:33.356329Z","shell.execute_reply.started":"2025-06-10T11:01:33.343825Z","shell.execute_reply":"2025-06-10T11:01:33.355083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_indices = np.asarray(range(1, 42))\ncombined_path = [(KaggleDatasets().get_gcs_path(f'leap-tfrecs-{idx}-xx')) \n                 for idx in combined_indices]\npickle.dump(combined_path,open('combined_path.p', 'bw'))\ntfrecords_path_hr = combined_path.copy()\ncombined_path","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T11:01:33.357681Z","iopub.execute_input":"2025-06-10T11:01:33.358101Z","iopub.status.idle":"2025-06-10T11:44:13.077485Z","shell.execute_reply.started":"2025-06-10T11:01:33.358064Z","shell.execute_reply":"2025-06-10T11:44:13.076318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_folder='/kaggle/working/input'\ntry:\n    os.mkdir(input_folder)\nexcept Exception as e:\n    print('exception error:')\n    print(e)\ntry:\n    os.mkdir(f'{input_folder}/lens')\nexcept Exception as e:\n    print('exception error:')\n    print(e)\ntry:\n    os.mkdir(f'{input_folder}/lens')\nexcept Exception as e:\n    print('exception error:')\n    print(e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T11:44:13.079653Z","iopub.execute_input":"2025-06-10T11:44:13.080004Z","iopub.status.idle":"2025-06-10T11:44:13.089062Z","shell.execute_reply.started":"2025-06-10T11:44:13.079973Z","shell.execute_reply":"2025-06-10T11:44:13.087793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\ncounter = 0\nall_lens_hr = []\ntffiles_hr = []\n\nfor curr_tfrecords_path in tfrecords_path_hr:\n    folders = !gsutil ls $curr_tfrecords_path\n    folders_num = len(folders)\n    counter+=folders_num\n\n    curr_tfrec_bucket = curr_tfrecords_path.split('-')[1]\n    curr_folder_path = f'{input_folder}/lens/{curr_tfrec_bucket}'\n\n    try:\n        os.mkdir(curr_folder_path)\n    except Exception as e:\n        print('exception error:')\n        print(e)\n\n    for i in range(counter-folders_num, counter):\n        curr_folder_path_2 = f'{curr_folder_path}/{i}'\n        lens_path = f'{curr_tfrecords_path}/{i}/tffiles_len.p'\n        id_path = f'{curr_tfrecords_path}/{i}/tffiles_id.p'\n\n        try:\n            os.mkdir(curr_folder_path_2)\n        except Exception as e:\n            print('exception error:')\n            print(e)\n\n        !gsutil -m cp -r $lens_path $curr_folder_path_2\n        curr_tffiles_len = pickle.load(open(f'{curr_folder_path_2}/tffiles_len.p', 'br'))\n        all_lens_hr.append(curr_tffiles_len)\n\n        !gsutil -m cp -r $id_path $curr_folder_path_2\n        curr_tffiles_id = pickle.load(open(f'{curr_folder_path_2}/tffiles_id.p', 'br'))\n\n        curr_tffiles = [f'{curr_tfrecords_path}/{i}/tfds/{x}' for x in curr_tffiles_id]\n        tffiles_hr.append(curr_tffiles)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T11:44:13.09088Z","iopub.execute_input":"2025-06-10T11:44:13.091238Z","iopub.status.idle":"2025-06-10T12:08:28.828853Z","shell.execute_reply.started":"2025-06-10T11:44:13.091208Z","shell.execute_reply":"2025-06-10T12:08:28.827398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_lens_hr_concat = np.concatenate(all_lens_hr)\nall_tffiles_hr_concat = np.concatenate(tffiles_hr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T12:08:28.830744Z","iopub.execute_input":"2025-06-10T12:08:28.831148Z","iopub.status.idle":"2025-06-10T12:08:28.929702Z","shell.execute_reply.started":"2025-06-10T12:08:28.831102Z","shell.execute_reply":"2025-06-10T12:08:28.928691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(all_lens_hr_concat))\nprint(len(all_tffiles_hr_concat))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T12:08:28.933159Z","iopub.execute_input":"2025-06-10T12:08:28.933509Z","iopub.status.idle":"2025-06-10T12:08:28.939395Z","shell.execute_reply.started":"2025-06-10T12:08:28.933479Z","shell.execute_reply":"2025-06-10T12:08:28.937858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gcp_data = '/kaggle/working/'\n\npickle.dump(all_lens_hr_concat, open(f'{gcp_data}/all_lens_hr_concat_TOTAL.p', 'bw'))\npickle.dump(all_tffiles_hr_concat, open(f'{gcp_data}/all_tffiles_hr_concat_TOTAL.p', 'bw'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-10T12:08:28.940935Z","iopub.execute_input":"2025-06-10T12:08:28.941277Z","iopub.status.idle":"2025-06-10T12:08:29.011225Z","shell.execute_reply.started":"2025-06-10T12:08:28.941249Z","shell.execute_reply":"2025-06-10T12:08:29.009888Z"}},"outputs":[],"execution_count":null}]}