{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"},{"sourceId":8432232,"sourceType":"datasetVersion","datasetId":4402985},{"sourceId":183559136,"sourceType":"kernelVersion"},{"sourceId":183559247,"sourceType":"kernelVersion"},{"sourceId":183559248,"sourceType":"kernelVersion"},{"sourceId":183559250,"sourceType":"kernelVersion"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nfrom huggingface_hub import hf_hub_download\nimport os\nimport shutil","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-02T22:28:15.3602Z","iopub.execute_input":"2024-06-02T22:28:15.360936Z","iopub.status.idle":"2024-06-02T22:28:17.17654Z","shell.execute_reply.started":"2024-06-02T22:28:15.36089Z","shell.execute_reply":"2024-06-02T22:28:17.17531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfrec_dir_list = ['/kaggle/input/leap-data-to-tfrecs-1-s', '/kaggle/input/leap-data-to-tfrecs-2-s',\n             '/kaggle/input/leap-data-to-tfrecs-3-s', '/kaggle/input/leap-data-to-tfrecs-4-s']\ntfrecs = [[x for x in os.listdir(f'{tfrec_dir}/tfds')] for tfrec_dir in tfrec_dir_list]","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:28:17.17875Z","iopub.execute_input":"2024-06-02T22:28:17.179412Z","iopub.status.idle":"2024-06-02T22:28:17.239581Z","shell.execute_reply.started":"2024-06-02T22:28:17.179368Z","shell.execute_reply":"2024-06-02T22:28:17.238592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"notebook_id = 1","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_folder = '/kaggle/data'\nos.mkdir(save_folder)\nprint(os.listdir('/kaggle'))","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:30:46.497899Z","iopub.execute_input":"2024-06-02T22:30:46.498279Z","iopub.status.idle":"2024-06-02T22:30:46.507265Z","shell.execute_reply.started":"2024-06-02T22:30:46.49825Z","shell.execute_reply":"2024-06-02T22:30:46.505591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(tfrec_dir_list)):\n    os.mkdir(f\"{save_folder}/{i}\")\n    os.mkdir(f\"{save_folder}/{i}/tfds\")\n    lens = pickle.load(open(f'{tfrec_dir_list[i]}/tffiles_len.p', 'br'))\n    pickle.dump(lens, open(f\"{save_folder}/{i}/lens.p\", 'bw'))\n\n    for x in tfrecs[i]:\n        shutil.copy(f'{tfrec_dir_list[i]}/tfds/{x}', f\"{save_folder}/{i}/tfds\")\n    shutil.make_archive(f'{save_folder}/{i}', 'zip', f'{save_folder}/{i}')\n    shutil.rmtree(f\"{save_folder}/{i}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ni = 0\nos.mkdir(f\"{save_folder}/{i}\")\nos.mkdir(f\"{save_folder}/{i}/tfds\")\nlens = pickle.load(open(f'{tfrec_dir_list[i]}/tffiles_len.p', 'br'))\npickle.dump(lens, open(f\"{save_folder}/{i}/lens.p\", 'bw'))\n\nfor x in tfrecs[i]:\n    shutil.copy(f'{tfrec_dir_list[i]}/tfds/{x}', f\"{save_folder}/{i}/tfds\")\nshutil.make_archive(f'{save_folder}/{i}', 'zip', f'{save_folder}/{i}')\nshutil.rmtree(f\"{save_folder}/{i}\")\n'''","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:32:52.674068Z","iopub.execute_input":"2024-06-02T22:32:52.67529Z","iopub.status.idle":"2024-06-02T22:32:52.684575Z","shell.execute_reply.started":"2024-06-02T22:32:52.675251Z","shell.execute_reply":"2024-06-02T22:32:52.682992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nf = open('/kaggle/input/kaggle-json/kaggle.json')\nkaggle_json = json.load(f)\n\nKAGGLE_USERNAME = kaggle_json['username']\nKAGGLE_KEY = kaggle_json['key']\n\n%env KAGGLE_USERNAME=$KAGGLE_USERNAME\n%env KAGGLE_KEY=$KAGGLE_KEY\n\nimport kaggle\n\n!kaggle datasets init -p $save_folder\n\nwith open(f'{save_folder}/dataset-metadata.json', 'r+') as f:\n    data = json.load(f)\n    data['title'] = f'LEAP: TFRecs combined {notebook_id} s ds'\n    data['id'] = f'shlomoron/leap-tfrecs-combined-{notebook_id}-s-ds'\n    data['licenses'][0]['name'] = 'CC0-1.0'\n    f.seek(0)        # <--- should reset file position to the beginning.\n    json.dump(data, f, indent=4)\n    f.truncate()     # remove remaining part\n\n!kaggle datasets create -p $f'{save_folder}' -u --dir-mode zip","metadata":{"execution":{"iopub.status.busy":"2024-06-02T23:42:57.277988Z","iopub.execute_input":"2024-06-02T23:42:57.278423Z","iopub.status.idle":"2024-06-03T00:05:18.940557Z","shell.execute_reply.started":"2024-06-02T23:42:57.278388Z","shell.execute_reply":"2024-06-03T00:05:18.938996Z"},"trusted":true},"execution_count":null,"outputs":[]}]}