{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-24T08:00:39.983167Z","iopub.execute_input":"2024-05-24T08:00:39.98377Z","iopub.status.idle":"2024-05-24T08:00:41.60227Z","shell.execute_reply.started":"2024-05-24T08:00:39.983624Z","shell.execute_reply":"2024-05-24T08:00:41.601064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc \ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-24T08:00:41.604861Z","iopub.execute_input":"2024-05-24T08:00:41.605368Z","iopub.status.idle":"2024-05-24T08:00:41.685517Z","shell.execute_reply.started":"2024-05-24T08:00:41.605336Z","shell.execute_reply":"2024-05-24T08:00:41.684253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = \"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\"\nsample_ratio = 0.16\nbatch_size = 50000\n\nreader = pl.read_csv_batched(\n    train_path,\n    batch_size=batch_size,\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T08:01:41.317396Z","iopub.execute_input":"2024-05-24T08:01:41.318396Z","iopub.status.idle":"2024-05-24T08:01:41.402259Z","shell.execute_reply.started":"2024-05-24T08:01:41.31835Z","shell.execute_reply":"2024-05-24T08:01:41.400867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batches = reader.next_batches(5)\nsample_data = []\ncounter = 0\n\nwhile batches:\n    batches = pl.concat(batches)\n    batches = batches.sample(fraction=sample_ratio, shuffle=True)\n    sample_data.append(batches)\n    \n    counter += len(batches)\n    print(counter)\n    \n    batches = reader.next_batches(5)","metadata":{"execution":{"iopub.status.busy":"2024-05-24T08:01:44.741583Z","iopub.execute_input":"2024-05-24T08:01:44.742252Z","iopub.status.idle":"2024-05-24T08:23:31.103656Z","shell.execute_reply.started":"2024-05-24T08:01:44.742209Z","shell.execute_reply":"2024-05-24T08:23:31.099865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data = pl.concat(sample_data)\nsample_data = sample_data.sample(1500000, shuffle=True)\nsample_data.write_parquet(\"train_sampled_1_n_half_m.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-05-24T08:23:31.108038Z","iopub.execute_input":"2024-05-24T08:23:31.108558Z","iopub.status.idle":"2024-05-24T08:25:42.305388Z","shell.execute_reply.started":"2024-05-24T08:23:31.108485Z","shell.execute_reply":"2024-05-24T08:25:42.303999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}