{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Data Division Plan\n\n1. **Total Dataset Size**: 10 million rows\n\n2. **Training Data**: First 1 million rows\n    - Divided into two files:\n        - Data1: Rows 1 to 500,000\n        - Data2: Rows 500,001 to 1,000,000\n\n3. **Validation Data**: 100,000 rows\n    - Taken from rows 1,000,001 to 1,100,000\n\n4. **Testing Data**: 300,000 rows\n    - Extracted from rows 1,100,001 to 1,400,000\n\n**Data Format**: Parquet\n","metadata":{}},{"cell_type":"code","source":"# traing data 1\n%%time\nimport pandas as pd\nsample_size = 500000  # Adjust the sample size as needed\ndata1 = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", nrows=sample_size)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving the training data from csv to parquet\ndata1.to_parquet(\"/kaggle//working//data1.parquet\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# traing data 2\nsample_size = 500000  # Adjust the sample size as needed\ndata2 = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", skiprows=500000,nrows=sample_size)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving the training data from csv to parquet\ndata2.to_parquet(\"/kaggle//working//data2.parquet\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation data \nsample_size = 100000  # Adjust the sample size as needed\nvalidation = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", skiprows=1000000,nrows=sample_size)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving the validation data from csv to parquet\nvalidation.to_parquet(\"/kaggle//working//valid.parquet\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data \nsample_size = 300000  # Adjust the sample size as needed\ntest = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv\", skiprows=1100000,nrows=sample_size)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saving the test data from csv to parquet\ntest.to_parquet(\"/kaggle//working//test.parquet\")                        ","metadata":{},"execution_count":null,"outputs":[]}]}