{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":179412149,"sourceType":"kernelVersion"},{"sourceId":179412956,"sourceType":"kernelVersion"},{"sourceId":179413671,"sourceType":"kernelVersion"},{"sourceId":179413908,"sourceType":"kernelVersion"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nimport gc\nimport json\nfrom tqdm import tqdm\n\n\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom warnings import simplefilter\nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nsubmission = True\n\nif submission:\n    divice = 'gpu'\nelse:\n    divice = 'cpu'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-31T06:06:35.354387Z","iopub.execute_input":"2024-05-31T06:06:35.354827Z","iopub.status.idle":"2024-05-31T06:06:39.746502Z","shell.execute_reply.started":"2024-05-31T06:06:35.354791Z","shell.execute_reply":"2024-05-31T06:06:39.744959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file1 = \"/kaggle/input/convert-original-dataset-read-pl/file1.parquet\"\nfile2 = \"/kaggle/input/convert-original-dataset-read-pl2/file2.parquet\"\nfile3 = \"/kaggle/input/fork-of-convert-original-dataset-read-pl3/file3.parquet\"\nfile4 = \"/kaggle/input/fork-of-convert-original-dataset-read-pl4/file4.parquet\"","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.749232Z","iopub.execute_input":"2024-05-31T06:06:39.749959Z","iopub.status.idle":"2024-05-31T06:06:39.756255Z","shell.execute_reply.started":"2024-05-31T06:06:39.749913Z","shell.execute_reply":"2024-05-31T06:06:39.754903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fileList = [file1, file2, file3, file4]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.763048Z","iopub.execute_input":"2024-05-31T06:06:39.763796Z","iopub.status.idle":"2024-05-31T06:06:39.772853Z","shell.execute_reply.started":"2024-05-31T06:06:39.763754Z","shell.execute_reply":"2024-05-31T06:06:39.771359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vertical_features = { 'state_t' : [0,59], 'state_q0001' : [0,59], 'state_q0002' : [0,59],\n                    'state_q0003' : [0,59] , 'state_u' : [0,59] , 'state_v' : [0,59]\n                   #  ,\n                  #'pbuf_ozone' : 60 , 'pbuf_CH4' : 27 , 'pbuf_N2O' : 27\n                    }\n\n\nsurface_features = { 'state_ps' : 1, 'pbuf_SOLIN' : 1, 'pbuf_LHFLX' : 1,\n                    'pbuf_SHFLX' : 1 , 'pbuf_TAUX' : 1 , 'pbuf_TAUY' : 1,\n                  'pbuf_COSZRS' : 1 , 'cam_in_ALDIF' : 1 , 'cam_in_ALDIR' : 1, \n                   'cam_in_ASDIF' : 1 , 'cam_in_ASDIR' : 1 , 'cam_in_LWUP' : 1,\n                   'cam_in_ICEFRAC' : 1 , 'cam_in_LANDFRAC' : 1 , 'cam_in_OCNFRAC' : 1,\n                   'cam_in_SNOWHLAND' : 1\n                    #, 'state_t_59' : 1, 'state_u_59' : 1, 'state_v_59' : 1,\n                   #'state_q0003_59' : 1, 'state_q0002_59' : 1, 'state_q0001_59' : 1\n                   }\n\n\nvertical_targets = { 'ptend_t' : [0,59], 'ptend_q0001' : [12,59], 'ptend_q0002' : [15,59],\n                    'ptend_q0003' : [12,59] , 'ptend_u' : [12,59] , 'ptend_v' : [12,59]}\n\n\nsurface_targets = { 'cam_out_NETSW' : 1, 'cam_out_FLWDS' : 1, 'cam_out_PRECSC' : 1,\n                    'cam_out_PRECC' : 1 , 'cam_out_SOLS' : 1 , 'cam_out_SOLL' : 1,\n                  'cam_out_SOLSD' : 1 , 'cam_out_SOLLD' : 1}","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.774607Z","iopub.execute_input":"2024-05-31T06:06:39.775517Z","iopub.status.idle":"2024-05-31T06:06:39.790429Z","shell.execute_reply.started":"2024-05-31T06:06:39.775474Z","shell.execute_reply":"2024-05-31T06:06:39.788976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"featureList = [\n    'state_t_0', 'state_t_1', 'state_t_2', 'state_t_3', 'state_t_4', 'state_t_5', 'state_t_6', 'state_t_7', 'state_t_8', 'state_t_9', 'state_t_10', 'state_t_11', 'state_t_12', 'state_t_13', 'state_t_14', 'state_t_15', 'state_t_16', 'state_t_17', 'state_t_18', 'state_t_19', 'state_t_20', 'state_t_21', 'state_t_22', 'state_t_23', 'state_t_24', 'state_t_25', 'state_t_26', 'state_t_27', 'state_t_28', 'state_t_29', 'state_t_30', 'state_t_31', 'state_t_32', 'state_t_33', 'state_t_34', 'state_t_35', 'state_t_36', 'state_t_37', 'state_t_38', 'state_t_39', 'state_t_40', 'state_t_41', 'state_t_42', 'state_t_43', 'state_t_44', 'state_t_45', 'state_t_46', 'state_t_47', 'state_t_48', 'state_t_49', 'state_t_50', 'state_t_51', 'state_t_52', 'state_t_53', 'state_t_54', 'state_t_55', 'state_t_56', 'state_t_57', 'state_t_58', 'state_t_59',\n    'state_q0001_0', 'state_q0001_1', 'state_q0001_2', 'state_q0001_3', 'state_q0001_4', 'state_q0001_5', 'state_q0001_6', 'state_q0001_7', 'state_q0001_8', 'state_q0001_9', 'state_q0001_10', 'state_q0001_11', 'state_q0001_12', 'state_q0001_13', 'state_q0001_14', 'state_q0001_15', 'state_q0001_16', 'state_q0001_17', 'state_q0001_18', 'state_q0001_19', 'state_q0001_20', 'state_q0001_21', 'state_q0001_22', 'state_q0001_23', 'state_q0001_24', 'state_q0001_25', 'state_q0001_26', 'state_q0001_27', 'state_q0001_28', 'state_q0001_29', 'state_q0001_30', 'state_q0001_31', 'state_q0001_32', 'state_q0001_33', 'state_q0001_34', 'state_q0001_35', 'state_q0001_36', 'state_q0001_37', 'state_q0001_38', 'state_q0001_39', 'state_q0001_40', 'state_q0001_41', 'state_q0001_42', 'state_q0001_43', 'state_q0001_44', 'state_q0001_45', 'state_q0001_46', 'state_q0001_47', 'state_q0001_48', 'state_q0001_49', 'state_q0001_50', 'state_q0001_51', 'state_q0001_52', 'state_q0001_53', 'state_q0001_54', 'state_q0001_55', 'state_q0001_56', 'state_q0001_57', 'state_q0001_58', 'state_q0001_59',\n    'state_q0002_0', 'state_q0002_1', 'state_q0002_2', 'state_q0002_3', 'state_q0002_4', 'state_q0002_5', 'state_q0002_6', 'state_q0002_7', 'state_q0002_8', 'state_q0002_9', 'state_q0002_10', 'state_q0002_11', 'state_q0002_12', 'state_q0002_13', 'state_q0002_14', 'state_q0002_15', 'state_q0002_16', 'state_q0002_17', 'state_q0002_18', 'state_q0002_19', 'state_q0002_20', 'state_q0002_21', 'state_q0002_22', 'state_q0002_23', 'state_q0002_24', 'state_q0002_25', 'state_q0002_26', 'state_q0002_27', 'state_q0002_28', 'state_q0002_29', 'state_q0002_30', 'state_q0002_31', 'state_q0002_32', 'state_q0002_33', 'state_q0002_34', 'state_q0002_35', 'state_q0002_36', 'state_q0002_37', 'state_q0002_38', 'state_q0002_39', 'state_q0002_40', 'state_q0002_41', 'state_q0002_42', 'state_q0002_43', 'state_q0002_44', 'state_q0002_45', 'state_q0002_46', 'state_q0002_47', 'state_q0002_48', 'state_q0002_49', 'state_q0002_50', 'state_q0002_51', 'state_q0002_52', 'state_q0002_53', 'state_q0002_54', 'state_q0002_55', 'state_q0002_56', 'state_q0002_57', 'state_q0002_58', 'state_q0002_59',\n    'state_q0003_0', 'state_q0003_1', 'state_q0003_2', 'state_q0003_3', 'state_q0003_4', 'state_q0003_5', 'state_q0003_6', 'state_q0003_7', 'state_q0003_8', 'state_q0003_9', 'state_q0003_10', 'state_q0003_11', 'state_q0003_12', 'state_q0003_13', 'state_q0003_14', 'state_q0003_15', 'state_q0003_16', 'state_q0003_17', 'state_q0003_18', 'state_q0003_19', 'state_q0003_20', 'state_q0003_21', 'state_q0003_22', 'state_q0003_23', 'state_q0003_24', 'state_q0003_25', 'state_q0003_26', 'state_q0003_27', 'state_q0003_28', 'state_q0003_29', 'state_q0003_30', 'state_q0003_31', 'state_q0003_32', 'state_q0003_33', 'state_q0003_34', 'state_q0003_35', 'state_q0003_36', 'state_q0003_37', 'state_q0003_38', 'state_q0003_39', 'state_q0003_40', 'state_q0003_41', 'state_q0003_42', 'state_q0003_43', 'state_q0003_44', 'state_q0003_45', 'state_q0003_46', 'state_q0003_47', 'state_q0003_48', 'state_q0003_49', 'state_q0003_50', 'state_q0003_51', 'state_q0003_52', 'state_q0003_53', 'state_q0003_54', 'state_q0003_55', 'state_q0003_56', 'state_q0003_57', 'state_q0003_58', 'state_q0003_59',\n    'state_u_0', 'state_u_1', 'state_u_2', 'state_u_3', 'state_u_4', 'state_u_5', 'state_u_6', 'state_u_7', 'state_u_8', 'state_u_9', 'state_u_10', 'state_u_11', 'state_u_12', 'state_u_13', 'state_u_14', 'state_u_15', 'state_u_16', 'state_u_17', 'state_u_18', 'state_u_19', 'state_u_20', 'state_u_21', 'state_u_22', 'state_u_23', 'state_u_24', 'state_u_25', 'state_u_26', 'state_u_27', 'state_u_28', 'state_u_29', 'state_u_30', 'state_u_31', 'state_u_32', 'state_u_33', 'state_u_34', 'state_u_35', 'state_u_36', 'state_u_37', 'state_u_38', 'state_u_39', 'state_u_40', 'state_u_41', 'state_u_42', 'state_u_43', 'state_u_44', 'state_u_45', 'state_u_46', 'state_u_47', 'state_u_48', 'state_u_49', 'state_u_50', 'state_u_51', 'state_u_52', 'state_u_53', 'state_u_54', 'state_u_55', 'state_u_56', 'state_u_57', 'state_u_58', 'state_u_59',\n    'state_v_0', 'state_v_1', 'state_v_2', 'state_v_3', 'state_v_4', 'state_v_5', 'state_v_6', 'state_v_7', 'state_v_8', 'state_v_9', 'state_v_10', 'state_v_11', 'state_v_12', 'state_v_13', 'state_v_14', 'state_v_15', 'state_v_16', 'state_v_17', 'state_v_18', 'state_v_19', 'state_v_20', 'state_v_21', 'state_v_22', 'state_v_23', 'state_v_24', 'state_v_25', 'state_v_26', 'state_v_27', 'state_v_28', 'state_v_29', 'state_v_30', 'state_v_31', 'state_v_32', 'state_v_33', 'state_v_34', 'state_v_35', 'state_v_36', 'state_v_37', 'state_v_38', 'state_v_39', 'state_v_40', 'state_v_41', 'state_v_42', 'state_v_43', 'state_v_44', 'state_v_45', 'state_v_46', 'state_v_47', 'state_v_48', 'state_v_49', 'state_v_50', 'state_v_51', 'state_v_52', 'state_v_53', 'state_v_54', 'state_v_55', 'state_v_56', 'state_v_57', 'state_v_58', 'state_v_59',\n    'state_ps', 'pbuf_SOLIN', 'pbuf_LHFLX', 'pbuf_SHFLX', 'pbuf_TAUX', 'pbuf_TAUY', 'pbuf_COSZRS', 'cam_in_ALDIF', 'cam_in_ALDIR', 'cam_in_ASDIF', 'cam_in_ASDIR', 'cam_in_LWUP', 'cam_in_ICEFRAC', 'cam_in_LANDFRAC', 'cam_in_OCNFRAC', 'cam_in_SNOWHLAND',\n    'pbuf_ozone_0', 'pbuf_ozone_1', 'pbuf_ozone_2', 'pbuf_ozone_3', 'pbuf_ozone_4', 'pbuf_ozone_5', 'pbuf_ozone_6', 'pbuf_ozone_7', 'pbuf_ozone_8', 'pbuf_ozone_9', 'pbuf_ozone_10', 'pbuf_ozone_11', 'pbuf_ozone_12', 'pbuf_ozone_13', 'pbuf_ozone_14', 'pbuf_ozone_15', 'pbuf_ozone_16', 'pbuf_ozone_17', 'pbuf_ozone_18', 'pbuf_ozone_19', 'pbuf_ozone_20', 'pbuf_ozone_21', 'pbuf_ozone_22', 'pbuf_ozone_23', 'pbuf_ozone_24', 'pbuf_ozone_25', 'pbuf_ozone_26', 'pbuf_ozone_27', 'pbuf_ozone_28', 'pbuf_ozone_29', 'pbuf_ozone_30', 'pbuf_ozone_31', 'pbuf_ozone_32', 'pbuf_ozone_33', 'pbuf_ozone_34', 'pbuf_ozone_35', 'pbuf_ozone_36', 'pbuf_ozone_37', 'pbuf_ozone_38', 'pbuf_ozone_39', 'pbuf_ozone_40', 'pbuf_ozone_41', 'pbuf_ozone_42', 'pbuf_ozone_43', 'pbuf_ozone_44', 'pbuf_ozone_45', 'pbuf_ozone_46', 'pbuf_ozone_47', 'pbuf_ozone_48', 'pbuf_ozone_49', 'pbuf_ozone_50', 'pbuf_ozone_51', 'pbuf_ozone_52', 'pbuf_ozone_53', 'pbuf_ozone_54', 'pbuf_ozone_55', 'pbuf_ozone_56', 'pbuf_ozone_57', 'pbuf_ozone_58', 'pbuf_ozone_59',\n    'pbuf_CH4_0', 'pbuf_CH4_1', 'pbuf_CH4_2', 'pbuf_CH4_3', 'pbuf_CH4_4', 'pbuf_CH4_5', 'pbuf_CH4_6', 'pbuf_CH4_7', 'pbuf_CH4_8', 'pbuf_CH4_9', 'pbuf_CH4_10', 'pbuf_CH4_11', 'pbuf_CH4_12', 'pbuf_CH4_13', 'pbuf_CH4_14', 'pbuf_CH4_15', 'pbuf_CH4_16', 'pbuf_CH4_17', 'pbuf_CH4_18', 'pbuf_CH4_19', 'pbuf_CH4_20', 'pbuf_CH4_21', 'pbuf_CH4_22', 'pbuf_CH4_23', 'pbuf_CH4_24', 'pbuf_CH4_25', 'pbuf_CH4_26',\n    'pbuf_N2O_0', 'pbuf_N2O_1', 'pbuf_N2O_2', 'pbuf_N2O_3', 'pbuf_N2O_4', 'pbuf_N2O_5', 'pbuf_N2O_6', 'pbuf_N2O_7', 'pbuf_N2O_8', 'pbuf_N2O_9', 'pbuf_N2O_10', 'pbuf_N2O_11', 'pbuf_N2O_12', 'pbuf_N2O_13', 'pbuf_N2O_14', 'pbuf_N2O_15', 'pbuf_N2O_16', 'pbuf_N2O_17', 'pbuf_N2O_18', 'pbuf_N2O_19', 'pbuf_N2O_20', 'pbuf_N2O_21', 'pbuf_N2O_22', 'pbuf_N2O_23', 'pbuf_N2O_24', 'pbuf_N2O_25', 'pbuf_N2O_26',\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.792256Z","iopub.execute_input":"2024-05-31T06:06:39.793344Z","iopub.status.idle":"2024-05-31T06:06:39.830287Z","shell.execute_reply.started":"2024-05-31T06:06:39.793275Z","shell.execute_reply":"2024-05-31T06:06:39.828784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_gradient(feature_name,df,featureList=None):\n    for i in range(vertical_features[feature_name][0],vertical_features[feature_name][1]+1):\n        name = 'grad_'+feature_name+'_'+str(i)\n        if featureList != None:\n            featureList.append(name)\n        if i == 0:\n            df[name] = df[feature_name+'_'+str(1)] - df[feature_name+'_'+str(0)]\n        elif i == vertical_features[feature_name][1]:\n            df[name] = df[feature_name+'_'+str(i)] - df[feature_name+'_'+str(i-1)]\n        else:\n            df[name] = 0.5 * (df[feature_name+'_'+str(i+1)] - df[feature_name+'_'+str(i-1)])\n            \n    vertical_features['grad_'+feature_name] = vertical_features[feature_name]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.831808Z","iopub.execute_input":"2024-05-31T06:06:39.83266Z","iopub.status.idle":"2024-05-31T06:06:39.848704Z","shell.execute_reply.started":"2024-05-31T06:06:39.832623Z","shell.execute_reply":"2024-05-31T06:06:39.847624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_mean(feature_name,df,featureList=None):\n    name = 'mean_'+feature_name\n    df[name] = df[feature_name+'_'+str(vertical_features[feature_name][0])]\n    j = 1\n    for i in range(vertical_features[feature_name][0]+1,vertical_features[feature_name][1]+1):\n        df[name] = df[name] + df[feature_name+'_'+str(i)]\n        j = j + 1\n    df[name] = df[name] / j\n    if featureList != None:\n            featureList.append(name)\n    surface_features[name] = 1","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.850304Z","iopub.execute_input":"2024-05-31T06:06:39.851258Z","iopub.status.idle":"2024-05-31T06:06:39.865626Z","shell.execute_reply.started":"2024-05-31T06:06:39.851213Z","shell.execute_reply":"2024-05-31T06:06:39.864258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_std(feature_name,df,featureList=None):\n    name = 'std_'+feature_name\n    df_temp = df[feature_name+'_'+str(vertical_features[feature_name][0])]\n    j = 1\n    for i in range(vertical_features[feature_name][0]+1,vertical_features[feature_name][1]+1):\n        df_temp = df_temp + df[feature_name+'_'+str(i)]\n        j = j + 1\n    df_temp = df_temp / j\n    df[name] = (df_temp - df[feature_name+'_'+str(vertical_features[feature_name][0])])**2\n    j = 1\n    for i in range(vertical_features[feature_name][0] +1,vertical_features[feature_name][1]+1):\n        df[name] = df[name] +  (df_temp - df[feature_name+'_'+str(i)])**2\n        j = j + 1\n    df[name] = (df[name] / j )**0.5\n    \n    if featureList != None:\n            featureList.append(name)\n    surface_features[name] = 1","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.866907Z","iopub.execute_input":"2024-05-31T06:06:39.867353Z","iopub.status.idle":"2024-05-31T06:06:39.88173Z","shell.execute_reply.started":"2024-05-31T06:06:39.867317Z","shell.execute_reply":"2024-05-31T06:06:39.880036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_kinetic_energy(feature_name,df,featureList=None):\n    for i in range(vertical_features[feature_name][0],vertical_features[feature_name][1]+1):\n        name = 'kappa_'+str(i)\n        if featureList != None:\n            featureList.append(name)\n        df[name] = 0.5 * (df['state_u_'+str(i)]**2 + df['state_v_'+str(i)]**2)\n            \n    vertical_features['kappa'] = vertical_features[feature_name]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.887429Z","iopub.execute_input":"2024-05-31T06:06:39.887901Z","iopub.status.idle":"2024-05-31T06:06:39.898213Z","shell.execute_reply.started":"2024-05-31T06:06:39.887861Z","shell.execute_reply":"2024-05-31T06:06:39.896468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_virtual_temperature(df,featureList=None):\n    feature_name1 = 'state_t'\n    feature_name2 = 'state_q0001'\n    for i in range(vertical_features[feature_name1][0],vertical_features[feature_name1][1]+1):\n        name = 'state_tv_'+str(i)\n        if featureList != None:\n            featureList.append(name)\n        df[name] =  (1.0 + 0.61 * df['state_q0001_'+str(i)]) * df['state_t_'+str(i)]\n            \n    vertical_features['state_tv'] = vertical_features[feature_name1]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.900054Z","iopub.execute_input":"2024-05-31T06:06:39.900532Z","iopub.status.idle":"2024-05-31T06:06:39.914134Z","shell.execute_reply.started":"2024-05-31T06:06:39.900493Z","shell.execute_reply":"2024-05-31T06:06:39.912526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def insert_total_water(df,featureList=None):\n    feature_name1 = 'state_q0001'\n    feature_name2 = 'state_q0002'\n    feature_name3 = 'state_q0003'\n    for i in range(vertical_features[feature_name1][0],vertical_features[feature_name1][1]+1):\n        name = 'state_total_water_'+str(i)\n        if featureList != None:\n            featureList.append(name)\n        df[name] = df[feature_name1+'_'+str(i)] + df[feature_name2+'_'+str(i)] + df[feature_name3+'_'+str(i)]\n            \n    vertical_features['state_total_water'] = vertical_features[feature_name1]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.915424Z","iopub.execute_input":"2024-05-31T06:06:39.915868Z","iopub.status.idle":"2024-05-31T06:06:39.927496Z","shell.execute_reply.started":"2024-05-31T06:06:39.915826Z","shell.execute_reply":"2024-05-31T06:06:39.926111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_target_relation(vertical_features,surface_features,vertical_targets,surface_targets,connections):\n    feature_to_target_map = {}\n    \n    for target in surface_targets:\n        feature_list = []\n        for surface_feature in surface_features:\n            feature_list.append(surface_feature)\n            \n        for vertical_feature in vertical_features:\n            for i in range(59,59-connections,-1):\n                if i < vertical_features[vertical_feature][1]:\n                    feature_name = vertical_feature+'_'+str(i)\n                    if feature_name not in feature_list:\n                        feature_list.append(vertical_feature+'_'+str(i))\n        \n        feature_to_target_map[target] = feature_list\n        \n    for target in vertical_targets:\n        for i in range(vertical_targets[target][0],vertical_targets[target][1]+1):\n            feature_list = []\n            target_string = target+'_'+str(i)\n            #if i > 59 - connections:\n            for surface_feature in surface_features:\n                feature_list.append(surface_feature)\n                    \n            for vertical_feature in vertical_features:\n                for j in range(vertical_targets[target][0],vertical_features[vertical_feature][1]+1):\n                    feature_string = vertical_feature+'_'+str(j)\n                    if np.abs(i-j) < connections:\n                        if feature_string not in feature_list:\n                            feature_list.append(feature_string)\n                        \n                \n            \n            feature_to_target_map[target_string] = feature_list\n                \n    \n  #  print(feature_to_target_map)\n    \n    return feature_to_target_map","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.929108Z","iopub.execute_input":"2024-05-31T06:06:39.929579Z","iopub.status.idle":"2024-05-31T06:06:39.945152Z","shell.execute_reply.started":"2024-05-31T06:06:39.929539Z","shell.execute_reply":"2024-05-31T06:06:39.94389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.947286Z","iopub.execute_input":"2024-05-31T06:06:39.947756Z","iopub.status.idle":"2024-05-31T06:06:39.961535Z","shell.execute_reply.started":"2024-05-31T06:06:39.947714Z","shell.execute_reply":"2024-05-31T06:06:39.960365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pl.read_csv(DATA_PATH + \"leap-atmospheric-physics-ai-climsim/train.csv\", n_rows=2_500_000)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.963963Z","iopub.execute_input":"2024-05-31T06:06:39.964684Z","iopub.status.idle":"2024-05-31T06:06:39.973484Z","shell.execute_reply.started":"2024-05-31T06:06:39.964634Z","shell.execute_reply":"2024-05-31T06:06:39.972168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_lines = 625_000","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:39.975102Z","iopub.execute_input":"2024-05-31T06:06:39.97557Z","iopub.status.idle":"2024-05-31T06:06:39.986828Z","shell.execute_reply.started":"2024-05-31T06:06:39.975528Z","shell.execute_reply":"2024-05-31T06:06:39.985436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#files_read = []\n\nprint(file1)\ntrain = pl.read_parquet(file1, \n                        n_rows=n_lines,row_index_offset=int(3*n_lines))\nprint(file2)\ntrain.extend(pl.read_parquet(file2, \n                             n_rows=n_lines,row_index_offset=int(3*n_lines)))\nprint(file3)\ntrain.extend(pl.read_parquet(file3, \n                             n_rows=n_lines,row_index_offset=int(3*n_lines)))\nprint(file4)\ntrain.extend(pl.read_parquet(file4, \n                             n_rows=n_lines,row_index_offset=int(3*n_lines)))\n\ngc.collect()\n\n#for file in fileList:\n#    print(file)\n#    files_read.append(pl.read_parquet(file, n_rows=n_lines))\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:43:10.494017Z","iopub.execute_input":"2024-05-31T06:43:10.494471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = pd.read_csv(DATA_PATH + \"leap-atmospheric-physics-ai-climsim/sample_submission.csv\", nrows=1)\ndel weights['sample_id']\nweights = weights.T\nweights = weights.to_dict()[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.006431Z","iopub.status.idle":"2024-05-31T06:06:40.006996Z","shell.execute_reply.started":"2024-05-31T06:06:40.006745Z","shell.execute_reply":"2024-05-31T06:06:40.006765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for target in weights:\n    train = train.with_columns(pl.col(target).mul(weights[target]))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.008915Z","iopub.status.idle":"2024-05-31T06:06:40.009611Z","shell.execute_reply.started":"2024-05-31T06:06:40.009324Z","shell.execute_reply":"2024-05-31T06:06:40.00935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_COLS = train.columns[1:557]\nTARGET_COLS = train.columns[557:]\n\nfor col in FEAT_COLS:\n    train = train.with_columns(pl.col(col).cast(pl.Float32))\n\nfor col in TARGET_COLS:\n    train = train.with_columns(pl.col(col).cast(pl.Float32))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.011499Z","iopub.status.idle":"2024-05-31T06:06:40.012174Z","shell.execute_reply.started":"2024-05-31T06:06:40.011826Z","shell.execute_reply":"2024-05-31T06:06:40.01185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pl.concat(files_read, how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.014146Z","iopub.status.idle":"2024-05-31T06:06:40.014725Z","shell.execute_reply.started":"2024-05-31T06:06:40.014453Z","shell.execute_reply":"2024-05-31T06:06:40.014476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pl.read_csv(DATA_PATH + \"leap-atmospheric-physics-ai-climsim/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.016401Z","iopub.status.idle":"2024-05-31T06:06:40.016956Z","shell.execute_reply.started":"2024-05-31T06:06:40.016684Z","shell.execute_reply":"2024-05-31T06:06:40.016706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.to_pandas()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.018358Z","iopub.status.idle":"2024-05-31T06:06:40.01889Z","shell.execute_reply.started":"2024-05-31T06:06:40.018625Z","shell.execute_reply":"2024-05-31T06:06:40.018647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.020782Z","iopub.status.idle":"2024-05-31T06:06:40.0214Z","shell.execute_reply.started":"2024-05-31T06:06:40.021079Z","shell.execute_reply":"2024-05-31T06:06:40.021103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for target in weights:\n#    if 'q0003' in target:\n#        #print(target.replace(\"ptend\", \"state\"))\n#        #print(train[target.replace(\"ptend\", \"state\")])\n#        train[target.replace(\"ptend\", \"state\")] = train[target.replace(\"ptend\", \"state\")]*weights[target]\n#        test[target.replace(\"ptend\", \"state\")] = test[target.replace(\"ptend\", \"state\")]*weights[target]\n#        #print(train[target.replace(\"ptend\", \"state\")])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.022999Z","iopub.status.idle":"2024-05-31T06:06:40.023927Z","shell.execute_reply.started":"2024-05-31T06:06:40.023634Z","shell.execute_reply":"2024-05-31T06:06:40.023658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#feature_to_target_map","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.025441Z","iopub.status.idle":"2024-05-31T06:06:40.025995Z","shell.execute_reply.started":"2024-05-31T06:06:40.025702Z","shell.execute_reply":"2024-05-31T06:06:40.025724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" train, valid = train_test_split(train, test_size=0.1, random_state=42,shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.028035Z","iopub.status.idle":"2024-05-31T06:06:40.028617Z","shell.execute_reply.started":"2024-05-31T06:06:40.028339Z","shell.execute_reply":"2024-05-31T06:06:40.028365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_gradient('state_u',train,featureList)\ninsert_gradient('state_u',valid)\ninsert_gradient('state_u',test)\n\ninsert_gradient('state_v',train,featureList)\ninsert_gradient('state_v',valid)\ninsert_gradient('state_v',test)\n\ninsert_gradient('state_t',train,featureList)\ninsert_gradient('state_t',valid)\ninsert_gradient('state_t',test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.034576Z","iopub.status.idle":"2024-05-31T06:06:40.035235Z","shell.execute_reply.started":"2024-05-31T06:06:40.034883Z","shell.execute_reply":"2024-05-31T06:06:40.034926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#insert_kinetic_energy('state_u',train,featureList)\n#insert_kinetic_energy('state_u',valid)\n#insert_kinetic_energy('state_u',test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.0366Z","iopub.status.idle":"2024-05-31T06:06:40.037161Z","shell.execute_reply.started":"2024-05-31T06:06:40.036879Z","shell.execute_reply":"2024-05-31T06:06:40.036902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_virtual_temperature(train,featureList)\ninsert_virtual_temperature(valid)\ninsert_virtual_temperature(test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.038466Z","iopub.status.idle":"2024-05-31T06:06:40.038996Z","shell.execute_reply.started":"2024-05-31T06:06:40.038731Z","shell.execute_reply":"2024-05-31T06:06:40.038752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_total_water(train,featureList)\ninsert_total_water(valid)\ninsert_total_water(test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.041851Z","iopub.status.idle":"2024-05-31T06:06:40.042496Z","shell.execute_reply.started":"2024-05-31T06:06:40.042176Z","shell.execute_reply":"2024-05-31T06:06:40.04222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_gradient('state_total_water',train,featureList)\ninsert_gradient('state_total_water',valid)\ninsert_gradient('state_total_water',test)\n\ninsert_mean('state_total_water',train,featureList)\ninsert_mean('state_total_water',valid)\ninsert_mean('state_total_water',test)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.044421Z","iopub.status.idle":"2024-05-31T06:06:40.045008Z","shell.execute_reply.started":"2024-05-31T06:06:40.044727Z","shell.execute_reply":"2024-05-31T06:06:40.044751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#insert_mean('state_u',train,featureList)\n#insert_mean('state_u',valid)\n#insert_mean('state_u',test)\n\n#insert_mean('state_v',train,featureList)\n#insert_mean('state_v',valid)\n#insert_mean('state_v',test)\n\ninsert_mean('state_t',train,featureList)\ninsert_mean('state_t',valid)\ninsert_mean('state_t',test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.047089Z","iopub.status.idle":"2024-05-31T06:06:40.047675Z","shell.execute_reply.started":"2024-05-31T06:06:40.0474Z","shell.execute_reply":"2024-05-31T06:06:40.047422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_mean('state_tv',train,featureList)\ninsert_mean('state_tv',valid)\ninsert_mean('state_tv',test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.049251Z","iopub.status.idle":"2024-05-31T06:06:40.049813Z","shell.execute_reply.started":"2024-05-31T06:06:40.049532Z","shell.execute_reply":"2024-05-31T06:06:40.049554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insert_mean('state_q0001',train,featureList)\ninsert_mean('state_q0001',valid)\ninsert_mean('state_q0001',test)\n\ninsert_mean('state_q0002',train,featureList)\ninsert_mean('state_q0002',valid)\ninsert_mean('state_q0002',test)\n\ninsert_mean('state_q0003',train,featureList)\ninsert_mean('state_q0003',valid)\ninsert_mean('state_q0003',test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.05137Z","iopub.status.idle":"2024-05-31T06:06:40.051915Z","shell.execute_reply.started":"2024-05-31T06:06:40.051637Z","shell.execute_reply":"2024-05-31T06:06:40.05166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_to_target_map = feature_target_relation(vertical_features,surface_features,\n                                                vertical_targets,surface_targets,connections=5)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.053696Z","iopub.status.idle":"2024-05-31T06:06:40.054275Z","shell.execute_reply.started":"2024-05-31T06:06:40.053979Z","shell.execute_reply":"2024-05-31T06:06:40.054001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_to_target_map['cam_out_NETSW']","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.05575Z","iopub.status.idle":"2024-05-31T06:06:40.056324Z","shell.execute_reply.started":"2024-05-31T06:06:40.056035Z","shell.execute_reply":"2024-05-31T06:06:40.056058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#valid['grad_state_u_59']","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.058072Z","iopub.status.idle":"2024-05-31T06:06:40.058615Z","shell.execute_reply.started":"2024-05-31T06:06:40.058349Z","shell.execute_reply":"2024-05-31T06:06:40.058371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#targetList = list(feature_to_target_map.keys())\n\nfor col in weights.keys():\n    if col not in feature_to_target_map.keys():\n        del train[col]\nfor col in weights.keys():\n    if col not in feature_to_target_map.keys():\n        del valid[col]\n        \ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.060408Z","iopub.status.idle":"2024-05-31T06:06:40.060965Z","shell.execute_reply.started":"2024-05-31T06:06:40.06068Z","shell.execute_reply":"2024-05-31T06:06:40.060703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params = {\n    'task': 'train',\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': ['l1','l2'],\n    'learning_rate': 0.15,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.7,\n    'bagging_freq': 10,\n    'verbose': 0,\n    \"max_depth\": 10,\n    \"num_leaves\": 128,  \n    \"max_bin\": 63,\n    \"num_iterations\": 200,\n    \"reg_alpha\": 0.1  , \n    \"reg_lambda\": 3.25, \n    \"device\" : divice\n}\n\nscore_dictionary = {}\npartial_target = []\n#for tnum, target in enumerate(targets):\ntnum = 0\nfor target in feature_to_target_map:\n    feat = target + '_pred'\n    M = train[target].mean()\n    \n    print(target)\n\n\n    if (train[target].nunique() != 1) and (train[target].abs().mean() > 1e-3):\n        \n        features = feature_to_target_map[target]\n        \n        print(features)\n        \n        model = lgb.LGBMRegressor(**lgbm_params)\n                                \n        model.fit(train[features], train[target],\n            eval_set=[(valid[features], valid[target])],\n            eval_metric='mean_squared_error',\n            callbacks=[lgb.log_evaluation(10), lgb.early_stopping(10)])\n\n        \n        #es = xgb.callback.EarlyStopping( rounds=30, min_delta=1e-3, save_best=False, maximize=False)        \n        \n        #model = xgb.XGBRegressor(**xgb_params, objective='reg:squarederror', callbacks=[es])\n        #model.fit(\n        #    train[features],\n        #    train[target],\n        #    eval_set=[(valid[features], valid[target])],\n        #    verbose=False,\n        #)\n        \n        \n        valid[feat] = model.predict(valid[features])\n        score_model = r2_score(valid[target], valid[feat], force_finite=True)\n\n        # If r2 is negative, just use mean(target)\n        if score_model <= 0:\n            valid[feat] = 0.\n            test[target] = 0.\n        else:\n            test[target] = model.predict(test[features])\n\n        #bi = model.best_iteration\n        del model; gc.collect()\n    else:\n        valid[feat] = 0.\n        test[target] = 0.\n        bi = 0\n        \n    tnum = tnum + 1\n    \n    if submission == False:\n        partial_target.append(target)\n        score0 = r2_score(valid[target], valid[feat], force_finite=True)\n        score1 = r2_score(valid[partial_target], valid[[f + '_pred' for f in partial_target]])\n        print(f\"{tnum} r2(accum): {score1:.4f} / r2({target}): {score0:.4f},  best iter: {0}\")\n    else:\n        score0 = r2_score(valid[target], valid[feat], force_finite=True)\n        del valid[feat]\n        gc.collect()\n    \n    \n    score_dictionary[target] = score0","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.062458Z","iopub.status.idle":"2024-05-31T06:06:40.063005Z","shell.execute_reply.started":"2024-05-31T06:06:40.062721Z","shell.execute_reply":"2024-05-31T06:06:40.062742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = list(feature_to_target_map.keys())\n\n#print(targets)\nif submission == False:\n    score = r2_score(valid[targets], valid[[f + '_pred' for f in targets]])\n    print(score)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.064649Z","iopub.status.idle":"2024-05-31T06:06:40.065227Z","shell.execute_reply.started":"2024-05-31T06:06:40.064908Z","shell.execute_reply":"2024-05-31T06:06:40.06493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(json.dumps(score_dictionary, indent = 4))\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.066548Z","iopub.status.idle":"2024-05-31T06:06:40.067077Z","shell.execute_reply.started":"2024-05-31T06:06:40.066802Z","shell.execute_reply":"2024-05-31T06:06:40.066823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean up\n\n#valid.to_pandas().to_parquet(f'validation_{score:.4f}.parquet')\n#test = test.to_pandas()\n\ndel train, valid; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.068392Z","iopub.status.idle":"2024-05-31T06:06:40.068908Z","shell.execute_reply.started":"2024-05-31T06:06:40.068642Z","shell.execute_reply":"2024-05-31T06:06:40.068667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_cols = []\nfor i in range(29):\n    use_cols.append(f\"ptend_q0002_{i}\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.070665Z","iopub.status.idle":"2024-05-31T06:06:40.071213Z","shell.execute_reply.started":"2024-05-31T06:06:40.070921Z","shell.execute_reply":"2024-05-31T06:06:40.070942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\")\nsubmission2 = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\")\n\n\nfor col in tqdm(targets):\n    if weights[col] > 0:\n        submission[col] = test[col].values\n        #*submission[col].values\n    else:\n        submission[col] = 0.\n        \nfor col in use_cols:\n    submission[col] = -test[col.replace(\"ptend\", \"state\")]*submission2[col]/1200.\n        \nsubmission.to_csv('submission.csv', index=False)        \nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T06:06:40.072685Z","iopub.status.idle":"2024-05-31T06:06:40.073256Z","shell.execute_reply.started":"2024-05-31T06:06:40.072929Z","shell.execute_reply":"2024-05-31T06:06:40.072951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}