{"metadata":{"accelerator":"TPU","colab":{"gpuType":"V28","machine_shape":"hm","provenance":[]},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"}],"dockerImageVersionId":30685,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":792.37307,"end_time":"2024-05-05T16:27:14.967127","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-05T16:14:02.594057","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{"id":"FP8uDmyQE3Hn"}},{"cell_type":"code","source":"%%capture\n\n!pip install polars keras==2.15","metadata":{"executionInfo":{"elapsed":7082,"status":"ok","timestamp":1715713761945,"user":{"displayName":"shlomo ron","userId":"09718489046984556743"},"user_tz":-180},"id":"5c3a8197","execution":{"iopub.status.busy":"2024-05-25T18:31:33.386227Z","iopub.execute_input":"2024-05-25T18:31:33.386613Z","iopub.status.idle":"2024-05-25T18:31:50.831894Z","shell.execute_reply.started":"2024-05-25T18:31:33.386583Z","shell.execute_reply":"2024-05-25T18:31:50.830436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport polars as pl\nimport numpy as np\nimport pickle","metadata":{"executionInfo":{"elapsed":15536,"status":"ok","timestamp":1715713777453,"user":{"displayName":"shlomo ron","userId":"09718489046984556743"},"user_tz":-180},"id":"f7bfcb59","execution":{"iopub.status.busy":"2024-05-25T18:31:50.834466Z","iopub.execute_input":"2024-05-25T18:31:50.834798Z","iopub.status.idle":"2024-05-25T18:32:10.532073Z","shell.execute_reply.started":"2024-05-25T18:31:50.834767Z","shell.execute_reply":"2024-05-25T18:32:10.530933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pl.read_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv')\ntest_df = pl.read_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/test_old.csv')","metadata":{"executionInfo":{"elapsed":2206,"status":"ok","timestamp":1715714196599,"user":{"displayName":"shlomo ron","userId":"09718489046984556743"},"user_tz":-180},"id":"a695a6ec","outputId":"452dc531-29e8-46f7-e877-5d333d0d4df7","execution":{"iopub.status.busy":"2024-05-25T18:32:25.631922Z","iopub.execute_input":"2024-05-25T18:32:25.63225Z","iopub.status.idle":"2024-05-25T18:32:38.24282Z","shell.execute_reply.started":"2024-05-25T18:32:25.632223Z","shell.execute_reply":"2024-05-25T18:32:38.242047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_COLS = train_df.columns[1:557]\nTARGET_COLS = train_df.columns[557:]","metadata":{"executionInfo":{"elapsed":9,"status":"ok","timestamp":1715714196599,"user":{"displayName":"shlomo ron","userId":"09718489046984556743"},"user_tz":-180},"id":"a320146d","execution":{"iopub.status.busy":"2024-05-25T18:32:38.247559Z","iopub.execute_input":"2024-05-25T18:32:38.247913Z","iopub.status.idle":"2024-05-25T18:32:38.25468Z","shell.execute_reply.started":"2024-05-25T18:32:38.247883Z","shell.execute_reply":"2024-05-25T18:32:38.253867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_cols = FEAT_COLS\ny_cols = TARGET_COLS\n\nX_cols_series_parents = ['state_t', 'state_q0001', 'state_q0002', 'state_q0003', 'state_u', 'state_v',\n                         'pbuf_ozone', 'pbuf_CH4', 'pbuf_N2O']\nX_cols_series = [x for x in X_cols if np.mean([x.startswith(y) for y in X_cols_series_parents])>0]\nX_cols_series_NOT = [x for x in X_cols if x not in X_cols_series]\nprint(len(X_cols_series)+len(X_cols_series_NOT) == len(X_cols))\nX_cols_series_indices = np.asarray([np.where(np.asarray(X_cols) == x)[0][0] for x in X_cols_series])\nX_cols_series_NOT_indices = np.asarray([np.where(np.asarray(X_cols) == x)[0][0] for x in X_cols_series_NOT])\n\ny_cols_series_parents = ['ptend_t', 'ptend_q0001', 'ptend_q0002', 'ptend_q0003', 'ptend_u', 'ptend_v']\ny_cols_series = [x for x in y_cols if np.mean([x.startswith(y) for y in y_cols_series_parents])>0]\ny_cols_series_NOT = [x for x in y_cols if x not in y_cols_series]\nprint(len(y_cols_series)+len(y_cols_series_NOT) == len(y_cols))\ny_cols_series_indices = np.asarray([np.where(np.asarray(y_cols) == x)[0][0] for x in y_cols_series])\ny_cols_series_NOT_indices = np.asarray([np.where(np.asarray(y_cols) == x)[0][0] for x in y_cols_series_NOT])","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1715714196600,"user":{"displayName":"shlomo ron","userId":"09718489046984556743"},"user_tz":-180},"id":"J8n0doCCz3jJ","outputId":"6e3f5fc0-0b5b-448f-d169-d90f8f1283ae","execution":{"iopub.status.busy":"2024-05-25T18:32:38.279866Z","iopub.execute_input":"2024-05-25T18:32:38.28029Z","iopub.status.idle":"2024-05-25T18:32:38.481926Z","shell.execute_reply.started":"2024-05-25T18:32:38.280253Z","shell.execute_reply":"2024-05-25T18:32:38.481016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mean_std(data_df, COLS):\n    data = data_df.select(COLS).to_numpy()\n    min_data = np.min(data, axis = 0, keepdims = True)\n    max_data = np.max(data, axis = 0, keepdims = True)\n    mean = np.mean(data, axis = 0, keepdims = True)\n    std = np.std(data, axis = (0), keepdims = True)\n    return min_data, max_data, mean, std\nmin_y, max_y, mean_y, std_y = get_mean_std(train_df, TARGET_COLS)\nmin_col_not, max_col_not, mean_col_not, std_col_not = get_mean_std(train_df, X_cols_series_NOT)\nmin_col_not_test, max_col_not_test, mean_col_not_test, std_col_not_test = get_mean_std(test_df, X_cols_series_NOT)\nmins_col_not = np.where(min_col_not_test<min_col_not,min_col_not_test, min_col_not)\nx_col_min, x_col_max, x_col_mean, x_col_std = get_mean_std(train_df, X_cols_series)\nx_col_min_test, x_col_max_test, x_col_mean_test, x_col_std_test = get_mean_std(test_df, X_cols_series)\nx_col_mins = np.where(x_col_min_test<x_col_min,x_col_min_test, x_col_min)\ndef get_mean_std_cols(data_df, COLS):\n    data = data_df.select(COLS).to_numpy()\n    data = np.reshape(data, [-1, int(len(data[1])//60), 60])\n    min_data = np.min(data, axis = (0,2), keepdims = True)\n    max_data = np.max(data, axis = (0,2), keepdims = True)\n    mean = np.mean(data, axis = (0,2), keepdims = True)\n    std = np.std(data, axis = (0,2), keepdims = True)\n    return min_data, max_data, mean, std\nX_total_min, X_total_max, X_total_mean, X_total_std = get_mean_std_cols(train_df, X_cols_series)\nX_total_min_test, X_total_max_test, X_total_mean_test, X_total_std_test = get_mean_std_cols(test_df, X_cols_series)\nx_total_mins = np.where(X_total_min_test<X_total_min,X_total_min_test, X_total_min)\ny_total_min, y_total_max, y_total_mean, y_total_std = get_mean_std_cols(train_df, y_cols_series)","metadata":{"execution":{"iopub.status.busy":"2024-05-25T18:32:38.537971Z","iopub.execute_input":"2024-05-25T18:32:38.540314Z","iopub.status.idle":"2024-05-25T18:32:42.692048Z","shell.execute_reply.started":"2024-05-25T18:32:38.540275Z","shell.execute_reply":"2024-05-25T18:32:42.690876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(min_y, open('min_y.p', 'bw'))\npickle.dump(max_y, open('max_y.p', 'bw'))\npickle.dump(mean_y, open('mean_y.p', 'bw'))\npickle.dump(std_y, open('std_y.p', 'bw'))\n\npickle.dump(min_col_not, open('min_col_not.p', 'bw'))\npickle.dump(max_col_not, open('max_col_not.p', 'bw'))\npickle.dump(mean_col_not, open('mean_col_not.p', 'bw'))\npickle.dump(std_col_not, open('std_col_not.p', 'bw'))\n\npickle.dump(min_col_not_test, open('min_col_not_test.p', 'bw'))\npickle.dump(max_col_not_test, open('max_col_not_test.p', 'bw'))\npickle.dump(mean_col_not_test, open('mean_col_not_test.p', 'bw'))\npickle.dump(std_col_not_test, open('std_col_not_test.p', 'bw'))\n\npickle.dump(x_col_min, open('x_col_min.p', 'bw'))\npickle.dump(x_col_max, open('x_col_max.p', 'bw'))\npickle.dump(x_col_mean, open('x_col_mean.p', 'bw'))\npickle.dump(x_col_std, open('x_col_std.p', 'bw'))\n\npickle.dump(x_col_min_test, open('x_col_min_test.p', 'bw'))\npickle.dump(x_col_max_test, open('x_col_max_test.p', 'bw'))\npickle.dump(x_col_mean_test, open('x_col_mean_test.p', 'bw'))\npickle.dump(x_col_std_test, open('x_col_std_test.p', 'bw'))\n\npickle.dump(X_total_min, open('X_total_min.p', 'bw'))\npickle.dump(X_total_max, open('X_total_max.p', 'bw'))\npickle.dump(X_total_mean, open('x_total_mean.p', 'bw'))\npickle.dump(X_total_std, open('x_total_std.p', 'bw'))\n\npickle.dump(X_total_min_test, open('X_total_min_test.p', 'bw'))\npickle.dump(X_total_max_test, open('X_total_max_test.p', 'bw'))\npickle.dump(X_total_mean_test, open('x_total_mean_test.p', 'bw'))\npickle.dump(X_total_std_test, open('x_total_std_test.p', 'bw'))\n\npickle.dump(y_total_min, open('y_total_min.p', 'bw'))\npickle.dump(y_total_max, open('y_total_max.p', 'bw'))\npickle.dump(y_total_mean, open('y_total_mean.p', 'bw'))\npickle.dump(y_total_std, open('y_total_std.p', 'bw'))","metadata":{"execution":{"iopub.status.busy":"2024-05-25T18:32:42.69362Z","iopub.execute_input":"2024-05-25T18:32:42.694045Z","iopub.status.idle":"2024-05-25T18:32:42.714931Z","shell.execute_reply.started":"2024-05-25T18:32:42.694008Z","shell.execute_reply":"2024-05-25T18:32:42.713774Z"},"trusted":true},"execution_count":null,"outputs":[]}]}