{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":8352923,"sourceType":"datasetVersion","datasetId":4866555,"isSourceIdPinned":true}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"import h5py","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:24.739858Z","iopub.execute_input":"2024-05-08T06:41:24.740253Z","iopub.status.idle":"2024-05-08T06:41:25.002301Z","shell.execute_reply.started":"2024-05-08T06:41:24.740221Z","shell.execute_reply":"2024-05-08T06:41:25.001076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '/kaggle/input/leap-climsim-hdf5-training-data/train.h5'","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:26.569974Z","iopub.execute_input":"2024-05-08T06:41:26.571149Z","iopub.status.idle":"2024-05-08T06:41:26.57629Z","shell.execute_reply.started":"2024-05-08T06:41:26.571096Z","shell.execute_reply":"2024-05-08T06:41:26.57492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_h5 = h5py.File(TRAIN_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:26.777658Z","iopub.execute_input":"2024-05-08T06:41:26.778064Z","iopub.status.idle":"2024-05-08T06:41:26.797457Z","shell.execute_reply.started":"2024-05-08T06:41:26.778031Z","shell.execute_reply":"2024-05-08T06:41:26.796309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_h5.keys())","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:27.530764Z","iopub.execute_input":"2024-05-08T06:41:27.531323Z","iopub.status.idle":"2024-05-08T06:41:27.555219Z","shell.execute_reply.started":"2024-05-08T06:41:27.531283Z","shell.execute_reply":"2024-05-08T06:41:27.553722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_h5['data']","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:27.778578Z","iopub.execute_input":"2024-05-08T06:41:27.779093Z","iopub.status.idle":"2024-05-08T06:41:27.78835Z","shell.execute_reply.started":"2024-05-08T06:41:27.779038Z","shell.execute_reply":"2024-05-08T06:41:27.78696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_data.keys())","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:28.549675Z","iopub.execute_input":"2024-05-08T06:41:28.550063Z","iopub.status.idle":"2024-05-08T06:41:28.557801Z","shell.execute_reply.started":"2024-05-08T06:41:28.55003Z","shell.execute_reply":"2024-05-08T06:41:28.556616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_table = train_data['table']","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:29.391261Z","iopub.execute_input":"2024-05-08T06:41:29.391665Z","iopub.status.idle":"2024-05-08T06:41:29.398414Z","shell.execute_reply.started":"2024-05-08T06:41:29.391632Z","shell.execute_reply":"2024-05-08T06:41:29.397109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_table.dtype)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:29.967879Z","iopub.execute_input":"2024-05-08T06:41:29.968589Z","iopub.status.idle":"2024-05-08T06:41:29.976593Z","shell.execute_reply.started":"2024-05-08T06:41:29.968554Z","shell.execute_reply":"2024-05-08T06:41:29.975047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_array = train_table.fields('values_block_0')","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:32.078923Z","iopub.execute_input":"2024-05-08T06:41:32.079278Z","iopub.status.idle":"2024-05-08T06:41:32.087249Z","shell.execute_reply.started":"2024-05-08T06:41:32.079251Z","shell.execute_reply":"2024-05-08T06:41:32.086098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_table.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:33.103347Z","iopub.execute_input":"2024-05-08T06:41:33.103831Z","iopub.status.idle":"2024-05-08T06:41:33.113326Z","shell.execute_reply.started":"2024-05-08T06:41:33.103797Z","shell.execute_reply":"2024-05-08T06:41:33.111626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_array[1000000])","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:33.895606Z","iopub.execute_input":"2024-05-08T06:41:33.896Z","iopub.status.idle":"2024-05-08T06:41:33.929883Z","shell.execute_reply.started":"2024-05-08T06:41:33.895968Z","shell.execute_reply":"2024-05-08T06:41:33.928178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_array[100:110])","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:35.273101Z","iopub.execute_input":"2024-05-08T06:41:35.273933Z","iopub.status.idle":"2024-05-08T06:41:35.294288Z","shell.execute_reply.started":"2024-05-08T06:41:35.273875Z","shell.execute_reply":"2024-05-08T06:41:35.293283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_array[[10, 1000, 100000]])","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:36.088498Z","iopub.execute_input":"2024-05-08T06:41:36.088936Z","iopub.status.idle":"2024-05-08T06:41:36.115859Z","shell.execute_reply.started":"2024-05-08T06:41:36.088897Z","shell.execute_reply":"2024-05-08T06:41:36.114856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_h5.close()","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:36.673915Z","iopub.execute_input":"2024-05-08T06:41:36.674277Z","iopub.status.idle":"2024-05-08T06:41:36.679926Z","shell.execute_reply.started":"2024-05-08T06:41:36.674248Z","shell.execute_reply":"2024-05-08T06:41:36.67881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:37.055083Z","iopub.execute_input":"2024-05-08T06:41:37.055502Z","iopub.status.idle":"2024-05-08T06:41:37.807756Z","shell.execute_reply.started":"2024-05-08T06:41:37.055444Z","shell.execute_reply":"2024-05-08T06:41:37.805934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with h5py.File(TRAIN_PATH) as t:\n    train_idx, valid_idx = train_test_split(np.arange(t['data']['table'].shape[0]), random_state = 0)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:37.991345Z","iopub.execute_input":"2024-05-08T06:41:37.992131Z","iopub.status.idle":"2024-05-08T06:41:38.771597Z","shell.execute_reply.started":"2024-05-08T06:41:37.992085Z","shell.execute_reply":"2024-05-08T06:41:38.769973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:41.248258Z","iopub.execute_input":"2024-05-08T06:41:41.248818Z","iopub.status.idle":"2024-05-08T06:41:41.559089Z","shell.execute_reply.started":"2024-05-08T06:41:41.248777Z","shell.execute_reply":"2024-05-08T06:41:41.55804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_WEIGHTS = pl.scan_csv(\n    '/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv',\n).drop('sample_id').head(1).collect().to_numpy()[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:41.571478Z","iopub.execute_input":"2024-05-08T06:41:41.571923Z","iopub.status.idle":"2024-05-08T06:41:41.774653Z","shell.execute_reply.started":"2024-05-08T06:41:41.57189Z","shell.execute_reply":"2024-05-08T06:41:41.773422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(TARGET_WEIGHTS)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:42.703742Z","iopub.execute_input":"2024-05-08T06:41:42.704137Z","iopub.status.idle":"2024-05-08T06:41:42.716955Z","shell.execute_reply.started":"2024-05-08T06:41:42.704106Z","shell.execute_reply":"2024-05-08T06:41:42.7158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare dataset","metadata":{}},{"cell_type":"code","source":"NUM_FEATURES = 556\nNUM_TARGETS = 368","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:43.231256Z","iopub.execute_input":"2024-05-08T06:41:43.231737Z","iopub.status.idle":"2024-05-08T06:41:43.237756Z","shell.execute_reply.started":"2024-05-08T06:41:43.2317Z","shell.execute_reply":"2024-05-08T06:41:43.23624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:43.815434Z","iopub.execute_input":"2024-05-08T06:41:43.815873Z","iopub.status.idle":"2024-05-08T06:41:59.887726Z","shell.execute_reply.started":"2024-05-08T06:41:43.815839Z","shell.execute_reply":"2024-05-08T06:41:59.885838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_dataset(h5file, indexes, target_weights,\n                 seed = None,\n                 batch_size = 32):\n    rng = np.random.default_rng(seed)\n    def generator():\n        with h5py.File(h5file, 'r') as hf:\n            t = hf['data']['table'].fields('values_block_0')\n            while True:\n                i = rng.choice(indexes)\n                z = t[i]\n                x = z[:NUM_FEATURES]\n                y = z[NUM_FEATURES:]\n                y = y * target_weights\n                yield x, y\n    return tf.data.Dataset.from_generator(\n        generator,\n        output_signature = (\n            tf.TensorSpec(shape = (NUM_FEATURES,), dtype = tf.float32),\n            tf.TensorSpec(shape = (NUM_TARGETS,), dtype = tf.float32),\n        ),\n    ).batch(batch_size).prefetch(tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:59.890108Z","iopub.execute_input":"2024-05-08T06:41:59.890884Z","iopub.status.idle":"2024-05-08T06:41:59.903196Z","shell.execute_reply.started":"2024-05-08T06:41:59.890844Z","shell.execute_reply":"2024-05-08T06:41:59.901804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = make_dataset(TRAIN_PATH, train_idx, TARGET_WEIGHTS)\nds_valid = make_dataset(TRAIN_PATH, valid_idx, TARGET_WEIGHTS)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:41:59.904714Z","iopub.execute_input":"2024-05-08T06:41:59.905206Z","iopub.status.idle":"2024-05-08T06:42:00.042083Z","shell.execute_reply.started":"2024-05-08T06:41:59.905157Z","shell.execute_reply":"2024-05-08T06:42:00.040719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(ds_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-08T06:42:14.649477Z","iopub.execute_input":"2024-05-08T06:42:14.649918Z","iopub.status.idle":"2024-05-08T06:42:14.658512Z","shell.execute_reply.started":"2024-05-08T06:42:14.649877Z","shell.execute_reply":"2024-05-08T06:42:14.657021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch in ds_train.take(1):\n    display(batch)","metadata":{"execution":{"iopub.status.busy":"2024-04-27T06:28:48.669446Z","iopub.execute_input":"2024-04-27T06:28:48.669866Z","iopub.status.idle":"2024-04-27T06:28:49.37354Z","shell.execute_reply.started":"2024-04-27T06:28:48.669832Z","shell.execute_reply":"2024-04-27T06:28:49.372539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model definition","metadata":{}},{"cell_type":"code","source":"def build_model():\n    inputs = tf.keras.layers.Input(shape = (NUM_FEATURES,))\n    x = inputs\n    x = tf.keras.layers.Dense(768)(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    y0 = tf.keras.layers.Dense(360)(x) # `ptend_t` to `ptend_v`. 60 x 6\n    y1 = tf.keras.layers.Dense(8, activation = 'relu')(x) # `cam_out_NETSW` to `cam_out_SOLLD`\n    y = tf.keras.layers.Concatenate()([y0, y1])\n    outputs = y\n    model = tf.keras.Model(\n        inputs = inputs,\n        outputs = outputs,\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-04-27T06:28:54.662556Z","iopub.execute_input":"2024-04-27T06:28:54.662942Z","iopub.status.idle":"2024-04-27T06:28:54.670089Z","shell.execute_reply.started":"2024-04-27T06:28:54.662915Z","shell.execute_reply":"2024-04-27T06:28:54.668776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_model().summary();","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:07:05.583929Z","iopub.execute_input":"2024-04-24T06:07:05.584933Z","iopub.status.idle":"2024-04-24T06:07:05.664611Z","shell.execute_reply.started":"2024-04-24T06:07:05.584877Z","shell.execute_reply":"2024-04-24T06:07:05.663307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"model = build_model()\nmodel.compile(\n    optimizer = 'adam',\n    loss = 'mean_squared_error',\n    metrics = ['r2_score'],\n)\nhistory = model.fit(\n    ds_train,\n    epochs = 3,\n    steps_per_epoch = 100,\n    validation_data = ds_valid,\n    validation_steps = 100,\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:07:07.671738Z","iopub.execute_input":"2024-04-24T06:07:07.672201Z","iopub.status.idle":"2024-04-24T06:08:50.218638Z","shell.execute_reply.started":"2024-04-24T06:07:07.672165Z","shell.execute_reply":"2024-04-24T06:08:50.217124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:08:50.221451Z","iopub.execute_input":"2024-04-24T06:08:50.22191Z","iopub.status.idle":"2024-04-24T06:08:50.22724Z","shell.execute_reply.started":"2024-04-24T06:08:50.221873Z","shell.execute_reply":"2024-04-24T06:08:50.226033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_history = pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:08:52.713647Z","iopub.execute_input":"2024-04-24T06:08:52.714572Z","iopub.status.idle":"2024-04-24T06:08:52.719683Z","shell.execute_reply.started":"2024-04-24T06:08:52.714536Z","shell.execute_reply":"2024-04-24T06:08:52.718738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(df_history[['loss', 'val_loss']]);","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:08:53.664733Z","iopub.execute_input":"2024-04-24T06:08:53.665147Z","iopub.status.idle":"2024-04-24T06:08:54.018728Z","shell.execute_reply.started":"2024-04-24T06:08:53.665114Z","shell.execute_reply":"2024-04-24T06:08:54.017601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(df_history[['r2_score', 'val_r2_score']]);","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:08:54.529918Z","iopub.execute_input":"2024-04-24T06:08:54.530441Z","iopub.status.idle":"2024-04-24T06:08:54.907179Z","shell.execute_reply.started":"2024-04-24T06:08:54.530403Z","shell.execute_reply":"2024-04-24T06:08:54.905584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights('model.weights.h5');","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:10:38.859458Z","iopub.execute_input":"2024-04-24T06:10:38.859886Z","iopub.status.idle":"2024-04-24T06:10:38.905511Z","shell.execute_reply.started":"2024-04-24T06:10:38.859854Z","shell.execute_reply":"2024-04-24T06:10:38.904166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:12:54.574626Z","iopub.execute_input":"2024-04-24T06:12:54.575332Z","iopub.status.idle":"2024-04-24T06:12:54.583343Z","shell.execute_reply.started":"2024-04-24T06:12:54.57525Z","shell.execute_reply":"2024-04-24T06:12:54.581041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lf_test = pl.scan_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/test.csv')\nlf_sample_submission = pl.scan_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv')\nTARGET_SCHEMA = lf_sample_submission.drop('sample_id').schema","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:14:52.075407Z","iopub.execute_input":"2024-04-24T06:14:52.075916Z","iopub.status.idle":"2024-04-24T06:14:52.105549Z","shell.execute_reply.started":"2024-04-24T06:14:52.075881Z","shell.execute_reply":"2024-04-24T06:14:52.104257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_predict_1(model, batch):\n    pred = model.predict(batch.to_numpy())\n    return pl.from_numpy(pred, schema = TARGET_SCHEMA)\n\ndef custom_predict(model, lf):\n    Y = lf.drop('sample_id').map_batches(\n        (lambda x: custom_predict_1(model, x)),\n        streamable = True,\n        schema = TARGET_SCHEMA\n    )\n    return pl.concat(\n        [lf.select('sample_id'), Y],\n        how = 'horizontal',\n    )","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:16:07.79618Z","iopub.execute_input":"2024-04-24T06:16:07.796628Z","iopub.status.idle":"2024-04-24T06:16:07.803901Z","shell.execute_reply.started":"2024-04-24T06:16:07.796598Z","shell.execute_reply":"2024-04-24T06:16:07.802391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lf_pred = custom_predict(model, lf_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:16:17.363989Z","iopub.execute_input":"2024-04-24T06:16:17.364496Z","iopub.status.idle":"2024-04-24T06:16:17.380029Z","shell.execute_reply.started":"2024-04-24T06:16:17.364459Z","shell.execute_reply":"2024-04-24T06:16:17.378087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lf_pred.collect().write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-24T06:16:33.892324Z","iopub.execute_input":"2024-04-24T06:16:33.893184Z","iopub.status.idle":"2024-04-24T06:19:12.462583Z","shell.execute_reply.started":"2024-04-24T06:16:33.893137Z","shell.execute_reply":"2024-04-24T06:19:12.46083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}