{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars keras==2.15","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.callbacks import LearningRateScheduler, ReduceLROnPlateau\nfrom tensorflow.keras import backend as K\nimport tensorflow as tf\n\nimport os\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.TPUStrategy(tpu)\n\nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pl.read_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv')\ntest_df = pl.read_csv('/kaggle/input/leap-atmospheric-physics-ai-climsim/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_COLS = train_df.columns[1:557]\nTARGET_COLS = train_df.columns[557:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in FEAT_COLS:\n    train_df = train_df.with_columns(pl.col(col).cast(pl.Float32))\n    test_df = test_df.with_columns(pl.col(col).cast(pl.Float32))\n\nfor col in TARGET_COLS:\n    train_df = train_df.with_columns(pl.col(col).cast(pl.Float32))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train_df.select(FEAT_COLS).to_numpy()\ny = train_df.select(TARGET_COLS).to_numpy()\ntest_feats = test_df.select(FEAT_COLS).to_numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nreference\nhttps://www.kaggle.com/code/ymatioun/leap-simple-nn/notebook\n\"\"\"\n# norm X\nmin_std = 1e-8\n\nmx = X_train.mean(axis=0)\nsx = np.maximum(X_train.std(axis=0), min_std)\nX_train = (X_train - mx.reshape(1,-1)) / sx.reshape(1,-1)\ntest_feats = (test_feats - mx.reshape(1,-1)) / sx.reshape(1,-1)\n\n# norm Y\nmy = y.mean(axis=0)\nsy = np.maximum(np.sqrt((y*y).mean(axis=0)), min_std)\ny = (y - my.reshape(1,-1)) / sy.reshape(1,-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(tf.keras.Model):\n    def __init__(self):\n        super(CustomModel, self).__init__()\n        \n        self.hidden_size = 128\n        \n        self.linear = tf.keras.Sequential([\n            tf.keras.layers.Dense(self.hidden_size, activation='relu'),\n            tf.keras.layers.BatchNormalization(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(self.hidden_size, activation='relu'),\n            tf.keras.layers.BatchNormalization(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(self.hidden_size, activation='relu'),\n            tf.keras.layers.BatchNormalization(),\n            # tf.keras.layers.Dropout(0.2),\n            tf.keras.layers.Dense(len(TARGET_COLS))\n            ])\n    \n    # def call(self, x, training=False):\n    def call(self, x):    \n        x = self.linear(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    model = CustomModel()\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10 # 100\nbatch_size = 512*strategy.num_replicas_in_sync\nlr_patience = 10\nearly_patience = 20\nmin_lr = 1e-6\nlr_factor = 0.1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# training, inference","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, y)):\n    x_train, x_valid = X_train[train_idx], X_train[val_idx]\n    y_train, y_valid = y[train_idx], y[val_idx]\n    \n    print(x_train.shape, x_valid.shape)\n\n    checkpoint_filepath = f\"folds{fold}.weights.h5\"\n    \n    K.clear_session()\n    with strategy.scope():\n        model = build_model()\n        \n        optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n        \n    model.compile(optimizer=optimizer, loss=\"mse\")\n    \n    monitor = \"val_loss\"\n    sv = ModelCheckpoint(\n            checkpoint_filepath, monitor=monitor, verbose=1, save_best_only=True,\n            save_weights_only=True, mode='min', save_freq='epoch'\n    )\n    reduce_lr = ReduceLROnPlateau(monitor=monitor, factor=lr_factor,\n                              patience=lr_patience, min_lr=min_lr)\n    early_stop = tf.keras.callbacks.EarlyStopping(\n        monitor=monitor,\n        min_delta=0.0,\n        patience=early_patience, \n        mode=\"min\"\n    )\n\n    history = model.fit(x_train, y_train, verbose=1,\n                        validation_data=(x_valid, y_valid), \n                        epochs=epochs, batch_size=batch_size, callbacks=[reduce_lr, sv, early_stop])\n    \n    oof_pred = model.predict(x_valid)    \n    oof_true = y_valid\n\n    test_pred = model.predict(test_feats)\n    \n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submit\n# override constant columns\nfor i in range(sy.shape[0]):\n    if sy[i] < min_std * 1.1:\n        test_pred[:,i] = 0\n\n# undo y scaling\ntest_pred = test_pred * sy.reshape(1,-1) + my.reshape(1,-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\")\nsub.iloc[:,1:] *= test_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nfast submission\nhttps://www.kaggle.com/competitions/leap-atmospheric-physics-ai-climsim/discussion/495128\n\"\"\"\ntest_polars = pl.from_pandas(sub[[\"sample_id\"]+TARGET_COLS])\ntest_polars.write_csv(\"submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}