{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-02T22:08:30.925598Z","iopub.execute_input":"2024-06-02T22:08:30.926534Z","iopub.status.idle":"2024-06-02T22:08:32.037629Z","shell.execute_reply.started":"2024-06-02T22:08:30.926498Z","shell.execute_reply":"2024-06-02T22:08:32.036579Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport seaborn as sns\nimport sklearn as sk\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport torch\nfrom torch.utils.data import DataLoader, TensorDataset\n","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:32.039667Z","iopub.execute_input":"2024-06-02T22:08:32.040177Z","iopub.status.idle":"2024-06-02T22:08:36.208988Z","shell.execute_reply.started":"2024-06-02T22:08:32.040147Z","shell.execute_reply":"2024-06-02T22:08:36.208024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leap_raw_df = pl.scan_csv(f'/kaggle/input/leap-atmospheric-physics-ai-climsim/train.csv')\nfetch_size =100000\ninput_columns = leap_raw_df.columns[1:556]\noutput_columns = leap_raw_df.columns[556:924]\n\nX = leap_raw_df.select(pl.col(input_columns)).fetch(fetch_size)\nY = leap_raw_df.select(pl.col(output_columns)).fetch(fetch_size)","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:36.21031Z","iopub.execute_input":"2024-06-02T22:08:36.21097Z","iopub.status.idle":"2024-06-02T22:08:45.195943Z","shell.execute_reply.started":"2024-06-02T22:08:36.210931Z","shell.execute_reply":"2024-06-02T22:08:45.194797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import FunctionTransformer\nconstant_epson = 1\nconstant_epson_r = 1\ndef log_transform(x):\n    return (np.log1p(x+constant_epson) +constant_epson_r)\n\n# Convert Polars DataFrame to NumPy array\nX_np = X.to_numpy()\nY_np = Y.to_numpy()\n\ntransformer_X = StandardScaler().fit(X_np)\ntransformer_Y = StandardScaler().fit(Y_np)\n\nX_transformed = transformer_X.transform(X_np)\nY_transformed = transformer_Y.transform(Y_np)\n\n\n\n# Log\n#transformer =FunctionTransformer(log_transform)\n#X_transformado=transformer.transform(X_np)\n#Y_transformado=transformer.transform(Y_np)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:45.198794Z","iopub.execute_input":"2024-06-02T22:08:45.199151Z","iopub.status.idle":"2024-06-02T22:08:47.253221Z","shell.execute_reply.started":"2024-06-02T22:08:45.199122Z","shell.execute_reply":"2024-06-02T22:08:47.252107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_transformed.dtype)\nprint(Y_transformed.dtype)","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:47.254843Z","iopub.execute_input":"2024-06-02T22:08:47.255676Z","iopub.status.idle":"2024-06-02T22:08:47.261307Z","shell.execute_reply.started":"2024-06-02T22:08:47.255635Z","shell.execute_reply":"2024-06-02T22:08:47.260165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nX_tensor = torch.tensor(X_transformed, dtype=torch.float64)\nY_tensor = torch.tensor(Y_transformed, dtype=torch.float64)","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:47.262652Z","iopub.execute_input":"2024-06-02T22:08:47.263044Z","iopub.status.idle":"2024-06-02T22:08:47.677326Z","shell.execute_reply.started":"2024-06-02T22:08:47.263015Z","shell.execute_reply":"2024-06-02T22:08:47.676159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:47.678677Z","iopub.execute_input":"2024-06-02T22:08:47.679072Z","iopub.status.idle":"2024-06-02T22:08:47.685544Z","shell.execute_reply.started":"2024-06-02T22:08:47.679043Z","shell.execute_reply":"2024-06-02T22:08:47.683725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, Y_train,Y_val = train_test_split(X_tensor, Y_tensor, test_size=0.2, random_state=42)\ntrain_dataset = TensorDataset(X_train, Y_train)\nval_dataset = TensorDataset(X_val, Y_val)\n\ndef create_dataloader(dataset, batch_size):\n    return DataLoader(dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:47.686996Z","iopub.execute_input":"2024-06-02T22:08:47.687365Z","iopub.status.idle":"2024-06-02T22:08:49.465363Z","shell.execute_reply.started":"2024-06-02T22:08:47.687329Z","shell.execute_reply":"2024-06-02T22:08:49.464157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.preprocessing import StandardScaler\nimport polars as pl\n\ndef objective(trial):\n    # Define the actual input size and output size based on your data\n    input_size = 555  # Number of features in X\n    output_size = 368  # Number of targets in Y\n\n    # Tune the number of hidden layers and units per layer\n    hidden_layers = trial.suggest_int('hidden_layers', 3, 8)\n    hidden_units = trial.suggest_int('hidden_units', 512, 1024)\n    batch_size = trial.suggest_int('batch_size', 1536, 4608)\n    learning_rate = trial.suggest_loguniform('learning_rate', 2.5e-4, 2.5e-2)\n\n    X_train_tensor = torch.tensor(X_train, dtype=torch.float64)\n    Y_train_tensor = torch.tensor(Y_train, dtype=torch.float64)\n    X_val_tensor = torch.tensor(X_val, dtype=torch.float64)\n    Y_val_tensor = torch.tensor(Y_val, dtype=torch.float64)\n\n    # Define the architecture of the MLP\n    class MLPRegressor(nn.Module):\n        def __init__(self, input_size, output_size, hidden_layers, hidden_units):\n            super(MLPRegressor, self).__init__()\n            layers = []\n            layers.append(nn.Linear(input_size, hidden_units, dtype=torch.float64))\n            layers.append(nn.LeakyReLU(0.15))\n            for _ in range(hidden_layers - 1):\n                layers.append(nn.Linear(hidden_units, hidden_units, dtype=torch.float64))\n                layers.append(nn.LeakyReLU(0.15))\n            layers.append(nn.Linear(hidden_units, output_size, dtype=torch.float64))\n            self.model = nn.Sequential(*layers)\n        \n        def forward(self, x):\n            return self.model(x)\n\n    model = MLPRegressor(input_size, output_size, hidden_layers, hidden_units).double()\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n    def create_dataloader(X, Y, batch_size):\n        dataset = torch.utils.data.TensorDataset(X, Y)\n        return torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\n    train_loader = create_dataloader(X_train_tensor, Y_train_tensor, batch_size)\n    val_loader = create_dataloader(X_val_tensor, Y_val_tensor, batch_size)\n\n    # Train the model\n    def train(model, criterion, optimizer, train_loader, val_loader, epochs=4):\n        model.train()\n        for epoch in range(epochs):\n            for inputs, targets in train_loader:\n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                loss.backward()\n                optimizer.step()\n        \n        # Evaluate on the validation set\n        model.eval()\n        val_preds = []\n        val_targets = []\n        with torch.no_grad():\n            for val_inputs, val_targets_batch in val_loader:\n                val_outputs = model(val_inputs)\n                val_preds.append(val_outputs)\n                val_targets.append(val_targets_batch)\n        \n        val_preds = torch.cat(val_preds).cpu().numpy()\n        val_targets = torch.cat(val_targets).cpu().numpy()\n        val_preds[:, 61] = 0\n        mse = mean_squared_error(val_targets, val_preds)\n        \n        r2_values = []\n        for i in range(output_size):\n            r2 = r2_score(val_targets[:, i], val_preds[:, i])\n            r2_values.append(r2)\n        print(f\"R² for column 62: {r2_values[61]}\")\n\n        # Calculate the average R² excluding the value at index 61\n        r2_values_excluding_61 = [r2 for idx, r2 in enumerate(r2_values) if idx != 61]\n        avg_r2_excluding_61 = sum(r2_values_excluding_61) / len(r2_values_excluding_61)\n        print(\"Average R² excluding column 61:\", avg_r2_excluding_61)\n        return avg_r2_excluding_61\n\n    avg_r2_excluding_61 = train(model, criterion, optimizer, train_loader, val_loader)\n    return avg_r2_excluding_61\n\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=100)\n\nprint(\"Best hyperparameters: \", study.best_params)\nprint(\"Best value (MSE): \", study.best_value)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-02T23:13:27.880034Z","iopub.execute_input":"2024-06-02T23:13:27.880799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.48361Z","iopub.status.idle":"2024-06-02T22:08:56.484032Z","shell.execute_reply.started":"2024-06-02T22:08:56.483843Z","shell.execute_reply":"2024-06-02T22:08:56.48386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna.visualization as vis\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.485542Z","iopub.status.idle":"2024-06-02T22:08:56.485979Z","shell.execute_reply.started":"2024-06-02T22:08:56.485793Z","shell.execute_reply":"2024-06-02T22:08:56.485811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importance_plot = vis.plot_param_importances(study)\nimportance_plot.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.487849Z","iopub.status.idle":"2024-06-02T22:08:56.488218Z","shell.execute_reply.started":"2024-06-02T22:08:56.488039Z","shell.execute_reply":"2024-06-02T22:08:56.488054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt_history_plot = vis.plot_optimization_history(study)\nopt_history_plot.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.489249Z","iopub.status.idle":"2024-06-02T22:08:56.489645Z","shell.execute_reply.started":"2024-06-02T22:08:56.489444Z","shell.execute_reply":"2024-06-02T22:08:56.489477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parallel_plot = vis.plot_parallel_coordinate(study)\nparallel_plot.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.491396Z","iopub.status.idle":"2024-06-02T22:08:56.491792Z","shell.execute_reply.started":"2024-06-02T22:08:56.491587Z","shell.execute_reply":"2024-06-02T22:08:56.491602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contour_plot = vis.plot_contour(study)\ncontour_plot.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-02T22:08:56.493112Z","iopub.status.idle":"2024-06-02T22:08:56.493463Z","shell.execute_reply.started":"2024-06-02T22:08:56.493283Z","shell.execute_reply":"2024-06-02T22:08:56.493297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}