{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# introduce","metadata":{}},{"cell_type":"markdown","source":"<strong style=\"font-size:20px;\">In this notebook I focus on the optimization of models and parameters, if you want a detailed interpretation of feature engineering, please check out <a href=\"https://www.kaggle.com/code/zyh1104/feature-engineering-no-hyperparamer-lgb-xgb-cat\" target=\"_blank\">this notebook</a></strong>","metadata":{}},{"cell_type":"markdown","source":"# Import library","metadata":{}},{"cell_type":"code","source":"!pip install ray==2.10.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:33:29.010736Z","iopub.execute_input":"2024-12-31T09:33:29.011056Z","iopub.status.idle":"2024-12-31T09:33:41.885196Z","shell.execute_reply.started":"2024-12-31T09:33:29.011018Z","shell.execute_reply":"2024-12-31T09:33:41.88436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install google-cloud-bigquery==3.10.0\n!pip install google-cloud-storage==2.11.0  \n!pip install numpy==1.25.0  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:33:41.886553Z","iopub.execute_input":"2024-12-31T09:33:41.886803Z","iopub.status.idle":"2024-12-31T09:34:08.822995Z","shell.execute_reply.started":"2024-12-31T09:33:41.886782Z","shell.execute_reply":"2024-12-31T09:34:08.821728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install autogluon.tabular --no-cache-dir -q\n!pip install -U ipywidgets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:08.824849Z","iopub.execute_input":"2024-12-31T09:34:08.825124Z","iopub.status.idle":"2024-12-31T09:34:24.692628Z","shell.execute_reply.started":"2024-12-31T09:34:08.825097Z","shell.execute_reply":"2024-12-31T09:34:24.691485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom collections import Counter\nfrom sklearn.feature_selection import mutual_info_regression, SelectKBest, f_regression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\nimport logging\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\nimport optuna\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:24.694343Z","iopub.execute_input":"2024-12-31T09:34:24.69461Z","iopub.status.idle":"2024-12-31T09:34:29.03321Z","shell.execute_reply.started":"2024-12-31T09:34:24.694587Z","shell.execute_reply":"2024-12-31T09:34:29.032486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from autogluon.tabular import TabularPredictor\nfrom autogluon.common import space\n\nprint('done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:29.034043Z","iopub.execute_input":"2024-12-31T09:34:29.03473Z","iopub.status.idle":"2024-12-31T09:34:29.397103Z","shell.execute_reply.started":"2024-12-31T09:34:29.034687Z","shell.execute_reply":"2024-12-31T09:34:29.396156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:29.39806Z","iopub.execute_input":"2024-12-31T09:34:29.398706Z","iopub.status.idle":"2024-12-31T09:34:38.957317Z","shell.execute_reply.started":"2024-12-31T09:34:29.398678Z","shell.execute_reply":"2024-12-31T09:34:38.956467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"markdown","source":"<h2 style=\"font-size:16px;\">The processing methods I used in data preprocessing were to obtain the optimal solution by comparing CV score and LB score</h2>","metadata":{}},{"cell_type":"code","source":"#Set id as index\n\ntrain.set_index('id', inplace=True)\ntest.set_index('id', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:38.95814Z","iopub.execute_input":"2024-12-31T09:34:38.958474Z","iopub.status.idle":"2024-12-31T09:34:38.969275Z","shell.execute_reply.started":"2024-12-31T09:34:38.958441Z","shell.execute_reply":"2024-12-31T09:34:38.96837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_cols = list(train.select_dtypes(include=['number']).columns)\ncat_cols = list(train.select_dtypes(exclude=['number', 'datetime64[ns]']).columns)\ndatetime_cols = ['Policy Start Date']\n\nif 'Premium Amount' in num_cols:\n    num_cols.remove('Premium Amount')\nif 'Policy Start Date' in cat_cols:\n    cat_cols.remove('Policy Start Date')\n\n    \nprint(\"Numerical columns:\")\nprint(num_cols)\nprint(\"\\nCategorical columns excluding datetime columns:\")\nprint(cat_cols)\nprint(\"\\nDatetime column:\")\nprint(datetime_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:38.970023Z","iopub.execute_input":"2024-12-31T09:34:38.970296Z","iopub.status.idle":"2024-12-31T09:34:39.823899Z","shell.execute_reply.started":"2024-12-31T09:34:38.970226Z","shell.execute_reply":"2024-12-31T09:34:39.822921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handling of missing values","metadata":{}},{"cell_type":"markdown","source":"#### For missing values, I processed only the classification features","metadata":{}},{"cell_type":"code","source":"\ndef fill_missing_values(df, num_cols, cat_cols):\n    \n    # The missing value for filling numerical features is -1\n    #for col in num_cols:\n    #    if col in df.columns:\n    #        df[col] = df[col].fillna(-1)\n            \n    # Missing value of fill classification feature is 'Unknown'\n    for col in cat_cols:\n        if col in df.columns:\n            df[col] = df[col].fillna('Unknown')\n    \n    return df\n\ntrain = fill_missing_values(train, num_cols, cat_cols)\ntest = fill_missing_values(test, num_cols, cat_cols)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:39.826628Z","iopub.execute_input":"2024-12-31T09:34:39.826896Z","iopub.status.idle":"2024-12-31T09:34:40.940019Z","shell.execute_reply.started":"2024-12-31T09:34:39.826872Z","shell.execute_reply":"2024-12-31T09:34:40.939319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Numerical feature","metadata":{}},{"cell_type":"markdown","source":"#### Here, I perform a log transformation on the features 'Annual Income' and 'Previous Claims'.\n","metadata":{}},{"cell_type":"code","source":"\ndef log_transform_features(df, num_cols):\n    X_num = df[num_cols]\n    X_num_log = pd.DataFrame(np.log1p(X_num), columns=X_num.columns)\n    df[num_cols] = X_num_log\n    return df\n    \nlog_transform_cols = ['Annual Income', 'Previous Claims']\n\n\ntrain = log_transform_features(train, log_transform_cols)\ntest = log_transform_features(test, log_transform_cols)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:40.941114Z","iopub.execute_input":"2024-12-31T09:34:40.941367Z","iopub.status.idle":"2024-12-31T09:34:41.011917Z","shell.execute_reply.started":"2024-12-31T09:34:40.941345Z","shell.execute_reply":"2024-12-31T09:34:41.011182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Datetime feature","metadata":{}},{"cell_type":"markdown","source":"#### Datetime features I only processed the year\n","metadata":{}},{"cell_type":"code","source":"\ndef process_date_features(df, date_col):\n    df[date_col] = pd.to_datetime(df[date_col])\n    \n    df['Year'] = df[date_col].dt.year\n    # df['Month'] = df[date_col].dt.month\n    # df['Day'] = df[date_col].dt.day\n\n    \n    #df[\"seconds since 1970\"] = df[date_col].astype(\"int64\") // 10**9\n    \n    df['YearSin'] = np.sin(2 * np.pi * df['Year'] / 4)\n    df['YearCos'] = np.cos(2 * np.pi * df['Year'] / 4)\n    # df['MonthSin'] = np.sin(2 * np.pi * df['Month'] / 12)\n    # df['MonthCos'] = np.cos(2 * np.pi * df['Month'] / 12)\n    # df['DaySin'] = np.sin(2 * np.pi * df['Day'] / 30)\n    # df['DayCos'] = np.cos(2 * np.pi * df['Day'] / 30)\n    \n    # df['Season'] = df['Month'].apply(lambda x: 'Winter' if x in [12, 1, 2] else\n    #                                            'Spring' if x in [3, 4, 5] else\n    #                                            'Summer' if x in [6, 7, 8] else\n    #                                            'Autumn')\n    # df = pd.get_dummies(df, columns=['Season'], prefix=['Season'])\n    df = df.drop(date_col, axis=1)\n    \n    return df\n\ntrain = process_date_features(train, 'Policy Start Date')\ntest = process_date_features(test, 'Policy Start Date')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:41.012744Z","iopub.execute_input":"2024-12-31T09:34:41.012966Z","iopub.status.idle":"2024-12-31T09:34:42.21975Z","shell.execute_reply.started":"2024-12-31T09:34:41.012947Z","shell.execute_reply":"2024-12-31T09:34:42.218992Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Classification feature","metadata":{}},{"cell_type":"markdown","source":"#### For the processing of classification features, I use frequency coding","metadata":{}},{"cell_type":"code","source":"# Remove the comment if you use the second date-time processing method\n#cat_cols.extend(['Monthname', 'Dayofweek'])\ndef frequency_encode(df, columns):\n    encoded_df = df.copy()\n    if 'Policy Start Date' in df.columns:\n        df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n        # Add the Monthname and Dayofweek columns\n        #df['Monthname'] = df['Policy Start Date'].dt.month_name()\n        #df['Dayofweek'] = df['Policy Start Date'].dt.day_name()\n\n    for col in columns:\n        frequency = df[col].value_counts(normalize=True).to_dict()\n        new_col_name = f\"{col}_Freq_Enc\"\n        encoded_df[new_col_name] = df[col].map(frequency)\n\n    encoded_df = encoded_df.drop(columns=columns)\n    return encoded_df\n\ntrain = frequency_encode(train, cat_cols)\ntest = frequency_encode(test, cat_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:42.220587Z","iopub.execute_input":"2024-12-31T09:34:42.220803Z","iopub.status.idle":"2024-12-31T09:34:45.42831Z","shell.execute_reply.started":"2024-12-31T09:34:42.220784Z","shell.execute_reply":"2024-12-31T09:34:45.427602Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## View preprocessed data","metadata":{}},{"cell_type":"markdown","source":"#### Check whether the data is processed successfully","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.428981Z","iopub.execute_input":"2024-12-31T09:34:45.42921Z","iopub.status.idle":"2024-12-31T09:34:45.455088Z","shell.execute_reply.started":"2024-12-31T09:34:45.429191Z","shell.execute_reply":"2024-12-31T09:34:45.454336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.455833Z","iopub.execute_input":"2024-12-31T09:34:45.456071Z","iopub.status.idle":"2024-12-31T09:34:45.477203Z","shell.execute_reply.started":"2024-12-31T09:34:45.456052Z","shell.execute_reply":"2024-12-31T09:34:45.47627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Optuna","metadata":{}},{"cell_type":"markdown","source":"### Here you can choose the parameter range yourself","metadata":{}},{"cell_type":"code","source":"\"\"\"\nlogging.getLogger('lightgbm').setLevel(logging.ERROR)\nwarnings.filterwarnings(\"ignore\", category=UserWarning, message=\".*Found whitespace in feature_names.*\")\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\ntrain.columns = train.columns.str.replace(\" \", \"_\", regex=True)\ntest.columns = test.columns.str.replace(\" \", \"_\", regex=True)\n\ny = train['Premium_Amount']\nX_train = train.drop('Premium_Amount', axis=1)\n\ny_log = np.log1p(y)\n\ntest = test[X_train.columns]\n\n\ndef objective(trial):\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 2000, 5000),\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"boosting_type\": \"gbdt\",\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 64, 128),\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.1),\n        \"feature_fraction\": trial.suggest_uniform(\"feature_fraction\", 0.7, 1.0),\n        \"bagging_fraction\": trial.suggest_uniform(\"bagging_fraction\", 0.7, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 5, 10),\n        \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 50, 100),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 6, 12),\n        \"lambda_l1\": trial.suggest_loguniform(\"lambda_l1\", 1e - 3, 10.0),\n        \"lambda_l2\": trial.suggest_loguniform(\"lambda_l2\", 1e - 3, 10.0),\n        \"device_type\": \"gpu\",\n        \"seed\": 42,\n        \"verbose\": -1,\n    }\n\n    n_splits = 5\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    cv_scores = []\n\n    for train_index, val_index in kf.split(X_train):\n        X_train_fold, X_val_fold = X_train.iloc[train_index], X_train.iloc[val_index]\n        y_train_fold, y_val_fold = y_log.iloc[train_index], y_log.iloc[val_index]\n\n        dtrain = lgb.Dataset(X_train_fold, label=y_train_fold)\n        dval = lgb.Dataset(X_val_fold, label=y_val_fold, reference=dtrain)\n\n        early_stopping = lgb.early_stopping(stopping_rounds=50, verbose=False)\n\n        model = lgb.train(\n            params,\n            dtrain,\n            valid_sets=[dval],\n            callbacks=[early_stopping],\n        )\n\n        y_val_pred = model.predict(X_val_fold)\n        y_val_pred = np.expm1(y_val_pred)\n        fold_rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_val_fold), np.maximum(y_val_pred, 0)))\n        cv_scores.append(fold_rmsle)\n\n    return np.mean(cv_scores)\n\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=10)\n\nprint(\"Best parameters:\", study.best_params)\nprint(f\"Best RMSLE: {study.best_value:.4f}\")\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.478209Z","iopub.execute_input":"2024-12-31T09:34:45.478587Z","iopub.status.idle":"2024-12-31T09:34:45.495048Z","shell.execute_reply.started":"2024-12-31T09:34:45.478552Z","shell.execute_reply":"2024-12-31T09:34:45.494283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nbest_params = study.best_params\nbest_params.update({\n    \"objective\": \"regression\",\n    \"metric\": \"rmse\",\n    \"device_type\": \"gpu\",\n    \"seed\": 42,\n    \"verbose\": -1,\n})\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\noof_predictions = np.zeros(len(X_train))\nfinal_test_predictions = np.zeros(len(test))\n\nbest_model = None\n\nfor train_index, val_index in kf.split(X_train):\n    X_train_fold, X_val_fold = X_train.iloc[train_index], X_train.iloc[val_index]\n    y_train_fold, y_val_fold = y_log.iloc[train_index], y_log.iloc[val_index]\n\n    dtrain = lgb.Dataset(X_train_fold, label=y_train_fold)\n    dval = lgb.Dataset(X_val_fold, label=y_val_fold, reference=dtrain)\n\n    early_stopping = lgb.early_stopping(stopping_rounds=50, verbose=False)\n\n    model = lgb.train(\n        best_params,\n        dtrain,\n        valid_sets=[dval],\n        callbacks=[early_stopping],\n    )\n\n    best_model = model\n\n    oof_predictions[val_index] = np.expm1(model.predict(X_val_fold))\n\n    final_test_predictions += np.expm1(model.predict(test)) / n_splits\n\nfinal_rmsle = np.sqrt(mean_squared_log_error(y, np.maximum(oof_predictions, 0)))\nprint(f\"Final RMSLE on out - of - fold predictions: {final_rmsle:.4f}\")\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.496022Z","iopub.execute_input":"2024-12-31T09:34:45.496391Z","iopub.status.idle":"2024-12-31T09:34:45.51485Z","shell.execute_reply.started":"2024-12-31T09:34:45.496357Z","shell.execute_reply":"2024-12-31T09:34:45.513939Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Default parameters & optimized parameters","metadata":{}},{"cell_type":"markdown","source":"### Default parameters CV score 1.0457, LB score 1.04531\n\n### optimized parameters CV score 1.0454, LB score 1.04463","metadata":{}},{"cell_type":"code","source":"\"\"\"\nlogging.getLogger('lightgbm').setLevel(logging.ERROR)  \nwarnings.filterwarnings(\"ignore\", category=UserWarning, message=\".*Found whitespace in feature_names.*\")\n\ny = train['Premium Amount']\nX_train = train.drop('Premium Amount', axis=1)\ny_log = np.log1p(y)\ntest = test[X_train.columns]\n\n#CV score 1.0457\n#LB score 1.04531\n\nparams = {\n    \"objective\": \"regression\",\n    \"metric\": \"rmse\",\n    \"seed\": 42,\n    \"verbose\": -1,  \n}\n\n#CV score 1.0454\n#LB score 1.04463\n\nparams = {\n    'num_leaves': 81,  \n    'learning_rate': 0.028996043553364135,  \n    'feature_fraction': 0.9329053997142247,  \n    'bagging_fraction': 0.9211443767433244,  \n    'bagging_freq': 6,  \n    'min_data_in_leaf': 64,  \n    'max_depth': 9,  \n    'lambda_l1': 0.10719079649349326,  \n    'lambda_l2': 9.687339663963053,  \n    'random_state': 42,\n    'eval_metric': 'rmse',\n    'n_estimators': 3053,  \n    'early_stopping_round': 200,\n    'verbose': -1,\n}\n\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\ncv_scores = []\noof_predictions = np.zeros(len(X_train))\nfinal_test_predictions = np.zeros(len(test))  \nbest_model = None\n\nfor train_index, val_index in kf.split(X_train):\n    X_train_fold, X_val_fold = X_train.iloc[train_index], X_train.iloc[val_index]\n    y_train_fold, y_val_fold = y_log.iloc[train_index], y_log.iloc[val_index]  \n\n    dtrain = lgb.Dataset(X_train_fold, label=y_train_fold)\n    dval = lgb.Dataset(X_val_fold, label=y_val_fold, reference=dtrain)\n\n    early_stopping = lgb.early_stopping(stopping_rounds=50, verbose=False)\n\n    model = lgb.train(\n        params,\n        dtrain,\n        valid_sets=[dval],\n        callbacks=[early_stopping],  \n    )\n\n    best_model = model\n\n    oof_predictions[val_index] = np.expm1(model.predict(X_val_fold))\n\n    final_test_predictions += np.expm1(model.predict(test)) / n_splits  \n\n    y_val_pred = model.predict(X_val_fold)\n    fold_rmsle = np.sqrt(mean_squared_log_error(y_val_fold, np.maximum(y_val_pred, 0)))\n    cv_scores.append(fold_rmsle)\n\nprint(f\"Mean RMSLE across {n_splits} folds: {np.mean(cv_scores):.4f}\")\nlgb_rmsle = np.sqrt(mean_squared_log_error(y, np.maximum(oof_predictions, 0)))\nprint(f\"Final RMSLE on out-of-fold predictions: {lgb_rmsle:.4f}\")\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.515653Z","iopub.execute_input":"2024-12-31T09:34:45.515907Z","iopub.status.idle":"2024-12-31T09:34:45.529198Z","shell.execute_reply.started":"2024-12-31T09:34:45.515877Z","shell.execute_reply":"2024-12-31T09:34:45.528533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Custom &optuna","metadata":{}},{"cell_type":"markdown","source":"### CV score 1.04556\n### LB score 1.0450","metadata":{}},{"cell_type":"code","source":"\"\"\"\nlogging.getLogger(\"lightgbm\").setLevel(logging.ERROR)\nwarnings.filterwarnings(\"ignore\", category=UserWarning, message=\".*Found whitespace in feature_names.*\")\n\n\ntrain.columns = train.columns.str.replace(\" \", \"_\", regex=True)\ntest.columns = test.columns.str.replace(\" \", \"_\", regex=True)\n\ny = train['Premium_Amount']\nX = train.drop('Premium_Amount', axis=1)\n\ny_log = np.log1p(y)\n\ntest = test[X.columns]\n\ndef objective(trial):\n    model_type = trial.suggest_categorical(\"model_type\", [\"lightgbm\", \"xgboost\", \"catboost\"])\n    params = {}\n    if model_type == \"lightgbm\":\n        params = {\n            \"objective\": \"regression\",\n            \"metric\": \"rmse\",\n            \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 200),\n            \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.1),\n            \"feature_fraction\": trial.suggest_uniform(\"feature_fraction\", 0.6, 1.0),\n            \"bagging_fraction\": trial.suggest_uniform(\"bagging_fraction\", 0.6, 1.0),\n            \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 10, 100),\n        }\n        model = lgb.LGBMRegressor(**params, random_state=42)\n    elif model_type == \"xgboost\":\n        params = {\n            \"objective\": \"reg:squarederror\",\n            \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.1),\n            \"max_depth\": trial.suggest_int(\"max_depth\", 3, 10),\n            \"colsample_bytree\": trial.suggest_uniform(\"colsample_bytree\", 0.6, 1.0),\n            \"subsample\": trial.suggest_uniform(\"subsample\", 0.6, 1.0),\n            \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 10),\n        }\n        model = xgb.XGBRegressor(**params, random_state=42)\n    else:\n        params = {\n            \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.1),\n            \"depth\": trial.suggest_int(\"depth\", 3, 10),\n            \"l2_leaf_reg\": trial.suggest_loguniform(\"l2_leaf_reg\", 1e-2, 10),\n            \"bagging_temperature\": trial.suggest_uniform(\"bagging_temperature\", 0.0, 1.0),\n        }\n        model = cb.CatBoostRegressor(**params, random_state=42, verbose=0)\n    \n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    rmsle_scores = []\n    for train_idx, valid_idx in kf.split(X):\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n        \n        if model_type == \"lightgbm\":\n            model.fit(\n                X_train, y_train, \n                eval_set=[(X_valid, y_valid)], \n                eval_metric=\"rmse\"\n            )\n        else:\n            model.fit(\n                X_train, y_train, \n                eval_set=[(X_valid, y_valid)], \n                verbose=0, \n                early_stopping_rounds=50\n            )\n        \n        y_pred = model.predict(X_valid)\n        rmsle = np.sqrt(mean_squared_log_error(np.expm1(y_valid), np.maximum(np.expm1(y_pred), 0)))\n        rmsle_scores.append(rmsle)\n    \n    return np.mean(rmsle_scores)\n\n\n\nstudy = optuna.create_study(direction=\"minimize\", study_name=\"Auto Model Selection\")\nstudy.optimize(objective, n_trials=50)\nprint(\"Best trial:\", study.best_trial.params)\nbest_params = study.best_trial.params\nif best_params[\"model_type\"] == \"lightgbm\":\n    final_model = lgb.LGBMRegressor(**{k: v for k, v in best_params.items() if k != \"model_type\"})\nelif best_params[\"model_type\"] == \"xgboost\":\n    final_model = xgb.XGBRegressor(**{k: v for k, v in best_params.items() if k != \"model_type\"})\nelse:\n    final_model = cb.CatBoostRegressor(**{k: v for k, v in best_params.items() if k != \"model_type\"}, verbose=0)\n\nfinal_model.fit(X, y_log)\ntest_pred_log = final_model.predict(test)\nfinal_test_predictions = np.expm1(test_pred_log)\n\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.530186Z","iopub.execute_input":"2024-12-31T09:34:45.530525Z","iopub.status.idle":"2024-12-31T09:34:45.54733Z","shell.execute_reply.started":"2024-12-31T09:34:45.530493Z","shell.execute_reply":"2024-12-31T09:34:45.546462Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Autogluon","metadata":{}},{"cell_type":"markdown","source":"### You can try changing the value of \"time_limit\" to get better results","metadata":{}},{"cell_type":"markdown","source":"### CV score 1.0450\n### LB score 1.04446","metadata":{}},{"cell_type":"code","source":"y = train['Premium Amount']\nX_train = train.drop('Premium Amount', axis=1)\ny_log = np.log1p(y)\n\n# 处理测试数据的缺失列\nmissing_cols = set(X_train.columns) - set(test.columns)\nfor col in missing_cols:\n    test[col] = 0\ntest = test[X_train.columns]\n\nn_splits = 5\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# 初始化存储变量\ncv_scores = []\noof_predictions = np.zeros(len(X_train))\nfinal_test_predictions = np.zeros(len(test))\n\npreset_params = {\n    \"time_limit\": 3600,\n    \"presets\": \"best_quality\",\n    \"verbosity\": 2,\n    \"use_bag_holdout\": True,\n    \"hyperparameters\": {\n        'GBM': {'num_boost_round': 1500, 'learning_rate': 0.05, 'max_depth': 20},\n        'CAT': {'iterations': 1500, 'learning_rate': 0.01, 'depth': 10},\n        'NN_TORCH': {'epochs': 200, 'learning_rate': 1e-3},\n    },\n    \"num_gpus\": 1,\n}\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n    print(f\"Starting fold {fold + 1}/{n_splits}...\")\n\n    # 构造训练和验证集\n    train_fold_data = pd.concat([\n        X_train.iloc[train_idx],\n        y_log.iloc[train_idx]\n    ], axis=1)\n    val_fold_data = pd.concat([\n        X_train.iloc[val_idx],\n        y_log.iloc[val_idx]\n    ], axis=1)\n\n    # 设置列名\n    train_fold_data.columns = list(X_train.columns) + [\"Premium Amount\"]\n    val_fold_data.columns = train_fold_data.columns\n\n    # 模型训练\n    predictor = TabularPredictor(label=\"Premium Amount\")\n    predictor.fit(train_data=train_fold_data, tuning_data=val_fold_data, **preset_params)\n\n    # 获取验证集预测值\n    val_preds = np.clip(predictor.predict(val_fold_data), 0, None)\n    val_preds = np.expm1(val_preds)\n    oof_predictions[val_idx] = val_preds\n\n    # 获取测试集预测值\n    test_preds = np.clip(predictor.predict(test), 0, None)\n    final_test_predictions += np.expm1(test_preds) / n_splits\n\n    # RMSLE 评估\n    fold_rmsle = np.sqrt(mean_squared_log_error(y.iloc[val_idx], np.maximum(val_preds, 0)))\n    cv_scores.append(fold_rmsle)\n    print(f\"Fold {fold + 1} RMSLE: {fold_rmsle:.4f}\")\n\n# 打印交叉验证结果\nprint(f\"Mean RMSLE across {n_splits} folds: {np.mean(cv_scores):.4f}\")\nfinal_rmsle = np.sqrt(mean_squared_log_error(y, np.maximum(oof_predictions, 0)))\nprint(f\"Final RMSLE on out-of-fold predictions: {final_rmsle:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:34:45.548389Z","iopub.execute_input":"2024-12-31T09:34:45.548667Z","iopub.status.idle":"2024-12-31T09:42:01.259623Z","shell.execute_reply.started":"2024-12-31T09:34:45.54864Z","shell.execute_reply":"2024-12-31T09:42:01.258661Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"\ntest_submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\nfinal_test_predictions = pd.Series(final_test_predictions, index=test_submission['id'])\n\nsubmission = pd.DataFrame({\n    'id': test_submission['id'],\n    'Premium Amount': final_test_predictions.values\n})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file created:\")\nprint(submission.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T09:42:01.534079Z","iopub.status.idle":"2024-12-31T09:42:01.534378Z","shell.execute_reply":"2024-12-31T09:42:01.534267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}