{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import LabelEncoder, OrdinalEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:08.527552Z","iopub.execute_input":"2024-12-31T11:54:08.527907Z","iopub.status.idle":"2024-12-31T11:54:09.945338Z","shell.execute_reply.started":"2024-12-31T11:54:08.527882Z","shell.execute_reply":"2024-12-31T11:54:09.944311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\nsample = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\ntrain.drop('id', axis=1, inplace=True)\ntest.drop('id', axis=1, inplace=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:10.105917Z","iopub.execute_input":"2024-12-31T11:54:10.106353Z","iopub.status.idle":"2024-12-31T11:54:19.767449Z","shell.execute_reply.started":"2024-12-31T11:54:10.106326Z","shell.execute_reply":"2024-12-31T11:54:19.766394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def date(Df):\n\n    Df['Policy Start Date'] = pd.to_datetime(Df['Policy Start Date'])\n    Df['Year'] = Df['Policy Start Date'].dt.year\n    Df['Day'] = Df['Policy Start Date'].dt.day\n    Df['Month'] = Df['Policy Start Date'].dt.month\n    Df['Month_name'] = Df['Policy Start Date'].dt.month_name()\n    Df['Day_of_week'] = Df['Policy Start Date'].dt.day_name()\n    Df['Week'] = Df['Policy Start Date'].dt.isocalendar().week\n    Df['Year_sin'] = np.sin(2 * np.pi * Df['Year'])\n    Df['Year_cos'] = np.cos(2 * np.pi * Df['Year'])\n    Df['Month_sin'] = np.sin(2 * np.pi * Df['Month'] / 12) \n    Df['Month_cos'] = np.cos(2 * np.pi * Df['Month'] / 12)\n    Df['Day_sin'] = np.sin(2 * np.pi * Df['Day'] / 31)  \n    Df['Day_cos'] = np.cos(2 * np.pi * Df['Day'] / 31)\n    Df['Group']=(Df['Year']-2020)*48+Df['Month']*4+Df['Day']//7\n    \n    Df.drop('Policy Start Date', axis=1, inplace=True)\n\n    return Df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:19.768602Z","iopub.execute_input":"2024-12-31T11:54:19.768923Z","iopub.status.idle":"2024-12-31T11:54:19.776239Z","shell.execute_reply.started":"2024-12-31T11:54:19.768888Z","shell.execute_reply":"2024-12-31T11:54:19.774621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fe(df):\n    \n    df['contract length'] = pd.cut(\n        df[\"Insurance Duration\"].fillna(99),  \n        bins=[-float('inf'), 1, 3, float('inf')],  \n        labels=[0, 1, 2]  \n    ).astype(int)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:19.77795Z","iopub.execute_input":"2024-12-31T11:54:19.778243Z","iopub.status.idle":"2024-12-31T11:54:19.797257Z","shell.execute_reply.started":"2024-12-31T11:54:19.778217Z","shell.execute_reply":"2024-12-31T11:54:19.796433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = date(train)\ntest = date(test)\n\ntrain = fe(train)\ntest = fe(test)\n\ncat_cols = [col for col in train.columns if train[col].dtype == 'object']\nfeature_cols = list(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:19.798256Z","iopub.execute_input":"2024-12-31T11:54:19.798589Z","iopub.status.idle":"2024-12-31T11:54:22.708016Z","shell.execute_reply.started":"2024-12-31T11:54:19.798558Z","shell.execute_reply":"2024-12-31T11:54:22.70707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CategoricalEncoder:\n    def __init__(self, train, test):\n        self.train = train\n        self.test = test\n\n    def frequency_encode(self, cat_cols, feature_cols, drop_org=False):\n        combined = pd.concat([self.train, self.test], axis=0, ignore_index=True)\n\n        new_cat_cols = [] \n        for col in cat_cols:\n            freq_encoding = combined[col].value_counts().to_dict()\n            \n            self.train[f\"{col}_freq\"] = self.train[col].map(freq_encoding).astype('float')\n            self.test[f\"{col}_freq\"] = self.test[col].map(freq_encoding).astype('float')\n\n            new_col_name = f\"{col}_freq\"\n            new_cat_cols.append(new_col_name)\n            feature_cols.append(new_col_name)\n            if drop_org:\n                feature_cols.remove(col)\n\n        return self.train, self.test, new_cat_cols, feature_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:22.708956Z","iopub.execute_input":"2024-12-31T11:54:22.709262Z","iopub.status.idle":"2024-12-31T11:54:22.715359Z","shell.execute_reply.started":"2024-12-31T11:54:22.709217Z","shell.execute_reply":"2024-12-31T11:54:22.714158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder = CategoricalEncoder(train, test)\ntrain, test, cat_cols, feature_cols = encoder.frequency_encode(cat_cols, feature_cols, drop_org=True)\n\ntrain = train[feature_cols + ['Premium Amount']]\ntest = test[feature_cols]\n\n# train = train.fillna(-111)\n# test = test.fillna(-111)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:22.716927Z","iopub.execute_input":"2024-12-31T11:54:22.71729Z","iopub.status.idle":"2024-12-31T11:54:26.736032Z","shell.execute_reply.started":"2024-12-31T11:54:22.717259Z","shell.execute_reply":"2024-12-31T11:54:26.735028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.drop('Premium Amount', axis=1)  \ny = train['Premium Amount']\n\ny_log = np.log1p(y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:26.737278Z","iopub.execute_input":"2024-12-31T11:54:26.737573Z","iopub.status.idle":"2024-12-31T11:54:26.911267Z","shell.execute_reply.started":"2024-12-31T11:54:26.737551Z","shell.execute_reply":"2024-12-31T11:54:26.91024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:54:26.913039Z","iopub.execute_input":"2024-12-31T11:54:26.91341Z","iopub.status.idle":"2024-12-31T11:54:26.917483Z","shell.execute_reply.started":"2024-12-31T11:54:26.913355Z","shell.execute_reply":"2024-12-31T11:54:26.916669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_squared_error\nimport joblib\n\n# Define the RMSLE metric\ndef rmsle(y_true, y_pred):\n    y_pred = np.maximum(y_pred, 0)  # Clamp predictions to prevent log issues\n    return np.sqrt(mean_squared_error(y_true, y_pred))\n\n# Optuna objective function\ndef objective(trial):\n    # Define the parameter search space\n    params = {\n        \"iterations\": trial.suggest_int('iterations', 1000, 4000),\n        \"learning_rate\": trial.suggest_loguniform('learning_rate', 0.01, 0.1),\n        \"depth\": trial.suggest_int('depth', 4, 10),\n        \"l2_leaf_reg\": trial.suggest_float('l2_leaf_reg', 0.1, 10),\n    }\n    \n    kf = KFold(n_splits=10, shuffle=True, random_state=42)\n    rmsle_scores = []\n    \n    for train_idx, valid_idx in kf.split(X):\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n        \n        model = CatBoostRegressor(\n            iterations=params['iterations'],\n            learning_rate=params['learning_rate'],\n            depth=params['depth'],\n            l2_leaf_reg=params['l2_leaf_reg'],\n            eval_metric=\"RMSE\",\n            random_seed=42,\n            verbose=0,\n            task_type='GPU',\n        )\n        \n        # Train model with early stopping\n        model.fit(\n            X_train,\n            y_train,\n            eval_set=(X_valid, y_valid),\n            early_stopping_rounds=200,\n        )\n        \n        # Validate and calculate RMSLE\n        predictions = np.maximum(0, model.predict(X_valid))\n        fold_rmsle = rmsle(np.expm1(y_valid), np.expm1(predictions))\n        rmsle_scores.append(fold_rmsle)\n    \n    # Return the mean RMSLE over all folds\n    return np.mean(rmsle_scores)\n\n# Run Optuna to find the best parameters\nstudy = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=20)\n\n# Get the best parameters\nbest_params = study.best_trial.params\nprint(\"Best parameters found:\")\nprint(best_params)\n\n# Train with best parameters using K-Fold\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = []\n\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n    print(f\"Training Fold {fold + 1}...\")\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n    \n    model = CatBoostRegressor(\n        iterations=best_params['iterations'],\n        learning_rate=best_params['learning_rate'],\n        depth=best_params['depth'],\n        l2_leaf_reg=best_params['l2_leaf_reg'],\n        eval_metric=\"RMSE\",\n        random_seed=42,\n        verbose=200,\n        task_type='GPU',\n    )\n    \n    # Train model\n    model.fit(\n        X_train,\n        y_train,\n        eval_set=(X_valid, y_valid),\n        early_stopping_rounds=200,\n    )\n    \n    # Save model for each fold\n    models.append(model)\n\nprint(\"All folds trained.\")\n\n# Make predictions on the test dataset by averaging across folds\ntest_predictions = np.zeros(len(test))\nfor model in models:\n    test_predictions += np.maximum(0, np.expm1(model.predict(test))) / len(models)\n\n# Save the trained models for future use\nfor i, model in enumerate(models):\n    joblib.dump(model, f\"fold_{i + 1}_model.pkl\")\n    print(f\"Fold {i + 1} model saved as 'fold_{i + 1}_model.pkl'\")\n\n# Example of loading one of the saved models\n# loaded_model = joblib.load(\"fold_1_model.pkl\")\n# print(\"Loaded Fold 1 Model\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T12:55:32.335764Z","iopub.execute_input":"2024-12-31T12:55:32.336065Z","iopub.status.idle":"2024-12-31T14:23:09.619176Z","shell.execute_reply.started":"2024-12-31T12:55:32.336039Z","shell.execute_reply":"2024-12-31T14:23:09.618421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.experimental import enable_hist_gradient_boosting  # Required to use HistGradientBoostingRegressor\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nfrom lightgbm.callback import log_evaluation\nfrom lightgbm.callback import early_stopping\nimport numpy as np\nimport pandas as pd\nimport joblib\nfrom catboost import CatBoostRegressor\n\n# Split validation data into a separate validation set for early stopping\nX_valid_train, X_valid_eval, y_valid_train, y_valid_eval = train_test_split(\n    X_valid, y_valid, test_size=0.2, random_state=42\n)\n\n# Define the ensemble function\ndef ensemble_predictions(models, test, weights=None):\n    \"\"\"Combine predictions from multiple models.\"\"\"\n    predictions = np.zeros(len(test))\n    if weights is None:\n        weights = [1 / len(models)] * len(models)\n    \n    for model, weight in zip(models, weights):\n        predictions += weight * np.maximum(0, np.expm1(model.predict(test)))\n    \n    return predictions\n\n# Train and evaluate models for each fold\nkf = KFold(n_splits=10, shuffle=True, random_state=42)\ncatboost_models = []\nlgbm_models = []\nxgb_models = []\nhgb_models = []  # For HistGradientBoostingRegressor\n\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n    print(f\"Training Fold {fold + 1}...\")\n\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n\n    # Train CatBoost model\n    catboost_model = CatBoostRegressor(\n        iterations=best_params['iterations'],\n        learning_rate=best_params['learning_rate'],\n        depth=best_params['depth'],\n        l2_leaf_reg=best_params['l2_leaf_reg'],\n        eval_metric=\"RMSE\",\n        random_seed=42,\n        verbose=200,\n        task_type='GPU',\n    )\n    catboost_model.fit(\n        X_train, y_train, eval_set=(X_valid, y_valid), early_stopping_rounds=200\n    )\n    catboost_models.append(catboost_model)\n\n    # Train LightGBM model\n    lgbm_model = LGBMRegressor(\n        n_estimators=best_params['iterations'],\n        learning_rate=best_params['learning_rate'],\n        max_depth=best_params['depth'],\n        reg_lambda=best_params['l2_leaf_reg'],\n        random_state=42,\n    )\n    lgbm_model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[early_stopping(stopping_rounds=200, verbose=True)],\n    )\n    lgbm_models.append(lgbm_model)\n\n    # Train XGBoost model\n    xgb_model = XGBRegressor(\n        n_estimators=best_params['iterations'],\n        learning_rate=best_params['learning_rate'],\n        max_depth=best_params['depth'],\n        reg_lambda=best_params['l2_leaf_reg'],\n        random_state=42,\n        tree_method='gpu_hist',\n    )\n    xgb_model.fit(\n        X_train, y_train, eval_set=[(X_valid, y_valid)], early_stopping_rounds=200, verbose=200\n    )\n    xgb_models.append(xgb_model)\n\n    # Train HistGradientBoosting model\n    hgb_model = HistGradientBoostingRegressor(\n        max_iter=best_params['iterations'],\n        learning_rate=best_params['learning_rate'],\n        max_depth=best_params['depth'],\n        l2_regularization=best_params['l2_leaf_reg'],\n        random_state=42,\n    )\n    hgb_model.fit(X_train, y_train)\n    hgb_models.append(hgb_model)\n\nprint(\"All folds trained.\")\n\n# Make predictions on the test dataset using ensemble\ntest_predictions = np.zeros(len(test))\nfor i in range(len(catboost_models)):\n    test_predictions += ensemble_predictions(\n        [catboost_models[i], lgbm_models[i], xgb_models[i], hgb_models[i]],\n        test,\n        weights=[0.4, 0.3, 0.2, 0.1]  # Adjust weights as needed\n    ) / len(catboost_models)\n\n# Prepare the submission file\nsample['Premium Amount'] = test_predictions\nsubmission_file = 'submission.csv'\nsample.to_csv(submission_file, index=False)\nprint(f\"Ensemble submission file saved as '{submission_file}'\")\n\n# Save the trained models for future use\nfor i, (cat_model, lgb_model, xgb_model, hgb_model) in enumerate(zip(catboost_models, lgbm_models, xgb_models, hgb_models)):\n    joblib.dump(cat_model, f\"fold_{i + 1}_catboost_model.pkl\")\n    joblib.dump(lgb_model, f\"fold_{i + 1}_lgbm_model.pkl\")\n    joblib.dump(xgb_model, f\"fold_{i + 1}_xgb_model.pkl\")\n    joblib.dump(hgb_model, f\"fold_{i + 1}_hgb_model.pkl\")\n    print(f\"Fold {i + 1} models saved.\")\n\n# Preview the submission file\nprint(sample.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:23:09.620128Z","iopub.execute_input":"2024-12-31T14:23:09.620473Z","iopub.status.idle":"2024-12-31T14:49:51.677793Z","shell.execute_reply.started":"2024-12-31T14:23:09.620449Z","shell.execute_reply":"2024-12-31T14:49:51.676907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.model_selection import KFold, GridSearchCV\nimport numpy as np\nimport joblib\n\n# Define parameter grid for GridSearchCV\nparams = {\n    'n_estimators': [1000, 1500, 2000],\n    'learning_rate': [0.01, 0.05],\n    'max_depth': [3, 5, 7],\n    'subsample': [0.6, 0.8],\n    'colsample_bytree': [0.6, 0.8]\n}\n\n# Prepare stacking data\nkf = KFold(n_splits=10, shuffle=True, random_state=42)\nn_splits = kf.get_n_splits(X)\n\nbase_model_predictions = {\n    'train': np.zeros((len(X), 3)),  # OOF predictions for 3 base models\n    'test': np.zeros((len(test), 3))  # Average test predictions\n}\n\n# Create OOF predictions\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n    print(f\"Processing Fold {fold + 1}...\")\n\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n\n    # Load pre-trained CatBoost model\n    cat_model = joblib.load(f\"/kaggle/working/fold_{fold + 1}_catboost_model.pkl\")\n    base_model_predictions['train'][valid_idx, 0] = cat_model.predict(X_valid)\n    base_model_predictions['test'][:, 0] += cat_model.predict(test) / n_splits\n\n    # Load pre-trained LightGBM model\n    lgb_model = joblib.load(f\"/kaggle/working/fold_{fold + 1}_lgbm_model.pkl\")\n    base_model_predictions['train'][valid_idx, 1] = lgb_model.predict(X_valid)\n    base_model_predictions['test'][:, 1] += lgb_model.predict(test) / n_splits\n\n    # Load pre-trained XGBoost model\n    xgb_model = joblib.load(f\"/kaggle/working/fold_{fold + 1}_xgb_model.pkl\")\n    base_model_predictions['train'][valid_idx, 2] = xgb_model.predict(X_valid)\n    base_model_predictions['test'][:, 2] += xgb_model.predict(test) / n_splits\n\nprint(\"OOF predictions generated for all base models.\")\n\n# Hyperparameter tuning using GridSearchCV\nprint(\"Starting hyperparameter tuning for meta-model...\")\ngrid_search = GridSearchCV(\n    estimator=XGBRegressor(random_state=42),\n    param_grid=params,\n    cv=3,\n    scoring='neg_mean_squared_error',\n    verbose=2\n)\n\n# Fit the meta-model using OOF predictions as training data\ngrid_search.fit(base_model_predictions['train'], y_log)\n\n# Use the best model from GridSearchCV\nmeta_model = grid_search.best_estimator_\nprint(\"Best hyperparameters for meta-model:\", grid_search.best_params_)\n\n# Make predictions using the tuned meta-model\nfinal_predictions = meta_model.predict(base_model_predictions['test'])\nfinal_predictions = np.maximum(0, np.expm1(final_predictions))  # Ensure predictions are non-negative\n\n# Prepare submission\nsample['Premium Amount'] = final_predictions\nstacking_submission_file = 'stacking_submission_GSCV.csv'\nsample.to_csv(stacking_submission_file, index=False)\nprint(f\"Stacking submission file saved as '{stacking_submission_file}'\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}