{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Kaggle Competition: Regression with an Insurance Data","metadata":{}},{"cell_type":"markdown","source":"## **1 Import Libraries**","metadata":{}},{"cell_type":"code","source":"# Import Libraries\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_squared_error\nfrom catboost import CatBoostRegressor, Pool\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.preprocessing import LabelEncoder\nimport warnings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:09:28.510768Z","iopub.execute_input":"2024-12-28T23:09:28.511152Z","iopub.status.idle":"2024-12-28T23:09:28.774537Z","shell.execute_reply.started":"2024-12-28T23:09:28.511124Z","shell.execute_reply":"2024-12-28T23:09:28.773654Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **2 Load Data**","metadata":{}},{"cell_type":"code","source":"# Load Data\ntrain_data = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv', index_col='id')\ntest_data = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv', index_col='id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:10:59.255021Z","iopub.execute_input":"2024-12-28T23:10:59.25538Z","iopub.status.idle":"2024-12-28T23:11:06.325484Z","shell.execute_reply.started":"2024-12-28T23:10:59.255352Z","shell.execute_reply":"2024-12-28T23:11:06.324311Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **3 Data Exploration**","metadata":{}},{"cell_type":"code","source":"#explore train data\ntrain_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:11:13.456856Z","iopub.execute_input":"2024-12-28T23:11:13.457183Z","iopub.status.idle":"2024-12-28T23:11:13.477718Z","shell.execute_reply.started":"2024-12-28T23:11:13.457157Z","shell.execute_reply":"2024-12-28T23:11:13.476822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# explore test data\ntest_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:11:15.835314Z","iopub.execute_input":"2024-12-28T23:11:15.835648Z","iopub.status.idle":"2024-12-28T23:11:15.856641Z","shell.execute_reply.started":"2024-12-28T23:11:15.835619Z","shell.execute_reply":"2024-12-28T23:11:15.855418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:11:18.196624Z","iopub.execute_input":"2024-12-28T23:11:18.196965Z","iopub.status.idle":"2024-12-28T23:11:18.817365Z","shell.execute_reply.started":"2024-12-28T23:11:18.196936Z","shell.execute_reply":"2024-12-28T23:11:18.816109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:11:20.516478Z","iopub.execute_input":"2024-12-28T23:11:20.516839Z","iopub.status.idle":"2024-12-28T23:11:20.941329Z","shell.execute_reply.started":"2024-12-28T23:11:20.516807Z","shell.execute_reply":"2024-12-28T23:11:20.94024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T22:54:16.636827Z","iopub.execute_input":"2024-12-28T22:54:16.637198Z","iopub.status.idle":"2024-12-28T22:54:17.251764Z","shell.execute_reply.started":"2024-12-28T22:54:16.637165Z","shell.execute_reply":"2024-12-28T22:54:17.250844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T23:11:23.136585Z","iopub.execute_input":"2024-12-28T23:11:23.136943Z","iopub.status.idle":"2024-12-28T23:11:23.551555Z","shell.execute_reply.started":"2024-12-28T23:11:23.136909Z","shell.execute_reply":"2024-12-28T23:11:23.550848Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **5 Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"# Convert 'Policy Start Date' to datetime format\ntrain_data['Policy Start Date'] = pd.to_datetime(train_data['Policy Start Date'], errors='coerce')\ntest_data['Policy Start Date'] = pd.to_datetime(test_data['Policy Start Date'], errors='coerce')\n\n# Feature Engineering\nfor df in [train_data, test_data]:\n    df['Policy_Start_Year'] = df['Policy Start Date'].dt.year\n    df['Policy_Start_Month'] = df['Policy Start Date'].dt.month\n    df['Policy_Start_Day'] = df['Policy Start Date'].dt.day\n    df['Policy_Age'] = 2024 - df['Policy_Start_Year']  # Assuming current year is 2024\n    df['Year_Month_Interaction'] = df['Policy_Start_Year'] * df['Policy_Start_Month']\n\n# Health Score Optimization\nif 'Health Score' in df.columns:\n    df['Health Score'] = df['Health Score'].fillna(-1)  # Replace NaN with -1\n    df['HealthScore'] = df['Health Score'].astype(int).astype(str)  # Convert to int, then to string\n\n\n# Drop the original datetime column\ntrain_data.drop('Policy Start Date', axis=1, inplace=True)\ntest_data.drop('Policy Start Date', axis=1, inplace=True)\n\n\n# Handle Missing Values for Numerical Columns\nnumerical_cols = train_data.select_dtypes(include=['number']).columns\nfor col in numerical_cols:\n    train_data[col].fillna(train_data[col].median(), inplace=True)\n    if col in test_data.columns:\n        test_data[col].fillna(test_data[col].median(), inplace=True)\n\n# Handle Missing Values for Categorical Features\ncategorical_cols = train_data.select_dtypes(include=['object']).columns.tolist()\nfor col in categorical_cols:\n    train_data[col] = train_data[col].fillna('missing')  # Replace NaN with 'missing'\n    if col in test_data.columns:\n        test_data[col] = test_data[col].fillna('missing')  # Replace NaN with 'missing'\n\n# Ensure 'Health Score' and 'HealthScore' are treated as categorical\nif 'Health Score' in train_data.columns:\n    categorical_cols.append('Health Score')\nif 'HealthScore' in train_data.columns:\n    categorical_cols.append('HealthScore')\n\n# Convert categorical features to strings\nfor col in categorical_cols:\n    train_data[col] = train_data[col].astype(str)\n    if col in test_data.columns:\n        test_data[col] = test_data[col].astype(str)\n\n# Log-transform target\ntarget_column = 'Premium Amount'\nif target_column not in train_data.columns:\n    raise KeyError(f\"Target column '{target_column}' not found in training data.\")\ny = np.log1p(train_data[target_column])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T22:54:22.516242Z","iopub.execute_input":"2024-12-28T22:54:22.516573Z","iopub.status.idle":"2024-12-28T22:54:29.196766Z","shell.execute_reply.started":"2024-12-28T22:54:22.516546Z","shell.execute_reply":"2024-12-28T22:54:29.195701Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **6 Building The Model**","metadata":{}},{"cell_type":"code","source":"# CatBoost Pool (Handles categorical features automatically)\ntrain_pool = Pool(X_train, y_train, cat_features=categorical_cols)\nval_pool = Pool(X_val, y_val, cat_features=categorical_cols)\ntest_pool = Pool(X_test, cat_features=categorical_cols)\n\n# Train CatBoost Model\ncatboost_model = CatBoostRegressor(\n    iterations=2000,               # Moderate number of iterations\n    learning_rate=0.02,            # Slightly higher learning rate\n    depth=8,                       # Tree depth for complexity control\n    l2_leaf_reg=8,                 # Regularization for overfitting\n    subsample=0.8,                 # Use 80% of the dataset in each iteration\n    colsample_bylevel=0.8,         # Use 80% of features per tree level\n    loss_function='RMSE',\n    eval_metric='RMSE',\n    early_stopping_rounds=100,     # Stop early if no improvement\n    random_seed=42,\n    verbose=100                    # Print progress every 100 iterations\n)\ncatboost_model.fit(train_pool, eval_set=val_pool, use_best_model=True)\n\n# Evaluate Model\nval_preds = catboost_model.predict(val_pool)\nval_preds = np.expm1(val_preds)  # Reverse log-transform\nactual_y_val = np.expm1(y_val)  # Reverse log-transform\nval_rmse = np.sqrt(mean_squared_error(actual_y_val, val_preds))\nprint(f\"Validation RMSE: {val_rmse}\")\n\n# Cross-Validation\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\ncv_scores = []\n\nfor train_idx, val_idx in kf.split(X):\n    X_kf_train, X_kf_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_kf_train, y_kf_val = y.iloc[train_idx], y.iloc[val_idx]\n\n    train_pool_kf = Pool(X_kf_train, y_kf_train, cat_features=categorical_cols)\n    val_pool_kf = Pool(X_kf_val, y_kf_val, cat_features=categorical_cols)\n\n    catboost_model.fit(train_pool_kf, eval_set=val_pool_kf, use_best_model=True, verbose=100)\n    preds = catboost_model.predict(val_pool_kf)\n    preds = np.expm1(preds)\n    y_kf_val = np.expm1(y_kf_val)\n    rmse = np.sqrt(mean_squared_error(y_kf_val, preds))\n    cv_scores.append(rmse)\n\nprint(f\"Mean CV RMSE: {np.mean(cv_scores)}\")\n\n# Predict on Test Data\ntest_preds = catboost_model.predict(test_pool)\ntest_preds = np.expm1(test_preds)  # Reverse log-transform\n\n# Prepare Submission\nsubmission = pd.DataFrame({\n    'id': test_ids,\n    'Premium Amount': test_preds\n})\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"Submission file created: submission.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}