{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom sklearn.metrics import make_scorer, mean_squared_log_error\nimport xgboost as xgb\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import plot_importance\nfrom lightgbm import early_stopping, log_evaluation","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:24.729776Z","iopub.execute_input":"2024-12-06T14:19:24.730129Z","iopub.status.idle":"2024-12-06T14:19:31.018428Z","shell.execute_reply.started":"2024-12-06T14:19:24.730081Z","shell.execute_reply":"2024-12-06T14:19:31.017434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:31.020348Z","iopub.execute_input":"2024-12-06T14:19:31.021133Z","iopub.status.idle":"2024-12-06T14:19:40.897706Z","shell.execute_reply.started":"2024-12-06T14:19:31.021085Z","shell.execute_reply":"2024-12-06T14:19:40.896848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:40.898856Z","iopub.execute_input":"2024-12-06T14:19:40.899198Z","iopub.status.idle":"2024-12-06T14:19:41.690781Z","shell.execute_reply.started":"2024-12-06T14:19:40.899166Z","shell.execute_reply":"2024-12-06T14:19:41.689797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:41.69269Z","iopub.execute_input":"2024-12-06T14:19:41.692989Z","iopub.status.idle":"2024-12-06T14:19:42.367586Z","shell.execute_reply.started":"2024-12-06T14:19:41.692959Z","shell.execute_reply":"2024-12-06T14:19:42.366535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = df_train.copy()\ntest = df_test.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:42.368704Z","iopub.execute_input":"2024-12-06T14:19:42.369048Z","iopub.status.idle":"2024-12-06T14:19:42.627746Z","shell.execute_reply.started":"2024-12-06T14:19:42.369016Z","shell.execute_reply":"2024-12-06T14:19:42.626619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# NAN fill","metadata":{}},{"cell_type":"code","source":"def fill_missing_values(train_df, test_df):\n    # 2.1 Fill 'Age' with median\n    age_median = train_df['Age'].median()\n    train_df['Age'].fillna(age_median, inplace=True)\n    test_df['Age'].fillna(age_median, inplace=True)\n\n    # 2.2 Fill 'Annual Income' based on skewness\n    income_skew = train_df['Annual Income'].skew()\n    income_fill = train_df['Annual Income'].median() if abs(income_skew) > 1 else train_df['Annual Income'].mean()\n    train_df['Annual Income'].fillna(income_fill, inplace=True)\n    test_df['Annual Income'].fillna(income_fill, inplace=True)\n\n    # 2.3 Fill 'Marital Status' with 'Single'\n    train_df['Marital Status'].fillna('Single', inplace=True)\n    test_df['Marital Status'].fillna('Single', inplace=True)\n\n    # 2.4 Fill 'Number of Dependents' with median\n    dependents_median = train_df['Number of Dependents'].median()\n    train_df['Number of Dependents'].fillna(dependents_median, inplace=True)\n    test_df['Number of Dependents'].fillna(dependents_median, inplace=True)\n\n    # 2.5 Fill 'Health Score' with median\n    health_median = train_df['Health Score'].median()\n    train_df['Health Score'].fillna(health_median, inplace=True)\n    test_df['Health Score'].fillna(health_median, inplace=True)\n\n    # 2.6 Fill 'Occupation' with 'Unemployed'\n    train_df['Occupation'].fillna('Unemployed', inplace=True)\n    test_df['Occupation'].fillna('Unemployed', inplace=True)\n\n    # 2.7 No specific operations for 'Gender'\n\n    # 2.8 No specific operations for 'Education Level'\n\n    # 2.9 Fill 'Previous Claims' with 0\n    train_df['Previous Claims'].fillna(0, inplace=True)\n    test_df['Previous Claims'].fillna(0, inplace=True)\n\n    # 2.10 Fill 'Vehicle Age' with median\n    vehicle_median = train_df['Vehicle Age'].median()\n    train_df['Vehicle Age'].fillna(vehicle_median, inplace=True)\n    test_df['Vehicle Age'].fillna(vehicle_median, inplace=True)\n\n    # 2.11 Fill 'Credit Score' with median\n    credit_score_median = train_df['Credit Score'].median()\n    train_df['Credit Score'].fillna(credit_score_median, inplace=True)\n    test_df['Credit Score'].fillna(credit_score_median, inplace=True)\n\n    # 2.12 Fill 'Insurance Duration' with mean or median based on skewness\n    duration_skew = train_df['Insurance Duration'].skew()\n    duration_fill = train_df['Insurance Duration'].median() if abs(duration_skew) > 1 else train_df['Insurance Duration'].mean()\n    train_df['Insurance Duration'].fillna(duration_fill, inplace=True)\n    test_df['Insurance Duration'].fillna(duration_fill, inplace=True)\n\n    # 2.13 Fill 'Policy Start Date' with median date\n    train_df['Policy Start Date'] = pd.to_datetime(train_df['Policy Start Date'], errors='coerce')\n    test_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'], errors='coerce')\n    policy_start_median = train_df['Policy Start Date'].median()\n    train_df['Policy Start Date'].fillna(policy_start_median, inplace=True)\n    test_df['Policy Start Date'].fillna(policy_start_median, inplace=True)\n    train_df['Policy Start Date'] = train_df['Policy Start Date'].apply(lambda x: x.toordinal())\n    test_df['Policy Start Date'] = test_df['Policy Start Date'].apply(lambda x: x.toordinal())\n\n    # 2.14 Fill 'Customer Feedback' with median or mode\n    if pd.api.types.is_numeric_dtype(train_df['Customer Feedback']):\n        feedback_median = train_df['Customer Feedback'].median()\n        train_df['Customer Feedback'].fillna(feedback_median, inplace=True)\n        test_df['Customer Feedback'].fillna(feedback_median, inplace=True)\n    else:\n        feedback_mode = train_df['Customer Feedback'].mode()[0]\n        train_df['Customer Feedback'].fillna(feedback_mode, inplace=True)\n        test_df['Customer Feedback'].fillna(feedback_mode, inplace=True)\n\n    # 2.15 No specific operations for 'Smoking Status'\n\n    # 2.16 No specific operations for 'Exercise Frequency'\n\n    # 2.17 No specific operations for 'Property Type'\n\n    # 2.18 No specific operations for 'Premium Amount'\n\n    return train_df, test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:42.629388Z","iopub.execute_input":"2024-12-06T14:19:42.630097Z","iopub.status.idle":"2024-12-06T14:19:42.647279Z","shell.execute_reply.started":"2024-12-06T14:19:42.63003Z","shell.execute_reply":"2024-12-06T14:19:42.64613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function to fill missing values\ntrain, test = fill_missing_values(train, test)\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:42.64849Z","iopub.execute_input":"2024-12-06T14:19:42.648837Z","iopub.status.idle":"2024-12-06T14:19:50.257826Z","shell.execute_reply.started":"2024-12-06T14:19:42.648809Z","shell.execute_reply":"2024-12-06T14:19:50.256939Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering with scoring","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import MinMaxScaler\n\n# Feature Engineering Function\ndef feature_engineering(train, test):\n    # Standardize column names to lowercase\n    train.columns = train.columns.str.strip().str.lower()\n    test.columns = test.columns.str.strip().str.lower()\n\n    # Gender - Ordinary encoding\n    gender_mapping = {'Male': 1, 'Female': 0}\n    train['gender'] = train['gender'].map(gender_mapping)\n    test['gender'] = test['gender'].map(gender_mapping)\n\n    # Marital Status - Customized scoring\n    marital_status_mapping = {'Single': 0, 'Married': 1, 'Divorced': -1, 'Widowed': -0.5}\n    train['marital status'] = train['marital status'].map(marital_status_mapping)\n    test['marital status'] = test['marital status'].map(marital_status_mapping)\n\n    # Education Level - Customized scoring\n    education_mapping = {'High School': -1, \"Bachelor's\": 0, \"Master's\": 1, 'PhD': 2}\n    train['education level'] = train['education level'].map(education_mapping)\n    test['education level'] = test['education level'].map(education_mapping)\n    \n    # Occupation - Customized scoring\n    occupation_mapping = {'Employed': 2, 'Self-Employed': 1, 'Unemployed': 0}\n    train['occupation'] = train['occupation'].map(occupation_mapping)\n    test['occupation'] = test['occupation'].map(occupation_mapping)\n    \n    # Location - Customized scoring\n    location_mapping = {'Rural': -1, 'Suburban': 0, 'Urban': 1}\n    train['location'] = train['location'].map(location_mapping)\n    test['location'] = test['location'].map(location_mapping)\n\n    # Policy Type - Customized scoring\n    policy_type_mapping = {'Basic': 0, 'Comprehensive': 1, 'Premium': 2}\n    train['policy type'] = train['policy type'].map(policy_type_mapping)\n    test['policy type'] = test['policy type'].map(policy_type_mapping)\n\n    # Smoking Status - Customized scoring\n    smoking_status_mapping = {'Yes': -1, 'No': 1}\n    train['smoking status'] = train['smoking status'].map(smoking_status_mapping)\n    test['smoking status'] = test['smoking status'].map(smoking_status_mapping)\n\n    # Exercise Frequency - Customized scoring\n    exercise_mapping = {'Rarely': -1, 'Monthly': 0, 'Weekly': 1, 'Daily': 2}\n    train['exercise frequency'] = train['exercise frequency'].map(exercise_mapping)\n    test['exercise frequency'] = test['exercise frequency'].map(exercise_mapping)\n\n    # Property Type - Assign scores based on average property value\n    property_values = {'Apartment': 300000, 'Condo': 500000, 'House': 407200}\n    max_value = max(property_values.values())\n    property_scores = {k: v / max_value for k, v in property_values.items()}  # Normalize values\n    train['property type'] = train['property type'].map(property_scores)\n    test['property type'] = test['property type'].map(property_scores)\n\n    # Policy Start Date - Extract year, month, day features\n    train['policy start date'] = pd.to_datetime(train['policy start date'], errors='coerce')\n    test['policy start date'] = pd.to_datetime(test['policy start date'], errors='coerce')\n\n    for df in [train, test]:\n        df['policy_start_year'] = df['policy start date'].dt.year\n        df['policy_start_month'] = df['policy start date'].dt.month\n        df['policy_start_day'] = df['policy start date'].dt.day\n        \n    # Customer Feedback - Customized scoring\n    feedback_mapping = {'Good': 1, 'Poor': -1, 'Average': 0}\n    train['customer feedback'] = train['customer feedback'].map(feedback_mapping).fillna(0)\n    test['customer feedback'] = test['customer feedback'].map(feedback_mapping).fillna(0)\n\n    # Drop the original 'Policy Start Date' column\n    train.drop('policy start date', axis=1, inplace=True)\n    test.drop('policy start date', axis=1, inplace=True)\n\n    # Derive new features\n    # 1. Income Per Dependent\n    train['income_per_dependent'] = train['annual income'] / (train['number of dependents'] + 1)\n    test['income_per_dependent'] = test['annual income'] / (test['number of dependents'] + 1)\n    \n    # 3. Claims Per Year\n    train['claims_per_year'] = train['previous claims'] / train['insurance duration']\n    test['claims_per_year'] = test['previous claims'] / test['insurance duration']\n\n    # 4. Credit Score Normalization\n    scaler = MinMaxScaler()\n    train['credit score'] = scaler.fit_transform(train[['credit score']])\n    test['credit score'] = scaler.transform(test[['credit score']])\n\n    return train, test\n\n\n# Apply the feature engineering function\ntrain, test = feature_engineering(train, test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:50.259463Z","iopub.execute_input":"2024-12-06T14:19:50.259874Z","iopub.status.idle":"2024-12-06T14:19:52.697302Z","shell.execute_reply.started":"2024-12-06T14:19:50.259829Z","shell.execute_reply":"2024-12-06T14:19:52.696214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:52.698614Z","iopub.execute_input":"2024-12-06T14:19:52.698943Z","iopub.status.idle":"2024-12-06T14:19:53.09797Z","shell.execute_reply.started":"2024-12-06T14:19:52.698912Z","shell.execute_reply":"2024-12-06T14:19:53.096845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\nimport matplotlib.pyplot as plt\nfrom catboost import CatBoostRegressor, Pool\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\n\n# RMSLE calculation function\ndef rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, np.maximum(0, y_pred)))\n\n# Model training function (5-Fold, supports RMSLE)\ndef train_model_with_kfold(\n    model_name, model_fn, X, y, test_data, params, n_splits=5\n):\n    # Initialize KFold for cross-validation\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n    test_preds = np.zeros(len(test_data))  # Placeholder for test predictions\n    fold_rmsle_scores = []  # Store RMSLE for each fold\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n        # Split data into training and validation sets\n        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        if model_name == \"CatBoost\":\n            # CatBoost-specific training\n            train_pool = Pool(X_train, y_train)\n            val_pool = Pool(X_val, y_val)\n            model = model_fn(loss_function=\"RMSE\", **params)  # CatBoost uses RMSE as an approximation for RMSLE\n            model.fit(\n                train_pool, eval_set=val_pool, early_stopping_rounds=50, verbose=0\n            )\n            val_preds = model.predict(X_val)\n        elif model_name == \"LightGBM\":\n            # LightGBM-specific training\n            model = model_fn(**params)\n            model.fit(\n                X_train,\n                y_train,\n                eval_set=[(X_val, y_val)],\n                eval_metric=\"rmse\",  # Use RMSE as the evaluation metric\n                callbacks=[\n                    lgb.early_stopping(stopping_rounds=50),  # Stop training if no improvement in 50 rounds\n                    lgb.log_evaluation(period=10000000)      # Suppress log output\n                ]\n            )\n            val_preds = model.predict(X_val)\n        elif model_name == \"XGBoost\":\n            # XGBoost-specific training\n            model = model_fn(objective=\"reg:squarederror\", **params)\n            model.fit(\n                X_train,\n                y_train,\n                eval_set=[(X_val, y_val)],\n                eval_metric=\"rmse\",  # Use RMSE as the evaluation metric\n                early_stopping_rounds=50,\n                verbose=0\n            )\n            val_preds = model.predict(X_val)\n        else:\n            raise ValueError(f\"Unsupported model: {model_name}\")\n\n        # Calculate RMSLE for the current fold\n        fold_rmsle = rmsle(y_val, val_preds)\n        fold_rmsle_scores.append(fold_rmsle)\n        print(f\"{model_name} Fold {fold + 1} RMSLE: {fold_rmsle:.4f}\")\n\n        # Predict on the test set\n        test_preds += model.predict(test_data) / n_splits\n\n    # Calculate average RMSLE across all folds\n    avg_rmsle = np.mean(fold_rmsle_scores)\n    print(f\"{model_name} Average RMSLE across folds: {avg_rmsle:.4f}\")\n\n    return test_preds, avg_rmsle\n\n\n# Features and target column\nX_train = train.drop(columns=[\"premium amount\", \"id\"])  # Remove target and ID from training data\ny_train = train[\"premium amount\"]  # Target variable\nX_test = test.drop(columns=[\"id\"])  # Remove ID from test data\ntest_ids = test[\"id\"]  # Save test IDs\n\n# CatBoost parameters\ncatboost_params = {\n    \"iterations\": 1000,\n    \"learning_rate\": 0.1,\n    \"depth\": 6,\n    \"verbose\": 0,\n    'task_type': 'GPU'\n}\n\n# LightGBM parameters\nlightgbm_params = {\n    \"n_estimators\": 1000,\n    \"learning_rate\": 0.1,\n    'device': 'gpu',\n    \"max_depth\": 6,\n    'verbosity':-1\n}\n\n# XGBoost parameters\nxgboost_params = {\n    \"n_estimators\": 1000,\n    \"learning_rate\": 0.1,\n    \"max_depth\": 6,\n    'tree_method': 'gpu_hist'\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:53.100788Z","iopub.execute_input":"2024-12-06T14:19:53.101179Z","iopub.status.idle":"2024-12-06T14:19:53.467882Z","shell.execute_reply.started":"2024-12-06T14:19:53.101145Z","shell.execute_reply":"2024-12-06T14:19:53.4668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CatBoost\ncatboost_preds, catboost_rmsle = train_model_with_kfold(\n    \"CatBoost\", CatBoostRegressor, X_train, y_train, X_test, catboost_params\n)\nprint(\"\\n=======================================================\\n\")\n# LightGBM\nlightgbm_preds, lightgbm_rmsle = train_model_with_kfold(\n    \"LightGBM\", lgb.LGBMRegressor, X_train, y_train, X_test, lightgbm_params\n)\nprint(\"\\n=======================================================\\n\")\n# XGBoost\nxgboost_preds, xgboost_rmsle = train_model_with_kfold(\n    \"XGBoost\", XGBRegressor, X_train, y_train, X_test, xgboost_params\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:19:53.468991Z","iopub.execute_input":"2024-12-06T14:19:53.469337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensemble Prediction\nensemble_preds = (catboost_preds + lightgbm_preds + xgboost_preds) / 3\n\n# Calculate the RMSLE (Root Mean Squared Logarithmic Error) for the ensemble model\n\n# Compute the RMSLE for the ensemble predictions\n# ensemble_rmsle = rmsle(y_val, ensemble_preds)\n# print(f\"Ensemble RMSLE: {ensemble_rmsle:.4f}\")\n\n# Save predictions to submission.csv\nsubmission = pd.DataFrame({\n    \"id\": test_ids,\n    \"Premium Amount\": ensemble_preds\n})\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"Predictions have been saved to the submission.csv file.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}