{"metadata":{"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.7"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"# Core imports\nimport pandas as pd\nimport numpy as np\nimport re\n\nfrom fastai.imports import *\n\n# Data Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nsns.set(style='darkgrid', font_scale=1.2)\npd.set_option('display.max_columns', None)\n# pd.set_option('display.max_rows', None)\n\n# Models\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\n\n# Preprocessing\nfrom sklearn.impute import SimpleImputer\n\n# Metrics\nfrom sklearn.metrics import mean_squared_error\n\n# Model Selection\nfrom sklearn.model_selection import KFold\n\nfrom sklearn.ensemble import VotingRegressor\n\n# Seed\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Utility Functions\n\n * **merge_train_and_test_to_X** - combines train and test data into a single DataFrame for less boilerplate during data preprocessing\n\n * **split_X_back_to_train_and_test** - does the reverse of 'merge_train_and_test_to_X'. Split the DataFrame back to train and test before training\n\n * **map_column** - maps a categorical column to an int, given a predefined map. It's assumed all NaN values are imputed\n\n * **impute_missing_values** - imputes missing values in a column by a selected strategy. Generally use 'mean' strategy for high-cardinality features and 'most-frequent' for most_frequent features\n\n * **extract_columns_from_date** - split timestamp column into partial columns, such as 'year', 'month', 'day-of-month', 'day-of-week', etc.\n\n * **plot_kfold_learning_curves** - plot each training fold's (train, eval) metrics, to monitor for overfitting. If you are going to use it for LightGBM & CatBoost, perhaps you'have to play around with the metrics.\n\n * **select_most_important_features** - passes all engineered features through a very basic model ('xgb', 'lgbm', 'catboost') and prints ranking of feature importances for the particular model. Also allows to cap N top performing features. I have taken the idea from this notebook, so props to the author: https://www.kaggle.com/code/arunklenin/ps4e11-mental-health-prediction-classification\n\n * **kfold_cross_val_and_test_pred** - Kfold cross validation for training + Kfold predictions for the final submission. The reason is, you can use early-stopping parameter when you have the train/val splits. Then select the mean of all folds as a final answer. My observations, lead me to believe it improves performance slightly, yet to confirm if it's true.\n\n * **encode_cyclic_column** - encoding cyclical columns such as date in month, day in week, month in year, etc.","metadata":{}},{"cell_type":"code","source":"# Data Preprocessing \ndef merge_train_and_test_to_X(train, test):\n    df = pd.concat([train, test])\n    return df\n\ndef split_X_back_to_train_and_test(df, train, test):\n    n_train, n_test = train.shape[0], test.shape[0]\n    test_chunk = df.iloc[n_train:, :]\n    train_chunk = df.iloc[:n_train, :]\n    return train_chunk, test_chunk\n\ndef map_column(df, col, map):\n    df[col] = df[col].map(map)\n    return df\n\ndef impute_missing_values(df, columns, strategy):\n    imputer = SimpleImputer(strategy=strategy)\n    df[columns] = imputer.fit_transform(df[columns])\n    return df\n\ndef get_unique_values_for_object_columns(df):\n    for col in df.select_dtypes('object').columns:\n        print(f'{col} - {df[col].unique()}')\n\ndef extract_columns_from_date(df, column):\n\n    def parse_date_to_columns(x):\n        date_format_pattern = r\"(\\d{4})-(\\d{2})-(\\d{2}) (\\d{2}:\\d{2})(:\\d{2})(\\.\\d+)\" \n        match_groups = re.match(date_format_pattern, x).groups()\n        return match_groups[0], match_groups[1], match_groups[2], match_groups[3].replace(':', '')\n\n    parsed_dates = df[column].apply(parse_date_to_columns)\n    df[f'{column}_year'] = parsed_dates.apply(lambda x: x[0]).astype(int)\n    df[f'{column}_month'] = parsed_dates.apply(lambda x: x[1]).astype(int)\n    df[f'{column}_day_of_month'] = parsed_dates.apply(lambda x: x[2]).astype(int)\n    df[f'{column}_time'] = parsed_dates.apply(lambda x: x[3]).astype(int)\n    df[f'{column}_day_of_week'] = pd.to_datetime(parsed_dates.apply(lambda x: (x[0] + '-' + x[1] + '-' + x[2]))).dt.dayofweek.astype(int)\n\n    return df\n\ndef encode_cyclic_column(df, column):\n    df_max = df[column].max()\n    df[f'{column}_sin'] = np.sin(2 * np.pi * df[column]/df_max)\n    df[f'{column}_cos'] = np.cos(2 * np.pi * df[column]/df_max)\n    return df\n\n# Training Visualization\n\ndef plot_kfold_learning_curves(eval_results, model_type):\n    folds = len(eval_results)\n\n    key1, key2 = None, None\n    metric = None\n    match model_type:\n        case 'xgb':\n            key1, key2 = 'validation_0', 'validation_1'\n            metric = 'rmse'\n        case 'lgbm':\n            key1, key2 = 'training', 'valid_1'\n            metric = 'rmse'\n        case 'cat':\n            key1, key2 = 'validation_0', 'validation_1'\n            metric = 'RMSE'\n\n    fig = plt.figure(figsize=(30, 5*((folds+10)//2)))\n\n    for i, results in enumerate(eval_results):\n        plt.subplot((folds+1)//2, 2, i+1)\n        # plot learning curves\n        plt.title(f'Fold {i}')\n        plt.plot(results[key1][metric], label='train')\n        plt.plot(results[key2][metric], label='val')\n        # show the legend\n        plt.legend()\n\n# Feature Selection\n\ndef select_most_important_features(features, target, n, model_type, seed, n_splits=5):\n    \n    lgbm_params = {\n        'random_state': RANDOM_STATE,\n        'verbose': -1,\n    }\n\n    xgb_params = {\n        'n_estimators': 50,\n        'subsample': 0.2,\n        'max_depth': 10,\n        'learning_rate': 0.2,\n        # 'colsample_bytree': 0.8,\n\n        # 'tree_method': 'hist',\n        'verbosity': 0,\n        'random_state': RANDOM_STATE,\n\n        'objective': 'reg:gamma',\n        'metric': 'rmsle',\n        'eval_metric': 'rmsle'\n    }\n    \n    cat_params = {\n        'random_state': RANDOM_STATE,\n        'verbose': 0,\n        'iterations': 100,\n    }\n\n    match model_type:\n        case 'lgbm':\n            model = LGBMRegressor(**lgbm_params)\n        case 'xgb':\n            model = XGBRegressor(**xgb_params)\n        case 'cat':\n            model = CatBoostRegressor(**cat_params)\n    \n    log_target = np.log1p(target)\n\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=RANDOM_STATE)\n    kfold_errors = []\n    kfold_feature_importance_list = []\n\n    for train_index, val_index in kf.split(features, target):\n        train_x, val_x = features.iloc[train_index], features.iloc[val_index]\n        train_y, val_y = log_target.iloc[train_index], log_target.iloc[val_index]\n\n        model.fit(train_x, train_y)\n\n        pred_y = model.predict(val_x)\n        \n        kfold_errors.append(np.sqrt(mean_squared_error(val_y, pred_y)))\n        kfold_feature_importance_list.append(model.feature_importances_)\n\n    avg_feature_importances = np.mean(kfold_feature_importance_list, axis=0)\n\n    feature_importance_list = list(zip(features, avg_feature_importances))\n    sorted_features = sorted(feature_importance_list, key=lambda x: x[1], reverse=True)\n    top_n_features = [feat[0] for feat in sorted_features[:n]]\n\n    # display_features = top_n_features[:n_display_features]\n    for i, feat in enumerate(top_n_features):\n        print(f'{i}: {feat} with feature importance score of {sorted_features[i]}')\n\n    return top_n_features, avg_feature_importances\n\n# Validation and Prediction\n\ndef kfold_cross_val_and_test_pred(model, features, target, seed, n_splits=5, test=None): \n    skf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)\n\n    test_preds = np.zeros((test.shape[0], n_splits))\n    evals_results = []\n    errors = []\n    i = 0\n\n    for fold, (train_index, val_index) in enumerate(skf.split(features, target)):\n        log_target = np.log1p(target)\n        train_x, val_x = features.iloc[train_index], features.iloc[val_index]\n        train_y, val_y = log_target.iloc[train_index], log_target.iloc[val_index]\n        try:\n            model.fit(train_x, train_y, eval_set=[(train_x, train_y), (val_x, val_y)])\n            if hasattr(model, 'evals_result'):\n                evals_results.append(model.evals_result())\n            elif hasattr(model, '_evals_result'):\n                evals_results.append(model._evals_result)\n            elif hasattr(model, 'evals_result_'):\n                evals_results.append(model.evals_result_)\n        except: \n            model.fit(train_x, train_y)\n        preds = model.predict(val_x)\n        test_preds[:, fold] = np.exp(model.predict(test))\n\n        err = np.sqrt(mean_squared_error(val_y, preds))\n        errors.append(err)\n        i+=1\n        print(f'fold {i}: error: {err:.8f}')\n    return evals_results, np.mean(test_preds, axis=1)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dictionaries & Mappings","metadata":{}},{"cell_type":"code","source":"gender_map = {\n    'Female': 1,\n    'Male': 0\n}\n\nmarital_status_map = {\n    'Married': 2,\n    'Divorced': 1,\n    'Single': 0,\n    'Other': -1\n}\n\neducation_level_map = {\n    'PhD': 3,\n    'Master\\'s': 2,\n    'Bachelor\\'s': 1,\n    'High School': 0\n}\n\noccupation_map = {\n    'Self-Employed': 1,\n    'Employed': 2,\n    'Unemployed': 0\n}\n\nlocation_map = {\n    'Urban': 2,\n    'Suburban': 1,\n    'Rural': 0\n}\n\npolicy_type_map = {\n    'Premium': 2,\n    'Comprehensive': 1,\n    'Basic': 0\n}\n\ncustomer_feedback_map = {\n    'Good': 2,\n    'Average': 1,\n    'Poor': 0,\n    'Other': -1\n}\n\nyes_no_map = {\n    'Yes': 1,\n    'No': 0\n}\n\nexercise_frequency_map = {\n    'Daily': 3,\n    'Weekly': 2,\n    'Monthly': 1,\n    'Rarely': 0\n}\n\nproperty_type = {\n    'House': 2,\n    'Condo': 1,\n    'Apartment': 0\n}","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Load, Visualization & Preparation","metadata":{}},{"cell_type":"code","source":"target_name = 'Premium Amount'\n# X_train = pd.read_csv('train.csv')\n# X_test = pd.read_csv('test.csv')\n\nX_train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\nX_test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nY_train = X_train[target_name]\nX_train = X_train.iloc[:, :-1]\n\nX = merge_train_and_test_to_X(X_train, X_test)\n\n# Impute missing values\ncolumns_w_missing_values = X.columns[X.isnull().any()]\nhigh_cardinality_cols = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', 'Credit Score']\nlow_cardinality_cols = ['Marital Status', 'Occupation', 'Previous Claims', 'Vehicle Age', 'Insurance Duration']\n\nX = impute_missing_values(X, high_cardinality_cols, 'mean')\nX = impute_missing_values(X, low_cardinality_cols, 'most_frequent')\n\n# dummies = pd.get_dummies(X[low_cardinality_cols], columns=low_cardinality_cols, dtype='int', drop_first=True)\n# X = pd.concat([X, dummies], axis=1)\n\n# Columns that are arguably not applicable to each individual\nX['Customer Feedback'] = X['Customer Feedback'].fillna('Other')\n\n# Encode Categorical columns with low cardinality\nX = map_column(X, 'Gender', gender_map)\nX = map_column(X, 'Marital Status', marital_status_map)\nX = map_column(X, 'Education Level', education_level_map)\nX = map_column(X, 'Occupation', occupation_map)\nX = map_column(X, 'Location', location_map)\nX = map_column(X, 'Policy Type', policy_type_map)\nX = map_column(X, 'Customer Feedback', customer_feedback_map)\nX = map_column(X, 'Smoking Status', yes_no_map)\nX = map_column(X, 'Exercise Frequency', exercise_frequency_map)\nX = map_column(X, 'Property Type', property_type)\n\n# Split timestamp to multiple columns to extract maximum signal ( hopefully ;) )\nX = extract_columns_from_date(X, 'Policy Start Date')\n\nX = encode_cyclic_column(X, 'Policy Start Date_month')\nX = encode_cyclic_column(X, 'Policy Start Date_day_of_month')\nX = encode_cyclic_column(X, 'Policy Start Date_time')\nX = encode_cyclic_column(X, 'Policy Start Date_day_of_week')\n\n# Convert any remaining cols of type 'object' to int or float\nobject_to_int_columns = ['Vehicle Age', 'Previous Claims', 'Insurance Duration']\nX[object_to_int_columns] = X[object_to_int_columns].astype(int)\n\n# 'Policy Start Date_month', 'Policy Start Date_day_of_month', 'Policy Start Date_time', 'Policy Start Date_day_of_week'\ncolumns_to_drop = ['id', 'Policy Start Date']\nX = X.drop(columns_to_drop, axis=1).reset_index(drop=True)\n\nX_train, X_test = split_X_back_to_train_and_test(X, X_train, X_test)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Selection","metadata":{}},{"cell_type":"code","source":"n_top_features = 30\n\n# print('=== LGBM Regressor')\n# top_feats_lgbm, avg_feat_importances_lgbm = select_most_important_features(X_train, Y_train, n_top_features, 'lgbm', RANDOM_STATE, n_splits=5)\n# print('=== XGB Regressor')\n# top_feats_xgb, avg_feat_importances_xgb = select_most_important_features(X_train, Y_train, n_top_features, 'xgb', RANDOM_STATE, n_splits=5)\nprint('=== CatBoost Regressor')\ntop_feats_cat, avg_feat_importances_cat = select_most_important_features(X_train, Y_train, n_top_features, 'cat', RANDOM_STATE, n_splits=5)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# drop redundant features\ncolumns_to_drop = ['Policy Start Date_time', 'Policy Start Date_time_sin']\nX = X.drop(columns_to_drop, axis=1).reset_index(drop=True)\n\nX_train, X_test = split_X_back_to_train_and_test(X, X_train, X_test)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"cat_params = {\n    # 'task_type':'GPU',\n    'random_state':RANDOM_STATE,\n    'silent': False,\n    # 'random_strength': 1,\n\n    'subsample': 0.4,\n    'learning_rate': 0.06,\n    'depth': 6,\n    'n_estimators': 450,\n    'l2_leaf_reg': 5,\n    # 'colsample_bylevel': 0.6,\n    # 'boosting_type': 'Ordered',\n    # # 'border_count': 254,\n    'objective': 'RMSE',\n    # 'eval_metrics' : ['RMSE'],\n    # 'max_bin': 400\n    # 'bootstrap_type': 'Bernoulli'\n    # 'use_best_model' : True,\n    'early_stopping_rounds': 10\n}\n\n\ncat = CatBoostRegressor(**cat_params)\n\neval_results, cat_test_preds = kfold_cross_val_and_test_pred(cat, X_train, Y_train, RANDOM_STATE, test=X_test)\nplot_kfold_learning_curves(eval_results, 'cat')\n\n# Average RMSLE: 1.05041612 with STD: 0.0007\n\n# Average RMSLE: 1.04896038 with STD: 0.0008\n\n# Average RMSLE: 1.04894 with STD: 0.0008","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGB","metadata":{}},{"cell_type":"code","source":"xgb_params = {\n    'n_estimators': 200,\n    'subsample': 0.4,\n    'max_depth': 10,\n    'learning_rate': 0.06,\n    # 'colsample_bytree': 0.8,\n    # 'reg_lambda': 100,\n\n    # 'tree_method': 'hist',\n    'verbosity': 0,\n    'random_state': RANDOM_STATE,\n\n    # 'objective': 'reg:gamma',\n    'metric': 'rmse',\n    'eval_metric': 'rmse'\n}\n\nxgb = XGBRegressor(**xgb_params, early_stopping_rounds=10)\n\neval_results, xgb_test_preds = kfold_cross_val_and_test_pred(xgb, X_train, Y_train, RANDOM_STATE, test=X_test)\nplot_kfold_learning_curves(eval_results, 'xgb')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble","metadata":{}},{"cell_type":"code","source":"mean_preds = np.mean([cat_test_preds, xgb_test_preds], axis=0)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\nsub[\"Premium Amount\"] = mean_preds\nsub.to_csv(\"submission.csv\", index=False)","metadata":{},"outputs":[],"execution_count":null}]}