{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌟 **S4E12 | LightGBM tuning** 🌟\n\nWelcome to the **Hyperparameter Tuning** phase of our journey! 🚀 \n\nWe’ve been progressing step-by-step, building a deeper understanding of our dataset. Here’s a quick recap of the milestones we’ve achieved so far:  \n\n---\n\n### 📚 **Our Journey So Far**\n1. **[Starter Pack](https://www.kaggle.com/code/isaranja/s4e12-starter-pack-by-ama)**  \n   *Launched with an overview of the dataset and initial setup.*  \n\n2. **[EDA and Transformation](https://www.kaggle.com/code/isaranja/s4e12-eda-transformation-by-ama)**  \n   *Explored the data, uncovered patterns, and applied necessary transformations.*  \n\n3. **[Feature Engineering](https://www.kaggle.com/code/isaranja/s4e12-feature-engineering-by-ama)**  \n   *Crafted and refined features to highlight hidden insights.*  \n\n4. **[Algorithm Spot Check](https://www.kaggle.com/code/isaranja/s4e12-algorithm-spot-check-by-ama)**  \n   *Assessed multiple algorithms to identify the best performers for our dataset.*  \n\n---\n\n### 🧠 **Hyperparameter Tuning**\nNow, it’s time to focus on **Hyperparameter Tuning**! 🔍  \nWe’ll tune LightGBM model with all the transformations, engineered features that we have created. Let’s dive in! 🚀  ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"#********** Loading libraries and configurations\n\n#***********************************************************************************************************\n# Importing libraries, Helper functions, Loading files and Transformations identified during EDA\n\n#!pip install pycaret -q\n\nfrom IPython.core.interactiveshell import InteractiveShell #\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom scipy.stats import f_oneway, pointbiserialr, pearsonr, spearmanr # statistical analysis\n\nfrom sklearn.preprocessing import LabelEncoder,OneHotEncoder, MinMaxScaler # transformation\n\nimport lightgbm as lgb # lightgbm \nfrom lightgbm import LGBMRegressor # lightgbm sklern api\n\nfrom sklearn.model_selection import train_test_split # data split\nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error # evaluation metrices\nimport optuna # Hyperparameter Tuning\n\nfrom tabulate import tabulate # tabulate printing\n\nimport seaborn as sns # plots\nimport matplotlib.pyplot as plt # plots\n\nimport warnings #\n\n#********** Settings **************************************************************************************\n# settings for jupyter envioronment\n\n# This ensures that plots are rendered inline\n%matplotlib inline\n\n# This ensures that all output, including text and plots, is shown automatically\nInteractiveShell.ast_node_interactivity = \"all\"\n\n# Switching off the future warrnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:17:40.456422Z","iopub.execute_input":"2024-12-31T06:17:40.456613Z","iopub.status.idle":"2024-12-31T06:17:45.152557Z","shell.execute_reply.started":"2024-12-31T06:17:40.456588Z","shell.execute_reply":"2024-12-31T06:17:45.151884Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#********** Loading Dataset\n# Loading dataset\ntrain_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\",parse_dates=['Policy Start Date'])\ntest_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\",parse_dates=['Policy Start Date'])\n\n# Merging two dataframes after adding trn and tst tag\ntrain_df['src']='trn'\ntest_df['src']='tst'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:17:45.153443Z","iopub.execute_input":"2024-12-31T06:17:45.154089Z","iopub.status.idle":"2024-12-31T06:17:54.660879Z","shell.execute_reply.started":"2024-12-31T06:17:45.154063Z","shell.execute_reply":"2024-12-31T06:17:54.660215Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Binnining** \nDataset was right skewed. Let's create few features in order to analyse whether continues feature binning with target mean and frequency helps for predicctions","metadata":{}},{"cell_type":"code","source":"#********** Helper functions\n\n# binning\ndef add_binned_feature_with_target_mean_and_frequency(column_name, target_column, bins=100):\n    \"\"\"\n    Adds two new columns to the global DataFrame:\n    - The mean of the target variable for each bin of the specified column.\n    - The frequency of each bin.\n\n    Args:\n    - column_name (str): The name of the column to be binned.\n    - target_column (str): The name of the target column whose mean will be calculated within bins.\n    - bins (int): The number of bins to use for discretization (default is 100).\n    \"\"\"\n    \n    # Check if the column exists in the DataFrame\n    if column_name not in df.columns or target_column not in df.columns:\n        print(f\"Error: {column_name} or {target_column} not found in the DataFrame.\")\n        return\n    \n    # Bin the specified column using pd.cut\n    df[f'{column_name}_binned'] = pd.cut(df[column_name], bins=bins, labels=False, include_lowest=True)\n    \n    # Calculate the mean of the target variable within each bin\n    mean_target_per_bin = df.groupby(f'{column_name}_binned')[target_column].mean()\n    \n    # Map the mean target values back to the original dataframe\n    df[f'{column_name}_binned_mean'] = df[f'{column_name}_binned'].map(mean_target_per_bin)\n    \n    # Calculate the frequency of each bin\n    bin_frequency = df[f'{column_name}_binned'].value_counts(normalize=False)\n    \n    # Map the frequency values back to the original dataframe\n    df[f'{column_name}_binned_freq'] = df[f'{column_name}_binned'].map(bin_frequency)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:18:00.977258Z","iopub.execute_input":"2024-12-31T06:18:00.977568Z","iopub.status.idle":"2024-12-31T06:18:00.983359Z","shell.execute_reply.started":"2024-12-31T06:18:00.977545Z","shell.execute_reply":"2024-12-31T06:18:00.982355Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset preperation\ndf = pd.concat([train_df, test_df], ignore_index=True)\n\n# Removing whitesapces in a dataframe since some libraries does not support it\ndf.columns = df.columns.str.replace(' ', '_', regex=False)\n\n#++++++++++++++++++++++++\ndf['annual_income_null'] = df['Annual_Income'].isnull().map({True: 1, False: 0})\ndf['credit_score_null'] = df['Credit_Score'].isnull().map({True: 1, False: 0})\ndf['previous_claims_null'] = df['Previous_Claims'].isnull().map({True: 1, False: 0})\n#++++++++++++++++++++++++\n\n#********** Transformation *********************************************************************************\n# Replace 'inf' and '-inf' with NaN\ndf.replace([np.inf, -np.inf], np.nan, inplace=True)\n\n# Missing value imputation\nfill_values = {'Age': 0, \n               'Annual_Income': df['Annual_Income'].max(), # since very low count there\n               'Marital_Status': 'unknown',\n               'Number_of_Dependents':5.0,\n               'Occupation':'unknown',\n               'Health_Score':0.0,\n               'Previous_Claims':10.0,\n               'Vehicle_Age':0.0,\n               'Credit_Score':900,\n               'Insurance_Duration':0.0,\n               'Customer_Feedback':'unknown'}\n\ndf.fillna(value=fill_values, inplace=True)\n\n# Log transformation\ndf['annual_income_log'] = np.log1p(df['Annual_Income'])\ndf['premium_amount_log'] = np.log1p(df['Premium_Amount'])\n\n#********** Feature Engineering *****************************************************************************\n\n# deriving year from the policy start date\ndf['policy_start_year'] = df['Policy_Start_Date'].dt.year\n\n# deriving the month from the policy start date \ndf['policy_start_month'] = df['Policy_Start_Date'].dt.month\n\n# deriving the day of policy start date\ndf['policy_start_day'] = df['Policy_Start_Date'].dt.day\n\nmax_date = df['Policy_Start_Date'].max()\ndf['policy_age'] = (max_date - df['Policy_Start_Date']).dt.days\ndf['policy_age_bins'] = pd.cut(df['policy_age'], bins=[df['policy_age'].min()-1,1700,df['policy_age'].max()], labels=['new','old'])\nadd_binned_feature_with_target_mean_and_frequency('policy_age', 'premium_amount_log', bins=100)\n\n# Annual Income\ndf['annual_income_bins'] = pd.cut(df['annual_income_log'], bins=[df['annual_income_log'].min()-1,1,10.9,df['annual_income_log'].max()], labels=['no-data','mid','high'])\nadd_binned_feature_with_target_mean_and_frequency('Annual_Income', 'premium_amount_log', bins=100)\n\n# Credit Score\ndf['credit_score_bins'] = pd.cut(df['Credit_Score'], bins=[df['Credit_Score'].min()-1,380,550,805,840,df['Credit_Score'].max()], labels=['S','M','L','XL','no-data'])\nadd_binned_feature_with_target_mean_and_frequency('Credit_Score', 'premium_amount_log', bins=100)\n\n# Health Score\ndf['health_score_bins'] = pd.cut(df['Health_Score'], bins=[df['Health_Score'].min()-1,1,50,df['Health_Score'].max()], labels=['S','M','L'])\nadd_binned_feature_with_target_mean_and_frequency('Health_Score', 'premium_amount_log', bins=50)\n\n# Annual Income per dependents\ndf['annual_income_to_dependent'] = np.log1p(df['Annual_Income']/(df['Number_of_Dependents']+1))\n\n# Health score and Credit Score combined\ndf['health_and_credit_score'] = df['Health_Score']+(df['Credit_Score'])\ndf['health_and_credit_score_bins'] = pd.cut(df['health_and_credit_score'], bins=[df['health_and_credit_score'].min()-1,440,575,df['health_and_credit_score'].max()], labels=['S','M','L'])\nadd_binned_feature_with_target_mean_and_frequency('health_and_credit_score', 'premium_amount_log', bins=100)\n\n#Annual Income per Insurance Duration\ndf['annual_income_to_insurance_duration'] = np.log1p(df['Annual_Income']/(df['Insurance_Duration']+1))\ndf['annual_income_to_insurance_duration_bins'] = pd.cut(df['annual_income_to_insurance_duration'], bins=[df['annual_income_to_insurance_duration'].min()-1,8.5,9.75,10.9,df['annual_income_to_insurance_duration'].max()], labels=['S','M','L','XL'])\n\n# Annual Income per Age\ndf['annual_income_to_age'] = np.log1p(df['Annual_Income']/(df['Age']+1))\ndf['annual_income_to_age_bins'] = pd.cut(df['annual_income_to_age'], bins=[df['annual_income_to_age'].min()-1,6.6,8.2,df['annual_income_to_age'].max()], labels=['S','M','L'])\nadd_binned_feature_with_target_mean_and_frequency('annual_income_to_age', 'premium_amount_log', bins=100)\n\n# Annual Income to Health Score ratio\ndf['annual_income_to_health_score'] = np.log1p(df['Annual_Income']/(df['Health_Score']+1))\ndf['annual_income_to_health_score_bins'] = pd.cut(df['annual_income_to_health_score'], bins=[df['annual_income_to_health_score'].min()-1,6.8,9,df['annual_income_to_health_score'].max()], labels=['S','M','L'])\nadd_binned_feature_with_target_mean_and_frequency('annual_income_to_health_score', 'premium_amount_log', bins=100)\n\n# Credit Score to Dependents ratio\ndf['credit_score_to_dependent'] = np.log1p(df['Credit_Score']/(df['Number_of_Dependents']+1))\ndf['credit_score_to_dependent_bins'] = pd.cut(df['credit_score_to_dependent'], bins=[df['credit_score_to_dependent'].min()-1,5,6.25,df['credit_score_to_dependent'].max()], labels=['S','M','L'])\nadd_binned_feature_with_target_mean_and_frequency('credit_score_to_dependent', 'premium_amount_log', bins=100)\n\n# Annual Income to Policy Age\ndf['annual_income_to_policy_age'] = np.log1p(df['Annual_Income']/(df['policy_age']+1))\nadd_binned_feature_with_target_mean_and_frequency('annual_income_to_policy_age', 'premium_amount_log', bins=100)\n\n# Credit Score to Previous Claims\ndf['credit_score_to_previous_claims'] = df['Credit_Score']/(df['Previous_Claims']+1)\ndf['credit_score_to_previous_claims_bins'] = pd.cut(df['credit_score_to_previous_claims'], bins=[df['credit_score_to_previous_claims'].min()-1,300,440,565,df['credit_score_to_previous_claims'].max()], labels=['S','M','L','XL'])\nadd_binned_feature_with_target_mean_and_frequency('credit_score_to_previous_claims', 'premium_amount_log', bins=100)\n\n# Annual Income to Previous Claims\ndf['annual_income_to_previous_claims'] = np.log1p(df['Annual_Income']/(df['Previous_Claims']+1))\nadd_binned_feature_with_target_mean_and_frequency('annual_income_to_previous_claims', 'premium_amount_log', bins=100)\n\n# Dependents and previous claims\ndf['dependents_and_previous_claims'] = (df['Number_of_Dependents'])*(df['Previous_Claims'])\n\n#********** Final Dataset *****************************************************************************\n# Dataset looks like                                             \nwith pd.option_context('display.max_columns', None): # setting the max rows\n    display(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:18:04.391335Z","iopub.execute_input":"2024-12-31T06:18:04.391667Z","iopub.status.idle":"2024-12-31T06:18:13.073885Z","shell.execute_reply.started":"2024-12-31T06:18:04.39164Z","shell.execute_reply":"2024-12-31T06:18:13.073017Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Feature List**\nLet's define the full feature list","metadata":{}},{"cell_type":"code","source":"# Feature list\n\nfeature_list = [\n# 'id',\n 'Age',\n 'Gender',\n# 'Annual_Income',\n 'Marital_Status',\n 'Number_of_Dependents',\n 'Education_Level',\n 'Occupation',\n 'Health_Score',\n 'Location',\n 'Policy_Type',\n 'Previous_Claims',\n 'Vehicle_Age',\n 'Credit_Score',\n 'Insurance_Duration',\n# 'Policy_Start_Date',\n 'Customer_Feedback',\n 'Smoking_Status',\n 'Exercise_Frequency',\n 'Property_Type',\n# 'Premium_Amount',\n# 'src',\n 'annual_income_null',\n 'credit_score_null',\n 'previous_claims_null',\n 'annual_income_log',\n# 'premium_amount_log',\n 'policy_start_year',\n 'policy_start_month',\n 'policy_start_day',\n 'policy_age',\n 'policy_age_bins',\n# 'policy_age_binned',\n 'policy_age_binned_mean',\n 'policy_age_binned_freq',\n 'annual_income_bins',\n# 'Annual_Income_binned',\n 'Annual_Income_binned_mean',\n 'Annual_Income_binned_freq',\n 'credit_score_bins',\n# 'Credit_Score_binned',\n 'Credit_Score_binned_mean',\n 'Credit_Score_binned_freq',\n 'health_score_bins',\n# 'Health_Score_binned',\n 'Health_Score_binned_mean',\n 'Health_Score_binned_freq',\n 'annual_income_to_dependent',\n 'health_and_credit_score',\n 'health_and_credit_score_bins',\n# 'health_and_credit_score_binned',\n 'health_and_credit_score_binned_mean',\n 'health_and_credit_score_binned_freq',\n 'annual_income_to_insurance_duration',\n 'annual_income_to_insurance_duration_bins',\n 'annual_income_to_age',\n 'annual_income_to_age_bins',\n# 'annual_income_to_age_binned',\n 'annual_income_to_age_binned_mean',\n 'annual_income_to_age_binned_freq',\n 'annual_income_to_health_score',\n 'annual_income_to_health_score_bins',\n# 'annual_income_to_health_score_binned',\n 'annual_income_to_health_score_binned_mean',\n 'annual_income_to_health_score_binned_freq',\n 'credit_score_to_dependent',\n 'credit_score_to_dependent_bins',\n# 'credit_score_to_dependent_binned',\n 'credit_score_to_dependent_binned_mean',\n 'credit_score_to_dependent_binned_freq',\n 'annual_income_to_policy_age',\n# 'annual_income_to_policy_age_binned',\n 'annual_income_to_policy_age_binned_mean',\n 'annual_income_to_policy_age_binned_freq',\n 'credit_score_to_previous_claims',\n 'credit_score_to_previous_claims_bins',\n# 'credit_score_to_previous_claims_binned',\n 'credit_score_to_previous_claims_binned_mean',\n 'credit_score_to_previous_claims_binned_freq',\n 'annual_income_to_previous_claims',\n# 'annual_income_to_previous_claims_binned',\n 'annual_income_to_previous_claims_binned_mean',\n 'annual_income_to_previous_claims_binned_freq',\n 'dependents_and_previous_claims']\n\n\ncat_cols = ['Gender','Marital_Status','Education_Level','Occupation','Location','Policy_Type','Customer_Feedback','Smoking_Status','Exercise_Frequency','Property_Type',\n            'policy_age_bins','annual_income_bins','credit_score_bins','health_score_bins','health_and_credit_score_bins','annual_income_to_age_bins','annual_income_to_health_score_bins',\n            'credit_score_to_dependent_bins','credit_score_to_previous_claims_bins',\n            'annual_income_to_insurance_duration_bins','annual_income_null','credit_score_null','previous_claims_null',\n            'policy_start_year','policy_start_month'\n           ]\n\ndf[cat_cols] = df[cat_cols].astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:18:17.465449Z","iopub.execute_input":"2024-12-31T06:18:17.465746Z","iopub.status.idle":"2024-12-31T06:18:19.173392Z","shell.execute_reply.started":"2024-12-31T06:18:17.465723Z","shell.execute_reply":"2024-12-31T06:18:19.172465Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Light GBM Tuned with Optuna** \nIt's time to tune a lighgbm model and see the performances.","metadata":{}},{"cell_type":"code","source":"# LGBM native API with non transformed target variable\n\n#dfs = df.sample(frac=0.1, random_state=42)\ndfs = df.copy()\n#dfs = trn.copy()\n\n# Split data into features and target\nX = dfs.loc[(dfs['src']=='trn'),feature_list]\ntest_df = dfs.loc[(dfs['src']=='tst'),feature_list]\n\ntest_data = lgb.Dataset(test_df[feature_list], categorical_feature=cat_cols, free_raw_data=False)\n\n#y = dfs.loc[df['src']=='trn','Premium_Amount']\ny = dfs.loc[(dfs['src']=='trn'),'premium_amount_log']\n\n# Split into train and validation sets\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# LightGBM Dataset (categorical features need to be specified)\n\ntrain_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_cols, free_raw_data=False)\nvalid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_cols, free_raw_data=False)\ntest_data = lgb.Dataset(test_df, categorical_feature=cat_cols, free_raw_data=False)\n\n# Define RMSLE as a custom evaluation metric\n\ndef rmsle(y_pred, y_true):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(np.log1p(y_true) - np.log1p(y_pred))))\n    rmsle = np.round(rmsle,3)\n    return 'rmsle', rmsle, False\n\ndef rmsle_metric(y_pred, dataset):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_true = dataset.get_label()\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(np.log1p(y_true) - np.log1p(y_pred))))\n    return 'rmsle', rmsle, False\n\ndef rmsle_metric_for_log(y_pred, dataset):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_true = dataset.get_label()\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(y_true - y_pred)))\n    return 'rmsle', rmsle, False\n\ndef rmsle_objective(preds, train_data):\n    \"\"\"Custom RMSLE objective function for LightGBM.\"\"\"\n    y_true = train_data.get_label()\n    preds = np.maximum(preds, 1e-7)  # Avoid log(0) issues by ensuring predictions are > 0\n    grad = -1 * (np.log1p(y_true) - np.log1p(preds)) / (preds + 1)\n    hess = 1 / (preds + 1)**2\n    return grad, hess\n\n# Hyperparameter tuning with Optuna\ndef objective(trial):\n    params = {\n        'objective': 'regression',                                                # rmsle_objective, 'regression', 'huber', 'poisson'\n        #'metric': 'rmse',                                                          # 'l2', 'rmse', 'mae', 'mape'.....\n        'boosting_type': 'gbdt',                                                  # 'dart', 'goss', 'rf'\n        'device': 'gpu',                                                          # Enable GPU\n        'gpu_platform_id': 0,                                                     # Select the GPU (typically 0 for Kaggle)\n        'gpu_device_id': 0,                                                       # Select the device (typically 0 for Kaggle)\n        'num_leaves': round(trial.suggest_int('num_leaves', 31, 128),2),          # Maximum number of leaves in a tree. Controls the model’s complexity and affects overfitting/underfitting\n        'learning_rate': 0.1,                                                     # Controls the step size during each iteration. Lower values lead to slower but more accurate convergence\n        #'n_estimators': 1000,                                                      # Number of boosting iterations (trees). More trees improve accuracy but increase computation time.\n        'bagging_freq': 5,                                                        # Frequency of bagging (subsampling rows). Controls how often bagging is performed\n        'min_data_in_leaf': trial.suggest_int(\"min_data_in_leaf\", 20, 1000),      # Minimum number of samples required in a leaf node. Prevents leaves from being too small, which helps avoid overfitting.\n        'max_depth': trial.suggest_int('max_depth', -1, 15),                      # Maximum depth of a tree. Limits tree growth to control overfitting\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1.0, 10),              # L1 regularization term on weights. Helps control model complexity by penalizing large weights.\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1.0, 10),            # L2 regularization term on weights. Similar to reg_alpha but penalizes squared weights.\n        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.5, 1.0),  # Fraction of features (columns) sampled for each tree.\n        'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),                # Row sampling ratio for each tree. Prevents overfitting by using a subset of the data.\n        'verbosity':-1\n    }\n    \n    # Train the model\n    # Train model with early stopping and custom metric\n    booster = lgb.train(\n        params,\n        train_data,\n        num_boost_round=1000,\n        valid_sets=[valid_data],\n        valid_names=['valid'],\n        #feval='rmse', #rmsle_metric_for_log, #rmsle_metric\n        callbacks=([lgb.early_stopping(stopping_rounds=50,verbose=0)]),\n    )\n    \n    # Predict and compute RMSLE (note that we exponentiate the predictions and true values)\n    y_pred = booster.predict(X_valid)\n    return (mean_squared_error(y_valid, y_pred, squared=False)) # with log transformed target variable\n\n# Run hyperparameter optimization using Optuna\nstudy = optuna.create_study(direction='minimize')\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nstudy.optimize(objective, n_trials=20, show_progress_bar=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T06:23:57.770466Z","iopub.execute_input":"2024-12-31T06:23:57.770796Z","iopub.status.idle":"2024-12-31T06:29:20.194398Z","shell.execute_reply.started":"2024-12-31T06:23:57.770769Z","shell.execute_reply":"2024-12-31T06:29:20.193642Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Training the model with best parameters**\nIts time to train the model with identified best parameters","metadata":{}},{"cell_type":"code","source":"# Best hyperparameters found by Optuna\nimport pprint\nprint('Best parameters are \\n')\nbest_params = study.best_params\nconst_params = {\n    'objective': 'regression',\n    'boosting_type': 'gbdt',\n    'device': 'gpu',\n    'gpu_platform_id': 0,\n    'gpu_device_id': 0,\n    'learning_rate': 0.05,\n    'verbosity':-1,\n    'bagging_freq': 5,\n}\n\nparams = best_params | const_params\npprint.pprint(params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:24:45.731997Z","iopub.execute_input":"2024-12-30T11:24:45.732218Z","iopub.status.idle":"2024-12-30T11:24:45.74451Z","shell.execute_reply.started":"2024-12-30T11:24:45.732198Z","shell.execute_reply":"2024-12-30T11:24:45.743862Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train final model with the best parameters\nfinal_model = lgb.train(params,\n                        train_data,\n                        num_boost_round=3000,\n                        valid_sets=[valid_data],\n                        valid_names=['valid'],\n                        feval=rmsle_metric_for_log,\n                        callbacks=([lgb.early_stopping(stopping_rounds=100,verbose=20)]),\n                       )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:24:45.746731Z","iopub.execute_input":"2024-12-30T11:24:45.746932Z","iopub.status.idle":"2024-12-30T11:25:14.008404Z","shell.execute_reply.started":"2024-12-30T11:24:45.746914Z","shell.execute_reply":"2024-12-30T11:25:14.007465Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Error Analysis**\nLet's view the prediction vs actual distribution. It is evident that predictions are closed to one region.","metadata":{}},{"cell_type":"code","source":"# Prediction\ny_pred = final_model.predict(X_valid)\n\nfig, axes = plt.subplots(2, 1, figsize=(20, 6)) \n\n_ = sns.histplot(data=np.expm1(y_valid), bins=100, kde=True, color=\"green\", ax=axes[0],binrange=(np.expm1(y_valid.min()), np.expm1(y_valid.max())))\n_ = sns.histplot(data=np.expm1(y_pred) , bins=100, kde=True, color=\"green\", ax=axes[1],binrange=(np.expm1(y_valid.min()), np.expm1(y_valid.max())))\n\n# Show the plot\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:25:14.009612Z","iopub.execute_input":"2024-12-30T11:25:14.011075Z","iopub.status.idle":"2024-12-30T11:25:18.253806Z","shell.execute_reply.started":"2024-12-30T11:25:14.011044Z","shell.execute_reply":"2024-12-30T11:25:18.252892Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Up Sampling**\nShall we try to upsample the number of records from low frequency bins and try the model.\n\nThat strategy didn't work because performance were poor","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import resample\n# upsampling\ntrn, val = train_test_split(df[df['src']=='trn'], test_size=0.2, random_state=42)\nbins = pd.cut(trn['premium_amount_log'], bins=10, labels=False)\ntrn['bin'] = bins\n\n# Step 2: Define the desired sample count for each bin\ndesired_counts = {  # Configure the desired sample counts for each bin\n    0: 100000,  # Bin 0 -> 150 samples\n    1: 100000,  # Bin 1 -> 120 samples\n    2: 100000,  # Bin 2 -> 100 samples\n    3: 100000,  # Bin 3 -> 90 samples\n    4: 100000,\n    5: 100000,\n    6: 100000,\n    7: 100000,\n    8: 100000,\n    9: 100000   # Adjust these values as needed\n}\n\n# Step 3: Resample each bin\nresampled_bins = []\nfor bin_value, desired_count in desired_counts.items():\n    bin_data = trn[trn['bin'] == bin_value]  # Get records for this bin\n    if len(bin_data) > 0:  # Check if the bin is not empty\n        resampled_bin = resample(\n            bin_data,\n            replace=True,  # Sample with replacement\n            n_samples=desired_count,  # Desired number of samples for this bin\n            random_state=42\n        )\n        resampled_bins.append(resampled_bin)\n\n# Combine all resampled bins\nresampled_df = pd.concat(resampled_bins).reset_index(drop=True)\n\n# Print results\n\nprint(tabulate(pd.merge(trn['bin'].value_counts().sort_index(), resampled_df['bin'].value_counts().sort_index(), left_index=True, right_index=True, how=\"outer\"), \n               headers=[\"input\", \"resampled\"], \n               tablefmt=\"simple_grid\"))\n\n\ntrn = resampled_df.copy()\n\n# Create a histogram for column 'Premium Amount'\nfig, axes = plt.subplots(2, 1, figsize=(20, 5)) \n\n_ = sns.histplot(data=trn, x='premium_amount_log', bins=10, kde=True, color=\"blue\", ax=axes[0])\n_ = sns.histplot(data=val, x='premium_amount_log', bins=10, kde=True, color=\"green\", ax=axes[1])\n\n# Show the plot\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:25:18.254673Z","iopub.execute_input":"2024-12-30T11:25:18.25491Z","iopub.status.idle":"2024-12-30T11:25:26.224395Z","shell.execute_reply.started":"2024-12-30T11:25:18.25489Z","shell.execute_reply":"2024-12-30T11:25:26.22347Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LGBM native API with non transformed target variable\n\ndfs = trn.copy()\n\n# Split into train and validation sets\nX_train = dfs[feature_list]\nX_valid = val[feature_list]\ny_train = dfs['premium_amount_log']\ny_valid = val['premium_amount_log']\n#X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# LightGBM Dataset (categorical features need to be specified)\n\ntrain_data = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_cols, free_raw_data=False)\nvalid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_cols, free_raw_data=False)\n\n# Define RMSLE as a custom evaluation metric\n\ndef rmsle(y_pred, y_true):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(np.log1p(y_true) - np.log1p(y_pred))))\n    rmsle = np.round(rmsle,3)\n    return 'rmsle', rmsle, False\n\ndef rmsle_metric(y_pred, dataset):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_true = dataset.get_label()\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(np.log1p(y_true) - np.log1p(y_pred))))\n    return 'rmsle', rmsle, False\n\ndef rmsle_metric_for_log(y_pred, dataset):\n    \"\"\"Custom RMSLE evaluation metric for LightGBM.\"\"\"\n    y_true = dataset.get_label()\n    y_pred = np.clip(y_pred, 1e-15, None)  # Avoid log(0)\n    rmsle = np.sqrt(np.mean(np.square(y_true - y_pred)))\n    return 'rmsle', rmsle, False\n\ndef rmsle_objective(preds, train_data):\n    \"\"\"Custom RMSLE objective function for LightGBM.\"\"\"\n    y_true = train_data.get_label()\n    preds = np.maximum(preds, 1e-7)  # Avoid log(0) issues by ensuring predictions are > 0\n    grad = -1 * (np.log1p(y_true) - np.log1p(preds)) / (preds + 1)\n    hess = 1 / (preds + 1)**2\n    return grad, hess\n\n# Hyperparameter tuning with Optuna\ndef objective(trial):\n    params = {\n        'objective': 'regression',                                                # rmsle_objective, 'regression', 'huber', 'poisson'\n        #'metric': 'rmse',                                                          # 'l2', 'rmse', 'mae', 'mape'.....\n        'boosting_type': 'gbdt',                                                  # 'dart', 'goss', 'rf'\n        'device': 'gpu',                                                          # Enable GPU\n        'gpu_platform_id': 0,                                                     # Select the GPU (typically 0 for Kaggle)\n        'gpu_device_id': 0,                                                       # Select the device (typically 0 for Kaggle)\n        'num_leaves': round(trial.suggest_int('num_leaves', 31, 128),2),          # Maximum number of leaves in a tree. Controls the model’s complexity and affects overfitting/underfitting\n        'learning_rate': 0.1,                                                     # Controls the step size during each iteration. Lower values lead to slower but more accurate convergence\n        #'n_estimators': 1000,                                                      # Number of boosting iterations (trees). More trees improve accuracy but increase computation time.\n        'bagging_freq': 5,                                                        # Frequency of bagging (subsampling rows). Controls how often bagging is performed\n        'min_data_in_leaf': trial.suggest_int(\"min_data_in_leaf\", 20, 1000),      # Minimum number of samples required in a leaf node. Prevents leaves from being too small, which helps avoid overfitting.\n        'max_depth': trial.suggest_int('max_depth', -1, 15),                      # Maximum depth of a tree. Limits tree growth to control overfitting\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1.0, 10),              # L1 regularization term on weights. Helps control model complexity by penalizing large weights.\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1.0, 10),            # L2 regularization term on weights. Similar to reg_alpha but penalizes squared weights.\n        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.5, 1.0),  # Fraction of features (columns) sampled for each tree.\n        'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),                # Row sampling ratio for each tree. Prevents overfitting by using a subset of the data.\n        'verbosity':-1\n    }\n    \n    # Train the model\n    # Train model with early stopping and custom metric\n    booster = lgb.train(\n        params,\n        train_data,\n        num_boost_round=1000,\n        valid_sets=[valid_data],\n        valid_names=['valid'],\n        feval=rmsle_metric, #rmsle_metric_for_log, #rmsle_metric\n        callbacks=([lgb.early_stopping(stopping_rounds=50,verbose=0)]),\n    )\n    \n    # Predict and compute RMSLE (note that we exponentiate the predictions and true values)\n    y_pred = booster.predict(X_valid)\n    \n    return (mean_squared_error(y_valid,y_pred,squared=False)) # with log transformed target variable\n\n# Run hyperparameter optimization using Optuna\nstudy = optuna.create_study(direction='minimize')\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nstudy.optimize(objective, n_trials=5, show_progress_bar=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:25:26.225392Z","iopub.execute_input":"2024-12-30T11:25:26.225728Z","iopub.status.idle":"2024-12-30T11:29:34.512321Z","shell.execute_reply.started":"2024-12-30T11:25:26.225704Z","shell.execute_reply":"2024-12-30T11:29:34.511425Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Feature Importance**\nFollowing are the important features","metadata":{}},{"cell_type":"code","source":"# Get feature importance\nfeature_importance = final_model.feature_importance(importance_type='gain')\nfeature_names = feature_list\n\n# Create a DataFrame for feature importance\nimportance_df = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': feature_importance\n}).sort_values(by='Importance', ascending=False)\n\n# Display feature importance\n#with pd.option_context('display.max_rows', None): # setting the max rows\n#    display(importance_df)\n\n# Plot feature importance\n_ = plt.figure(figsize=(10, 15))\n_ = sns.barplot(x='Importance', y='Feature', data=importance_df, palette='viridis')\n_ = plt.title('Feature Importance (Best Model)')\n_ = plt.xlabel('Importance Score')\n_ = plt.ylabel('Feature')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:34.513612Z","iopub.execute_input":"2024-12-30T11:29:34.513955Z","iopub.status.idle":"2024-12-30T11:29:35.259851Z","shell.execute_reply.started":"2024-12-30T11:29:34.513919Z","shell.execute_reply":"2024-12-30T11:29:35.258977Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Submission**","metadata":{}},{"cell_type":"code","source":"# Make predictions on the test dataset\n\n# Split into train and validation sets\ntest_df = df[(df['src']=='tst')].copy()\n\n#tst = np.expm1(final_model.predict(test_df[feature_list]))\ntest_df.loc[:, 'Premium Amount'] = np.expm1(final_model.predict(test_df[feature_list]))\n\n# Submission file generation\n\nsubmission_df = test_df[['id','Premium Amount']].copy()\nsubmission_df['Premium Amount'] = submission_df['Premium Amount'].round(decimals=3)\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"submission.csv file generation completed\")","metadata":{"execution":{"iopub.status.busy":"2024-12-30T11:29:35.260661Z","iopub.execute_input":"2024-12-30T11:29:35.260886Z","iopub.status.idle":"2024-12-30T11:29:42.349139Z","shell.execute_reply.started":"2024-12-30T11:29:35.260866Z","shell.execute_reply":"2024-12-30T11:29:42.348311Z"},"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null}]}