{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading libraries & data\n## Loading Libraries","metadata":{}},{"cell_type":"code","source":"# Library load\nimport pandas as pd\nimport numpy as np\nimport torch\nimport cupy as cp\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.impute import KNNImputer\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\n\nimport optuna\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nimport matplotlib.pyplot as plt\nfrom cuml.neighbors import KNeighborsClassifier\n\n# Setting display options\npd.set_option('display.max_columns', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:18.932631Z","iopub.execute_input":"2024-12-23T15:54:18.932975Z","iopub.status.idle":"2024-12-23T15:54:37.473431Z","shell.execute_reply.started":"2024-12-23T15:54:18.932937Z","shell.execute_reply":"2024-12-23T15:54:37.472747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"# Import data\ntrain = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\") \ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\n# Having a look at the train data\ntrain.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:37.474432Z","iopub.execute_input":"2024-12-23T15:54:37.474868Z","iopub.status.idle":"2024-12-23T15:54:46.285245Z","shell.execute_reply.started":"2024-12-23T15:54:37.47484Z","shell.execute_reply":"2024-12-23T15:54:46.284352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Pre-Processing","metadata":{}},{"cell_type":"markdown","source":"## Handling Date Columns","metadata":{}},{"cell_type":"code","source":"# Recreating the date related function\ndef date(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    \n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Quarter'] = df['Policy Start Date'].dt.quarter\n    df['Day of Week'] = df['Policy Start Date'].dt.dayofweek\n    \n    df.drop('Policy Start Date', axis=1, inplace=True)\n    \n    return df\n\ntrain = date(train)\ntest = date(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:46.287166Z","iopub.execute_input":"2024-12-23T15:54:46.28744Z","iopub.status.idle":"2024-12-23T15:54:47.5789Z","shell.execute_reply.started":"2024-12-23T15:54:46.287414Z","shell.execute_reply":"2024-12-23T15:54:47.578203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For now I'm thinking about only imputing columns where data is missing for ~10% or greater\n# Columns to be imputed \ndata_missing_cols = ['Credit Score', 'Previous Claims', 'Number of Dependents', 'Occupation']\n\n# Let's try and understand the distribution of each missing variable\ntrain[data_missing_cols].hist(bins = 20, figsize=(10, 8), layout=(2, 2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:47.579865Z","iopub.execute_input":"2024-12-23T15:54:47.580189Z","iopub.status.idle":"2024-12-23T15:54:48.36086Z","shell.execute_reply.started":"2024-12-23T15:54:47.580162Z","shell.execute_reply":"2024-12-23T15:54:48.359768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Credit Score data is slightly skewed to the left and hence we can apply KNN algorithm for imputation \n# The data has also been analyzed and there doesn't seem to be any unusual values assocuated with columns ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:48.362094Z","iopub.execute_input":"2024-12-23T15:54:48.362373Z","iopub.status.idle":"2024-12-23T15:54:48.3662Z","shell.execute_reply.started":"2024-12-23T15:54:48.36234Z","shell.execute_reply":"2024-12-23T15:54:48.365438Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Building","metadata":{}},{"cell_type":"code","source":"# Data split\nX = train.drop(columns=['Premium Amount'])\ny = train['Premium Amount']\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:48.367546Z","iopub.execute_input":"2024-12-23T15:54:48.367954Z","iopub.status.idle":"2024-12-23T15:54:49.009153Z","shell.execute_reply.started":"2024-12-23T15:54:48.367915Z","shell.execute_reply":"2024-12-23T15:54:49.008407Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Building a baseline","metadata":{}},{"cell_type":"code","source":"# As suggested by Chris Deotte given the evaluation we can make our predictions to be exponented mean logs\npred = np.exp(np.mean(np.log1p(train['Premium Amount']))) - 1 \nlog_pred = np.log1p(pred)\nlog_y_test = np.log1p(y_test)\n\n# Calculate a function using RMSLE\ndef RMSLE(true,pred):\n    true_log = np.log1p(true)\n    pred_log = np.log1p(pred)\n    m = np.sqrt(np.mean( (true_log-pred_log)**2.0 ))\n    return m\n\nRMSLE(y_test, pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:54:49.01036Z","iopub.execute_input":"2024-12-23T15:54:49.010736Z","iopub.status.idle":"2024-12-23T15:54:49.027469Z","shell.execute_reply.started":"2024-12-23T15:54:49.010696Z","shell.execute_reply":"2024-12-23T15:54:49.02673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function for applying OOF \ndef evaluate_oof(model_class, X, y, params, n_splits=5):\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n    oof_predictions = np.zeros(len(y))\n\n    # Ensure X and y indices are consistent\n    X = X.reset_index(drop=True)\n    y = y.reset_index(drop=True)\n    \n    if model_class == xgb.XGBRegressor :\n        for train_idx, valid_idx in kf.split(X):\n            X_train_fold, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n            y_train_fold, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n            # Applying preprocessor to the data\n            X_train_fold = preprocessor.fit_transform(X_train_fold)\n            X_valid = preprocessor.transform(X_valid)\n    \n            # Move data to GPU\n            X_train_fold, X_valid, y_train_fold, y_valid = cp.array(X_train_fold), cp.array(X_valid), cp.array(y_train_fold), cp.array(y_valid) \n    \n            # Instantiate and train the model\n            model = model_class(**params, random_state=42)\n            model.fit(X_train_fold, y_train_fold)\n            \n            # Predict on the validation fold\n            y_pred = model.predict(X_valid)\n            y_pred = np.clip(cp.asnumpy(y_pred), 0, None)  # Convert to CPU and ensure non-negative\n            \n            oof_predictions[valid_idx] = y_pred\n\n        # Calculate RMSLE\n        y = cp.asnumpy(y)  # Ensure y is on CPU\n        rmsle = np.sqrt(mean_squared_log_error(y, oof_predictions))\n        return rmsle, oof_predictions\n\n    elif model_class == CatBoostRegressor : \n        for train_idx, valid_idx in kf.split(X):\n            X_train_fold, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n            y_train_fold, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n            # Applying preprocessor to the data\n            X_train_fold = preprocessor.fit_transform(X_train_fold)\n            X_valid = preprocessor.transform(X_valid)\n    \n            # Move data to GPU\n            # X_train_fold, X_valid, y_train_fold, y_valid = cp.array(X_train_fold), cp.array(X_valid), cp.array(y_train_fold), cp.array(y_valid) \n    \n            # Instantiate and train the model\n            model = model_class(**params, random_state=42)\n            model.fit(X_train_fold, y_train_fold)\n            \n            # Predict on the validation fold\n            y_pred = model.predict(X_valid)\n            y_pred = np.clip(y_pred, 0, None)  # Convert to CPU and ensure non-negative\n            \n            oof_predictions[valid_idx] = y_pred\n    \n        # Calculate RMSLE\n        # y = cp.asnumpy(y)  # Ensure y is on CPU\n        rmsle = np.sqrt(mean_squared_log_error(y, oof_predictions))\n        return rmsle, oof_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T15:57:36.970283Z","iopub.execute_input":"2024-12-23T15:57:36.970634Z","iopub.status.idle":"2024-12-23T15:57:36.980629Z","shell.execute_reply.started":"2024-12-23T15:57:36.970604Z","shell.execute_reply":"2024-12-23T15:57:36.979944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select model\ndef objective(trial, X, y, model_class):    \n    # Define the hyperparameter search space\n    # XGBRegression\n    if model == xgb.XGBRegressor : \n        params = {\n            ## Params for XGBoost\n            # Booster params\n            'booster' : trial.suggest_categorical('booster', ['gbtree','dart']),\n            'eta' : trial.suggest_float('eta', 0.01,0.3),\n            'gamma' : trial.suggest_float('gamma', 0,10),\n            'max_depth': trial.suggest_int('max_depth', 3, 15),\n            'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n            'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n            'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),\n            'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),\n            'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n            # General params\n            'tree_method': 'hist',  # Use GPU acceleration,\n            'device' :'cuda:0',\n            'predictor': 'gpu_predictor' }\n    # CatboostRegression\n    elif model == CatBoostRegressor :\n        params = {\n            ## Params for CatBoost\n            # Booster params\n            'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),  # Similar to XGBoost's eta\n            'depth': trial.suggest_int('depth', 3, 15),  # Equivalent to max_depth in XGBoost\n            'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-8, 10.0, log=True),  # Similar to reg_lambda in XGBoost\n            'colsample_bylevel': trial.suggest_float('colsample_bylevel', 0.5, 1.0),  # Similar to colsample_bytree in XGBoost\n            'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 1, 10),  # Equivalent to min_child_weight in XGBoost\n            'border_count': trial.suggest_int('border_count', 32, 255),  # Controls number of bins for discretization\n            # General params\n            # 'task_type': 'GPU'\n            }\n    \n    # Call the OOF evaluation function\n    rmsle, _ = evaluate_oof(model_class, X, y, params)\n    return rmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:02:28.580293Z","iopub.execute_input":"2024-12-23T16:02:28.581126Z","iopub.status.idle":"2024-12-23T16:02:28.588248Z","shell.execute_reply.started":"2024-12-23T16:02:28.581092Z","shell.execute_reply":"2024-12-23T16:02:28.587408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data imputation step is dropped based on the suggestion by Chris Deotte's wonderful observation where he clearly explains \n# correlation between NaNs and other variables\ndef impute_knn(x_train, col = 'remainder__Credit Score'):\n    knn = KNeighborsClassifier(n_neighbors=5)\n    \n    if not isinstance(x_train, pd.DataFrame):\n        x_train = pd.DataFrame(\n            x_train,\n            columns=preprocessor.get_feature_names_out()\n        )\n    \n    miss_idx = x_train[x_train[col].isna()].index\n    not_miss_idx = x_train.index.difference(miss_idx)\n    x_train_knn = x_train.drop(columns = [col]).loc[not_miss_idx]\n    y_train_knn = x_train[col].loc[not_miss_idx]\n    \n    # Fit the model on non-missing data\n    knn.fit(\n        x_train_knn,\n        y_train_knn\n    )\n\n    # Predict for rows with missing values\n    predictions = knn.predict(x_train.drop(columns = [col]).loc[miss_idx])\n    x_train.loc[miss_idx][col] = predictions\n    \n    return x_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:02:28.589937Z","iopub.execute_input":"2024-12-23T16:02:28.590637Z","iopub.status.idle":"2024-12-23T16:02:28.607917Z","shell.execute_reply.started":"2024-12-23T16:02:28.590608Z","shell.execute_reply":"2024-12-23T16:02:28.607086Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pipeline Creation","metadata":{}},{"cell_type":"code","source":"# Example columns in your dataset\ntrain_numeric_col = train.select_dtypes(exclude = 'object').columns.drop(['Premium Amount'])\ncategorical_nominal = ['Gender', 'Marital Status', 'Location', 'Smoking Status', 'Property Type', 'Occupation']\ncategorical_ordinal = ['Education Level','Policy Type', 'Customer Feedback', 'Exercise Frequency']\n\n# Ordinal encoding for ordered categories\nmapping = [['missing','High School', \"Bachelor's\", \"Master's\", 'PhD'],\n                    ['missing','Basic','Premium','Comprehensive' ],\n                    ['missing','Poor', 'Average', 'Good'],\n                    ['missing','Rarely', 'Monthly', 'Weekly', 'Daily']]\n\n# Preprocessing steps\npreprocessor = ColumnTransformer(\n    transformers=[\n        # Impute missing and One-Hot Encode Nominal Categorical Variables\n        ('nominal', Pipeline(steps=[\n            ('imputer', SimpleImputer(strategy='constant', fill_value = \"missing\")),  # Impute with most frequent category\n            ('encoder', OneHotEncoder(handle_unknown='ignore', drop='first'))\n        ]), categorical_nominal),\n        # Impute missing and Ordinal Encode Ordered Categories\n        ('ordinal', Pipeline(steps=[\n            ('imputer', SimpleImputer(strategy='constant', fill_value = \"missing\")),  # Impute with most frequent category\n            ('encoder', OrdinalEncoder(categories=mapping))\n        ]), categorical_ordinal),\n        # Impute missing for Numerical Data\n        ('numerical', SimpleImputer(strategy='constant', fill_value = -1), train_numeric_col) # Fill missing with median\n        # ('credit_score_imputation', ImputeKNN(missing_column='Credit Score'), ['Credit Score'])\n   ],\n    remainder='passthrough'  # Keep unused columns\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:02:28.608869Z","iopub.execute_input":"2024-12-23T16:02:28.609147Z","iopub.status.idle":"2024-12-23T16:02:28.679007Z","shell.execute_reply.started":"2024-12-23T16:02:28.609122Z","shell.execute_reply":"2024-12-23T16:02:28.678283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')  # Minimize RMSLE\n# Selecting the model type\nmodel = CatBoostRegressor\nstudy.optimize(lambda trial : objective(trial, X = X, y = y, model_class = model), \n               n_trials=1,\n               timeout=300,\n               show_progress_bar=True)\n\nprint(\"Best hyperparameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)\n\n# Train final model with best hyperparameters\nbest_params = study.best_params\n# best_params.update({\n    # 'tree_method': 'gpu_hist',\n    # 'device': 'cuda',\n    # 'predictor': 'gpu_predictor'\n# })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:02:28.681021Z","iopub.execute_input":"2024-12-23T16:02:28.68138Z","iopub.status.idle":"2024-12-23T16:09:22.384478Z","shell.execute_reply.started":"2024-12-23T16:02:28.68134Z","shell.execute_reply":"2024-12-23T16:09:22.383651Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Predictions","metadata":{}},{"cell_type":"code","source":"X_train = preprocessor.fit_transform(X_train) \nX_test = preprocessor.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:34:20.7772Z","iopub.execute_input":"2024-12-23T16:34:20.777545Z","iopub.status.idle":"2024-12-23T16:34:25.057393Z","shell.execute_reply.started":"2024-12-23T16:34:20.777513Z","shell.execute_reply":"2024-12-23T16:34:25.056408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = CatBoostRegressor(**best_params, random_state=42)\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:34:25.058993Z","iopub.execute_input":"2024-12-23T16:34:25.059285Z","iopub.status.idle":"2024-12-23T16:35:41.744646Z","shell.execute_reply.started":"2024-12-23T16:34:25.059257Z","shell.execute_reply":"2024-12-23T16:35:41.743782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict and evaluate\ny_pred = model.predict(X_test)\n\nrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))\nprint(\"Root Mean Squared Logarithmic Error:\", rmsle)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:35:41.745829Z","iopub.execute_input":"2024-12-23T16:35:41.746229Z","iopub.status.idle":"2024-12-23T16:35:43.623775Z","shell.execute_reply.started":"2024-12-23T16:35:41.746189Z","shell.execute_reply":"2024-12-23T16:35:43.622856Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Observations \n- The initial model should always be a simple baseline\n- In a discussion post Chris Deotte suggests that we do not impute variables because of the correlation between NaNs and \n- Miniscule incremental improvement in the XGBoost model after initial trials. The model performs poorly wrt to the baseline\n- Trying out Catboost didn't result in any improvement as well","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Making our nth submission !\ntest_processed = preprocessor.transform(test)\ntest_predictions = model.predict(test_processed)\nsubmission = pd.DataFrame({'id': test['id'], 'Premium Amount': test_predictions})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:35:59.164468Z","iopub.execute_input":"2024-12-23T16:35:59.164812Z","iopub.status.idle":"2024-12-23T16:36:09.018016Z","shell.execute_reply.started":"2024-12-23T16:35:59.164781Z","shell.execute_reply":"2024-12-23T16:36:09.017328Z"}},"outputs":[],"execution_count":null}]}