{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"colab":{"provenance":[],"gpuType":"T4","toc_visible":true},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sb","metadata":{"trusted":true,"id":"VzqH_UUvVRx2","execution":{"iopub.status.busy":"2024-12-30T14:47:33.885552Z","iopub.execute_input":"2024-12-30T14:47:33.885873Z","iopub.status.idle":"2024-12-30T14:47:35.218977Z","shell.execute_reply.started":"2024-12-30T14:47:33.885838Z","shell.execute_reply":"2024-12-30T14:47:35.218171Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Explore data","metadata":{"id":"m-wrqQtXVRx2"}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\nX_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\ny_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"id":"g3PopKudVRx3","execution":{"iopub.status.busy":"2024-12-30T14:47:35.220105Z","iopub.execute_input":"2024-12-30T14:47:35.220574Z","iopub.status.idle":"2024-12-30T14:47:44.211497Z","shell.execute_reply.started":"2024-12-30T14:47:35.220523Z","shell.execute_reply":"2024-12-30T14:47:44.210638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(5)","metadata":{"trusted":true,"id":"uNtleY-yVRx3","executionInfo":{"status":"ok","timestamp":1735146894152,"user_tz":-420,"elapsed":5,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"ac0e6c06-53cd-4d8a-c0f3-013a184d0ac1","execution":{"iopub.status.busy":"2024-12-30T14:47:44.213523Z","iopub.execute_input":"2024-12-30T14:47:44.213862Z","iopub.status.idle":"2024-12-30T14:47:44.248513Z","shell.execute_reply.started":"2024-12-30T14:47:44.21384Z","shell.execute_reply":"2024-12-30T14:47:44.247606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"id":"SBcgl5oeVRx3","executionInfo":{"status":"ok","timestamp":1735146895319,"user_tz":-420,"elapsed":1170,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"2948c41a-1eee-4d14-8159-b4353cb7e007","execution":{"iopub.status.busy":"2024-12-30T14:47:44.249974Z","iopub.execute_input":"2024-12-30T14:47:44.2503Z","iopub.status.idle":"2024-12-30T14:47:44.793498Z","shell.execute_reply.started":"2024-12-30T14:47:44.250266Z","shell.execute_reply":"2024-12-30T14:47:44.792596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"id":"sdzIupIOVRx3","executionInfo":{"status":"ok","timestamp":1735146896438,"user_tz":-420,"elapsed":1121,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"4134fbfc-348f-455b-d582-a1ba3baa4058","execution":{"iopub.status.busy":"2024-12-30T14:47:44.794485Z","iopub.execute_input":"2024-12-30T14:47:44.794809Z","iopub.status.idle":"2024-12-30T14:47:45.323148Z","shell.execute_reply.started":"2024-12-30T14:47:44.794776Z","shell.execute_reply":"2024-12-30T14:47:45.322231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor = [x for x in train.columns if x not in X_test.columns]\npredictor","metadata":{"trusted":true,"id":"SJl5lUPSVRx4","outputId":"7e41d437-ee37-443c-d6b1-e4e578407ebc","executionInfo":{"status":"ok","timestamp":1735146915335,"user_tz":-420,"elapsed":318,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:45.324234Z","iopub.execute_input":"2024-12-30T14:47:45.324582Z","iopub.status.idle":"2024-12-30T14:47:45.329701Z","shell.execute_reply.started":"2024-12-30T14:47:45.324531Z","shell.execute_reply":"2024-12-30T14:47:45.328962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Premium Amount'].describe()","metadata":{"trusted":true,"id":"RBoA1C9DVRx4","executionInfo":{"status":"ok","timestamp":1735146916109,"user_tz":-420,"elapsed":476,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"97b8c0b1-e676-4eca-d7e4-48f3f984ec42","execution":{"iopub.status.busy":"2024-12-30T14:47:45.330726Z","iopub.execute_input":"2024-12-30T14:47:45.331053Z","iopub.status.idle":"2024-12-30T14:47:45.393066Z","shell.execute_reply.started":"2024-12-30T14:47:45.331017Z","shell.execute_reply":"2024-12-30T14:47:45.392179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.isnull().sum()","metadata":{"trusted":true,"id":"Bi_0fV08VRx4","outputId":"8c8ee924-431a-422f-f3e6-b806c63228c0","executionInfo":{"status":"ok","timestamp":1735146916110,"user_tz":-420,"elapsed":4,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:45.393892Z","iopub.execute_input":"2024-12-30T14:47:45.394107Z","iopub.status.idle":"2024-12-30T14:47:45.746494Z","shell.execute_reply.started":"2024-12-30T14:47:45.394089Z","shell.execute_reply":"2024-12-30T14:47:45.74564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example\ntrain.iloc[1]","metadata":{"trusted":true,"id":"DXqmLhQfVRx4","executionInfo":{"status":"ok","timestamp":1735146916110,"user_tz":-420,"elapsed":3,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"9fb749db-ae55-4962-891f-4d7b67cc632e","execution":{"iopub.status.busy":"2024-12-30T14:47:45.750763Z","iopub.execute_input":"2024-12-30T14:47:45.751066Z","iopub.status.idle":"2024-12-30T14:47:45.758054Z","shell.execute_reply.started":"2024-12-30T14:47:45.751038Z","shell.execute_reply":"2024-12-30T14:47:45.757046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.concat([train, X_test])\ndf.head(5)","metadata":{"trusted":true,"id":"5cwkgcDPVRx4","outputId":"01241b24-c190-468a-f686-d4ba2a5be8ac","executionInfo":{"status":"ok","timestamp":1735146916433,"user_tz":-420,"elapsed":326,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:45.760113Z","iopub.execute_input":"2024-12-30T14:47:45.760405Z","iopub.status.idle":"2024-12-30T14:47:46.163459Z","shell.execute_reply.started":"2024-12-30T14:47:45.760376Z","shell.execute_reply":"2024-12-30T14:47:46.162518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.set_index('id')","metadata":{"trusted":true,"id":"gJrVHZfsVRx4","outputId":"0930be71-44a0-425b-d3c9-3e72fe65dcbf","executionInfo":{"status":"ok","timestamp":1735146917536,"user_tz":-420,"elapsed":1104,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:46.164375Z","iopub.execute_input":"2024-12-30T14:47:46.16473Z","iopub.status.idle":"2024-12-30T14:47:47.246612Z","shell.execute_reply.started":"2024-12-30T14:47:46.164697Z","shell.execute_reply":"2024-12-30T14:47:47.24563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# List values of categorical columns (except Policy Start Date)\ncat_columns = df.select_dtypes(include = 'object').columns\n\nfor col in cat_columns:\n    if col != 'Policy Start Date':\n        print(f'Column {col} has {len(set(df[col]))} values: [{set(df[col])}]')","metadata":{"trusted":true,"id":"FAtY6zMqVRx5","outputId":"4a9b59e8-1451-4fac-8336-c20525199e1e","executionInfo":{"status":"ok","timestamp":1735146921353,"user_tz":-420,"elapsed":3819,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:47.247472Z","iopub.execute_input":"2024-12-30T14:47:47.247791Z","iopub.status.idle":"2024-12-30T14:47:51.480202Z","shell.execute_reply.started":"2024-12-30T14:47:47.247761Z","shell.execute_reply":"2024-12-30T14:47:51.479167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Date range: [{df['Policy Start Date'].min()} - {df['Policy Start Date'].max()}]\")","metadata":{"trusted":true,"id":"oWBFkb9JVRx5","outputId":"551eaa0e-2e8d-407d-80f0-2512309e72ee","executionInfo":{"status":"ok","timestamp":1735146921664,"user_tz":-420,"elapsed":313,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"execution":{"iopub.status.busy":"2024-12-30T14:47:51.481082Z","iopub.execute_input":"2024-12-30T14:47:51.481404Z","iopub.status.idle":"2024-12-30T14:47:52.014748Z","shell.execute_reply.started":"2024-12-30T14:47:51.481369Z","shell.execute_reply":"2024-12-30T14:47:52.013834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.boxplot(column = 'Premium Amount', by = 'Marital Status')","metadata":{"trusted":true,"id":"JY1DrtPFVRx5","executionInfo":{"status":"ok","timestamp":1735146924020,"user_tz":-420,"elapsed":2357,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"1c35afe2-a6e6-4036-fd58-6e7db6b4a593","execution":{"iopub.status.busy":"2024-12-30T14:47:52.015403Z","iopub.execute_input":"2024-12-30T14:47:52.015684Z","iopub.status.idle":"2024-12-30T14:47:53.764799Z","shell.execute_reply.started":"2024-12-30T14:47:52.015657Z","shell.execute_reply":"2024-12-30T14:47:53.763884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.boxplot(column = 'Premium Amount', by = 'Smoking Status')","metadata":{"trusted":true,"id":"fjhRUVkLVRx5","executionInfo":{"status":"ok","timestamp":1735146925797,"user_tz":-420,"elapsed":1780,"user":{"displayName":"Thành Đạt Tô","userId":"00907423653334339718"}},"outputId":"f7c2d57b-d125-4549-e7a1-476fabb03374","execution":{"iopub.status.busy":"2024-12-30T14:47:53.765817Z","iopub.execute_input":"2024-12-30T14:47:53.766192Z","iopub.status.idle":"2024-12-30T14:47:55.166048Z","shell.execute_reply.started":"2024-12-30T14:47:53.76616Z","shell.execute_reply":"2024-12-30T14:47:55.16517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering","metadata":{"id":"NEgy0Hh2VRx5"}},{"cell_type":"markdown","source":"# Training","metadata":{"id":"v71ZgbGpfp2j"}},{"cell_type":"markdown","source":"## Build pipeline","metadata":{"id":"6umw16YXNyUV"}},{"cell_type":"code","source":"train = train.set_index('id')\nX_test = X_test.set_index('id')","metadata":{"id":"sVGn-cWlubUV","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:47:55.166957Z","iopub.execute_input":"2024-12-30T14:47:55.167177Z","iopub.status.idle":"2024-12-30T14:47:55.444403Z","shell.execute_reply.started":"2024-12-30T14:47:55.167158Z","shell.execute_reply":"2024-12-30T14:47:55.443454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder, LabelEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import make_pipeline","metadata":{"id":"0Z8GrCUNN8BB","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:47:55.445333Z","iopub.execute_input":"2024-12-30T14:47:55.44561Z","iopub.status.idle":"2024-12-30T14:47:55.768056Z","shell.execute_reply.started":"2024-12-30T14:47:55.445577Z","shell.execute_reply":"2024-12-30T14:47:55.767114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_columns = train.select_dtypes(include=['number']).columns\nfor col in numeric_columns:\n    if col in X_test.columns:\n        med_val = train[col].median()\n        train[col].fillna(med_val, inplace=True)\n        X_test[col].fillna(med_val, inplace=True)\n\nobject_columns = train.select_dtypes(include=['object']).columns\nfor col in object_columns:\n    if col in X_test.columns:\n        train[col].fillna(\"<UNK>\", inplace=True)\n        X_test[col].fillna(\"<UNK>\", inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:47:55.769064Z","iopub.execute_input":"2024-12-30T14:47:55.769444Z","iopub.status.idle":"2024-12-30T14:47:57.598281Z","shell.execute_reply.started":"2024-12-30T14:47:55.769398Z","shell.execute_reply":"2024-12-30T14:47:57.597378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"col_name = 'Policy Start Date'\ntrain[col_name] = pd.to_datetime(train[col_name], errors='raise')\nX_test[col_name] = pd.to_datetime(X_test[col_name], errors='raise')\n\ntrain[col_name] = train[col_name].astype(np.int64) / 10**9\nX_test[col_name] = X_test[col_name].astype(np.int64) / 10**9","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:47:57.599163Z","iopub.execute_input":"2024-12-30T14:47:57.599371Z","iopub.status.idle":"2024-12-30T14:47:58.262999Z","shell.execute_reply.started":"2024-12-30T14:47:57.599342Z","shell.execute_reply":"2024-12-30T14:47:58.262047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bin_features = ['Gender', 'Smoking Status']\nord_features = {\n    'Exercise Frequency': ['Rarely', 'Monthly', 'Weekly', 'Daily']\n}\nnomi_features = ['Marital Status', 'Education Level', 'Occupation', 'Location', 'Policy Type', 'Customer Feedback', 'Property Type']\n\nlabel_enc = LabelEncoder()\nfor feature in bin_features:\n    train[feature] = label_enc.fit_transform(train[feature])\n    X_test[feature] = label_enc.transform(X_test[feature])\n\nfor feature, order in ord_features.items():\n    ord_enc = OrdinalEncoder(categories=[order])\n    train[feature] = ord_enc.fit_transform(train[[feature]]).flatten()\n    X_test[feature] = ord_enc.transform(X_test[[feature]]).flatten()\n\ntrain = pd.get_dummies(train, columns=nomi_features, drop_first=True)\nX_test = pd.get_dummies(X_test, columns=nomi_features, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:47:58.263838Z","iopub.execute_input":"2024-12-30T14:47:58.264087Z","iopub.status.idle":"2024-12-30T14:48:00.500748Z","shell.execute_reply.started":"2024-12-30T14:47:58.264066Z","shell.execute_reply":"2024-12-30T14:48:00.499814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_columns = train.select_dtypes(include=['float64']).columns\nnum_columns = num_columns[num_columns != 'Premium Amount']\n\ntrain['Annual Income'] = np.log1p(train['Annual Income'])\nX_test['Annual Income'] = np.log1p(X_test['Annual Income'])\n\nscaler = StandardScaler()\ntrain[num_columns] = scaler.fit_transform(train[num_columns])\nX_test[num_columns] = scaler.transform(X_test[num_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:00.501655Z","iopub.execute_input":"2024-12-30T14:48:00.501949Z","iopub.status.idle":"2024-12-30T14:48:00.891828Z","shell.execute_reply.started":"2024-12-30T14:48:00.501916Z","shell.execute_reply":"2024-12-30T14:48:00.890902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.drop(['Premium Amount'], axis = 1)\ny = train['Premium Amount']\ny_log = np.log1p(y)","metadata":{"id":"b9NpXZd4Z1Fd","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:00.892744Z","iopub.execute_input":"2024-12-30T14:48:00.892991Z","iopub.status.idle":"2024-12-30T14:48:00.944304Z","shell.execute_reply.started":"2024-12-30T14:48:00.89297Z","shell.execute_reply":"2024-12-30T14:48:00.943602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:00.945026Z","iopub.execute_input":"2024-12-30T14:48:00.945254Z","iopub.status.idle":"2024-12-30T14:48:01.135219Z","shell.execute_reply.started":"2024-12-30T14:48:00.945234Z","shell.execute_reply":"2024-12-30T14:48:01.134288Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"from catboost import Pool, CatBoostRegressor, EShapCalcType, EFeaturesSelectionAlgorithm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:01.188601Z","iopub.execute_input":"2024-12-30T14:48:01.188802Z","iopub.status.idle":"2024-12-30T14:48:01.601441Z","shell.execute_reply.started":"2024-12-30T14:48:01.188783Z","shell.execute_reply":"2024-12-30T14:48:01.600569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CatBoostAndOptunaParams:\n    def __init__(self, X, y, n_folds=10, hyperparam_bounds=None, random_state=86, task_type='GPU', verbose=0):\n        self.X = X\n        self.y = y\n        self.n_folds = n_folds\n        self.random_state = random_state\n        self.task_type = task_type\n        self.verbose = verbose\n        self.hyperparam_bounds = hyperparam_bounds or {\n            'iterations': (500, 800),\n            'depth': (6, 9),\n            'learning_rate': (0.05, 0.2),\n            'l2_leaf_reg': (0.3, 1.0),\n            'loss_function': ['RMSE']\n        }\n\n        # Placeholders\n        self.hyperparams_history = []\n        self._oof_predictions = []\n        self.best_score = float('inf')\n\n    def _suggest_hyperparams(self, trial):\n        return {\n            'iterations': trial.suggest_int('iterations', *self.hyperparam_bounds['iterations']),\n            'depth': trial.suggest_int('depth', *self.hyperparam_bounds['depth']),\n            'learning_rate': trial.suggest_float('learning_rate', *self.hyperparam_bounds['learning_rate'], log=True),\n            'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', *self.hyperparam_bounds['l2_leaf_reg'], log=True),\n            'loss_function': trial.suggest_categorical('loss_function', self.hyperparam_bounds['loss_function'])\n        }\n\n\n    def objective(self, trial):\n\n        params = self._suggest_hyperparams(trial)\n\n        # Assigning string type variables to cat_features for fitting\n        cat_features = [col for col in self.X.select_dtypes(include=['object', 'string']).columns]\n\n        # K-Fold cross-validation\n        folds = KFold(n_splits=self.n_folds, shuffle = True, random_state=self.random_state)\n        fold_rmsle = []\n        oof_preds = np.zeros(len(self.y))\n\n\n        for fold, (train_idx, val_idx) in enumerate(folds.split(self.X, self.y)):\n            X_train, y_train = self.X.iloc[train_idx], self.y.iloc[train_idx]\n            X_val, y_val = self.X.iloc[val_idx], self.y.iloc[val_idx]\n\n            # Model creation and training\n            model = CatBoostRegressor(\n                iterations =params['iterations'],\n                depth =params['depth'],\n                learning_rate =params['learning_rate'],\n                loss_function =params['loss_function'],\n                random_state =self.random_state,\n                l2_leaf_reg =params['l2_leaf_reg'],\n                task_type =self.task_type,\n                verbose =self.verbose\n            )\n            model.fit(X_train, y_train, cat_features=cat_features)\n\n\n            val_pred = model.predict(X_val)\n            val_pred = np.maximum(0, val_pred) # Clipping prediction values\n\n            # Getting oof predictions\n            oof_preds[val_idx] = val_pred\n\n\n            fold_rmsle.append(mean_squared_log_error(y_val, val_pred, squared = False))\n\n        # Saving oof predictions\n        self._oof_predictions.append(oof_preds.copy())\n\n\n        # Calculating mean rmsle across folds\n        mean_rmsle = np.mean(fold_rmsle)\n\n        self.hyperparams_history.append({**params, 'RMSE': mean_rmsle})\n        return mean_rmsle\n\n    def optimize(self, n_trials=5, direction=\"minimize\"):\n        study = optuna.create_study(direction=direction)\n        study.optimize(self.objective, n_trials=n_trials)\n        self.best_params = study.best_params\n        self.best_value = study.best_value\n        return study\n\n    @property\n    def oof(self):\n        if self._oof_predictions is None:\n            raise ValueError(\"No OOF predictions available\")\n        return self._oof_predictions\n\n    @property\n    def history(self):\n        if not self.hyperparams_history:\n            raise ValueError(\"No hyperparameter history available\")\n        self.hyperparams_history = sorted(self.hyperparams_history, key=lambda x: x['RMSE'])\n        return self.hyperparams_history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:01.602356Z","iopub.execute_input":"2024-12-30T14:48:01.602613Z","iopub.status.idle":"2024-12-30T14:48:01.614474Z","shell.execute_reply.started":"2024-12-30T14:48:01.602588Z","shell.execute_reply":"2024-12-30T14:48:01.613498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_log1p = np.log1p(y)\n\nhyperparam_search_1 = CatBoostAndOptunaParams(X, y_log1p, n_folds = 5)\nfirst_study = hyperparam_search_1.optimize(n_trials=5)\nhyperparam_search_2 = CatBoostAndOptunaParams(X, y_log1p, n_folds = 5)\nsecond_study = hyperparam_search_2.optimize(n_trials=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T14:48:01.615298Z","iopub.execute_input":"2024-12-30T14:48:01.615602Z","iopub.status.idle":"2024-12-30T14:53:33.522342Z","shell.execute_reply.started":"2024-12-30T14:48:01.615572Z","shell.execute_reply":"2024-12-30T14:53:33.521433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=86)\n\ny_train_log1p = np.log1p(y_train)\ny_val_log1p = np.log1p(y_val)\n\ncat_features = [col for col in X_train.select_dtypes(include=['object', 'string']).columns]\n\ntrain_pool = Pool(X_train, y_train_log1p, feature_names = X_train.columns.tolist(), cat_features=cat_features)\nval_pool = Pool(X_val, y_val_log1p, feature_names = X_val.columns.tolist(), cat_features=cat_features)\n\ncat_1 = CatBoostRegressor(**first_study.best_params, random_state=86, task_type='GPU', verbose = 0, cat_features=cat_features)\ncat_2 = CatBoostRegressor(**second_study.best_params, random_state=86, task_type='GPU', verbose = 0, cat_features=cat_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:09:45.747812Z","iopub.execute_input":"2024-12-30T15:09:45.748133Z","iopub.status.idle":"2024-12-30T15:09:46.089019Z","shell.execute_reply.started":"2024-12-30T15:09:45.748104Z","shell.execute_reply":"2024-12-30T15:09:46.088097Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom lightgbm import early_stopping, log_evaluation","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:10:19.377836Z","iopub.execute_input":"2024-12-30T15:10:19.378123Z","iopub.status.idle":"2024-12-30T15:10:22.014612Z","shell.execute_reply.started":"2024-12-30T15:10:19.378099Z","shell.execute_reply":"2024-12-30T15:10:22.013743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_LGBM(trial):\n  #Define param\n  param = {\n      \"objective\": \"regression\",\n      \"metric\": \"rmse\",\n      \"boosting_type\": \"gbdt\",\n      \"device\": \"gpu\",\n      \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.2),\n      \"n_estimators\": trial.suggest_int(\"n_estimators\", 200, 1000),\n      \"max_depth\": trial.suggest_int(\"max_depth\", 3, 15),\n      \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 300),\n      \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100),\n      \"subsample\": trial.suggest_uniform(\"subsample\", 0.5, 1.0),\n      \"colsample_bytree\": trial.suggest_uniform(\"colsample_bytree\", 0.5, 1.0),\n      \"reg_alpha\": trial.suggest_loguniform(\"reg_alpha\", 1e-3, 3.0),\n      \"reg_lambda\": trial.suggest_loguniform(\"reg_lambda\", 1e-3, 3.0),\n      \"verbose\": -1  # Suppress warnings and messages\n  }\n\n  X_train_split, X_val, y_train_split, y_val = train_test_split(X, y_log, test_size=0.2, random_state=86)\n\n  model = lgb.LGBMRegressor(**param)\n  model.fit(X_train_split, y_train_split, eval_set=[(X_val, y_val)], callbacks=[early_stopping(stopping_rounds=30, verbose=False), log_evaluation(10)])\n\n  y_pred = model.predict(X_val)\n  rmsle = np.sqrt(mean_squared_log_error(y_val, y_pred))\n\n  return rmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:10:22.667433Z","iopub.execute_input":"2024-12-30T15:10:22.667959Z","iopub.status.idle":"2024-12-30T15:10:22.673769Z","shell.execute_reply.started":"2024-12-30T15:10:22.667932Z","shell.execute_reply":"2024-12-30T15:10:22.672846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"third_study = optuna.create_study(direction=\"minimize\")\nthird_study.optimize(objective_LGBM, n_trials=10)\nfourth_study = optuna.create_study(direction=\"minimize\")\nfourth_study.optimize(objective_LGBM, n_trials=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:14:52.538242Z","iopub.execute_input":"2024-12-30T15:14:52.538534Z","iopub.status.idle":"2024-12-30T15:19:51.195982Z","shell.execute_reply.started":"2024-12-30T15:14:52.538511Z","shell.execute_reply":"2024-12-30T15:19:51.195036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params_lgbm_1 = third_study.best_params\nbest_params_lgbm_1[\"objective\"] = \"regression\"\nbest_params_lgbm_1[\"metric\"] = \"rmse\"\nbest_params_lgbm_1[\"device\"] = \"gpu\"\n\nbest_params_lgbm_2 = fourth_study.best_params\nbest_params_lgbm_2[\"objective\"] = \"regression\"\nbest_params_lgbm_2[\"metric\"] = \"rmse\"\nbest_params_lgbm_2[\"device\"] = \"gpu\"\n\n# Train the final model\nlgbm_1 = lgb.LGBMRegressor(**best_params_lgbm_1)\nlgbm_2 = lgb.LGBMRegressor(**best_params_lgbm_2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:31:37.047988Z","iopub.execute_input":"2024-12-30T15:31:37.048316Z","iopub.status.idle":"2024-12-30T15:31:37.053411Z","shell.execute_reply.started":"2024-12-30T15:31:37.048286Z","shell.execute_reply":"2024-12-30T15:31:37.052458Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Random Forest","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:31:43.963966Z","iopub.execute_input":"2024-12-30T15:31:43.964264Z","iopub.status.idle":"2024-12-30T15:31:44.086199Z","shell.execute_reply.started":"2024-12-30T15:31:43.964239Z","shell.execute_reply":"2024-12-30T15:31:44.085451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf_params = {'n_estimators': 423, 'max_depth': 15, 'min_samples_split': 3, 'min_samples_leaf': 2}\nrf = RandomForestRegressor(**rf_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:31:45.984466Z","iopub.execute_input":"2024-12-30T15:31:45.984817Z","iopub.status.idle":"2024-12-30T15:31:45.988711Z","shell.execute_reply.started":"2024-12-30T15:31:45.984786Z","shell.execute_reply":"2024-12-30T15:31:45.987817Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Stacking Regressor","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import VotingRegressor, StackingRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:31:51.645891Z","iopub.execute_input":"2024-12-30T15:31:51.646293Z","iopub.status.idle":"2024-12-30T15:31:51.650344Z","shell.execute_reply.started":"2024-12-30T15:31:51.646259Z","shell.execute_reply":"2024-12-30T15:31:51.649395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cv_test_preds(X, y, X_test, n_folds=5, random_state=86, task_type='GPU', verbose=0):\n\n    folds = KFold(n_splits=n_folds, shuffle=True, random_state=random_state)\n    test_preds = []\n\n    for fold, (train_idx, val_idx) in enumerate(folds.split(X, y)):\n        print(f\"Current fold: {fold}...\")\n        X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n\n        cat_features = [col for col in X.select_dtypes(include=['object', 'string']).columns]\n\n        # model = StackingRegressor(estimators=[('cat_2', cat_2), ('lgbm_1', lgbm_1), ('lgbm_2', lgbm_2), ('rf', rf)],\n        #                          final_estimator=cat_1,\n        #                          n_jobs = -1)\n        model = StackingRegressor(estimators=[('cat_1', cat_1), ('lgbm_1', lgbm_1)],\n                                 final_estimator=rf)\n        model.fit(X_train, y_train)\n\n        test_preds.append(model.predict(X_test))\n\n    return test_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:47:56.926358Z","iopub.execute_input":"2024-12-30T15:47:56.926735Z","iopub.status.idle":"2024-12-30T15:47:56.932973Z","shell.execute_reply.started":"2024-12-30T15:47:56.926698Z","shell.execute_reply":"2024-12-30T15:47:56.931988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_final_preds = cv_test_preds(X, y_log1p, X_test, n_folds = 5)\n\ny_pred = sum(np.expm1(X_final_preds))/len(X_final_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T15:47:58.360278Z","iopub.execute_input":"2024-12-30T15:47:58.360595Z","iopub.status.idle":"2024-12-30T16:11:56.090252Z","shell.execute_reply.started":"2024-12-30T15:47:58.360539Z","shell.execute_reply":"2024-12-30T16:11:56.088892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": X_test.index, 'Premium Amount': y_pred})\n\nprint(submission.head(10))\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file created successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T15:43:25.830697Z","iopub.execute_input":"2024-12-29T15:43:25.83098Z","iopub.status.idle":"2024-12-29T15:43:27.125856Z","shell.execute_reply.started":"2024-12-29T15:43:25.830957Z","shell.execute_reply":"2024-12-29T15:43:27.124731Z"}},"outputs":[],"execution_count":null}]}