{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.metrics import (mean_squared_log_error, mean_absolute_error, \nmean_squared_error, r2_score, mean_absolute_percentage_error)\nimport optuna\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\n\nfrom lightgbm import LGBMRegressor\nfrom statsmodels.regression.linear_model import OLS\n\npd.set_option('display.max_columns', 25)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:41.148037Z","iopub.execute_input":"2024-12-01T17:17:41.148435Z","iopub.status.idle":"2024-12-01T17:17:42.959383Z","shell.execute_reply.started":"2024-12-01T17:17:41.1484Z","shell.execute_reply":"2024-12-01T17:17:42.957783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv', \n                    parse_dates=['Policy Start Date'])\ntrain = train.drop(columns = ['id'])\nprint(train.shape)\ntrain.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:42.961994Z","iopub.execute_input":"2024-12-01T17:17:42.962541Z","iopub.status.idle":"2024-12-01T17:17:48.381345Z","shell.execute_reply.started":"2024-12-01T17:17:42.962504Z","shell.execute_reply":"2024-12-01T17:17:48.38023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:48.383277Z","iopub.execute_input":"2024-12-01T17:17:48.383627Z","iopub.status.idle":"2024-12-01T17:17:49.074599Z","shell.execute_reply.started":"2024-12-01T17:17:48.383592Z","shell.execute_reply":"2024-12-01T17:17:49.073423Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"code","source":"plt.hist(train['Premium Amount'], bins=100, density=1.0)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:49.0775Z","iopub.execute_input":"2024-12-01T17:17:49.078Z","iopub.status.idle":"2024-12-01T17:17:49.446479Z","shell.execute_reply.started":"2024-12-01T17:17:49.077943Z","shell.execute_reply":"2024-12-01T17:17:49.445428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# some visualizing functions\ndef violin(data, target, indep):\n    plt.figure(figsize=(6, 4))\n    sns.violinplot(data=data, x=indep, y=target)\n    plt.title('Violinplot')\n    plt.xlabel(indep)\n    plt.ylabel(target)\n    plt.show()\n\ndef scatter(data, x, y):\n    plt.figure(figsize=(6, 4))\n    sns.scatterplot(data=data, x=x, y=y)\n    plt.title('Violinplot')\n    plt.xlabel(x)\n    plt.ylabel(y)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:49.448027Z","iopub.execute_input":"2024-12-01T17:17:49.448449Z","iopub.status.idle":"2024-12-01T17:17:49.455916Z","shell.execute_reply.started":"2024-12-01T17:17:49.448405Z","shell.execute_reply":"2024-12-01T17:17:49.454525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = train.select_dtypes('object').columns.to_list()\ncat_cols = cat_cols + ['Number of Dependents']\nfor i in cat_cols:\n    print(f'Number of unique values in {i} columns', train[i].nunique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:49.457429Z","iopub.execute_input":"2024-12-01T17:17:49.457847Z","iopub.status.idle":"2024-12-01T17:17:50.231445Z","shell.execute_reply.started":"2024-12-01T17:17:49.457815Z","shell.execute_reply":"2024-12-01T17:17:50.230268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in cat_cols:\n    violin(train, 'Premium Amount', i)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:17:50.232733Z","iopub.execute_input":"2024-12-01T17:17:50.233065Z","iopub.status.idle":"2024-12-01T17:18:25.859373Z","shell.execute_reply.started":"2024-12-01T17:17:50.233034Z","shell.execute_reply":"2024-12-01T17:18:25.857831Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We cannot see any visual difference in Premium Amount across each categorical variable","metadata":{}},{"cell_type":"code","source":"# we add the feature which shows the number of days from Policy start date\ntrain['policy_duration'] = (datetime.now() - train['Policy Start Date']).dt.days\ntrain = train.drop(columns=['Policy Start Date'])\n\nnum_cols = [i for i in train.select_dtypes(['float64', 'int64']).columns.tolist() \n            if (i != 'log_premium')&(i != 'Premium Amount')&(i!='Number of Dependents')]\nnum_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:18:25.86102Z","iopub.execute_input":"2024-12-01T17:18:25.861565Z","iopub.status.idle":"2024-12-01T17:18:26.130191Z","shell.execute_reply.started":"2024-12-01T17:18:25.861503Z","shell.execute_reply":"2024-12-01T17:18:26.128973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in num_cols:\n    scatter(train, i, 'Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:18:26.131471Z","iopub.execute_input":"2024-12-01T17:18:26.131843Z","iopub.status.idle":"2024-12-01T17:18:46.852254Z","shell.execute_reply.started":"2024-12-01T17:18:26.131808Z","shell.execute_reply":"2024-12-01T17:18:46.850986Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We cannot see any correlation between variables and target","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nsns.heatmap(train[num_cols+['Premium Amount']].corr(), annot=True, fmt=\".2f\", cmap=\"coolwarm\", cbar=True)\nplt.title(\"Pearson Correlation Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:18:46.857126Z","iopub.execute_input":"2024-12-01T17:18:46.857614Z","iopub.status.idle":"2024-12-01T17:18:47.762097Z","shell.execute_reply.started":"2024-12-01T17:18:46.857565Z","shell.execute_reply":"2024-12-01T17:18:47.760933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nsns.heatmap(train[num_cols+['Premium Amount']].corr(method='spearman'), \n            annot=True, fmt=\".2f\", cmap=\"coolwarm\", cbar=True)\nplt.title(\"Spearman Correlation Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:18:47.763746Z","iopub.execute_input":"2024-12-01T17:18:47.764857Z","iopub.status.idle":"2024-12-01T17:19:02.43413Z","shell.execute_reply.started":"2024-12-01T17:18:47.764806Z","shell.execute_reply":"2024-12-01T17:19:02.432777Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can hardly see any significant correlation between target and predictors","metadata":{}},{"cell_type":"code","source":"for i in cat_cols:\n    le = LabelEncoder()\n    train[i] = le.fit_transform(train[i])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:02.435966Z","iopub.execute_input":"2024-12-01T17:19:02.436311Z","iopub.status.idle":"2024-12-01T17:19:04.92525Z","shell.execute_reply.started":"2024-12-01T17:19:02.436279Z","shell.execute_reply":"2024-12-01T17:19:04.923931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isna().mean()*100","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:04.926763Z","iopub.execute_input":"2024-12-01T17:19:04.927081Z","iopub.status.idle":"2024-12-01T17:19:04.963983Z","shell.execute_reply.started":"2024-12-01T17:19:04.927052Z","shell.execute_reply":"2024-12-01T17:19:04.962841Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# we split the data into dependent variable and independent ones\nX, y = train.drop(columns = 'Premium Amount'), train['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:04.965729Z","iopub.execute_input":"2024-12-01T17:19:04.96606Z","iopub.status.idle":"2024-12-01T17:19:05.080433Z","shell.execute_reply.started":"2024-12-01T17:19:04.966031Z","shell.execute_reply":"2024-12-01T17:19:05.079382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# we fill the nans with mean value\nmeans = X.mean()\nX = X.fillna(means)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:05.081535Z","iopub.execute_input":"2024-12-01T17:19:05.081885Z","iopub.status.idle":"2024-12-01T17:19:05.53034Z","shell.execute_reply.started":"2024-12-01T17:19:05.081852Z","shell.execute_reply":"2024-12-01T17:19:05.529033Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0,test_size= 0.2, shuffle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:05.531656Z","iopub.execute_input":"2024-12-01T17:19:05.53201Z","iopub.status.idle":"2024-12-01T17:19:06.051343Z","shell.execute_reply.started":"2024-12-01T17:19:05.531979Z","shell.execute_reply":"2024-12-01T17:19:06.049453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"m = LGBMRegressor(random_state=0).fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:06.052994Z","iopub.execute_input":"2024-12-01T17:19:06.053437Z","iopub.status.idle":"2024-12-01T17:19:11.591311Z","shell.execute_reply.started":"2024-12-01T17:19:06.05339Z","shell.execute_reply":"2024-12-01T17:19:11.590317Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pred =m.predict(X_train)\ntest_pred = m.predict(X_test)\n\nprint('RMSLE for Train %.3f'%mean_squared_log_error(y_train, train_pred,squared= False))\nprint('RMSLE for Test %.3f'%mean_squared_log_error(y_test, test_pred, squared= False))\nprint('-'*100)\nprint('RMSE for Train %.3f'%mean_squared_error(y_train, train_pred,squared= False))\nprint('RMSE for Test %.3f'%mean_squared_error(y_test, test_pred, squared= False))\nprint('-'*100)\nprint('MAE for Train %.3f'%mean_absolute_error(y_train, train_pred))\nprint('MAE for Test %.3f'%mean_absolute_error(y_test, test_pred))\nprint('-'*100)\nprint('MAPE for Train %.3f'%mean_absolute_percentage_error(y_train, train_pred))\nprint('MAPE for Test %.3f'%mean_absolute_percentage_error(y_test,test_pred))\nprint('-'*100)\nprint('R2 for Train %.3f'%r2_score(y_train, train_pred))\nprint('R2 for Test %.3f'%r2_score(y_test, test_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:11.592694Z","iopub.execute_input":"2024-12-01T17:19:11.593143Z","iopub.status.idle":"2024-12-01T17:19:18.060494Z","shell.execute_reply.started":"2024-12-01T17:19:11.593099Z","shell.execute_reply":"2024-12-01T17:19:18.059069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hyperparams optimization","metadata":{}},{"cell_type":"code","source":"# def objective(trial):\n#     params = {\n#         'random_state': 0,\n#         'verbosity': -1,\n#         'boosting_type': 'gbdt',\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.2),\n#         'num_leaves': trial.suggest_int('num_leaves', 30, 100),\n#         'max_depth': trial.suggest_int('max_depth', 3, 8),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 10**-3, 10**3),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 10**-3, 10**3)\n#     }\n\n#     model = LGBMRegressor(**params).fit(X_train, y_train)\n    \n#     return mean_squared_log_error(y_test, model.predict(X_test),squared= False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:18.061903Z","iopub.execute_input":"2024-12-01T17:19:18.062229Z","iopub.status.idle":"2024-12-01T17:19:18.067989Z","shell.execute_reply.started":"2024-12-01T17:19:18.062199Z","shell.execute_reply":"2024-12-01T17:19:18.066567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# study = optuna.create_study(direction='minimize')\n# study.optimize(objective, n_trials=50, n_jobs=-1, show_progress_bar=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:18.069368Z","iopub.execute_input":"2024-12-01T17:19:18.069844Z","iopub.status.idle":"2024-12-01T17:19:18.082267Z","shell.execute_reply.started":"2024-12-01T17:19:18.069766Z","shell.execute_reply":"2024-12-01T17:19:18.081101Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(study.best_params)\n# print(study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:18.083753Z","iopub.execute_input":"2024-12-01T17:19:18.084182Z","iopub.status.idle":"2024-12-01T17:19:18.099447Z","shell.execute_reply.started":"2024-12-01T17:19:18.084137Z","shell.execute_reply":"2024-12-01T17:19:18.09813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns = X_train.columns\n\nparams = { 'random_state': 0,\n        'verbosity': -1,\n        'boosting_type': 'gbdt', 'learning_rate': 0.16359124820620413, \n          'num_leaves': 69, 'max_depth': 8, 'reg_alpha': 84.41084709500603, 'reg_lambda': 270.64466611895875}\n\nmodel_final = LGBMRegressor(**params).fit(X_train, y_train)\nprint('RMSLE for Train %.3f'%mean_squared_log_error(y_train, model_final.predict(X_train),squared= False))\nprint('RMSLE for Test %.3f'%mean_squared_log_error(y_test, model_final.predict(X_test), squared= False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:18.100971Z","iopub.execute_input":"2024-12-01T17:19:18.10141Z","iopub.status.idle":"2024-12-01T17:19:38.307979Z","shell.execute_reply.started":"2024-12-01T17:19:18.101354Z","shell.execute_reply":"2024-12-01T17:19:38.306829Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv', parse_dates=['Policy Start Date'])\ntest.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:38.309303Z","iopub.execute_input":"2024-12-01T17:19:38.309613Z","iopub.status.idle":"2024-12-01T17:19:41.924769Z","shell.execute_reply.started":"2024-12-01T17:19:38.309575Z","shell.execute_reply":"2024-12-01T17:19:41.923151Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.isna().mean()*100","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:41.926172Z","iopub.execute_input":"2024-12-01T17:19:41.92655Z","iopub.status.idle":"2024-12-01T17:19:42.320254Z","shell.execute_reply.started":"2024-12-01T17:19:41.926514Z","shell.execute_reply":"2024-12-01T17:19:42.318755Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in cat_cols:\n    le = LabelEncoder()\n    test[i] = le.fit_transform(test[i])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:42.321709Z","iopub.execute_input":"2024-12-01T17:19:42.322053Z","iopub.status.idle":"2024-12-01T17:19:43.955812Z","shell.execute_reply.started":"2024-12-01T17:19:42.322022Z","shell.execute_reply":"2024-12-01T17:19:43.954501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test.fillna(means)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:43.957239Z","iopub.execute_input":"2024-12-01T17:19:43.957576Z","iopub.status.idle":"2024-12-01T17:19:44.153208Z","shell.execute_reply.started":"2024-12-01T17:19:43.957544Z","shell.execute_reply":"2024-12-01T17:19:44.151925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['policy_duration'] = (datetime.now() - test['Policy Start Date']).dt.days","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:44.154616Z","iopub.execute_input":"2024-12-01T17:19:44.155078Z","iopub.status.idle":"2024-12-01T17:19:44.189889Z","shell.execute_reply.started":"2024-12-01T17:19:44.155022Z","shell.execute_reply":"2024-12-01T17:19:44.188602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['Premium Amount'] = model_final.predict(test[columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:44.194355Z","iopub.execute_input":"2024-12-01T17:19:44.194836Z","iopub.status.idle":"2024-12-01T17:19:50.27946Z","shell.execute_reply.started":"2024-12-01T17:19:44.194801Z","shell.execute_reply":"2024-12-01T17:19:50.278152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test[['id', 'Premium Amount']].to_csv('submission.csv', index=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T17:19:50.281208Z","iopub.execute_input":"2024-12-01T17:19:50.2817Z","iopub.status.idle":"2024-12-01T17:19:52.017061Z","shell.execute_reply.started":"2024-12-01T17:19:50.281618Z","shell.execute_reply":"2024-12-01T17:19:52.015802Z"}},"outputs":[],"execution_count":null}]}