{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076},{"sourceId":10176199,"sourceType":"datasetVersion","datasetId":6278280}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import numpy as np\nimport pandas as pd\n#import matplotlib.pyplot as plt\n#import seaborn as sns\n#import scipy.stats as stats\n#import math\n#import warnings\n#import pickle\n#warnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"pg = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\npg.drop('id',axis=1,inplace=True)\npg.dropna(subset=['Premium Amount'],inplace=True)\npg\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"orig = pd.read_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv')\norig.dropna(subset=['Premium Amount'],inplace=True)\norig\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"cat_col = ['Gender','Marital Status','Education Level','Occupation','Location',\n           'Policy Type','Smoking Status','Property Type','Exercise Frequency','Customer Feedback']\ndate_col = ['Policy Start Date']\ntarget_col = ['Premium Amount']\nnum_col = list(set(orig.columns)-set(cat_col)-set(date_col)-set(target_col))\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"from sklearn.model_selection import train_test_split\ntrain,test = train_test_split(pg,test_size=0.2,shuffle=True,random_state=42)\ntrain.shape,test.shape\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"train = pd.concat([train,orig],axis=0)\ntrain.reset_index(inplace=True,drop=True)\ntrain.shape\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"train.duplicated().sum()\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"train.info()\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"(train.isna().sum()/len(train))*100\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"for col in cat_col:\n    try:\n        unique = np.unique(train[col])\n        print(f'{col} unique category: ', unique)\n    except:\n        train[col] = train[col].fillna('0')\n        unique = np.unique(train[col])\n        print(f'{col} unique category: ', unique)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"for col in cat_col:\n    test[col] = test[col].fillna('0')\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"train['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])\ntest['Policy Start Date'] = pd.to_datetime(test['Policy Start Date'])\n\ntrain['Policy Start Year'] = train['Policy Start Date'].dt.year\ntrain['Policy Start Month'] = train['Policy Start Date'].dt.month\ntrain['Policy Start Day'] = train['Policy Start Date'].dt.day\ntrain['Policy Start DayOfWeek'] = train['Policy Start Date'].dt.dayofweek\ntrain.drop('Policy Start Date',axis=1,inplace=True)\n\ntest['Policy Start Year'] = test['Policy Start Date'].dt.year\ntest['Policy Start Month'] = test['Policy Start Date'].dt.month\ntest['Policy Start Day'] = test['Policy Start Date'].dt.day\ntest['Policy Start DayOfWeek'] = test['Policy Start Date'].dt.dayofweek\ntest.drop('Policy Start Date',axis=1,inplace=True)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"num_col = num_col + ['Policy Start Year','Policy Start Month',\n                     'Policy Start Day', 'Policy Start DayOfWeek']\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"plt.figure(figsize=(20,10))\nsns.heatmap(train[num_col + ['Premium Amount']].corr(),\n            vmin=-1,\n            vmax=1,\n            center=0,\n            annot=True)\nplt.show()\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"_, ax = plt.subplots(3, 4, figsize=(30,20))\nfor i in range(len(num_col)):\n    sns.histplot(x=train[num_col[i]],kde=True,ax=ax[i//4,i%4])\n    ax[i//4,i%4].set_xlabel(num_col[i])\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"train[num_col].describe()\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"_, ax = plt.subplots(3, 4, figsize=(30,20))\nfor i in range(len(cat_col)):\n    unique,counts=np.unique(train[cat_col[i]],return_counts=True)\n    ax[i//4,i%4].pie(x=counts,labels=unique,autopct='%.0f%%')\n    ax[i//4,i%4].set_xlabel(cat_col[i])\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"_, ax = plt.subplots(3, 4, figsize=(30,20))\nfor i in range(len(cat_col)):\n    sns.boxplot(x=train[cat_col[i]],y=train['Premium Amount'],ax=ax[i//4,i%4])\n    ax[i//4,i%4].set_ylabel(cat_col[i])\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"import statsmodels.api as sm\n\nencoded = pd.get_dummies(train[cat_col],drop_first=True,dtype=float)\nencoded = sm.add_constant(encoded)\nfitted = sm.OLS(train['Premium Amount'],encoded).fit_regularized(alpha=0.1)\nimportances = fitted.params.sort_values(ascending=False)[1:]\nindices = importances.index\nsns.barplot(x=importances,y=indices,orient='h')\nplt.show()\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"dropped_candidates = ['Location','Gender','Policy Type','Exercise Frequency',\n                      'Smoking Status','Property Type']\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"x_train,y_train = train.drop('Premium Amount',axis=1), train['Premium Amount']\nx_test,y_test = test.drop('Premium Amount',axis=1), test['Premium Amount']\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"from sklearn.model_selection import StratifiedKFold\nimport optuna\nfrom lightgbm import LGBMRegressor\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.impute import SimpleImputer\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"def feature_engineering(df:pd.DataFrame):\n    df1 = df.copy()\n    time_col = ['Policy Start Month','Policy Start Day',\n                'Policy Start DayOfWeek','Policy Start Year']\n    for col in num_col:\n        if col == 'Age':\n            bins = [0, 20, 40, 60, 1000]\n            labels = [0, 1, 2, 3]\n            df1[col] = pd.cut(df1[col],bins=bins,labels=labels).astype(float)\n        elif col in time_col:\n            continue\n        else:\n            df1[f'{col}_log'] = np.log1p(df1[col])\n    \n    df1[\"Annual_Income_Health_Score_Ratio\"] = df1[\"Annual Income\"] / df1[\"Health Score\"] \n    df1[\"Annual_Income_Health_Score\"] = df1[\"Annual Income\"] * df1[\"Health Score\"]\n    \n    df1[\"Annual_Income_Credit_Score_Ratio\"] = df1[\"Annual Income\"] / df1[\"Credit Score\"]\n    df1[\"Annual_Income_Credit_Score\"] = df1[\"Annual Income\"] * df1[\"Credit Score\"]\n\n    df1[\"Vehicle_Age_Insurance_Duration\"] = df1[\"Vehicle Age\"] / df1[\"Insurance Duration\"]\n    df1['Annual_Income_Previous_Claims'] = df1['Previous Claims'] * df1['Annual Income']\n\n    df1['Policy Start Month Sin'] = np.sin(df1['Policy Start Month'] / 12 * 2 * np.pi)\n    df1['Policy Start Month Cos'] = np.cos(df1['Policy Start Month'] / 12 * 2 * np.pi)\n    df1['Policy Start Day Sin'] = np.sin(df1['Policy Start Day'] / 31 * 2 * np.pi)\n    df1['Policy Start Day Cos'] = np.cos(df1['Policy Start Day'] / 31 * 2 * np.pi)\n    df1['Policy Start DayOfWeek Sin'] = np.sin(df1['Policy Start DayOfWeek'] / 7 * 2 * np.pi)\n    df1['Policy Start DayOfWeek Cos'] = np.cos(df1['Policy Start DayOfWeek'] / 7 * 2 * np.pi)\n    \n    return df1\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"def objective(trial):\n    cv = StratifiedKFold(n_splits=5,shuffle=True,random_state=42)\n    \n    params={'max_depth':trial.suggest_int(\"max_depth\",6,20,step=1),\n            'n_estimators':trial.suggest_int(\"n_estimators\",100,900,step=100),\n            'subsample':trial.suggest_float(\"subsample\",0.5,1,step=0.1),\n            'colsample_bytree':trial.suggest_float(\"colsample_bytree\",0.5,1,step=0.1),\n            'reg_lambda':trial.suggest_float(\"reg_lambda\",0,2e-1,step=0.025),\n            'reg_alpha':trial.suggest_float(\"reg_alpha\",0,2e-1,step=0.025),\n            'num_leaves':trial.suggest_int(\"num_leaves\",12,40,step=2),\n            'learning_rate':trial.suggest_float(\"learning_rate\",0.05,0.3,step=0.05),\n            'subsample_for_bin':trial.suggest_int(\"subsample_for_bin\",\n                                                  200000,1200000,step=100),\n            'early_stopping_rounds':10,\n            'eval_metric':'rmse',\n            'random_state':42,\n            'n_jobs':-1,\n            'verbosity':-1}\n    \n    engineer_feature = trial.suggest_categorical('engineer_feature',\n                                                 [True,False])\n    drop_column = trial.suggest_categorical('drop_column',\n                                             [True,False])\n    impute = trial.suggest_categorical('impute',\n                                     [None,'mean','median'])\n    \n    onehot = OneHotEncoder(sparse_output=False,dtype=np.float64,\n                           handle_unknown='ignore',drop='first')\n    local_cat_col = cat_col\n    local_num_col = num_col\n    if drop_column:\n        local_cat_col = list(set(local_cat_col) - set(dropped_candidates))\n    if impute:\n        sequence = [('onehot',onehot, local_cat_col),\n                    ('impute',SimpleImputer(strategy=impute),local_num_col)]\n    else:\n        sequence = [('onehot',onehot, local_cat_col)]\n    \n    transformer = ColumnTransformer(sequence,\n                                    remainder='passthrough',\n                                    n_jobs=-1,\n                                    verbose_feature_names_out=False,\n                                    verbose=False)\n    transformer.set_output(transform='pandas')\n    scores = []\n    for train,val in cv.split(x_train,y_train):\n        X1,y1 = x_train.iloc[train],y_train.iloc[train]\n        X2,y2 = x_train.iloc[val],y_train.iloc[val]\n        if drop_column:\n            X1 = X1.drop(dropped_candidates,axis=1)\n            X2 = X2.drop(dropped_candidates,axis=1)\n        X1 = transformer.fit_transform(X1)\n        X2 = transformer.transform(X2)\n        if engineer_feature:\n            X1 = feature_engineering(X1)\n            X2 = feature_engineering(X2)\n        reg = LGBMRegressor(**params)\n        reg.fit(X1,y1,eval_set=[(X2,y2)])\n        pred = reg.predict(X2)\n        scores.append(math.sqrt(mean_squared_log_error(y2,pred)))\n    return sum(scores)/len(scores)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"sampler=optuna.samplers.TPESampler(seed=42)\nstudy=optuna.create_study(sampler=sampler,direction='minimize')\nstudy.optimize(objective,n_trials=50)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"print('Best RMSLE Score: ', study.best_value)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"best_params = study.best_params\nbest_params['random_state'] = 42\nbest_params['n_jobs'] = -1\nbest_params['early_stopping_rounds'] = 10\nbest_params['eval_metric'] = 'rmse'\nbest_params['verbosity'] = -1\n\nengineer_feature = best_params['engineer_feature']\ndel best_params['engineer_feature']\ndrop_column = best_params['drop_column']\ndel best_params['drop_column']\nimpute = best_params['impute']\ndel best_params['impute']\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"cat_col_copy = cat_col\n\nif drop_column:\n    cat_col_copy = list(set(cat_col_copy) - set(dropped_candidates))\n    x_train.drop(dropped_candidates,axis=1,inplace=True)\n    x_test.drop(dropped_candidates,axis=1,inplace=True)\n\nonehot = OneHotEncoder(sparse_output=False,dtype=np.float64,\n                       handle_unknown='ignore',drop='first')\nif impute:\n    sequence = [('onehot',onehot, cat_col_copy),\n                ('impute',SimpleImputer(strategy=impute),num_col)]\nelse:\n    sequence = [('onehot',onehot, cat_col_copy)]\n\ntransformer = ColumnTransformer(sequence,\n                                remainder='passthrough',\n                                n_jobs=-1,\n                                verbose_feature_names_out=False,\n                                verbose=False)\ntransformer.set_output(transform='pandas')\n    \nx_train = transformer.fit_transform(x_train)\nx_test = transformer.transform(x_test)\n\nif engineer_feature:\n    x_train = feature_engineering(x_train)\n    x_test = feature_engineering(x_test)\n\nreg = LGBMRegressor(**best_params)\nreg.fit(x_train,y_train,eval_set=[(x_test,y_test)])\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"with open('/kaggle/working/transformer.pkl','wb') as f:\n    pickle.dump(transformer,f)\n\nwith open('/kaggle/working/lgbm.pkl','wb') as f:\n    pickle.dump(reg,f)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"submission_features = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nsubmission_features.drop('id',axis=1,inplace=True)\nfor col in cat_col:\n    submission_features[col] = submission_features[col].fillna('0')\n    \nsubmission_features['Policy Start Date'] = pd.to_datetime(\n    submission_features['Policy Start Date']\n)\n\nsubmission_features['Policy Start Year'] = submission_features['Policy Start Date'].dt.year\nsubmission_features['Policy Start Month'] = submission_features['Policy Start Date'].dt.month\nsubmission_features['Policy Start Day'] = submission_features['Policy Start Date'].dt.day\nsubmission_features['Policy Start DayOfWeek'] = submission_features['Policy Start Date'].dt.dayofweek\nsubmission_features.drop('Policy Start Date',axis=1,inplace=True)\n\nif drop_column:\n    submission_features.drop(dropped_candidates,axis=1,inplace=True)\n    \nsubmission_features = transformer.transform(submission_features)\n\nif engineer_feature:\n    submission_features = feature_engineering(submission_features)\n\npred = reg.predict(submission_features)\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_xgb = pd.read_csv('/kaggle/input/kaggleplayground-s4-e12/submission_xgb.csv')['Premium Amount']\nsubmission_lgb = pd.read_csv('/kaggle/input/kaggleplayground-s4-e12/submission_lgb.csv')['Premium Amount']\n\npred = 0.3 * submission_xgb + 0.7 * submission_lgb\npred","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsubmission['Premium Amount'] = pred\nsubmission.to_csv('/kaggle/working/submission.csv',index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}