{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport missingno as msno\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import PowerTransformer, OneHotEncoder, StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.metrics import (\n    mean_absolute_error, \n    mean_squared_error, \n    r2_score, \n    mean_squared_log_error\n)\n\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import KFold\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:08:33.189875Z","iopub.execute_input":"2024-12-06T06:08:33.190244Z","iopub.status.idle":"2024-12-06T06:08:35.163189Z","shell.execute_reply.started":"2024-12-06T06:08:33.190207Z","shell.execute_reply":"2024-12-06T06:08:35.162155Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsub = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:57:10.175899Z","iopub.execute_input":"2024-12-06T13:57:10.176323Z","iopub.status.idle":"2024-12-06T13:57:18.139603Z","shell.execute_reply.started":"2024-12-06T13:57:10.176269Z","shell.execute_reply":"2024-12-06T13:57:18.138535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploring Dataset","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:57:18.141495Z","iopub.execute_input":"2024-12-06T13:57:18.1418Z","iopub.status.idle":"2024-12-06T13:57:18.165397Z","shell.execute_reply.started":"2024-12-06T13:57:18.141769Z","shell.execute_reply":"2024-12-06T13:57:18.164362Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exploring missing values","metadata":{}},{"cell_type":"code","source":"missing_summary = train.isnull().sum().sort_values(ascending=False)\nmissing_percentage = (missing_summary / len(train)) * 100\nmissing_data = pd.DataFrame({'Missing Values': missing_summary, \n                             'Percentage (%)': missing_percentage})\nmissing_data[missing_data['Missing Values'] > 0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:57:18.16647Z","iopub.execute_input":"2024-12-06T13:57:18.166831Z","iopub.status.idle":"2024-12-06T13:57:18.784542Z","shell.execute_reply.started":"2024-12-06T13:57:18.166799Z","shell.execute_reply":"2024-12-06T13:57:18.783377Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"msno.matrix(train)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:57:18.78639Z","iopub.execute_input":"2024-12-06T13:57:18.786704Z","iopub.status.idle":"2024-12-06T13:57:25.988965Z","shell.execute_reply.started":"2024-12-06T13:57:18.786674Z","shell.execute_reply":"2024-12-06T13:57:25.987975Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analysing numerical features using histograms, box plots, and density plots","metadata":{}},{"cell_type":"code","source":"numerical_features = [\n    'Age', 'Annual Income', 'Health Score', 'Previous Claims', \n    'Vehicle Age', 'Credit Score', 'Insurance Duration', 'Premium Amount',  'Number of Dependents'\n]\n\nplt.figure(figsize=(20, 20))\nfor i, feature in enumerate(numerical_features):\n    plt.subplot(len(numerical_features), 3, i * 3 + 1)\n    sns.histplot(train[feature], kde=True, bins=30, color='skyblue')\n    plt.title(f'Histogram of {feature}')\n    \n    plt.subplot(len(numerical_features), 3, i * 3 + 2)\n    sns.boxplot(x=train[feature], color='salmon')\n    plt.title(f'Box Plot of {feature}')\n    \n    plt.subplot(len(numerical_features), 3, i * 3 + 3)\n    sns.kdeplot(train[feature], fill=True, color='green')\n    plt.title(f'Density Plot of {feature}')\n    \nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:57:25.991233Z","iopub.execute_input":"2024-12-06T13:57:25.991715Z","iopub.status.idle":"2024-12-06T13:58:55.363999Z","shell.execute_reply.started":"2024-12-06T13:57:25.991669Z","shell.execute_reply":"2024-12-06T13:58:55.363034Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analysing distribution in categorical features using bar plots","metadata":{}},{"cell_type":"code","source":"categorical_features = ['Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location', \n                        'Policy Type', 'Smoking Status', 'Exercise Frequency', 'Property Type']\n\nplt.figure(figsize=(20, 15))\nfor i, feature in enumerate(categorical_features):\n    plt.subplot((len(categorical_features) + 2) // 3, 3, i + 1)\n    sns.countplot(y=train[feature], order=train[feature].value_counts().index, palette='pastel')\n    plt.title(f'Frequency of {feature}')\n    plt.xlabel('Count')\n    plt.ylabel(feature)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:58:55.365272Z","iopub.execute_input":"2024-12-06T13:58:55.365706Z","iopub.status.idle":"2024-12-06T13:59:01.156554Z","shell.execute_reply.started":"2024-12-06T13:58:55.365657Z","shell.execute_reply":"2024-12-06T13:59:01.155535Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exploring Correlation","metadata":{}},{"cell_type":"code","source":"corr_matrix = train[['Premium Amount', 'Annual Income', 'Credit Score', 'Vehicle Age', 'Health Score']].corr()\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt=\".2f\")\nplt.title('Correlation Heatmap')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:59:01.158932Z","iopub.execute_input":"2024-12-06T13:59:01.159784Z","iopub.status.idle":"2024-12-06T13:59:01.537467Z","shell.execute_reply.started":"2024-12-06T13:59:01.159736Z","shell.execute_reply":"2024-12-06T13:59:01.536457Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"g = sns.FacetGrid(train, col='Marital Status', hue='Property Type')\ng.map(sns.scatterplot, 'Annual Income', 'Premium Amount')\ng.add_legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:59:01.538615Z","iopub.execute_input":"2024-12-06T13:59:01.538885Z","iopub.status.idle":"2024-12-06T13:59:10.933545Z","shell.execute_reply.started":"2024-12-06T13:59:01.538858Z","shell.execute_reply":"2024-12-06T13:59:10.932443Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.boxplot(train, x='Education Level', y='Premium Amount', hue='Marital Status')\nplt.title('Premium Amount by Education Level and Marital Status')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:59:10.934956Z","iopub.execute_input":"2024-12-06T13:59:10.935373Z","iopub.status.idle":"2024-12-06T13:59:12.838734Z","shell.execute_reply.started":"2024-12-06T13:59:10.935329Z","shell.execute_reply":"2024-12-06T13:59:12.837738Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.histplot(train, x='Premium Amount', hue='Property Type', multiple='stack')\nplt.title('Premium Amount Distribution by Property Type')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:59:12.840048Z","iopub.execute_input":"2024-12-06T13:59:12.840485Z","iopub.status.idle":"2024-12-06T13:59:15.98674Z","shell.execute_reply.started":"2024-12-06T13:59:12.840439Z","shell.execute_reply":"2024-12-06T13:59:15.985682Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Finding multicollinearity among numerical features","metadata":{}},{"cell_type":"code","source":"numerical_features = [\n    'Age', 'Annual Income', 'Health Score', 'Previous Claims', \n    'Vehicle Age', 'Credit Score', 'Insurance Duration', 'Premium Amount',  'Number of Dependents'\n]\n\ntemp_df= train[numerical_features].dropna()\nvif_data = pd.DataFrame()\nvif_data['Feature'] = temp_df.columns\nvif_data['VIF'] = [variance_inflation_factor(temp_df.values, i) for i in range(temp_df.shape[1])]\nprint(vif_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T14:01:36.381859Z","iopub.execute_input":"2024-12-06T14:01:36.382578Z","iopub.status.idle":"2024-12-06T14:01:39.736032Z","shell.execute_reply.started":"2024-12-06T14:01:36.38254Z","shell.execute_reply":"2024-12-06T14:01:39.732873Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Processing","metadata":{}},{"cell_type":"markdown","source":"## Imputing missing values","metadata":{}},{"cell_type":"code","source":"numerical_cols = ['Previous Claims', 'Health Score', 'Credit Score', 'Annual Income', \n                  'Age', 'Vehicle Age', 'Insurance Duration', 'Number of Dependents']\n\ncategorical_cols = ['Marital Status', 'Customer Feedback', 'Occupation']\n\ndef impute_values(df, numerical_cols, categorical_cols):\n    num_imputer = SimpleImputer(strategy='median') \n    cat_imputer = SimpleImputer(strategy='constant', fill_value='Unknown') \n\n    df[numerical_cols] = num_imputer.fit_transform(df[numerical_cols])\n    df[categorical_cols] = cat_imputer.fit_transform(df[categorical_cols])\n\n    return df\n\ntrain = impute_values(train, numerical_cols, categorical_cols)\ntest = impute_values(test, numerical_cols, categorical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:10.279145Z","iopub.execute_input":"2024-12-06T06:09:10.279551Z","iopub.status.idle":"2024-12-06T06:09:13.429375Z","shell.execute_reply.started":"2024-12-06T06:09:10.279514Z","shell.execute_reply":"2024-12-06T06:09:13.428276Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def date(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Month_name'] = df['Policy Start Date'].dt.month_name()\n    df['Day_of_week'] = df['Policy Start Date'].dt.day_name()\n    df['Week'] = df['Policy Start Date'].dt.isocalendar().week\n    df['Year_sin'] = np.sin(2 * np.pi * df['Year'])\n    df['Year_cos'] = np.cos(2 * np.pi * df['Year'])\n    min_year = df['Year'].min()\n    max_year = df['Year'].max()\n    df['Year_sin'] = np.sin(2 * np.pi * (df['Year'] - min_year) / (max_year - min_year))\n    df['Year_cos'] = np.cos(2 * np.pi * (df['Year'] - min_year) / (max_year - min_year))\n    df['Month_sin'] = np.sin(2 * np.pi * df['Month'] / 12) \n    df['Month_cos'] = np.cos(2 * np.pi * df['Month'] / 12)\n    df['Day_sin'] = np.sin(2 * np.pi * df['Day'] / 31)  \n    df['Day_cos'] = np.cos(2 * np.pi * df['Day'] / 31)\n    df['Group']=(df['Year']-2020)*48+df['Month']*4+df['Day']//7\n    \n    df.drop('Policy Start Date', axis=1, inplace=True)\n\n    return df\n\ntrain = date(train)\ntest = date(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:16.338024Z","iopub.execute_input":"2024-12-06T06:09:16.338567Z","iopub.status.idle":"2024-12-06T06:09:19.458559Z","shell.execute_reply.started":"2024-12-06T06:09:16.338526Z","shell.execute_reply":"2024-12-06T06:09:19.457484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Encoding categorical values","metadata":{}},{"cell_type":"code","source":"categorical_cols = ['Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location', \n                    'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type', 'Month_name', 'Day_of_week']\n\ndef encode_values(df, categorical_cols):\n    encoder = OneHotEncoder(drop='first', sparse_output=False)\n    encoded_data = encoder.fit_transform(df[categorical_cols])\n    encoded_df = pd.DataFrame(encoded_data, columns=encoder.get_feature_names_out(categorical_cols), index=df.index)\n    \n    df = df.drop(columns=categorical_cols)\n    df = pd.concat([df, encoded_df], axis=1)\n    return df\n\ntrain = encode_values(train, categorical_cols)\ntest = encode_values(test, categorical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:23.136185Z","iopub.execute_input":"2024-12-06T06:09:23.136674Z","iopub.status.idle":"2024-12-06T06:09:33.02467Z","shell.execute_reply.started":"2024-12-06T06:09:23.13663Z","shell.execute_reply":"2024-12-06T06:09:33.023589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_summary = train.isnull().sum().sort_values(ascending=False)\nmissing_percentage = (missing_summary / len(train)) * 100\n\nmissing_data = pd.DataFrame({'Missing Values': missing_summary, \n                             'Percentage (%)': missing_percentage})\nprint(missing_data[missing_data['Missing Values'] > 0])\nprint(\"-\"*70)\nmissing_summary = test.isnull().sum().sort_values(ascending=False)\nmissing_percentage = (missing_summary / len(train)) * 100\n\nmissing_data = pd.DataFrame({'Missing Values': missing_summary, \n                             'Percentage (%)': missing_percentage})\nprint(missing_data[missing_data['Missing Values'] > 0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:36.365474Z","iopub.execute_input":"2024-12-06T06:09:36.365852Z","iopub.status.idle":"2024-12-06T06:09:36.536126Z","shell.execute_reply.started":"2024-12-06T06:09:36.365818Z","shell.execute_reply":"2024-12-06T06:09:36.535105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Training (LightGBM + Optuna)","metadata":{}},{"cell_type":"code","source":"X = train.drop(columns = ['id', 'Premium Amount'])\ny = train['Premium Amount']\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:40.20741Z","iopub.execute_input":"2024-12-06T06:09:40.208166Z","iopub.status.idle":"2024-12-06T06:09:41.115072Z","shell.execute_reply.started":"2024-12-06T06:09:40.208126Z","shell.execute_reply":"2024-12-06T06:09:41.114128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 5\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\ndef root_mean_squared_log_error(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\ndef objective(trial):\n    params = {\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"dart\"]),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 200, 512),\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 1e-4, 1e-1),\n        \"feature_fraction\": trial.suggest_uniform(\"feature_fraction\", 0.6, 1.0),\n        \"bagging_fraction\": trial.suggest_uniform(\"bagging_fraction\", 0.6, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 5, 12),\n        \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 20, 100),\n        \"max_depth\": trial.suggest_int(\"max_depth\", -1, 16),\n        \"lambda_l1\": trial.suggest_loguniform(\"lambda_l1\", 1e-4, 10.0),\n        \"lambda_l2\": trial.suggest_loguniform(\"lambda_l2\", 1e-4, 10.0),\n        \"feature_pre_filter\": False,\n       # \"device_type\": \"gpu\",\n        \"seed\": 42\n    }\n\n    rmsle_scores = []\n\n    for train_idx, val_idx in kf.split(X, y):\n        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        train_data = lgb.Dataset(X_train, label=y_train)\n        val_data = lgb.Dataset(X_val, label=y_val, reference=train_data)\n\n        model = lgb.train(params, train_data,\n                          num_boost_round=1000,\n                          valid_sets=[val_data],\n                          callbacks=[lgb.early_stopping(stopping_rounds=10)])\n\n        y_pred = model.predict(X_val, num_iteration=model.best_iteration)\n        rmsle = root_mean_squared_log_error(y_val, np.maximum(y_pred, 0))\n        rmsle_scores.append(rmsle)\n\n    return np.mean(rmsle_scores)\n\nstudy = optuna.create_study(direction=\"minimize\", sampler=optuna.samplers.TPESampler())\nstudy.optimize(objective, n_trials=10)\n\nprint(\"Best params:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T06:09:56.365646Z","iopub.execute_input":"2024-12-06T06:09:56.366023Z","iopub.status.idle":"2024-12-06T13:17:20.131205Z","shell.execute_reply.started":"2024-12-06T06:09:56.365985Z","shell.execute_reply":"2024-12-06T13:17:20.130247Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training Model with with best","metadata":{}},{"cell_type":"code","source":"best_params = study.best_params\n\ntrain_data = lgb.Dataset(X_train, label=y_train)\nval_data = lgb.Dataset(X_val, label=y_val, reference=train_data)\n\nmodel = lgb.train(best_params, train_data,  \n                valid_sets=[val_data],\n                callbacks=[lgb.early_stopping(stopping_rounds=10)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:18:15.148014Z","iopub.execute_input":"2024-12-06T13:18:15.148439Z","iopub.status.idle":"2024-12-06T13:19:03.134239Z","shell.execute_reply.started":"2024-12-06T13:18:15.148394Z","shell.execute_reply":"2024-12-06T13:19:03.133168Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_id = test.copy()\ntest.drop('id', axis=1, inplace=True, errors='ignore')\ntest_predictions = model.predict(test, num_iteration=model.best_iteration)\nsubmission = pd.DataFrame({'id': test_id['id'], 'Premium Amount': test_predictions})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission saved to 'submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:52:41.124921Z","iopub.execute_input":"2024-12-06T13:52:41.125374Z","iopub.status.idle":"2024-12-06T13:52:41.130169Z","shell.execute_reply.started":"2024-12-06T13:52:41.125336Z","shell.execute_reply":"2024-12-06T13:52:41.129145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T13:50:07.583339Z","iopub.execute_input":"2024-12-06T13:50:07.583756Z","iopub.status.idle":"2024-12-06T13:50:07.595426Z","shell.execute_reply.started":"2024-12-06T13:50:07.583714Z","shell.execute_reply":"2024-12-06T13:50:07.594389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}