{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":446.633933,"end_time":"2024-12-23T06:12:54.351821","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-23T06:05:27.717888","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setting Up\r\nImporting essential libraries and loading data to see data overview.","metadata":{"papermill":{"duration":0.010606,"end_time":"2024-12-23T06:05:31.597115","exception":false,"start_time":"2024-12-23T06:05:31.586509","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#importing libraires\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random \nimport missingno as msno\nfrom scipy.stats import shapiro\n\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:04:52.975997Z","iopub.execute_input":"2024-12-24T03:04:52.97635Z","iopub.status.idle":"2024-12-24T03:04:54.211801Z","shell.execute_reply.started":"2024-12-24T03:04:52.976295Z","shell.execute_reply":"2024-12-24T03:04:54.210722Z"},"papermill":{"duration":4.448163,"end_time":"2024-12-23T06:05:36.056284","exception":false,"start_time":"2024-12-23T06:05:31.608121","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Loading available dataset - Train and Test**","metadata":{"papermill":{"duration":0.009353,"end_time":"2024-12-23T06:05:36.075254","exception":false,"start_time":"2024-12-23T06:05:36.065901","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#load and check test.csv\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:04:54.213078Z","iopub.execute_input":"2024-12-24T03:04:54.21351Z","iopub.status.idle":"2024-12-24T03:04:58.471107Z","shell.execute_reply.started":"2024-12-24T03:04:54.213483Z","shell.execute_reply":"2024-12-24T03:04:58.470113Z"},"papermill":{"duration":4.11864,"end_time":"2024-12-23T06:05:40.203489","exception":false,"start_time":"2024-12-23T06:05:36.084849","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#load and check train.csv\ntrain = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:04:58.472614Z","iopub.execute_input":"2024-12-24T03:04:58.472875Z","iopub.status.idle":"2024-12-24T03:05:04.941261Z","shell.execute_reply.started":"2024-12-24T03:04:58.472852Z","shell.execute_reply":"2024-12-24T03:05:04.94021Z"},"papermill":{"duration":6.221609,"end_time":"2024-12-23T06:05:46.435228","exception":false,"start_time":"2024-12-23T06:05:40.213619","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"concat train and test. to perform preprocessing: missing data and label encode","metadata":{"papermill":{"duration":0.009921,"end_time":"2024-12-23T06:05:46.45561","exception":false,"start_time":"2024-12-23T06:05:46.445689","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#concatting train and test set into \"df\" set for simple parsing and preprocessing \ndf = pd.concat([train, test])\nprint(\"train shape:\", train.shape)\nprint(\"test shape:\", test.shape)\nprint(\"df shape:\", df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:04.942647Z","iopub.execute_input":"2024-12-24T03:05:04.942993Z","iopub.status.idle":"2024-12-24T03:05:05.332959Z","shell.execute_reply.started":"2024-12-24T03:05:04.942951Z","shell.execute_reply":"2024-12-24T03:05:05.331932Z"},"papermill":{"duration":0.394316,"end_time":"2024-12-23T06:05:46.860028","exception":false,"start_time":"2024-12-23T06:05:46.465712","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#test has a missing column, check to see which \nmissing_cols = set(train.columns) - set(test.columns)\nfor col in missing_cols:\n    print(f\"{col} is missing\")","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:05.334009Z","iopub.execute_input":"2024-12-24T03:05:05.334306Z","iopub.status.idle":"2024-12-24T03:05:05.339881Z","shell.execute_reply.started":"2024-12-24T03:05:05.33428Z","shell.execute_reply":"2024-12-24T03:05:05.338743Z"},"papermill":{"duration":0.019732,"end_time":"2024-12-23T06:05:46.890477","exception":false,"start_time":"2024-12-23T06:05:46.870745","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Premium Amount in test dataset will be the predicted target, thus is missing in test set. ","metadata":{"papermill":{"duration":0.010251,"end_time":"2024-12-23T06:05:46.911275","exception":false,"start_time":"2024-12-23T06:05:46.901024","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#using df for parsing \ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:05.340955Z","iopub.execute_input":"2024-12-24T03:05:05.341426Z","iopub.status.idle":"2024-12-24T03:05:05.374983Z","shell.execute_reply.started":"2024-12-24T03:05:05.341381Z","shell.execute_reply":"2024-12-24T03:05:05.373864Z"},"papermill":{"duration":0.041817,"end_time":"2024-12-23T06:05:46.963468","exception":false,"start_time":"2024-12-23T06:05:46.921651","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import missingno as msno\nmsno.matrix(df)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:05.37608Z","iopub.execute_input":"2024-12-24T03:05:05.376484Z"},"papermill":{"duration":12.807221,"end_time":"2024-12-23T06:05:59.781276","exception":false,"start_time":"2024-12-23T06:05:46.974055","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Missing data above is likely random, except for Premium Amount. Finding out more about this","metadata":{"papermill":{"duration":0.012493,"end_time":"2024-12-23T06:05:59.806948","exception":false,"start_time":"2024-12-23T06:05:59.794455","status":"completed"},"tags":[]}},{"cell_type":"code","source":"missing_column = set(train.columns) - set(test.columns)\nfor col in missing_column:\n    print(f\"{col} is missing\")","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:18.661045Z","iopub.execute_input":"2024-12-24T03:05:18.661449Z","iopub.status.idle":"2024-12-24T03:05:18.667208Z"},"papermill":{"duration":0.023013,"end_time":"2024-12-23T06:05:59.843122","exception":false,"start_time":"2024-12-23T06:05:59.820109","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check Data Overview","metadata":{"papermill":{"duration":0.012643,"end_time":"2024-12-23T06:05:59.868676","exception":false,"start_time":"2024-12-23T06:05:59.856033","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:18.66924Z","iopub.execute_input":"2024-12-24T03:05:18.669578Z","iopub.status.idle":"2024-12-24T03:05:18.699121Z"},"papermill":{"duration":0.026503,"end_time":"2024-12-23T06:05:59.907998","exception":false,"start_time":"2024-12-23T06:05:59.881495","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.describe().round(3)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:18.700294Z","iopub.execute_input":"2024-12-24T03:05:18.700704Z","iopub.status.idle":"2024-12-24T03:05:19.961249Z","shell.execute_reply.started":"2024-12-24T03:05:18.700673Z","shell.execute_reply":"2024-12-24T03:05:19.959784Z"},"papermill":{"duration":1.28302,"end_time":"2024-12-23T06:06:01.204183","exception":false,"start_time":"2024-12-23T06:05:59.921163","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#checking if data is categorical \ncategory_col = df.select_dtypes(include = 'object').columns.tolist()\nfor col in df[category_col].columns:\n    print(f\"{col} is\", df[col].nunique())","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:19.962904Z","iopub.execute_input":"2024-12-24T03:05:19.96334Z","iopub.status.idle":"2024-12-24T03:05:22.835383Z","shell.execute_reply.started":"2024-12-24T03:05:19.963278Z","shell.execute_reply":"2024-12-24T03:05:22.834163Z"},"papermill":{"duration":2.825732,"end_time":"2024-12-23T06:06:04.043861","exception":false,"start_time":"2024-12-23T06:06:01.218129","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Most object columns are not ordial and continuous with certain few unique, thus columns type likely to be category. **Policy Start DAte** is not object, it should be datetime. The rest of object column astype to **category**.","metadata":{"papermill":{"duration":0.013771,"end_time":"2024-12-23T06:06:04.071675","exception":false,"start_time":"2024-12-23T06:06:04.057904","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#change Policy Start Date to to_datetime\ndf['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\ndf['Policy Year'] = df['Policy Start Date'].dt.year\ndf['Policy Month'] = df['Policy Start Date'].dt.month\ndf['Policy Day'] = df['Policy Start Date'].dt.day\n\ndf = df.drop(columns = ['Policy Start Date'])","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:22.836365Z","iopub.execute_input":"2024-12-24T03:05:22.83665Z","iopub.status.idle":"2024-12-24T03:05:24.251503Z","shell.execute_reply.started":"2024-12-24T03:05:22.836624Z","shell.execute_reply":"2024-12-24T03:05:24.250213Z"},"papermill":{"duration":1.334645,"end_time":"2024-12-23T06:06:05.419664","exception":false,"start_time":"2024-12-23T06:06:04.085019","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"category_col = [col for col in category_col if col !='Policy Start Date']\ndf[category_col] = df[category_col].astype('category')\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:24.252664Z","iopub.execute_input":"2024-12-24T03:05:24.252975Z","iopub.status.idle":"2024-12-24T03:05:25.761807Z","shell.execute_reply.started":"2024-12-24T03:05:24.252946Z","shell.execute_reply":"2024-12-24T03:05:25.760725Z"},"papermill":{"duration":1.53274,"end_time":"2024-12-23T06:06:06.965714","exception":false,"start_time":"2024-12-23T06:06:05.432974","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{"papermill":{"duration":0.013368,"end_time":"2024-12-23T06:06:06.992918","exception":false,"start_time":"2024-12-23T06:06:06.97955","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:25.762693Z","iopub.execute_input":"2024-12-24T03:05:25.762977Z","iopub.status.idle":"2024-12-24T03:05:25.820901Z","shell.execute_reply.started":"2024-12-24T03:05:25.762931Z","shell.execute_reply":"2024-12-24T03:05:25.819984Z"},"papermill":{"duration":0.078275,"end_time":"2024-12-23T06:06:07.084698","exception":false,"start_time":"2024-12-23T06:06:07.006423","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Categorical Columns**","metadata":{"papermill":{"duration":0.013287,"end_time":"2024-12-23T06:06:07.112558","exception":false,"start_time":"2024-12-23T06:06:07.099271","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#checking category columns value counts\nfor col in df[category_col].columns:\n    if df[col].isna().sum() > 0:\n        plt.figure(figsize = (5,5))\n        sns.countplot(data = df, x = col)\n        plt.title(f\"{col} value counts\")\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:25.822112Z","iopub.execute_input":"2024-12-24T03:05:25.822531Z","iopub.status.idle":"2024-12-24T03:05:26.599507Z","shell.execute_reply.started":"2024-12-24T03:05:25.822493Z","shell.execute_reply":"2024-12-24T03:05:26.598444Z"},"papermill":{"duration":0.736831,"end_time":"2024-12-23T06:06:07.863517","exception":false,"start_time":"2024-12-23T06:06:07.126686","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#checking sum of missing data in category col.\ncategory_missing = [col for col in category_col if df[col].isna().sum() > 0]\ndf[category_missing].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:26.600427Z","iopub.execute_input":"2024-12-24T03:05:26.600789Z","iopub.status.idle":"2024-12-24T03:05:26.633873Z","shell.execute_reply.started":"2024-12-24T03:05:26.600757Z","shell.execute_reply":"2024-12-24T03:05:26.632905Z"},"papermill":{"duration":0.052892,"end_time":"2024-12-23T06:06:07.931702","exception":false,"start_time":"2024-12-23T06:06:07.87881","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Sighting from the above count plot, there is no highest or lowest, while most of the values are almost the same we will be using ffill to fillna. ","metadata":{"papermill":{"duration":0.01474,"end_time":"2024-12-23T06:06:07.961792","exception":false,"start_time":"2024-12-23T06:06:07.947052","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#using ffill to fillna\ndf[category_missing] = df[category_missing].ffill()\ndf[category_col].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:26.634733Z","iopub.execute_input":"2024-12-24T03:05:26.635009Z","iopub.status.idle":"2024-12-24T03:05:26.692744Z","shell.execute_reply.started":"2024-12-24T03:05:26.634984Z","shell.execute_reply":"2024-12-24T03:05:26.691935Z"},"papermill":{"duration":0.071559,"end_time":"2024-12-23T06:06:08.04841","exception":false,"start_time":"2024-12-23T06:06:07.976851","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Numerical Columns**","metadata":{"papermill":{"duration":0.01497,"end_time":"2024-12-23T06:06:08.078558","exception":false,"start_time":"2024-12-23T06:06:08.063588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#since we already have category columns, now we going to include numeric data\nnumeric_col = df.select_dtypes(include = 'number')\nnumeric_col = [col for col in numeric_col if col != 'Premium Amount' and col != 'id']","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:26.694065Z","iopub.execute_input":"2024-12-24T03:05:26.694364Z","iopub.status.idle":"2024-12-24T03:05:26.853661Z","shell.execute_reply.started":"2024-12-24T03:05:26.694294Z","shell.execute_reply":"2024-12-24T03:05:26.852638Z"},"papermill":{"duration":0.184229,"end_time":"2024-12-23T06:06:08.278048","exception":false,"start_time":"2024-12-23T06:06:08.093819","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_col_df = df[numeric_col]\nnum_col_df","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:26.854698Z","iopub.execute_input":"2024-12-24T03:05:26.854983Z","iopub.status.idle":"2024-12-24T03:05:26.925181Z","shell.execute_reply.started":"2024-12-24T03:05:26.854959Z","shell.execute_reply":"2024-12-24T03:05:26.924178Z"},"papermill":{"duration":0.09507,"end_time":"2024-12-23T06:06:08.388218","exception":false,"start_time":"2024-12-23T06:06:08.293148","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Plot Numeric Columns to check how's data doing ","metadata":{"papermill":{"duration":0.015387,"end_time":"2024-12-23T06:06:08.419096","exception":false,"start_time":"2024-12-23T06:06:08.403709","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def histbox_plot(df):\n    \n    for i,col in enumerate(df.columns):\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize = (10,5))\n        sns.histplot(df[col], bins = 'auto', kde = True, ax = ax1)\n        sns.boxplot(x = df[col], ax = ax2)\n        ax1.set_title(f\" {col} Histplot\")\n        ax2.set_title(f\" {col} Boxplot\")\n\n    plt.tight_layout()\n    plt.show()\n\nhistbox_plot(num_col_df)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:05:26.926239Z","iopub.execute_input":"2024-12-24T03:05:26.926548Z","iopub.status.idle":"2024-12-24T03:07:05.606002Z","shell.execute_reply.started":"2024-12-24T03:05:26.926521Z","shell.execute_reply":"2024-12-24T03:07:05.604784Z"},"papermill":{"duration":97.300405,"end_time":"2024-12-23T06:07:45.735172","exception":false,"start_time":"2024-12-23T06:06:08.434767","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"histbox_plot(df[['Premium Amount']])","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:05.606989Z","iopub.execute_input":"2024-12-24T03:07:05.60728Z","iopub.status.idle":"2024-12-24T03:07:12.47257Z","shell.execute_reply.started":"2024-12-24T03:07:05.607255Z","shell.execute_reply":"2024-12-24T03:07:12.471254Z"},"papermill":{"duration":6.745241,"end_time":"2024-12-23T06:07:52.509345","exception":false,"start_time":"2024-12-23T06:07:45.764104","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Perform Correlation prior to cleaning**","metadata":{"papermill":{"duration":0.023417,"end_time":"2024-12-23T06:07:52.556989","exception":false,"start_time":"2024-12-23T06:07:52.533572","status":"completed"},"tags":[]}},{"cell_type":"code","source":"num_df_col = df.select_dtypes(include = ['number'])\ndf_corr = num_df_col.corr()\n\nplt.figure(figsize = (10,10))\nsns.heatmap(df_corr, \n           annot=True, fmt = '.2f',\n           linewidth = 1, cmap = 'inferno')\n","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:12.473613Z","iopub.execute_input":"2024-12-24T03:07:12.474019Z","iopub.status.idle":"2024-12-24T03:07:14.740927Z","shell.execute_reply.started":"2024-12-24T03:07:12.47398Z","shell.execute_reply":"2024-12-24T03:07:14.739759Z"},"papermill":{"duration":2.291731,"end_time":"2024-12-23T06:07:54.872325","exception":false,"start_time":"2024-12-23T06:07:52.580594","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Preprocess numeric data**","metadata":{"papermill":{"duration":0.025186,"end_time":"2024-12-23T06:07:54.923185","exception":false,"start_time":"2024-12-23T06:07:54.897999","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#Filling missing data\n\n#Age Ffill \ndf['Age'] = df['Age'].ffill()\n\n# Annual Income mode\ndf['Annual Income'] = df['Annual Income'].fillna(df['Annual Income'].mode()[0])\n\n# Number of Dependents ffill\ndf['Number of Dependents'] = df['Number of Dependents'].ffill()\n\n# Health Score mean \ndf['Health Score'] = df['Health Score'].fillna(df['Health Score'].mean())\n\n# Previous Claim mode\ndf['Previous Claims'] = df['Previous Claims'].fillna(df['Previous Claims'].mode()[0])\n\n#Credit Score Median \ndf['Credit Score'] = df['Credit Score'].fillna(df['Credit Score'].median())\n\n#Vehicle Age ffill\ndf['Vehicle Age'] = df['Vehicle Age'].ffill()\n\n#Insurance Duration ffill\ndf['Insurance Duration'] = df['Insurance Duration'].ffill()\n\ndf.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:14.745633Z","iopub.execute_input":"2024-12-24T03:07:14.745945Z","iopub.status.idle":"2024-12-24T03:07:15.053001Z","shell.execute_reply.started":"2024-12-24T03:07:14.745917Z","shell.execute_reply":"2024-12-24T03:07:15.05206Z"},"papermill":{"duration":0.35143,"end_time":"2024-12-23T06:07:55.300248","exception":false,"start_time":"2024-12-23T06:07:54.948818","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Standard sCaler or mixmaxscaler. for numeric columns. Label encoding categorical. then use all df for corr. then start to split to train test again \n1. Random Forest Reg:\n   - need binning for age and vehicle age\n   - Label Encode cat\n   - no need scaler\n  \n2. Gboost:\n   - need binning for age and vehicle\n   - -Label Encode cat\n   - need scaler","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":0.025176,"end_time":"2024-12-23T06:07:55.350824","exception":false,"start_time":"2024-12-23T06:07:55.325648","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Binning Age and Vehicle Age ","metadata":{"papermill":{"duration":0.025336,"end_time":"2024-12-23T06:07:55.401624","exception":false,"start_time":"2024-12-23T06:07:55.376288","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#Bin age and vehicle age\nage_bins = [0, 18, 30, 45, 60, 100]\nage_labels = ['0-17', '18-29', '30-44', '45-59', '60+']\n\ndf['age_binned'] = pd.cut(df['Age'], bins=age_bins, labels=age_labels, right = False)\n\nvage_bins = [0, 4, 8, 13, 20]\nvage_labels = ['0-3', '4-7', '8-12','13+'] \n\ndf['vehicle_age_binned'] = pd.cut(df['Vehicle Age'], bins=vage_bins, labels=vage_labels, right = False)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:15.054421Z","iopub.execute_input":"2024-12-24T03:07:15.054788Z","iopub.status.idle":"2024-12-24T03:07:15.168142Z","shell.execute_reply.started":"2024-12-24T03:07:15.05476Z","shell.execute_reply":"2024-12-24T03:07:15.167232Z"},"papermill":{"duration":0.144555,"end_time":"2024-12-23T06:07:55.572342","exception":false,"start_time":"2024-12-23T06:07:55.427787","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.drop(columns = ['Age', 'Vehicle Age'])","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:15.169251Z","iopub.execute_input":"2024-12-24T03:07:15.169617Z","iopub.status.idle":"2024-12-24T03:07:15.24013Z","shell.execute_reply.started":"2024-12-24T03:07:15.169589Z","shell.execute_reply":"2024-12-24T03:07:15.239285Z"},"papermill":{"duration":0.113831,"end_time":"2024-12-23T06:07:55.711774","exception":false,"start_time":"2024-12-23T06:07:55.597943","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Label Encode Categorical columns","metadata":{"papermill":{"duration":0.025403,"end_time":"2024-12-23T06:07:55.762952","exception":false,"start_time":"2024-12-23T06:07:55.737549","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#Label Encoding all cat\nfrom sklearn.preprocessing import LabelEncoder\n\ncat_col2 = df.select_dtypes(include = 'category').columns.tolist()\nlabel_encoders = {}\nfor col in cat_col2:\n    le = LabelEncoder()\n    df[col] = le.fit_transform(df[col])  \n    label_encoders[col] = le    \n\nfor col,le in label_encoders.items():\n    mapping = dict(zip(le.classes_, range(len(le.classes_))))\n    print(f\"\\n Mapping for {col}: \\n\")\n    print(mapping)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:15.24129Z","iopub.execute_input":"2024-12-24T03:07:15.241774Z","iopub.status.idle":"2024-12-24T03:07:19.402741Z","shell.execute_reply.started":"2024-12-24T03:07:15.241736Z","shell.execute_reply":"2024-12-24T03:07:19.400991Z"},"papermill":{"duration":4.45058,"end_time":"2024-12-23T06:08:00.239314","exception":false,"start_time":"2024-12-23T06:07:55.788734","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#convert le encoded columns back to category, and also number of dependents\nto_cat = ['Gender', 'Marital Status', 'Education Level', 'Occupation', 'Location', 'Policy Type', \\\n          'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type', 'age_binned', 'vehicle_age_binned', 'Number of Dependents']\ndf[to_cat] = df[to_cat].astype('category')\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:19.403789Z","iopub.execute_input":"2024-12-24T03:07:19.40411Z","iopub.status.idle":"2024-12-24T03:07:19.766677Z","shell.execute_reply.started":"2024-12-24T03:07:19.40408Z","shell.execute_reply":"2024-12-24T03:07:19.765493Z"},"papermill":{"duration":0.39194,"end_time":"2024-12-23T06:08:00.656755","exception":false,"start_time":"2024-12-23T06:08:00.264815","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Correlation parsing after label encoding","metadata":{"papermill":{"duration":0.025686,"end_time":"2024-12-23T06:08:00.708195","exception":false,"start_time":"2024-12-23T06:08:00.682509","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_corr2 = df.corr()\nplt.figure(figsize=(10,10))\nsns.heatmap(df_corr2, \n           annot = True, fmt = '.2f',\n           annot_kws = {\"fontsize\":6},\n           linewidth = 1, cmap = \"coolwarm\")","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:19.767647Z","iopub.execute_input":"2024-12-24T03:07:19.767935Z","iopub.status.idle":"2024-12-24T03:07:24.699743Z","shell.execute_reply.started":"2024-12-24T03:07:19.767911Z","shell.execute_reply":"2024-12-24T03:07:24.698534Z"},"papermill":{"duration":4.958111,"end_time":"2024-12-23T06:08:05.692181","exception":false,"start_time":"2024-12-23T06:08:00.73407","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Still no correlation between variables. \n\n**Splitting df into train and test again.**","metadata":{"papermill":{"duration":0.028451,"end_time":"2024-12-23T06:08:05.749252","exception":false,"start_time":"2024-12-23T06:08:05.720801","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#splitting train and test set based on na in premium amount \ntrain_df = df[df['Premium Amount'].notna()]\ntest_df = df[df['Premium Amount'].isna()]\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:24.700833Z","iopub.execute_input":"2024-12-24T03:07:24.701226Z","iopub.status.idle":"2024-12-24T03:07:24.885751Z","shell.execute_reply.started":"2024-12-24T03:07:24.701181Z","shell.execute_reply":"2024-12-24T03:07:24.884593Z"},"papermill":{"duration":0.223452,"end_time":"2024-12-23T06:08:06.001055","exception":false,"start_time":"2024-12-23T06:08:05.777603","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:24.886806Z","iopub.execute_input":"2024-12-24T03:07:24.887095Z","iopub.status.idle":"2024-12-24T03:07:24.91503Z","shell.execute_reply.started":"2024-12-24T03:07:24.887069Z","shell.execute_reply":"2024-12-24T03:07:24.913913Z"},"papermill":{"duration":0.059813,"end_time":"2024-12-23T06:08:06.08925","exception":false,"start_time":"2024-12-23T06:08:06.029437","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_df:\", train_df.shape, \"test_df:\", test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-12-24T03:07:24.916084Z","iopub.execute_input":"2024-12-24T03:07:24.916465Z","iopub.status.idle":"2024-12-24T03:07:24.936606Z","shell.execute_reply.started":"2024-12-24T03:07:24.916423Z","shell.execute_reply":"2024-12-24T03:07:24.935434Z"},"papermill":{"duration":0.039184,"end_time":"2024-12-23T06:08:06.157245","exception":false,"start_time":"2024-12-23T06:08:06.118061","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T03:07:24.937634Z","iopub.execute_input":"2024-12-24T03:07:24.937953Z","iopub.status.idle":"2024-12-24T03:07:24.9753Z","shell.execute_reply.started":"2024-12-24T03:07:24.937911Z","shell.execute_reply":"2024-12-24T03:07:24.974281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T03:08:02.154489Z","iopub.execute_input":"2024-12-24T03:08:02.154842Z","iopub.status.idle":"2024-12-24T03:08:02.186686Z","shell.execute_reply.started":"2024-12-24T03:08:02.154812Z","shell.execute_reply":"2024-12-24T03:08:02.185462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.to_csv(\"train_df.csv\")\ntest_df.to_csv(\"test_df.csv\")","metadata":{"papermill":{"duration":0.029594,"end_time":"2024-12-23T06:12:51.601019","exception":false,"start_time":"2024-12-23T06:12:51.571425","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T03:08:23.580026Z","iopub.execute_input":"2024-12-24T03:08:23.580418Z","iopub.status.idle":"2024-12-24T03:08:40.14925Z","shell.execute_reply.started":"2024-12-24T03:08:23.580384Z","shell.execute_reply":"2024-12-24T03:08:40.148199Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Conclusion \n\nPrior to model development, the above data is cleaned. \n\n1. Unique without ordinal columns have been converted to categorical data type\n2. Numeric and Category data's missing data filled\n3. Label Encoded for Category data\n4. Data column converted to pd.Datetime and split into day, month and year.\n5. Age and Vehicle age are binned","metadata":{}}]}