{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":10070859,"sourceType":"datasetVersion","datasetId":6207291}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Clean Dataset Creation Notebook","metadata":{}},{"cell_type":"markdown","source":"Aim of this notebook is to create the best quality data that will be further used to train best models. I will try to come up with different solutions to each type of null data.","metadata":{}},{"cell_type":"markdown","source":"You may want to double check the last part. Previous Claims and Credit Score seems to have a relation inbetween, but there are few outlier values in the Previous Claims that may require excluding.","metadata":{}},{"cell_type":"markdown","source":"**WARNING:** Also you may want to keep the columns which I directly dropped while generating the database. May be of use later.","metadata":{}},{"cell_type":"markdown","source":"# Import Libraries\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom sklearn.metrics import make_scorer, mean_squared_log_error  # Import mean_squared_log_error\nimport xgboost as xgb\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import plot_importance","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:08.329661Z","iopub.execute_input":"2024-12-01T21:08:08.330464Z","iopub.status.idle":"2024-12-01T21:08:08.335196Z","shell.execute_reply.started":"2024-12-01T21:08:08.330431Z","shell.execute_reply":"2024-12-01T21:08:08.334227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Upload ","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:08.336714Z","iopub.execute_input":"2024-12-01T21:08:08.337064Z","iopub.status.idle":"2024-12-01T21:08:13.703527Z","shell.execute_reply.started":"2024-12-01T21:08:08.337028Z","shell.execute_reply":"2024-12-01T21:08:13.702812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Analysis","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:13.704819Z","iopub.execute_input":"2024-12-01T21:08:13.705112Z","iopub.status.idle":"2024-12-01T21:08:14.314175Z","shell.execute_reply.started":"2024-12-01T21:08:13.705086Z","shell.execute_reply":"2024-12-01T21:08:14.313182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:14.315184Z","iopub.execute_input":"2024-12-01T21:08:14.315458Z","iopub.status.idle":"2024-12-01T21:08:14.851315Z","shell.execute_reply.started":"2024-12-01T21:08:14.315432Z","shell.execute_reply":"2024-12-01T21:08:14.85044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:14.853546Z","iopub.execute_input":"2024-12-01T21:08:14.854019Z","iopub.status.idle":"2024-12-01T21:08:15.385921Z","shell.execute_reply.started":"2024-12-01T21:08:14.853978Z","shell.execute_reply":"2024-12-01T21:08:15.385044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter for numeric columns only\nnumeric_columns = df_train.select_dtypes(include=[np.number])\n\n# Calculate the correlation matrix\ncorrelation_matrix = numeric_columns.corr()\n\n# Plot the heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\nplt.title(\"Correlation Matrix\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:15.386961Z","iopub.execute_input":"2024-12-01T21:08:15.38727Z","iopub.status.idle":"2024-12-01T21:08:16.478429Z","shell.execute_reply.started":"2024-12-01T21:08:15.387243Z","shell.execute_reply":"2024-12-01T21:08:16.477589Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"train = df_train.copy()\ntest = df_test.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:16.479394Z","iopub.execute_input":"2024-12-01T21:08:16.479632Z","iopub.status.idle":"2024-12-01T21:08:16.718767Z","shell.execute_reply.started":"2024-12-01T21:08:16.479608Z","shell.execute_reply":"2024-12-01T21:08:16.717916Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**FILLING NULL VALUES**","metadata":{}},{"cell_type":"markdown","source":"## Vehince Age","metadata":{}},{"cell_type":"code","source":"# Fill missing values in 'Vehicle Age'\nvehicle_median = train['Vehicle Age'].median()\ntrain['Vehicle Age'] = train['Vehicle Age'].fillna(vehicle_median)\ntest['Vehicle Age'] = test['Vehicle Age'].fillna(vehicle_median)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:16.71977Z","iopub.execute_input":"2024-12-01T21:08:16.720086Z","iopub.status.idle":"2024-12-01T21:08:16.756865Z","shell.execute_reply.started":"2024-12-01T21:08:16.720059Z","shell.execute_reply":"2024-12-01T21:08:16.755941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Insurance Duration","metadata":{}},{"cell_type":"code","source":"duration_median = train['Insurance Duration'].median()\ntrain['Insurance Duration'] = train['Insurance Duration'].fillna(duration_median)\ntest['Insurance Duration'] = test['Insurance Duration'].fillna(duration_median)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:16.758054Z","iopub.execute_input":"2024-12-01T21:08:16.758416Z","iopub.status.idle":"2024-12-01T21:08:16.794487Z","shell.execute_reply.started":"2024-12-01T21:08:16.758379Z","shell.execute_reply":"2024-12-01T21:08:16.793566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Customer Feedback\nSeemed unimportant :) dropping the column.","metadata":{}},{"cell_type":"code","source":"# Print unique values and their counts in 'Customer Feedback'\nfeedback_counts = train['Customer Feedback'].value_counts(dropna=False)\n\nprint(\"Unique values and their counts in 'Customer Feedback':\")\nprint(feedback_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:16.795673Z","iopub.execute_input":"2024-12-01T21:08:16.796316Z","iopub.status.idle":"2024-12-01T21:08:16.839299Z","shell.execute_reply.started":"2024-12-01T21:08:16.796277Z","shell.execute_reply":"2024-12-01T21:08:16.838467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'Customer Feedback' column from train and test datasets\ntrain.drop('Customer Feedback', axis=1, inplace=True)\ntest.drop('Customer Feedback', axis=1, inplace=True)\n\n# Verify that the column has been removed\nprint(\"Columns in train dataset after dropping 'Customer Feedback':\")\nprint(train.columns)\n\nprint(\"\\nColumns in test dataset after dropping 'Customer Feedback':\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:16.841858Z","iopub.execute_input":"2024-12-01T21:08:16.842109Z","iopub.status.idle":"2024-12-01T21:08:17.141726Z","shell.execute_reply.started":"2024-12-01T21:08:16.842085Z","shell.execute_reply":"2024-12-01T21:08:17.140829Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Annual Income","metadata":{}},{"cell_type":"code","source":"# Temporarily map 'Education Level' to numerical values for correlation analysis\neducation_mapping = {level: idx for idx, level in enumerate(train['Education Level'].unique())}\n\n# Map the values in both train and test datasets\ntrain['Education Level Temp'] = train['Education Level'].map(education_mapping)\ntest['Education Level Temp'] = test['Education Level'].map(education_mapping)\n\n# Calculate the correlation of 'Annual Income' with 'Premium Amount', 'Credit Score', 'Age', and 'Education Level Temp'\ncorrelations = train[['Annual Income', 'Premium Amount', 'Credit Score', 'Age', 'Education Level Temp']].corr()\n\n# Display the correlation matrix\nprint(\"Correlation matrix:\")\nprint(correlations)\n\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(correlations, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\nplt.title(\"Correlation of Annual Income with Premium Amount, Credit Score, Age, and Education Level (Temp)\")\nplt.show()\n\n# Drop the temporary column after analysis\ntrain.drop('Education Level Temp', axis=1, inplace=True)\ntest.drop('Education Level Temp', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:17.143077Z","iopub.execute_input":"2024-12-01T21:08:17.143684Z","iopub.status.idle":"2024-12-01T21:08:17.956025Z","shell.execute_reply.started":"2024-12-01T21:08:17.143642Z","shell.execute_reply":"2024-12-01T21:08:17.955057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of 'Annual Income'\nplt.figure(figsize=(10, 6))\nsns.histplot(train['Annual Income'], kde=True, bins=50, color='blue')\n\nplt.title(\"Distribution of Annual Income\")\nplt.xlabel(\"Annual Income\")\nplt.ylabel(\"Frequency\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:17.957297Z","iopub.execute_input":"2024-12-01T21:08:17.957653Z","iopub.status.idle":"2024-12-01T21:08:22.731515Z","shell.execute_reply.started":"2024-12-01T21:08:17.957615Z","shell.execute_reply":"2024-12-01T21:08:22.730511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check the skewness of 'Annual Income'\nincome_skewness = train['Annual Income'].skew()\nprint(f\"Skewness of 'Annual Income': {income_skewness}\")\n\n# Decide on mean or median based on skewness\nif abs(income_skewness) > 1:  # Heavily skewed\n    income_fill = train['Annual Income'].median()\n    print(\"Using median to fill missing values in 'Annual Income'.\")\nelse:  # Less skewed\n    income_fill = train['Annual Income'].mean()\n    print(\"Using mean to fill missing values in 'Annual Income'.\")\n\n# Fill missing values in 'Annual Income' without inplace=True\ntrain['Annual Income'] = train['Annual Income'].fillna(income_fill)\ntest['Annual Income'] = test['Annual Income'].fillna(income_fill)\n\n# Verify that there are no missing values left\nprint(f\"Null values in 'Annual Income' after imputation (Train): {train['Annual Income'].isnull().sum()}\")\nprint(f\"Null values in 'Annual Income' after imputation (Test): {test['Annual Income'].isnull().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:22.732915Z","iopub.execute_input":"2024-12-01T21:08:22.733714Z","iopub.status.idle":"2024-12-01T21:08:22.792869Z","shell.execute_reply.started":"2024-12-01T21:08:22.733667Z","shell.execute_reply":"2024-12-01T21:08:22.791874Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Let's Check Again","metadata":{}},{"cell_type":"code","source":"# Check for null values in all columns of train and test datasets\nprint(\"Null values in each column (Train):\")\nprint(train.isnull().sum())\n\nprint(\"\\nNull values in each column (Test):\")\nprint(test.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:22.794211Z","iopub.execute_input":"2024-12-01T21:08:22.794643Z","iopub.status.idle":"2024-12-01T21:08:23.668306Z","shell.execute_reply.started":"2024-12-01T21:08:22.794598Z","shell.execute_reply":"2024-12-01T21:08:23.667467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Age\nTrying to be somewhat creative. :(","metadata":{}},{"cell_type":"code","source":"# Boxplot of 'Exercise Frequency' vs 'Age'\nplt.figure(figsize=(10, 6))\nsns.boxplot(x='Exercise Frequency', y='Age', data=train, order=['Rarely', 'Monthly', 'Weekly', 'Daily'])\n\nplt.title(\"Age Distribution across Exercise Frequency Categories\")\nplt.xlabel(\"Exercise Frequency\")\nplt.ylabel(\"Age\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:23.669336Z","iopub.execute_input":"2024-12-01T21:08:23.669627Z","iopub.status.idle":"2024-12-01T21:08:24.079631Z","shell.execute_reply.started":"2024-12-01T21:08:23.669599Z","shell.execute_reply":"2024-12-01T21:08:24.078657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'Location' and calculate the mean of 'Age'\nlocation_age_mean = train.groupby('Location')['Age'].mean().sort_values()\n\n# Plot the average age by location\nplt.figure(figsize=(12, 6))\nlocation_age_mean.plot(kind='bar', color='skyblue', edgecolor='black')\n\nplt.title(\"Average Age by Location\")\nplt.xlabel(\"Location\")\nplt.ylabel(\"Average Age\")\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:24.080923Z","iopub.execute_input":"2024-12-01T21:08:24.081386Z","iopub.status.idle":"2024-12-01T21:08:24.389898Z","shell.execute_reply.started":"2024-12-01T21:08:24.081334Z","shell.execute_reply":"2024-12-01T21:08:24.38897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check skewness of 'Age' to decide between mean or median\nage_skewness = train['Age'].skew()\nprint(f\"Skewness of 'Age': {age_skewness}\")\n\n# Use median for imputation if distribution is skewed, otherwise use mean\nif abs(age_skewness) > 1:  # Heavily skewed\n    age_fill = train['Age'].median()\n    print(\"Using median to fill missing values in 'Age'.\")\nelse:  # Less skewed\n    age_fill = train['Age'].mean()\n    print(\"Using mean to fill missing values in 'Age'.\")\n\n# Impute missing values in 'Age'\ntrain['Age'] = train['Age'].fillna(age_fill)\ntest['Age'] = test['Age'].fillna(age_fill)\n\n# Verify imputation\nprint(f\"Null values in 'Age' after imputation (Train): {train['Age'].isnull().sum()}\")\nprint(f\"Null values in 'Age' after imputation (Test): {test['Age'].isnull().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:24.390974Z","iopub.execute_input":"2024-12-01T21:08:24.391242Z","iopub.status.idle":"2024-12-01T21:08:24.426466Z","shell.execute_reply.started":"2024-12-01T21:08:24.391216Z","shell.execute_reply":"2024-12-01T21:08:24.425619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Martial Status","metadata":{}},{"cell_type":"code","source":"# Group by 'Marital Status' and compute the mean of 'Age', 'Annual Income', and 'Education Level'\nmarital_status_summary = train.groupby('Marital Status')[['Age', 'Annual Income']].mean()\nprint(\"Mean values grouped by Marital Status:\")\nprint(marital_status_summary)\n\n# Check distribution of 'Education Level' across 'Marital Status'\neducation_summary = train.groupby('Marital Status')['Education Level'].value_counts(normalize=True)\nprint(\"\\nEducation Level distribution by Marital Status:\")\nprint(education_summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:24.427595Z","iopub.execute_input":"2024-12-01T21:08:24.427909Z","iopub.status.idle":"2024-12-01T21:08:24.701072Z","shell.execute_reply.started":"2024-12-01T21:08:24.427882Z","shell.execute_reply":"2024-12-01T21:08:24.700164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.boxplot(x='Marital Status', y='Age', data=train, order=train['Marital Status'].value_counts().index)\n\nplt.title(\"Age Distribution across Marital Status\")\nplt.xlabel(\"Marital Status\")\nplt.ylabel(\"Age\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:24.702018Z","iopub.execute_input":"2024-12-01T21:08:24.70226Z","iopub.status.idle":"2024-12-01T21:08:25.274578Z","shell.execute_reply.started":"2024-12-01T21:08:24.702236Z","shell.execute_reply":"2024-12-01T21:08:25.273687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.boxplot(x='Marital Status', y='Annual Income', data=train, order=train['Marital Status'].value_counts().index)\n\nplt.title(\"Annual Income Distribution across Marital Status\")\nplt.xlabel(\"Marital Status\")\nplt.ylabel(\"Annual Income\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:25.275497Z","iopub.execute_input":"2024-12-01T21:08:25.275727Z","iopub.status.idle":"2024-12-01T21:08:25.978636Z","shell.execute_reply.started":"2024-12-01T21:08:25.275703Z","shell.execute_reply":"2024-12-01T21:08:25.977703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Impute missing values in 'Marital Status' with the mode\nmarital_status_mode = train['Marital Status'].mode()[0]\nprint(f\"Most common value in 'Marital Status': {marital_status_mode}\")\n\n# Fill missing values without inplace=True\ntrain['Marital Status'] = train['Marital Status'].fillna(marital_status_mode)\ntest['Marital Status'] = test['Marital Status'].fillna(marital_status_mode)\n\n# Verify imputation\nprint(f\"Null values in 'Marital Status' after imputation (Train): {train['Marital Status'].isnull().sum()}\")\nprint(f\"Null values in 'Marital Status' after imputation (Test): {test['Marital Status'].isnull().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:25.97964Z","iopub.execute_input":"2024-12-01T21:08:25.979919Z","iopub.status.idle":"2024-12-01T21:08:26.240143Z","shell.execute_reply.started":"2024-12-01T21:08:25.979892Z","shell.execute_reply":"2024-12-01T21:08:26.239265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Let's Check Again Again","metadata":{}},{"cell_type":"code","source":"# Check for null values in all columns of train and test datasets\nprint(\"Null values in each column (Train):\")\nprint(train.isnull().sum())\n\nprint(\"\\nNull values in each column (Test):\")\nprint(test.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:26.241091Z","iopub.execute_input":"2024-12-01T21:08:26.241348Z","iopub.status.idle":"2024-12-01T21:08:27.046394Z","shell.execute_reply.started":"2024-12-01T21:08:26.241323Z","shell.execute_reply":"2024-12-01T21:08:27.045529Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Number of Dependents","metadata":{}},{"cell_type":"code","source":"# Count zero values and null values in 'Number of Dependents'\nzero_count_dependents = train['Number of Dependents'].eq(0).sum()\nnull_count_dependents = train['Number of Dependents'].isnull().sum()\n\n# Print the counts\nprint(f\"Number of zero values in 'Number of Dependents': {zero_count_dependents}\")\nprint(f\"Number of null values in 'Number of Dependents': {null_count_dependents}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:27.047263Z","iopub.execute_input":"2024-12-01T21:08:27.047536Z","iopub.status.idle":"2024-12-01T21:08:27.056497Z","shell.execute_reply.started":"2024-12-01T21:08:27.04751Z","shell.execute_reply":"2024-12-01T21:08:27.055687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter rows where 'Number of Dependents' is not null\ndependents_not_null = train[train['Number of Dependents'].notnull()]\n\n# Check correlation with Age (numerical)\ncorrelation_with_age = dependents_not_null[['Number of Dependents', 'Age']].corr()\nprint(\"Correlation between Number of Dependents and Age:\")\nprint(correlation_with_age)\n\n# Analyze correlation with Marital Status, Gender, and Occupation (categorical)\nprint(\"\\nMean Number of Dependents by Marital Status:\")\nprint(dependents_not_null.groupby('Marital Status')['Number of Dependents'].mean())\n\nprint(\"\\nMean Number of Dependents by Gender:\")\nprint(dependents_not_null.groupby('Gender')['Number of Dependents'].mean())\n\n# Check Occupation only for non-null values\nif 'Occupation' in dependents_not_null.columns and dependents_not_null['Occupation'].notnull().any():\n    print(\"\\nMean Number of Dependents by Occupation (Non-null):\")\n    print(dependents_not_null.groupby('Occupation')['Number of Dependents'].mean())\nelse:\n    print(\"\\nOccupation contains too many null values for meaningful analysis.\")\n\n# Optional: Visualize distribution by Gender and Marital Status\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='Marital Status', y='Number of Dependents', data=dependents_not_null)\nplt.title(\"Distribution of Number of Dependents by Marital Status\")\nplt.show()\n\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='Gender', y='Number of Dependents', data=dependents_not_null)\nplt.title(\"Distribution of Number of Dependents by Gender\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:27.057525Z","iopub.execute_input":"2024-12-01T21:08:27.058364Z","iopub.status.idle":"2024-12-01T21:08:28.739896Z","shell.execute_reply.started":"2024-12-01T21:08:27.058321Z","shell.execute_reply":"2024-12-01T21:08:28.738971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of 'Number of Dependents' (non-null values only)\nplt.figure(figsize=(10, 6))\nsns.histplot(train['Number of Dependents'].dropna(), kde=True, bins=30, color='blue', edgecolor='black')\n\nplt.title(\"Distribution of Number of Dependents\")\nplt.xlabel(\"Number of Dependents\")\nplt.ylabel(\"Frequency\")\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:28.74087Z","iopub.execute_input":"2024-12-01T21:08:28.741112Z","iopub.status.idle":"2024-12-01T21:08:32.717003Z","shell.execute_reply.started":"2024-12-01T21:08:28.741087Z","shell.execute_reply":"2024-12-01T21:08:32.715893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select relevant insurance-related columns for correlation\ninsurance_columns = ['Number of Dependents', 'Insurance Duration', 'Previous Claims', 'Premium Amount']\n\n# Calculate correlation matrix\ncorrelations = train[insurance_columns].corr()\n\n# Display correlation matrix\nprint(\"Correlation matrix for insurance-related columns:\")\nprint(correlations)\n\nplt.figure(figsize=(8, 6))\nsns.heatmap(correlations, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\nplt.title(\"Correlation Matrix of Number of Dependents with Insurance Columns\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:32.718359Z","iopub.execute_input":"2024-12-01T21:08:32.71873Z","iopub.status.idle":"2024-12-01T21:08:33.104701Z","shell.execute_reply.started":"2024-12-01T21:08:32.718691Z","shell.execute_reply":"2024-12-01T21:08:33.103839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'Number of Dependents' from train and test datasets\ntrain.drop('Number of Dependents', axis=1, inplace=True)\ntest.drop('Number of Dependents', axis=1, inplace=True)\n\n# Verify that the column has been removed\nprint(\"Columns in train dataset after dropping 'Number of Dependents':\")\nprint(train.columns)\n\nprint(\"\\nColumns in test dataset after dropping 'Number of Dependents':\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:33.105776Z","iopub.execute_input":"2024-12-01T21:08:33.106069Z","iopub.status.idle":"2024-12-01T21:08:33.40768Z","shell.execute_reply.started":"2024-12-01T21:08:33.106042Z","shell.execute_reply":"2024-12-01T21:08:33.406808Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Health Score","metadata":{}},{"cell_type":"code","source":"# Plot the distribution of 'Health Score' (non-null values only)\nplt.figure(figsize=(10, 6))\nsns.histplot(train['Health Score'].dropna(), kde=True, bins=30, color='green', edgecolor='black')\n\nplt.title(\"Distribution of Health Score\")\nplt.xlabel(\"Health Score\")\nplt.ylabel(\"Frequency\")\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:33.412067Z","iopub.execute_input":"2024-12-01T21:08:33.412362Z","iopub.status.idle":"2024-12-01T21:08:37.855317Z","shell.execute_reply.started":"2024-12-01T21:08:33.412335Z","shell.execute_reply":"2024-12-01T21:08:37.854469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select relevant columns for correlation\nhealth_correlation_columns = ['Health Score', 'Age', 'Annual Income', 'Premium Amount', 'Insurance Duration']\n\n# Calculate correlation matrix\nhealth_correlations = train[health_correlation_columns].corr()\n\n# Display the correlation matrix\nprint(\"Correlation matrix for Health Score:\")\nprint(health_correlations)\n\n# Visualize the correlation matrix\nplt.figure(figsize=(8, 6))\nsns.heatmap(health_correlations, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\nplt.title(\"Correlation Matrix of Health Score with Relevant Columns\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:37.856525Z","iopub.execute_input":"2024-12-01T21:08:37.856912Z","iopub.status.idle":"2024-12-01T21:08:38.283475Z","shell.execute_reply.started":"2024-12-01T21:08:37.856873Z","shell.execute_reply":"2024-12-01T21:08:38.282602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Scatter plot of 'Health Score' vs 'Age'\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x=train['Age'], y=train['Health Score'], alpha=0.6)\n\nplt.title(\"Health Score vs Age\")\nplt.xlabel(\"Age\")\nplt.ylabel(\"Health Score\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:38.284667Z","iopub.execute_input":"2024-12-01T21:08:38.285047Z","iopub.status.idle":"2024-12-01T21:08:40.856583Z","shell.execute_reply.started":"2024-12-01T21:08:38.285007Z","shell.execute_reply":"2024-12-01T21:08:40.855837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'Health Score' from train and test datasets\ntrain.drop('Health Score', axis=1, inplace=True)\ntest.drop('Health Score', axis=1, inplace=True)\n\n# Verify that the column has been removed\nprint(\"Columns in train dataset after dropping 'Health Score':\")\nprint(train.columns)\n\nprint(\"\\nColumns in test dataset after dropping 'Health Score':\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:40.857669Z","iopub.execute_input":"2024-12-01T21:08:40.858121Z","iopub.status.idle":"2024-12-01T21:08:41.113925Z","shell.execute_reply.started":"2024-12-01T21:08:40.85807Z","shell.execute_reply":"2024-12-01T21:08:41.113015Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Let's Check Again Again Again","metadata":{}},{"cell_type":"code","source":"# Check for null values in all columns of train and test datasets\nprint(\"Null values in each column (Train):\")\nprint(train.isnull().sum())\n\nprint(\"\\nNull values in each column (Test):\")\nprint(test.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:41.115103Z","iopub.execute_input":"2024-12-01T21:08:41.115868Z","iopub.status.idle":"2024-12-01T21:08:41.940107Z","shell.execute_reply.started":"2024-12-01T21:08:41.115824Z","shell.execute_reply":"2024-12-01T21:08:41.939347Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Occupation","metadata":{}},{"cell_type":"code","source":"# Plot the distribution of 'Occupation' (non-null values only)\nplt.figure(figsize=(12, 6))\nsns.countplot(y=train['Occupation'].dropna(), order=train['Occupation'].value_counts().index)\n\nplt.title(\"Distribution of Occupation\")\nplt.xlabel(\"Count\")\nplt.ylabel(\"Occupation\")\nplt.grid(axis='x')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:41.941079Z","iopub.execute_input":"2024-12-01T21:08:41.941347Z","iopub.status.idle":"2024-12-01T21:08:42.508524Z","shell.execute_reply.started":"2024-12-01T21:08:41.941321Z","shell.execute_reply":"2024-12-01T21:08:42.507755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'Occupation' and calculate mean of 'Annual Income' and 'Age'\noccupation_summary = train.groupby('Occupation')[['Annual Income', 'Age']].mean()\nprint(\"Mean Annual Income and Age by Occupation:\")\nprint(occupation_summary)\n\n# Visualize Annual Income by Occupation\nplt.figure(figsize=(12, 6))\noccupation_summary['Annual Income'].plot(kind='bar', color='skyblue', edgecolor='black')\n\nplt.title(\"Average Annual Income by Occupation\")\nplt.xlabel(\"Occupation\")\nplt.ylabel(\"Average Annual Income\")\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:42.509617Z","iopub.execute_input":"2024-12-01T21:08:42.509988Z","iopub.status.idle":"2024-12-01T21:08:42.837629Z","shell.execute_reply.started":"2024-12-01T21:08:42.509947Z","shell.execute_reply":"2024-12-01T21:08:42.836828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Distribution of Education Level within each Occupation\neducation_occupation = train.groupby('Occupation')['Education Level'].value_counts(normalize=True)\nprint(\"Education Level distribution within each Occupation:\")\nprint(education_occupation)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:42.838739Z","iopub.execute_input":"2024-12-01T21:08:42.839034Z","iopub.status.idle":"2024-12-01T21:08:43.028555Z","shell.execute_reply.started":"2024-12-01T21:08:42.839008Z","shell.execute_reply":"2024-12-01T21:08:43.027656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'Occupation' and calculate the mean of insurance-related columns\ninsurance_columns = ['Previous Claims', 'Insurance Duration', 'Premium Amount', 'Credit Score']\noccupation_insurance_summary = train.groupby('Occupation')[insurance_columns].mean()\n\n# Display the mean values\nprint(\"Mean insurance-related values by Occupation:\")\nprint(occupation_insurance_summary)\n\n# Visualize Premium Amount by Occupation\nplt.figure(figsize=(12, 6))\noccupation_insurance_summary['Premium Amount'].plot(kind='bar', color='salmon', edgecolor='black')\n\nplt.title(\"Average Premium Amount by Occupation\")\nplt.xlabel(\"Occupation\")\nplt.ylabel(\"Average Premium Amount\")\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:43.029481Z","iopub.execute_input":"2024-12-01T21:08:43.029718Z","iopub.status.idle":"2024-12-01T21:08:43.374921Z","shell.execute_reply.started":"2024-12-01T21:08:43.029694Z","shell.execute_reply":"2024-12-01T21:08:43.374082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'Occupation' and calculate mean for null-containing columns\nnull_columns = ['Previous Claims', 'Credit Score']\noccupation_null_summary = train.groupby('Occupation')[null_columns].mean()\n\n# Display the summary\nprint(\"Mean values of null-containing columns by Occupation:\")\nprint(occupation_null_summary)\n\n# Visualize the relationship\nplt.figure(figsize=(12, 6))\noccupation_null_summary['Credit Score'].plot(kind='bar', color='blue', edgecolor='black')\n\nplt.title(\"Average Credit Score by Occupation\")\nplt.xlabel(\"Occupation\")\nplt.ylabel(\"Average Credit Score\")\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y')\nplt.show()\n\nplt.figure(figsize=(12, 6))\noccupation_null_summary['Previous Claims'].plot(kind='bar', color='orange', edgecolor='black')\n\nplt.title(\"Average Previous Claims by Occupation\")\nplt.xlabel(\"Occupation\")\nplt.ylabel(\"Average Previous Claims\")\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:43.375865Z","iopub.execute_input":"2024-12-01T21:08:43.376093Z","iopub.status.idle":"2024-12-01T21:08:43.933319Z","shell.execute_reply.started":"2024-12-01T21:08:43.37607Z","shell.execute_reply":"2024-12-01T21:08:43.9325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'Occupation' from train and test datasets\ntrain.drop('Occupation', axis=1, inplace=True)\ntest.drop('Occupation', axis=1, inplace=True)\n\n# Verify that the column has been removed\nprint(\"Columns in train dataset after dropping 'Occupation':\")\nprint(train.columns)\n\nprint(\"\\nColumns in test dataset after dropping 'Occupation':\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:43.934514Z","iopub.execute_input":"2024-12-01T21:08:43.93487Z","iopub.status.idle":"2024-12-01T21:08:44.192695Z","shell.execute_reply.started":"2024-12-01T21:08:43.934843Z","shell.execute_reply":"2024-12-01T21:08:44.191773Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Previous Claims","metadata":{}},{"cell_type":"code","source":"# Count zero values and null values in 'Previous Claims'\nzero_count = train['Previous Claims'].eq(0).sum()\nnull_count = train['Previous Claims'].isnull().sum()\n\n# Print the counts\nprint(f\"Number of zero values in 'Previous Claims': {zero_count}\")\nprint(f\"Number of null values in 'Previous Claims': {null_count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:44.193755Z","iopub.execute_input":"2024-12-01T21:08:44.194076Z","iopub.status.idle":"2024-12-01T21:08:44.203126Z","shell.execute_reply.started":"2024-12-01T21:08:44.194049Z","shell.execute_reply":"2024-12-01T21:08:44.202316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of 'Previous Claims' (non-null values only)\nplt.figure(figsize=(10, 6))\nsns.histplot(train['Previous Claims'].dropna(), kde=True, bins=30, color='purple', edgecolor='black')\n\nplt.title(\"Distribution of Previous Claims\")\nplt.xlabel(\"Previous Claims\")\nplt.ylabel(\"Frequency\")\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:44.204328Z","iopub.execute_input":"2024-12-01T21:08:44.204654Z","iopub.status.idle":"2024-12-01T21:08:47.474049Z","shell.execute_reply.started":"2024-12-01T21:08:44.204618Z","shell.execute_reply":"2024-12-01T21:08:47.473358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count occurrences of each claim value in 'Previous Claims'\nclaims_count = train['Previous Claims'].value_counts(dropna=False).sort_index()\n\n# Display the counts\nprint(\"Number of occurrences for each claim value in 'Previous Claims':\")\nprint(claims_count)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:47.475433Z","iopub.execute_input":"2024-12-01T21:08:47.475825Z","iopub.status.idle":"2024-12-01T21:08:47.498355Z","shell.execute_reply.started":"2024-12-01T21:08:47.475769Z","shell.execute_reply":"2024-12-01T21:08:47.497502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Group by 'Previous Claims' and calculate mean for null-containing columns\nnull_columns = ['Credit Score']\nprevious_claims_null_summary = train.groupby('Previous Claims')[null_columns].mean()\n\n# Display the summary\nprint(\"Mean values of Credit Score by Previous Claims:\")\nprint(previous_claims_null_summary)\n\n# Visualize the relationship\nplt.figure(figsize=(10, 6))\nprevious_claims_null_summary['Credit Score'].plot(kind='line', marker='o', color='red')\n\nplt.title(\"Credit Score vs Previous Claims\")\nplt.xlabel(\"Previous Claims\")\nplt.ylabel(\"Average Credit Score\")\nplt.grid(axis='y')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:47.499334Z","iopub.execute_input":"2024-12-01T21:08:47.499593Z","iopub.status.idle":"2024-12-01T21:08:47.703413Z","shell.execute_reply.started":"2024-12-01T21:08:47.499568Z","shell.execute_reply":"2024-12-01T21:08:47.702563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count rows with both 'Credit Score' and 'Previous Claims' null\nboth_null_count = train[(train['Credit Score'].isnull()) & (train['Previous Claims'].isnull())].shape[0]\n\nprint(f\"Number of rows with both 'Credit Score' and 'Previous Claims' null: {both_null_count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:47.704365Z","iopub.execute_input":"2024-12-01T21:08:47.704636Z","iopub.status.idle":"2024-12-01T21:08:47.732604Z","shell.execute_reply.started":"2024-12-01T21:08:47.704609Z","shell.execute_reply":"2024-12-01T21:08:47.731825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate average Premium Amount for each Previous Claims value\nclaims_premium_avg = train.groupby('Previous Claims')['Premium Amount'].mean()\n\n# Plot a line chart\nplt.figure(figsize=(10, 6))\nclaims_premium_avg.plot(kind='line', marker='o', color='blue')\n\nplt.title(\"Average Premium Amount vs Previous Claims\")\nplt.xlabel(\"Previous Claims\")\nplt.ylabel(\"Average Premium Amount\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:47.733563Z","iopub.execute_input":"2024-12-01T21:08:47.733839Z","iopub.status.idle":"2024-12-01T21:08:47.924951Z","shell.execute_reply.started":"2024-12-01T21:08:47.733805Z","shell.execute_reply":"2024-12-01T21:08:47.924095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate average Premium Amount for each Credit Score value\ncredit_score_premium_avg = train.groupby('Credit Score')['Premium Amount'].mean()\n\n# Plot a line chart\nplt.figure(figsize=(10, 6))\ncredit_score_premium_avg.plot(kind='line', marker='o', color='green')\n\nplt.title(\"Average Premium Amount vs Credit Score\")\nplt.xlabel(\"Credit Score\")\nplt.ylabel(\"Average Premium Amount\")\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:47.925949Z","iopub.execute_input":"2024-12-01T21:08:47.926213Z","iopub.status.idle":"2024-12-01T21:08:48.139206Z","shell.execute_reply.started":"2024-12-01T21:08:47.926189Z","shell.execute_reply":"2024-12-01T21:08:48.13844Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Policy Start Date","metadata":{}},{"cell_type":"code","source":"# Ensure 'Policy Start Date' is in datetime format\ntrain['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'], errors='coerce')\ntest['Policy Start Date'] = pd.to_datetime(test['Policy Start Date'], errors='coerce')\n\n# Extract date-related features\nfor df in [train, test]:\n    df['Policy_Start_Year'] = df['Policy Start Date'].dt.year\n    df['Policy_Start_Month'] = df['Policy Start Date'].dt.month\n    df['Policy_Start_Day'] = df['Policy Start Date'].dt.day\n    df['Policy_Start_Weekday'] = df['Policy Start Date'].dt.weekday  # Monday=0, Sunday=6\n    df['Policy_Start_Elapsed'] = (df['Policy Start Date'] - pd.Timestamp('2000-01-01')).dt.days","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:48.140503Z","iopub.execute_input":"2024-12-01T21:08:48.141167Z","iopub.status.idle":"2024-12-01T21:08:49.079922Z","shell.execute_reply.started":"2024-12-01T21:08:48.141126Z","shell.execute_reply":"2024-12-01T21:08:49.079186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop the original 'Policy Start Date' column\ntrain.drop('Policy Start Date', axis=1, inplace=True)\ntest.drop('Policy Start Date', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:49.080906Z","iopub.execute_input":"2024-12-01T21:08:49.081198Z","iopub.status.idle":"2024-12-01T21:08:49.279577Z","shell.execute_reply.started":"2024-12-01T21:08:49.081172Z","shell.execute_reply":"2024-12-01T21:08:49.278846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Sample of extracted features:\")\nprint(train[['Policy_Start_Year', 'Policy_Start_Month', 'Policy_Start_Day', 'Policy_Start_Weekday', 'Policy_Start_Elapsed']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:49.280562Z","iopub.execute_input":"2024-12-01T21:08:49.280838Z","iopub.status.idle":"2024-12-01T21:08:49.302703Z","shell.execute_reply.started":"2024-12-01T21:08:49.280811Z","shell.execute_reply":"2024-12-01T21:08:49.301911Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Encode Non-Numeric Columns","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Combine train and test for consistent encoding\ncombined = pd.concat([train, test], axis=0)\n\n# Initialize LabelEncoder\nlabel_encoders = {}\n\n# Updated categorical columns list (excluding 'Policy Start Date')\ncategorical_cols = ['Gender', 'Marital Status', 'Education Level', 'Location',\n                    'Policy Type', 'Smoking Status', 'Exercise Frequency', 'Property Type']\n\n# Encode categorical columns\nfor col in categorical_cols:\n    le = LabelEncoder()\n    combined[col] = le.fit_transform(combined[col].astype(str))\n    label_encoders[col] = le  # Save the encoder for potential inverse transformations\n\n# Split back into train and test\ntrain = combined.iloc[:train.shape[0], :]\ntest = combined.iloc[train.shape[0]:, :]\n\n# Ensure extracted features from Policy Start Date are present\nprint(\"Sample of extracted features in train:\")\nprint(train[['Policy_Start_Year', 'Policy_Start_Month', 'Policy_Start_Day', 'Policy_Start_Weekday', 'Policy_Start_Elapsed']].head())\n\nprint(\"Sample of extracted features in test:\")\nprint(test[['Policy_Start_Year', 'Policy_Start_Month', 'Policy_Start_Day', 'Policy_Start_Weekday', 'Policy_Start_Elapsed']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:49.303655Z","iopub.execute_input":"2024-12-01T21:08:49.303919Z","iopub.status.idle":"2024-12-01T21:08:52.087887Z","shell.execute_reply.started":"2024-12-01T21:08:49.303894Z","shell.execute_reply":"2024-12-01T21:08:52.086991Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Credit Score","metadata":{}},{"cell_type":"code","source":"# Ensure XGBoost is imported\nimport xgboost as xgb\n\n# Exclude 'Previous Claims', 'Premium Amount', and 'id' from features\nfeatures = train.columns.drop(['Credit Score', 'Previous Claims', 'Premium Amount', 'id'])\n\n# Separate data where 'Credit Score' is not null\ntrain_credit = train[train['Credit Score'].notnull()].copy()\n\n# Prepare training data\nX_train_credit = train_credit[features]\ny_train_credit = train_credit['Credit Score']\n\n# Create DMatrix for training\ndtrain_c = xgb.DMatrix(X_train_credit, label=y_train_credit)\n\n# Define parameters\nparams_c = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'seed': 42\n}\n\n# Train the XGBoost model\nmodel_credit = xgb.train(params_c, dtrain_c, num_boost_round=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:52.089046Z","iopub.execute_input":"2024-12-01T21:08:52.089364Z","iopub.status.idle":"2024-12-01T21:08:56.254591Z","shell.execute_reply.started":"2024-12-01T21:08:52.089335Z","shell.execute_reply":"2024-12-01T21:08:56.253845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare feature matrix for rows with missing 'Credit Score'\nX_missing_credit = train[train['Credit Score'].isnull()][features]\ndmissing_credit = xgb.DMatrix(X_missing_credit)\n\n# Predict missing 'Credit Score'\ncredit_score_imputed = model_credit.predict(dmissing_credit)\n\n# Assign the predictions back to the missing rows in 'Credit Score'\ntrain.loc[train['Credit Score'].isnull(), 'Credit Score'] = credit_score_imputed\n\n# Verify that 'Credit Score' no longer contains null values\nprint(f\"Null values in 'Credit Score' after imputation (Train): {train['Credit Score'].isnull().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:08:56.255412Z","iopub.execute_input":"2024-12-01T21:08:56.25566Z","iopub.status.idle":"2024-12-01T21:08:56.44681Z","shell.execute_reply.started":"2024-12-01T21:08:56.255635Z","shell.execute_reply":"2024-12-01T21:08:56.445944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare the feature matrix for rows with missing 'Credit Score' in the test set\nX_test_missing_credit = test[test['Credit Score'].isnull()][features]  # Use the same features list as in training\ndtest_missing_credit = xgb.DMatrix(X_test_missing_credit)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:09:40.539965Z","iopub.execute_input":"2024-12-01T21:09:40.54032Z","iopub.status.idle":"2024-12-01T21:09:40.580482Z","shell.execute_reply.started":"2024-12-01T21:09:40.54029Z","shell.execute_reply":"2024-12-01T21:09:40.579826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict missing 'Credit Score' in the test dataset\ntest_credit_score_imputed = model_credit.predict(dtest_missing_credit)\n\n# Assign the predictions back to the test dataset\ntest.loc[test['Credit Score'].isnull(), 'Credit Score'] = test_credit_score_imputed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:09:48.857298Z","iopub.execute_input":"2024-12-01T21:09:48.857634Z","iopub.status.idle":"2024-12-01T21:09:48.936705Z","shell.execute_reply.started":"2024-12-01T21:09:48.857605Z","shell.execute_reply":"2024-12-01T21:09:48.93607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify that 'Credit Score' no longer contains null values\nprint(f\"Null values in 'Credit Score' after imputation (Test): {test['Credit Score'].isnull().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:09:55.157647Z","iopub.execute_input":"2024-12-01T21:09:55.158371Z","iopub.status.idle":"2024-12-01T21:09:55.164132Z","shell.execute_reply.started":"2024-12-01T21:09:55.158335Z","shell.execute_reply":"2024-12-01T21:09:55.163228Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### The following is necessary because of a mistake i made :|","metadata":{}},{"cell_type":"code","source":"# Drop 'Premium Amount' from the test dataset\ntest.drop('Premium Amount', axis=1, inplace=True)\n\n# Verify that the column has been removed\nprint(\"Columns in test dataset after dropping 'Premium Amount':\")\nprint(test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:10:49.348509Z","iopub.execute_input":"2024-12-01T21:10:49.349356Z","iopub.status.idle":"2024-12-01T21:10:49.385206Z","shell.execute_reply.started":"2024-12-01T21:10:49.349321Z","shell.execute_reply":"2024-12-01T21:10:49.384302Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Previous Claims","metadata":{}},{"cell_type":"code","source":"# Features for the predictive model\nfeatures_previous_claims = train.columns.drop(['Previous Claims', 'id', 'Premium Amount'])\n\n# Separate rows where 'Previous Claims' is not null\ntrain_previous_claims = train[train['Previous Claims'].notnull()]\nX_train_pc = train_previous_claims[features_previous_claims]\ny_train_pc = train_previous_claims['Previous Claims']\n\n# Train a regression model (e.g., XGBoost)\ndtrain_pc = xgb.DMatrix(X_train_pc, label=y_train_pc)\nparams_pc = {\n    'objective': 'reg:squarederror',\n    'eval_metric': 'rmse',\n    'seed': 42\n}\nmodel_previous_claims = xgb.train(params_pc, dtrain_pc, num_boost_round=100)\n\n# Predict missing 'Previous Claims' for train and test\nX_missing_pc_train = train[train['Previous Claims'].isnull()][features_previous_claims]\nX_missing_pc_test = test[test['Previous Claims'].isnull()][features_previous_claims]\n\ndmissing_pc_train = xgb.DMatrix(X_missing_pc_train)\ndmissing_pc_test = xgb.DMatrix(X_missing_pc_test)\n\ntrain.loc[train['Previous Claims'].isnull(), 'Previous Claims'] = model_previous_claims.predict(dmissing_pc_train)\ntest.loc[test['Previous Claims'].isnull(), 'Previous Claims'] = model_previous_claims.predict(dmissing_pc_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:13:05.890339Z","iopub.execute_input":"2024-12-01T21:13:05.891188Z","iopub.status.idle":"2024-12-01T21:13:10.437033Z","shell.execute_reply.started":"2024-12-01T21:13:05.89114Z","shell.execute_reply":"2024-12-01T21:13:10.436271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Save the Updated Datasets","metadata":{}},{"cell_type":"code","source":"# Save the updated datasets\ntrain.to_csv(\"train_cleaned.csv\", index=False)\ntest.to_csv(\"test_cleaned.csv\", index=False)\n\nprint(\"Cleaned datasets have been saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:14:11.416246Z","iopub.execute_input":"2024-12-01T21:14:11.416596Z","iopub.status.idle":"2024-12-01T21:14:24.431053Z","shell.execute_reply.started":"2024-12-01T21:14:11.416565Z","shell.execute_reply":"2024-12-01T21:14:24.430148Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Review of Our New Data\n","metadata":{}},{"cell_type":"code","source":"# Reload the updated datasets\ntrain = pd.read_csv(\"train_cleaned.csv\")\ntest = pd.read_csv(\"test_cleaned.csv\")\n\n# Check for null values in each column\nprint(\"Null values in each column (Train):\")\nprint(train.isnull().sum())\n\nprint(\"\\nNull values in each column (Test):\")\nprint(test.isnull().sum())\n\n# Display data types of each column\nprint(\"\\nData types of each column (Train):\")\nprint(train.dtypes)\n\nprint(\"\\nData types of each column (Test):\")\nprint(test.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T21:14:24.432489Z","iopub.execute_input":"2024-12-01T21:14:24.432773Z","iopub.status.idle":"2024-12-01T21:14:26.991151Z","shell.execute_reply.started":"2024-12-01T21:14:24.432745Z","shell.execute_reply":"2024-12-01T21:14:26.990256Z"}},"outputs":[],"execution_count":null}]}