{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:39:53.786628Z","iopub.execute_input":"2024-12-04T17:39:53.78765Z","iopub.status.idle":"2024-12-04T17:39:53.792895Z","shell.execute_reply.started":"2024-12-04T17:39:53.787592Z","shell.execute_reply":"2024-12-04T17:39:53.791457Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:39:53.794925Z","iopub.execute_input":"2024-12-04T17:39:53.795368Z","iopub.status.idle":"2024-12-04T17:40:01.013253Z","shell.execute_reply.started":"2024-12-04T17:39:53.795307Z","shell.execute_reply":"2024-12-04T17:40:01.012186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:01.014531Z","iopub.execute_input":"2024-12-04T17:40:01.014858Z","iopub.status.idle":"2024-12-04T17:40:01.039781Z","shell.execute_reply.started":"2024-12-04T17:40:01.014825Z","shell.execute_reply":"2024-12-04T17:40:01.038667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:01.041089Z","iopub.execute_input":"2024-12-04T17:40:01.041506Z","iopub.status.idle":"2024-12-04T17:40:01.689511Z","shell.execute_reply.started":"2024-12-04T17:40:01.041454Z","shell.execute_reply":"2024-12-04T17:40:01.688375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:01.691952Z","iopub.execute_input":"2024-12-04T17:40:01.692293Z","iopub.status.idle":"2024-12-04T17:40:02.388388Z","shell.execute_reply.started":"2024-12-04T17:40:01.692258Z","shell.execute_reply":"2024-12-04T17:40:02.387172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:02.389917Z","iopub.execute_input":"2024-12-04T17:40:02.390401Z","iopub.status.idle":"2024-12-04T17:40:03.025947Z","shell.execute_reply.started":"2024-12-04T17:40:02.390326Z","shell.execute_reply":"2024-12-04T17:40:03.024712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_percentage = (train.isnull().sum() / len(train)) * 100\nmissing_percentage = missing_percentage[missing_percentage > 0].sort_values(ascending=False)\n\nplt.figure(figsize=(8, 6))\nsns.barplot(x=missing_percentage.index, y=missing_percentage.values, palette=\"coolwarm\")\nplt.title(\"Percentage of Missing Values by Column\", fontsize=14, weight='bold')\nplt.ylabel(\"Percentage (%)\", fontsize=12)\nplt.xlabel(\"Columns\", fontsize=12)\nplt.xticks(rotation=45, fontsize=10)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nsns.despine(left=True, bottom=True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:03.027136Z","iopub.execute_input":"2024-12-04T17:40:03.027472Z","iopub.status.idle":"2024-12-04T17:40:03.904762Z","shell.execute_reply.started":"2024-12-04T17:40:03.02744Z","shell.execute_reply":"2024-12-04T17:40:03.903503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing values","metadata":{}},{"cell_type":"code","source":"train['Previous Claims'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:03.906642Z","iopub.execute_input":"2024-12-04T17:40:03.907079Z","iopub.status.idle":"2024-12-04T17:40:03.933653Z","shell.execute_reply.started":"2024-12-04T17:40:03.90703Z","shell.execute_reply":"2024-12-04T17:40:03.93243Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Occupation'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:03.935042Z","iopub.execute_input":"2024-12-04T17:40:03.935516Z","iopub.status.idle":"2024-12-04T17:40:03.993588Z","shell.execute_reply.started":"2024-12-04T17:40:03.935475Z","shell.execute_reply":"2024-12-04T17:40:03.992328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Number of Dependents'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:03.995171Z","iopub.execute_input":"2024-12-04T17:40:03.996205Z","iopub.status.idle":"2024-12-04T17:40:04.020699Z","shell.execute_reply.started":"2024-12-04T17:40:03.996152Z","shell.execute_reply":"2024-12-04T17:40:04.019581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Education Level'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.022055Z","iopub.execute_input":"2024-12-04T17:40:04.022553Z","iopub.status.idle":"2024-12-04T17:40:04.10128Z","shell.execute_reply.started":"2024-12-04T17:40:04.02251Z","shell.execute_reply":"2024-12-04T17:40:04.099948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"occupation_probs = train[train['Occupation'].notnull()].groupby('Education Level')['Occupation'].value_counts(normalize=True)\nprint(occupation_probs)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.102809Z","iopub.execute_input":"2024-12-04T17:40:04.103221Z","iopub.status.idle":"2024-12-04T17:40:04.409231Z","shell.execute_reply.started":"2024-12-04T17:40:04.103184Z","shell.execute_reply":"2024-12-04T17:40:04.408098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = {\n    \"Education Level\": [\"Bachelor's\", \"Bachelor's\", \"Bachelor's\",\n                        \"High School\", \"High School\", \"High School\",\n                        \"Master's\", \"Master's\", \"Master's\",\n                        \"PhD\", \"PhD\", \"PhD\"],\n    \"Occupation\": [\"Employed\", \"Self-Employed\", \"Unemployed\",\n                   \"Self-Employed\", \"Employed\", \"Unemployed\",\n                   \"Self-Employed\", \"Employed\", \"Unemployed\",\n                   \"Self-Employed\", \"Employed\", \"Unemployed\"],\n    \"Proportion\": [0.338023, 0.334912, 0.327065,\n                   0.336438, 0.335161, 0.328402,\n                   0.335537, 0.334553, 0.329910,\n                   0.336001, 0.335582, 0.328418]\n}\n\ndf = pd.DataFrame(data)\n\n# Visualization\nplt.figure(figsize=(10, 6))\nsns.barplot(\n    data=df, \n    x=\"Education Level\", \n    y=\"Proportion\", \n    hue=\"Occupation\", \n    palette=\"coolwarm\"\n)\nplt.title(\"Proportion of Occupations by Education Level\", fontsize=14, weight='bold')\nplt.ylabel(\"Proportion\", fontsize=12)\nplt.xlabel(\"Education Level\", fontsize=12)\nplt.xticks(rotation=45, fontsize=10)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nsns.despine(left=True, bottom=True)\nplt.legend(title=\"Occupation\", fontsize=10)\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.412776Z","iopub.execute_input":"2024-12-04T17:40:04.413107Z","iopub.status.idle":"2024-12-04T17:40:04.725323Z","shell.execute_reply.started":"2024-12-04T17:40:04.413076Z","shell.execute_reply":"2024-12-04T17:40:04.724183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert proportions into a dictionary for easy lookup\noccupation_probs = {\n    \"Bachelor's\": [0.338023, 0.334912, 0.327065],\n    \"High School\": [0.336438, 0.335161, 0.328402],\n    \"Master's\": [0.335537, 0.334553, 0.329910],\n    \"PhD\": [0.336001, 0.335582, 0.328418],\n}\n\noccupation_labels = ['Employed', 'Self-Employed', 'Unemployed']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.726748Z","iopub.execute_input":"2024-12-04T17:40:04.727106Z","iopub.status.idle":"2024-12-04T17:40:04.732903Z","shell.execute_reply.started":"2024-12-04T17:40:04.727071Z","shell.execute_reply":"2024-12-04T17:40:04.731785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_levels = set(train['Education Level'].unique()) - set(occupation_probs.keys())\nprint(\"Education Levels missing from the dictionary:\", missing_levels)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.73406Z","iopub.execute_input":"2024-12-04T17:40:04.734406Z","iopub.status.idle":"2024-12-04T17:40:04.816987Z","shell.execute_reply.started":"2024-12-04T17:40:04.734333Z","shell.execute_reply":"2024-12-04T17:40:04.81583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for level, probs in occupation_probs.items():\n    print(f\"{level}: Sum = {sum(probs)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.818184Z","iopub.execute_input":"2024-12-04T17:40:04.818507Z","iopub.status.idle":"2024-12-04T17:40:04.82409Z","shell.execute_reply.started":"2024-12-04T17:40:04.818474Z","shell.execute_reply":"2024-12-04T17:40:04.823036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Re-normalize probabilities to handle precision issues\nfor level in occupation_probs:\n    total = sum(occupation_probs[level])\n    occupation_probs[level] = [p / total for p in occupation_probs[level]]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.825397Z","iopub.execute_input":"2024-12-04T17:40:04.825753Z","iopub.status.idle":"2024-12-04T17:40:04.834858Z","shell.execute_reply.started":"2024-12-04T17:40:04.82572Z","shell.execute_reply":"2024-12-04T17:40:04.833822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for level, probs in occupation_probs.items():\n    print(f\"{level}: Sum = {sum(probs)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.83606Z","iopub.execute_input":"2024-12-04T17:40:04.836434Z","iopub.status.idle":"2024-12-04T17:40:04.848977Z","shell.execute_reply.started":"2024-12-04T17:40:04.836394Z","shell.execute_reply":"2024-12-04T17:40:04.847966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def impute_occupation(row):\n    if pd.isnull(row['Occupation']):\n        # Fetch probabilities for the given 'Education Level'\n        probs = occupation_probs[row['Education Level']]\n        return np.random.choice(occupation_labels, p=probs)\n    return row['Occupation']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.85019Z","iopub.execute_input":"2024-12-04T17:40:04.850546Z","iopub.status.idle":"2024-12-04T17:40:04.865481Z","shell.execute_reply.started":"2024-12-04T17:40:04.850512Z","shell.execute_reply":"2024-12-04T17:40:04.864256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Occupation'] = train.apply(impute_occupation, axis=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:04.867Z","iopub.execute_input":"2024-12-04T17:40:04.867399Z","iopub.status.idle":"2024-12-04T17:40:27.498236Z","shell.execute_reply.started":"2024-12-04T17:40:04.867325Z","shell.execute_reply":"2024-12-04T17:40:27.497161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train['Occupation'].isnull().sum()) \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:27.49974Z","iopub.execute_input":"2024-12-04T17:40:27.500073Z","iopub.status.idle":"2024-12-04T17:40:27.572986Z","shell.execute_reply.started":"2024-12-04T17:40:27.50004Z","shell.execute_reply":"2024-12-04T17:40:27.571794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.groupby('Education Level')['Occupation'].value_counts(normalize=True))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:27.574601Z","iopub.execute_input":"2024-12-04T17:40:27.575053Z","iopub.status.idle":"2024-12-04T17:40:27.78611Z","shell.execute_reply.started":"2024-12-04T17:40:27.575014Z","shell.execute_reply":"2024-12-04T17:40:27.784851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:27.787707Z","iopub.execute_input":"2024-12-04T17:40:27.788101Z","iopub.status.idle":"2024-12-04T17:40:28.460684Z","shell.execute_reply.started":"2024-12-04T17:40:27.788064Z","shell.execute_reply":"2024-12-04T17:40:28.459561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Numeric columns\ntrain['Age'] = train['Age'].fillna(train['Age'].median())\ntrain['Annual Income'] = train.groupby('Occupation')['Annual Income'].transform(lambda x: x.fillna(x.median()))\ntrain['Number of Dependents'] = train['Number of Dependents'].fillna(0)\ntrain['Health Score'] = train['Health Score'].fillna(train['Health Score'].median())\ntrain['Previous Claims'] = train['Previous Claims'].fillna(0)\ntrain['Vehicle Age'] = train['Vehicle Age'].fillna(train['Vehicle Age'].mean())\ntrain['Credit Score'] = train['Credit Score'].fillna(train['Credit Score'].median())\ntrain['Insurance Duration'] = train['Insurance Duration'].fillna(train['Insurance Duration'].median())\n\n# Categorical columns\ntrain['Marital Status'] = train['Marital Status'].fillna('Missing')\ntrain['Customer Feedback'] = train['Customer Feedback'].fillna('Missing')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:28.461992Z","iopub.execute_input":"2024-12-04T17:40:28.462321Z","iopub.status.idle":"2024-12-04T17:40:29.038497Z","shell.execute_reply.started":"2024-12-04T17:40:28.462288Z","shell.execute_reply":"2024-12-04T17:40:29.037174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:29.039976Z","iopub.execute_input":"2024-12-04T17:40:29.040324Z","iopub.status.idle":"2024-12-04T17:40:29.695981Z","shell.execute_reply.started":"2024-12-04T17:40:29.040288Z","shell.execute_reply":"2024-12-04T17:40:29.695003Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Getting better view","metadata":{}},{"cell_type":"code","source":"categorical_columns = train.select_dtypes(include=['object']).columns\nfor col in categorical_columns:\n    print(f\"{col}: {train[col].unique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:29.697354Z","iopub.execute_input":"2024-12-04T17:40:29.697797Z","iopub.status.idle":"2024-12-04T17:40:30.960918Z","shell.execute_reply.started":"2024-12-04T17:40:29.69776Z","shell.execute_reply":"2024-12-04T17:40:30.95969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_columns = train.select_dtypes(include=['float64', 'int64']).columns\ntrain[numeric_columns].hist(bins=20, figsize=(15, 10))\nplt.suptitle(\"Distributions of Numeric Columns\", fontsize=16, weight='bold')\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nfor col in numeric_columns:\n    sns.histplot(data=train, x=col, kde=True, color='blue', alpha=0.7, bins=30)\n    plt.title(f\"Distribution of {col}\", fontsize=14, weight='bold')\n    plt.xlabel(col, fontsize=12)\n    plt.ylabel(\"Frequency\", fontsize=12)\n    plt.grid(axis='y', linestyle='--', alpha=0.7)\n    sns.despine(left=True, bottom=True)\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:40:30.96206Z","iopub.execute_input":"2024-12-04T17:40:30.962372Z","iopub.status.idle":"2024-12-04T17:41:20.151823Z","shell.execute_reply.started":"2024-12-04T17:40:30.962327Z","shell.execute_reply":"2024-12-04T17:41:20.150786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\nsns.heatmap(\n    data=train[numeric_columns].corr(), \n    annot=True, \n    cmap=\"coolwarm\", \n    fmt=\".2f\", \n    linewidths=0.5\n)\nplt.title(\"Correlation Heatmap\", fontsize=14, weight='bold')\nplt.xticks(fontsize=10)\nplt.yticks(fontsize=10)\nsns.despine(left=True, bottom=True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:46:53.049907Z","iopub.execute_input":"2024-12-04T17:46:53.050458Z","iopub.status.idle":"2024-12-04T17:46:54.070401Z","shell.execute_reply.started":"2024-12-04T17:46:53.050416Z","shell.execute_reply":"2024-12-04T17:46:54.068853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\ngrouped = train.groupby('Education Level')['Annual Income'].mean().sort_values()\nsns.barplot(\n    x=grouped.index, \n    y=grouped.values, \n    palette=\"coolwarm\"\n)\nplt.title(\"Average Annual Income by Education Level\", fontsize=14, weight='bold')\nplt.ylabel(\"Average Annual Income\", fontsize=12)\nplt.xlabel(\"Education Level\", fontsize=12)\nplt.xticks(rotation=45, fontsize=10)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nsns.despine(left=True, bottom=True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:49:29.81159Z","iopub.execute_input":"2024-12-04T17:49:29.811989Z","iopub.status.idle":"2024-12-04T17:49:30.183996Z","shell.execute_reply.started":"2024-12-04T17:49:29.811952Z","shell.execute_reply":"2024-12-04T17:49:30.182662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.kdeplot(\n    data=train, \n    x=\"Credit Score\", \n    hue=\"Marital Status\", \n    fill=True, \n    common_norm=False, \n    palette=\"coolwarm\", \n    alpha=0.6\n)\nplt.title(\"Distribution of Credit Score by Marital Status\", fontsize=14, weight='bold')\nplt.xlabel(\"Credit Score\", fontsize=12)\nplt.ylabel(\"Density\", fontsize=12)\nplt.grid(axis='y', linestyle='--', alpha=0.7)\nsns.despine(left=True, bottom=True)\nplt.legend(title=\"Marital Status\", fontsize=10)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T17:50:35.206554Z","iopub.execute_input":"2024-12-04T17:50:35.206881Z","iopub.status.idle":"2024-12-04T17:50:41.255018Z","shell.execute_reply.started":"2024-12-04T17:50:35.206848Z","shell.execute_reply":"2024-12-04T17:50:41.253833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}