{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup 📚","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.model_selection import train_test_split \n# Matplotlib config\n%matplotlib inline\n%config InlineBackend.figure_formats = ['svg']\n%config InlineBackend.rc = {'figure.figsize': (5.0, 3.0)}\n\n","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-21T23:13:29.440569Z","iopub.execute_input":"2024-12-21T23:13:29.441029Z","iopub.status.idle":"2024-12-21T23:13:32.70054Z","shell.execute_reply.started":"2024-12-21T23:13:29.440975Z","shell.execute_reply":"2024-12-21T23:13:32.699236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%HTML\n","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-05T23:51:12.929322Z","iopub.execute_input":"2024-12-05T23:51:12.929947Z","iopub.status.idle":"2024-12-05T23:51:12.93799Z","shell.execute_reply.started":"2024-12-05T23:51:12.929901Z","shell.execute_reply":"2024-12-05T23:51:12.936921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# functions 🔗","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"# Distribution plots for quantitative variables\n# ich glaube dass nicht für boolean-datatypes funktioniert\n\nimport warnings\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\ndef plot_quantitative_var_distributions(df, var_list, bin_dict=None):\n \"\"\"\n Plottet Histogramm, Boxplot und KDE für jede Variable in der übergebenen Liste.\n \n Parameters:\n df (pd.DataFrame): Der DataFrame, der die Daten enthält.\n var_list (list): Liste der numerischen Variablen, die geplottet werden sollen.\n bin_dict (dict): Dictionary mit benutzerdefinierten Bins für spezifische Variablen. \n Falls nicht angegeben, wird die Anzahl irgendwie festgelegt\n \"\"\"\n \n for col in var_list:\n plt.figure(figsize=(12, 3))\n \n \n # Histogramm\n plt.subplot(1, 3, 1)\n if col in bin_dict:\n df[col].plot.hist(color='skyblue', bins=bin_dict[col])\n else: \n df[col].plot.hist(color='skyblue')\n plt.title(f'Histogram of {col}')\n \n \n # Boxplot\n plt.subplot(1, 3, 2)\n sns.boxplot(x=df[col], color='skyblue')\n plt.title(f'Boxplot of {col}')\n plt.xlabel('')\n\n # KDE\n plt.subplot(1, 3, 3)\n sns.kdeplot(df[col], fill=True)\n plt.title(f'KDE of {col}')\n plt.xlabel('')\n\n plt.tight_layout()\n plt.show()\n\n#################################\n\n\n\nimport warnings\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\ndef plot_quantitative_qualitative_distributions(df, numerical_var, categorical_var, ordinal_variable_order=None):\n \"\"\"\n Erstellt 6 verschiedene Visualisierungen zur Analyse der Beziehung zwischen einer quantitativen und einer qualitativen Variable.\n Die Funktion unterstützt zwei Anwendungsfälle:\n\n Fall 1: Analyse numerischer Variablen nach Kategorien (Classification)\n - numerical_var: Quantitativ/Numerisch (z.B. 'Age', 'Income')\n - categorical_var: Qualitativ/Kategorisch (z.B. 'PurchaseStatus')\n Beispiel:\n >>> target = 'PurchaseStatus'\n >>> for num_var in ['Age', 'Income']:\n >>> plot_quantitative_qualitative_distributions(df, num_var, target, {})\n\n Fall 2: Analyse kategorischer Features für numerisches Target (Regression)\n - numerical_var: Quantitativ/Numerisch (z.B. 'Price', 'CarbonEmission')\n - categorical_var: Qualitativ/Kategorisch (z.B. 'Gender', 'Vehicle_Type')\n Beispiel:\n >>> target = 'Price'\n >>> for cat_var in ['Gender', 'Vehicle_Type']:\n >>> plot_quantitative_qualitative_distributions(df, target, cat_var, ordinal_variable_order)\n\n Parameter:\n ----------\n df (pd.DataFrame): Der DataFrame, der die Daten enthält\n numerical_var (str): numerische Variable auf der y-Achse\n - Fall 1: Name der numerischen Variable (z.B. 'Age')\n - Fall 2: Name der numerischen Zielvariable (z.B. 'Price')\n categorical_var (str): kategoriale Variable auf der x-Achse\n - Fall 1: Name der kategorialen Zielvariable (z.B. 'PurchaseStatus')\n - Fall 2: Name eines kategorialen Features (z.B. 'Gender')\n ordinal_variable_order (dict): Ordnung der ordinalen Variablen\n \"\"\"\n \n ordinal_variable_order = ordinal_variable_order or {}\n \n # Bestimme die Ordnung für das aktuelle Feature\n if ordinal_variable_order and categorical_var in ordinal_variable_order:\n order = ordinal_variable_order[categorical_var]\n else:\n order = None\n \n # Erstelle eine große Figure für alle 6 Subplots\n plt.figure(figsize=(15, 10))\n \n # Dictionary für Plot-Konfigurationen\n plot_configs = {\n (0, 0): {\n 'func': sns.countplot,\n 'kwargs': {'data': df, 'x': categorical_var, 'palette': \"Set2\", 'order': order},\n 'title': f'Häufigkeitsverteilung: {categorical_var}'\n },\n (0, 1): {\n 'func': sns.boxplot,\n 'kwargs': {'x': categorical_var, 'y': numerical_var, 'data': df, 'palette': \"Set2\", 'order': order},\n# 'kwargs': {'x': numerical_var, 'y': categorical_var, 'data': df, 'palette': \"Set2\", 'order': order},\n 'title': f'Boxplot: {categorical_var} vs. {numerical_var}'\n },\n (0, 2): {\n 'func': sns.violinplot,\n 'kwargs': {'x': categorical_var, 'y': numerical_var, 'data': df, 'palette': \"Set2\", 'order': order},\n# 'kwargs': {'x': numerical_var, 'y': categorical_var, 'data': df, 'palette': \"Set2\", 'order': order},\n 'title': f'Violinplot: {categorical_var} vs. {numerical_var}'\n },\n (1, 0): {\n 'func': sns.barplot,\n 'kwargs': {'x': categorical_var, 'y': numerical_var, 'data': df, 'palette': \"Set2\", 'order': order, 'ci': 95},\n 'title': f'Mittelwerte mit KI: {categorical_var} vs. {numerical_var}'\n },\n (1, 1): {\n 'func': sns.stripplot,\n 'kwargs': {'x': categorical_var, 'y': numerical_var, 'data': df, 'palette': \"Set2\", 'order': order, \n 'alpha': 0.1, 'jitter': 0.2},\n 'title': f'Stripplot: {categorical_var} vs. {numerical_var}'\n },\n (1, 2): {\n 'func': sns.pointplot,\n 'kwargs': {'x': categorical_var, 'y': numerical_var, 'data': df, 'palette': \"Set2\", 'order': order},\n 'title': f'Punktplot: {categorical_var} vs. {numerical_var}'\n }\n }\n \n # Erstelle alle Plots\n for (row, col), config in plot_configs.items():\n plt.subplot(2, 3, row * 3 + col + 1)\n config['func'](**config['kwargs'])\n plt.title(config['title'])\n plt.xticks(rotation=45, ha='right')\n \n # Adjustiere das Layout\n plt.tight_layout()\n plt.show()\n \n # Statistische Zusammenfassung\n print(f\"\\nStatistische Zusammenfassung für {categorical_var}:\")\n summary = df.groupby(categorical_var)[numerical_var].agg(['count', 'mean', 'std', 'min', 'max']).round(2)\n if order is not None:\n summary = summary.reindex(order)\n print(summary)\n print(\"\\n\" + \"=\"*70 + \"\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T21:57:38.398663Z","iopub.execute_input":"2024-12-03T21:57:38.399089Z","iopub.status.idle":"2024-12-03T21:57:38.422289Z","shell.execute_reply.started":"2024-12-03T21:57:38.399043Z","shell.execute_reply":"2024-12-03T21:57:38.420982Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading 📥","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/playground-series-s4e12/'\ndf = pd.read_csv(path + 'train.csv')#.sample(10000)\ndf_test = pd.read_csv(path + 'test.csv').sample(10000)\ndf_sub = pd.read_csv(path + 'sample_submission.csv').sample(10000)\ndf_org=pd.read_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv').sample(10000)\n\n# delete unnecessary columns\ndf=df.drop('id',axis=1)\n\n# dataset shape\nfor id_df, dataset in enumerate([df, df_test, df_sub, df_org]):\n # print(f\"dataset shape of dataframe {id_df} is: {dataset.shape}\")\n print(f\"Dataset {id_df} contains {dataset.shape[0]} rows and {dataset.shape[1]} columns.\")\nprint(\"=\"*80)\n\n\nmissing_columns = set(df.columns) - set(df_test.columns)\nprint(\"These columns are in df but are not in df_test:\", missing_columns)\ntoo_much_columns = set(df_test.columns) - set(df.columns) \nprint(\"These columns are in df_test but are not in df:\", too_much_columns)\nmissing_columns = set(df.columns) - set(df_org.columns)\nprint(\"These columns are in df but are not in df_org:\", missing_columns)\ntoo_much_columns = set(df_org.columns) - set(df.columns) \nprint(\"These columns are in df_org but are not in df:\", too_much_columns)\n\n#change order of columns to the same order as df\ndf_org = df_org[df.columns.tolist()] \n\n############################\n# Rename Columns\nfor dataset in [df, df_test, df_org]:\n\n #andere Überschriften geben\n dataset.rename(columns= {'Gender':'gender_male'}, inplace = True)\n\n # Replace spaces with underscores & make all column names lowercase\n dataset.columns = dataset.columns.str.replace(' ', '_').str.lower()\n\n############################\n# change data-type to Boolean\nfor dataset in [df, df_test, df_org]:\n dataset['gender_male'] = dataset['gender_male'].map({'Male': 1, 'Female': 0})\n dataset['smoking_status'] = dataset['smoking_status'].map({'Yes': 1, 'No': 0})\n \n############################\n# change display settings\npd.set_option('display.max_columns', None) #to show all columns\n\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T23:14:27.999182Z","iopub.execute_input":"2024-12-21T23:14:27.999599Z","iopub.status.idle":"2024-12-21T23:14:42.158562Z","shell.execute_reply.started":"2024-12-21T23:14:27.999562Z","shell.execute_reply":"2024-12-21T23:14:42.157084Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Dictionary\n\n| Variable Name | Description | Values | Variable Type |\n|--------------|-------------|---------|---------------|\n| age | | Range: 18-64 | continuous |\n| gender_male | | 0 (=female), 1 (=male) | nominal_boolean |\n| annual_income | Annual income in thousands |between 20 and 149847 | continuous |\n| marital_status | | Divorced, Married, Single | nominal |\n| number_of_dependents | | Range: 0-4 | discrete |\n| education_level | | Bachelor's < High School < Master's < PhD | ordinal |\n| occupation | Employment status | Employed, Self-Employed, Unemployed | nominal |\n| health_score | Overall health assessment | between 2 and 56 | continuous |\n| location | Residential area type | Rural, Suburban, Urban | nominal |\n| policy_type | Insurance coverage level | Basic < Comprehensive < Premium | ordinal |\n| previous_claims | Number of prior insurance claims | Range: 0-6 | discrete |\n| vehicle_age | Age of insured vehicle in years | Range: 0-19 | discrete |\n| credit_score | | between 300 and 849 | continuous |\n| insurance_duration | Length of insurance coverage in years | Range: 1-9 | discrete |\n| policy_start_date | Date and time when policy began | | |\n| customer_feedback | Customer satisfaction level | Poor < Average < Good | ordinal |\n| smoking_status | Binary indicator for smoker | 0 (=No), 1 (=Yes) | nominal_boolean |\n| exercise_frequency | Frequency of physical activity | Rarely < Monthly < Weekly < Daily | ordinal |\n| property_type | Type of insured property | Apartment, Condo, House | nominal |\n| premium_amount | | between 20 and 4978 | continuous |","metadata":{}},{"cell_type":"markdown","source":"| Variable Type | Variables |\n|--------------|-----------|\n| Date | policy_start_date |\n| Continuous | credit_score, annual_income, health_score |\n| Continuous, Target | premium_amount|\n| Discrete | number_of_dependents, insurance_duration, previous_claims, vehicle_age, age |\n| Nominal | location, property_type, marital_status, occupation |\n| Nominal (Boolean) | gender_male, smoking_status |\n| Ordinal | policy_type, customer_feedback , education_level , exercise_frequency |","metadata":{}},{"cell_type":"markdown","source":"# ordering values of ordinal variables 🗂️","metadata":{}},{"cell_type":"code","source":"ordinal_variable_order = {\n 'policy_type': ['Basic', 'Comprehensive', 'Premium'],\n 'customer_feedback': ['Poor', 'Average', 'Good'],\n 'education_level': ['High School', \"Bachelor's\", \"Master's\", 'PhD'],\n 'exercise_frequency': ['Rarely', 'Monthly', 'Weekly', 'Daily']\n}\n\n# set the ordering\nfor column, value_ordering in ordinal_variable_order.items():\n for dataset in [df, df_test]:\n dataset[column] = pd.Categorical(dataset[column], categories=value_ordering, ordered=True) \n\n# example. to see the change\nfor col in ordinal_variable_order.keys():\n print(df[col].unique(),\"\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T21:57:49.021242Z","iopub.execute_input":"2024-12-03T21:57:49.021732Z","iopub.status.idle":"2024-12-03T21:57:49.046814Z","shell.execute_reply.started":"2024-12-03T21:57:49.021686Z","shell.execute_reply":"2024-12-03T21:57:49.045683Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# changing datatyp ⚖️","metadata":{}},{"cell_type":"code","source":"for dataset in [df, df_test]:\n dataset['policy_start_date'] = pd.to_datetime(dataset['policy_start_date'], format='%Y-%m-%d %H:%M:%S.%f')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T22:03:59.543218Z","iopub.execute_input":"2024-12-03T22:03:59.543957Z","iopub.status.idle":"2024-12-03T22:03:59.564595Z","shell.execute_reply.started":"2024-12-03T22:03:59.543884Z","shell.execute_reply":"2024-12-03T22:03:59.563272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Datumsebene\n* Tag des Monats (day): Der Tag des Monats, z. B. 1 bis 31.\n* Wochentag (weekday): Der Wochentag als Zahl (0 für Montag bis 6 für Sonntag).\n* Name des Wochentags (weekday_name): Der vollständige Name des Wochentags, z. B. \"Monday\" oder \"Dienstag\".\n* Quartal (quarter): Das Quartal des Jahres, z. B. 1 bis 4.\n* Jahrestag (day_of_year): Die laufende Nummer des Tages im Jahr, z. B. 1 für den 1. Januar bis 365 (oder 366) für den letzten Tag des Jahres.\n* Kalenderwoche (week oder isocalendar().week): Die Woche im Jahr nach ISO-Standard (1 bis 52/53).\n* Ist es ein Schaltjahr? (is_leap_year): Ein boolescher Wert, der angibt, ob das Jahr ein Schaltjahr ist.\n* Tage im Monat (days_in_month): Die Anzahl der Tage im jeweiligen Monat (z. B. 28, 30 oder 31).\n\nZeitebene\n* Stunde (hour): Die Stunde des Tages, z. B. 0 bis 23.\n* Minute (minute): Die Minute der Stunde, z. B. 0 bis 59.\n* Sekunde (second): Die Sekunde der Minute, z. B. 0 bis 59.\n* Millisekunde (microsecond): Die Mikrosekunde, falls enthalten.\n* Zeit (time): Die Zeitkomponente allein ohne Datum, z. B. \"14:30:00\".\n\nWeitere Informationen\n* Zeitzone (tz): Falls vorhanden, die Zeitzoneninformation.\n* Epoch-Zeit (timestamp): Zeit in Sekunden seit dem 1. Januar 1970 (Unix-Zeitstempel).\n* ISO-Format (isoformat): Die Zeitangabe im ISO-8601-Format.\n\nZusätzliche Zeitmerkmale (benutzerdefiniert)\n* Arbeitszeit vs. Nicht-Arbeitszeit: Indikator dafür, ob die Zeit innerhalb normaler Arbeitszeiten liegt (z. B. 9:00–17:00).\n* Wochenende vs. Wochentag: Ein boolescher Indikator, ob der Tag ein Wochenende ist (Samstag/Sonntag).\n* Saison (Frühling, Sommer, Herbst, Winter): Basierend auf dem Monat oder spezifischen Datumsbereichen.\n* Halbjahr: Ob das Datum im ersten oder zweiten Halbjahr liegt.\n* Tageszeit (Morgen, Nachmittag, Abend, Nacht): Basierend auf der Stunde in z. B. Kategorien wie 0–6 (Nacht), 6–12 (Morgen), 12–18 (Nachmittag), 18–24 (Abend).","metadata":{}},{"cell_type":"code","source":"\n# **Datumsebene**\ndf['policy_start_year'] = df['policy_start_date'].dt.year # Jahr\ndf_test['policy_start_year'] = df_test['policy_start_date'].dt.year\n\ndf['policy_start_month'] = df['policy_start_date'].dt.month # Monat\ndf_test['policy_start_month'] = df_test['policy_start_date'].dt.month\n\ndf['policy_start_day'] = df['policy_start_date'].dt.day # Tag des Monats\ndf_test['policy_start_day'] = df_test['policy_start_date'].dt.day\n\ndf['policy_start_weekday'] = df['policy_start_date'].dt.weekday # Wochentag (0=Montag)\ndf_test['policy_start_weekday'] = df_test['policy_start_date'].dt.weekday\n\ndf['policy_start_weekday_name'] = df['policy_start_date'].dt.day_name() # Name des Wochentags\ndf_test['policy_start_weekday_name'] = df_test['policy_start_date'].dt.day_name()\n\ndf['policy_start_quarter'] = df['policy_start_date'].dt.quarter # Quartal\ndf_test['policy_start_quarter'] = df_test['policy_start_date'].dt.quarter\n\ndf['policy_start_day_of_year'] = df['policy_start_date'].dt.dayofyear # Tag des Jahres\ndf_test['policy_start_day_of_year'] = df_test['policy_start_date'].dt.dayofyear\n\ndf['policy_start_week'] = df['policy_start_date'].dt.isocalendar().week # Kalenderwoche\ndf_test['policy_start_week'] = df_test['policy_start_date'].dt.isocalendar().week\n\n\n# **Zeitebene**\ndf['policy_start_hour'] = df['policy_start_date'].dt.hour # Stunde\ndf_test['policy_start_hour'] = df_test['policy_start_date'].dt.hour\n\ndf['policy_start_minute'] = df['policy_start_date'].dt.minute # Minute\ndf_test['policy_start_minute'] = df_test['policy_start_date'].dt.minute\n\ndf['policy_start_time'] = df['policy_start_date'].dt.time # Nur die Zeit\ndf_test['policy_start_time'] = df_test['policy_start_date'].dt.time\n\n# **Zusätzliche Zeitmerkmale**\ndf['policy_start_is_weekend'] = df['policy_start_weekday'].apply(lambda x: x in [5, 6]) # Wochenende\ndf_test['policy_start_is_weekend'] = df_test['policy_start_weekday'].apply(lambda x: x in [5, 6])\n\ndf['policy_start_season'] = df['policy_start_month'].apply(lambda x: \n 'Spring' if x in [3, 4, 5] else \n 'Summer' if x in [6, 7, 8] else \n 'Autumn' if x in [9, 10, 11] else \n 'Winter')\ndf_test['policy_start_season'] = df_test['policy_start_month'].apply(lambda x: \n 'Spring' if x in [3, 4, 5] else \n 'Summer' if x in [6, 7, 8] else \n 'Autumn' if x in [9, 10, 11] else \n 'Winter')\n\ndf['policy_start_half_year'] = df['policy_start_month'].apply(lambda x: 1 if x <= 6 else 2) # Halbjahr\ndf_test['policy_start_half_year'] = df_test['policy_start_month'].apply(lambda x: 1 if x <= 6 else 2)\n\ndf['policy_start_part_of_day'] = df['policy_start_hour'].apply(lambda x: # Tageszeit\n 'Night' if x < 6 else \n 'Morning' if x < 12 else \n 'Afternoon' if x < 18 else \n 'Evening')\ndf_test['policy_start_part_of_day'] = df_test['policy_start_hour'].apply(lambda x: \n 'Night' if x < 6 else \n 'Morning' if x < 12 else \n 'Afternoon' if x < 18 else \n 'Evening')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T22:11:42.621346Z","iopub.execute_input":"2024-12-03T22:11:42.622452Z","iopub.status.idle":"2024-12-03T22:11:42.70468Z","shell.execute_reply.started":"2024-12-03T22:11:42.622412Z","shell.execute_reply":"2024-12-03T22:11:42.703546Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# basic EDA 🔬","metadata":{}},{"cell_type":"code","source":"# Berechnet den maximalen Wert einer Spalte, wenn sie numerisch ist\ndef max_value(column):\n if pd.api.types.is_numeric_dtype(column): # Überprüfe, ob der Datentyp numerisch ist\n return column.dropna().max() if not column.dropna().empty else np.nan\n return \"\"\n\n\n# Gibt die einzigartigen Werte einer Spalte zurück, oder eine Range (falls es eine gibt)\ndef get_unique_values(column):\n if pd.api.types.is_integer_dtype(column): # Überprüfe, ob der Datentyp eine Ganzzahl ist\n unique_vals = sorted(set(column.dropna()))\n min_val, max_val = column.min(), column.max()\n if unique_vals == list(range(min_val, max_val + 1)):\n# return f\"range({min_val},{max_val + 1})\"\n return f\"(range:{min_val}-{max_val})\"\n return unique_vals\n# return f\"between {min_val} and {max_val}\"\n return sorted(set(column.dropna()))\n\n\ndef summary(df=df):\n summary_df = pd.DataFrame({\n 'dtypes': df.dtypes.astype(str),\n# 'missing data': df.isna().sum(),\n 'missing count': df.isna().sum(),\n# 'missing data percent': round(df.isna().sum() / df.shape[0],2),\n 'missing ratio': round(df.isna().sum() / df.shape[0],2),\n# 'unique values': [get_unique_values(df[col]) for col in df.columns],\n 'values': [get_unique_values(df[col]) for col in df.columns],\n# 'unique values max': [max_value(df[col]) for col in df.columns],\n 'values max': [max_value(df[col]) for col in df.columns],\n# 'z Cardinality': df.nunique() #datatype float if NaN in df\n 'unique': df.nunique() #datatype float if NaN in df\n })\n return summary_df\n\n\ndef prepare_and_merge(base_df, additional_df, name_df):\n \"\"\"Bereitet den Merge zwischen base_df und additional_df vor und gibt das gemergte DataFrame zurück.\"\"\"\n \n summary_additional = summary(additional_df)\n\n # Fehlende Spalten in additional_df hinzufügen\n missing_columns = set(base_df.index.tolist()) - set(additional_df.columns)\n for col in missing_columns:\n# summary_additional.loc[col] = np.nan\n summary_additional.loc[col] = \"\"\n \n # Überzählige Spalten in summary_additional entfernen # for example column ID\n too_much_columns = set(additional_df.columns) - set(base_df.index.tolist())\n # summary_additional = summary_additional.drop(index=too_much_columns)\n summary_additional = summary_additional.drop(too_much_columns, axis=0)\n \n # Spaltennamen umbenennen damit keine Verwechslung mit base_df kommt\n summary_additional.columns = [f\"{col} ({name_df})\" for col in summary_additional.columns]\n \n # Zusammenführen der Zusammenfassung-DataFrames\n summary_df = pd.concat([base_df, summary_additional], axis=1)\n\n # Spalten alphabetisch sortieren damit nicht base_df vorne und additional_df hinten\n summary_df = summary_df.reindex(sorted(summary_df.columns), axis=1)\n\n return summary_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:03:24.911678Z","iopub.execute_input":"2024-12-01T16:03:24.912609Z","iopub.status.idle":"2024-12-01T16:03:24.924039Z","shell.execute_reply.started":"2024-12-01T16:03:24.912566Z","shell.execute_reply":"2024-12-01T16:03:24.922798Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# change display settings if there are many columns in df\npd.set_option(\"display.max_rows\", df.shape[1]+1) #zeige 81 Zeilen. wenn >81 \"normale\" Anzeige\n\nsummary_df = prepare_and_merge(summary(df), df_test,'test')\n\nsummary_df.sort_values(by=['dtypes', 'unique','missing count'])\n#summary_df#.sort_values(by=['missing data percent'], ascending = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:03:25.205146Z","iopub.execute_input":"2024-12-01T16:03:25.205536Z","iopub.status.idle":"2024-12-01T16:03:25.362059Z","shell.execute_reply.started":"2024-12-01T16:03:25.205497Z","shell.execute_reply":"2024-12-01T16:03:25.361065Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Checking for Duplicate","metadata":{}},{"cell_type":"code","source":"\n#Checking for Duplicates\nfor id_df, dataset in enumerate([df, df_test]):\n print(f\"Number of Duplicates in dataframe {id_df}: {dataset.duplicated().sum()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T16:18:01.633476Z","iopub.execute_input":"2024-12-01T16:18:01.633893Z","iopub.status.idle":"2024-12-01T16:18:01.672646Z","shell.execute_reply.started":"2024-12-01T16:18:01.633822Z","shell.execute_reply":"2024-12-01T16:18:01.671413Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# visualize the missing values 🗑️","metadata":{}},{"cell_type":"code","source":"#for id_df, dataset in enumerate([df, df_test]):\n # Columns with missing values\n# na_cols=dataset.columns[dataset.isna().any()].tolist()\n\n # Heatmap of missing values for df\n# plt.figure(figsize=(16,4))\n# sns.heatmap(dataset[na_cols].isna().T, cmap='summer')\n# plt.title(f'Heatmap of missing values in dataframe {id_df}')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T08:25:44.650557Z","iopub.execute_input":"2024-12-01T08:25:44.65102Z","iopub.status.idle":"2024-12-01T08:25:44.655711Z","shell.execute_reply.started":"2024-12-01T08:25:44.650981Z","shell.execute_reply":"2024-12-01T08:25:44.65449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Correlation Matrix","metadata":{}},{"cell_type":"code","source":"\n# Correlation including Boolean-nominal + ordinal variables \n# not included are the nominal variables & datatype\n\ncorr_columns = df.select_dtypes(include=[np.number, 'category','bool']).columns.tolist() \ndf_corr_ordinal = df[corr_columns].copy()\n\n# encoding for ordinal variables based on defined order\nordinal_variable_order = {\n 'policy_type': ['Basic', 'Comprehensive', 'Premium'],\n 'customer_feedback': ['Poor', 'Average', 'Good'],\n 'education_level': ['High School', \"Bachelor's\", \"Master's\", 'PhD'],\n 'exercise_frequency': ['Rarely', 'Monthly', 'Weekly', 'Daily']\n}\nfor column, column_ordering in ordinal_variable_order.items():\n mapping = {category: idx for idx, category in enumerate(column_ordering)}\n df_corr_ordinal[column] = df[column].map(mapping)\n\n# delete upper diagonal matrix\nmask = np.triu(np.ones_like(df_corr_ordinal.corr(), dtype=bool))\n\nplt.figure(figsize=(21, 25)) \nsns.heatmap(df_corr_ordinal.corr(),fmt = '.2f', cmap=\"seismic\", annot=True, mask=mask,vmax=1,vmin=-1)\nplt.title('Correlation Matrix')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T22:12:12.90749Z","iopub.execute_input":"2024-12-03T22:12:12.907884Z","iopub.status.idle":"2024-12-03T22:12:14.280015Z","shell.execute_reply.started":"2024-12-03T22:12:12.907848Z","shell.execute_reply":"2024-12-03T22:12:14.278186Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA 📊","metadata":{}},{"cell_type":"code","source":"variables = df.select_dtypes(include='number').columns.tolist()\n\n# Dictionary mit benutzerdefinierten Bins für Histogramm für Variablen die standardmäßig nunique bins bekommen sollen\ncol = ['gender_male', 'smoking_status', 'number_of_dependents', 'insurance_duration', 'previous_claims']\ncustom_bins = df[col].nunique().to_dict() #anzahl der einzigartiges Values\n\nplot_quantitative_var_distributions(df, variables, custom_bins)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T08:25:45.632102Z","iopub.status.idle":"2024-12-01T08:25:45.63247Z","shell.execute_reply.started":"2024-12-01T08:25:45.632304Z","shell.execute_reply":"2024-12-01T08:25:45.632321Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = 'premium_amount'\n\nnominal_features = ['location','property_type', 'marital_status', 'occupation', 'gender_male', 'smoking_status']\nordinal_variable_order = {\n 'policy_type': ['Basic', 'Comprehensive', 'Premium'],\n 'customer_feedback': ['Poor', 'Average', 'Good'],\n 'education_level': ['High School', \"Bachelor's\", \"Master's\", 'PhD'],\n 'exercise_frequency': ['Rarely', 'Monthly', 'Weekly', 'Daily']\n}\n\ncategorical_variables = list(ordinal_variable_order.keys()) + nominal_features\n\nfor cat_var in categorical_variables:\n plot_quantitative_qualitative_distributions(df, target, cat_var, ordinal_variable_order)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T08:25:45.633904Z","iopub.status.idle":"2024-12-01T08:25:45.634294Z","shell.execute_reply.started":"2024-12-01T08:25:45.634082Z","shell.execute_reply":"2024-12-01T08:25:45.634099Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**I always appreciate Feedback because I want to improve. Feel free to add any of your thoughts below. Thank you!**","metadata":{}}]}