{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sb\nimport matplotlib.pyplot as plt\n\nfrom scipy.stats import pearsonr, skew, kurtosis\n\nfrom prettytable import PrettyTable\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        FILE_PATH = os.path.join(dirname, filename)\n\n        if 'train' in filename:\n            TRAIN_PATH = FILE_PATH\n\n        elif 'test' in filename:\n            TEST_PATH = FILE_PATH\n\n        else:\n            SUBMISSION_PATH = FILE_PATH\n\npd.set_option('display.max_columns', None)\n\nTARGET_FEATURE = 'Premium Amount'\nSAMPLE_SIZE = 600000","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:18:52.964067Z","iopub.execute_input":"2025-10-06T07:18:52.964408Z","iopub.status.idle":"2025-10-06T07:18:52.975826Z","shell.execute_reply.started":"2025-10-06T07:18:52.964384Z","shell.execute_reply":"2025-10-06T07:18:52.974698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_PATH)\ntest = pd.read_csv(TEST_PATH)\nsubmission = pd.read_csv(SUBMISSION_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:18:52.977441Z","iopub.execute_input":"2025-10-06T07:18:52.977774Z","iopub.status.idle":"2025-10-06T07:19:00.843363Z","shell.execute_reply.started":"2025-10-06T07:18:52.977749Z","shell.execute_reply":"2025-10-06T07:19:00.84229Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Memory Optimization","metadata":{}},{"cell_type":"code","source":"def reduce_memory_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f\"Memory usage of dataframe is {start_mem:.2f} MB\")\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        \n        if col_type != object:\n            if pd.api.types.is_float_dtype(col_type):\n                df[col] = pd.to_numeric(df[col], downcast='float')\n            elif pd.api.types.is_integer_dtype(col_type):\n                df[col] = pd.to_numeric(df[col], downcast='integer')\n        else:\n            num_unique_values = df[col].nunique()\n            num_total_values = len(df[col])\n            if num_unique_values / num_total_values < 0.5:\n                df[col] = df[col].astype('category')\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f\"Memory usage after optimization is: {end_mem:.2f} MB\")\n    reduction_percentage = ((start_mem - end_mem) / start_mem) * 100\n    print(f\"Reduction in memory usage: {reduction_percentage:.1f}%\")\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:00.844766Z","iopub.execute_input":"2025-10-06T07:19:00.845053Z","iopub.status.idle":"2025-10-06T07:19:00.853767Z","shell.execute_reply.started":"2025-10-06T07:19:00.845032Z","shell.execute_reply":"2025-10-06T07:19:00.852673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = reduce_memory_usage(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:00.854799Z","iopub.execute_input":"2025-10-06T07:19:00.85521Z","iopub.status.idle":"2025-10-06T07:19:04.049589Z","shell.execute_reply.started":"2025-10-06T07:19:00.855185Z","shell.execute_reply":"2025-10-06T07:19:04.048261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop(columns = 'id', inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:04.051797Z","iopub.execute_input":"2025-10-06T07:19:04.052167Z","iopub.status.idle":"2025-10-06T07:19:04.094811Z","shell.execute_reply.started":"2025-10-06T07:19:04.052143Z","shell.execute_reply":"2025-10-06T07:19:04.093728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### First Look at Data","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:04.09566Z","iopub.execute_input":"2025-10-06T07:19:04.096015Z","iopub.status.idle":"2025-10-06T07:19:04.286281Z","shell.execute_reply.started":"2025-10-06T07:19:04.095985Z","shell.execute_reply":"2025-10-06T07:19:04.284987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_info(dataframe: pd.core.frame.DataFrame):\n    \n    \"\"\"\n    This function takes a dataframe as input and\n    returns a short summary.\n    \"\"\"\n    print(f\"Total Records: {dataframe.shape[0]}\")\n    print(f\"Total Features: {dataframe.shape[1]}\")\n    print(f\"Total Duplicate Records: {dataframe.duplicated().sum()}\")\n    info = PrettyTable()\n    info.field_names = ['Column', \n                        'Data Type', \n                        'Missing Values', \n                        'Missing Percentage', \n                        'Unique Values', \n                       'Percentage Unique']\n    \n    for column in dataframe.columns:\n        \n        data_type = dataframe[column].dtypes\n        missing_values = dataframe[column].isnull().sum()\n        missing_percentage = np.round(100 * dataframe[column].isnull().sum() / len(dataframe), 2)\n        unique_values = dataframe[column].nunique()\n        percentage_unique = np.round(100 * dataframe[column].nunique() / len(dataframe), 2)\n        \n        info.add_row([column, data_type, missing_values, missing_percentage, unique_values, percentage_unique])\n    \n    print(info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:04.287253Z","iopub.execute_input":"2025-10-06T07:19:04.28751Z","iopub.status.idle":"2025-10-06T07:19:04.296717Z","shell.execute_reply.started":"2025-10-06T07:19:04.28749Z","shell.execute_reply":"2025-10-06T07:19:04.295525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"get_info(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:04.297919Z","iopub.execute_input":"2025-10-06T07:19:04.298396Z","iopub.status.idle":"2025-10-06T07:19:05.783314Z","shell.execute_reply.started":"2025-10-06T07:19:04.298362Z","shell.execute_reply":"2025-10-06T07:19:05.782019Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### Insights:\n\n - Some features have < 10% missing values\n - **Occupation** has almost 30% missing values.\n - All the categorical features have less than 5 categories.\n - **Policy Start Date** is category type, instead of being date type","metadata":{}},{"cell_type":"code","source":"train['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:05.784487Z","iopub.execute_input":"2025-10-06T07:19:05.784834Z","iopub.status.idle":"2025-10-06T07:19:06.476147Z","shell.execute_reply.started":"2025-10-06T07:19:05.784804Z","shell.execute_reply":"2025-10-06T07:19:06.475046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:06.477153Z","iopub.execute_input":"2025-10-06T07:19:06.477397Z","iopub.status.idle":"2025-10-06T07:19:07.236015Z","shell.execute_reply.started":"2025-10-06T07:19:06.477378Z","shell.execute_reply":"2025-10-06T07:19:07.235173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##### Insights:\n\n - Average age is 41\n - Annual Income seems to be right skewed given the Q1 has a high value.","metadata":{}},{"cell_type":"markdown","source":"### Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"In this data, we even have people having annual income as 1. To combat this, we'll only go ahead with people who have an annual income of minimum 5000 dollars.","metadata":{}},{"cell_type":"code","source":"train['Annual Income'].min(), train['Annual Income'].max()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:07.239584Z","iopub.execute_input":"2025-10-06T07:19:07.239902Z","iopub.status.idle":"2025-10-06T07:19:07.250212Z","shell.execute_reply.started":"2025-10-06T07:19:07.239879Z","shell.execute_reply":"2025-10-06T07:19:07.249013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train[train['Annual Income'] >= 5000]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:07.25124Z","iopub.execute_input":"2025-10-06T07:19:07.251659Z","iopub.status.idle":"2025-10-06T07:19:07.35285Z","shell.execute_reply.started":"2025-10-06T07:19:07.251627Z","shell.execute_reply":"2025-10-06T07:19:07.351873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Total Records: {len(train)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:07.353633Z","iopub.execute_input":"2025-10-06T07:19:07.35389Z","iopub.status.idle":"2025-10-06T07:19:07.359715Z","shell.execute_reply.started":"2025-10-06T07:19:07.353868Z","shell.execute_reply":"2025-10-06T07:19:07.358409Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Since the data contains 1M records, we'll draw 600k records randomly so execution becomes fast.","metadata":{}},{"cell_type":"code","source":"train_sample = train.sample(n = SAMPLE_SIZE, random_state = 4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:07.360814Z","iopub.execute_input":"2025-10-06T07:19:07.361138Z","iopub.status.idle":"2025-10-06T07:19:07.513754Z","shell.execute_reply.started":"2025-10-06T07:19:07.361115Z","shell.execute_reply":"2025-10-06T07:19:07.512614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Distribution of Target Variable","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (12, 6))\nsb.kdeplot(train_sample['Premium Amount'])\n\nplt.title(f'Distribution of {TARGET_FEATURE}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:07.514786Z","iopub.execute_input":"2025-10-06T07:19:07.515073Z","iopub.status.idle":"2025-10-06T07:19:10.632491Z","shell.execute_reply.started":"2025-10-06T07:19:07.515051Z","shell.execute_reply":"2025-10-06T07:19:10.631275Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Insights:\n\n - The target feature seems to be right skewed and bimodal.\n - Majority of customers have less premium amount and few of them had a higher value leading to right skewed data.","metadata":{}},{"cell_type":"code","source":"from scipy.stats import skew, kurtosis\n\nskewness = skew(train_sample[TARGET_FEATURE]).round(2)\nkurtosis = kurtosis(train_sample[TARGET_FEATURE]).round(2)\n\nprint(f\"Skewness: {skewness}\")\nprint(f\"Kurtosis: {kurtosis}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:10.633705Z","iopub.execute_input":"2025-10-06T07:19:10.634059Z","iopub.status.idle":"2025-10-06T07:19:10.657097Z","shell.execute_reply.started":"2025-10-06T07:19:10.63403Z","shell.execute_reply":"2025-10-06T07:19:10.65608Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Feature Distributions","metadata":{}},{"cell_type":"code","source":"def plot_feature_distributions(df, kind: str) -> None:\n\n    \"\"\"\n    This function plots the feature distribution from the\n    given dataframe\n\n    Args:\n        df(pd.core.frame.dataframe): The dataframe\n        kind(str): The type of plot\n\n    Raises: \n        Exception: When provided plot is not among the options\n    \"\"\"\n    \n    if kind in ['kde', 'box']:\n        features = df.select_dtypes(include = np.number).columns\n        \n    elif kind in ['count']:\n        features = df.select_dtypes(include = ['object', 'category']).columns\n\n    else:\n        raise Exception(\"Invalid plot type! Expected values are 'kind', 'box', 'count'\")\n    \n    num_features = len(features)\n    num_rows = (num_features // 3) + 1\n    plt.figure(figsize=(15, num_rows * 5))\n\n    for i, column in enumerate(features):\n        plt.subplot(num_rows, 3, i + 1)\n        \n        if kind == 'kde':\n            sb.kdeplot(x = df[column])\n            \n        elif kind == 'box':\n            sb.boxplot(x = df[column])\n            \n        elif kind == 'count':\n            sb.countplot(x = df[column])\n            plt.xticks(rotation = 60)\n            \n        plt.title(f'Distribution of {column}')\n        plt.xlabel(column)\n        plt.ylabel('Frequency')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:10.658424Z","iopub.execute_input":"2025-10-06T07:19:10.658768Z","iopub.status.idle":"2025-10-06T07:19:10.668798Z","shell.execute_reply.started":"2025-10-06T07:19:10.658736Z","shell.execute_reply":"2025-10-06T07:19:10.667731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_feature_distributions(train_sample, kind = 'kde')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:10.670006Z","iopub.execute_input":"2025-10-06T07:19:10.670881Z","iopub.status.idle":"2025-10-06T07:19:33.917438Z","shell.execute_reply.started":"2025-10-06T07:19:10.670847Z","shell.execute_reply":"2025-10-06T07:19:33.916309Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Insights:\n\n - **Age** feature can be practically treated as **uniform** distribution.\n - **Annual Income** is highly right skewed and hence contains lots of outliers.\n - **Number of Dependents**, **Previous Claims**, **Vehicle Age**, **Insurance Duration** are behaving like categorical features.","metadata":{}},{"cell_type":"code","source":"plot_feature_distributions(train_sample, kind = 'count')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:33.918523Z","iopub.execute_input":"2025-10-06T07:19:33.918803Z","iopub.status.idle":"2025-10-06T07:19:36.106706Z","shell.execute_reply.started":"2025-10-06T07:19:33.91878Z","shell.execute_reply":"2025-10-06T07:19:36.105571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Insights:\n\n - All features are uniformly distributed.","metadata":{}},{"cell_type":"markdown","source":"#### Missing value analysis","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\n\nsb.heatmap(train_sample.isnull())\n\nplt.title(\"Heatmap for missing values\")\nplt.xlabel(\"Features\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:36.108203Z","iopub.execute_input":"2025-10-06T07:19:36.10859Z","iopub.status.idle":"2025-10-06T07:19:48.788139Z","shell.execute_reply.started":"2025-10-06T07:19:36.108564Z","shell.execute_reply":"2025-10-06T07:19:48.786991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def handle_missing_values(df):\n\n    data_copy = df.copy()\n    for feature in data_copy.columns:\n        if data_copy[feature].dtype in ['category', 'object']:\n            data_copy[feature] = data_copy[feature].fillna(data_copy[feature].mode()[0])\n\n        else:\n            data_copy[feature] = data_copy[feature].fillna(data_copy[feature].median())\n\n    return data_copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:48.788909Z","iopub.execute_input":"2025-10-06T07:19:48.789196Z","iopub.status.idle":"2025-10-06T07:19:48.796425Z","shell.execute_reply.started":"2025-10-06T07:19:48.789175Z","shell.execute_reply":"2025-10-06T07:19:48.794966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dataset in [train_sample, train, test]:\n    dataset = handle_missing_values(dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:48.79795Z","iopub.execute_input":"2025-10-06T07:19:48.798268Z","iopub.status.idle":"2025-10-06T07:19:51.268846Z","shell.execute_reply.started":"2025-10-06T07:19:48.79824Z","shell.execute_reply":"2025-10-06T07:19:51.267563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dataset in [train, train_sample]:\n    dataset['Year'] = dataset['Policy Start Date'].dt.year\n    dataset['Month'] = dataset['Policy Start Date'].dt.month\n    dataset['Day'] = dataset['Policy Start Date'].dt.day\n    dataset.drop(columns = ['Policy Start Date'], inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:51.270346Z","iopub.execute_input":"2025-10-06T07:19:51.271076Z","iopub.status.idle":"2025-10-06T07:19:51.653564Z","shell.execute_reply.started":"2025-10-06T07:19:51.271043Z","shell.execute_reply":"2025-10-06T07:19:51.652393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:51.654831Z","iopub.execute_input":"2025-10-06T07:19:51.655213Z","iopub.status.idle":"2025-10-06T07:19:51.687306Z","shell.execute_reply.started":"2025-10-06T07:19:51.655182Z","shell.execute_reply":"2025-10-06T07:19:51.68643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features = train_sample.select_dtypes(include=['category', 'object']).columns\n\nfor feature in categorical_features:\n    mean_encoding = train_sample.groupby(feature)[TARGET_FEATURE].mean()\n    train_sample[feature] = train_sample[feature].map(mean_encoding)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:51.688222Z","iopub.execute_input":"2025-10-06T07:19:51.688529Z","iopub.status.idle":"2025-10-06T07:19:51.794336Z","shell.execute_reply.started":"2025-10-06T07:19:51.688505Z","shell.execute_reply":"2025-10-06T07:19:51.793184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_df = train_sample.corr()\n\nsb.heatmap(correlation_df)\nplt.title(\"Correlation Heatmap\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:51.79563Z","iopub.execute_input":"2025-10-06T07:19:51.796029Z","iopub.status.idle":"2025-10-06T07:19:53.507836Z","shell.execute_reply.started":"2025-10-06T07:19:51.795995Z","shell.execute_reply":"2025-10-06T07:19:53.506814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Top 5 correlated features\ncorrelation_df['Premium Amount'].abs().nlargest(6)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:53.508738Z","iopub.execute_input":"2025-10-06T07:19:53.509013Z","iopub.status.idle":"2025-10-06T07:19:53.51843Z","shell.execute_reply.started":"2025-10-06T07:19:53.508994Z","shell.execute_reply":"2025-10-06T07:19:53.517415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Outlier Treatment","metadata":{}},{"cell_type":"code","source":"def cap_all_numerical_features_iqr(df, factor=1.5) -> pd.core.frame.DataFrame:\n    \"\"\"\n    Identifies all numerical columns in a DataFrame and performs \n    outlier capping on each one using the IQR method.\n\n    Args:\n        df (pd.DataFrame): The input DataFrame (e.g., train_sample).\n        factor (float): The multiplication factor for the IQR (default is 1.5,\n                        which typically defines outliers).\n\n    Returns:\n        pd.DataFrame: A new DataFrame with all numerical features capped.\n    \"\"\"\n    df_capped = df.copy()\n    numerical_features = df.select_dtypes(include=np.number).columns\n    \n    capping_summary = {}\n\n    for column_name in numerical_features:\n        data = df_capped[column_name].dropna()\n        if len(data.unique()) < 4:\n            print(f\"Skipping '{column_name}': Too few unique values.\")\n            continue\n\n        Q1 = data.quantile(0.25)\n        Q3 = data.quantile(0.75)\n        IQR = Q3 - Q1\n\n        lower_bound = Q1 - (factor * IQR)\n        upper_bound = Q3 + (factor * IQR)\n\n        original_outlier_count = (df_capped[column_name] < lower_bound).sum() + (df_capped[column_name] > upper_bound).sum()\n        \n        if original_outlier_count > 0:\n            df_capped[column_name] = np.clip(\n                df_capped[column_name], \n                lower_bound, \n                upper_bound\n            )\n\n            \n        capping_summary[column_name] = {\n            \"Lower Bound\": round(lower_bound, 2),\n            \"Upper Bound\": round(upper_bound, 2),\n            \"Outliers Capped\": original_outlier_count\n        }\n    \n    return df_capped, pd.DataFrame.from_dict(capping_summary, orient='index')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:53.519848Z","iopub.execute_input":"2025-10-06T07:19:53.520328Z","iopub.status.idle":"2025-10-06T07:19:53.545679Z","shell.execute_reply.started":"2025-10-06T07:19:53.520299Z","shell.execute_reply":"2025-10-06T07:19:53.5445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample, capping_report = cap_all_numerical_features_iqr(train_sample)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:53.550139Z","iopub.execute_input":"2025-10-06T07:19:53.550688Z","iopub.status.idle":"2025-10-06T07:19:54.164234Z","shell.execute_reply.started":"2025-10-06T07:19:53.550663Z","shell.execute_reply":"2025-10-06T07:19:54.162883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (12, 5))\n\nsb.barplot(\n    data = capping_report.sort_values(by = 'Outliers Capped', ascending = False), \n    x = capping_report.index, \n    y = 'Outliers Capped'\n)\nplt.title(\"Outliers by features\")\nplt.xticks(rotation = 60)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:54.165118Z","iopub.execute_input":"2025-10-06T07:19:54.165406Z","iopub.status.idle":"2025-10-06T07:19:54.451994Z","shell.execute_reply.started":"2025-10-06T07:19:54.165385Z","shell.execute_reply":"2025-10-06T07:19:54.450602Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Feature Engineering","metadata":{}},{"cell_type":"code","source":"train['IsCovidYear'] = np.where(\n    train['Year'] == 2020,\n    1,\n    0\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:54.453364Z","iopub.execute_input":"2025-10-06T07:19:54.453644Z","iopub.status.idle":"2025-10-06T07:19:54.466657Z","shell.execute_reply.started":"2025-10-06T07:19:54.453623Z","shell.execute_reply":"2025-10-06T07:19:54.465661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def target_guided_encoding(train, test, target_feature, categorical_features=None, fillna_value=None):\n    \"\"\"\n    Apply target-guided encoding to categorical features in train and test datasets.\n    \n    Parameters:\n        train (pd.DataFrame): Training dataset\n        test (pd.DataFrame): Test dataset\n        target_feature (str): Name of the target column\n        categorical_features (list, optional): List of categorical feature names.\n        fillna_value (float, optional): Value to fill NaNs in test data. If None, uses mean of target_feature from train\n    \n    Returns:\n        train (pd.DataFrame): Training data with encoded features\n        test (pd.DataFrame): Test data with encoded features\n    \"\"\"\n    if categorical_features is None:\n        categorical_features = train.select_dtypes(include=['category', 'object']).columns\n\n    if fillna_value is None:\n        fillna_value = train[target_feature].mean()\n    \n    encoding_maps = {}\n    \n    for feature in categorical_features:\n        mean_encoding = train.groupby(feature)[target_feature].mean()\n        encoding_maps[feature] = mean_encoding\n        train[feature] = train[feature].map(mean_encoding)\n    \n    for feature in categorical_features:\n        test[feature] = test[feature].map(encoding_maps[feature]).fillna(fillna_value)\n    \n    return train, test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:54.467762Z","iopub.execute_input":"2025-10-06T07:19:54.468068Z","iopub.status.idle":"2025-10-06T07:19:54.492756Z","shell.execute_reply.started":"2025-10-06T07:19:54.468045Z","shell.execute_reply":"2025-10-06T07:19:54.491482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train, test = target_guided_encoding(train, test, target_feature = TARGET_FEATURE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:54.49414Z","iopub.execute_input":"2025-10-06T07:19:54.494512Z","iopub.status.idle":"2025-10-06T07:19:55.249978Z","shell.execute_reply.started":"2025-10-06T07:19:54.494484Z","shell.execute_reply":"2025-10-06T07:19:55.248963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from statsmodels.stats.outliers_influence import variance_inflation_factor\n\ndef calculate_vif(train, target_feature=None):\n\n    numeric_features = train.select_dtypes(include=[np.number]).columns\n    if target_feature and target_feature in numeric_features:\n        numeric_features = numeric_features.drop(target_feature)\n    \n    vif_data = pd.DataFrame()\n    vif_data['Feature'] = numeric_features\n    X = train[numeric_features].copy()\n\n    X = X.fillna(0)\n    vif_data['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]\n    \n    return vif_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:55.250994Z","iopub.execute_input":"2025-10-06T07:19:55.25138Z","iopub.status.idle":"2025-10-06T07:19:55.25855Z","shell.execute_reply.started":"2025-10-06T07:19:55.251357Z","shell.execute_reply":"2025-10-06T07:19:55.257385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vif_report = calculate_vif(train, TARGET_FEATURE)\nvif_report","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:19:55.259545Z","iopub.execute_input":"2025-10-06T07:19:55.259852Z","iopub.status.idle":"2025-10-06T07:20:05.153842Z","shell.execute_reply.started":"2025-10-06T07:19:55.259824Z","shell.execute_reply":"2025-10-06T07:20:05.152452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"useful_features = vif_report[vif_report['VIF'] < 10]['Feature']\ntrain = train[useful_features]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:20:05.154841Z","iopub.execute_input":"2025-10-06T07:20:05.155132Z","iopub.status.idle":"2025-10-06T07:20:05.194653Z","shell.execute_reply.started":"2025-10-06T07:20:05.155111Z","shell.execute_reply":"2025-10-06T07:20:05.193508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-06T07:20:05.195738Z","iopub.execute_input":"2025-10-06T07:20:05.196231Z","iopub.status.idle":"2025-10-06T07:20:05.222543Z","shell.execute_reply.started":"2025-10-06T07:20:05.196196Z","shell.execute_reply":"2025-10-06T07:20:05.220967Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## To be continued...","metadata":{}}]}