{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective","metadata":{}},{"cell_type":"markdown","source":"The objective of this notebook is to make linearly my schema of thought before modelling and in modelling. I hope you like my tools ideas and I hope you to give me new ideas as well ! :).","metadata":{}},{"cell_type":"code","source":"!pip install feature-engine\n!pip install catboost\n!pip install seaborn\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\n\nimport os\nimport math\nimport itertools\nfrom itertools import combinations\n\nimport numpy as np  # linear algebra\nimport pandas as pd  # data processing\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split, KFold, GridSearchCV\nfrom sklearn.metrics import root_mean_squared_error, pairwise_distances\nfrom sklearn.impute import KNNImputer\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.preprocessing import RobustScaler\nimport sklearn\n\nfrom catboost import CatBoostRegressor, Pool\n\nfrom feature_engine.creation import MathFeatures, RelativeFeatures\n\nfrom IPython.display import display, HTML\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read the Data","metadata":{}},{"cell_type":"code","source":"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/train.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Global variables\n","metadata":{}},{"cell_type":"code","source":"target_column = 'logY'  # Replace with your target column name\nF_HYPERPARAMETER= False","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Distribution of the Y variable","metadata":{}},{"cell_type":"code","source":"train_data[\"Premium Amount\"].hist()\nprint(f\"mean: {train_data['Premium Amount'].mean()}\")\nprint(f\"sd: {train_data['Premium Amount'].std()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The Y variable is very skewed to the left. Therefore, and given the fact that it is said that the metric to evaluate the error is the Root Mean Squared Logarithmic Error (RMSLE), if I log my objective variable, then I can use directly the RMSE formula in my objective function.","metadata":{}},{"cell_type":"markdown","source":"# Basic feature Engineering and Analysis","metadata":{},"attachments":{}},{"cell_type":"markdown","source":"As a matter of a fact, I know that tree based algorithms, such as Catboost are not good in understanding timestamps. Therefore, I am gonna create the typical variables from timestamps. ","metadata":{}},{"cell_type":"code","source":"numerical_columns = train_data.select_dtypes(include=['number']).columns.tolist()\nprint(numerical_columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert \"Policy Start Date\" to datetime\ntrain_data[\"Policy Start Date\"] = pd.to_datetime(train_data[\"Policy Start Date\"])\ntest_data[\"Policy Start Date\"] = pd.to_datetime(test_data[\"Policy Start Date\"])\n\n# Extract date-related features\nfor df in [train_data, test_data]:\n    df[\"year\"] = df[\"Policy Start Date\"].dt.year\n    df[\"month\"] = df[\"Policy Start Date\"].dt.month\n    df[\"day\"] = df[\"Policy Start Date\"].dt.day\n    df[\"day_of_week\"] = df[\"Policy Start Date\"].dt.dayofweek  # Monday=0, Sunday=6\n    df[\"day_name\"] = df[\"Policy Start Date\"].dt.day_name()    # Full name of the day\n    df[\"is_weekend\"] = df[\"Policy Start Date\"].dt.dayofweek >= 5  # True if Saturday or Sunday\n    df[\"quarter\"] = df[\"Policy Start Date\"].dt.quarter         # Quarter of the year (1-4)\n    df[\"is_month_start\"] = df[\"Policy Start Date\"].dt.is_month_start  # True if the start of the month\n    df[\"is_month_end\"] = df[\"Policy Start Date\"].dt.is_month_end      # True if the end of the month\n    df[\"week\"] = df[\"Policy Start Date\"].dt.isocalendar().week        # Week of the year (ISO standard)\n    df[\"f_miss_pos\"] = df[[\"Marital Status\", \"Health Score\", \"Vehicle Age\", \n                           \"Insurance Duration\", \"Customer Feedback\"]].isna().sum(axis=1)\n    df[\"f_miss_neg\"] = df[[\"Credit Score\", \"Annual Income\"]].isna().sum(axis=1)\n\n# Drop the original \"Policy Start Date\" column\ntrain_data = train_data.drop(columns=[\"Policy Start Date\"])\ntest_data = test_data.drop(columns=[\"Policy Start Date\"])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Premium Amount to log to have a non-skewed distribution\n- Annual Income to log to have a non-skewed distribution\n- Previous Claims to clip after 4 to value 4. This is to facilitate the cut of trees for when there is not enough data.\n\n","metadata":{},"attachments":{}},{"cell_type":"code","source":"def log_plus_one(x):\n    \"\"\"\n    Returns the natural logarithm of (x + 1).\n    Assumes x >= -1, since log is undefined for negative or zero arguments.\n    \"\"\"\n    return np.log(x + 1)\n\ndef exp_minus_one(y):\n    \"\"\"\n    Returns e^(y) - 1.\n    \"\"\"\n    return np.exp(y) - 1","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creation of KNN distance medoid variable","metadata":{}},{"cell_type":"code","source":"# knn_train = train_data.copy()\n# knn_train.fillna(-1, inplace = True)\n\n# knn_test = test_data[knn_cols].copy()\n# knn_test.fillna(-1, inplace = True)\n\n# # Create pY as an integer percentile rank (1 through 100)\n# knn_train[\"pY\"] = (knn_train[\"Premium Amount\"]\n#             .rank(pct=True)      # rank by percentage\n#             .mul(100)            # scale to 0-100\n#             .round(0)            # round to nearest integer\n#             .astype(int))        # convert to integer\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # List numeric columns\n# knn_cols = ['Age',\n#  'Annual Income',\n#  'Number of Dependents',\n#  'Health Score',\n#  'Previous Claims',\n#  'Vehicle Age',\n#  'Credit Score',\n#  'Insurance Duration',\n#  'year',\n#  'month',\n#  'day',\n#  'day_of_week',\n#  'f_missing']\n\n# robust_scaler = RobustScaler()\n# robust_scaler.fit(knn_train[knn_cols])  \n\n# knn_train[knn_cols] = robust_scaler.transform(knn_train[knn_cols])  \n# knn_test[knn_cols] = robust_scaler.transform(knn_test[knn_cols])  \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def compute_medoid(df_group, feature_cols):\n#     \"\"\"\n#     Returns:\n#       - the index of the medoid row in df_group\n#       - the row values of the medoid (numpy array)\n#     \"\"\"\n#     X = df_group[feature_cols].values\n#     # Pairwise distances: shape (n, n)\n#     dist_matrix = pairwise_distances(X, metric=\"euclidean\")\n#     # Sum across each row\n#     dist_sums = dist_matrix.sum(axis=1)\n#     # The medoid = row w/ minimal total distance\n#     medoid_idx = np.argmin(dist_sums)\n    \n#     # Return both the *original index* and the row data\n#     # (the original index helps us refer back to df)\n#     row_index = df_group.index[medoid_idx]\n#     medoid_values = X[medoid_idx]\n#     return row_index, medoid_values\n\n\n# species_to_medoid = {}\n# for sp, subdf in knn_train[knn_cols+[\"pY\"]].groupby(\"pY\"):\n#     midx, mvals = compute_medoid(subdf, knn_cols)\n#     species_to_medoid[sp] = (midx, mvals)\n\n# # print(\"Species -> Medoid index, values\")\n# # for sp in species_to_medoid:\n# #     print(sp, species_to_medoid[sp])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def predict_species_by_medoid(sample, species_to_medoid, feature_cols):\n#     \"\"\"\n#     sample: pd.Series or array of shape (4,) with numeric_cols in the same order\n#     species_to_medoid: dict of {species_name: (index, medoid_values)}\n#     feature_cols: list of the columns that define the medoid\n    \n#     Returns: The species (string) whose medoid is closest to 'sample'\n#     \"\"\"\n#     # Convert the sample to a numpy array if it's a Series\n#     if isinstance(sample, pd.Series):\n#         sample_vals = sample[feature_cols].values\n#     elif isinstance(sample, (list, np.ndarray)):\n#         sample_vals = np.array(sample)\n#     else:\n#         raise ValueError(\"Sample must be a pd.Series, list, or np.array\")\n    \n#     best_species = None\n#     best_dist = float(\"inf\")\n    \n#     for sp, (_, medoid_vals) in species_to_medoid.items():\n#         dist = np.linalg.norm(sample_vals - medoid_vals)\n#         if dist < best_dist:\n#             best_dist = dist\n#             best_species = sp\n    \n#     return best_species\n\n\n# # Example usage:\n# # Let's take one random row from the original df and see what species\n# # the medoid-based classifier would predict.\n\n\n# train_data[\"knn_medoid\"] = knn_train.apply(\n#     lambda row: predict_species_by_medoid(row, species_to_medoid, knn_cols), \n#     axis=1\n# )\n\n# test_data[\"knn_medoid\"] = knn_test.apply(\n#     lambda row: predict_species_by_medoid(row, species_to_medoid, knn_cols), \n#     axis=1\n# )\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data transformation on univariate analysis","metadata":{}},{"cell_type":"code","source":"for df in [train_data, test_data]:\n    df[\"log_Annual Income\"] = log_plus_one(df[\"Annual Income\"])\n    df[\"Previous Claims\"] = df[\"Previous Claims\"].clip(upper=4)\nfor df in [train_data]:\n    df[target_column] = log_plus_one(df[\"Premium Amount\"])\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Engineering","metadata":{}},{"cell_type":"code","source":"\n\ndef create_pairwise_features(df, numerical_columns, categorical_columns):\n    \"\"\"\n    Create pairwise features for numerical and categorical variables in a dataframe.\n\n    Parameters:\n    df (pd.DataFrame): The input dataframe.\n    numerical_columns (list): List of numerical column names.\n    categorical_columns (list): List of categorical column names.\n\n    Returns:\n    pd.DataFrame: A dataframe with new features.\n    \"\"\"\n    # Pairwise numerical features\n    if numerical_columns:\n        # Generate pairwise permutations for numerical columns\n        for num_pair in combinations(numerical_columns, 2):\n            col1, col2 = num_pair\n\n            # Add pairwise sum\n            df[f\"{col1}_plus_{col2}\"] = df[col1] + df[col2]\n\n            # # Add pairwise min\n            # df[f\"{col1}_min_{col2}\"] = df[[col1, col2]].min(axis=1)\n\n            # # Add pairwise max\n            # df[f\"{col1}_max_{col2}\"] = df[[col1, col2]].max(axis=1)\n\n            # Add pairwise mean\n            df[f\"{col1}_mean_{col2}\"] = df[[col1, col2]].mean(axis=1)\n\n            # Add pairwise division (avoiding division by zero)\n            df[f\"{col1}_div_{col2}\"] = df[col1] / df[col2] #.replace(0, np.nan)\n\n            # Add pairwise multiplication\n            df[f\"{col1}_mul_{col2}\"] = df[col1] * df[col2]\n\n            # Add pairwise modulus (avoiding division by zero)\n            df[f\"{col1}_mod_{col2}\"] = df[col1] % df[col2] #.replace(0, np.nan)\n\n    # Pairwise categorical combinations\n    if categorical_columns:\n       for pair in combinations(categorical_columns, 2):\n           col_name = f\"{pair[0]}_and_{pair[1]}\"\n           df[col_name] = df[pair[0]].astype(str) + \"_\" + df[pair[1]].astype(str)\n\n    return df\n\ndef generate_single_variable_features(\n    df: pd.DataFrame,\n    columns: list = None\n) -> pd.DataFrame:\n    \"\"\"\n    Generates new features based on single-column transformations.\n    \n    Parameters\n    ----------\n    df : pd.DataFrame\n        The input DataFrame.\n    columns : list, optional\n        A list of columns to transform. If None, all numeric columns are used.\n    include_original : bool, optional\n        If True, keep the original columns in the output. \n        If False, return only the newly generated columns.\n    \n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with the original columns (optionally) plus new transformed columns.\n    \"\"\"\n    \n    # If columns is None, select all numeric columns\n    \n    \n    \n    # For each column, we will apply some transformations:\n    # 1) Squared\n    # 2) Cube\n    # 3) Log (with a small shift to avoid log(0 or negative))\n    # 4) Square-root (with shift to avoid sqrt of negative)\n    # 5) Inverse (with a shift to avoid division by zero)\n    # 6) Centered squared: (x - mean(x))^2\n    \n    for col in columns:\n        # Squared\n        df[f\"{col}_squared\"] = df[col] ** 2\n        \n        # Cubed\n        # df[f\"{col}_cubed\"] = df[col] ** 3\n        \n        # Log transform — shift by a small value to avoid log(0)\n        # Make sure to handle negative values. One approach is to take absolute value:\n        df[f\"{col}_log\"] = np.log(np.abs(df[col]) + 1e-5)\n        \n        # Square-root transform — again, account for negative or zero values\n        df[f\"{col}_sqrt\"] = np.sqrt(np.abs(df[col]) + 1e-5)\n        \n        # # Inverse transform\n        # df[f\"{col}_inv\"] = 1.0 / (df[col] + 1e-5)\n        \n    \n    return df\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As a matter of a fact, I could be creating variables using all numerical and categorical combinations, but due to the fact of time and not making the notebook too heavy, I will only be chossing some small part of them.","metadata":{}},{"cell_type":"code","source":"categorical_columns = train_data.select_dtypes(include=['object']).columns.tolist()\nnumerical_columns = train_data.select_dtypes(include=['number']).columns.tolist()\ndrop_cols = [\"id\", \"Policy Start Date\", \"Premium Amount\", \"logY\"]\ncategorical_columns = [col for col in categorical_columns if col not in drop_cols]\nnumerical_columns = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', \n                     'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration', \n                     'log_Annual Income', 'f_miss_neg', 'f_miss_pos']\n# numerical_columns = [col for col in numerical_columns if col not in drop_cols]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"numerical: {numerical_columns}\")\nprint(f\"categorical: {categorical_columns}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = create_pairwise_features(train_data, numerical_columns, categorical_columns)\ntest_data = create_pairwise_features(test_data, numerical_columns, categorical_columns)\nprint(train_data.shape)\nprint(test_data.shape)\n\ntrain_data = generate_single_variable_features(train_data, numerical_columns)\ntest_data = generate_single_variable_features(test_data, numerical_columns)\nprint(train_data.shape)\nprint(test_data.shape)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Variables for evaluate to model\ndef replace_infs(df, columns=None):\n    \"\"\"\n    Replace +inf with the finite maximum and -inf with the finite minimum \n    for each selected column in the DataFrame.\n    \n    Parameters\n    ----------\n    df : pd.DataFrame\n        The input DataFrame.\n    columns : list or None\n        The columns to process. If None, all numeric columns will be used.\n        \n    Returns\n    -------\n    pd.DataFrame\n        The DataFrame with infinities replaced.\n    \"\"\"\n    # Work on a copy if you don't want to modify the original DataFrame:\n    # df = df.copy()\n\n    if columns is None:\n        # Use all numeric columns if none are specified\n        columns = df.select_dtypes(include=[np.number]).columns.tolist()\n    \n    for col in columns:\n        # Get only the finite values in this column\n        finite_vals = df.loc[np.isfinite(df[col]), col]\n        if len(finite_vals) == 0:\n            # If the column has no finite values at all, skip it\n            continue\n        \n        col_min = finite_vals.min()\n        col_max = finite_vals.max()\n        \n        # Replace +inf with the column's finite maximum\n        df.loc[df[col] == np.inf, col] = col_max\n        # Replace -inf with the column's finite minimum\n        df.loc[df[col] == -np.inf, col] = col_min\n    \n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = replace_infs(train_data)\ntest_data = replace_infs(test_data)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = train_data.select_dtypes(include=['object']).columns.tolist()\nnumerical_columns = train_data.select_dtypes(include=['number']).columns.tolist()\ndrop_cols = [\"id\", \"Policy Start Date\", \"Premium Amount\"]\n\ncols_to_evaluate = [col for col in train_data.columns if col not in drop_cols]\nprint(cols_to_evaluate)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drop highly correlated variables","metadata":{}},{"cell_type":"markdown","source":"> I will just drop variables that have a very high correlation (>0.9) due to the fact that of time in modelling. In this case I will not be very interested in model explainability, else the choice of variables would be more thorough and with much lower correlation.","metadata":{}},{"cell_type":"code","source":"def drop_highly_correlated_columns(df, threshold=0.8, y=None):\n    \"\"\"\n    Drops columns from a dataset where the correlation with another variable exceeds the threshold.\n\n    Parameters:\n    df (pd.DataFrame): The input dataframe.\n    threshold (float): The correlation threshold for dropping columns. Default is 0.8.\n    y (str): A column name to exclude from the correlation checks.\n\n    Returns:\n    pd.DataFrame: A dataframe with highly correlated columns removed.\n    \"\"\"\n    # Exclude the specified column y from the correlation matrix\n    numerical_columns = train_data.select_dtypes(include=['number']).columns.tolist()\n\n    if y and y in df.columns:\n        corr_matrix = df[numerical_columns].drop(columns=[y]).corr().abs()\n    else:\n        corr_matrix = df[numerical_columns].corr().abs()\n\n    # Create a set to keep track of columns to drop\n    columns_to_drop = set()\n\n    # Iterate over the correlation matrix\n    for i in range(len(corr_matrix.columns)):\n        for j in range(i + 1, len(corr_matrix.columns)):\n            col1 = corr_matrix.columns[i]\n            col2 = corr_matrix.columns[j]\n\n            # Check if the correlation exceeds the threshold\n            if corr_matrix.iloc[i, j] > threshold:\n                # Print the details of the drop\n                print(f\"Dropping column '{col2}' because it is {corr_matrix.iloc[i, j] * 100:.2f}% correlated with '{col1}'\")\n\n                # Add the second column to the set of columns to drop\n                columns_to_drop.add(col2)\n\n    # Drop the columns from the dataframe\n    df = df.drop(columns=columns_to_drop, errors='ignore')\n\n    return df\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data= drop_highly_correlated_columns(train_data, threshold=0.95, y=target_column)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bivariate regression analysis","metadata":{}},{"cell_type":"markdown","source":"The idea behind this part is \n1. to create a system to evaluate from deciles (much faster) whether some variable has very high or high enough power of discrimination solely such that it is important for us to put it into the algorithm. To do this, we will evaluate the mean of that specific cut against the mean of the general data. If that mean is larger than 3% (it is a poor criteria, but still it could be much higher, then we will be taking that variable for Catboost modelling.\n2. Evaluate whether missings should be a category (if the mean of the regression differs a lot from the others) or it should be inputted (if that one does not differ). I will do though the inputation but rather just use at the end everything as a category. ","metadata":{}},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def jitter(a_series, noise_reduction=1000000):\n    return (np.random.random(len(a_series))*a_series.std()/noise_reduction)-(a_series.std()/(2*noise_reduction))\n    \ndef numeric_decilecuts(df, column_name, n_deciles=10):\n    # Use qcut on non-null values to create deciles\n    df['X_decile'] = pd.qcut(df[column_name].dropna()+ jitter(df[column_name].dropna()),\n                             n_deciles,\n                             labels=range(1, n_deciles + 1),\n                             duplicates='drop')\n    \n    # Only add a 'Missing' category if there are NaN values\n    if df[column_name].isna().any():\n        df['X_decile'] = df['X_decile'].cat.add_categories(['Missing'])\n        df['X_decile'] = df['X_decile'].fillna('Missing')   \n\n    return df\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def numeric_treecuts(df, column_name, Y):\n\n    breakpoints = tree_cuts(df[df[column_name].notnull()],column_name, Y)\n    # Apply the updated function to the 'X_max' column with the breakpoints DataFrame\n    df['X_decile'] = df[column_name].apply(lambda x: find_group(x, breakpoints))\n    # Only add a 'Missing' category if there are NaN values\n    df.loc[df[column_name].isna(), \"X_decile\"]= \"Missing\"\n    return df\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tree_cuts(df, X_var, Y_var): \n    X = df[[X_var]]  # Features\n    y = df[Y_var]  # Target\n\n    # Initialize and train the decision tree\n    tree = DecisionTreeRegressor(random_state=42, min_samples_leaf=int(len(X)/10))\n    tree.fit(X, y)\n\n    # Extract information from the tree\n    n_nodes = tree.tree_.node_count\n    children_left = tree.tree_.children_left\n    children_right = tree.tree_.children_right\n    feature = tree.tree_.feature\n    threshold = tree.tree_.threshold\n    values = tree.tree_.value\n    impurities = tree.tree_.impurity\n    samples = tree.tree_.n_node_samples\n\n    # Initialize the table\n    table = pd.DataFrame(columns=[\"min\", \"max\", \"sample\", \"value\", \"squared error\"])\n\n    # Function to traverse the tree and extract information for the table\n    def recurse(node, lower_bound=float('-inf'), upper_bound=float('inf')):\n        if children_left[node] == children_right[node]:  # It's a leaf\n            value = values[node][0, 0]\n            sample_count = samples[node]\n            mse = impurities[node] * samples[node]  # Total squared error for this node\n            table.loc[len(table)] = [lower_bound, upper_bound, sample_count, value, mse]\n        else:\n            # Continue the recursion on both children\n            if children_left[node] != -1:\n                recurse(children_left[node], lower_bound, threshold[node])\n            if children_right[node] != -1:\n                recurse(children_right[node], threshold[node], upper_bound)\n\n    # Start recursion from root\n    recurse(0)\n    return table\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef find_group(value, breakpoints):\n    \"\"\"\n    Determines the group index for a given value based on specified breakpoints.\n\n    Parameters:\n    - value: The value to classify into a group.\n    - breakpoints: DataFrame with 'min' and 'max' columns defining group boundaries.\n\n    Returns:\n    - The index of the group if a match is found, otherwise None.\n    \"\"\"\n    for index, row in breakpoints.iterrows():\n        if row['min'] <= value < row['max']:\n            return index\n    return None  # Return None if no group matches\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def categorical_cuts(df, column_name):\n    df['X_decile'] = df[column_name].astype(str)\n\n    # Get unique categories of X_decile\n    unique_deciles = df['X_decile'].unique()\n\n    # Create an empty DataFrame with the specified columns and unique categories of X_decile\n    columns = ['X_min', 'X_max', 'X_median', 'X_25%', 'X_75%']\n    decile_stats = pd.DataFrame(index=unique_deciles, columns=columns)\n\n    return decile_stats","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Bivariant analysis\nimport numpy as np\nimport pandas as pd\n\ndef categorize_into_deciles_with_stats(df, column_name, Y, n_deciles=10, f_decile_tree =False):\n    \"\"\"\n    Categorizes a specified column in a DataFrame into deciles (or specified quantiles) for numerical data,\n    or uses existing categories for categorical data, and calculates various statistics for each group.\n\n    Parameters:\n    - df: Pandas DataFrame containing the data.\n    - column_name: string, the name of the column to categorize.\n    - Y: string, the name of another column to compute statistics for each group.\n    - n_deciles: integer, default 10. Specifies the number of groups for numerical data.\n\n    Returns:\n    - DataFrame: Returns a DataFrame with group statistics.\n    \"\"\"\n    # Check if the column is numeric or categorical\n    if pd.api.types.is_numeric_dtype(df[column_name]):\n        n_cuts = len(df[column_name].unique())\n        print(n_cuts)\n        \n        # if too little numerical values, then it is treated as categorical\n        if n_deciles>= n_cuts or n_cuts<=20:\n            decile_stats = categorical_cuts(df, column_name)\n        else:\n            if f_decile_tree:\n                df = numeric_treecuts(df, column_name, Y) \n            else:\n                try: \n                    df = numeric_decilecuts(df, column_name, n_deciles=n_deciles-1)\n                except:\n                    df[column_name] =df[column_name].astype(str)\n                    decile_stats = categorical_cuts(df, column_name)\n            decile_stats = df.groupby('X_decile')[column_name].agg([\n            'min', \n            'max', \n            'median', \n            lambda x: x.quantile(0.25), \n            lambda x: x.quantile(0.75)\n        ])\n\n        # Rename the columns\n        decile_stats.columns = ['X_min', 'X_max', 'X_median', 'X_25%', 'X_75%']\n    \n    else:\n        \n        decile_stats = categorical_cuts(df, column_name)\n    decile_stats.reset_index()    \n\n    # Calculate statistics for Y within each group\n    y_stats = df.groupby('X_decile',dropna = False)[Y].agg(['mean', 'std', 'median', lambda x: x.quantile(0.25), lambda x: x.quantile(0.75)])\n    y_stats.columns = [f'{Y}_mean', f'{Y}_std', f'{Y}_median', f'{Y}_25%', f'{Y}_75%']\n    \n    # Join decile_stats and y_stats\n    combined_stats = decile_stats.join(y_stats)\n\n    # Calculate overall mean for Y and discrepancy metrics\n    overall_median_Y = df[Y].median()\n    combined_stats[f'gen_{Y}_median'] = overall_median_Y\n    combined_stats['discr'] = abs(combined_stats[f'{Y}_median'] - overall_median_Y) / overall_median_Y\n    combined_stats['max_discr'] = combined_stats['discr'].max()\n    \n\n    # Insert variable name at the start\n    combined_stats.insert(0, 'varname', column_name)\n    combined_stats.reset_index(inplace=True)\n    combined_stats.columns = ['X_decile'] + combined_stats.columns[1:].tolist()\n    combined_stats['X_min_str'] = combined_stats['X_min'].apply(lambda x: f\"{x:.2f}\" if pd.notna(x) else \"\")\n    combined_stats['X_max_str'] = combined_stats['X_max'].apply(lambda x: f\"{x:.2f}\" if pd.notna(x) else \"\")\n    # combined_stats.insert(0, 'X_decile', combined_stats.index)\n    # Create x_string column based on the conditions\n    combined_stats['x_string'] = np.where(\n        combined_stats['X_min'].notna(),\n        '[' + combined_stats['X_min_str'] + '-' + combined_stats['X_max_str'] + ']',\n        combined_stats['X_decile']\n    )\n    combined_stats.drop(columns=['X_min_str',\"X_max_str\"], inplace=True)\n\n    return combined_stats\n\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_data_by_varname(df, var_name, Y):\n    \"\"\"\n    Plots Y_mean against x_string for a specified varname in the provided DataFrame\n    and adds a horizontal line at the value of gen_Y_mean.\n\n    Parameters:\n    - df: DataFrame containing the data.\n    - var_name: string, the varname to filter by and plot.\n    - Y: the response variable name (string).\n    \"\"\"\n    # Filter the DataFrame for the specified varname\n    filtered_df = df[df['varname'] == var_name].copy()  # Use .copy() to avoid SettingWithCopyWarning\n\n    # Check if the filtered DataFrame is empty\n    if filtered_df.empty:\n        display(HTML(f\"<p><strong>No data available for '{var_name}'.</strong></p>\"))\n        return\n\n    # Check for NaN values and warn\n    if filtered_df[['x_string', f'{Y}_mean']].isna().any().any():\n        display(HTML(f\"<p><strong>Warning: NaN values detected in data for '{var_name}'.</strong></p>\"))\n\n    # Fill NaN in `x_string` with a placeholder\n    filtered_df['x_string'] = filtered_df['x_string'].fillna('Missing')\n\n    # Ensure `x_string` is treated as a string\n    filtered_df['x_string'] = filtered_df['x_string'].astype(str)\n\n    # Determine general Y mean\n    gen_y_mean = filtered_df[f'gen_{Y}_median'].iloc[0]\n\n    # Create the plot\n    plt.figure(figsize=(10, 6))\n    ax = sns.lineplot(\n        data=filtered_df,\n        x='x_string',\n        y=f'{Y}_median',\n        marker='o',\n        linewidth=2.5,\n        label=f'Median {Y}'\n    )\n    plt.title(f'Median {Y} by Decile: {var_name}', fontsize=14, fontweight='bold')\n    plt.xlabel(f'{var_name}', fontsize=12)\n    plt.ylabel(f'Median {Y}', fontsize=12)\n    plt.grid(True)\n\n    # Add a horizontal line for gen_Y_mean\n    plt.axhline(y=gen_y_mean, color='red', linestyle='--', linewidth=1.5, \n                label=f'General {Y} Median: {gen_y_mean:.2f}')\n    plt.legend()\n\n    # Rotate x-axis labels to prevent overlapping\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n\n    # Display the plot\n    plt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Main","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_rows', 100)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drop_cols = [\"id\", \"Policy Start Date\", \"Premium Amount\", \"Annual Income\", target_column]\n\ncols_to_evaluate = [col for col in train_data.columns if col not in drop_cols]\nprint(cols_to_evaluate)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data[\"Health Score_div_Vehicle Age\"].value_counts(dropna = False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_result = pd.DataFrame()\n# Loop through columns and append results\nfor column in cols_to_evaluate:\n    print(f\"----------{column}--------------\")\n    try:\n        result = categorize_into_deciles_with_stats(train_data, column, target_column, n_deciles = 20, f_decile_tree = False)\n        final_result = pd.concat([final_result, result], ignore_index=True)\n    except:\n            print(f\"column {column} did not work\")\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_rows', 200)\ndisplay(final_result[final_result[\"varname\"]==\"Marital Status\"])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"times_std_by_mean =40\nthreshold=train_data[target_column].std()/times_std_by_mean/train_data[target_column].mean()\nprint(f\"threshold = {threshold}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(final_result.loc[final_result[\"max_discr\"]>threshold,['varname','max_discr']].drop_duplicates().sort_values(by = \"max_discr\", ascending=False))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_result[\"x_string\"]=final_result[\"x_string\"].astype(str)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vars_in = final_result.loc[final_result[\"max_discr\"]>threshold, \"varname\"].unique().tolist()\nprint(f\"The variables that are chosen for modelling ABOVE {round(threshold*100,2)}% discrimination are: {vars_in}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vars_in","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot","metadata":{}},{"cell_type":"code","source":"for var in vars_in:\n    plot_data_by_varname(final_result, var, target_column)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Column choice for modelling\n","metadata":{}},{"cell_type":"code","source":"train_data = train_data[vars_in]\ntest_data = test_data[vars_in]\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vars_in","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Replace NaN values in categorical columns with a string \"nan\nconversion_to_string= [ 'Number of Dependents', 'year', 'month',\"Insurance Duration\", \"Previous Claims\"]\n\nfor col_to_string in conversion_to_string:\n    if col_to_string in train_data.columns.tolist():\n        train_data[col_to_string] = train_data[col_to_string].astype(str)\n        test_data[col_to_string] = test_data[col_to_string].astype(str)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_columns = train_data.select_dtypes(include=['object']).columns.tolist()\nnumerical_columns = train_data.select_dtypes(include=['number']).columns.tolist()\nfor col in categorical_columns:\n    train_data[col]=train_data[col].astype(str)\n    test_data[col]=test_data[col].astype(str)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modelling","metadata":{}},{"cell_type":"markdown","source":"## Separate features and target","metadata":{}},{"cell_type":"code","source":"train_data2 = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/train.csv\")\ntrain_data2[target_column] = log_plus_one(train_data2[\"Premium Amount\"])\n\nprint(\"Categorical columns:\", categorical_columns)\nX = train_data\ny = train_data2[target_column]\nbins = 30  # Number of bins\ny_binned = pd.cut(y, bins=bins, labels=False)\n# Step 3: Train/Test Split\n# Perform stratified split\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.2, random_state=142, stratify=y_binned\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluation of the cut","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 5))\nsns.kdeplot(y_train, label='y_train', shade=True)\nsns.kdeplot(y_val,   label='y_val',   shade=True)\nplt.title(\"Distribution of y (Train vs. Val)\")\nplt.legend()\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# Step 4: CatBoost Dataset Pool (handles categorical features natively)\ntrain_pool = Pool(X_train, y_train, cat_features=categorical_columns)\nval_pool = Pool(X_val, y_val, cat_features=categorical_columns)\ntest_pool = Pool(test_data, cat_features=categorical_columns)\n\n# Step 5: Model Initialization\nmodel = CatBoostRegressor(\n    iterations=2000,  # Number of boosting iterations\n    learning_rate=0.1,  # Learning rate\n    depth=8,  # Depth of the trees\n    l2_leaf_reg=2,\n    loss_function='RMSE',  # Loss function for regression\n    cat_features=categorical_columns,\n    colsample_bylevel = 0.6,\n    subsample = 0.6,\n    min_data_in_leaf= 5,\n    verbose= None  # Prints progress every 200 iterations\n)\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6: Train the Model\nmodel.fit(train_pool, eval_set=val_pool, early_stopping_rounds=60)\n\n# # Step 7: Evaluate Model\n# y_val_pred = model.predict(val_pool)\n# rmse = mean_squared_error(y_val, y_val_pred, squared=False)\n# print(f'Validation RMSE: {rmse}')\nif F_HYPERPARAMETER:\n    # Step 8: Hyperparameter Optimization (Grid Search)\n    param_grid = {\n        \"iterations\": [1000],\n        \"learning_rate\": [ 0.1],\n        \"depth\": [3, 5, 6, 8, 10],\n        \"subsample\": [0.4], #[0.4, 0.5, 0.6, 0.7, 0.8],\n        \"colsample_bylevel\": [0.4], #[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n        \"min_data_in_leaf\": [30], #[10, 40, 100, 300, 1000],\n        \"l2_leaf_reg\": [2] #[0.5,1,2],\n        # You can add more if needed\n    }\n\n    \n    grid_search = GridSearchCV(\n    estimator=CatBoostRegressor(\n        cat_features=categorical_columns,\n        loss_function='RMSE',\n        # You can set some defaults here (like l2_leaf_reg=2 if you want to fix it)\n    ),\n    param_grid=param_grid,\n    scoring='neg_root_mean_squared_error',\n    cv=3,             # 3-fold cross-validation for scoring\n    verbose=1,\n    n_jobs=-1         # Use all CPU cores if you want\n)\n\n    # We pass early_stopping_rounds, eval_set, and use_best_model as extra fit parameters\n    fit_params = {\n        \"early_stopping_rounds\": 50,\n        \"eval_set\": (X_val, y_val),\n        \"use_best_model\": True,\n        \"verbose\": False,         # or True/None, depending on your preference\n    }\n    \n    grid_search.fit(X_train, y_train, **fit_params)\n    \n    best_model = grid_search.best_estimator_\n    print(\"Best parameters found: \", grid_search.best_params_)\n\n    \n    # Step 9: Retrain with Best Parameters\n    final_model = CatBoostRegressor(\n        **grid_search.best_params_,\n        cat_features=categorical_columns,\n        loss_function='RMSE'\n    )\n    final_model.fit(train_pool)\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if F_HYPERPARAMETER:\n    model = final_model\n# Predict on train: Predict on train Data\npredictions_train = model.predict(X_train)\n\nrmse = root_mean_squared_error(y_train, predictions_train)\nprint(f\"RMSE: {rmse}\")\n\n# Step 10: Predict on Val Data\npredictions = model.predict(X_val)\n\nrmse = root_mean_squared_error(y_val, predictions)\nprint(f\"RMSE: {rmse}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Prediction**","metadata":{}},{"cell_type":"code","source":"predicts=model.predict(test_data)\npredicts=exp_minus_one(predicts) #detransforming changing to real premium","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data2 = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    'Id': test_data2['id'],  # Replace 'Id' with your test dataset identifier column\n    'Premium Amount': predicts\n})\nsubmission.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}