{"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[],"gpuType":"V28"},"accelerator":"TPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math\nimport scipy.stats as stats\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.pipeline import Pipeline\nfrom scipy.stats import chi2_contingency, f_oneway, shapiro, normaltest, ks_2samp, chi2, boxcox, norm\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:36:42.69475Z","iopub.execute_input":"2025-01-02T18:36:42.695174Z","iopub.status.idle":"2025-01-02T18:36:44.529579Z","shell.execute_reply.started":"2025-01-02T18:36:42.695127Z","shell.execute_reply":"2025-01-02T18:36:44.528462Z"},"id":"YmM03EHBYCB-"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"train.csv\")\noriginal = pd.read_csv(\"original.csv\")\ntest = pd.read_csv(\"test.csv\")\ny_target = 'Premium Amount'\ncbrt_y_target = 'CBRT Premium Amount'\npd.set_option('display.max_columns', None)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:36:45.566311Z","iopub.execute_input":"2025-01-02T18:36:45.567857Z","iopub.status.idle":"2025-01-02T18:36:56.45037Z","shell.execute_reply.started":"2025-01-02T18:36:45.567793Z","shell.execute_reply":"2025-01-02T18:36:56.449213Z"},"id":"KHoEuiB6YCB_","outputId":"69593069-1d01-43d3-9e34-661e79c5d8e7"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[y_target].size","metadata":{"id":"pwGIidvKxv9f","outputId":"12131711-5787-4e2a-99c8-acc0199fb047"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_to_dt(dataset):\n    dataset['Policy Start Date'] = pd.to_datetime(dataset['Policy Start Date'], format='mixed')\n    dataset['Policy Start Date Epoch'] = pd.to_datetime(dataset['Policy Start Date'], format='mixed').astype(int) / 10**9\n\ndef transform_target(dataset, deg):\n    dataset[cbrt_y_target] = dataset[y_target]**(deg)\n    #dataset[cbrt_y_target] = (dataset[cbrt_y_target]- dataset[cbrt_y_target].mean())/dataset[cbrt_y_target].std()\n\ndef get_categorical_columns(dataset):\n    return dataset.select_dtypes(include='object').columns.tolist()\n\ndef fill_nulls(dataset):\n    dataset.fillna({'Occupation': 'Unknown'}, inplace=True)\n    dataset.fillna({'Marital Status': 'Unknown'}, inplace=True)\n    dataset.fillna({'Customer Feedback': 'Unknown'}, inplace=True)\n\ndef get_continuous_columns(dataset):\n    continuous_columns = set(train.columns) - set(categorical_columns)\n    continuous_columns.discard('Policy Start Date')\n    continuous_columns.discard(y_target)\n    continuous_columns.discard(cbrt_y_target)\n    return continuous_columns\n\ndef trim_data(dataset):\n    #count = (train[y_target] > 4000).sum()\n    #print(f\"Number of instances where y_target > 4000: {count}\")\n    dataset = dataset[dataset[y_target] <= 4000]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:36:56.452572Z","iopub.execute_input":"2025-01-02T18:36:56.453057Z","iopub.status.idle":"2025-01-02T18:36:56.46149Z","shell.execute_reply.started":"2025-01-02T18:36:56.453007Z","shell.execute_reply":"2025-01-02T18:36:56.460311Z"},"id":"RAAHWX-gYCB_"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dt= 2.7\nconvert_to_dt(train)\nconvert_to_dt(original)\n#transform_target(train, 1/dt)\n#transform_target(original, 1/dt)\n\n#train.drop(columns=['id'], inplace=True)\n#original.drop(columns=['id'], inplace=True)\n\n#original[cbrt_y_target] = np.log1p(original[y_target] + 0.1)\n# Ensure data is positive (Box-Cox only works on positive values)\ndata = np.array(train[y_target])\nif np.any(data <= 0):\n    data = data - np.min(data) + 1\ndata = data[np.isfinite(data)]\ntrain[cbrt_y_target], lambda_value = boxcox(data)\n\noriginal.dropna(subset=[y_target], inplace=True)\nd = np.array(original[y_target])\nif np.any(d <= 0):\n    d = d - np.min(d) + 1\nd = d[np.isfinite(d)]\noriginal[cbrt_y_target], lambda_value = boxcox(d)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:36:56.462989Z","iopub.execute_input":"2025-01-02T18:36:56.463426Z","iopub.status.idle":"2025-01-02T18:36:56.929244Z","shell.execute_reply.started":"2025-01-02T18:36:56.463379Z","shell.execute_reply":"2025-01-02T18:36:56.92797Z"},"id":"kos5vbffYCB_"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{"id":"AWDO9DMlYCB_"}},{"cell_type":"code","source":"\n#ee, lambda_value = boxcox(original[y_target])\n\n\nfig, axes = plt.subplots(2, 2, figsize=(15, 7))\naxes = axes.flatten()\nsns.histplot(train[y_target], kde=True, bins=20, color='red', stat='percent', ax=axes[0])\naxes[0].set_title('Train: Distribution of Premium Amount', fontsize=12)\naxes[0].set_xlabel('Premium Amount, $', fontsize=10)\naxes[0].set_ylabel('Frequency, %', fontsize=10)\naxes[0].tick_params(axis='x', rotation=0)\naxes[0].grid(axis='y')\n\nsns.histplot(train[cbrt_y_target], kde=True, bins=20, color='blue', stat='percent', ax=axes[1])\naxes[1].set_title('Train: Distribution of box_cox(Premium Amount)', fontsize=12)\naxes[1].set_xlabel('box_cox(Premium Amount), $', fontsize=10)\naxes[1].set_ylabel('Frequency, %', fontsize=10)\naxes[1].tick_params(axis='x', rotation=0)\naxes[1].grid(axis='y')\n\nsns.histplot(original[y_target], kde=True, bins=20, color='red', stat='percent', ax=axes[2])\naxes[2].set_title('Original: Distribution of Premium Amount', fontsize=12)\naxes[2].set_xlabel('Premium Amount, $', fontsize=10)\naxes[2].set_ylabel('Frequency, %', fontsize=10)\naxes[2].tick_params(axis='x', rotation=0)\naxes[2].grid(axis='y')\n\nsns.histplot(original[cbrt_y_target], kde=True, bins=20, color='blue', stat='percent', ax=axes[3])\naxes[3].set_title('Original: Distribution of box_cox(Premium Amount)', fontsize=12)\naxes[3].set_xlabel('box_cox(Premium Amount), $', fontsize=10)\naxes[3].set_ylabel('Frequency, %', fontsize=10)\naxes[3].tick_params(axis='x', rotation=0)\naxes[3].grid(axis='y')\n\nplt.subplots_adjust(hspace=0.3, wspace=0.3)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:36:56.931749Z","iopub.execute_input":"2025-01-02T18:36:56.932255Z","iopub.status.idle":"2025-01-02T18:37:08.320449Z","shell.execute_reply.started":"2025-01-02T18:36:56.932206Z","shell.execute_reply":"2025-01-02T18:37:08.319208Z"},"jupyter":{"outputs_hidden":true},"id":"ECOo4MdhYCCA","outputId":"a58333a8-c607-4591-8e3e-705e745082b7","collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = train[cbrt_y_target]\n# Step 1: Bin the data\nnum_bins = 30  # Choose the number of bins\nobserved_freq, bin_edges = np.histogram(data, bins=num_bins)\n\n# Step 2: Calculate expected frequencies\nmean, std = np.mean(data), np.std(data)\nexpected_freq = []\nfor i in range(len(bin_edges) - 1):\n    prob = norm.cdf(bin_edges[i + 1], mean, std) - norm.cdf(bin_edges[i], mean, std)\n    expected_freq.append(prob * len(data))\nexpected_freq = np.array(expected_freq)\n\n# Step 3: Compute the Chi-Square statistic\nchi_square_stat = np.sum((observed_freq - expected_freq) ** 2 / expected_freq)\n\n# Step 4: Compare with critical value\ndof = num_bins - 3  # Degrees of freedom = bins - 1 - estimated params (mean, std)\ncritical_value = chi2.ppf(0.95, dof)  # 95% confidence level\np_value = 1 - chi2.cdf(chi_square_stat, dof)\n\nprint(f\"Chi-Square Statistic: {chi_square_stat}\")\nprint(f\"Critical Value: {critical_value}\")\nprint(f\"P-value: {p_value}\")\n\nif chi_square_stat < critical_value:\n    print(\"Fail to reject the null hypothesis: Data is normally distributed.\")\nelse:\n    print(\"Reject the null hypothesis: Data is not normally distributed.\")\n\n","metadata":{"id":"la9_lU3wgRiw","outputId":"051a6f9c-2cd2-426e-e7af-b0c72ae87d29"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Shapiro-Wilk Test\nstat, p = shapiro(original[cbrt_y_target])\nprint(f\"Shapiro-Wilk Test: p-value={p}\")\n\n# D’Agostino and Pearson’s Test\nstat, p = normaltest(original[cbrt_y_target])\nprint(f\"D’Agostino Test: p-value={p}\")","metadata":{"id":"cyDWPq5mgY_e","outputId":"f7b9061f-97fb-4d10-a79a-a0c9ce09a646"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Shapiro-Wilk Test\nstat, p = shapiro(train[cbrt_y_target])\nprint(f\"Shapiro-Wilk Test: p-value={p}\")\n\n# D’Agostino and Pearson’s Test\nstat, p = normaltest(train[cbrt_y_target])\nprint(f\"D’Agostino Test: p-value={p}\")","metadata":{"id":"ngeEINpphTsV","outputId":"2ad8bbc6-874a-4931-a26a-94b8e73eb2f6"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Kolmogorov-Smirnov test to check weather the target distr is similar\n#in train and original\nstatistic, p_value = ks_2samp(original[y_target], train[y_target])\n\nprint(f\"KS Statistic: {statistic}\")\nprint(f\"P-value: {p_value}\")","metadata":{"id":"OcxvW3HVDm_E","outputId":"eb8e390b-19a9-4010-afd4-9f67512fdf80"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train[y_target].shape[0]\nchi2_cdf = chi2.cdf(np.sort(train[y_target]), df)\n\n# Perform KS test comparing the empirical distribution with the chi-squared CDF\nks_statistic, p_value = ks_2samp(train[y_target], chi2.rvs(df=5, size=df))\nprint(f\"KS Statistic: {ks_statistic}\")\nprint(f\"P-value: {p_value}\")","metadata":{"id":"1Ftx2oAlEfE7","outputId":"122ed800-eb27-4e61-81f0-0e3252714637"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"eEovp4Y2Etp9"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The disstributions do not seem to be exactly identical (KS is not 0), but are somewhat similar","metadata":{"id":"MAyD0LfmEDTY"}},{"cell_type":"code","source":"missing_values = train.isna().sum()  # Count NaN values in each column\nmissing_percentage = (train.isna().mean() * 100)  # Percentage of NaN values in each column\nprint(\"Missing Values per column:\\n\", missing_values, \"\\n\")\nprint(\"Percent of Missing Values per column:\\n\", missing_percentage)","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"collapsed":true,"execution":{"iopub.status.busy":"2025-01-02T18:37:08.322088Z","iopub.execute_input":"2025-01-02T18:37:08.322523Z","iopub.status.idle":"2025-01-02T18:37:09.469141Z","shell.execute_reply.started":"2025-01-02T18:37:08.322475Z","shell.execute_reply":"2025-01-02T18:37:09.468005Z"},"id":"W0klpZFeYCCA","outputId":"c4e7e824-15b8-4b46-c2c6-e02232d6c2ba"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_null_values = train.drop(columns=[y_target, cbrt_y_target, 'Policy Start Date Epoch']).isnull().T\ntest_null_values = test.isnull().T\n\nfig, axes = plt.subplots(nrows=1, ncols=2, figsize=(14, 5))\n\nsns.heatmap(train_null_values, cbar=False, ax=axes[0], cmap=sns.light_palette('grey', as_cmap=True))\naxes[0].set_title('Train')\naxes[0].set_xticks([])\naxes[0].set_xlabel('')\n\nsns.heatmap(test_null_values, cbar=False, ax=axes[1], cmap=sns.light_palette('grey', as_cmap=True))\naxes[1].set_title('Test')\naxes[1].set_xticks([])\naxes[1].set_xlabel('')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"collapsed":true,"execution":{"iopub.status.busy":"2025-01-02T18:37:09.470612Z","iopub.execute_input":"2025-01-02T18:37:09.47107Z","iopub.status.idle":"2025-01-02T18:39:33.406008Z","shell.execute_reply.started":"2025-01-02T18:37:09.471023Z","shell.execute_reply":"2025-01-02T18:39:33.4049Z"},"id":"b8jtLgnwYCCA","outputId":"ccff2e9e-195a-4b21-9177-91cd7bedb3d9"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Some of the columns have a significant number of missing values, for example: Previous Claims, Occupation and Credit Score.","metadata":{"id":"aX4tXZe9YCCA"}},{"cell_type":"markdown","source":"## Categorical Variables","metadata":{"id":"CeBiijcaYCCB"}},{"cell_type":"code","source":"categorical_columns = get_categorical_columns(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:39:33.407221Z","iopub.execute_input":"2025-01-02T18:39:33.407537Z","iopub.status.idle":"2025-01-02T18:39:33.905324Z","shell.execute_reply.started":"2025-01-02T18:39:33.407506Z","shell.execute_reply":"2025-01-02T18:39:33.904057Z"},"id":"gpI4J7qKYCCC"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(5, 3))\nsns.barplot(\n    x=train[categorical_columns].nunique().values,\n    y=train[categorical_columns].nunique().index,\n    ax=ax,\n    color=\"grey\"\n)\nax.set_title('Unique Values in Categorical Columns')\nax.set_xticks(range(5))\nax.set_xticklabels(range(5))\n\nfor i, value in enumerate(train[categorical_columns].nunique().values):\n    ax.text(value, i, f'{value}', va='center')\n\nsns.despine()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T23:43:50.187565Z","iopub.execute_input":"2024-12-30T23:43:50.187894Z","iopub.status.idle":"2024-12-30T23:43:52.831923Z","shell.execute_reply.started":"2024-12-30T23:43:50.187864Z","shell.execute_reply":"2024-12-30T23:43:52.830901Z"},"id":"zVeyf9TiYCCC","outputId":"280076d6-3334-44f4-95c7-32136951a487"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(5, 2, figsize=(15, 20))\naxes = axes.flatten()\n\nfor i, category in enumerate(categorical_columns):\n    group_means = train.groupby(category).size() / train[category].count() * 100\n    bar_plot = axes[i].bar(group_means.index.astype(str), group_means, color='red')\n    # Add text inside each bar\n    height = 0\n    for bar, label in zip(bar_plot, group_means.index):\n        if not height:\n            height = bar.get_height()\n\n        # Dynamically filter the train dataset based on the category\n        mean_premium = train[train[category] == label]['Premium Amount'].mean()\n\n        # Add text to the bar\n        axes[i].text(\n            bar.get_x() + bar.get_width() / 2,\n            height / 2,\n            f'${mean_premium:.2f}',\n            ha='center',\n            va='center',\n            fontsize=10,\n            color='black'\n        )\n    axes[i].set_title(category, fontsize=12)\n    axes[i].set_ylabel('%', fontsize=10)\n    axes[i].tick_params(axis='x', rotation=0)\n    axes[i].grid(axis='y')\n\n#plt.title(\"Distribution of Groups Within Categories and Their Mean Premium Amount\")\nplt.subplots_adjust(hspace=0.3, wspace=0.3)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:39:33.907796Z","iopub.execute_input":"2025-01-02T18:39:33.908203Z","iopub.status.idle":"2025-01-02T18:39:41.617302Z","shell.execute_reply.started":"2025-01-02T18:39:33.908168Z","shell.execute_reply":"2025-01-02T18:39:41.616016Z"},"id":"k9szWdWNYCCC","outputId":"6878cd6b-1f97-439b-9df4-8f67fc81e465"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The categories appear to be well-balanced, meaning that the groups within each category almost uniformly distributed. It also is evident that mean Premium Amount doesn't vary much intergroup.","metadata":{"id":"HRECu9m1YCCC"}},{"cell_type":"code","source":"fill_nulls(train)\nfill_nulls(original)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:39:41.618768Z","iopub.execute_input":"2025-01-02T18:39:41.619161Z","iopub.status.idle":"2025-01-02T18:39:41.877407Z","shell.execute_reply.started":"2025-01-02T18:39:41.619127Z","shell.execute_reply":"2025-01-02T18:39:41.875903Z"},"id":"C07BFvhmYCCC"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cramers_v(confusion_matrix):\n    chi2, p, dof, expected = chi2_contingency(confusion_matrix)\n    n = confusion_matrix.sum().sum()\n    phi2 = chi2 / n\n    r, k = confusion_matrix.shape\n    phi2corr = max(0, phi2 - ((k - 1) * (r - 1)) / (n - 1))\n    rcorr = r - (r - 1)**2 / (n - 1)\n    kcorr = k - (k - 1)**2 / (n - 1)\n\n    return np.sqrt(phi2corr / min((kcorr - 1), (rcorr - 1))), p\n\nresults = {}\n\nfor col in categorical_columns:\n    confusion_matrix = pd.crosstab(train[col], train[cbrt_y_target])\n    if not confusion_matrix.empty:\n        cv, p_value = cramers_v(confusion_matrix)\n        results[col] = {\"Cramér's V\": cv, \"p-value\": p_value}\ncramers_v_df = pd.DataFrame.from_dict(results, orient='index')\ncramers_v_df.sort_values(by=\"p-value\", ascending=False, inplace=True)\nprint(cramers_v_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:39:41.879639Z","iopub.execute_input":"2025-01-02T18:39:41.88002Z","iopub.status.idle":"2025-01-02T18:39:45.867332Z","shell.execute_reply.started":"2025-01-02T18:39:41.879986Z","shell.execute_reply":"2025-01-02T18:39:45.866146Z"},"id":"FNC_QDdIYCCD","outputId":"b111eaef-d5bd-4393-a5c2-f79c6f8f96d1"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The association is quite weak but appears to be significant for all categories.\nANOVA is typically used for normally distributed data, and performing it on cubic-root transformed target yields better F-statistic and associated p-values.","metadata":{"id":"O5iTIOwzYCCD"}},{"cell_type":"code","source":"# ANOVA test\nresults = {}\nfor col in categorical_columns:\n    groups = [train[train[col] == level][cbrt_y_target] for level in train[col].unique()]\n    f_stat, p_val = f_oneway(*groups)\n    results[col] = f_stat, p_val\nanova_df = pd.DataFrame.from_dict(results, orient='index', columns=['f-stat', 'p-value'])\nanova_df.sort_values(by='p-value', inplace=True)\nprint(anova_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:39:45.86867Z","iopub.execute_input":"2025-01-02T18:39:45.869098Z","iopub.status.idle":"2025-01-02T18:39:51.906809Z","shell.execute_reply.started":"2025-01-02T18:39:45.869064Z","shell.execute_reply":"2025-01-02T18:39:51.905547Z"},"id":"TUN3wCa4YCCD","outputId":"b59fd60d-f2c3-4571-8557-62f45cdaf741"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The ANOVA F-test suggests that Education Level, Property Type, Location, Policy Type, Exercise Frequency, Smoking Status and Gender may have little to no realtionship with the target variable (Premium Amount). We still cannot easily iliminate those variables without additional testing. Cramer's V results (previous section) somewhat align with the F-test results.","metadata":{"id":"mpOotojSYCCD"}},{"cell_type":"markdown","source":"## Continuous Variables","metadata":{"id":"6Tu9LZFeYCCD"}},{"cell_type":"code","source":"continuous_columns = get_continuous_columns(train)\nget_continuous_columns(original)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:40:52.852612Z","iopub.execute_input":"2025-01-02T18:40:52.85306Z","iopub.status.idle":"2025-01-02T18:40:52.858476Z","shell.execute_reply.started":"2025-01-02T18:40:52.853021Z","shell.execute_reply":"2025-01-02T18:40:52.857307Z"},"id":"O8KQswV2YCCD","outputId":"ee62068f-fa5d-425d-aded-fefc6f28c0a0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(15, 15))\naxes = axes.flatten()\nfor i, category in enumerate(continuous_columns):\n    if category in ['Health Score', 'Age']:\n        min_bin = math.floor(train[category].min() / 5) * 5\n        max_bin = math.ceil(train[category].max() / 5) * 5\n        bins = range(min_bin, max_bin + 5, 5)\n        bin_labels = [x + 2.5 for x in bins[:-1]]\n        grouping_criteria = pd.cut(train[category], bins=bins, labels=bin_labels, right=False)\n    elif category in ['Credit Score']:\n        min_bin = math.floor(train[category].min() / 50) * 50\n        max_bin = math.ceil(train[category].max() / 50) * 50\n        bins = range(min_bin, max_bin + 50, 50)\n        bin_labels = [x + 25 for x in bins[:-1]]\n        grouping_criteria = pd.cut(train[category], bins=bins, labels=bin_labels, right=False)\n    elif category in ['Annual Income']:\n        min_bin = math.floor(train[category].min() / 10000) * 10000\n        max_bin = math.ceil(train[category].max() / 10000) * 10000\n        bins = range(min_bin, max_bin + 10000, 10000)\n        bin_labels = [x + 10000 for x in bins[:-1]]\n        grouping_criteria = pd.cut(train[category], bins=bins, labels=bin_labels, right=False)\n    elif category in ['Policy Start Date Epoch']:\n        min_bin = math.floor(train[category].min() / (60*60*24*30*6)) * 60*60*24*30*6 #6 months\n        max_bin = math.ceil(train[category].max() / (60*60*24*30*6)) * 60*60*24*30*6\n        bins = range(min_bin, max_bin + 60*60*24*30*6, 60*60*24*30*6)\n        bin_labels = [x + 60*60*24*30*6 for x in bins[:-1]]\n        grouping_criteria = pd.cut(train[category], bins=bins, labels=bin_labels, right=False)\n    else:\n        grouping_criteria = category\n\n    group_means = train.groupby(grouping_criteria, observed=True)[cbrt_y_target].mean()\n    axes[i].plot(group_means.index, group_means, color='red', alpha=0.7)\n\n    axes[i].set_title(category, fontsize=12)\n    axes[i].set_ylabel('$\\sqrt[3]{Mean\\ Premium\\ Cost}$', fontsize=8)\n    axes[i].tick_params(axis='x', rotation=0)\n    axes[i].set_ylim(8.9, 11.8)\n    axes[i].grid(axis='y')\n\nplt.subplots_adjust(hspace=0.3, wspace=0.3)\nplt.show()","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"collapsed":true,"execution":{"iopub.status.busy":"2025-01-02T18:40:54.900098Z","iopub.execute_input":"2025-01-02T18:40:54.900505Z","iopub.status.idle":"2025-01-02T18:40:57.185891Z","shell.execute_reply.started":"2025-01-02T18:40:54.900469Z","shell.execute_reply":"2025-01-02T18:40:57.184704Z"},"id":"0AzG4GwxYCCD","outputId":"dd19627a-98aa-4425-8ec4-614000a985fa"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"When scaled to the same y-axis it appears that half of continuous values means have little correlation with the Premium Amount.","metadata":{"id":"2pwvaFogYCCD"}},{"cell_type":"code","source":"corr_mat_df_nc = train[list(continuous_columns) + [y_target]]\ncorr_mat_nc = corr_mat_df_nc.corr(method=\"spearman\")\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr_mat_nc, annot=True, fmt=\".2f\", cmap=\"coolwarm\", cbar=True, vmin=-1, vmax=1)\nplt.title(\"Spearman's Correlation Of Non Categorical Features with the Target\")\nplt.show()","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"execution":{"iopub.status.busy":"2025-01-02T18:41:51.430103Z","iopub.execute_input":"2025-01-02T18:41:51.430496Z","iopub.status.idle":"2025-01-02T18:41:52.542929Z","shell.execute_reply.started":"2025-01-02T18:41:51.430461Z","shell.execute_reply":"2025-01-02T18:41:52.541694Z"},"id":"v8m2Cj73YCCD","outputId":"b8c0cda0-7bf4-451f-f2ab-35e2b5a2936f","collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_mat_df_nc = original[list(continuous_columns) + [y_target]]\ncorr_mat_nc = corr_mat_df_nc.corr(method=\"spearman\")\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr_mat_nc, annot=True, fmt=\".2f\", cmap=\"coolwarm\", cbar=True, vmin=-1, vmax=1)\nplt.title(\"Original DS: Spearman's Correlation Of Non Categorical Features with the Target\")\nplt.show()","metadata":{"id":"Sh9K9NNtyU3t","outputId":"28dfe508-df6e-4aee-946b-132a2f50d2ae"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"According to the covariance matrix above and the line plots from the previous section, 'Previous Claims' shows the strongest positive correlation with the target followed by 'Health Score', while 'Credit Score' and 'Annual Income' exhibit the strongest negative correlations.\nOverall, Pearson's correlation coefficient did not effectively reveal potential relationships or the impact of features on the cubic root target. This is likely because the formula heavily depends on the differences between observations and their mean. As observed in the line plots of the continuous variables, many features display nearly flat patterns with minimal variance. This lack of variance could be one of the reasons for the zero (or near-zero) correlation coefficients in the variance-covariance matrix for these features. Another reasons for the low correlation coefficients is: non-linear realtionships between the target and the features, and non-normal distibution of the data (the cubic root transformed data is still not perfectly normal).\n\n\nAn interesting counter-intuitive observation from the above matrix is that the 'Annual Income' and 'Credit Score' are negatively correlated. Let's just quickly confirm it with the plot below:","metadata":{"id":"882dprFkYCCD"}},{"cell_type":"code","source":"min_bin = math.floor(train['Annual Income'].min() / 10000) * 10000\nmax_bin = math.ceil(train['Annual Income'].max() / 10000) * 10000\nbins = range(min_bin, max_bin + 10000, 10000)\nbin_labels = [x + 5000 for x in bins[:-1]]\ngrouping_criteria = pd.cut(train['Annual Income'], bins=bins, labels=bin_labels, right=False)\n\ngroup_means = train.groupby(grouping_criteria, observed=True)['Credit Score'].mean()\n\nfig, ax = plt.subplots(figsize=(5, 3))\nax.plot(group_means.index, group_means, color='red', alpha=0.7)\nax.set_title('Annual Income vs Credit Score', fontsize=12)\nax.set_ylabel('Mean Credit Score', fontsize=10)\nax.set_xlabel('Annual Income', fontsize=10)\nax.tick_params(axis='x', rotation=45)\nax.grid(axis='y')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"collapsed":true,"id":"201cjgY0YCCE","outputId":"09fc10c9-73a8-42bf-f0b8-6cacaec11479"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imputing nans for the continuous variables:\ndef impute_continuous(dataset, imputer):\n    imputer = imputer\n    for col in ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', 'Previous Claims', 'Credit Score']:\n        dataset[col] = imputer.fit_transform(dataset[[col]])\n    dataset.dropna(subset=['Vehicle Age'], inplace=True)\n    dataset.dropna(subset=['Insurance Duration'], inplace=True)\n    dataset.dropna(subset=['Premium Amount'], inplace=True)\n    dataset.dropna(subset=['CBRT Premium Amount'], inplace=True)\n\n\nimpute_continuous(train, SimpleImputer(strategy='mean'))\nimpute_continuous(original, SimpleImputer(strategy='mean'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:42:06.898547Z","iopub.execute_input":"2025-01-02T18:42:06.899061Z","iopub.status.idle":"2025-01-02T18:42:07.572292Z","shell.execute_reply.started":"2025-01-02T18:42:06.899025Z","shell.execute_reply":"2025-01-02T18:42:07.571065Z"},"id":"mYaN_f85YCCE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = []\n\nfor col in continuous_columns:\n    pearson_corr, pearson_pval = stats.pearsonr(train[col], train[cbrt_y_target])  # Pearson\n    spearman_corr, spearman_pval = stats.spearmanr(train[col], train[cbrt_y_target])  # Spearman\n    kendall_corr, kendall_pval = stats.kendalltau(train[col], train[cbrt_y_target])  # Kendall's Tau\n    kruskal_corr, kruskal_pval = stats.kruskal(train[col], train[cbrt_y_target]) #Kruskal_Wallis\n\n    # Store the results for each feature\n    results.append({\n        'Feature': col,\n        'Pearson Corr': pearson_corr,\n        'Pearson P-value': pearson_pval,\n        'Spearman Corr': spearman_corr,\n        'Spearman P-value': spearman_pval,\n        'Kendall Corr': kendall_corr,\n        'Kendall P-value': kendall_pval,\n        'Kruskal Corr': kruskal_corr,\n        'Kruskal P-value': kruskal_pval\n    })\n\nresults_df = pd.DataFrame(results)\npd.set_option('display.max_columns', None)\npd.set_option('display.width', None)\npd.set_option('display.max_colwidth', None)\nresults_df.sort_values(by=\"Pearson Corr\", ascending=False, inplace=True)\nprint(\"Train DS (transformed target):\")\nprint(results_df.to_string(index=False))\n\nresults = []\nfor col in continuous_columns:\n    pearson_corr, pearson_pval = stats.pearsonr(original[col], original[cbrt_y_target])  # Pearson\n    spearman_corr, spearman_pval = stats.spearmanr(original[col], original[cbrt_y_target])  # Spearman\n    kendall_corr, kendall_pval = stats.kendalltau(original[col], original[cbrt_y_target])  # Kendall's Tau\n    kruskal_corr, kruskal_pval = stats.kruskal(original[col], original[cbrt_y_target]) #Kruskal_Wallis\n\n    # Store the results for each feature\n    results.append({\n        'Feature': col,\n        'Pearson Corr': pearson_corr,\n        'Pearson P-value': pearson_pval,\n        'Spearman Corr': spearman_corr,\n        'Spearman P-value': spearman_pval,\n        'Kendall Corr': kendall_corr,\n        'Kendall P-value': kendall_pval,\n        'Kruskal Corr': kruskal_corr,\n        'Kruskal P-value': kruskal_pval\n    })\n\nresults_df = pd.DataFrame(results)\npd.set_option('display.max_columns', None)\npd.set_option('display.width', None)\npd.set_option('display.max_colwidth', None)\nresults_df.sort_values(by=\"Pearson Corr\", ascending=False, inplace=True)\nprint(\"\\nOriginal DS (transformed target):\")\nprint(results_df.to_string(index=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:42:09.505737Z","iopub.execute_input":"2025-01-02T18:42:09.506151Z","iopub.status.idle":"2025-01-02T18:42:21.267258Z","shell.execute_reply.started":"2025-01-02T18:42:09.506114Z","shell.execute_reply":"2025-01-02T18:42:21.266059Z"},"id":"KcJpwag2YCCE","outputId":"4bef1ebf-c4c3-4c36-dc82-2acc76f1db6c"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Models","metadata":{"id":"2aW9q4hccGdo"}},{"cell_type":"code","source":"# One-Hot Encoding for categorical variable\nencoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False, drop='first')\ntrain_encoded = encoder.fit_transform(train[categorical_columns])\nencoded_train_df = pd.DataFrame(train_encoded, columns=encoder.get_feature_names_out(categorical_columns))\n\noriginal_encoded = encoder.fit_transform(original[categorical_columns])\nencoded_original_df = pd.DataFrame(original_encoded, columns=encoder.get_feature_names_out(categorical_columns))\n\n# Split into features (X) and target (y)\n#X = train.drop([y_target, cbrt_y_target, 'Policy Start Date'], axis=1)\n#X = X.drop(categorical_columns, axis=1)\n#X = pd.concat([X.reset_index(drop=True), encoded_df.reset_index(drop=True)], axis=1)\n\nkf = KFold(n_splits=10, random_state=42, shuffle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:45:21.550629Z","iopub.execute_input":"2025-01-02T18:45:21.55105Z","iopub.status.idle":"2025-01-02T18:45:26.795309Z","shell.execute_reply.started":"2025-01-02T18:45:21.551012Z","shell.execute_reply":"2025-01-02T18:45:26.794076Z"},"id":"S9oqafv0YCCE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, config, dataset, enc_ds, y_trans, transformed=True, transf_degree=1):\n    model = model\n    config = config\n    dataset = dataset\n    transformed=transformed\n    td = transf_degree\n    if transformed:\n        #y = dataset[cbrt_y_target]\n        y = y_trans  # Reset index for target\n    else:\n        y = dataset[y_target]\n\n\n    # Split into features (X) and target (y)\n    X = dataset.drop([y_target, cbrt_y_target, 'Policy Start Date'], axis=1)\n    X = X.drop(categorical_columns, axis=1)\n    X = pd.concat([X.reset_index(drop=True), enc_ds.reset_index(drop=True)], axis=1)\n    #kf = KFold(n_splits=5, random_state=42, shuffle=True)\n\n    r2_scores = []\n    mse_scores = []\n    accuracy_scores = []\n    accuracy_threshold = 0.1\n    #fig, axes = plt.subplots(4, 5, figsize=(20, 10))\n    #axes = axes.ravel()\n\n    for fold_idx, (train_index, test_index) in enumerate(kf.split(X)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        model.fit(X_train, y_train)\n\n        y_pred = model.predict(X_test)\n\n        if transformed:\n            if transf_degree != 1:\n                y_pred = np.abs(y_pred)**td\n                y_pred = np.nan_to_num(y_pred, nan=0.0, posinf=np.max(y_pred), neginf=np.min(y_pred))\n\n                y_test = np.abs(y_test)**td\n                y_test = np.nan_to_num(y_test, nan=0.0, posinf=np.max(y_test), neginf=np.min(y_test))\n\n            #else:\n                #y_pred = transformer.inverse_transform(y_pred)\n                #y_test = transformer.inverse_transform(y_test)\n\n        accuracy = np.mean(np.abs((y_pred - y_test) / y_test) <= accuracy_threshold)\n\n        r2 = r2_score(y_test, y_pred)\n        mse = mean_squared_error(y_test, y_pred)\n        r2_scores.append(r2)\n        mse_scores.append(mse)\n        accuracy_scores.append(accuracy)\n        residuals = y_test - y_pred\n        '''\n          sns.histplot(residuals, kde=True, ax=axes[fold_idx])\n          axes[fold_idx].set_title(f\"Fold {fold_idx} Residuals\")\n          axes[fold_idx].set_xlabel(\"Residuals\")\n          axes[fold_idx].set_ylabel(\"Frequency\")\n\n          stats.probplot(residuals, dist=\"norm\", plot=axes[fold_idx+10])\n          axes[fold_idx+10].set_title(f\"Fold {fold_idx} Q-Q Plot\")\n        '''\n        print(f'Fold: {fold_idx} | R2: {r2:.6f} | MSE: {mse:.6f} | Accuracy: {accuracy:.6f}')\n\n        #print(np.mean(np.abs((y_pred - y_test) / y_test)), np.abs(y_pred - y_test))\n    # Summary details\n    summary = {\n        \"Average R²\": np.mean(r2_scores),\n        \"Average MSE\": np.mean(mse_scores),\n        \"Root Average MSE\": np.sqrt(np.mean(mse_scores)),\n        \"Average accuracy\": np.mean(accuracy_scores)\n    }\n    print(\"\\nLinear Regression Summary\")\n    print(pd.DataFrame([summary]))\n    '''\n      plt.tight_layout()\n      plt.show()\n\n      r = np.log1p(residuals)\n\n      sns.histplot(r, kde=True)\n      plt.title(\"Log Distribution\")\n      plt.xlabel(\"Residuals\")\n      plt.ylabel(\"Frequency\")\n      plt.show()\n\n      # Q-Q plot\n      stats.probplot(r, dist=\"norm\", plot=plt)\n      plt.title(\"Q-Q Plot of Log\")\n      plt.show()\n    '''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:45:55.058327Z","iopub.execute_input":"2025-01-02T18:45:55.058859Z","iopub.status.idle":"2025-01-02T18:45:55.074481Z","shell.execute_reply.started":"2025-01-02T18:45:55.058819Z","shell.execute_reply":"2025-01-02T18:45:55.072846Z"},"id":"behRMblIYCCE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# target is cubic root\ntrain_copy = train.copy()\noriginal_copy = original.copy()\ncombined_df = pd.concat([train_copy, original_copy], ignore_index=True)\ncombined_df = combined_df.reset_index(drop=True)\nencoded_combined_df = pd.concat([encoded_train_df, encoded_original_df], ignore_index=True)\n\ntrain_model(LinearRegression(), None, combined_df, encoded_combined_df, combined_df[cbrt_y_target], True, dt)\n#train_model(LinearRegression(), None, combined_df, encoded_combined_df, None, False, 1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:45:58.98525Z","iopub.execute_input":"2025-01-02T18:45:58.985645Z","iopub.status.idle":"2025-01-02T18:46:18.003081Z","shell.execute_reply.started":"2025-01-02T18:45:58.98561Z","shell.execute_reply":"2025-01-02T18:46:18.000933Z"},"id":"QBPQfm-TYCCE","outputId":"145d12c4-c14a-47be-8e19-d78fe7fb433a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_copy = train.copy()\noriginal_copy = original.copy()\ncombined_df = pd.concat([train_copy, original_copy], ignore_index=True)\n\ntrain_model(LinearRegression(), None, train_copy, encoded_train_df, None, False, 1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:46:18.005679Z","iopub.execute_input":"2025-01-02T18:46:18.006211Z","iopub.status.idle":"2025-01-02T18:46:36.97915Z","shell.execute_reply.started":"2025-01-02T18:46:18.006161Z","shell.execute_reply":"2025-01-02T18:46:36.977335Z"},"id":"O95_mD4NYCCE","outputId":"403220e2-5eb4-423a-db97-a1ee22cbc4a7"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_copy = train.copy()\n\ntrain_model(LinearRegression(), None, train_copy, encoded_train_df, train_copy[cbrt_y_target], True, dt)","metadata":{"id":"ljHIBoWH3jkH","outputId":"60dbfce0-3713-4639-fd9d-9f887f207392"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom sklearn.preprocessing import PowerTransformer\n#no transformation\ntrain_copy = train.copy()\ntransformer = PowerTransformer(method='yeo-johnson')\ny_transformed = transformer.fit_transform(train_copy[y_target].values.reshape(-1, 1))\ntrain_model(LinearRegression(), None, train_copy, encoded_train_df, pd.DataFrame(y_transformed), True, 1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T01:34:30.370411Z","iopub.execute_input":"2024-12-31T01:34:30.370859Z","iopub.status.idle":"2024-12-31T01:34:52.472591Z","shell.execute_reply.started":"2024-12-31T01:34:30.370823Z","shell.execute_reply":"2024-12-31T01:34:52.470976Z"},"jupyter":{"outputs_hidden":true},"collapsed":true,"id":"92iAECyxYCCE","outputId":"c442c2bc-ef34-4208-c028-db6aaded3705"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#no transformation\ntrain_copy = train.copy()\nmodel = RandomForestRegressor(n_estimators=50, random_state=42)\ntrain_model(model, None, train_copy, encoded_train_df, None, False, 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-02T18:53:08.946395Z","iopub.execute_input":"2025-01-02T18:53:08.946772Z"},"id":"_Amit3vUYCCE","collapsed":true,"outputId":"2c02b81c-dd9f-4508-8cc2-ac10f6fa2e52","jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#no transformation\ntd = 3\ntrain_copy = train.copy()\ntransform_target(train_copy, 1/td)\nmodel = RandomForestRegressor(n_estimators=10, random_state=42)\ntrain_model(model, None, train_copy, encoded_train_df, train_copy[cbrt_y_target], True, td)","metadata":{"trusted":true,"id":"5JqO0sAhYCCE"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### to be continued","metadata":{"id":"MKQt8GYfb7Bz"},"outputs":[],"execution_count":null}]}