{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:15.903247Z","iopub.execute_input":"2024-12-30T16:25:15.903767Z","iopub.status.idle":"2024-12-30T16:25:15.909231Z","shell.execute_reply.started":"2024-12-30T16:25:15.903713Z","shell.execute_reply":"2024-12-30T16:25:15.907541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_dataset = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\ntrain_dataset.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:15.911176Z","iopub.execute_input":"2024-12-30T16:25:15.911714Z","iopub.status.idle":"2024-12-30T16:25:23.840111Z","shell.execute_reply.started":"2024-12-30T16:25:15.911648Z","shell.execute_reply":"2024-12-30T16:25:23.838758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:23.842331Z","iopub.execute_input":"2024-12-30T16:25:23.842763Z","iopub.status.idle":"2024-12-30T16:25:24.481104Z","shell.execute_reply.started":"2024-12-30T16:25:23.842728Z","shell.execute_reply":"2024-12-30T16:25:24.479985Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Visualization","metadata":{}},{"cell_type":"code","source":"data_to_visualize = train_dataset.drop(\"id\", axis=1)\n\ndata_to_visualize.hist(bins=100, figsize=(12,12))\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:24.482568Z","iopub.execute_input":"2024-12-30T16:25:24.482931Z","iopub.status.idle":"2024-12-30T16:25:27.823629Z","shell.execute_reply.started":"2024-12-30T16:25:24.482899Z","shell.execute_reply":"2024-12-30T16:25:27.822385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"![](https://web.fibion.com/wp-content/uploads/2024/01/25-Innovative-ESM-Data-Analysis-Identifying-Missing-Elements-in-Research-1.jpg)\n## As we can see and as the info method show us before there are missing data in the dataset, instead other data is well distributed and we dosen't have the problem of having data of only some categories","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))  # Dimensione del grafico\n\ntrain_dataset[\"Premium Amount\"].hist(bins=50, edgecolor='black')\n\nplt.title('Premium Amount Distribution')\nplt.xlabel('Premium Amount')\nplt.ylabel('Number of Values')\n\nplt.grid(True, alpha=0.3)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:27.824917Z","iopub.execute_input":"2024-12-30T16:25:27.825273Z","iopub.status.idle":"2024-12-30T16:25:28.120471Z","shell.execute_reply.started":"2024-12-30T16:25:27.825242Z","shell.execute_reply":"2024-12-30T16:25:28.119206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n\n# Premium Amount\nsns.histplot(data=train_dataset, x='Premium Amount', bins=50, ax=axes[0,0])\naxes[0,0].set_title('Premium Amount Distribution')\n\n# Age\nsns.histplot(data=train_dataset, x='Age', bins=30, ax=axes[0,1])\naxes[0,1].set_title('Age Distribution')\n\n# Annual Income\nsns.histplot(data=train_dataset, x='Annual Income', bins=40, ax=axes[1,0])\naxes[1,0].set_title('Annual Income Distribution')\n\n# Health Score\nsns.histplot(data=train_dataset, x='Health Score', bins=30, ax=axes[1,1])\naxes[1,1].set_title('Health Score Distribution')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:28.121625Z","iopub.execute_input":"2024-12-30T16:25:28.121984Z","iopub.status.idle":"2024-12-30T16:25:30.71571Z","shell.execute_reply.started":"2024-12-30T16:25:28.121954Z","shell.execute_reply":"2024-12-30T16:25:30.714508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.boxplot(data=train_dataset, x='Policy Type', y='Premium Amount')\nplt.xticks(rotation=45)\nplt.title('Premium Amount per Policy Type')\nplt.show()\n\n# Scatter plot per vedere la relazione con variabili numeriche\nplt.figure(figsize=(10, 6))\nsns.scatterplot(data=train_dataset, x='Age', y='Premium Amount', alpha=0.5)\nplt.title('Premium Amount vs Age')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:30.718157Z","iopub.execute_input":"2024-12-30T16:25:30.718507Z","iopub.status.idle":"2024-12-30T16:25:34.815806Z","shell.execute_reply.started":"2024-12-30T16:25:30.718474Z","shell.execute_reply":"2024-12-30T16:25:34.814546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.barplot(data=train_dataset, x='Gender', y='Premium Amount', hue='Marital Status')\nplt.title('Premium Amount medio per Gender e Marital Status')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:34.818207Z","iopub.execute_input":"2024-12-30T16:25:34.818633Z","iopub.status.idle":"2024-12-30T16:25:45.832761Z","shell.execute_reply.started":"2024-12-30T16:25:34.818592Z","shell.execute_reply":"2024-12-30T16:25:45.831421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix = train_dataset.corr(numeric_only=True)\n\nplt.figure(figsize=(12, 8))\nsns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0)\nplt.title('Correlation Matrix')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:45.833514Z","iopub.execute_input":"2024-12-30T16:25:45.833914Z","iopub.status.idle":"2024-12-30T16:25:46.829877Z","shell.execute_reply.started":"2024-12-30T16:25:45.833882Z","shell.execute_reply":"2024-12-30T16:25:46.828732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation_matrix[\"Premium Amount\"].sort_values(ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:46.831019Z","iopub.execute_input":"2024-12-30T16:25:46.831385Z","iopub.status.idle":"2024-12-30T16:25:46.838958Z","shell.execute_reply.started":"2024-12-30T16:25:46.83135Z","shell.execute_reply":"2024-12-30T16:25:46.83799Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.boxplot(data=train_dataset, x='Location', y='Premium Amount')\nplt.xticks(rotation=45)\nplt.title('Premium Amount per Location')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:46.840522Z","iopub.execute_input":"2024-12-30T16:25:46.841039Z","iopub.status.idle":"2024-12-30T16:25:47.68456Z","shell.execute_reply.started":"2024-12-30T16:25:46.840972Z","shell.execute_reply":"2024-12-30T16:25:47.683542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.violinplot(data=train_dataset, x='Exercise Frequency', y='Premium Amount', hue='Smoking Status')\nplt.title('Premium Amount per Exercise Frequency e Smoking Status')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:47.685521Z","iopub.execute_input":"2024-12-30T16:25:47.68579Z","iopub.status.idle":"2024-12-30T16:25:52.150307Z","shell.execute_reply.started":"2024-12-30T16:25:47.685765Z","shell.execute_reply":"2024-12-30T16:25:52.149233Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Normalize and encode data, after that we will search if some new correlation of previous non numeric values will show to us","metadata":{}},{"cell_type":"code","source":"import sklearn\n\nsklearn.__version__","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:52.151669Z","iopub.execute_input":"2024-12-30T16:25:52.151983Z","iopub.status.idle":"2024-12-30T16:25:52.157702Z","shell.execute_reply.started":"2024-12-30T16:25:52.151957Z","shell.execute_reply":"2024-12-30T16:25:52.156668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:52.158699Z","iopub.execute_input":"2024-12-30T16:25:52.15904Z","iopub.status.idle":"2024-12-30T16:25:52.172894Z","shell.execute_reply.started":"2024-12-30T16:25:52.159003Z","shell.execute_reply":"2024-12-30T16:25:52.171969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_pipeline = Pipeline([\n    (\"impute\", SimpleImputer(strategy=\"median\")),\n    (\"standardize\", StandardScaler()),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:52.174369Z","iopub.execute_input":"2024-12-30T16:25:52.174797Z","iopub.status.idle":"2024-12-30T16:25:52.186713Z","shell.execute_reply.started":"2024-12-30T16:25:52.174753Z","shell.execute_reply":"2024-12-30T16:25:52.185735Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_pipeline = make_pipeline(\n    SimpleImputer(strategy=\"most_frequent\"),\n    OneHotEncoder(sparse=False, handle_unknown=\"ignore\", max_categories=10)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:52.187741Z","iopub.execute_input":"2024-12-30T16:25:52.188125Z","iopub.status.idle":"2024-12-30T16:25:52.200689Z","shell.execute_reply.started":"2024-12-30T16:25:52.188084Z","shell.execute_reply":"2024-12-30T16:25:52.199659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.compose import make_column_selector, make_column_transformer\n\npreprocessing = make_column_transformer(\n    (num_pipeline, make_column_selector(dtype_include=np.number)),\n    (cat_pipeline, make_column_selector(dtype_include=\"object\"))\n)\n\nparsed_train_data = train_dataset.drop(['id', 'Premium Amount'], axis=1)\ntrain_labels = train_dataset[\"Premium Amount\"]\n\nparsed_train_data = preprocessing.fit_transform(parsed_train_data)\n\nparsed_train_data_dataframe = pd.DataFrame(\n    parsed_train_data,\n    columns = preprocessing.get_feature_names_out(),\n    index = train_dataset.index\n)\n\nparsed_train_data_dataframe.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:25:52.201577Z","iopub.execute_input":"2024-12-30T16:25:52.201956Z","iopub.status.idle":"2024-12-30T16:26:10.940106Z","shell.execute_reply.started":"2024-12-30T16:25:52.201919Z","shell.execute_reply":"2024-12-30T16:26:10.939094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:26:10.941076Z","iopub.execute_input":"2024-12-30T16:26:10.941377Z","iopub.status.idle":"2024-12-30T16:26:10.948373Z","shell.execute_reply.started":"2024-12-30T16:26:10.941348Z","shell.execute_reply":"2024-12-30T16:26:10.947327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"parsed_train_data_dataframe[\"Premium Amount\"] = train_labels\ncorrelation_matrix = parsed_train_data_dataframe.corr(numeric_only=True)\n\ncorrelation_matrix[\"Premium Amount\"].sort_values(ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:26:10.95211Z","iopub.execute_input":"2024-12-30T16:26:10.952397Z","iopub.status.idle":"2024-12-30T16:26:19.606721Z","shell.execute_reply.started":"2024-12-30T16:26:10.952371Z","shell.execute_reply":"2024-12-30T16:26:19.605711Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engenieering and additional parsing removing outliers","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\n\ndef add_engineered_features(df):\n    \"\"\"\n    Aggiunge feature engineered al dataframe originale\n    \"\"\"\n\n    df = df.copy()\n    \n    df['health_risk'] = df['Health Score'] * df['Smoking Status'].map({'Yes': 1.5, 'No': 1.0})\n    df['lifestyle_score'] = (df['Exercise Frequency']\n                           .map({'Daily': 1.0, 'Weekly': 0.8, 'Monthly': 0.6, 'Rarely': 0.4}) \n                           * df['Health Score'])\n    \n    df['credit_income_ratio'] = df['Credit Score'] / df['Annual Income']\n    \n    df['claims_per_year'] = df['Previous Claims'] / df['Insurance Duration']\n    df['age_health_factor'] = df['Age'] * (df['Health Score'] / 100)\n\n    df['financial_risk_score'] = (df['credit_income_ratio'] * df['claims_per_year'])\n\n    # Score che combina età e claims\n    df['age_claims_interaction'] = df['Age'] * df['Previous Claims']\n    \n    # Ratio tra health_risk e credit_score\n    df['health_credit_ratio'] = df['health_risk'] / df['Credit Score']\n\n    df['weighted_risk_score'] = (\n        df['Previous Claims'] * 0.4 +\n        (100 - df['Health Score']) * 0.3 +\n        df['claims_per_year'] * 0.3\n    )\n    \n    # Bucketing di alcune variabili numeriche\n    df['age_group'] = pd.qcut(df['Age'], q=5, labels=['very_young', 'young', 'middle', 'senior', 'elderly'])\n    df['income_group'] = pd.qcut(df['Annual Income'], q=5, labels=['very_low', 'low', 'medium', 'high', 'very_high'])\n\n    df['premium_risk_score'] = (df['Previous Claims'] * df['Credit Score']) / df['Annual Income']\n    df['wealth_indicator'] = df['Annual Income'] * (df['Credit Score'] / 100)\n\n    return df\n\nclass OutlierRemover(BaseEstimator, TransformerMixin):\n    def __init__(self, factor=1.5):\n        self.factor = factor\n        self.lower_bounds = None\n        self.upper_bounds = None\n        \n    def fit(self, X, y=None):\n        # Converte in array numpy se non lo è già\n        X_array = np.asarray(X)\n        \n        # Calcola i bounds per ogni colonna\n        q1 = np.quantile(X_array, 0.25, axis=0)\n        q3 = np.quantile(X_array, 0.75, axis=0)\n        iqr = q3 - q1\n        \n        self.lower_bounds = q1 - (self.factor * iqr)\n        self.upper_bounds = q3 + (self.factor * iqr)\n        \n        return self\n    \n    def transform(self, X):\n        # Converte in array numpy se non lo è già\n        X_array = np.asarray(X)\n        \n        # Applica i bounds\n        X_transformed = np.minimum(np.maximum(X_array, self.lower_bounds), self.upper_bounds)\n        \n        return X_transformed\n\n    def get_feature_names_out(self, feature_names=None):\n        \"\"\"Restituisce i nomi delle feature dopo la trasformazione\"\"\"\n        return feature_names if feature_names is not None else np.array([f\"feature{i}\" for i in range(self.n_features_in_)])\n\nnum_pipeline = Pipeline([\n    (\"impute\", SimpleImputer(strategy=\"median\")),\n    (\"standardize\", StandardScaler()),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:09:08.85114Z","iopub.execute_input":"2024-12-30T19:09:08.851624Z","iopub.status.idle":"2024-12-30T19:09:08.867213Z","shell.execute_reply.started":"2024-12-30T19:09:08.851586Z","shell.execute_reply":"2024-12-30T19:09:08.865677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessing = make_column_transformer(\n    (num_pipeline, make_column_selector(dtype_include=np.number)),\n    (cat_pipeline, make_column_selector(dtype_include=\"object\")),\n    remainder='drop'\n)\n\nparsed_train_data = train_dataset.drop(['id', 'Premium Amount'], axis=1)\ntrain_labels = train_dataset[\"Premium Amount\"]\n\nparsed_train_data = add_engineered_features(parsed_train_data)\n\nparsed_train_data = preprocessing.fit_transform(parsed_train_data)\n\nparsed_train_data_dataframe = pd.DataFrame(\n    parsed_train_data,\n    columns = preprocessing.get_feature_names_out(),\n    index = train_dataset.index\n)\n\nparsed_train_data_dataframe.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:26:19.62324Z","iopub.execute_input":"2024-12-30T16:26:19.623601Z","iopub.status.idle":"2024-12-30T16:26:42.488246Z","shell.execute_reply.started":"2024-12-30T16:26:19.623567Z","shell.execute_reply":"2024-12-30T16:26:42.487187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"parsed_train_data_dataframe[\"Premium Amount\"] = train_labels\ncorrelation_matrix = parsed_train_data_dataframe.corr(numeric_only=True)\n\ncorrelation_matrix[\"Premium Amount\"].sort_values(ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T16:26:42.489002Z","iopub.execute_input":"2024-12-30T16:26:42.489268Z","iopub.status.idle":"2024-12-30T16:26:55.234453Z","shell.execute_reply.started":"2024-12-30T16:26:42.489244Z","shell.execute_reply":"2024-12-30T16:26:55.233301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Different Models","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\nfrom sklearn.preprocessing import PowerTransformer\n\ncolumns_to_keep = [\n    'Previous Claims',\n    'age_claims_interaction',\n    'claims_per_year',\n    'credit_income_ratio',\n    'health_credit_ratio',\n    'financial_risk_score',\n    'Health Score',\n    'health_risk',\n    'lifestyle_score',\n    'age_health_factor',\n    'Credit Score',\n    'Customer Feedback',\n    'Policy Start Date'\n    'Annual Income',\n    'weighted_risk_score',\n    'age_group',\n    'income_group',\n    'premium_risk_score',\n    'wealth_indicator',\n]\n\nlog_pipeline = make_pipeline(\n   SimpleImputer(strategy=\"median\"),\n   PowerTransformer(method='yeo-johnson'),\n   RobustScaler()\n)\n\nnum_pipeline = Pipeline([\n    (\"impute\", SimpleImputer(strategy=\"mean\")),\n    (\"standardize\", RobustScaler()),\n])\n\ncat_pipeline = make_pipeline(\n    SimpleImputer(strategy=\"most_frequent\"),\n    OneHotEncoder(sparse=False, handle_unknown=\"ignore\", max_categories=15)\n)\n\npreprocessing = make_column_transformer(\n    (log_pipeline, [\"Annual Income\"]),\n    (num_pipeline, make_column_selector(dtype_include=np.number)),\n    (cat_pipeline, make_column_selector(dtype_include=\"object\")),\n    remainder='drop'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:13:34.134571Z","iopub.execute_input":"2024-12-30T17:13:34.134956Z","iopub.status.idle":"2024-12-30T17:13:34.142848Z","shell.execute_reply.started":"2024-12-30T17:13:34.134923Z","shell.execute_reply":"2024-12-30T17:13:34.141586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = add_engineered_features(train_dataset)\n\ntrain_data = train_dataset[columns_to_keep]\ntrain_labels = train_dataset[\"Premium Amount\"]\n\ntest_data = test_dataset.drop(['id'], axis=1)\n\ntrain_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:04:51.678306Z","iopub.execute_input":"2024-12-30T17:04:51.678716Z","iopub.status.idle":"2024-12-30T17:04:53.085387Z","shell.execute_reply.started":"2024-12-30T17:04:51.67868Z","shell.execute_reply":"2024-12-30T17:04:53.084268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nclass PositiveLinearRegression(LinearRegression):\n    def predict(self, X):\n        return np.maximum(0, super().predict(X))\n\nlin_reg = make_pipeline(preprocessing, PositiveLinearRegression())\nlin_reg.fit(train_data, train_labels)\n\npredictions = lin_reg.predict(train_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:13:41.995827Z","iopub.execute_input":"2024-12-30T17:13:41.996155Z","iopub.status.idle":"2024-12-30T17:14:01.501283Z","shell.execute_reply.started":"2024-12-30T17:13:41.996128Z","shell.execute_reply":"2024-12-30T17:14:01.50004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error\n\ndef calculate_mean_squared_log_error(labels, predictions):\n    return np.sqrt(mean_squared_log_error(labels, predictions))\n\ncalculate_mean_squared_log_error(train_labels, predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:14:09.904607Z","iopub.execute_input":"2024-12-30T17:14:09.904998Z","iopub.status.idle":"2024-12-30T17:14:09.967105Z","shell.execute_reply.started":"2024-12-30T17:14:09.904968Z","shell.execute_reply":"2024-12-30T17:14:09.96606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\n# Creazione del modello XGBoost con alcuni iperparametri di base\nxgb_reg = make_pipeline(preprocessing,\n                       XGBRegressor(\n                           n_estimators=100,\n                           learning_rate=0.1,\n                           max_depth=5,\n                           random_state=42\n                       ))\ntrain_labels_log = np.log1p(train_labels)\n\nxgb_reg.fit(train_data, train_labels_log)\n\nxgb_predictions = np.expm1(xgb_reg.predict(train_data))\n\ncalculate_mean_squared_log_error(train_labels, xgb_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:05:41.067778Z","iopub.execute_input":"2024-12-30T17:05:41.06818Z","iopub.status.idle":"2024-12-30T17:06:02.357097Z","shell.execute_reply.started":"2024-12-30T17:05:41.068149Z","shell.execute_reply":"2024-12-30T17:06:02.355915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_log_error\nimport numpy as np\n\n# Creazione del modello LightGBM con iperparametri simili a XGBoost\nlgbm_reg = make_pipeline(preprocessing,\n                        LGBMRegressor(\n                            n_estimators=100,\n                            learning_rate=0.1,\n                            max_depth=5,\n                            random_state=42\n                        ))\n\ntrain_labels_log = np.log1p(train_labels)\n\nlgbm_reg.fit(train_data, train_labels_log)\n\nlgbm_predictions = np.expm1(lgbm_reg.predict(train_data))\n\ncalculate_mean_squared_log_error(train_labels, lgbm_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:06:11.855507Z","iopub.execute_input":"2024-12-30T17:06:11.855932Z","iopub.status.idle":"2024-12-30T17:06:36.10031Z","shell.execute_reply.started":"2024-12-30T17:06:11.855896Z","shell.execute_reply":"2024-12-30T17:06:36.099077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot degli errori\nerrors = abs(lgbm_predictions - train_labels)\nplt.figure(figsize=(10,6))\nplt.scatter(train_labels, errors, alpha=0.5)\nplt.xlabel('Actual Premium Amount')\nplt.ylabel('Prediction Error')\nplt.title('Error Analysis - Mean Imputation')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:14:16.435271Z","iopub.execute_input":"2024-12-30T17:14:16.435688Z","iopub.status.idle":"2024-12-30T17:14:20.256941Z","shell.execute_reply.started":"2024-12-30T17:14:16.435654Z","shell.execute_reply":"2024-12-30T17:14:20.255731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# As We Know before the dataset is unbalanced in the higher and loer price range so that error distribution for no is normal\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV\nfrom scipy.stats import uniform, randint, loguniform\n\nparam_distributions = {\n   'lgbmregressor__n_estimators': randint(100, 500),\n   'lgbmregressor__learning_rate': loguniform(1e-3, 1e-1),\n   'lgbmregressor__max_depth': randint(4, 12),\n   'lgbmregressor__num_leaves': randint(30, 150),\n   'lgbmregressor__min_child_samples': randint(5, 30),\n   'lgbmregressor__subsample': uniform(0.5, 0.5),\n   'lgbmregressor__colsample_bytree': uniform(0.5, 0.5),\n   'lgbmregressor__reg_alpha': loguniform(1e-3, 10),\n   'lgbmregressor__reg_lambda': loguniform(1e-3, 10)\n}\n\nrandom_search = RandomizedSearchCV(\n   estimator=lgbm_reg,\n   param_distributions=param_distributions,\n   n_iter=70,\n   cv=3,\n   scoring='neg_mean_squared_log_error',\n   random_state=42,\n   verbose=2,\n)\n\nrandom_search.fit(train_data, train_labels_log)\n\nprint(\"Migliori parametri:\", random_search.best_params_)\nprint(\"Miglior punteggio:\", -random_search.best_score_)\n\nbest_lgbm_model = random_search.best_estimator_\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:17:28.201123Z","iopub.execute_input":"2024-12-30T17:17:28.201799Z","iopub.status.idle":"2024-12-30T18:57:47.788803Z","shell.execute_reply.started":"2024-12-30T17:17:28.201742Z","shell.execute_reply":"2024-12-30T18:57:47.785861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm_predictions = np.expm1(best_lgbm_model.predict(train_data))\n\ncalculate_mean_squared_log_error(train_labels, lgbm_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:04:52.464732Z","iopub.execute_input":"2024-12-30T19:04:52.46535Z","iopub.status.idle":"2024-12-30T19:05:12.55886Z","shell.execute_reply.started":"2024-12-30T19:04:52.465306Z","shell.execute_reply":"2024-12-30T19:05:12.557535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## awesome we reduce the error with automatic hyperparameters searching!","metadata":{}},{"cell_type":"markdown","source":"# Final Submission\nwe now know what model to use (lightGBM) and know best hyperparameters\nso it's time to have our final model and get the final results for the submission","metadata":{}},{"cell_type":"code","source":"test_dataset.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:06:48.360094Z","iopub.execute_input":"2024-12-30T19:06:48.360701Z","iopub.status.idle":"2024-12-30T19:06:48.394395Z","shell.execute_reply.started":"2024-12-30T19:06:48.36066Z","shell.execute_reply":"2024-12-30T19:06:48.393095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = add_engineered_features(test_dataset)\n\ntest_dataset['Policy Start Date']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:12:00.190937Z","iopub.execute_input":"2024-12-30T19:12:00.191311Z","iopub.status.idle":"2024-12-30T19:12:00.746392Z","shell.execute_reply.started":"2024-12-30T19:12:00.19128Z","shell.execute_reply":"2024-12-30T19:12:00.74546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_to_keep = [\n    'Previous Claims',\n    'age_claims_interaction',\n    'claims_per_year',\n    'credit_income_ratio',\n    'health_credit_ratio',\n    'financial_risk_score',\n    'Health Score',\n    'health_risk',\n    'lifestyle_score',\n    'age_health_factor',\n    'Credit Score',\n    'Customer Feedback',\n    'Policy Start Date',\n    'Annual Income',\n    'weighted_risk_score',\n    'age_group',\n    'income_group',\n    'premium_risk_score',\n    'wealth_indicator',\n]\n\ntest_dataset.head()\n\ntest_dataset_cleaned = test_dataset[columns_to_keep]\n\ntest_predictions = np.expm1(best_lgbm_model.predict(test_dataset_cleaned))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:15:12.903208Z","iopub.execute_input":"2024-12-30T19:15:12.903658Z","iopub.status.idle":"2024-12-30T19:15:26.527723Z","shell.execute_reply.started":"2024-12-30T19:15:12.903614Z","shell.execute_reply":"2024-12-30T19:15:26.526676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({'id': test_dataset['id'],'Premium Amount': test_predictions})\nsubmission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:15:38.403306Z","iopub.execute_input":"2024-12-30T19:15:38.403728Z","iopub.status.idle":"2024-12-30T19:15:38.420206Z","shell.execute_reply.started":"2024-12-30T19:15:38.403686Z","shell.execute_reply":"2024-12-30T19:15:38.418819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('submission.csv',index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T19:15:44.299755Z","iopub.execute_input":"2024-12-30T19:15:44.300127Z","iopub.status.idle":"2024-12-30T19:15:45.914612Z","shell.execute_reply.started":"2024-12-30T19:15:44.300096Z","shell.execute_reply":"2024-12-30T19:15:45.913547Z"}},"outputs":[],"execution_count":null}]}