{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"_uuid":"23568bf6-b483-4001-ad1d-86e3be74eed4","_cell_guid":"3a936ab5-7f05-401c-a532-f32d2a34477f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Hi!\nThis is new playground competition and in this month we're having an insurance dataset. \n\nBefore diving deeper into code, let's describe a little topic in which we will work.\n\nFor start, a little meme which describes all this thing :)\n\n![](https://leadsurance.com/wp-content/uploads/2020/08/insurance-meme-6.jpg)\r\n","metadata":{}},{"cell_type":"markdown","source":"In my previous notebook I was combined EDA and training\nhttps://www.kaggle.com/code/iwways/calculating-premiums-like-a-pro-or-not\n\nBut I decided to make training in separate notebook, cause a lot of things is going here :)","metadata":{}},{"cell_type":"markdown","source":"# Import libraries","metadata":{"_uuid":"23e0c399-9c79-4374-8847-08df1b1784aa","_cell_guid":"096ad279-cbc5-49fd-b80c-9f34e221487e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_log_error\nimport pandas as pd\nimport missingno as mnso\nimport numpy as np\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy.stats import boxcox\nfrom scipy.special import boxcox1p\nfrom sklearn.model_selection import cross_val_score, StratifiedKFold\n# To ignore warinings\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"90871205-85e8-45d8-97a6-490aee449832","_cell_guid":"f9eafd57-6931-4703-8bb0-eef4e35e86e7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:29:13.995264Z","iopub.execute_input":"2024-12-30T11:29:13.995685Z","iopub.status.idle":"2024-12-30T11:29:14.0014Z","shell.execute_reply.started":"2024-12-30T11:29:13.995652Z","shell.execute_reply":"2024-12-30T11:29:14.000595Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load data","metadata":{"_uuid":"d005054f-58e9-4091-b065-00c10dbfee18","_cell_guid":"229240ab-e333-4f0a-a562-7749e4add3aa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"_kg_hide-output":true}},{"cell_type":"code","source":"X_train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\nX_test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\nX_train = X_train.drop(['id'], axis=1)\n\nid_test = X_test.id\nX_test = X_test.drop('id', axis=1)\n\n# y = X_train['Premium Amount']\n# y_log = np.log1p(y)\n\npd.set_option('display.max_columns', None)\n","metadata":{"_uuid":"cbbb9373-1d55-4852-8548-09338a7de6d5","_cell_guid":"b811f64a-b2c4-4899-bdcf-9f948b67f048","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:15.732155Z","iopub.execute_input":"2024-12-30T11:29:15.73297Z","iopub.status.idle":"2024-12-30T11:29:21.810751Z","shell.execute_reply.started":"2024-12-30T11:29:15.732912Z","shell.execute_reply":"2024-12-30T11:29:21.810005Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"After exploring discussions it seems reasonable to log transform our target variable to ensure that we optimizing directly the evaluation metric for this competition","metadata":{}},{"cell_type":"markdown","source":"# Feature enginereering","metadata":{"_uuid":"4b8e2aa6-1e78-4757-9db2-b301d24ff973","_cell_guid":"ddf59f25-6dc9-44f0-b027-5d3cb723cf8c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"# Log Transformation of Annual Income","metadata":{}},{"cell_type":"code","source":"X_train['Annual Income'] = np.log1p(X_train['Annual Income'])\nX_test['Annual Income'] = np.log1p(X_test['Annual Income'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:21.812032Z","iopub.execute_input":"2024-12-30T11:29:21.81237Z","iopub.status.idle":"2024-12-30T11:29:21.828637Z","shell.execute_reply.started":"2024-12-30T11:29:21.812339Z","shell.execute_reply":"2024-12-30T11:29:21.827969Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It's will be useful if we separate Policy Start Date feature to different features, as day, month and other.","metadata":{"_uuid":"3f384643-3c00-4786-aba8-2ad1befb2f17","_cell_guid":"0dac9a9b-b742-49ad-b992-a13a11eb3c68","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Time features","metadata":{"_uuid":"e1b37d48-d5fb-4d6a-a474-eda9b345157e","_cell_guid":"8e4d91ba-46d2-4576-a5a3-7cd1468c373b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"X_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:21.829906Z","iopub.execute_input":"2024-12-30T11:29:21.830287Z","iopub.status.idle":"2024-12-30T11:29:21.852536Z","shell.execute_reply.started":"2024-12-30T11:29:21.830249Z","shell.execute_reply":"2024-12-30T11:29:21.851555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_policy_start_date_separated_features(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'], errors='coerce')\n    df['Policy_Start_Year'] = df['Policy Start Date'].dt.year\n    df['Policy_Start_Month'] = df['Policy Start Date'].dt.month\n    df['Policy_Start_Day'] = df['Policy Start Date'].dt.day\n    \n\n    return df\n","metadata":{"_uuid":"68bb2d92-fc50-4001-86c9-c901fc3b898b","_cell_guid":"f91e09f7-6847-47c2-bebc-445f3801544f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:29:21.854627Z","iopub.execute_input":"2024-12-30T11:29:21.854878Z","iopub.status.idle":"2024-12-30T11:29:21.864849Z","shell.execute_reply.started":"2024-12-30T11:29:21.854853Z","shell.execute_reply":"2024-12-30T11:29:21.864236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef circular_features_time(df, date_column='Policy Start Date'):\n    df[date_column] = pd.to_datetime(df[date_column])\n    \n    df['day_of_week'] = df[date_column].dt.dayofweek\n    df['day_sin'] = np.sin(2 * np.pi * df['day_of_week'] / 7)\n    df['day_cos'] = np.cos(2 * np.pi * df['day_of_week'] / 7)\n    \n    df['day_of_year'] = df[date_column].dt.dayofyear\n    df['year_day_sin'] = np.sin(2 * np.pi * df['day_of_year'] / 365)\n    df['year_day_cos'] = np.cos(2 * np.pi * df['day_of_year'] / 365)\n    \n    df['month'] = df[date_column].dt.month\n    df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)\n    df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)\n    \n    df[\"seconds_since_1970\"] = df[date_column].astype(\"int64\") // 10**9\n    df = df.drop('Policy Start Date', axis=1)\n\n    return df\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:21.865736Z","iopub.execute_input":"2024-12-30T11:29:21.866055Z","iopub.status.idle":"2024-12-30T11:29:21.875992Z","shell.execute_reply.started":"2024-12-30T11:29:21.866018Z","shell.execute_reply":"2024-12-30T11:29:21.875394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = create_policy_start_date_separated_features(X_train)\ntest = create_policy_start_date_separated_features(X_test)\n\ntrain = circular_features_time(X_train)\ntest = circular_features_time(X_test)","metadata":{"_uuid":"e3611915-3b15-4555-8d02-2d15fb9c27d4","_cell_guid":"9fc5b863-5c93-439b-a9de-6b701d58fc1b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:29:21.876992Z","iopub.execute_input":"2024-12-30T11:29:21.877236Z","iopub.status.idle":"2024-12-30T11:29:23.589965Z","shell.execute_reply.started":"2024-12-30T11:29:21.877211Z","shell.execute_reply":"2024-12-30T11:29:23.589263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:29:23.591469Z","iopub.execute_input":"2024-12-30T11:29:23.591713Z","iopub.status.idle":"2024-12-30T11:29:23.622653Z","shell.execute_reply.started":"2024-12-30T11:29:23.591689Z","shell.execute_reply":"2024-12-30T11:29:23.621699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"_uuid":"4971d1cd-87e7-45ed-9091-55f75b305fc3","_cell_guid":"aefcfb19-7315-4afd-be9b-852d089a44de","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:25:46.032661Z","iopub.execute_input":"2024-12-30T11:25:46.032937Z","iopub.status.idle":"2024-12-30T11:25:46.657571Z","shell.execute_reply.started":"2024-12-30T11:25:46.032911Z","shell.execute_reply":"2024-12-30T11:25:46.656601Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Handling missing values TODO","metadata":{"_uuid":"bb704129-5119-44cc-867e-57d918bebf63","_cell_guid":"b912347f-8bf8-49c1-96e4-7e33686de49b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"After Chris's Deotte post i decided not to impute values for NaN's, so let's check how our results will be.\n\nUPD: adding an column indicator for missing values","metadata":{"_uuid":"bb704129-5119-44cc-867e-57d918bebf63","_cell_guid":"b912347f-8bf8-49c1-96e4-7e33686de49b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def impute_missing_numerical_data(df):\n    num = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score',\n       'Previous Claims', 'Vehicle Age', 'Credit Score', 'Insurance Duration']\n    for n in num:\n        df[f\"is_{n}_na\"] = df[n].isna()\n        df[n] = df[n].fillna(df[n].mean())\n\n    return df\n\ndef impute_missing_categorical_data(df):\n    cat = ['Marital Status', 'Occupation', 'Customer Feedback']\n    for c in cat:\n        df[f\"is_{c}_na\"] = df[c].isna()\n        df[c] = df[c].fillna(df[c].mode())\n\n    return df","metadata":{"_uuid":"978063c4-d8aa-4079-b287-266dc8b606f4","_cell_guid":"cdf31001-966e-42f1-a89a-a0d09f9feaa0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:38:43.383083Z","iopub.execute_input":"2024-12-30T11:38:43.383908Z","iopub.status.idle":"2024-12-30T11:38:43.389181Z","shell.execute_reply.started":"2024-12-30T11:38:43.383873Z","shell.execute_reply":"2024-12-30T11:38:43.38824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = impute_missing_categorical_data(train)\ntest = impute_missing_categorical_data(test)","metadata":{"_uuid":"cc5b2bf9-68b3-437f-9bc9-e82e0015c662","_cell_guid":"733caeea-2d3b-4435-a1ca-58da742f6987","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-30T11:38:45.239077Z","iopub.execute_input":"2024-12-30T11:38:45.239494Z","iopub.status.idle":"2024-12-30T11:38:46.3256Z","shell.execute_reply.started":"2024-12-30T11:38:45.239462Z","shell.execute_reply":"2024-12-30T11:38:46.324895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = impute_missing_numerical_data(train)\ntest = impute_missing_numerical_data(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:38:50.214089Z","iopub.execute_input":"2024-12-30T11:38:50.21492Z","iopub.status.idle":"2024-12-30T11:38:50.384293Z","shell.execute_reply.started":"2024-12-30T11:38:50.214882Z","shell.execute_reply":"2024-12-30T11:38:50.383543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T11:38:54.369691Z","iopub.execute_input":"2024-12-30T11:38:54.370574Z","iopub.status.idle":"2024-12-30T11:38:54.422748Z","shell.execute_reply.started":"2024-12-30T11:38:54.370524Z","shell.execute_reply":"2024-12-30T11:38:54.421674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:48.202347Z","iopub.execute_input":"2024-12-27T10:10:48.202629Z","iopub.status.idle":"2024-12-27T10:10:48.210653Z","shell.execute_reply.started":"2024-12-27T10:10:48.202585Z","shell.execute_reply":"2024-12-27T10:10:48.209939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Scaling numerical features","metadata":{}},{"cell_type":"code","source":"num_cols = X_train.select_dtypes(exclude=['object', 'datetime', 'bool']).columns.tolist()\nnum_cols.remove('Premium Amount')\ncat_cols = X_train.select_dtypes(include='object').columns.tolist()\n\nscaler = StandardScaler()\n\nX_train[num_cols] = scaler.fit_transform(X_train[num_cols])\nX_test[num_cols] = scaler.transform(X_test[num_cols])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:48.211525Z","iopub.execute_input":"2024-12-27T10:10:48.211787Z","iopub.status.idle":"2024-12-27T10:10:50.116966Z","shell.execute_reply.started":"2024-12-27T10:10:48.211751Z","shell.execute_reply":"2024-12-27T10:10:50.115846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train[num_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:50.118687Z","iopub.execute_input":"2024-12-27T10:10:50.119081Z","iopub.status.idle":"2024-12-27T10:10:50.264556Z","shell.execute_reply.started":"2024-12-27T10:10:50.11904Z","shell.execute_reply":"2024-12-27T10:10:50.263659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Encoding categorical features","metadata":{"_uuid":"d0f10256-3737-41b5-b176-266f1c05a523","_cell_guid":"e715bb0a-fc8a-4b6d-9362-2d8f2b3e5e33","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"Now it's time for encoding!","metadata":{"_uuid":"0b3e07c5-1117-4a33-8740-f6c32aefdbd8","_cell_guid":"b4cb4831-74c9-4a96-bf33-1f62c88c029f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def encode_ordinal(df):\n    educ = {\"High School\":0, \"Bachelor's\":1, \"Master's\":2, \"PhD\":3}\n    policy = {'Basic':0, 'Comprehensive':1, 'Premium':2}\n    exerc = {'Rarely':0, 'Daily':1, 'Weekly':2, 'Monthly': 3}\n    feedback = {'Poor':0, 'Average':1, 'Good':2, \"Unknown\": 0}\n\n    df['Education Level'] = df['Education Level'].map(educ)\n    df['Policy Type'] = df['Policy Type'].map(policy)\n    df['Exercise Frequency'] = df['Exercise Frequency'].map(exerc)\n    df['Customer Feedback'] = df['Customer Feedback'].map(feedback)\n    return df","metadata":{"_uuid":"ef800310-dfdb-418d-aa4a-6aecd3e290c4","_cell_guid":"4ac89d06-a052-4ea1-8025-1c85b8b13a76","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.26553Z","iopub.execute_input":"2024-12-27T10:10:50.265792Z","iopub.status.idle":"2024-12-27T10:10:50.270908Z","shell.execute_reply.started":"2024-12-27T10:10:50.265764Z","shell.execute_reply":"2024-12-27T10:10:50.270051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_binary(df):\n    df['Gender'] = df['Gender'].map({'Male':0, 'Female':1})\n    df['Smoking Status'] = df['Smoking Status'].map({'Yes':1, 'No':0})\n    return df","metadata":{"_uuid":"a47d5d6d-2303-4dfe-96d2-d14e539f7638","_cell_guid":"dcf14fb1-7862-4e65-80fc-24ee42a77b01","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.271834Z","iopub.execute_input":"2024-12-27T10:10:50.272051Z","iopub.status.idle":"2024-12-27T10:10:50.283248Z","shell.execute_reply.started":"2024-12-27T10:10:50.272028Z","shell.execute_reply":"2024-12-27T10:10:50.282462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def one_hot_dummies(df, categorical):\n    oh = pd.get_dummies(df[categorical])\n    df = df.drop(categorical, axis=1)\n    return pd.concat([df, oh], axis=1)","metadata":{"_uuid":"91a2932b-4f4b-4578-bbbf-5f437745e712","_cell_guid":"10d6970a-2459-455e-9073-852c88eb3e03","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.284198Z","iopub.execute_input":"2024-12-27T10:10:50.284429Z","iopub.status.idle":"2024-12-27T10:10:50.293102Z","shell.execute_reply.started":"2024-12-27T10:10:50.284406Z","shell.execute_reply":"2024-12-27T10:10:50.292376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = X_train['Premium Amount']\ntrain = train.drop('Premium Amount', axis=1)\ny_log = np.log1p(y)","metadata":{"_uuid":"9fe74935-1a3c-46be-a1de-73363fe5b242","_cell_guid":"e894ad87-229e-494e-9b26-cac4c354e6ab","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.294027Z","iopub.execute_input":"2024-12-27T10:10:50.294273Z","iopub.status.idle":"2024-12-27T10:10:50.509392Z","shell.execute_reply.started":"2024-12-27T10:10:50.294249Z","shell.execute_reply":"2024-12-27T10:10:50.508663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = encode_binary(train)\ntest = encode_binary(test)","metadata":{"_uuid":"859a0f4c-cafa-4366-ad38-74a726574da5","_cell_guid":"b8f96678-9790-49cf-9d0a-1ab5ee68f31f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.510376Z","iopub.execute_input":"2024-12-27T10:10:50.510628Z","iopub.status.idle":"2024-12-27T10:10:50.720808Z","shell.execute_reply.started":"2024-12-27T10:10:50.510603Z","shell.execute_reply":"2024-12-27T10:10:50.720072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = encode_ordinal(train)\ntest = encode_ordinal(test)","metadata":{"_uuid":"4c6401e7-6c9d-4388-b89c-7cef7fa22409","_cell_guid":"b9cc9b7c-e170-47d2-ac45-3947df051995","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:50.724329Z","iopub.execute_input":"2024-12-27T10:10:50.724618Z","iopub.status.idle":"2024-12-27T10:10:51.173449Z","shell.execute_reply.started":"2024-12-27T10:10:50.724589Z","shell.execute_reply":"2024-12-27T10:10:51.172743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"_uuid":"eef5cc6b-d5e7-4c80-9fe5-06562a8a8123","_cell_guid":"4f0d6343-6a8b-4202-8389-c840f24fc039","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:51.17487Z","iopub.execute_input":"2024-12-27T10:10:51.17524Z","iopub.status.idle":"2024-12-27T10:10:51.201267Z","shell.execute_reply.started":"2024-12-27T10:10:51.175196Z","shell.execute_reply":"2024-12-27T10:10:51.200288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_features_total = test.select_dtypes(exclude='object').columns\nnumerical_features_continuos = test.select_dtypes(exclude=['object', 'int']).columns\nnumerical_features_discrete = test.select_dtypes(exclude=['object', 'float']).columns\ncategorical_features = test.select_dtypes(include='object').columns","metadata":{"_uuid":"06db1183-c99a-4903-b517-80d44642c047","_cell_guid":"a4542063-81ea-48dc-890c-e3597f25207d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:51.202705Z","iopub.execute_input":"2024-12-27T10:10:51.20342Z","iopub.status.idle":"2024-12-27T10:10:51.652049Z","shell.execute_reply.started":"2024-12-27T10:10:51.203352Z","shell.execute_reply":"2024-12-27T10:10:51.651313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_features","metadata":{"_uuid":"f19f295e-8f93-4a3d-b114-7fb24085270e","_cell_guid":"a804551f-b0cd-4a19-939c-62d2a91cbad4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:51.653155Z","iopub.execute_input":"2024-12-27T10:10:51.653513Z","iopub.status.idle":"2024-12-27T10:10:51.659615Z","shell.execute_reply.started":"2024-12-27T10:10:51.653474Z","shell.execute_reply":"2024-12-27T10:10:51.658724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_enc = one_hot_dummies(train, categorical_features)\ntest_enc = one_hot_dummies(test, categorical_features)","metadata":{"_uuid":"1ade058c-fdcc-47ff-ae5a-9e8fb9f74a24","_cell_guid":"368bebaf-659b-40bd-a133-d4232bdeebcb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:51.660763Z","iopub.execute_input":"2024-12-27T10:10:51.661039Z","iopub.status.idle":"2024-12-27T10:10:53.11187Z","shell.execute_reply.started":"2024-12-27T10:10:51.661012Z","shell.execute_reply":"2024-12-27T10:10:53.111172Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Feature selection","metadata":{}},{"cell_type":"markdown","source":"Saving some samples for latter to evaluate perfomance","metadata":{}},{"cell_type":"code","source":"# train_enc, X_test_separated, y_log, y_test_separated = train_test_split(train_enc, y_log, test_size=0.15, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:53.112795Z","iopub.execute_input":"2024-12-27T10:10:53.113077Z","iopub.status.idle":"2024-12-27T10:10:53.116957Z","shell.execute_reply.started":"2024-12-27T10:10:53.113049Z","shell.execute_reply":"2024-12-27T10:10:53.116056Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Some new features","metadata":{}},{"cell_type":"code","source":"def add_new_features(df):\n    df['Income to Dependents Ratio'] = df['Annual Income'] / (df['Number of Dependents'].fillna(0) + 1)\n    df['Income_per_Dependent'] = df['Annual Income'] / (df['Number of Dependents'] + 1)\n    df['CreditScore_InsuranceDuration'] = df['Credit Score'] * df['Insurance Duration']\n    df['Health_Risk_Score'] = df['Smoking Status'].apply(lambda x: 1 if x == 'Smoker' else 0) + \\\n                                df['Exercise Frequency'].apply(lambda x: 1 if x == 'Low' else (0.5 if x == 'Medium' else 0)) + \\\n                                (100 - df['Health Score']) / 20\n    df['Credit_Health_Score'] = df['Credit Score'] * df['Health Score']\n    df['Health_Age_Interaction'] = df['Health Score'] * df['Age']\n    return df\n\ntrain_enc = add_new_features(train_enc)\ntest_enc = add_new_features(test_enc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:53.118121Z","iopub.execute_input":"2024-12-27T10:10:53.118475Z","iopub.status.idle":"2024-12-27T10:10:54.459136Z","shell.execute_reply.started":"2024-12-27T10:10:53.118435Z","shell.execute_reply":"2024-12-27T10:10:54.458205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training model","metadata":{"_uuid":"16443379-d65b-427a-9a6e-7b49c8c930e0","_cell_guid":"ae41cbed-96e9-4477-991c-314ea61abf46","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def rmsle(y_true, y_pred):\n    \n    log_true = np.log1p(y_true)  # log(1 + y_true)\n    log_pred = np.log1p(y_pred)  # log(1 + y_pred)\n\n    return np.sqrt(mean_squared_error(log_true, log_pred))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:54.460347Z","iopub.execute_input":"2024-12-27T10:10:54.460905Z","iopub.status.idle":"2024-12-27T10:10:54.465369Z","shell.execute_reply.started":"2024-12-27T10:10:54.460862Z","shell.execute_reply":"2024-12-27T10:10:54.464567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#experiment with feauture selection (Annual Income, Policy Type, Health Score)\ndef select_important_features(df, columns = ['Annual Income', 'Policy Type', 'Health Score']):\n    return df[columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:54.466528Z","iopub.execute_input":"2024-12-27T10:10:54.467196Z","iopub.status.idle":"2024-12-27T10:10:54.476266Z","shell.execute_reply.started":"2024-12-27T10:10:54.467114Z","shell.execute_reply":"2024-12-27T10:10:54.475398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_enc = select_important_features(train_enc)\n# test_enc = select_important_features(test_enc)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:10:54.477152Z","iopub.execute_input":"2024-12-27T10:10:54.477448Z","iopub.status.idle":"2024-12-27T10:10:54.484284Z","shell.execute_reply.started":"2024-12-27T10:10:54.477399Z","shell.execute_reply":"2024-12-27T10:10:54.483584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(train_enc, y_log, test_size=0.10, random_state=42)","metadata":{"_uuid":"9a747767-f858-4ef1-bbfd-2244ee59fcf5","_cell_guid":"46c7e689-ad2d-4983-a045-fb402f1a39b5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T10:10:54.48514Z","iopub.execute_input":"2024-12-27T10:10:54.485364Z","iopub.status.idle":"2024-12-27T10:10:54.876116Z","shell.execute_reply.started":"2024-12-27T10:10:54.485341Z","shell.execute_reply":"2024-12-27T10:10:54.875415Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Get in hand with Optuna","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom sklearn.ensemble import HistGradientBoostingRegressor as hgbc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:34:27.674113Z","iopub.execute_input":"2024-12-27T10:34:27.674852Z","iopub.status.idle":"2024-12-27T10:34:27.678472Z","shell.execute_reply.started":"2024-12-27T10:34:27.674817Z","shell.execute_reply":"2024-12-27T10:34:27.677566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import make_scorer\n\ndef rmsle_on_original_scale(y_true, y_pred):\n    y_true = np.expm1(y_true)\n    y_pred = np.expm1(y_pred)\n    \n    y_pred = np.maximum(y_pred, np.min(y_true))\n    \n    log_true = np.log1p(y_true)\n    log_pred = np.log1p(y_pred)\n    return np.sqrt(np.mean((log_true - log_pred) ** 2))\n\nrmsle_scorer = make_scorer(rmsle_on_original_scale, greater_is_better=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:14:30.664093Z","iopub.execute_input":"2024-12-27T10:14:30.664444Z","iopub.status.idle":"2024-12-27T10:14:30.67012Z","shell.execute_reply.started":"2024-12-27T10:14:30.664411Z","shell.execute_reply":"2024-12-27T10:14:30.668959Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBoost Regressor","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:34:05.745839Z","iopub.execute_input":"2024-12-27T12:34:05.746693Z","iopub.status.idle":"2024-12-27T12:34:05.750471Z","shell.execute_reply.started":"2024-12-27T12:34:05.746656Z","shell.execute_reply":"2024-12-27T12:34:05.749546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_xgb(trial):\n    # params = {\n    #     'n_estimators': trial.suggest_int('n_estimators', 20, 500),\n    #     'max_depth': int(trial.suggest_float('max_depth', 1, 100, log=True)),\n    #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n    #     'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n    #     'gamma': trial.suggest_float('gamma', 0, 2),\n    #     'max_delta_step': trial.suggest_float('max_delta_step', 0, 10),\n    #     'subsample': trial.suggest_float('subsample', 0, 1)\n    # }\n\n    params = {        \n            \"n_estimators\": trial.suggest_int(\"n_estimators\", 50, 1000, step=100),\n            \"max_depth\":trial.suggest_int(\"max_depth\", 4, 10),\n            \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 7, 8),\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-4, 1e-1, log=True), \n            \"subsample\": trial.suggest_float(\"subsample\", 0.7, 1.0),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.1, 1.0),\n            \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 1e-2, 10.),\n            \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 1e-2, 10.),\n            \"gamma\": trial.suggest_float(\"gamma\", 0.7, 1.0, step=0.1),\n    }    \n    \n    xgb_model = xgb.XGBRegressor(verbosity = 0, device='gpu', objective = 'reg:squarederror', tree_method='gpu_hist', **params)\n\n    return -1*cross_val_score(xgb_model, x_train, y_train, n_jobs=-1, cv=3, scoring=rmsle_scorer).mean()\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective_xgb, n_trials=100)\n# trial = study.best_trial","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T10:54:28.068654Z","iopub.execute_input":"2024-12-27T10:54:28.069464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print('Accuracy: {}'.format(trial.value))\n\n# print(\"Best hyperparameters: {}\".format(trial.params)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:34:14.617529Z","iopub.execute_input":"2024-12-27T12:34:14.617891Z","iopub.status.idle":"2024-12-27T12:34:14.623009Z","shell.execute_reply.started":"2024-12-27T12:34:14.617859Z","shell.execute_reply":"2024-12-27T12:34:14.622095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {'n_estimators': 950, 'max_depth': 9, 'min_child_weight': 8, 'learning_rate': 0.005157092015323985, 'subsample': 0.8137283875432285, 'colsample_bytree': 0.9983658272411838, 'reg_alpha': 6.517426484808757, 'reg_lambda': 0.8771067924995002, 'gamma': 0.7}\nxgb_model = xgb.XGBRegressor(verbosity = 0, device='gpu', objective = 'reg:squarederror', tree_method='gpu_hist', **params)\nxgb_model.fit(x_train, y_train, eval_set=[(x_val, y_val)])","metadata":{"_uuid":"5293784c-b6dc-4bf7-a69a-328589bfd086","_cell_guid":"e1b4c54f-ae0b-4c7c-920e-7d1a28451fe0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-27T12:34:55.155646Z","iopub.execute_input":"2024-12-27T12:34:55.156396Z","iopub.status.idle":"2024-12-27T12:35:26.84147Z","shell.execute_reply.started":"2024-12-27T12:34:55.156361Z","shell.execute_reply":"2024-12-27T12:35:26.840628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_xgb = np.expm1(xgb_model.predict(x_val))\ny_val_true = np.expm1(y_val)\ntrain_min = y_train.min()\nclipped_predictions = np.maximum(pred_xgb, train_min)\n\nprint(rmsle(y_val_true, pred_xgb)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:35:30.809639Z","iopub.execute_input":"2024-12-27T12:35:30.810006Z","iopub.status.idle":"2024-12-27T12:35:31.339992Z","shell.execute_reply.started":"2024-12-27T12:35:30.809974Z","shell.execute_reply":"2024-12-27T12:35:31.338988Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM Regressor","metadata":{}},{"cell_type":"code","source":"import lightgbm as light\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:36:21.26791Z","iopub.execute_input":"2024-12-27T12:36:21.268284Z","iopub.status.idle":"2024-12-27T12:36:23.736785Z","shell.execute_reply.started":"2024-12-27T12:36:21.268251Z","shell.execute_reply":"2024-12-27T12:36:23.735684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_light(trial):\n    # params = {\n    #     'n_estimators': trial.suggest_int('n_estimators', 20, 500),\n    #     'max_depth': int(trial.suggest_float('max_depth', 1, 100, log=True)),\n    #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n    #     'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n    #     'gamma': trial.suggest_float('gamma', 0, 2),\n    #     'max_delta_step': trial.suggest_float('max_delta_step', 0, 10),\n    #     'subsample': trial.suggest_float('subsample', 0, 1)\n    # }\n\n    params = {        \n            \"n_estimators\": trial.suggest_int(\"n_estimators\", 50, 1000, step=100),\n            \"max_depth\":trial.suggest_int(\"max_depth\", 4, 10),\n            \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 7, 8),\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-4, 1e-1, log=True), \n            \"subsample\": trial.suggest_float(\"subsample\", 0.7, 1.0),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.1, 1.0),\n            \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 1e-2, 10.),\n            \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 1e-2, 10.),\n            \"num_leaves\": trial.suggest_int(\"num_leaves\", 2^4+1, 2^10+1)\n    }    \n    \n    light_model = light.LGBMRegressor(verbosity = 0, device='gpu', **params)\n\n    return -1*cross_val_score(light_model, x_train, y_train, n_jobs=-1, cv=3, scoring=rmsle_scorer).mean()\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective_light, n_trials=100)\n# trial = study.best_trial","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:40:25.0556Z","iopub.execute_input":"2024-12-27T12:40:25.056011Z","iopub.status.idle":"2024-12-27T12:45:04.113087Z","shell.execute_reply.started":"2024-12-27T12:40:25.055973Z","shell.execute_reply":"2024-12-27T12:45:04.111163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print('Accuracy: {}'.format(trial.value))\n\n# print(\"Best hyperparameters: {}\".format(trial.params)) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {'n_estimators': 550, 'max_depth': 10, 'min_child_weight': 8, 'learning_rate': 0.08399292742344162, 'subsample': 0.9931188111029522, 'colsample_bytree': 0.9549872057382064, 'reg_alpha': 9.447214278486067, 'reg_lambda': 9.191170124319523, 'num_leaves': 9}\nlight_model = light.LGBMRegressor(**params, verbose=0, device='gpu')\nlight_model.fit(x_train, y_train, eval_set=[(x_val, y_val)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:06:44.051283Z","iopub.execute_input":"2024-12-25T11:06:44.051658Z","iopub.status.idle":"2024-12-25T11:06:50.489504Z","shell.execute_reply.started":"2024-12-25T11:06:44.05161Z","shell.execute_reply":"2024-12-25T11:06:50.488626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_light = np.expm1(light_model.predict(x_val))\ny_val_true = np.expm1(y_val)\ntrain_min = y_train.min()\nclipped_predictions = np.maximum(pred_light, train_min)\n\nprint(rmsle(y_val_true, pred_light)) \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:06:51.827815Z","iopub.execute_input":"2024-12-25T11:06:51.828201Z","iopub.status.idle":"2024-12-25T11:06:53.016117Z","shell.execute_reply.started":"2024-12-25T11:06:51.828167Z","shell.execute_reply":"2024-12-25T11:06:53.015082Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"from catboost import CatBoostRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:48:54.334672Z","iopub.execute_input":"2024-12-27T12:48:54.335538Z","iopub.status.idle":"2024-12-27T12:48:54.50148Z","shell.execute_reply.started":"2024-12-27T12:48:54.335498Z","shell.execute_reply":"2024-12-27T12:48:54.500514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_cat(trial):\n    # params = {\n    #     'n_estimators': trial.suggest_int('n_estimators', 20, 500),\n    #     'max_depth': int(trial.suggest_float('max_depth', 1, 100, log=True)),\n    #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n    #     'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n    #     'gamma': trial.suggest_float('gamma', 0, 2),\n    #     'max_delta_step': trial.suggest_float('max_delta_step', 0, 10),\n    #     'subsample': trial.suggest_float('subsample', 0, 1)\n    # }\n\n    params = {\n        'iterations': trial.suggest_int('iterations', 100, 1000), \n        'early_stopping_rounds': trial.suggest_int('early_stopping_rounds', 10, 200),  \n        'depth': trial.suggest_int('depth', 1, 16),  \n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 0.1, 10),  \n        'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1, log=True)\n    }\n\n    \n    cat_model = CatBoostRegressor(verbose = 0, task_type='GPU', **params)\n\n    return -1*cross_val_score(cat_model, x_train, y_train, n_jobs=1, cv=3, scoring=rmsle_scorer).mean()\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective_cat, n_trials=100)\n# trial = study.best_trial","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:50:55.192088Z","iopub.execute_input":"2024-12-27T12:50:55.192705Z","iopub.status.idle":"2024-12-27T12:51:27.516709Z","shell.execute_reply.started":"2024-12-27T12:50:55.192667Z","shell.execute_reply":"2024-12-27T12:51:27.515551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print('Accuracy: {}'.format(trial.value))\n\n# print(\"Best hyperparameters: {}\".format(trial.params)) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_model = CatBoostRegressor(iterations=1000,\n                          early_stopping_rounds=100,\n                          grow_policy = 'Depthwise',\n                          depth=8,\n                          random_state=42,\n                          l2_leaf_reg = 1,\n                          learning_rate=0.03,\n                             verbose=0, task_type='GPU')\ncat_model.fit(train_enc, y_log)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:06:54.759542Z","iopub.execute_input":"2024-12-25T11:06:54.759912Z","iopub.status.idle":"2024-12-25T11:07:12.989222Z","shell.execute_reply.started":"2024-12-25T11:06:54.759881Z","shell.execute_reply":"2024-12-25T11:07:12.988524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = np.expm1(cat_model.predict(x_val))\ny_val_true = np.expm1(y_val)\ntrain_min = y_train.min()\nclipped_predictions = np.maximum(pred, train_min)\n\nprint(rmsle(y_val_true, pred)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:07:12.990397Z","iopub.execute_input":"2024-12-25T11:07:12.990717Z","iopub.status.idle":"2024-12-25T11:07:14.544711Z","shell.execute_reply.started":"2024-12-25T11:07:12.990682Z","shell.execute_reply":"2024-12-25T11:07:14.543864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from catboost import CatBoostRegressor\n# from sklearn.model_selection import GridSearchCV\n\n# param_grid = {\n#     'iterations': [2000, 5000],\n#     'depth': [8, 10, 12],\n#     'learning_rate': [0.01, 0.03],\n#     'l2_leaf_reg': [1, 3, 5],\n#     'verbose': [0]\n# }\n\n# model = CatBoostRegressor()\n# grid = GridSearchCV(model, param_grid, cv=3)\n# grid.fit(train_enc, y_log)\n\n# print(grid.best_params_)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## HistGradientBoostingRegressor","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective_hgbc(trial):\n    # params = {\n    #     'n_estimators': trial.suggest_int('n_estimators', 20, 500),\n    #     'max_depth': int(trial.suggest_float('max_depth', 1, 100, log=True)),\n    #     'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n    #     'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n    #     'gamma': trial.suggest_float('gamma', 0, 2),\n    #     'max_delta_step': trial.suggest_float('max_delta_step', 0, 10),\n    #     'subsample': trial.suggest_float('subsample', 0, 1)\n    # }\n\n    params = {\n    'l2_regularization': trial.suggest_float('l2_regularization', 0.01, 10), \n    'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1, log=True),\n    'max_iter': trial.suggest_int('max_iter', 100, 1000),  \n    'max_depth': trial.suggest_int('max_depth', 3, 50), \n    'min_samples_leaf': trial.suggest_int('min_samples_leaf', 1, 20), \n    'max_leaf_nodes': trial.suggest_int('max_leaf_nodes', 10, 100), \n    'max_bins': trial.suggest_int('max_bins', 64, 255),  \n    'early_stopping': trial.suggest_categorical('early_stopping', [True, False]),  \n    'verbose': 0\n    }\n    hgbc_model = hgbc(**params)\n    \n    return -1*cross_val_score(hgbc_model, x_train, y_train, n_jobs=1, cv=3, scoring=rmsle_scorer).mean()\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective_hgbc, n_trials=100)\n# trial = study.best_trial\n\n# print('Accuracy: {}'.format(trial.value))\n\n# print(\"Best hyperparameters: {}\".format(trial.params)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T12:56:24.836375Z","iopub.execute_input":"2024-12-27T12:56:24.836743Z","iopub.status.idle":"2024-12-27T12:58:13.337116Z","shell.execute_reply.started":"2024-12-27T12:56:24.836702Z","shell.execute_reply":"2024-12-27T12:58:13.335974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {'l2_regularization': 2.5118741012643135, 'learning_rate': 0.018200213523365188, 'max_iter': 502, 'max_depth': 34, 'min_samples_leaf': 8, 'max_leaf_nodes': 68, 'max_bins': 238, 'early_stopping': False}\n\nhgbc_model = hgbc(**params)\nhgbc_model.fit(x_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:07:31.306883Z","iopub.execute_input":"2024-12-25T11:07:31.307719Z","iopub.status.idle":"2024-12-25T11:07:39.702299Z","shell.execute_reply.started":"2024-12-25T11:07:31.307667Z","shell.execute_reply":"2024-12-25T11:07:39.701539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = np.expm1(hgbc_model.predict(x_val))\ny_val_true = np.expm1(y_val)\ntrain_min = y_train.min()\nclipped_predictions = np.maximum(pred, train_min)\n\nprint(rmsle(y_val_true, pred)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:07:39.703536Z","iopub.execute_input":"2024-12-25T11:07:39.703812Z","iopub.status.idle":"2024-12-25T11:07:40.527583Z","shell.execute_reply.started":"2024-12-25T11:07:39.703785Z","shell.execute_reply":"2024-12-25T11:07:40.526391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import eli5\n# from eli5.sklearn import PermutationImportance\n\n# # Calculate the feature importance\n# importances = PermutationImportance(hgbc_model).fit(x_val, y_val)\n\n# # Print the feature importance\n# eli5.show_weights(importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T10:52:19.037131Z","iopub.execute_input":"2024-12-25T10:52:19.037924Z","iopub.status.idle":"2024-12-25T10:57:30.656508Z","shell.execute_reply.started":"2024-12-25T10:52:19.037875Z","shell.execute_reply":"2024-12-25T10:57:30.655547Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"![{A3FD84FA-81EA-42A5-B8AE-D221E42303B1}.png](attachment:fc36fee7-67ec-4f2d-9037-165b53fbbaa7.png)","metadata":{},"attachments":{"fc36fee7-67ec-4f2d-9037-165b53fbbaa7.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAANIAAAIGCAYAAAAhqdFSAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAEnQAABJ0Ad5mH3gAAEc5SURBVHhe7Z3fbxVHnui/s//BPCTSQhBge0yCIyUPu3s1CAWQAwYTmJjNIk0iTXYAm6wCCeFA4jAPo3kYcEJOwgZmMtiBmYyUjJQ7i4kJv3FsEGJ0d+chkWKCPfaJvfx4SB5y78u9u1e661u/uruqurq72q7qPjbfT6mFu0+fOvWtqm/Vt75d/eV70wRAEGRW/JX4F0GQWYCKhCAOQEVCEAegIiGIA1CR5gKDe6ChoYEce2BYXLJluEK/R47178NtcQ1xDyrSrLgN769P6Khh52+Adb3yJ9J3KnZqMXxuQPx1E8anxJ/zBqk+9MOyfuoBVKRZsQiaWsSfY1/ChPiTEnV+gPFTFyMlm7oIH4/xPze3r+Z/ZLC6fbP4azk0LRZ/OmcY9rAOvA7en3fKKph6H9YxGfPP7FmgIs2SqJMPwOlB8SdpptP94k+KrGTjX8I4+6MJHm1if2TTegRqtRo5joCd6s2AwdNEgpLpOCnkFEfVrbS3L34s6t49qEizpelRohKcmxNi3pkaJ0YYoblJfBYp2e0J9gn5bCu0SbNLuJYRh2wO3u5dJ66rI6nyHWJaDifcFxKOyPSIZh6WT2egRuNwcE1KHiUR1YE4NLNPrz96RHXIzcdVPYEaDcA2eg8zx4OZOH4/yyf4Hanu9gzGZ29UpNmyuA22NvM/x0f5vBOMfE1bTsCBDnYpVLKJUTEfbWkjhiGFN9o2eQYjjPes0tZWKrTjKN8ZOwjbwo5igHy+as1BaUQmCrNzbjggqKyREgj6t0nKdBvGR8SfEll1OFMGOrepszdZD6MizZpF0LZFzEkj46RJb8PFU7zRlzcugsZl/DO+TopMPvoZY7AKB9maqQkODAmTpo+bi8raSoaMjgdFPk3dV4UpdBICIzOJzX08/5NCuQOTc3U1+s2oHB7NyCSocohRn4/85Joka1D+2tABPtP3nxaz5iLYcUF8Jo5ARl6H/POr3YHtsBlO0vsu7BCDWU6aD8BV9juXYMdiMhAeHkBFcsGixuX8j7GP4eLUBHzJFGMzPN1KPmvbyhuddtrA5BOfUUJTLzSpyBGYWZoDI0RaZ21tC7rCaqiEHcUAafyK+M1AueeEFzCUlc4Eon7CmTUqv2766TO8SyJrgsLbGxXJBa1Pi9lgHL58Vyzamx+FRvpvaPqRddK7olN0PB2O9oGpZ8bc0SPlm2dozoYjRPHTZSX1TauPmFah6SfyCGddD4TWBEUMjqhITmiER8U6aaCfzybRqBW5yMPPljEVY0Szg2TahQc1HcTHEuEMSDrSxxcD428YqmlrpDlKJKtk2kmHomxUiZinz7xmyiJY48qPKDJZ3AS0hH/1ySefwL/+67/yi8gMkdZJDNnkkl3kFPWzRV0HwtksNO3EwdYIJlorcCBwcJAFNb9fWwDnJfQ+1pnXTpI1NO3EEXMkhGusVWLdqRIppey1iwbB8PuKUyYLblLjjOSIcC3E0B6cSi7y+EPV1XCkdjXsLHbQxbPqXKBOh2gxPQMW74ATs/m+N7ijIM1UW9R1Sfmc1oXx/tYjhuvxutzcl6896O/ji33zBupGFyMx9SrN1COFzAickeYowxXV9Lrduz00Z1SvElIEOCPNUWIPZANwNioFdDbMI9jDWVSiUvje6dOnp//6r/8a/u7v/k5cyuazzz4TfyEIQpmRaddLEoIgEehsQBAHoCIhiANQkRDEAahICOIAVCQEcQAqEoI4wKv7+7veI/BWz7fi7AHYMLQHViZFwZm6DkfWnIfg7qz7xyo/g3PL9sGeru+LK9/B9fVvwfmE7e+P9f0StooX20LYb96D9tpWyLVndCYMfgw/6/xCnCSUR2EMPm74AMJvdDwPv6xqpczIU61/gikPm98plXi7PtAtt3t94E2ReCMugOdFJ9XPdcYqHwNUo8/M96uValOhLJ9TfwP7LqyE8E6lAz6WWCZnsN+7Fw0M+nkM3rnvhfLp5wSLPMd6r8MDXYHcQmEURbH4nbIhcn0MW6MBQgy4CzIHomLxZNqNwWUyEj7WF3XQ73c9Dxuav4Bzvd+JKyrNkhJRvt/VTrr4F/C59E7Od70fwHnYAPtq+0he4mIqohz7JSWinYUoER29f9n3mLiWD6qcRxLkiEOU//AXROmfj5SmdSs83/EtnH/XPH1+13sOvmjeAM+HnbkZtpKyfttzmZSeYpdnc6hElGZY2/0AwMg35NuCwc9JDT8G7dLv0Hu+PTUS3VM2RC5FYRavhPYOgC/OmeuuLPwokmigx5UR4/vwYAvAt6OSqZGT73ftgV/KM0sGQYdcq5SDdMpagaPZ1Aj8eewB+Js2tdQPLNM6dch3MHLqW3hgS4sqZ9MCYuzeg2/oq+e586xfqIn+s4aPxQDBYdfWX59TcnhRpO8m7pH+ShteJVdDG5UxD6bZqATG75F13wJ4UDPhvt+4gBSRfqbzLdwjvWpBo1bqxQ+SXMhn9NXN3HkSiEn0gV4frWuZlfBBJejGvM5iSuyR5iq1LiRLhZhyH/QTcztxwByDz/uJQd7u1RjPTX167agdTMyvB7rXKuZeLqgixmajmcNHTn7QBfy3PW+F5z8LO2K9Qdc8oozMqaLPxN+HlRdIRx75QMhSxvqIlGF/YLYGZnfSmpWukck6z2G7uqLuFImuP34mFpMzb1CxhnA4sjZXyZqKdER67CPrCOroCM7ry8slw81YXs7H4XOqLLLJRAcsojxwPLjnl9A+SgaIos0qtr4jM6NwhhjNblbWt+B8y/O5zPui8KJISSYGWx+1PJhYCXTU5566Wa5hEtYQpSCvbSSSzF9iAMMCopf3JrSuPPUNyYV8RsMq5M6TQpRqaAM8MPZnGBHfG3v3PIDssCA0V5+Hx8bOw+WkwCueYGZ/EtQjKQbXeh20/MxIxob+Dr4ZSbZtqRJ9MEI9crN3RX938c/wbfPfQIu2higFeW0jQQcV84yZ4JSh66JAptx5muDtURew9RsdQIkS938AH8tKHLr168vdreNHkZiL8ls4vzMyEQLXdWDbMhMu9NbQBeQDsOG4iyk7wevlEOo9tDc7uUv5i07JMyUW1KHbmZktPws7UPNLZOZQOpRYO4SOArs8P1Zc9MTc3XleGmCCtckHcF0a8MYq9OHsbJw8eSGykdkG2Ho4LtfYOc3NX6d43dnAZpkgrgBZIMoPRZUHrrQjKbsaJLTvceii8y348xbTwpgusIndb/PAjo12YPVAVpFFx2I3AJc3kFB7CCzkVx4yanVi2gmRmqeoh2jfA8FUTlYH8l0FPKAO4e3Ing2GbSxfa4ERbVdDRMZOmYLBN2QRxAH16f5GkDkGKhKCOAAVCUEcMKM10nB9hFdHkLoBZyQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiAO8KtI3vfvh2YZnxbEfzmrb/pNg36t8Ls7MfF55Fvb3fiPOTHwOxxJ/k34WlIscGb/lhMFjUl08C8cyX1OwKGOOPJPrq4S6yIsmZ3q7l4M3RaLKsKdnMbxa+wg+IseRboAP1xwjzZYMbWxaUXt67oorOt/A2fX8njeTNpBOnYX9rMLfhBvikgrtOG/CVPcRVq6Paq/Civ43/TYO7QidU/DcEK+Lj/pWwI3OtIHFooxWeWbVVwl1kRddzqHnAHr2WAxExeJJkT6Hj4kyrOjbBY+LKw927Yfnmm/Ah0mNRCrszf4VTPFeTfiPd7/pPQwfwnNwpHaE5CUuKpCOs/NDANoxSIUvFFdlvun9EG40Pwf7ux4UVx6HXaQT3u35OFXJZeggYd/ZSJkO34CF3fthY7BTuXUXkfEufPiu+Rezy2iXZ2Z9DV4ng80KeE76na3dC+HuqX8jv1APGORcvBH2kzLeOHy2TsrI8aNIooFWKtv+H4SFLQB3R++Jcw3SET6qRYpn4sGuw/DRhY0kpyQehI0XPoLDYcfQ+Qb+7dRdWLjlb9U8mpYSpZuCu5amZy6m/g2GxxbC6ja1TAuWETUfuWvoDBZltMwzu77Kh1shqqXCrq2PFGVxoyrBg22rYeHY15DQk0rBiyJ9M0Fau3kpLBDnAcmdpyjuwddj8YaBxQthMdyFr7U3Tp0w/jXJeTEs1N6bebCRXDB2Bosy5s4zgdatzEp4M1wXcUsipsQeebxKZ0vJUgksE2kAmJrQegyT39PAN0O8OhvmE8H6LVjD3SV2enBelwt0K+gMTjryyJtCFr5eSp7RfUDKsD8wW8marZPYMuGS4EH42y3E1Ow5LK39+D31BiqSJY9XxWKXHEeIjb4wXKCTo5pmkNYxzDFzGOB4JNtzo2SAkMyqQmDrOzIzUgdRx6uwS1oSUPOUO6qCgew6rGTr3/iMXCZeFCnJxLg3ehegZWFhZkOcBbCULLpjpsIUNRQWwlIaocc1CeuvJPPXqoy58zTz+bvUMSMt5AmPV1+FFWMfqgFICoCZ/QmwtV4waNF1NDXtcshZBH5mJGNDfwN3RwBWtJc5eic4PFjDrIa/9THCJay/6KBiXotYlDF3niZ4e9QFZGY8zB6VcPd7umtbePIKXMfZ4EeRFm+E56grdmdkIgSu2CCAB39Ym/5cyQePv0TMAqWxhF2+3967RUdI+3UEdynf6JRkFQvq0O0snn0FZcouo0WemQRrE3n9QdeC9Pmb7nH1CZFtDZ0ZtxKpTHKd1cq3h/Wj6NFAfeD1xT66QA8fBDY/B0ckT0z0wDbu8mbfg1dT1h5kVFq/B4a3pCyMaedcMwyrhw4rpguDffYhGdM5K/o+UuxyE4osOsSuz1oncXnDX1TlFuVZLJfDooypeSqk1Bd74Ckv3tPycQ0vF3vWFfYN7ZpePou6LgN8QxZBHIBeOwRxACoSgjgAFQlBHDCjNdLdcAmMIAgFZyQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiAO8ur/v9T4Lz/RMirMlsHvoI/hx2g7rwZ/Dis7L4gRgbd8N+IW0v0zNj9BRhRvVH4oTzp8qK6Ai74kz3KP/zpLuP8JHXZ435WfIFudP8POGCoTfsJAjLU9aL0eXxeW0qq/S0OpAwaI/FYi3GYl3+h9AtXYDbpDjj90AR9f8nFRNAqxT/IVUDr//Rt9auNz5LPxB2vk7BZvgjyK/G7UqrO2vwIqKlOPUH+AC6Sz88+ieZ3ulVxL03xnaDdDzDPzc5/s3FrKp8A70F6Lg1nIY87wHf1i/AlY0aMoSYFNfpfJD+EVYtuio0uA4HbvrRokonhTpT3CczBxr+35BqoKzoOst2N18GY4aG4k0+OHLZGZ4K6qc1l+QCpuEo+9GivLDrh9LL3P9EHZ2LwEYmYxeIFz8Y/iFMuLyeyZPXRX3GH6HfOctcs/lw3+I8smADhL2nc1ONpl7vUfhcvNueCuUhXQooiiTPcfFQGSX573efXAUdpPB54+k7sVFmcz6qkOI8h/tJ7PRS/Uya3L8KNLgBTIdr4X1ipmxAJa0AEyOGobhqatwZmwJbGpTzY7FyzRFccQPGtXfWdC2CZaM3SIzngdyy3YPrp6ahCVbVqlvgDY9TIyZv8AkLaRlngu6PoIbF+TBp/6gpuWKBtVSYdfWmwe2P717FCbrbDaieFGkexN/AWh+GHRZEzvP+C2YhB/AEu0LCxp/AJDUwcnItI/OevvTOoq5U/5lQisB+33RSV2TW7YpuDUWV3ZYvITkMgm3WBShGdSXFQlK7JEfVulsKVkqxGSt9K+FqmkAqNPZiDLHvHZ07cBt/hVrbsFuYi+nLdj/VHkGjo6thd2h+bIAVm0hpkvPPmktQfKUFuxJ8JGTH9ThMUnWVcG5sk6bw8TrqwgWwI/3B2Yrbwt5SSBz7+KZupyNKHNMkeTF53q4QDux0QTgClcZoesDtVGoucMdH4FiXID1Q7uJ2RQf4WV+WI0Wu38k6wjq6QvO68fLNVOS66sQ2PruMlSoh66jmjA4inV3e33WtRdFSjIxpkYnAVqWxKds2f6XSDIROUSpqAKMnYGr8vfI9P9s4PFKWB+wtUOgBLTjUFMp8XdmSW7ZFsPDzQbzc4oan0vgYRZFaCb1lYBFfRUBM/vTMK676wc/M5Kxoe/B5AiYRxTZ/pegipfLXqedYs1R+EHfjRzPhYQHzNe6ILdsCU4ZpuybYBXVklLrywOkHPvYoxLufjc9ivjTOWJ+d6wvfra0xI8iLf4x7Kau2J2R2RW4YneKEYW6kCNvDXe7Xu6UvDdi0Rna66Syf664nIkC7DwadS56hdnQSaaBYPAPyrMWti4g5YpczdnQGc2+49nJ9iwxrYIO9MOXyEyrdCixdggdKxZ5WmBVX94hshFlhu6dRCqDXIw/wYX+hEG4TvC6s0F5at5M7G/JdIge2EY2ubpzYa3yGatw/Sm39hQ+9pReInzqzx5kSrlYPslPy9smj1TZpJkh7NTiWvgNw66F9PqSIYPO+mfgzBZ1Z4NVfXmFl4s96wr7huEaq4szsKmOdjLo4BuyCOKAOea1Q5D6BBUJQRyAioQgDpjRGul/koQgSATOSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB3hVpLu9W+DxhsfFsQV+r21ojjH4unT/4/B6SkCS65XHYUtvfKsSvS7nQQ/TfYyp38OWhtfhujj1Sg7ZONfhden+xyuGUlrkmacN+L0W7VQod+H36yMZgyO7/orFmyLRRtnYswyO1T6Hz8lxthvg7TUpnZZ2is5R2DvE7/+8bwOc79QbNarUXUkbSAkN3Wd5HuI41aX9j9lBB1zzNtTEJa9YySZDlWgXjIZyHIMN/bvUAcEiz3xtcB2O9RRSGzNiQ5+QUxyHSt2xHseTIvFG2dB3CFaKKwu7jsLe5vPwtnF2IApy+DxRgKPwk2B3b+shONZRg7ffjZr9bu9ueBv2wtnaWZKXuKhwF2qZ/1M36aSd53nDkM43E2gHTZzlYtjJJnO3920437wXjoYDwEo4RMpa6zkmlMAmz3xtcL1CFLdjAzSI8/rha/hqTPxZx/hRpMGzQLoqbFRGjYXQ0AJQG/1anEtMDcLpsQZ4uk2dOZYuI806UiPdhrOw6xR8fuEnJKd0ljWm3UE6ZZEjmqVsEXdh8FQNGra0qnI2PUI6+SjU6Ixjk2eeNiCz264RorgvPSIuFAc3xdVZkl1b/3upbhrgEfpmcB3jRZHuTowCND8CS8V5QGLnGf+KmFjLoEF712Rh4zKAsa/ImGQLH73Od3Lzjx72M4cncsvGZYgNBosbSC41+IpFEcrO07oN6DqRmojHswcoH6ysUutCmiWpUvdvgGPBgDlVo8MHMUmjNq239RHFq7OhePhsE9rSQ3sBejY6USbZibGRmEw1km9wbnQEzAmImUvWicv6TkUmYuEshJ/sD8zWwOyOzFFY/BM4JbcpWwveR86GuoA2grK2mDkrq1Fjnu1uUB0a1bDZ5xDUccMdGqUv3Nn67jzsatgF5zuOpZeH3Evr//xh2fQrHy+KlGS2fD1aA2hpiJsQsv0vkWSezClyy7YUHmkGGJ3QugkzccRawSLPzDYYPAZvExNSmVmZF1OYUQXPsszknMP4mZGMDc09ahvaDaO3bP9L0EaPLbpzUroy5pYtwSFA10XNT0MrNcFs8sxqAzKyhzNqcBBTmMy13KVe5CxL1mm7mZueu/mzzLbEAblE/CgSMan2Ulfszmj6DVzXu8S0zR/+Bd6albCLTtedkvdGLDr36s+A0iAN8rq8HmINNHtl1KHew9izqUQsZGMPhiO7f+VLpEMrHUqsHfYHDgGLPC3aoD7g6zTo3kWkMsulKBaTM2FALhFvayS6pjjW8jZsFGbDxlNPw9kU1zXtnGe7R4mdLMyMTiAjlLTotGRUM1WW9RkeyM4A046J8Mgwg3LLRtd2ZHYYDb2PuwCIHPLawSbPvG1QPHydJj8zC5517ZLc37IXNngIXW8PZL+Hb8giyOyZ3147BCkIVCQEcQAqEoI4YEZrpP9FEoIgETgjIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEc4FWR7vZ2wGMNj4mjIzs6zWC3dP9j0J2yC/h65THoiL2wR/duRd9PzGPq99Ah3RPPxwM5ZONch27p/seMUYSy80xvA8v6qhdYu3VHG1oNMHlLeNHSmyJRgdp7lsGval/AF+Q41w1QXZNSCbRTdI5CZYjf/wV7EzK54V80RhH6GmDLOf79MA+tc9DGWFOFZX3inqEKQE+73w5kJZsMVaIXYbQ7kOVXsKH/RVXhLfLMbgOL+qoHggGDtFtSnCM6sNJ72kuKhORJka7DURbBpifcjbyw6xhUms9D1Tj6EwVhUXGOSVFxeuBXHTWoKlGEdkEVKnCudo7kJS4qrISfyDu9W3ez++SX5O5e7IdacwV2B7uHF/8EKh0A58/Zj2K0g9rPYnayydztrcJ5UsZjoSwroYd08lrPUaEENnnatEF2fZUPGVTY6+dc0Y0QRXuxfwMbMH5F2rIM/CjS4Kcsgs1Tylb39ChC/WMN0JEZRagfvqir1wAssJQtIimK0PLoRT2bPPO2QUnwmUS1VNi18DUKMogQBelJe22CDCJf1KIBowy8KBJ/K3V57K3UxM4zfpNM2UlRcW7GXpe2hc1gYxugIo26C7sqsGGsCruCUZmYelWfL4rllu1ruJkSRegmiyKUnWfuNiCY6ss3K6vUupBmyWB2mWMDpldnQykE9jS1l0crhpGKjnC/gmVkXRTY3XS9lDriEQIbPLDDa8H36TFnowgRMuvLN0EUIWq2BmZcubPLTJh/isSmebF4bv+UdZDYIr3hU3gquIest5YfzlaGldXgfrpop1GEpEX6nIwiJMiqryJg67vz8GLDi3C+41eZg1o94kWRksyWxKAVsv0vkWSeWEMaiHb62qlBYcrwRbo64pER8XgFGvqr2e75mZBbtqWwPCWK0HIWRSg7z9xtQInVV3Ewk3MO42dGMjZ0dhQhZv9LJEfamSl8/VEouWVLiiJE1kXNHUoUodQ887ZBmZB16i7mpudu/rp+lpWAH0ViLuUaVJUINtx1Hbid+YPCwFuzEnaz6DGS90YsOvMsfO/2dquzCmsgucMafofOUjurUSe1gHoP+63LZSEbe9AYPb9Z+RKdIeUOJdYOUhShzDyt2iCrvoqAyEbWqdC9m0hlap+5wfd8vthHF+jhg9PmCpyTPDHRw8LIzOLXggdq9LlA0qKTdP717dC/5Zzaoen6h3Q4Gfr8Qbe51d8haGUzociiQ+z6rHVSqmxUkXSnh7gWfiNTDnN9pbWBbX35g7cjezYYlst0TcDKCyn9QsgL2e3hGq+KhCD3C/PPa4cgJYCKhCAOQEVCEAfMaI30f0hCECQCZyQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiAO8KtKd3k2wvGG5ODbB77Rt/zEG90v3L4f9hl3AyXnegd+tj76rH1Fe8fs29d4Rn3nEQjaVa7Bfun955Zq4LjGr+pLRfmv970gt1SFTv4NNDftJadMQspjqyyf0OVJe/rdFGjveOr1k6YvTFxLOY+nKi+Tz1ulfTyack5Q7T5LYPW2/nh4LrpB8X7wi/qZp8tfTrUuXqNdcJwvZ1HRh+kVSptbjQan1c5Jc1VcR8s82MdmWiMOivel9e9Pucp88KRJveLVxxqZ/3aZ1hjCZP7uwV66QvHnSZPpOPKm/k51oYyX/pp5sZFNTTPlpYp0p6ESu6ouf17USyXIodWBIbFAgA2XO9nSR/Jh2g2fgLGyETcp2/IfYC2sT9A1Nnakr8C9jjfD3bQ+JC5yGZY0sUAczM/LmSbjTexjONr8KLxf2WoABG9kU7sCVUxPQuOVJIp1EUws0wi3+op6r+hr8Z3hzTL+nWK5VqDmpmmvsWmhePgGHazfhcGYZicm+8014uO8wbBJXisSLIt2ZuAXQ3AL6y8OJnWd8BCbg4VhUnIcaHwYYG2Hv5OTOkzTNP/dMwMb9/6h2yBjX4Ew/wMb2J8S5YyxkU6nByBjAw41aqRc3kFwmYIRFEXJTX+yejk3QkHct65Anqpfg1eazcDhYp5J13wv9G+E3F7LaTeVaZR282fIbC4Xzw/z12tEROXM2oo6HF6xmLT5y8mMdUdCJnnVR5yt6YeuI2ugEQP8L8E/wHtwkoz49LnUDvLkma0HvkofgH/dvJPX5z+Q3r8H+TjKPklklz7BGHSovjLwKl6qeBkML5qkiEQU5fDZuHskwDxAfxW5ajH5PVHlH452tERq7L4XnN0tswFlDBpH3uiLpH+raTwxCaYYogtbD8JuOs/BCAxnUOnLOKmQGW9fzcO4ZzDVeFCnJbGEjYEtDXGDZ/peQzZNceSasIUKo23gNtacLUAIL2VQaoKUZ4NaE1pGnaiSXRmhhUYTc1Bcz82Lw3y8ac1my4AMmEMV/IbAOyPECfbWezLRFmql+ZiRjQ99hEWyMaxHZ/pegjR7OKjnyvHPxX2Ci+e/hSVMwE6pEnbfg1SGbBawDbGRTSHCg0HVRIJOj+jIrW8IazSfEOvgnOqvUfgMbiQJkP2MLIGbhhchSCI7f0PjfZGa7WTsD/2jqAz4QHu1cyG6/pMRcsZILV3fp6s809HOTqzMrT3E11SVO87B3XbtJmbLpz3Jiz3birmw39cXrSnYV69/xn7hsQZvE5JJTlvtbJCZDwe5vry/20QU6m2YpxBa/JNmxdIHIbNtatLDk14jpwdiofBaQlieHPtl+AYCYbfEZhzoXyLrIGNuukcxSySOY8rs6dPTLMBFTZaPrNWFqhmUW18JvGORxU19anRjv8YX4bZB/03RNwKwJMMopw2SG7DZxCb4hiyAOmL/ubwQpEFQkBHEAKhKCOGBGa6T/JAlBkAickRDEAahICOIAVCQEcQAqEoI4ABUJQRyAioQgDvCqSHd6N8LDDQ+LYyP8NmtL++A+6f6HYZ8xKk56ntcq0feDY2Ps3ZprsE++p4gX8yxkU7Eo46zr6w78dn30/aQ8yiVexnh7zqCvuYY+R8rLf1ikcbaLd9f0pYTzWLqyi3zeOv3eZMI5STZ5XmK7u8elK3q6NL2L7TYO7tHPPSQL2dRkUUYn9XVp+r1Ynkumd10Jr5SfSJmU8ky+x3bGy9ds+oXv5EmReMOrDTI+/R57vcHUYc2fUaVYsjeoDps8036DJ1bJbe+RO6WrrAPZVzzNI+031GQjm5qyy+iqvvSUXX/1kFQ5TWU2ye43+THtBgfgU3gKNivb/h+CxsQoQpfhj2ON8IwxKs4EmdwJOfJ8JPGltDtwmUXoWatuz2cvwX0FEz7MARvZFCzK6Li+yoSb4vuIIRvBrq3/raFu6hcvinRn4qvwlWeZxM7DouI8Ao2xqDiPSFFxbPLkb3d+2hnYyro9zT+PKdriRvLr8TdOnWAhm4pFGZ3Vl8qd3p3wxthT8JoUw8E3T1SvwGvNn8IbYRShfdDV/xT0XvipOpCEXIOBfoCnwreiH4Kf7n8KJnp2huuiO71vGAYRv8wzr90T8FbtFtwKjqHXAHqeNC5O8yI7MZ5kUYSeDM8LcVb4QnJYPDn6Gqm3t1JfmnNPoAhHiIpcg32dRAX6kspAHQ9d8Gnza7BHVpLWt+BW3yPwxpqgfR6B3oLlmN/u78U/hbN9QSPNjieqkYJeYVGErkQKO5ejCNFOGMjRPsA6oouBJxekDL0dn0JXA1GSjl54yzSTTP0WNjY8CW+09MItbbZig9y5zZEcQy3wBpGjSA+kF0VKMlt4BJvG+JSdsEaRzZPceRrhEXK+ikXomSC/LiL0uMZCNhWLMvqqL9Kh6SAxcepy4esTZnImQWfNNW/AI32GQYso2BvUFJSvkwH0OJHj08PFrbP8zEjGhr4DEyOybSuRsEbhUXHEojtvngK1wyYstlmEnmdgrY+IMzayKViU0WN9lQJRhp3MHOuFp/q71JmEKlHnV/Da0C3zTMXWi3WA8GjnQnb7JSXmopRcuLpLV/f16+cml3RWnvQZwy7ZDSqeOSiu0dhzCP+u0kzZ9DJZlNFFfY0f36U+yzLVl/fEZQt+U5eLypBeHu7+Vh4llCCHN0WiiTUkEYgdcocnKdYRwmvBd9TPgpSWZ1CBUR4JCqLdZ6NEyu/qR8LzIDmlyhZTnOha8B1TGWddX0z5pM/JYVMX7pJQAqVc8jXxt1ZGfsgPn+P3FTsY/MfMwnHhG7IIojK/vXYIUhCoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAK+KhFGEJCxkUykiipAKv7eECDypZEQR0upAOYp8XZ3tQs2JvFkvKembUk2bVJXENlDONiqOzW5hdbdx/NxDspBNTRZldFRfUeK/kV6uEhKRy7SZN31zLZelyA24GEWIJtYJU5RcSzSPtN9Qk41sasouo6v6ihIbgPbuIp20zhTJkNLqjiZj/XlOfkw7jCIUMReiCBHzqGvkNTj+EvmwYNxHEboGR3om4Kn9ScFT/OBFkXJHsHEWFQejCOWrLwKNhUDfQD1ebMcLmH0UIRUWQUgPjlIA88xrh1GE8nEN9rFYCGfhpz5es7fCQRShED6bFz0bUea3+xujCKXAO+VXRA5jLIQimWUUoZDBIywuX5Hx7AK8KFLuCDYJa5RZR8WJgVGEwvpinY6sl+SZlcxOZK7l8eEKnmWZyZkEdXEnRRGSuHbuU4COzQXH5eP4mZGMDY1RhGQU2RQKiiIkx7MLDmIKk7mWRewpdJadTRShkPS1k3eERzsXstsvKTEXpeSC1F2S7DztGYfBJZ2VJ33GgFGExDlJmfWlJ/a7Rbu/uWxBG+lyMbe8wV0fSzkfYbhOGEUo6R4tKb+rH3M1ipCeClck/lxLLZd8TfwdyigfhofPWfJ5TBhFCEEcML+9dghSEKhICOIAVCQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiANQkRDEAahICOIAr4pURhQhmcSoOOzdluh3CvlfvC1kUyk6ihD9vXqLIETJiCIkUKNHlSAH24WaE3mzXlLSN6Xq57HEdu9KGxH1c5Ly5cl3Fcei4ugbRE0bRl0nC9nUpO6Ijp+T5Kq+lM2xaWUqKRG5TJt55WtsY660YTWzr3lI8yyKUJTod01RcVgla7uE1d/JTjQPsxymZCObmkxl5IoSdA5X9SWdF77ze+ZJkdNY7uR+4Sv5Me1KjiJETZ6youLEqOsoQg/BTy/cgrNdaj5F4iaKkB4I5iFYu6XR3Nc84UWRrCPYBBQYFeehrtfgqbE3YGdgZ5P73/D5ZmW9RxEqGTdRhOKv3bNX6guUc5557Wyi4tBIQ73wSBCrQMQCyAoAMj+jCNUDs4witHgtPNM8AW/slGYwpozi74KYR4pkGRWHeboGYHMYq+AKtBzOVob5F0WojphVFCFqnpJZDd6AJ4OB7dxm1kb2QXFmjxdFyh3xxxioQzVP3ETFIcp2WB/xSEMcfw0a+9/w4zK1kE2loChC4rxeYCZnEnTwS40ixNd68sDGAsEsi9euL/zMSMaG9hxFyCoqDl9/FEq9RhGqJ5xEEZJg6964M8YrwqOdC9ntl5SYi1L37af4+vVz1d3LU1aesWRwjcZ+R7hKU/OZZcqUTX82EntWEndlO6+v0tzf6jMyXS4qQ5Yb+9JxWSY1v6LS/IoipKeEzqH+jkU+JCm/qx8Wz6BSZYspTnQt+I76PIgnp/VViiKZBjH5mvg7lFE+orLqbWOqK98JowghiAPmmfsbQcoBFQlBHICKhCAOmNEa6f+ShCBIBM5ICOIAVCQEcQAqEoI4ABUJQRyAioQgDkBFQhAHeFWkO73tsKxhmTja4aS27T/GYEW6fxlUjFFx0vK8AyfXR98353ENKtLnVuVygYVsKlo5jVGEZltf9IXF6PvsqPOXFHl5K6R2JKZOQrssQ1FtKkOfI+XlPy3ShNiUejnhPJbY7mW6ETHhnKTsPC9Pv3d8IjzjedBNjOGV6f+cfG/6Pemc56H+jvNkIZuaLosdzIEs+jlJLuqL1MUuOU/T79RTCjfyqv3o8l71PN4v/CdPisQbROnA0xMisoupkcyfXaa7evcG1ZE3T5qyPqfJlG96og1l39lsZFMT6wht75FvSleZogSdw1d9Jfx2XSRe9l1EabKVJH+bzjb5Me0GP2ERbH6kvIj1EH3XLCGK0KWUqDjjxGAj5M2zXrCRTeEOXGJRhNapL/2JF/XGqckyj+rLZKqxa+tPKnVzp7cL3mjpg2q7uFBneFGk9Ag2hs4jouI0zTgqjqlDisofewq6U8JNXat0smAar9i+fZkXC9lUkqIINUVvxXqqL/ItsxJ75InqIIsi1BNGEapAZ/9T0HdhW1QGcq215xHos42NYRxE/DL/vHbSArx1tBtGa9VYRBp5Ad6zbBBG5UZLQF6Ut7IoQq3heb0v0G25VmnNHHjc8xBsY1GE3iGz0jWosChCUptRRwJ71TzejkbY/Z9CY/crdvc7Yv4pUmuVKM8oP9o/YR29XYsV/VDXufCeXuhi92R50Z6oijzJMciiCBEFFOejcz6KEPcQdo68BoOGgcc7pM36Oj6FzgZiHXQQ8y2cSUi5WNCTc7AtMbxaBBsg2f2jcK7goJdeFCk9gk1TfPSX7X8J+6g4hjwppIFop584dSnBlOFK1dcB8Ok5T7OKhWwqSVGExkkuahQhJ/XFXMed8BUdGCxmZl8wk1PjTm8PMdFI23RG1sAyMtuQK0Tp1AGSWgzM/CMDW6SIxeFnRjI29B1q2idEEZLsfwkeFUfY63nzrBdsZFNIcAjQdVHzM7CORRFyVF9UiUoawRVIObqYEvTBU/2doXUgWw7h0fcU+YSsocjfQZmpEpU2mwYIj3YuZLdfUmKuWMmNqrtVdV+/fq66e3nKznOX+mxGPHeQ3b36Mwf+O35dpZmyiXKGZdDPyZ26O9dNfZHzBBd8cYnLFrRRTC49xeSk3/f8HNAieVMkmlhDkkpih9SANJkqjF8LvmOuzLQ8A6WQD11B1N+gh10jKL+rHxadMVW2mOJE14Lv6HLQNNv6SpPJ9HvuE382pJbLdE1KCYOQSYbEPDwkfEMWQRww/7x2CFICqEgI4gBUJARxwIzWSP9FEoIgETgjIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEc4FWRbve2QVNDkzja4IS27T/G4CvS/U3wiuEdoew8b8OJ9VEeTetPkCsaUyegLcyjCdp6Y3e4x0I2lavwinR/U+WquC7hoL6uVqLvs8P0O6Wi1UNCP1LlsOhrrqHPkfLy/yzS5PEnp5cufWn6s4TzWLryEvn8yeneyYRzkrLznJzubVs6vXRveId2TtJk7/STS5dOv3Ql4dxHspBNTZ9Nv0TK9OTxyYRzklzUF5H9JTlP0++UnUgZe6W24TKocn62l7RxWy9pbX4e7xf+kydF4g2idk7eqc2NZP6MVVCoBBZ5ss6kVSBTlKjiWSVLlU6T+jvZieZh39lsZFOTqYyqbI7qy5CMv11XSZNLa1+esuV0nfyYdoOn4QxsgqeVNxUXmV9Yo0xdgo/HGmFr2yJxgRME6mCGl0WeV8+dAeh4GlaxM8HiJlgOE/Cl9hJcYdjIpnA7DECifKPpUWiEm/xFPUf1VQ9wk+wVYsBFsGsmkzyR5VogmEWwbktjoXJ6UaTbEzcBmh9NjGATq6DxL0lX1yuDVEfjcoCxL9nr0tl53mZvfzYui90BjzYD3Jzgv7qo6wBsGjsEO4J1EVkvHewH2NSuqJ87LGRTqcGXY6RrNKpKogwITurLRIISe2RVdQhebz4DB4P2IOu+7f2b4MSF7cYyXK1shzPNr8NeZYAQA4wEe6U+UU733Ideu1XwTu0ELO9Zwxemaw7B8r5xeCfjPX95MbuGRRES36dH3S3QZ8bVyho4NLYJDnQVpUaURbB9/yZSn2+TWekqvNJJ5tG+dxSrQnaYHFw2BOOyki1eB1ubJ+DQTmkGY8oo/i6I+0+RmKfrNDxdG4dxdgzBo4ezlWFVNbh/HIZYFCHSoEEeVU+zWWFwz9j2kddhqKZ24kJofQdOdJyB7Q1ktuk4ERvUFnVdDOv6fdjBFCryUBJFvEBmNTgEa4SyNZ17mrURDfJS1JDgRZGSzJYggk1MONn+l5DNk+w8k+x/2VS6DScO6yMeaYjjr0Nj/0E/LlML2VRUUzRkapzk0giPsihCLupLwB4FbIebdGBIMKeKgJmcFlClOtEBcOacPPBRZYoGOjqwsUAwMTPfH35mJGND8zWMcS2S4BBglRHY6xZ5Gu1/1gGDRTdXqkKxkU0hYUCg66LmrWEUIRf1xZRImLYXCzXnNEg5dvQshxPE5N7Uv93iGVsGbN0bd8Z4RXi0cyG7/ZISc8Xqvv0UX79+bnJlZ+UZuEYjt2fcDRr7HXGPT5dvpmz6syz9XMglu7Jd1Bc7z+H295PUNtPl+myvKhOXU62Lz46n9YFikjdFook1JBGKHVpHjXWE8FrwHa0CRUrLkydekcE9pgpVfycpHzUpv6sfFp0xVbaY4kTXgu8on4k02/pKk8n0e+6TaRBTr8XaKvbMKC5HMWVXE74hiyAOuA/d3wjiHlQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHGAV0Wq2yhCFPb6gHSf75fzLGRTKSaKkHV9lYZWPnIkRX1SZbWpY4ewXag5kTfrJSW+2TDaSKmfxxLb1SttSNTPScrOU2x4DDeR6uciGfL2mixkU5O+g9mwo7nI+iozEblMm3n1jals46rF5mNf6b6LIsTzSevE2Yl2SLMcpmQjm5pYh9c7hSJbkfVVfylWd1SOEpWIJj+mXR1HEbrdexDOBC/IFUEdRxGqh6hLs48ixN96LjJgiwkvipQ7gk2BUYSCt0hrUjATvSGdUrdRhOzqyzd5owjRteNpJeoTrS86qNSy15Qeuc+8drzz0AhAp9ujd/xZ4I2MEfB+iCJUDtlRhCKo40ELx8VCCUzAoTVyQBv+ynohoagF96f7W4tUs+ql16Fx7BC8neLlmd9RhEomI4oQg3lZ18ChlhPGIC2q8q2CvaSNuHIWgxdFso5gE+AkKo5NFCF+TwxmNnnCQjaVoqII2dRXcTCTMwnq5hdBWmKDVkLbMfkLxM+MZBPBRiZhgZs3Ko5xDaZEEUq7R3RS19jIppDQwT1EEbKpr0JIiyJElajzJrw+lBTE0zzwJA9UnhAe7VzIbr+kxFyUkktSd+lmPuMwuGaz8gxcvpFb2OQmNt/j89lJpmz6s5HYs5K4K7u4+vKd1DLoclEZMsvDZJdc9gnPmnym+zKKkH6PjRIpv6sfFt9Plc3U8OJa8B1TpyiuvnwlMYgp5ZKvib+l8kWH9qyLKVP0eZFKRBNGEUIQB9yfXjsEcQwqEoI4ABUJQRwwozXSNEkIgkTgjIQgDkBFQhAHoCIhiANQkRDEAahICOIAVCQEcYBXRaJRXRobGsXRBu9r2/5jDL4i3d+YGBUnPc/b8P76KI/G9e+ru5v1z8lRyAtgFrKpDMMr0v2NlWFxXaKQ+qovhiu0nK+Q2tHIXb+Ooc+R8vJfFmnq+NrppUtfnv4s4TyWrrxMPl873TuZcE5Sdp5T4U5u8zlJJN+Xr4Rn0/812Tu9dulS9ZrrZCGbmj6bfpmUae3xqYRzkoqqr3pKoq1i/Sh3/bpPnhSJN7zaOXkjKZ0hTObP2M7lsFEt8mQVqFUyq/z0SlV/JzvRDmmWw5RsZFMT6/BtveSb0lVFtnLrq5zEy/7yXr3M+evXR/Jj2g1+AgOwGX6kvIjFX1gbp29o6kxdhP8+1gT/oEXFaVzWFL14ZpHn8LkBgI4fwWp2JmAvwY3TbMrBRjaF23Dx1Dg0bWlTX/ojgjYFL+rNo/oymWrsmmZi3u7dwV4zf6ddXAjIXb9+8KJI/O3EFtBfHk4UbnyENF1SVJwRoM2enSd/+7Mp9spyI7SkRsUZhk/6ATa3K93JHRayqUzASGIUIdHBS60vt6yuDsPrzQNwSIkitBlOXNgRDSTk2mr6Bm3V0Ea569cP97nXji60t8OAHJUmAT5y8mN1zziM96wOz42OAMSSRbBj/2ZSn2+TIW0YXukk82jfO9EsOfU+tLFXzaVrdcj9q0i0gRpWM3NhQh79ElhdnYCJGj+Gu5ugqXs4PJ8wjZSIPSyK0ACLIjSgRBEiisWCnlyEHdqMU294UaSkaXVilNglxihCkv0vIZsn2XkmrcEMphJ1lbIGKkAJLGRTSTCtWFCSJipq8fVVAMzk1Ljde4is8wAGOiNroJHMWOQKUTrx2CJ3/frBz4xkFI7b5Ma1SMIClzZ6uOi2yNO4BmMdUFp0UyVipsJEQlQax9jIppDQwelaoPkfoI2OzEXWVxEQ6yCIIrRZiiK0qOtiNOsHR99m8glZQ5G/L3YRSXPXryeERzsXstsvKTH3o+TC1V26mc84DK7ZrDzJHdrzlrhrlOahu0p9p0zZ9GdZsWdbcVd2UfXlP6lliMmlJ4OcNnXhO3lTJJpYQ5JKYofSgOYK49eC75grIi1PnnjDBPeonYJ3lPD7ymHxrMn4PXJYPK9IlS2mONG14Dvq8yCe/NeX7yTaQymX6ZqUEpTEpi58JnxDFkEccJ+7vxHEDahICOIAVCQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiANQkRDEAahICOIAr4pUn1GEJNg7SYaIND6wkE2lqChCAfT3LNqoZIxRhLR6KCUSEtuFmhN5s15S0jel6uexxDYjpkeCyc5TbHgMN5Hq5yKxvAvc4Gghm5r0Xdn6OUlO6oskZXNsWpnqIIVlVWWYOt4rnWdsevWU7sMoQlI+pvstEu2QZjlMyUY2NbEOr3cEpayO6ks+j9VTvSVe1ngUIUOaYbvOJvkx7eo6itBqeKdW0Et9lDqOIkTPd1wQL8iVxKyjCNUJXhQpO4KNhkUkmOw86yMqTgwL2VSKiiJUH8w6ilAMHkClqXtvocFS0GtnCUYR8oWLKEKyg+kQtAwVP8uiIlmCUYQ8MusoQtxE5e3xPsBOqlAFeWMFXhQpO4KNhjFQh2qezLWoOCEWsqkUFUWovmAmp4ZVFKEYVKlOkNXhAHxSYCB9PzOSsaHrJIpQ0eSOclNcFKG6YTZRhOoF4dHOhez2S0rMFSu5cHWXbuYzDoMLMyvPwOVrdO+aUkFu0kzZxPOR0FWtnwu5ZFe2m/qSUmnub7XNYnLpKSYn+b72GIHJnZaHh3SfRRHSUg5FUn5XPxKeB8kpVbaY4kTXgu+oz4N4clNfIpWiSHygU8tluialWJuJ+8N6SPmux4RRhBDEAei1QxAHoCIhiANQkRDEAahICOIAVCQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiAO8KhJGEZKwkE0FowiZMEYRUhD1VvTLlmwXak7kzXpJSd+UatqkqiS2GVHaNKmfk5Sdp9jAGG4i1c9FYnkHmxxTyuQqWcimJn0Xu35OkpP6IknZHJtWpjpIYVmT24zLSO6x2EjsMmEUoRkoEm0ssxymZCObmlhn0HcwK2V1VF/yeSm7v/MkXtbUKEJMhpfJPcUrkh/TDqMIRWAUoVTcRREiJv1O+lr6O/AjcaVIvChS7gg2TqLiYBShfPVVH7iKIjRcWc0VraQ3odFrZwlGEfLF7KMIUYfK9pHXYbjEIDSoSJZgFCGPzCaKUDBbyTNYCXhRpNwRbIyBOlTzJDtPjCKUr77qC2ZyamRHEfof8P7h6Dy4Z3s/udS/nfxdnDvfz4xkbGiMIiSjyKaAUYTsowj9NymeXXSc6CC3kJltopYVD88hwqOdC9ntl5SYK1Zy4eouXf2Zhn5uck1n5Unu0J63mN3EYZqh+ztvypRNPB8JXdX6uZBLdmW7qS8pleb+VtssJpeeLNqMyT0/niPxxAQilcQOrQFNFcavBd8xV1ZanjzxhgnuSVQimnIokvK7+mHRaKmyxRQnuhZ8R30exJOb+hKpFEXiA51aLtM1KdWpImEUIQRxAHrtEMQBqEgI4gBUJARxwIzWSAiCqOCMhCAOQEVCEAegIiGIA1CREMQBqEgI4gBUJARxgFdFut27DhoaGsSxziKK0B7p/gbYY4yKk5UnjSIU5dFgjCI0DHvCPMhRxIt5FrKpWJSxsPoqE6185Fhn+k+Yc9evY+hzJB/8u9iUOpRwHoNtRlw73TeZcE7IzvPfp/vohse94R3aOWVI7Db+94RzD1jIpmJRxsLqq2SIXC9fEX9TJvvEBl9xTsldv+7xpEi84RVhRSOZO6z5syG2izdoVIs8WQVqysoqPqpU1pna+sg3JUzfS4HmYZbDhI1sKtllLK6+6hFVzvz16wM/pt3gaRbB5unUCDYSUxfh47Em2JoaFSc7Tx5F6GljFKEv2UtwSRF6HjW+ceoEG9kULMpYWH35Z7hCTbE9xJCNYNdsTczc9esHL4rEX3l+1D6CzfiXpOmSouJ8KUXFScszOYrQo+Gr2xPwZUqEHi+dx0I2FYsyFlZf/lldvQoHmgfgYBhFaA9s698MJxNjMAzD6X7pLd/c9esH9NpZwkdOfqxiUYRWheeFOCvmLUEUoSpRkWHYw6IIHUmIGEQdD9tgoPkAVMoKHZAAKpIlq6s1qNX4cZVFEboantcwitDsaD0CJzsGYFsDUZKOk3DEpCRT78O6hlVwsOUk1EqOGGTCiyIlTauJEWwS1iiyeZKdZ3IUochUSjBbRISeR4nV4xwL2VQsylhYfRUHMzmToK7tNQdheZ9h0Mpdv37wMyMZhcuOIqSvUdSoONl5GtdgIooQX3QndB5qZzdv5RF6XGMjm4JFGQurr4Igs832nuVwsnYSNvdvU58BUSXqvAkHhmrmmSp3/XpCeO+cw9yPkgtXd+nqzzT0c5NrNivPwOUbuUINrtHYcwiTm9gtmbLpZbIoY2H15R21DLpcVIas8tjUhW+8KRKFNSSpJB7BRm5Ag/AEfi34jrki0vLk8IYJ7jE2guiowT02SqT8rn5YPK9IlS2mOASLMhZWX97giquWS74m/pbKFx2mh8/BZ8UqEQXfkEUQB6DXDkEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA7wqkgYRUgid5SboqIIBdDfs2ijkjG9mi7D5C3jRUu2484D+qZU/TwG27GbHgkmO0+xyVELjKFuKtV3POvnHrCQTcWijE7qi6Bsjk0rUx0QljXej5TNuQUGPQnAKEIU0/dSoHmY5TBhI5tKdhkd1Zd8rtVT/cHL+vJeQ1tJdZNWrz7xY9phFKGIOo4iRM93XKjBpS41nyKxjSJ0u3c7e838SLu4INN6BGq1pDgPxeBFkbIj2GgUFhUHowglK3A5WEURItdW0Tdo6zg2BnrtLMEoQr7IiCJEg56wV83LnXGyQEWyBKMIeSQxihBRLBb05BLs8BFPwyFeFCk7go1GwhrFfVQcjCKU2AYlw0xOjdu9B8k6D2CgM7IGGsiMRa4QpUsIpl8SfmYkY0NjFCGZ5Cg3FmV0VF91Q0IUoUVdl6JZPzj6NpNPyBqK/F2mkySG8N45h7khJReu7tLVn2no5yaXdFaegcvX6N4NEM8iIrewyU3slkzZ9DJZlNFNfUmU5v5W2ywml45BTpmy3N8YRYgcNkqk/K5+WDRcqmwxxSFYlNFNfQlKUSQ+0KnlMl2TqFNFwihCCOIA9NohiANQkRDEAahICOIAVCQEcQAqEoI4ABUJQRyAioQgDkBFQhAHoCIhiANQkRDEAV4VCaMISdRtFCGb+ioTrR5MMrD/8TzjHt+wHXce0DelmjapKrDNiNKmSf2ckJ2n2PAYblrUzyn6DnH93AMWsqlYlLGw+iqZyb7pvtiud1XOob1qv4rL6R+MIkQxfS8FmodZDhM2sqlkl7G4+qo/THLp2NzjFj+mHUYRiqjjKELZ9eUf2yhC9Y4XRcodwQajCEkUFUXIpr78YxVFSGK4sg0Gmg9ARRkgNIyDiF/Qa2cJRhHyRUYUIYLsMDm47CrUEpSMwaIODUBTd6XQqEOoSJZgFCGPJEYR4sixG07AdqZQiR5KFnWo+HgOXhQpdwSbhDVKvqg4SWswjCIkk6++ioOZnBZQpTrZATBwTrUCqMXAgkgSZdMVsQj8zEjGhsYoQjKKbAoWZSysvgoiIYqQLVSJto0cgKtlhi0W3jvnMFes5MLVXbq6r18/N7lms/IM3J6RW9jgJhZBRSLXqH9XaaZsepksylhYfXlHLYMul/6MiMup10357nqMIkQOGyVSflc/LB5gpsoWUxyCRRkLqy9vcMVVy6VeU2Wkh6Y0Wj0ph1FeP2AUIQRxAHrtEMQBqEgI4gBUJARxACoSgjgAFQlBHICKhCAOQEVCEAegIiGIA1CREMQBqEgI4gCvioRRhCQwitAsKKG98sJ23HlA35Sqn8dgu3qlDYn6OSE7T7HhMdxEqp9T9B3P+rkHLGRTsShjYfVVNiW01wzAKEIU0/dSoHnYN6SNbCrZZSyuvsrGRXsVgR/TDqMIRWAUoVTSowiV0F4zxIsiZUew0cAoQhIYRSiKIlRCe80Q9NpZglGEfJEdRWgugIpkCUYR8khGFKG5gBdFyo5go5Fg87qPioNRhPLVV3EwkzNGCe01Q/zMSMaGxihCMopsChZlLKy+CiIxilAJ7TVThPfOOcwVK7ktdTcmO5dcmPq5ycWZlWfg8o3cwgY3cSzQiMlN7JZM2fQyWZSxsPryjlqGmFwltNdMwChC5LBpFOV39cPiAWaqbLHOQrAoY2H15Q2uuGq5DNdm0F5Fg1GEEMQB6LVDEAegIiGIA1CREMQBqEgI4gBUJARxACoSgjgAFQlBHICKhCAOQEVCEAegIiGIA7wqEkYRksAoQjNEq4eMfsTkLeNFS7bjzgP6plTTJlUFtnu5iKg4+o5n/dwDFrKpWJSxsPoqmcm+6T5pkyqXIV53yubcEsqPUYQopu+lQPMwy2HCRjaV7DIWV1/1h0EuSY60evWJH9MOowhFYBShVNKjCFnSegRqtXLjPHhRpOwINhqFRcXBKEL56ss/6VGE4gxXtsFA8wGoFPkGrwXotbMEowj5IjuKkOwwObjsKtQSlKxMUJEswShCHsmIIrSo61JY1ydgO1OobK9nsXhRpOwINhoJaxT3UXEwilC++ioOZnJaQJXqZAfAwLn6sgL8zEjGhsYoQjKKbAoWZSysvgoiMYrQHEJ475zD3JCSC1d36fLnAZHrVT83uWaz8gxco5Fb2OAmFoE0IvepyU3slkzZ9DJZlLGw+vKOWgZdrqG9qkxczuT2Ksv9jVGEyGGjRMrv6odFw6XKFlMcgkUZC6svb3DFVculXlNlpEf6M66yFAmjCCGIA9BrhyAOQEVCEAegIiGIA1CREMQBqEgI4gBUJARxACoSgjgAFQlBHICKhCAOQEVCEAegIhXF1PuwLngRkB2GaDhaVCB21F1UH8QEKlJBDL/7JRwIXgQkx9VugINr1FgFjOYDcFW6rx7fBkXioCIVxOqq+vr0oq4DsBkG4LT07g17MQ+Zk6Ai1RumN4hTYLEkKsNKTIl1LJCIGviRX9PQTUndjGTmKDFBBwOzVJpBdVP1fo9bwV6mQIon6UU8+d0b4/tDKsF3wveVxItv8Wum4JHqNZ6XVKbgnSi9HOy6/F3xDlEJ7wHVC6hIZSA6aPpLdKaX3uLEX2QzdWp+Lfq9+Bu3HO266YVDAv3NWNkNA8P9BJp2BcNCS605CMv7anCpK82IWwQ7LpyEzWMHoTqDGAbxeHUSiXEZVsPTHXrgFT0oDI/7oIQjo0fngPj8/gQVqUDoOmYVC/JRi4WcmmtsJgNB6FkMj3KjnZYJKlJBUCXaNkJd2zk6m68wYSwKkeox5AzD6f6sUFw8ylG9hcMqG1QkK/h/LRJ5vrRz4cFKDiNFO2gTHDie9kzoNrxfkb1m5HznQU9hwlZDpbsJBjrVh8K24YBXt28G0MNmkTrYY/IMMoQHMfTs6ed6/c49UJGKgM0s43BwjbSmCA7Z5TxyEFaFn62Cgy0nvT2QpYEW+UPhqCzbwPL3aND6vs1EEaPvNqz5Ep5OXfPNbzCKEII4AGckBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUIQB6AiIYgDUJEQxAGoSAjiAFQkBHEAKhKCOAAVCUEcgIqEIA5ARUKQWQPw/wEs6It4VgDbmwAAAABJRU5ErkJggg=="}}},{"cell_type":"markdown","source":"0: Age\n1: Gender\n2: Annual Income\n3: Marital Status\n4: Number of Dependents\n5: Education Level\n6: Occupation\n7: Health Score\n8: Location\n9: Policy Type\n10: Previous Claims\n11: Vehicle Age\n12: Credit Score\n13: Insurance Duration\n14: Policy Start Date\n15: Customer Feedback\n16: Smoking Status\n17: Exercise Frequency\n18: Property Type\n19: Premium Amount\n20: Policy_Start_Year\n21: Policy_Start_Month\n22: Policy_Start_Day\n23: day_of_week\n24: day_sin\n25: day_cos\n26: day_of_year\n27: year_day_sin\n28: year_day_cos\n29: month\n30: month_sin\n31: month_cos\n32: seconds_since_1970","metadata":{}},{"cell_type":"code","source":"\n\n\n# # Get the column names as a list\n# column_names = X_train.columns.tolist()\n\n# # Print the index number and column name for each column\n# for i, column_name in enumerate(column_names):\n#     print(f\"{i}: {column_name}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T10:58:58.824324Z","iopub.execute_input":"2024-12-25T10:58:58.825147Z","iopub.status.idle":"2024-12-25T10:58:58.830361Z","shell.execute_reply.started":"2024-12-25T10:58:58.825108Z","shell.execute_reply":"2024-12-25T10:58:58.829697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Ensemble method (Stacking)","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.ensemble import StackingRegressor\n\nbase_estimators = [\n    ('model1', cat_model),\n    ('model2', light_model),\n    ('model3', xgb_model),\n    ('model4', hgbc_model)\n]\n\nstacking_model = StackingRegressor(estimators=base_estimators, final_estimator=Ridge())\n\nstacking_model.fit(x_train, y_train)\n\nensemble_pred = stacking_model.predict(x_val)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:38:15.095849Z","iopub.execute_input":"2024-12-25T11:38:15.096486Z","iopub.status.idle":"2024-12-25T11:40:38.482413Z","shell.execute_reply.started":"2024-12-25T11:38:15.096452Z","shell.execute_reply":"2024-12-25T11:40:38.480719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = ensemble_pred\ny_val_true = np.expm1(y_val)\ntrain_min = y_train.min()\nclipped_predictions = np.maximum(pred, train_min)\n\nprint(rmsle(y_val_true, pred)) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:40:38.483255Z","iopub.status.idle":"2024-12-25T11:40:38.483704Z","shell.execute_reply.started":"2024-12-25T11:40:38.48347Z","shell.execute_reply":"2024-12-25T11:40:38.483497Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bias-Variance Tradeoff","metadata":{}},{"cell_type":"markdown","source":"Recently I started diving into bias-variance tradeoff, and one of the suggestions were to plot graph of residuals between model's predicted and true values. So, let's take a look","metadata":{}},{"cell_type":"code","source":"# pred = cat_model.predict(X_test_separated)\n# # y_test_separated = (y_test_separated)\n# residuals = y_test_separated-pred\n# print(residuals)\n# plt.scatter(y_test_separated, residuals)\n# plt.axhline(y=0, color='red', linestyle='--')  # Лінія нульових залишків\n\n# plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# sns.kdeplot(x=pred)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# sns.kdeplot(x=y_test_separated)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.model_selection import learning_curve\n\n# train_sizes, train_scores, test_scores = learning_curve(cat_model, train_enc, y_log, cv=5)\n\n# train_mean = np.mean(train_scores, axis=1)\n# test_mean = np.mean(test_scores, axis=1)\n# train_std = np.std(train_scores, axis=1)\n# test_std = np.std(test_scores, axis=1)\n\n# plt.plot(train_sizes, train_mean, label=\"Training score\", color=\"blue\")\n# plt.plot(train_sizes, test_mean, label=\"Cross-validation score\", color=\"green\")\n# plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color=\"blue\", alpha=0.2)\n# plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color=\"green\", alpha=0.2)\n\n# plt.title(\"Learning Curve (Bias-Variance Tradeoff)\")\n# plt.xlabel(\"Training Size\")\n# plt.ylabel(\"Score\")\n# plt.legend(loc=\"best\")\n# plt.grid(True)\n# plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CV score","metadata":{}},{"cell_type":"markdown","source":"After my posted discussion I decided not to follow public leaderboard, but instead follow cross validation and kfold score for models","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T22:04:28.408808Z","iopub.execute_input":"2024-12-23T22:04:28.409134Z","iopub.status.idle":"2024-12-23T22:04:28.41435Z","shell.execute_reply.started":"2024-12-23T22:04:28.409107Z","shell.execute_reply":"2024-12-23T22:04:28.413468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def perform_cv_score(estimator, X, y, scoring):\n    \n    scores = cross_val_score(estimator, X=X, \n                             y=y,scoring=scoring, cv = 10, \n                             n_jobs=1,  error_score=\"raise\")\n\n    return scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T22:04:30.472026Z","iopub.execute_input":"2024-12-23T22:04:30.472401Z","iopub.status.idle":"2024-12-23T22:04:30.477207Z","shell.execute_reply.started":"2024-12-23T22:04:30.472361Z","shell.execute_reply":"2024-12-23T22:04:30.476336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# data = {\n#         \"XGBRegressor\": np.average(-1*perform_cv_score(xgb_model, train_enc, y_log, rmsle_scorer)),       \n#        \"LightGBM\": np.average(-1*perform_cv_score(light_model, train_enc, y_log, rmsle_scorer)),\n#         \"CatBoost\": np.average(-1*perform_cv_score(cat_model, train_enc, y_log, rmsle_scorer)),\n#     'HistGradient': np.average(-1*perform_cv_score(hgbc_model, train_enc, y_log, rmsle_scorer)),\n#     'StackingEnsemble': np.average(-1*perform_cv_score(stacking_model, train_enc, y_log, rmsle_scorer))\n#        }\n\n# data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T22:04:46.888873Z","iopub.execute_input":"2024-12-23T22:04:46.889196Z","iopub.status.idle":"2024-12-23T22:08:24.085665Z","shell.execute_reply.started":"2024-12-23T22:04:46.889169Z","shell.execute_reply":"2024-12-23T22:08:24.084165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"_uuid":"138f9e70-775a-4d9b-98f1-751b9696ecc4","_cell_guid":"d25d9ca9-ec27-4ed0-b49a-f17cfbf1abd8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"predictions = {\n    'xgb': xgb_model.predict(test_enc),\n    'light': light_model.predict(test_enc),\n    'cat': cat_model.predict(test_enc),\n    'hgbc': hgbc_model.predict(test_enc),\n    'ensemble': stacking_model.predict(test_enc)\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T11:42:02.30244Z","iopub.execute_input":"2024-12-25T11:42:02.303279Z","iopub.status.idle":"2024-12-25T11:42:18.363367Z","shell.execute_reply.started":"2024-12-25T11:42:02.303245Z","shell.execute_reply":"2024-12-25T11:42:18.362386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for key, pred in predictions.items():\n    submission = pd.DataFrame({'id': id_test, 'Premium Amount': np.expm1(pred)})\n    submission.to_csv(f\"submission_insurance_{key}.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}