{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"toc_visible":true,"machine_shape":"hm","authorship_tag":"ABX9TyOQkUWXRyXI9jIOuEkDVPhf"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **EDA Insurance Prediction Premium - First Submission with autogluon**\n\nLets take advantage of autogluon ecosystem for this first submission. ","metadata":{"id":"gtCHSHc_b3Wb"}},{"cell_type":"code","source":"%%capture\n%pip install -q autogluon autogluon.eda ray==2.2.0 pandas","metadata":{"id":"ZxwwboJucI6K","executionInfo":{"status":"ok","timestamp":1734123668764,"user_tz":180,"elapsed":204702,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:46:48.785675Z","iopub.execute_input":"2024-12-13T22:46:48.785942Z","iopub.status.idle":"2024-12-13T22:55:09.406696Z","shell.execute_reply.started":"2024-12-13T22:46:48.785914Z","shell.execute_reply":"2024-12-13T22:55:09.405419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import basic libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport warnings\nimport cloudpickle\nimport plotly.io as pio\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\npd.options.plotting.backend = \"plotly\"\npio.templates.default = \"simple_white\"\nwarnings.filterwarnings('ignore')\n\n# Import specific libraries\nimport autogluon.eda.auto as auto\nfrom autogluon.tabular import TabularDataset, TabularPredictor","metadata":{"id":"ksay2atjbwbZ","executionInfo":{"status":"ok","timestamp":1734125235759,"user_tz":180,"elapsed":548,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T00:54:57.575777Z","iopub.execute_input":"2024-12-14T00:54:57.576111Z","iopub.status.idle":"2024-12-14T00:54:57.597697Z","shell.execute_reply.started":"2024-12-14T00:54:57.576087Z","shell.execute_reply":"2024-12-14T00:54:57.596803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\noriginal = pd.read_csv(\"/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv\")","metadata":{"id":"FW9nGCRsb2fg","executionInfo":{"status":"ok","timestamp":1734124199630,"user_tz":180,"elapsed":5628,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:12.284262Z","iopub.execute_input":"2024-12-13T22:55:12.284688Z","iopub.status.idle":"2024-12-13T22:55:22.09326Z","shell.execute_reply.started":"2024-12-13T22:55:12.284662Z","shell.execute_reply":"2024-12-13T22:55:22.092326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Lets prepare the datasets","metadata":{}},{"cell_type":"code","source":"train.set_index('id', inplace=True)\ntest.set_index('id', inplace=True)\n\n# Renaming columns for consistency\ntrain.columns = train.columns.str.lower()\ntest.columns = test.columns.str.lower()\noriginal.columns = original.columns.str.lower()\ntrain.columns = [col.replace(\" \", \"_\") for col in train.columns]\ntest.columns = [col.replace(\" \", \"_\") for col in test.columns]\noriginal.columns = [col.replace(\" \", \"_\") for col in original.columns]\noriginal  = original[train.columns]\noriginal = original.dropna(subset=['premium_amount'])\n\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:22.095236Z","iopub.execute_input":"2024-12-13T22:55:22.095554Z","iopub.status.idle":"2024-12-13T22:55:22.198407Z","shell.execute_reply.started":"2024-12-13T22:55:22.095527Z","shell.execute_reply":"2024-12-13T22:55:22.197236Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Autogluon EDA**\n\nI've recently found autogluon has also a EDA component. Lets try it out.","metadata":{"id":"a37M02TzfUCa"}},{"cell_type":"markdown","source":"## **Dataset Overview**","metadata":{"id":"tJ57DAPNkWUO"}},{"cell_type":"code","source":"target_col = 'premium_amount'\nauto.dataset_overview(train_data=train, label=target_col, sample=0.2)","metadata":{"id":"qKP3XLdQdSFH","executionInfo":{"status":"ok","timestamp":1733884391058,"user_tz":180,"elapsed":15543,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"outputId":"e833b382-860d-41c0-c9eb-ec9a31e106a1","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:22.199701Z","iopub.execute_input":"2024-12-13T22:55:22.200059Z","iopub.status.idle":"2024-12-13T22:55:26.575561Z","shell.execute_reply.started":"2024-12-13T22:55:22.200024Z","shell.execute_reply":"2024-12-13T22:55:26.574375Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- There are mostly categorical variables, with one date variable.\n- We could do feature engineering with the policy start date, for example, relate the start with the age. Probable there could be some information as you took the policy later in life.\n- We do have missing values but it does not seem severe. ","metadata":{}},{"cell_type":"markdown","source":"## **Target Analysis**","metadata":{"id":"mqWGtUaHkaG5"}},{"cell_type":"code","source":"auto.target_analysis(train_data=train, label=target_col, sample=0.2, problem_type=\"regression\", fit_distributions=False)","metadata":{"id":"2BdpqniWkDdQ","executionInfo":{"status":"ok","timestamp":1733886672322,"user_tz":180,"elapsed":1641889,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"outputId":"1d3b3166-87bb-4217-c1c5-7e1736663677","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:26.577241Z","iopub.execute_input":"2024-12-13T22:55:26.577738Z","iopub.status.idle":"2024-12-13T22:55:34.483976Z","shell.execute_reply.started":"2024-12-13T22:55:26.577682Z","shell.execute_reply":"2024-12-13T22:55:34.482995Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- We have a long tail scenario, with a peak very near zero.\n- It seems that correlation is low with the target.\n\nLets transform the target for better analysis.","metadata":{}},{"cell_type":"code","source":"train_log = train.copy()\ntrain_log['premium_amount'] = np.log10(train_log['premium_amount'])\nauto.target_analysis(train_data=train_log, label=\"premium_amount\", sample=0.2, problem_type=\"regression\", fit_distributions=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:34.485244Z","iopub.execute_input":"2024-12-13T22:55:34.486025Z","iopub.status.idle":"2024-12-13T22:55:42.509209Z","shell.execute_reply.started":"2024-12-13T22:55:34.485983Z","shell.execute_reply":"2024-12-13T22:55:42.508517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Lets compare with the **original dataset**","metadata":{}},{"cell_type":"code","source":"original_log = original.copy()\noriginal_log['premium_amount'] = np.log10(original_log['premium_amount'])\nauto.target_analysis(train_data=original_log, label=\"premium_amount\", sample=0.2, problem_type=\"regression\", fit_distributions=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:42.510344Z","iopub.execute_input":"2024-12-13T22:55:42.511143Z","iopub.status.idle":"2024-12-13T22:55:44.466451Z","shell.execute_reply.started":"2024-12-13T22:55:42.511103Z","shell.execute_reply":"2024-12-13T22:55:44.465748Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Overall they have similar distribution, although the training data looks noisy for the target.\n- The training data has a bunch of low value observations which could be problematic.\n- The original data has a higher mean. ","metadata":{}},{"cell_type":"markdown","source":"## **Missing values Analysis**","metadata":{"id":"YU8do2WqlEZJ"}},{"cell_type":"code","source":"auto.missing_values_analysis(train_data=train, sample=0.5)","metadata":{"id":"iwaQPu8rlHlQ","executionInfo":{"status":"ok","timestamp":1733886720158,"user_tz":180,"elapsed":4653,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"outputId":"c39e243e-9cda-444f-f6d8-55cda94bbc29","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:44.467555Z","iopub.execute_input":"2024-12-13T22:55:44.46791Z","iopub.status.idle":"2024-12-13T22:55:49.739602Z","shell.execute_reply.started":"2024-12-13T22:55:44.467872Z","shell.execute_reply":"2024-12-13T22:55:49.738725Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- `occupation` and `previous_claims` have the highest missing values.\n- It doesn't seem that a there are concentrated rows with missing values. ","metadata":{}},{"cell_type":"markdown","source":"## **Covariate shift analysis**","metadata":{"id":"wMrRyqlQksSK"}},{"cell_type":"markdown","source":"## **Train - Test**","metadata":{}},{"cell_type":"code","source":"auto.covariate_shift_detection(train_data=train, test_data=test, label=target_col, sample=0.2)","metadata":{"id":"rR9Un632kewK","executionInfo":{"status":"ok","timestamp":1733886715511,"user_tz":180,"elapsed":43195,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"outputId":"9b4a2ce8-9d03-4bfc-a0bb-8c2ad2b761e0","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:55:49.74218Z","iopub.execute_input":"2024-12-13T22:55:49.742508Z","iopub.status.idle":"2024-12-13T22:56:17.133163Z","shell.execute_reply.started":"2024-12-13T22:55:49.742481Z","shell.execute_reply":"2024-12-13T22:56:17.132263Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Train - Original**","metadata":{}},{"cell_type":"code","source":"auto.covariate_shift_detection(train_data=train, test_data=original, label=target_col, sample=0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:56:17.134496Z","iopub.execute_input":"2024-12-13T22:56:17.135189Z","iopub.status.idle":"2024-12-13T22:58:00.115072Z","shell.execute_reply.started":"2024-12-13T22:56:17.135145Z","shell.execute_reply":"2024-12-13T22:58:00.114185Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Interesting, its seems the original dataset is significantly different from train.\n- ¿Could this worsen the scores? Maybe, for now we will train with only training set. ","metadata":{}},{"cell_type":"markdown","source":"## **Quick Fit**","metadata":{"id":"ITzSy14FlZtd"}},{"cell_type":"markdown","source":"## **Without Original Dataset**","metadata":{}},{"cell_type":"code","source":"auto.quick_fit(train, target_col, show_feature_importance_barplots=True)","metadata":{"id":"B83a7xfclVzO","executionInfo":{"status":"ok","timestamp":1734125009785,"user_tz":180,"elapsed":4011,"user":{"displayName":"Fabián Alberto Reyes Madrid","userId":"11772448517145602220"}},"outputId":"d9a47f1e-c174-48d8-83d1-372f08c1a917","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:58:00.116177Z","iopub.execute_input":"2024-12-13T22:58:00.116469Z","iopub.status.idle":"2024-12-13T23:02:00.233669Z","shell.execute_reply.started":"2024-12-13T22:58:00.116437Z","shell.execute_reply":"2024-12-13T23:02:00.232832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **With Original Dataset**","metadata":{}},{"cell_type":"code","source":"original.index = list(range(train.index.max()+1, train.index.max()+1 + len(original)))\ntrain_or = pd.concat([train, original], axis=0)\nauto.quick_fit(train_or, target_col, show_feature_importance_barplots=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T23:02:00.234616Z","iopub.execute_input":"2024-12-13T23:02:00.234875Z","iopub.status.idle":"2024-12-13T23:04:09.624418Z","shell.execute_reply.started":"2024-12-13T23:02:00.23485Z","shell.execute_reply":"2024-12-13T23:04:09.623546Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- A quickfit using original data made the score worse. ","metadata":{}},{"cell_type":"markdown","source":"# **Autogluon Train 8 hours**","metadata":{"id":"XaRagHNL6IcG"}},{"cell_type":"code","source":"# Lets transform the target and variables\ntrain['premium_amount_log'] = np.log10(train['premium_amount'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T23:04:09.6254Z","iopub.execute_input":"2024-12-13T23:04:09.625694Z","iopub.status.idle":"2024-12-13T23:04:09.634243Z","shell.execute_reply.started":"2024-12-13T23:04:09.625667Z","shell.execute_reply":"2024-12-13T23:04:09.633446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Setting up\neval_metric = 'rmse'\nlabel = 'premium_amount_log'\nproblem_type='regression'\nhours = 8\n\n# Models to exclude\nexcluded_model_types = ['KNN']\n\n# Initialize the TabularPredictor\npredictor = TabularPredictor(label=label, eval_metric=eval_metric, problem_type=problem_type,\n                             path = \"/kaggle/working/Autogluon/202412_ps4s12_8hr_training\")\n\n\n\n# Fit the model\npredictor.fit(train_data=train.drop(columns=\"premium_amount\"),\n              time_limit=3600*hours,\n              presets=\"best_quality\",\n              excluded_model_types=excluded_model_types,\n              num_bag_folds=20,\n              num_bag_sets = 20,\n              num_stack_levels=3,\n              ag_args_fit={'num_gpus': 1},\n              \n)","metadata":{"id":"HW8tAB-b6Tcu","trusted":true,"execution":{"iopub.status.busy":"2024-12-13T23:04:09.635487Z","iopub.execute_input":"2024-12-13T23:04:09.635861Z","iopub.status.idle":"2024-12-14T00:04:20.905498Z","shell.execute_reply.started":"2024-12-13T23:04:09.635822Z","shell.execute_reply":"2024-12-14T00:04:20.904661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"leaderboard_test = predictor.leaderboard(silent=True)\nleaderboard_test","metadata":{"id":"PsoZUUls72uC","trusted":true,"execution":{"iopub.status.busy":"2024-12-14T00:55:52.764627Z","iopub.execute_input":"2024-12-14T00:55:52.765451Z","iopub.status.idle":"2024-12-14T00:55:52.78162Z","shell.execute_reply.started":"2024-12-14T00:55:52.765416Z","shell.execute_reply":"2024-12-14T00:55:52.780888Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Submission**","metadata":{}},{"cell_type":"code","source":"models = leaderboard_test.head(5)['model'].to_list()\nbest_model = models[0]\nprint(\"Best model by autogluon is\", models[0])\nprint(\"With a score of RMSLE\", np.abs(leaderboard_test[leaderboard_test['model']==best_model]['score_val'][0]))\n\nsub_autogluon = submission.copy()\nsub_autogluon['premium_amount_log'] = predictor.predict(test, as_pandas=False, model=best_model)\nsub_autogluon['Premium Amount'] = np.power(10, sub_autogluon['premium_amount_log'])\nsub_autogluon.drop(columns=\"premium_amount_log\", inplace=True)\nsub_autogluon.to_csv(f\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T00:42:09.090747Z","iopub.execute_input":"2024-12-14T00:42:09.09108Z","iopub.status.idle":"2024-12-14T00:42:23.307755Z","shell.execute_reply.started":"2024-12-14T00:42:09.091053Z","shell.execute_reply":"2024-12-14T00:42:23.307022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oofs = {}\nfor model in models:\n    print(\"Model:\", model)\n    oofs_model = predictor.get_oof_pred(model=model)\n    oofs[model] = pd.DataFrame(oofs_model.values, index=train.index, columns=[f'{model}_preds'])\n\nwith open(\"/kaggle/working/oofs.pkl\", 'wb') as f:\n    cloudpickle.dump(oofs, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T00:55:06.838237Z","iopub.execute_input":"2024-12-14T00:55:06.839203Z","iopub.status.idle":"2024-12-14T00:55:07.563108Z","shell.execute_reply.started":"2024-12-14T00:55:06.83915Z","shell.execute_reply":"2024-12-14T00:55:07.562207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}