{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Loading Data ","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport polars as pl # data processing, CSV file I/O (e.g. pd.read_csv)\npl.Config.set_tbl_cols(20)\npl.Config.set_tbl_rows(30)\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n    \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:00.974583Z","iopub.execute_input":"2024-12-24T14:52:00.974908Z","iopub.status.idle":"2024-12-24T14:52:01.319344Z","shell.execute_reply.started":"2024-12-24T14:52:00.974869Z","shell.execute_reply":"2024-12-24T14:52:01.318245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kaggle_path = '/kaggle/input/playground-series-s4e12'\n\ntrain_df = pl.scan_csv(f'{kaggle_path}/train.csv').collect()\ntest_df = pl.scan_csv(f'{kaggle_path}/test.csv').collect()\nsample_df = pl.scan_csv(f'{kaggle_path}/sample_submission.csv').collect()\n\ndisplay (train_df.collect_schema())\n\ndisplay (train_df.head(5))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:01.321611Z","iopub.execute_input":"2024-12-24T14:52:01.32198Z","iopub.status.idle":"2024-12-24T14:52:05.3991Z","shell.execute_reply.started":"2024-12-24T14:52:01.321914Z","shell.execute_reply":"2024-12-24T14:52:05.397857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars.selectors as cs\n\ntrain_df.select (cs.numeric()).describe()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:05.404382Z","iopub.execute_input":"2024-12-24T14:52:05.404674Z","iopub.status.idle":"2024-12-24T14:52:05.653738Z","shell.execute_reply.started":"2024-12-24T14:52:05.404646Z","shell.execute_reply":"2024-12-24T14:52:05.652434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\n\nsns.histplot (train_df, x= 'Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:05.657527Z","iopub.execute_input":"2024-12-24T14:52:05.658053Z","iopub.status.idle":"2024-12-24T14:52:08.268635Z","shell.execute_reply.started":"2024-12-24T14:52:05.658004Z","shell.execute_reply":"2024-12-24T14:52:08.267394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features =   ['Gender', 'Marital Status',  'Education Level', 'Occupation', 'Location', 'Policy Type', 'Customer Feedback',  \n                  'Smoking Status', 'Exercise Frequency', 'Property Type', 'Age', 'Number of Dependents', 'Previous Claims', 'Vehicle Age']\n\nfor c in cat_features:\n    print (f'values for {c}')\n    distribution = train_df.group_by(by = c).agg (pl.col('Premium Amount').mean(), pl.col('id').len()   )\n    print (distribution)\n    sns.barplot (data = distribution.to_pandas(), x = 'by', y = 'id')\n    \n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T15:03:00.881294Z","iopub.execute_input":"2024-12-24T15:03:00.881691Z","iopub.status.idle":"2024-12-24T15:03:02.386469Z","shell.execute_reply.started":"2024-12-24T15:03:00.881649Z","shell.execute_reply":"2024-12-24T15:03:02.385144Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"cell_type":"code","source":"def add_features  (raw : pl.DataFrame) -> pl.DataFrame :\n    result = raw.with_columns (pl.when (pl.col('Number of Dependents').is_null()).then(\n                                             pl.lit (None)).otherwise(\n                                           pl.col('Annual Income')  / (pl.col('Number of Dependents') +1)).alias ('Income per person'), \n                                pl.when (pl.col ('Insurance Duration').is_null()).then(\n                                              pl.lit (None)).otherwise \n                                 ( pl.col('Previous Claims') / pl.col('Insurance Duration') ).alias ('Claims per year'),\n                                 pl.col('Annual Income').log().alias ('log Annual Income'), \n                                 pl.col('Previous Claims').log().alias ('log Previous Claims'))        \n    return result \n\t","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.135532Z","iopub.status.idle":"2024-12-24T14:52:09.136002Z","shell.execute_reply.started":"2024-12-24T14:52:09.135785Z","shell.execute_reply":"2024-12-24T14:52:09.135807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_null_count (raw : pl.DataFrame) -> pl.DataFrame :\n    \n    result = raw.with_columns (pl.lit(0).alias ('null count'))\n    for c in raw.columns :\n        result = result.with_columns ((raw.get_column (c).is_null() + pl.col ('null count')).alias ('null count') ) \n    return result ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.13831Z","iopub.status.idle":"2024-12-24T14:52:09.138897Z","shell.execute_reply.started":"2024-12-24T14:52:09.138619Z","shell.execute_reply":"2024-12-24T14:52:09.138647Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Autogluon requires minimal cleaning. Using the log of the premium ammount makes the loss function a lot easier and improves the results. I also assume that the time of day does not impact the result.  ","metadata":{}},{"cell_type":"code","source":"num_to_cat_features = ['Age', 'Number of Dependents', 'Previous Claims', 'Vehicle Age']\ndef data_clean (raw : pl.DataFrame) -> pl.DataFrame :\n    drop_columns = ['Policy Start Date']\n    result = raw.with_columns(pl.col('Policy Start Date').str.head (10).alias('Policy Start Day'))\n    if ('Premium Amount' in raw.columns) :\n        result = result.filter (~pl.col('Premium Amount').is_nan())\n        result = result.filter (pl.col('Premium Amount') > 0)\n        result = result.with_columns(pl.col('Premium Amount').log(). alias('Premium log'))\n        \n        result = result.with_columns(pl.when (pl.col('Premium Amount').is_between (0, 1000)\n                                             ).then (pl.lit(1)).otherwise (pl.lit(2)).alias('my_weight'))    \n    \n    result = add_null_count(result) \n    # result = add_features (result)  \n    # for c in num_to_cat_features :\n    #   result = result.with_columns (pl.col(c).cast(pl.Utf8))\n    \n    return result\n\ntrain_clean_df = data_clean (train_df)\ntest_clean_df = data_clean (test_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.140366Z","iopub.status.idle":"2024-12-24T14:52:09.140929Z","shell.execute_reply.started":"2024-12-24T14:52:09.140651Z","shell.execute_reply":"2024-12-24T14:52:09.14068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# looking at the added features \nshow_me = [n for n in [ 'Claims per year','Family size', 'null count'] if n in train_clean_df.columns]\n\ntrain_clean_df.select (show_me).describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.143782Z","iopub.status.idle":"2024-12-24T14:52:09.144365Z","shell.execute_reply.started":"2024-12-24T14:52:09.144085Z","shell.execute_reply":"2024-12-24T14:52:09.144113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Optional : combining train and original data ","metadata":{}},{"cell_type":"code","source":"\n#original_df = pl.scan_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv', \n#                        ).collect()\n\n#display  (original_df.collect_schema())                       \n#display (original_df.head(5))  \n#train_clean_df = train_clean_df.with_columns (pl.lit (2).alias ('my_weight'))\n\n\n#original_clean_df = data_clean (original_df)\n#original_clean_df = original_clean_df.with_columns (pl.lit (1).alias ('my_weight'))\n#train_clean_df = pl.concat ([train_clean_df, original_clean_df], how = 'vertical_relaxed')\n#train_clean_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.146323Z","iopub.status.idle":"2024-12-24T14:52:09.146878Z","shell.execute_reply.started":"2024-12-24T14:52:09.146593Z","shell.execute_reply":"2024-12-24T14:52:09.14662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_clean_df = train_clean_df.sample (fraction = 1, shuffle = True) \n\nn = train_clean_df.shape [0]\n\ntrain_size = int (np.rint (n * 0.95))\nvalidation_size = n - train_size\n\ntrain_clean_df = train_clean_df.head (train_size)\n\nvalidation_clean_df = train_clean_df.tail (validation_size)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.148273Z","iopub.status.idle":"2024-12-24T14:52:09.148826Z","shell.execute_reply.started":"2024-12-24T14:52:09.148532Z","shell.execute_reply":"2024-12-24T14:52:09.148558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print (validation_clean_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.15049Z","iopub.status.idle":"2024-12-24T14:52:09.151078Z","shell.execute_reply.started":"2024-12-24T14:52:09.150766Z","shell.execute_reply":"2024-12-24T14:52:09.150792Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Installing Libraries ","metadata":{}},{"cell_type":"code","source":"!pip install ray==2.10.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.152198Z","iopub.status.idle":"2024-12-24T14:52:09.152713Z","shell.execute_reply.started":"2024-12-24T14:52:09.152436Z","shell.execute_reply":"2024-12-24T14:52:09.152462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install google-cloud-bigquery==3.10.0\n!pip install google-cloud-storage==1.44.0\n!pip install numpy==2.2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.153817Z","iopub.status.idle":"2024-12-24T14:52:09.154349Z","shell.execute_reply.started":"2024-12-24T14:52:09.154082Z","shell.execute_reply":"2024-12-24T14:52:09.154108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install autogluon.tabular --no-cache-dir -q\n!pip install -U ipywidgets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.15632Z","iopub.status.idle":"2024-12-24T14:52:09.156856Z","shell.execute_reply.started":"2024-12-24T14:52:09.156567Z","shell.execute_reply":"2024-12-24T14:52:09.156593Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The training loop","metadata":{}},{"cell_type":"code","source":"from autogluon.tabular import TabularPredictor\n\nfrom autogluon.common import space\n\nimport warnings\n\nwarnings.simplefilter(\"ignore\")\n\nprint ('done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.15856Z","iopub.status.idle":"2024-12-24T14:52:09.15906Z","shell.execute_reply.started":"2024-12-24T14:52:09.158835Z","shell.execute_reply":"2024-12-24T14:52:09.158855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" # hyperparameter tuning (optional)","metadata":{}},{"cell_type":"code","source":"\n# cat_options = { 'iterations' : space.Int(lower=50, upper=2000, default=1000),\n#                 'learning_rate' : space.Real(0.00001, 0.5, 0.03, log=True),\n#                 'depth' : space.Int(lower=4, upper=10, default=6),\n#                 'random_strength' : space.Real(1.0, 8.5, 3),\n#                 'l2_leaf_reg' : space.Real(0.3, 1.0, 0.5)\n#                 }\n\n# hyperparameters = {  # hyperparameters of each model type\n#                    'CAT': cat_options}\n\n# #time_limit = 2*60  # train various models for ~2 min\n# #num_trials = 80  # try at most 80 different hyperparameter configurations for each type of model\n# #search_strategy = 'auto'  # to tune hyperparameters using random search routine with a local scheduler\n\n# hyperparameter_tune_kwargs = {  # HPO is not performed unless hyperparameter_tune_kwargs is specified\n#     'num_trials': num_trials,\n#     'scheduler' : 'local',\n#     'searcher': search_strategy,\n# }  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.16145Z","iopub.status.idle":"2024-12-24T14:52:09.1621Z","shell.execute_reply.started":"2024-12-24T14:52:09.161757Z","shell.execute_reply":"2024-12-24T14:52:09.161785Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Training loop","metadata":{}},{"cell_type":"code","source":"%%time\n\npredictor_with_log = TabularPredictor(path = '/kaggle/working/Autogluon/with_log',\n                                       label='Premium log', \n                               problem_type = 'regression', \n                               # eval_metric = 'root_mean_squared_error',  \n                               # sample_weight = 'my_weight',\n                               learner_kwargs = {'ignored_columns' : [\n                                   'id', \n                                   'Premium Amount', \n                                  #'Premium log'\n                                  'my_weight'\n                                    ]})\n\n\npredictor_with_log.fit(train_data= train_clean_df.to_pandas(), \n                        dynamic_stacking=False, num_stack_levels=5,\n                        num_bag_folds=5,\n                        num_bag_sets=2,    \n                        presets='experimental_quality',\n# best_quality,  medium_quality                         \n                        time_limit = 33000,\n                        num_gpus=1\n#                        hyperparameters=hyperparameters,\n#                        hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n                        )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.164833Z","iopub.status.idle":"2024-12-24T14:52:09.165431Z","shell.execute_reply.started":"2024-12-24T14:52:09.165144Z","shell.execute_reply":"2024-12-24T14:52:09.165173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print ('leaderboard with log') \npredictor_with_log.leaderboard()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.166576Z","iopub.status.idle":"2024-12-24T14:52:09.167147Z","shell.execute_reply.started":"2024-12-24T14:52:09.166848Z","shell.execute_reply":"2024-12-24T14:52:09.166874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor_with_log.features()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.168631Z","iopub.status.idle":"2024-12-24T14:52:09.16916Z","shell.execute_reply.started":"2024-12-24T14:52:09.168918Z","shell.execute_reply":"2024-12-24T14:52:09.16897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor_with_log.evaluate (validation_clean_df.to_pandas())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.171004Z","iopub.status.idle":"2024-12-24T14:52:09.171592Z","shell.execute_reply.started":"2024-12-24T14:52:09.171308Z","shell.execute_reply":"2024-12-24T14:52:09.171337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = pl.Series (predictor_with_log.predict (validation_clean_df.to_pandas())).exp()\n\ny_pred.describe ()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.173522Z","iopub.status.idle":"2024-12-24T14:52:09.174089Z","shell.execute_reply.started":"2024-12-24T14:52:09.173783Z","shell.execute_reply":"2024-12-24T14:52:09.173819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_true = validation_clean_df.get_column ('Premium Amount')\n\nsns.scatterplot (x = y_true.to_numpy(), y = y_pred.to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.175985Z","iopub.status.idle":"2024-12-24T14:52:09.176499Z","shell.execute_reply.started":"2024-12-24T14:52:09.176233Z","shell.execute_reply":"2024-12-24T14:52:09.176259Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fetching Results ","metadata":{}},{"cell_type":"code","source":"\n# result_with_log = pl.scan_csv('/kaggle/input/previous-results/submission (14).csv').collect()\n# result_without_log = pl.scan_csv('/kaggle/input/previous-results/submission (15).csv').collect()\n# submission = sample_df.with_columns ( (result_with_log.get_column('Premium Amount') + pl.lit (200)\n#                                      ).alias('Premium Amount'))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.178377Z","iopub.status.idle":"2024-12-24T14:52:09.178914Z","shell.execute_reply.started":"2024-12-24T14:52:09.178632Z","shell.execute_reply":"2024-12-24T14:52:09.17866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # this cell allows to reload the trained model from disk\n# predictor = TabularPredictor.load(\"/kaggle/working/Autogluon\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.180967Z","iopub.status.idle":"2024-12-24T14:52:09.181522Z","shell.execute_reply.started":"2024-12-24T14:52:09.181235Z","shell.execute_reply":"2024-12-24T14:52:09.181261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions_with_log = pl.Series (predictor_with_log.predict(test_clean_df.to_pandas () ))\n\nprint ('done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.182996Z","iopub.status.idle":"2024-12-24T14:52:09.183524Z","shell.execute_reply.started":"2024-12-24T14:52:09.183254Z","shell.execute_reply":"2024-12-24T14:52:09.183279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # transforming the results back from the log conversion \npredictions_transform = predictions_with_log.exp()\n\nprint (predictions_transform.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.184869Z","iopub.status.idle":"2024-12-24T14:52:09.185422Z","shell.execute_reply.started":"2024-12-24T14:52:09.185152Z","shell.execute_reply":"2024-12-24T14:52:09.185178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.schema","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.186486Z","iopub.status.idle":"2024-12-24T14:52:09.187026Z","shell.execute_reply.started":"2024-12-24T14:52:09.186729Z","shell.execute_reply":"2024-12-24T14:52:09.186755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = sample_df.with_columns ((predictions_transform).alias('Premium Amount'))\nsubmission.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.189338Z","iopub.status.idle":"2024-12-24T14:52:09.189892Z","shell.execute_reply.started":"2024-12-24T14:52:09.189599Z","shell.execute_reply":"2024-12-24T14:52:09.189628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.write_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.191546Z","iopub.status.idle":"2024-12-24T14:52:09.192096Z","shell.execute_reply.started":"2024-12-24T14:52:09.191789Z","shell.execute_reply":"2024-12-24T14:52:09.191824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time \nimport zipfile\n\ndef zip_files_in_directory(directory, zip_name):\n    num_files_deleted = 0 \n    with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf:\n        for root, dirs, files in os.walk(directory):\n            for file in files:\n                file_path = os.path.join(root, file)\n                zipf.write(file_path, os.path.relpath(file_path, directory))\n                os.remove(file_path)\n                num_files_deleted += 1\n    return num_files_deleted  \n# Example usage\ndirectory = '/kaggle/working/Autogluon'\nzip_name = 'Autogluon.zip'\nn = zip_files_in_directory(directory, zip_name)\n\nprint(f\"deleted {n} files in {directory}\")","metadata":{"trusted":true,"scrolled":true,"execution":{"iopub.status.busy":"2024-12-24T14:52:09.194551Z","iopub.status.idle":"2024-12-24T14:52:09.195116Z","shell.execute_reply.started":"2024-12-24T14:52:09.194813Z","shell.execute_reply":"2024-12-24T14:52:09.19484Z"}},"outputs":[],"execution_count":null}]}