{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":10118268,"sourceType":"datasetVersion","datasetId":6215722},{"sourceId":10145531,"sourceType":"datasetVersion","datasetId":6262475},{"sourceId":211773270,"sourceType":"kernelVersion"},{"sourceId":211945583,"sourceType":"kernelVersion"},{"sourceId":211653094,"sourceType":"kernelVersion"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install autogluon\n!pip install -U ipywidgets","metadata":{"_uuid":"503e2554-66dd-479c-87cd-f7c69205f94f","_cell_guid":"e7cd2864-5e97-42fd-b873-12a84779a992","trusted":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-09T07:58:00.30161Z","iopub.execute_input":"2024-12-09T07:58:00.302084Z","iopub.status.idle":"2024-12-09T07:59:19.522627Z","shell.execute_reply.started":"2024-12-09T07:58:00.302046Z","shell.execute_reply":"2024-12-09T07:59:19.521209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom autogluon.tabular import TabularDataset, TabularPredictor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:19.524678Z","iopub.execute_input":"2024-12-09T07:59:19.525054Z","iopub.status.idle":"2024-12-09T07:59:21.017396Z","shell.execute_reply.started":"2024-12-09T07:59:19.525Z","shell.execute_reply":"2024-12-09T07:59:21.016487Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import Data","metadata":{}},{"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/insurance-4-12/DATABASE (1)/datasets/fe_enccat_best/train.csv')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:46.110141Z","iopub.execute_input":"2024-12-09T07:59:46.110926Z","iopub.status.idle":"2024-12-09T07:59:49.918684Z","shell.execute_reply.started":"2024-12-09T07:59:46.110891Z","shell.execute_reply":"2024-12-09T07:59:49.917519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test=pd.read_csv('/kaggle/input/insurance-4-12/DATABASE (1)/datasets/fe_enccat_best/test.csv')\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:49.920636Z","iopub.execute_input":"2024-12-09T07:59:49.921313Z","iopub.status.idle":"2024-12-09T07:59:52.280819Z","shell.execute_reply.started":"2024-12-09T07:59:49.921266Z","shell.execute_reply":"2024-12-09T07:59:52.279598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:54.43618Z","iopub.execute_input":"2024-12-09T07:59:54.436549Z","iopub.status.idle":"2024-12-09T07:59:55.444759Z","shell.execute_reply.started":"2024-12-09T07:59:54.436519Z","shell.execute_reply":"2024-12-09T07:59:55.44351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label = 'Premium Amount'\ntrain[label].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:57.543361Z","iopub.execute_input":"2024-12-09T07:59:57.543935Z","iopub.status.idle":"2024-12-09T07:59:57.610215Z","shell.execute_reply.started":"2024-12-09T07:59:57.543877Z","shell.execute_reply":"2024-12-09T07:59:57.608984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Premium Amount'] = np.log1p(train['Premium Amount'])\ntrain['Premium Amount'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T07:59:59.615831Z","iopub.execute_input":"2024-12-09T07:59:59.616344Z","iopub.status.idle":"2024-12-09T07:59:59.70978Z","shell.execute_reply.started":"2024-12-09T07:59:59.616298Z","shell.execute_reply":"2024-12-09T07:59:59.708537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## Zeroshot Portfolio 2024 ","metadata":{}},{"cell_type":"code","source":"# \"\"\"TabRepo Rerun + manual changes.\n\n# We re-ran the work of the TabRepo paper, but use a portfolio of size 200 instead of the\n# 100 size portfolio used by AutoGluon’s best_quality setting. We also included more model\n# families: Linear models and KNN.\n\n# # Removed because too slow predict for large datasets\n# LightGBM_r19\n# LightGBM_r96\n# LightGBM_r94_BAG_L1\n# LightGBM_r15_BAG_L1\n# LightGBM_r133_BAG_L1\n# LightGBM_r174\n# XGBoost_r31_BAG_L1\n# XGBoost_r33_BAG_L1\n\n# # Added manually:\n# LightGBM, XGBoost, and CatBoost with different max_bin values.\n# \"\"\"\n\n# zeroshot2024 = {\n#     \"NN_TORCH\": [\n#         {},\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.10077639529843717,\n#             \"hidden_size\": 108,\n#             \"learning_rate\": 0.002735937344002146,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.356433327634438e-12,\n#             \"ag_args\": {\"name_suffix\": \"_r79\", \"priority\": -2},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.11897478034205347,\n#             \"hidden_size\": 213,\n#             \"learning_rate\": 0.0010474382260641949,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 5.594471067786272e-10,\n#             \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": -8},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.10276665747466765,\n#             \"hidden_size\": 165,\n#             \"learning_rate\": 0.0038022969797074676,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 8.916967887098654e-09,\n#             \"ag_args\": {\"name_suffix\": \"_r164\", \"priority\": -19},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.14717103221481653,\n#             \"hidden_size\": 222,\n#             \"learning_rate\": 0.002928569897768998,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 3.3157591552416304e-06,\n#             \"ag_args\": {\"name_suffix\": \"_r115\", \"priority\": -22},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.01030258381183309,\n#             \"hidden_size\": 111,\n#             \"learning_rate\": 0.01845979186513771,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 0.00020238017476912164,\n#             \"ag_args\": {\"name_suffix\": \"_r158\", \"priority\": -26},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.12415233039192458,\n#             \"hidden_size\": 131,\n#             \"learning_rate\": 0.016469659989857787,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 7.620839071729229e-06,\n#             \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -30},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.3905837860053583,\n#             \"hidden_size\": 106,\n#             \"learning_rate\": 0.0018297905295930797,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 9.178069874232892e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r14\", \"priority\": -41},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.3457125770744979,\n#             \"hidden_size\": 37,\n#             \"learning_rate\": 0.006435774191713849,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 2.4012185204155345e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r36\", \"priority\": -48},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.10584781187588926,\n#             \"hidden_size\": 89,\n#             \"learning_rate\": 0.014791925972420345,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 2.0956656927351057e-11,\n#             \"ag_args\": {\"name_suffix\": \"_r171\", \"priority\": -50},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.3717900746799333,\n#             \"hidden_size\": 154,\n#             \"learning_rate\": 0.005908762713584694,\n#             \"num_layers\": 2,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 5.681050909765253e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r200\", \"priority\": -54},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.1703783780377607,\n#             \"hidden_size\": 212,\n#             \"learning_rate\": 0.0004107199833213839,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.105439140660822e-07,\n#             \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -59},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.013288954106470907,\n#             \"hidden_size\": 81,\n#             \"learning_rate\": 0.005340914647396154,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 8.762168370775353e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -64},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.15375268291707994,\n#             \"hidden_size\": 95,\n#             \"learning_rate\": 0.016181912728325788,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 1.6514168330341684e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r2\", \"priority\": -71},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.059123470678908954,\n#             \"hidden_size\": 18,\n#             \"learning_rate\": 0.0002939890353572163,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.1744592227570546e-11,\n#             \"ag_args\": {\"name_suffix\": \"_r122\", \"priority\": -76},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.13685500504288658,\n#             \"hidden_size\": 229,\n#             \"learning_rate\": 0.0001934231631502052,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.0282176418614337e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r157\", \"priority\": -83},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.2211285919550286,\n#             \"hidden_size\": 196,\n#             \"learning_rate\": 0.011307978270179143,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.8441764217351068e-06,\n#             \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -85},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.36669080773207274,\n#             \"hidden_size\": 95,\n#             \"learning_rate\": 0.015280159186761077,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.3082489374636015e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r87\", \"priority\": -94},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.23713784729000734,\n#             \"hidden_size\": 200,\n#             \"learning_rate\": 0.00311256170909018,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 4.573016756474468e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -97},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.06134755114373829,\n#             \"hidden_size\": 144,\n#             \"learning_rate\": 0.005834535148903801,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 2.0826540090463355e-09,\n#             \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -101},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.09976801642258049,\n#             \"hidden_size\": 135,\n#             \"learning_rate\": 0.001631450730978947,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 3.867683394425807e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r86\", \"priority\": -103},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.3027114570947557,\n#             \"hidden_size\": 196,\n#             \"learning_rate\": 0.006482759295309238,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 1.2806509958776e-12,\n#             \"ag_args\": {\"name_suffix\": \"_r71\", \"priority\": -109},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.01759017280963624,\n#             \"hidden_size\": 250,\n#             \"learning_rate\": 0.00027701426675638685,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 2.941370491259882e-10,\n#             \"ag_args\": {\"name_suffix\": \"_r190\", \"priority\": -111},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.14075719847721704,\n#             \"hidden_size\": 107,\n#             \"learning_rate\": 0.02102016621542758,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 8.583524549723169e-10,\n#             \"ag_args\": {\"name_suffix\": \"_r117\", \"priority\": -112},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.34251047184911276,\n#             \"hidden_size\": 199,\n#             \"learning_rate\": 0.00013693277454657866,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 0.005425887964567897,\n#             \"ag_args\": {\"name_suffix\": \"_r134\", \"priority\": -120},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.2232260403696044,\n#             \"hidden_size\": 10,\n#             \"learning_rate\": 0.00016660531025826497,\n#             \"num_layers\": 2,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.851165282762072e-12,\n#             \"ag_args\": {\"name_suffix\": \"_r100\", \"priority\": -129},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.383666641214089,\n#             \"hidden_size\": 14,\n#             \"learning_rate\": 0.0017662443351764005,\n#             \"num_layers\": 2,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 1.180743534035588e-11,\n#             \"ag_args\": {\"name_suffix\": \"_r72\", \"priority\": -135},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.006531401073483156,\n#             \"hidden_size\": 192,\n#             \"learning_rate\": 0.012418052210914356,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 3.0406866089493607e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r76\", \"priority\": -138},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.35413506384383425,\n#             \"hidden_size\": 128,\n#             \"learning_rate\": 0.02291819787571232,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 2.347385665763067e-10,\n#             \"ag_args\": {\"name_suffix\": \"_r43\", \"priority\": -145},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.13744220125923948,\n#             \"hidden_size\": 245,\n#             \"learning_rate\": 0.01760548103080169,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 4.619518091833427e-09,\n#             \"ag_args\": {\"name_suffix\": \"_r57\", \"priority\": -147},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.33567564890346097,\n#             \"hidden_size\": 245,\n#             \"learning_rate\": 0.006746560197328548,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.6470047305392933e-10,\n#             \"ag_args\": {\"name_suffix\": \"_r89\", \"priority\": -148},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.3834963692226372,\n#             \"hidden_size\": 30,\n#             \"learning_rate\": 0.0006387876089505596,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 0.020005485780993044,\n#             \"ag_args\": {\"name_suffix\": \"_r120\", \"priority\": -153},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.12166942295569863,\n#             \"hidden_size\": 151,\n#             \"learning_rate\": 0.0018866871631794007,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 9.190843763153802e-05,\n#             \"ag_args\": {\"name_suffix\": \"_r185\", \"priority\": -157},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.17934767155919895,\n#             \"hidden_size\": 26,\n#             \"learning_rate\": 0.0002670236000453949,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 9.550309584226972e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r169\", \"priority\": -158},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.28421136108893824,\n#             \"hidden_size\": 233,\n#             \"learning_rate\": 0.004756576344566884,\n#             \"num_layers\": 2,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 0.004694424874123613,\n#             \"ag_args\": {\"name_suffix\": \"_r98\", \"priority\": -165},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.19838811814938784,\n#             \"hidden_size\": 91,\n#             \"learning_rate\": 0.0018766153235863243,\n#             \"num_layers\": 5,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.9551641972196325e-11,\n#             \"ag_args\": {\"name_suffix\": \"_r139\", \"priority\": -167},\n#         },\n#         {\n#             \"activation\": \"elu\",\n#             \"dropout_prob\": 0.08014572877647762,\n#             \"hidden_size\": 238,\n#             \"learning_rate\": 0.003193230959659184,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 0.006268749322131784,\n#             \"ag_args\": {\"name_suffix\": \"_r114\", \"priority\": -168},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.22890076231634937,\n#             \"hidden_size\": 188,\n#             \"learning_rate\": 0.016161752294108052,\n#             \"num_layers\": 1,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 0.0581139255429484,\n#             \"ag_args\": {\"name_suffix\": \"_r21\", \"priority\": -181},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.31096302473950127,\n#             \"hidden_size\": 115,\n#             \"learning_rate\": 0.00027659068642753017,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 1.2781148440039829e-12,\n#             \"ag_args\": {\"name_suffix\": \"_r64\", \"priority\": -183},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.05930745685905112,\n#             \"hidden_size\": 14,\n#             \"learning_rate\": 0.00011882187473200266,\n#             \"num_layers\": 4,\n#             \"use_batchnorm\": False,\n#             \"weight_decay\": 1.2573893223363365e-08,\n#             \"ag_args\": {\"name_suffix\": \"_r99\", \"priority\": -191},\n#         },\n#         {\n#             \"activation\": \"relu\",\n#             \"dropout_prob\": 0.22187512346456761,\n#             \"hidden_size\": 97,\n#             \"learning_rate\": 0.01090384075529055,\n#             \"num_layers\": 3,\n#             \"use_batchnorm\": True,\n#             \"weight_decay\": 9.24700655189339e-09,\n#             \"ag_args\": {\"name_suffix\": \"_r83\", \"priority\": -192},\n#         },\n#     ],\n#     \"GBM\": [\n#         {},\n#         {  # Added manually.\n#             \"max_bin\": 4095,\n#             \"ag_args\": {\"priority\": -1, \"name_suffix\": \"Bin4095\"},\n#         },\n#         {\"extra_trees\": True, \"ag_args\": {\"name_suffix\": \"XT\"}},\n#         {  # Old GBMLarge\n#             \"learning_rate\": 0.03,\n#             \"num_leaves\": 128,\n#             \"feature_fraction\": 0.9,\n#             \"min_data_in_leaf\": 3,\n#             \"ag_args\": {\"name_suffix\": \"Large\", \"priority\": 0, \"hyperparameter_tune_kwargs\": None},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.7023601671276614,\n#             \"learning_rate\": 0.012144796373999013,\n#             \"min_data_in_leaf\": 14,\n#             \"num_leaves\": 53,\n#             \"ag_args\": {\"name_suffix\": \"_r131\", \"priority\": -3},\n#         },\n#         # { # TOo slow for large datasets\n#         #     \"extra_trees\": True,\n#         #     \"feature_fraction\": 0.5636931414546802,\n#         #     \"learning_rate\": 0.01518660230385841,\n#         #     \"min_data_in_leaf\": 48,\n#         #     \"num_leaves\": 16,\n#         #     \"ag_args\": {\"name_suffix\": \"_r96\", \"priority\": -7},\n#         # },\n#         # { # Too slow\n#         #     \"extra_trees\": True,\n#         #     \"feature_fraction\": 0.7291114577678862,\n#         #     \"learning_rate\": 0.02371597110187949,\n#         #     \"min_data_in_leaf\": 5,\n#         #     \"num_leaves\": 90,\n#         #     \"ag_args\": {\"name_suffix\": \"_r133\", \"priority\": -13},\n#         # },\n#         # { # Too slow\n#         #     \"extra_trees\": False,\n#         #     \"feature_fraction\": 0.7421180622507277,\n#         #     \"learning_rate\": 0.018603888565740096,\n#         #     \"min_data_in_leaf\": 6,\n#         #     \"num_leaves\": 22,\n#         #     \"ag_args\": {\"name_suffix\": \"_r15\", \"priority\": -15},\n#         # },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.8999894845710796,\n#             \"learning_rate\": 0.051087336729504676,\n#             \"min_data_in_leaf\": 18,\n#             \"num_leaves\": 167,\n#             \"ag_args\": {\"name_suffix\": \"_r191\", \"priority\": -24},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.7325454610506641,\n#             \"learning_rate\": 0.009447054356012436,\n#             \"min_data_in_leaf\": 4,\n#             \"num_leaves\": 85,\n#             \"ag_args\": {\"name_suffix\": \"_r54\", \"priority\": -27},\n#         },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.8682559906624081,\n#             \"learning_rate\": 0.09561511371136407,\n#             \"min_data_in_leaf\": 9,\n#             \"num_leaves\": 121,\n#             \"ag_args\": {\"name_suffix\": \"_r81\", \"priority\": -28},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.8254432681390782,\n#             \"learning_rate\": 0.031251656439648626,\n#             \"min_data_in_leaf\": 50,\n#             \"num_leaves\": 210,\n#             \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -32},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.9668244885378855,\n#             \"learning_rate\": 0.07254551525590439,\n#             \"min_data_in_leaf\": 14,\n#             \"num_leaves\": 31,\n#             \"ag_args\": {\"name_suffix\": \"_r145\", \"priority\": -40},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.45835595623790437,\n#             \"learning_rate\": 0.09533195017847339,\n#             \"min_data_in_leaf\": 7,\n#             \"num_leaves\": 231,\n#             \"ag_args\": {\"name_suffix\": \"_r41\", \"priority\": -44},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.6245777099925497,\n#             \"learning_rate\": 0.04711573688184715,\n#             \"min_data_in_leaf\": 56,\n#             \"num_leaves\": 89,\n#             \"ag_args\": {\"name_suffix\": \"_r130\", \"priority\": -52},\n#         },\n#         # { # Too slow for large datasets\n#         #     \"extra_trees\": True,\n#         #     \"feature_fraction\": 0.5179494170080321,\n#         #     \"learning_rate\": 0.015090113680567405,\n#         #     \"min_data_in_leaf\": 7,\n#         #     \"num_leaves\": 110,\n#         #     \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -62},\n#         # },\n#         # { # Too slow for large datasets\n#         #     \"extra_trees\": True,\n#         #     \"feature_fraction\": 0.4341088458599442,\n#         #     \"learning_rate\": 0.04034449862560467,\n#         #     \"min_data_in_leaf\": 33,\n#         #     \"num_leaves\": 16,\n#         #     \"ag_args\": {\"name_suffix\": \"_r94\", \"priority\": -65},\n#         # },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.9666234339903601,\n#             \"learning_rate\": 0.04582977995120822,\n#             \"min_data_in_leaf\": 4,\n#             \"num_leaves\": 127,\n#             \"ag_args\": {\"name_suffix\": \"_r55\", \"priority\": -68},\n#         },\n#         # { # Too slow\n#         #     \"extra_trees\": False,\n#         #     \"feature_fraction\": 0.9995667963533027,\n#         #     \"learning_rate\": 0.01434806540259691,\n#         #     \"min_data_in_leaf\": 37,\n#         #     \"num_leaves\": 183,\n#         #     \"ag_args\": {\"name_suffix\": \"_r174\", \"priority\": -88},\n#         # },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.7016257244614168,\n#             \"learning_rate\": 0.007922167829715967,\n#             \"min_data_in_leaf\": 7,\n#             \"num_leaves\": 132,\n#             \"ag_args\": {\"name_suffix\": \"_r149\", \"priority\": -91},\n#         },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.9046840778713597,\n#             \"learning_rate\": 0.07515257316211908,\n#             \"min_data_in_leaf\": 42,\n#             \"num_leaves\": 18,\n#             \"ag_args\": {\"name_suffix\": \"_r43\", \"priority\": -100},\n#         },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.4601361323873807,\n#             \"learning_rate\": 0.07856777698860955,\n#             \"min_data_in_leaf\": 12,\n#             \"num_leaves\": 198,\n#             \"ag_args\": {\"name_suffix\": \"_r42\", \"priority\": -105},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.7532437659821729,\n#             \"learning_rate\": 0.08944644189688526,\n#             \"min_data_in_leaf\": 39,\n#             \"num_leaves\": 53,\n#             \"ag_args\": {\"name_suffix\": \"_r153\", \"priority\": -118},\n#         },\n#         {\n#             \"extra_trees\": True,\n#             \"feature_fraction\": 0.43613528297756193,\n#             \"learning_rate\": 0.03685135839677242,\n#             \"min_data_in_leaf\": 57,\n#             \"num_leaves\": 27,\n#             \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -121},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.7579932437770318,\n#             \"learning_rate\": 0.052301563688720604,\n#             \"min_data_in_leaf\": 37,\n#             \"num_leaves\": 136,\n#             \"ag_args\": {\"name_suffix\": \"_r51\", \"priority\": -131},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.41239059967943725,\n#             \"learning_rate\": 0.04848901712678711,\n#             \"min_data_in_leaf\": 5,\n#             \"num_leaves\": 67,\n#             \"ag_args\": {\"name_suffix\": \"_r61\", \"priority\": -132},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.40585986135777,\n#             \"learning_rate\": 0.012590980616372347,\n#             \"min_data_in_leaf\": 32,\n#             \"num_leaves\": 22,\n#             \"ag_args\": {\"name_suffix\": \"_r106\", \"priority\": -139},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.9744705133953723,\n#             \"learning_rate\": 0.020546267996855768,\n#             \"min_data_in_leaf\": 60,\n#             \"num_leaves\": 99,\n#             \"ag_args\": {\"name_suffix\": \"_r66\", \"priority\": -163},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.6937293621346563,\n#             \"learning_rate\": 0.013803836586316339,\n#             \"min_data_in_leaf\": 38,\n#             \"num_leaves\": 16,\n#             \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -164},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.6090855934200983,\n#             \"learning_rate\": 0.04590490414627263,\n#             \"min_data_in_leaf\": 56,\n#             \"num_leaves\": 144,\n#             \"ag_args\": {\"name_suffix\": \"_r144\", \"priority\": -171},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.5730390983988963,\n#             \"learning_rate\": 0.010305352949119608,\n#             \"min_data_in_leaf\": 10,\n#             \"num_leaves\": 215,\n#             \"ag_args\": {\"name_suffix\": \"_r121\", \"priority\": -172},\n#         },\n#         {\n#             \"extra_trees\": False,\n#             \"feature_fraction\": 0.45118655387122203,\n#             \"learning_rate\": 0.009705399613761859,\n#             \"min_data_in_leaf\": 9,\n#             \"num_leaves\": 45,\n#             \"ag_args\": {\"name_suffix\": \"_r198\", \"priority\": -173},\n#         },\n#     ],\n#     \"CAT\": [\n#         {},\n#         {\n#             # Added manually.\n#             \"max_bin\": 4095,\n#             \"ag_args\": {\"priority\": -3, \"name_suffix\": \"Bin4095\"},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 2.1542798306067823,\n#             \"learning_rate\": 0.06864209415792857,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r177\", \"priority\": -1},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 2.7997999596449104,\n#             \"learning_rate\": 0.031375015734637225,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r9\", \"priority\": -5},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.559174625782161,\n#             \"learning_rate\": 0.04939557741379516,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r137\", \"priority\": -12},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.3274013177541373,\n#             \"learning_rate\": 0.017301189655111057,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r13\", \"priority\": -18},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 1.9842527399638579,\n#             \"learning_rate\": 0.03675129858005787,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -20},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 2.9106604692853995,\n#             \"learning_rate\": 0.05696733223175933,\n#             \"max_ctr_complexity\": 1,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r3\", \"priority\": -29},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 1.3584121369544215,\n#             \"learning_rate\": 0.03743901034980473,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r70\", \"priority\": -34},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.353268454214423,\n#             \"learning_rate\": 0.06028218319511302,\n#             \"max_ctr_complexity\": 1,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -38},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.609235810966101,\n#             \"learning_rate\": 0.05862063297323188,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r135\", \"priority\": -42},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 1.3976655774456335,\n#             \"learning_rate\": 0.0119117922642152,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r118\", \"priority\": -49},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.487142939021972,\n#             \"learning_rate\": 0.022543679111584062,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r124\", \"priority\": -53},\n#         },\n#         {\n#             \"depth\": 5,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 1.0457098345001241,\n#             \"learning_rate\": 0.050294288910022224,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r69\", \"priority\": -56},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 3.7631391046261182,\n#             \"learning_rate\": 0.09628094704738965,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r159\", \"priority\": -60},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 2.7018061518087038,\n#             \"learning_rate\": 0.07092851311746352,\n#             \"max_ctr_complexity\": 1,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r50\", \"priority\": -78},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.637071465711953,\n#             \"learning_rate\": 0.04387418552563314,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r198\", \"priority\": -80},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.835797074498082,\n#             \"learning_rate\": 0.03534026385152556,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r12\", \"priority\": -87},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 4.693224271556653,\n#             \"learning_rate\": 0.06007466728599504,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r125\", \"priority\": -92},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.901696537625293,\n#             \"learning_rate\": 0.05205851115876207,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r171\", \"priority\": -95},\n#         },\n#         {\n#             \"depth\": 5,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.7454481983750014,\n#             \"learning_rate\": 0.09328642499990342,\n#             \"max_ctr_complexity\": 1,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r163\", \"priority\": -98},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 1.3843936315758523,\n#             \"learning_rate\": 0.09319080384124982,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r16\", \"priority\": -104},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 1.640921865280573,\n#             \"learning_rate\": 0.036232951900213306,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r128\", \"priority\": -107},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 2.894432181094842,\n#             \"learning_rate\": 0.055078095725390575,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r5\", \"priority\": -110},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.218751234645676,\n#             \"learning_rate\": 0.011814970512055711,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r96\", \"priority\": -122},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 1.6761016245166451,\n#             \"learning_rate\": 0.06566144806528762,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r143\", \"priority\": -133},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 3.5217917474671645,\n#             \"learning_rate\": 0.0686190772732043,\n#             \"max_ctr_complexity\": 2,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r62\", \"priority\": -142},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 4.43335055453705,\n#             \"learning_rate\": 0.055406199833457785,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r180\", \"priority\": -149},\n#         },\n#         {\n#             \"depth\": 8,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.829803422635814,\n#             \"learning_rate\": 0.020546267996855768,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r52\", \"priority\": -154},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.522712492188319,\n#             \"learning_rate\": 0.08481607830570326,\n#             \"max_ctr_complexity\": 3,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r167\", \"priority\": -156},\n#         },\n#         {\n#             \"depth\": 5,\n#             \"grow_policy\": \"Depthwise\",\n#             \"l2_leaf_reg\": 1.2612168286071208,\n#             \"learning_rate\": 0.05223739392014652,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r94\", \"priority\": -162},\n#         },\n#         {\n#             \"depth\": 4,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 1.9515712854980345,\n#             \"learning_rate\": 0.08211271991437913,\n#             \"max_ctr_complexity\": 5,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r41\", \"priority\": -170},\n#         },\n#         {\n#             \"depth\": 5,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.299190822458588,\n#             \"learning_rate\": 0.037054953070982596,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r183\", \"priority\": -175},\n#         },\n#         {\n#             \"depth\": 6,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 4.3146276582229515,\n#             \"learning_rate\": 0.079869267564513,\n#             \"max_ctr_complexity\": 1,\n#             \"one_hot_max_size\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r97\", \"priority\": -176},\n#         },\n#         {\n#             \"depth\": 7,\n#             \"grow_policy\": \"SymmetricTree\",\n#             \"l2_leaf_reg\": 1.2565899853951374,\n#             \"learning_rate\": 0.039801276672212574,\n#             \"max_ctr_complexity\": 4,\n#             \"one_hot_max_size\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -186},\n#         },\n#     ],\n#     \"XGB\": [\n#         {},\n#         {\n#             # Added manually.\n#             \"max_bin\": 4095,\n#             \"ag_args\": {\"priority\": -2, \"name_suffix\": \"Bin4095\"},\n#         },\n#         {\n#             # Added manually.\n#             \"max_bin\": 16383,\n#             \"ag_args\": {\"priority\": -4, \"name_suffix\": \"Bin4095\"},\n#         },\n#         # {  # Too slow\n#         #     \"colsample_bytree\": 0.6917311125174739,\n#         #     \"enable_categorical\": False,\n#         #     \"learning_rate\": 0.018063876087523967,\n#         #     \"max_depth\": 10,\n#         #     \"min_child_weight\": 0.6028633586934382,\n#         #     \"ag_args\": {\"name_suffix\": \"_r33\", \"priority\": -9},\n#         # },\n#         {\n#             \"colsample_bytree\": 0.6326947454697227,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.07792091886639502,\n#             \"max_depth\": 6,\n#             \"min_child_weight\": 1.0759464955561793,\n#             \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": -25},\n#         },\n#         {\n#             \"colsample_bytree\": 0.8387059147359721,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.010427977953050884,\n#             \"max_depth\": 7,\n#             \"min_child_weight\": 1.2521345209879338,\n#             \"ag_args\": {\"name_suffix\": \"_r78\", \"priority\": -35},\n#         },\n#         {\n#             \"colsample_bytree\": 0.922117480776512,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.09278141584600721,\n#             \"max_depth\": 10,\n#             \"min_child_weight\": 1.3428548878354571,\n#             \"ag_args\": {\"name_suffix\": \"_r193\", \"priority\": -47},\n#         },\n#         {\n#             \"colsample_bytree\": 0.7452294043087835,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.038404229910104046,\n#             \"max_depth\": 7,\n#             \"min_child_weight\": 0.5564183327139662,\n#             \"ag_args\": {\"name_suffix\": \"_r49\", \"priority\": -51},\n#         },\n#         # { # Too slow\n#         #     \"colsample_bytree\": 0.7506621909633511,\n#         #     \"enable_categorical\": False,\n#         #     \"learning_rate\": 0.009974712407899168,\n#         #     \"max_depth\": 4,\n#         #     \"min_child_weight\": 0.9238550485581797,\n#         #     \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -55},\n#         # },\n#         {\n#             \"colsample_bytree\": 0.5078030322234142,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.03502844787947471,\n#             \"max_depth\": 9,\n#             \"min_child_weight\": 0.7519409882460929,\n#             \"ag_args\": {\"name_suffix\": \"_r102\", \"priority\": -69},\n#         },\n#         {\n#             \"colsample_bytree\": 0.975937238416368,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.06634196266155237,\n#             \"max_depth\": 5,\n#             \"min_child_weight\": 1.4088437184127383,\n#             \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -90},\n#         },\n#         {\n#             \"colsample_bytree\": 0.8551684144871067,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.01984858936348865,\n#             \"max_depth\": 10,\n#             \"min_child_weight\": 1.0305372145627818,\n#             \"ag_args\": {\"name_suffix\": \"_r130\", \"priority\": -116},\n#         },\n#         {\n#             \"colsample_bytree\": 0.7317254886922415,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.014069637236426217,\n#             \"max_depth\": 4,\n#             \"min_child_weight\": 1.108252865854013,\n#             \"ag_args\": {\"name_suffix\": \"_r44\", \"priority\": -117},\n#         },\n#         {\n#             \"colsample_bytree\": 0.6363281472900566,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.05358312362730344,\n#             \"max_depth\": 10,\n#             \"min_child_weight\": 1.0680445610939322,\n#             \"ag_args\": {\"name_suffix\": \"_r3\", \"priority\": -119},\n#         },\n#         {\n#             \"colsample_bytree\": 0.5019301757551305,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.028332242590352767,\n#             \"max_depth\": 5,\n#             \"min_child_weight\": 0.5813695988533053,\n#             \"ag_args\": {\"name_suffix\": \"_r184\", \"priority\": -124},\n#         },\n#         {\n#             \"colsample_bytree\": 0.6167101277340481,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.01585533568837928,\n#             \"max_depth\": 6,\n#             \"min_child_weight\": 1.230709099127476,\n#             \"ag_args\": {\"name_suffix\": \"_r113\", \"priority\": -127},\n#         },\n#         {\n#             \"colsample_bytree\": 0.5619099914247208,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.06672278511715538,\n#             \"max_depth\": 9,\n#             \"min_child_weight\": 1.0691007386145932,\n#             \"ag_args\": {\"name_suffix\": \"_r39\", \"priority\": -146},\n#         },\n#         {\n#             \"colsample_bytree\": 0.8576039170061793,\n#             \"enable_categorical\": False,\n#             \"learning_rate\": 0.012446976122406065,\n#             \"max_depth\": 8,\n#             \"min_child_weight\": 0.5030995410843868,\n#             \"ag_args\": {\"name_suffix\": \"_r147\", \"priority\": -184},\n#         },\n#         {\n#             \"colsample_bytree\": 0.9679160401083943,\n#             \"enable_categorical\": True,\n#             \"learning_rate\": 0.07231523643035528,\n#             \"max_depth\": 9,\n#             \"min_child_weight\": 0.7115338977263024,\n#             \"ag_args\": {\"name_suffix\": \"_r159\", \"priority\": -185},\n#         },\n#     ],\n#     \"FASTAI\": [\n#         {},\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.5411770367537934,\n#             \"epochs\": 43,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.01519848858318159,\n#             \"ps\": 0.23782946566604385,\n#             \"ag_args\": {\"name_suffix\": \"_r191\", \"priority\": -4},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.3939481870086508,\n#             \"epochs\": 50,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.05474993731798857,\n#             \"ps\": 0.312476090838242,\n#             \"ag_args\": {\"name_suffix\": \"_r120\", \"priority\": -10},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.44339037504795686,\n#             \"epochs\": 31,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.008615195908919904,\n#             \"ps\": 0.19220253419114286,\n#             \"ag_args\": {\"name_suffix\": \"_r145\", \"priority\": -14},\n#         },\n#         {\n#             \"bs\": 2048,\n#             \"emb_drop\": 0.4149912327575128,\n#             \"epochs\": 20,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.04711168148637163,\n#             \"ps\": 0.5930762171488877,\n#             \"ag_args\": {\"name_suffix\": \"_r1\", \"priority\": -17},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.026897798530914306,\n#             \"epochs\": 31,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.08045277634470181,\n#             \"ps\": 0.4569532219038436,\n#             \"ag_args\": {\"name_suffix\": \"_r11\", \"priority\": -23},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.5074958658302495,\n#             \"epochs\": 42,\n#             \"layers\": [200, 100, 50],\n#             \"lr\": 0.026342427824862867,\n#             \"ps\": 0.34814978753283593,\n#             \"ag_args\": {\"name_suffix\": \"_r187\", \"priority\": -33},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.0052278224214431955,\n#             \"epochs\": 39,\n#             \"layers\": [200, 100, 50],\n#             \"lr\": 0.019675935745941037,\n#             \"ps\": 0.49360327844735585,\n#             \"ag_args\": {\"name_suffix\": \"_r165\", \"priority\": -37},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.08669109226243704,\n#             \"epochs\": 45,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.0041554361714983635,\n#             \"ps\": 0.2669780074016213,\n#             \"ag_args\": {\"name_suffix\": \"_r138\", \"priority\": -39},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.5325204955732282,\n#             \"epochs\": 36,\n#             \"layers\": [200],\n#             \"lr\": 0.021387722847117794,\n#             \"ps\": 0.41116271561673834,\n#             \"ag_args\": {\"name_suffix\": \"_r179\", \"priority\": -43},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.304332450990704,\n#             \"epochs\": 37,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.022114884908502042,\n#             \"ps\": 0.3661044725262556,\n#             \"ag_args\": {\"name_suffix\": \"_r123\", \"priority\": -45},\n#         },\n#         {\n#             \"bs\": 2048,\n#             \"emb_drop\": 0.006251885504130949,\n#             \"epochs\": 47,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.01329622020483052,\n#             \"ps\": 0.2677080696008348,\n#             \"ag_args\": {\"name_suffix\": \"_r134\", \"priority\": -58},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.09607594536689695,\n#             \"epochs\": 36,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.05050764650138042,\n#             \"ps\": 0.12558704891144218,\n#             \"ag_args\": {\"name_suffix\": \"_r131\", \"priority\": -63},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.05604276533830355,\n#             \"epochs\": 32,\n#             \"layers\": [400],\n#             \"lr\": 0.027320709383189166,\n#             \"ps\": 0.022591301744255762,\n#             \"ag_args\": {\"name_suffix\": \"_r172\", \"priority\": -70},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.6656668277387758,\n#             \"epochs\": 32,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.019326244622675428,\n#             \"ps\": 0.04084945128641206,\n#             \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -73},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.6539497985473556,\n#             \"epochs\": 27,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.008540379133171428,\n#             \"ps\": 0.4129369834481997,\n#             \"ag_args\": {\"name_suffix\": \"_r39\", \"priority\": -77},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.4329361816589235,\n#             \"epochs\": 50,\n#             \"layers\": [400],\n#             \"lr\": 0.09501311551121323,\n#             \"ps\": 0.2863378667611431,\n#             \"ag_args\": {\"name_suffix\": \"_r88\", \"priority\": -81},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.48799913177984827,\n#             \"epochs\": 37,\n#             \"layers\": [200, 100, 50],\n#             \"lr\": 0.09585563123235417,\n#             \"ps\": 0.17459402873951585,\n#             \"ag_args\": {\"name_suffix\": \"_r94\", \"priority\": -84},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.03949283289977636,\n#             \"epochs\": 37,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.05323385634320209,\n#             \"ps\": 0.6834830706151297,\n#             \"ag_args\": {\"name_suffix\": \"_r42\", \"priority\": -93},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.013991365786131115,\n#             \"epochs\": 30,\n#             \"layers\": [200, 100],\n#             \"lr\": 0.08974875871045038,\n#             \"ps\": 0.25161112477852504,\n#             \"ag_args\": {\"name_suffix\": \"_r16\", \"priority\": -96},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.6036229827851316,\n#             \"epochs\": 26,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.0019169423883858754,\n#             \"ps\": 0.3122948588614413,\n#             \"ag_args\": {\"name_suffix\": \"_r106\", \"priority\": -102},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.3132611095327173,\n#             \"epochs\": 22,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.0037111611381388542,\n#             \"ps\": 0.07002243450258397,\n#             \"ag_args\": {\"name_suffix\": \"_r101\", \"priority\": -108},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.31956392388385874,\n#             \"epochs\": 25,\n#             \"layers\": [200, 100],\n#             \"lr\": 0.08552736732040143,\n#             \"ps\": 0.0934076022219228,\n#             \"ag_args\": {\"name_suffix\": \"_r127\", \"priority\": -113},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.38627138139328115,\n#             \"epochs\": 42,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.08483598191521528,\n#             \"ps\": 0.22469806896144823,\n#             \"ag_args\": {\"name_suffix\": \"_r29\", \"priority\": -115},\n#         },\n#         {\n#             \"bs\": 2048,\n#             \"emb_drop\": 0.5482641068196692,\n#             \"epochs\": 47,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.0017967245032803941,\n#             \"ps\": 0.46375320007287313,\n#             \"ag_args\": {\"name_suffix\": \"_r90\", \"priority\": -123},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.3358277091379548,\n#             \"epochs\": 39,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.08407418785434553,\n#             \"ps\": 0.06997096091499505,\n#             \"ag_args\": {\"name_suffix\": \"_r136\", \"priority\": -125},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.1714226355940696,\n#             \"epochs\": 45,\n#             \"layers\": [200],\n#             \"lr\": 0.0012051775395048954,\n#             \"ps\": 0.6459296271577557,\n#             \"ag_args\": {\"name_suffix\": \"_r159\", \"priority\": -130},\n#         },\n#         {\n#             \"bs\": 2048,\n#             \"emb_drop\": 0.000382175427896958,\n#             \"epochs\": 46,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.002028652260418399,\n#             \"ps\": 0.24384972715881284,\n#             \"ag_args\": {\"name_suffix\": \"_r169\", \"priority\": -134},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.4599138419358,\n#             \"epochs\": 47,\n#             \"layers\": [200, 100],\n#             \"lr\": 0.03888383281136287,\n#             \"ps\": 0.28193673177122863,\n#             \"ag_args\": {\"name_suffix\": \"_r128\", \"priority\": -143},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.22771721361129746,\n#             \"epochs\": 38,\n#             \"layers\": [400],\n#             \"lr\": 0.0005383511954451698,\n#             \"ps\": 0.3734259772256502,\n#             \"ag_args\": {\"name_suffix\": \"_r65\", \"priority\": -144},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.1978897556618756,\n#             \"epochs\": 49,\n#             \"layers\": [400, 200, 100],\n#             \"lr\": 0.021551443035084644,\n#             \"ps\": 0.005518872455908264,\n#             \"ag_args\": {\"name_suffix\": \"_r109\", \"priority\": -152},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.3794906175059568,\n#             \"epochs\": 49,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.01841936985537725,\n#             \"ps\": 0.2009834831789172,\n#             \"ag_args\": {\"name_suffix\": \"_r118\", \"priority\": -160},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.24892891649496893,\n#             \"epochs\": 48,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.02884073803446048,\n#             \"ps\": 0.5240645338953831,\n#             \"ag_args\": {\"name_suffix\": \"_r53\", \"priority\": -161},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.2684235575244635,\n#             \"epochs\": 29,\n#             \"layers\": [200, 100],\n#             \"lr\": 0.052518805993807795,\n#             \"ps\": 0.07200435108540675,\n#             \"ag_args\": {\"name_suffix\": \"_r28\", \"priority\": -166},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.0463046234818683,\n#             \"epochs\": 36,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.04669680619450405,\n#             \"ps\": 0.11348418249618325,\n#             \"ag_args\": {\"name_suffix\": \"_r146\", \"priority\": -174},\n#         },\n#         {\n#             \"bs\": 256,\n#             \"emb_drop\": 0.6272268712563539,\n#             \"epochs\": 44,\n#             \"layers\": [800, 400],\n#             \"lr\": 0.05629418126052196,\n#             \"ps\": 0.4760388986243886,\n#             \"ag_args\": {\"name_suffix\": \"_r23\", \"priority\": -182},\n#         },\n#         {\n#             \"bs\": 1024,\n#             \"emb_drop\": 0.4530218981312888,\n#             \"epochs\": 42,\n#             \"layers\": [200],\n#             \"lr\": 0.002032928260082698,\n#             \"ps\": 0.2784745269310403,\n#             \"ag_args\": {\"name_suffix\": \"_r20\", \"priority\": -188},\n#         },\n#         {\n#             \"bs\": 512,\n#             \"emb_drop\": 0.4875242012567504,\n#             \"epochs\": 36,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.0006167746422948765,\n#             \"ps\": 0.041961008774701276,\n#             \"ag_args\": {\"name_suffix\": \"_r122\", \"priority\": -189},\n#         },\n#         {\n#             \"bs\": 2048,\n#             \"emb_drop\": 0.6343202884164582,\n#             \"epochs\": 21,\n#             \"layers\": [400, 200],\n#             \"lr\": 0.08479209380262258,\n#             \"ps\": 0.48362560779595565,\n#             \"ag_args\": {\"name_suffix\": \"_r111\", \"priority\": -190},\n#         },\n#         {\n#             \"bs\": 128,\n#             \"emb_drop\": 0.5990623396748277,\n#             \"epochs\": 33,\n#             \"layers\": [200, 100, 50],\n#             \"lr\": 0.023916641022656154,\n#             \"ps\": 0.12014077408300836,\n#             \"ag_args\": {\"name_suffix\": \"_r35\", \"priority\": -193},\n#         },\n#     ],\n#     \"RF\": [\n#         {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n#         {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n#         {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]}},\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 18392,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r42\", \"priority\": -11},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 25177,\n#             \"min_samples_leaf\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r125\", \"priority\": -31},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 45899,\n#             \"min_samples_leaf\": 20,\n#             \"ag_args\": {\"name_suffix\": \"_r50\", \"priority\": -46},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 18704,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r144\", \"priority\": -57},\n#         },\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 48569,\n#             \"min_samples_leaf\": 40,\n#             \"ag_args\": {\"name_suffix\": \"_r200\", \"priority\": -66},\n#         },\n#         {\n#             \"max_features\": \"sqrt\",\n#             \"max_leaf_nodes\": 11995,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r148\", \"priority\": -74},\n#         },\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 36230,\n#             \"min_samples_leaf\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r15\", \"priority\": -79},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 26368,\n#             \"min_samples_leaf\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r19\", \"priority\": -106},\n#         },\n#         {\n#             \"max_features\": 0.5,\n#             \"max_leaf_nodes\": 22329,\n#             \"min_samples_leaf\": 20,\n#             \"ag_args\": {\"name_suffix\": \"_r111\", \"priority\": -126},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 42420,\n#             \"min_samples_leaf\": 10,\n#             \"ag_args\": {\"name_suffix\": \"_r83\", \"priority\": -140},\n#         },\n#         {\n#             \"max_features\": 0.5,\n#             \"max_leaf_nodes\": 43008,\n#             \"min_samples_leaf\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r95\", \"priority\": -141},\n#         },\n#         {\n#             \"max_features\": 0.5,\n#             \"max_leaf_nodes\": 19705,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r134\", \"priority\": -150},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 34081,\n#             \"min_samples_leaf\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r194\", \"priority\": -178},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 12845,\n#             \"min_samples_leaf\": 4,\n#             \"ag_args\": {\"name_suffix\": \"_r172\", \"priority\": -179},\n#         },\n#         {\n#             \"max_features\": \"log2\",\n#             \"max_leaf_nodes\": 42644,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r166\", \"priority\": -180},\n#         },\n#         {\n#             \"max_features\": \"sqrt\",\n#             \"max_leaf_nodes\": 30080,\n#             \"min_samples_leaf\": 80,\n#             \"ag_args\": {\"name_suffix\": \"_r113\", \"priority\": -187},\n#         },\n#     ],\n#     \"XT\": [\n#         {\"criterion\": \"gini\", \"ag_args\": {\"name_suffix\": \"Gini\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n#         {\"criterion\": \"entropy\", \"ag_args\": {\"name_suffix\": \"Entr\", \"problem_types\": [\"binary\", \"multiclass\"]}},\n#         {\"criterion\": \"squared_error\", \"ag_args\": {\"name_suffix\": \"MSE\", \"problem_types\": [\"regression\", \"quantile\"]}},\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 18729,\n#             \"min_samples_leaf\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r137\", \"priority\": -16},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 15825,\n#             \"min_samples_leaf\": 3,\n#             \"ag_args\": {\"name_suffix\": \"_r196\", \"priority\": -61},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 48136,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r16\", \"priority\": -72},\n#         },\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 46988,\n#             \"min_samples_leaf\": 40,\n#             \"ag_args\": {\"name_suffix\": \"_r102\", \"priority\": -75},\n#         },\n#         {\n#             \"max_features\": \"log2\",\n#             \"max_leaf_nodes\": 33436,\n#             \"min_samples_leaf\": 2,\n#             \"ag_args\": {\"name_suffix\": \"_r119\", \"priority\": -86},\n#         },\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 12012,\n#             \"min_samples_leaf\": 5,\n#             \"ag_args\": {\"name_suffix\": \"_r25\", \"priority\": -89},\n#         },\n#         {\n#             \"max_features\": 0.5,\n#             \"max_leaf_nodes\": 30392,\n#             \"min_samples_leaf\": 4,\n#             \"ag_args\": {\"name_suffix\": \"_r31\", \"priority\": -99},\n#         },\n#         {\n#             \"max_features\": 0.75,\n#             \"max_leaf_nodes\": 37308,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r195\", \"priority\": -128},\n#         },\n#         {\n#             \"max_features\": \"log2\",\n#             \"max_leaf_nodes\": 24441,\n#             \"min_samples_leaf\": 20,\n#             \"ag_args\": {\"name_suffix\": \"_r123\", \"priority\": -155},\n#         },\n#         {\n#             \"max_features\": 1.0,\n#             \"max_leaf_nodes\": 31951,\n#             \"min_samples_leaf\": 4,\n#             \"ag_args\": {\"name_suffix\": \"_r142\", \"priority\": -169},\n#         },\n#         {\n#             \"max_features\": 0.5,\n#             \"max_leaf_nodes\": 36718,\n#             \"min_samples_leaf\": 1,\n#             \"ag_args\": {\"name_suffix\": \"_r158\", \"priority\": -177},\n#         },\n#     ],\n#     \"KNN\": [\n#         {\"weights\": \"uniform\", \"ag_args\": {\"name_suffix\": \"Unif\"}},\n#         {\"weights\": \"distance\", \"ag_args\": {\"name_suffix\": \"Dist\"}},\n#         {\"weights\": \"distance\", \"p\": 2, \"n_neighbors\": 50, \"ag_args\": {\"name_suffix\": \"_r32\", \"priority\": -56}},\n#         {\"weights\": \"distance\", \"p\": 1, \"n_neighbors\": 5, \"ag_args\": {\"name_suffix\": \"_r16\", \"priority\": -82}},\n#         {\"weights\": \"uniform\", \"p\": 1, \"n_neighbors\": 3, \"ag_args\": {\"name_suffix\": \"_r23\", \"priority\": -136}},\n#         {\"weights\": \"uniform\", \"p\": 1, \"n_neighbors\": 30, \"ag_args\": {\"name_suffix\": \"_r45\", \"priority\": -151}},\n#     ],\n#     \"LR\": [\n#         {\n#             \"C\": 978.6204803985407,\n#             \"penalty\": \"L2\",\n#             \"proc.impute_strategy\": \"mean\",\n#             \"proc.skew_threshold\": None,\n#             \"ag_args\": {\"name_suffix\": \"_r9\", \"priority\": 95},\n#         },\n#         {\n#             \"C\": 958.9533736976579,\n#             \"penalty\": \"L1\",\n#             \"proc.impute_strategy\": \"mean\",\n#             \"proc.skew_threshold\": 0.99,\n#             \"ag_args\": {\"name_suffix\": \"_r22\", \"priority\": 15},\n#         },\n#         {\n#             \"C\": 55.80912223223615,\n#             \"penalty\": \"L1\",\n#             \"proc.impute_strategy\": \"mean\",\n#             \"proc.skew_threshold\": None,\n#             \"ag_args\": {\"name_suffix\": \"_r30\", \"priority\": -21},\n#         },\n#         {\n#             \"C\": 647.2094227735367,\n#             \"penalty\": \"L2\",\n#             \"proc.impute_strategy\": \"median\",\n#             \"proc.skew_threshold\": 0.999,\n#             \"ag_args\": {\"name_suffix\": \"_r38\", \"priority\": -36},\n#         },\n#         {\n#             \"C\": 857.9598230609945,\n#             \"penalty\": \"L1\",\n#             \"proc.impute_strategy\": \"mean\",\n#             \"proc.skew_threshold\": None,\n#             \"ag_args\": {\"name_suffix\": \"_r2\", \"priority\": -67},\n#         },\n#         {\n#             \"C\": 791.7458655788563,\n#             \"penalty\": \"L1\",\n#             \"proc.impute_strategy\": \"median\",\n#             \"proc.skew_threshold\": 0.99,\n#             \"ag_args\": {\"name_suffix\": \"_r5\", \"priority\": -159},\n#         },\n#     ],\n# }","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# allowed_models = [\n#     # \"LR\",\n#     \"FASTAI\",\n#     \"NN_TORCH\",\n#     \"GBM\",\n#     \"CAT\",\n#     \"XGB\",\n#     \"RF\",\n#     \"XT\",\n# ]\n\n# for k in list(zeroshot2024.keys()):\n#     if k not in allowed_models:\n#         del zeroshot2024[k]","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Autogluon","metadata":{}},{"cell_type":"code","source":"# hyperparameter_tune_kwargs = {  \n#     'num_trials': 40,\n#     'scheduler' : 'local',\n#     'searcher': 'auto',\n# }\n\npredictor = TabularPredictor(\n    label=label,\n    eval_metric ='rmse',\n    problem_type=\"regression\",\n    verbosity = 2,\n)\n\npredictor.fit(\n    time_limit=int(60 * 60 * 11),\n    train_data=train,\n    presets=\"best_quality\",\n    # hyperparameters = 'light',\n    # included_model_types=['GBM','XGB','CAT'],\n    # hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,\n    # ag_args_fit={'num_gpus': 1},\n    # dynamic_stacking=False,\n    # hyperparameters=zeroshot2024,\n    num_bag_folds=5,\n    num_bag_sets=1,\n    num_stack_levels=1,\n)\n\nresults = predictor.fit_summary()","metadata":{"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"predictor = TabularPredictor.load('/kaggle/input/pss4e12/AutogluonModels/ag-20241207_195649')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:00:07.402816Z","iopub.execute_input":"2024-12-09T08:00:07.403261Z","iopub.status.idle":"2024-12-09T08:00:07.757093Z","shell.execute_reply.started":"2024-12-09T08:00:07.403226Z","shell.execute_reply":"2024-12-09T08:00:07.754671Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Leaderboard","metadata":{}},{"cell_type":"code","source":"predictor.leaderboard()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:00:10.33237Z","iopub.execute_input":"2024-12-09T08:00:10.33276Z","iopub.status.idle":"2024-12-09T08:00:10.368688Z","shell.execute_reply.started":"2024-12-09T08:00:10.332727Z","shell.execute_reply":"2024-12-09T08:00:10.367522Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"df = predictor.predict(test).to_frame(name=label)\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:01:43.513497Z","iopub.execute_input":"2024-12-09T08:01:43.51391Z","iopub.status.idle":"2024-12-09T08:38:10.866668Z","shell.execute_reply.started":"2024-12-09T08:01:43.513875Z","shell.execute_reply":"2024-12-09T08:38:10.865317Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['Premium Amount'] = np.expm1(df['Premium Amount']) \ndf['Premium Amount'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:41:01.60726Z","iopub.execute_input":"2024-12-09T08:41:01.609114Z","iopub.status.idle":"2024-12-09T08:41:01.670053Z","shell.execute_reply.started":"2024-12-09T08:41:01.60906Z","shell.execute_reply":"2024-12-09T08:41:01.668808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\npred.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:41:04.707619Z","iopub.execute_input":"2024-12-09T08:41:04.708155Z","iopub.status.idle":"2024-12-09T08:41:05.145239Z","shell.execute_reply.started":"2024-12-09T08:41:04.708107Z","shell.execute_reply":"2024-12-09T08:41:05.143815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred['Premium Amount']= df[label]\npred.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:41:09.671277Z","iopub.execute_input":"2024-12-09T08:41:09.671639Z","iopub.status.idle":"2024-12-09T08:41:09.683737Z","shell.execute_reply.started":"2024-12-09T08:41:09.67161Z","shell.execute_reply":"2024-12-09T08:41:09.682732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pred.to_csv('fe_encat_autog_best.csv',index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Blending","metadata":{}},{"cell_type":"code","source":"# rid_train_h2o = pd.read_csv('/kaggle/input/rid-train-h2o/submission.csv') # 1.02922\nregression_ESB = pd.read_csv('/kaggle/input/regression-with-an-insurance-ensemble/submission.csv') # 1.0289\nLGBR_STACK_1 = pd.read_csv('/kaggle/input/insurance-competition-database/LGBR_STACK_1.03088.csv') # 1.02866","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:45:40.751967Z","iopub.execute_input":"2024-12-09T08:45:40.753223Z","iopub.status.idle":"2024-12-09T08:45:41.634428Z","shell.execute_reply.started":"2024-12-09T08:45:40.753166Z","shell.execute_reply":"2024-12-09T08:45:41.63334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a copy for blending\nblended = regression_ESB.copy()\n\n# more stack\nblended['Premium Amount'] = (\n    (0.31) * regression_ESB['Premium Amount'] +\n    (0.53) * LGBR_STACK_1['Premium Amount'] +\n    (0.16) * pred['Premium Amount']\n)\n\n# Save the blended results\nblended.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T08:46:19.724366Z","iopub.execute_input":"2024-12-09T08:46:19.724729Z","iopub.status.idle":"2024-12-09T08:46:21.409324Z","shell.execute_reply.started":"2024-12-09T08:46:19.724699Z","shell.execute_reply":"2024-12-09T08:46:21.408223Z"}},"outputs":[],"execution_count":null}]}