{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch, numpy as np, pandas as pd, matplotlib.pyplot as plt\nimport xgboost as xgb, lightgbm as lgb\nfrom pathlib import Path\nfrom fastai.tabular.all import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T17:01:06.001398Z","iopub.execute_input":"2024-12-31T17:01:06.001641Z","iopub.status.idle":"2024-12-31T17:01:13.245846Z","shell.execute_reply.started":"2024-12-31T17:01:06.001618Z","shell.execute_reply":"2024-12-31T17:01:13.244916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:46.394089Z","iopub.execute_input":"2024-12-30T17:38:46.394496Z","iopub.status.idle":"2024-12-30T17:38:46.398072Z","shell.execute_reply.started":"2024-12-30T17:38:46.394476Z","shell.execute_reply":"2024-12-30T17:38:46.397231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ndef clear():\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:46.399821Z","iopub.execute_input":"2024-12-30T17:38:46.400097Z","iopub.status.idle":"2024-12-30T17:38:46.411975Z","shell.execute_reply.started":"2024-12-30T17:38:46.400077Z","shell.execute_reply":"2024-12-30T17:38:46.411085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = Path('/kaggle/input/playground-series-s4e12')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:46.413143Z","iopub.execute_input":"2024-12-30T17:38:46.413423Z","iopub.status.idle":"2024-12-30T17:38:46.422076Z","shell.execute_reply.started":"2024-12-30T17:38:46.413394Z","shell.execute_reply":"2024-12-30T17:38:46.421354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:46.422859Z","iopub.execute_input":"2024-12-30T17:38:46.423135Z","iopub.status.idle":"2024-12-30T17:38:51.236864Z","shell.execute_reply.started":"2024-12-30T17:38:46.423107Z","shell.execute_reply":"2024-12-30T17:38:51.236185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def proc_data(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Month'] = df['Policy Start Date'].dt.month.astype(float)\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Week']  = df['Policy Start Date'].dt.isocalendar().week\n    df['Weekday'] = df['Policy Start Date'].dt.weekday.astype(float)\n    df['Income-Age Ratio'] = df['Annual Income'] / df['Age']\n    for col in df:\n        if(df[col].isna().sum() > 0): df[col + '_na'] = df[col].isna().astype(float)\n    df = df.fillna(df.mode().iloc[0])\n    conts,cats = cont_cat_split(df)\n    for col in cats:\n        #df[col] = pd.Categorical(df[col])\n        #df[col] = df[col].cat.codes\n        df[col] = df[col].astype('category')\n    indeps = df.drop(['id', 'Policy Start Date'], axis=1)\n    deps = None\n    if 'Premium Amount' in df.columns:\n        indeps = indeps.drop('Premium Amount', axis=1)\n        #deps = np.log1p(df['Premium Amount'])\n        deps = df['Premium Amount']\n    return indeps,deps","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:55.051933Z","iopub.execute_input":"2024-12-30T17:38:55.052216Z","iopub.status.idle":"2024-12-30T17:38:55.058172Z","shell.execute_reply.started":"2024-12-30T17:38:55.052192Z","shell.execute_reply":"2024-12-30T17:38:55.057326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"indeps,deps = proc_data(df)\nindeps.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:38:56.851718Z","iopub.execute_input":"2024-12-30T17:38:56.851989Z","iopub.status.idle":"2024-12-30T17:39:04.650608Z","shell.execute_reply.started":"2024-12-30T17:38:56.851969Z","shell.execute_reply":"2024-12-30T17:39:04.649737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.data.transforms import RandomSplitter\ntrain_split,valid_split = RandomSplitter(valid_pct=0.2)(df)\ntrain_indeps,train_deps = indeps.iloc[train_split],deps.iloc[train_split]\nvalid_indeps,valid_deps = indeps.iloc[valid_split],deps.iloc[valid_split]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:39:06.785222Z","iopub.execute_input":"2024-12-30T17:39:06.785529Z","iopub.status.idle":"2024-12-30T17:39:07.472121Z","shell.execute_reply.started":"2024-12-30T17:39:06.785494Z","shell.execute_reply":"2024-12-30T17:39:07.471212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error\ndef rmsle(preds, targs):\n    preds = np.maximum(0, preds)\n    targs = np.maximum(0, targs)\n    return np.sqrt(mean_squared_log_error(targs, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:39:17.19852Z","iopub.execute_input":"2024-12-30T17:39:17.198847Z","iopub.status.idle":"2024-12-30T17:39:17.203359Z","shell.execute_reply.started":"2024-12-30T17:39:17.198818Z","shell.execute_reply":"2024-12-30T17:39:17.202406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def graph_params(losses, x_label='', log=False):\n    x, y = losses.keys(), losses.values()\n    plt.xlabel(x_label)\n    if log: plt.xscale('log')\n    plt.ylabel('RMSLE')\n    plt.plot(x, y)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:28:47.185012Z","iopub.execute_input":"2024-12-30T05:28:47.185306Z","iopub.status.idle":"2024-12-30T05:28:47.189717Z","shell.execute_reply.started":"2024-12-30T05:28:47.185284Z","shell.execute_reply":"2024-12-30T05:28:47.188967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(arch, optimal_params):\n    if(arch == 'LGB'): return lgb.LGBMRegressor(**optimal_params, n_jobs=-1, verbose=-1)\n    if(arch == 'XGB'): return xgb.XGBRFRegressor(**optimal_params, n_jobs=-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:44:20.908603Z","iopub.execute_input":"2024-12-30T17:44:20.90894Z","iopub.status.idle":"2024-12-30T17:44:20.913213Z","shell.execute_reply.started":"2024-12-30T17:44:20.908915Z","shell.execute_reply":"2024-12-30T17:44:20.912211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def optimize_params(param_ranges, arch, optimal_params={}, graph_settings=None, print_log=True, print_graphs=True):\n    if graph_settings == None: print_graphs = False\n    for param,vals in param_ranges.items():\n        losses = {}\n        for val in vals:\n            start_time = time.time()\n            optimal_params[param] = val\n            model = get_model(arch=arch, optimal_params=optimal_params)\n            model.fit(train_indeps, train_deps)\n            preds = model.predict(valid_indeps)\n            losses[val] = rmsle(valid_deps, preds)\n            end_time = time.time()\n            print(f'tested {param}: {val:<10.2e} rmsle: {losses[val]:<8.4f} elapsed time: {(end_time - start_time):.2f}s')\n        optimal_params[param] = min(losses, key=losses.get)\n        print(f'optimal value for {param} is {optimal_params[param]:.2f}')\n        if print_graphs: graph_params(losses, **graph_settings[param])\n    return optimal_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:28:49.233372Z","iopub.execute_input":"2024-12-30T05:28:49.233684Z","iopub.status.idle":"2024-12-30T05:28:49.239292Z","shell.execute_reply.started":"2024-12-30T05:28:49.233657Z","shell.execute_reply":"2024-12-30T05:28:49.238411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def random_search(param_ranges, arch, num_searches, optimal_params={}, graph_settings=None, print_log=True):\n    min_loss = sys.maxsize\n    optimal_params = {}\n    for i in range(num_searches):\n        start = time.time()\n        random_vals = {}\n        for param,vals in param_ranges.items():\n            random_vals[param] = np.random.choice(vals)\n        model = get_model(arch=arch, optimal_params=random_vals)\n        model.fit(train_indeps, train_deps)\n        preds = model.predict(valid_indeps)\n        loss = rmsle(valid_deps, preds)\n        end = time.time()\n        print(f'search number {i:<4} elapsed time: {(end - start):.2f}s')\n        if loss < min_loss:\n            min_loss = loss\n            optimal_params = random_vals\n            print(f'new minimum loss: {loss:.4f}')\n    return optimal_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:28:49.459745Z","iopub.execute_input":"2024-12-30T05:28:49.46003Z","iopub.status.idle":"2024-12-30T05:28:49.465392Z","shell.execute_reply.started":"2024-12-30T05:28:49.460007Z","shell.execute_reply":"2024-12-30T05:28:49.464567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"param_ranges = {\n    'colsample_bytree': np.linspace(0.6, 1, num=10),\n    'learning_rate': np.logspace(-4, 0.1, num=30),\n    'max_bins': np.logspace(1, 5, num=10, dtype=int),\n    'max_depth': np.linspace(1, 20, num=20, dtype=int),\n    'min_data_in_leaf': np.linspace(5, 50, num=10, dtype=int),\n    'min_split_gain': np.linspace(0, 1, num=10),\n    'num_leaves': np.linspace(2, 40, num=20, dtype=int), # 2^max_depth\n    'n_estimators': np.linspace(10, 200, num=10, dtype=int),\n    'reg_alpha': np.logspace(-3, 3, num=10),\n    'reg_lambda': np.logspace(-3, 3, num=10),\n    'subsample': np.linspace(0.6, 1, num=10)\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:00:07.565302Z","iopub.execute_input":"2024-12-27T20:00:07.565604Z","iopub.status.idle":"2024-12-27T20:00:07.579456Z","shell.execute_reply.started":"2024-12-27T20:00:07.565576Z","shell.execute_reply":"2024-12-27T20:00:07.578606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_search(param_ranges, arch='LGB', num_searches=10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimal_params = {'learning_rate': 0.046642320150933586, 'num_leaves': 463, 'n_estimators': 151, 'max_depth': 20}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"s = time.time()\nmodel = get_model(arch='LGB', optimal_params={'learning_rate': 0.0466, 'num_leaves': 463, 'n_estimators': 151, 'max_depth': 20})\nmodel.fit(train_indeps, train_deps)\npreds = model.predict(valid_indeps)\nloss = rmsle(valid_deps, preds)\nloss, time.time() - s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T19:28:18.977508Z","iopub.execute_input":"2024-12-27T19:28:18.977922Z","iopub.status.idle":"2024-12-27T19:29:01.293346Z","shell.execute_reply.started":"2024-12-27T19:28:18.977887Z","shell.execute_reply":"2024-12-27T19:29:01.29276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fixed_params = {\n    'n_estimators': 150,\n}\nparam_ranges = {\n    'learning_rate': np.linspace(0.03, 0.06, num=10),\n    'num_leaves': np.linspace(400, 500, num=10, dtype=int),\n    'max_depth': np.linspace(15, 25, num=10, dtype=int),\n}\noptimal_params = optimize_params(param_ranges, arch='LGB', optimal_params=optimal_params, graph_settings=graph_settings)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimal_lgb = {'n_estimators': 10,\n 'colsample_bytree': 1,\n 'learning_rate': 1.2,\n 'max_bins': 35938,\n 'max_depth': 13, # everything above test more\n 'min_data_in_leaf': 30, # test more\n 'num_leaves': 16, # test more\n 'subsample': 1\n}\n\noptimal_lgb_2 = {\n    'n_estimators': 150,\n    'learning_rate': 0.05,\n    'num_leaves': 444,\n}\n\ns = time.time()\nmodel = get_model(arch='LGB', optimal_params=optimal_lgb_2)\nmodel.fit(train_indeps, train_deps)\npreds = model.predict(valid_indeps)\nloss = rmsle(valid_deps, preds)\nloss, time.time() - s","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T19:51:47.461513Z","iopub.execute_input":"2024-12-27T19:51:47.461924Z","iopub.status.idle":"2024-12-27T19:52:30.548337Z","shell.execute_reply.started":"2024-12-27T19:51:47.461887Z","shell.execute_reply":"2024-12-27T19:52:30.545052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calc_loss(params):\n    model = lgb.LGBMRegressor(**params, n_jobs=-1, verbose=-1)\n    model.fit(train_indeps, train_deps)\n    model.fit(train_indeps, np.log1p(train_deps))\n    preds = model.predict(valid_indeps)\n    valid_deps_log = np.log1p(valid_deps)\n    return valid_deps, preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:42:41.479046Z","iopub.execute_input":"2024-12-30T17:42:41.4794Z","iopub.status.idle":"2024-12-30T17:42:41.484153Z","shell.execute_reply.started":"2024-12-30T17:42:41.479373Z","shell.execute_reply":"2024-12-30T17:42:41.483019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'n_estimators': 170,\n    'learning_rate': 0.05,\n    'num_leaves': 444,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:31:32.005428Z","iopub.execute_input":"2024-12-30T05:31:32.005729Z","iopub.status.idle":"2024-12-30T05:31:32.009561Z","shell.execute_reply.started":"2024-12-30T05:31:32.005705Z","shell.execute_reply":"2024-12-30T05:31:32.008522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'n_estimators': 100,\n    'num_leaves': 50,\n    'reg_alpha': 0.2,\n    'reg_lambda': 100\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:45:54.733568Z","iopub.execute_input":"2024-12-30T17:45:54.73399Z","iopub.status.idle":"2024-12-30T17:45:54.738929Z","shell.execute_reply.started":"2024-12-30T17:45:54.733957Z","shell.execute_reply":"2024-12-30T17:45:54.737847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vdl, p = calc_loss(params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:41:36.999537Z","iopub.execute_input":"2024-12-30T17:41:36.999899Z","iopub.status.idle":"2024-12-30T17:41:54.951487Z","shell.execute_reply.started":"2024-12-30T17:41:36.999868Z","shell.execute_reply":"2024-12-30T17:41:54.950736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n#rmsle(vdl, p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:42:16.657446Z","iopub.execute_input":"2024-12-30T17:42:16.657977Z","iopub.status.idle":"2024-12-30T17:42:16.671528Z","shell.execute_reply.started":"2024-12-30T17:42:16.657929Z","shell.execute_reply":"2024-12-30T17:42:16.670493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vdl.min()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:49:39.481444Z","iopub.execute_input":"2024-12-30T05:49:39.481867Z","iopub.status.idle":"2024-12-30T05:49:39.489556Z","shell.execute_reply.started":"2024-12-30T05:49:39.481812Z","shell.execute_reply":"2024-12-30T05:49:39.488345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calc_loss(params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T05:31:36.057943Z","iopub.execute_input":"2024-12-30T05:31:36.058331Z","iopub.status.idle":"2024-12-30T05:32:23.91216Z","shell.execute_reply.started":"2024-12-30T05:31:36.058301Z","shell.execute_reply":"2024-12-30T05:32:23.908911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calc_loss(params) #n_estimators = 150","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:04:24.761061Z","iopub.execute_input":"2024-12-27T20:04:24.761333Z","iopub.status.idle":"2024-12-27T20:05:09.567868Z","shell.execute_reply.started":"2024-12-27T20:04:24.761312Z","shell.execute_reply":"2024-12-27T20:05:09.565908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"calc_loss(params) #n_estimators = 160","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T20:07:49.349124Z","iopub.execute_input":"2024-12-27T20:07:49.349399Z","iopub.status.idle":"2024-12-27T20:08:28.891658Z","shell.execute_reply.started":"2024-12-27T20:07:49.349378Z","shell.execute_reply":"2024-12-27T20:08:28.890978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(path/'test.csv')\ntest_indeps,_ = proc_data(test_df)\nsubmit_df = pd.read_csv(path/'sample_submission.csv')\nmodel = lgb.LGBMRegressor(**params, n_jobs=-1, verbose=-1)\nmodel.fit(indeps, np.log1p(deps))\npreds = model.predict(test_indeps)\nsubmit_df['Premium Amount'] = np.e**preds\nsubmit_df.to_csv(f'insurance_v8.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:47:00.133499Z","iopub.execute_input":"2024-12-30T17:47:00.133944Z","iopub.status.idle":"2024-12-30T17:47:31.928819Z","shell.execute_reply.started":"2024-12-30T17:47:00.133908Z","shell.execute_reply":"2024-12-30T17:47:31.927899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit(arch='LGB', params=params, name='submission_v6')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T17:44:25.743935Z","iopub.execute_input":"2024-12-30T17:44:25.744331Z","iopub.status.idle":"2024-12-30T17:44:58.429067Z","shell.execute_reply.started":"2024-12-30T17:44:25.744296Z","shell.execute_reply":"2024-12-30T17:44:58.428146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"graph_settings = {\n    'alpha': {'x_label': 'L1 Regularization', 'log': True},\n    'colsample_bytree': {'x_label': 'Fraction of Column Samples in Training Set'},\n    'gamma': {'x_label': 'Minimum Loss Required for Split', 'log': True},\n    'lambda': {'x_label': 'L2 Regularization', 'log': True},\n    'learning_rate': {'x_label': 'Learning Rate', 'log': True},\n    'max_bins': {'x_label': 'Maximum Bins', 'log': True},\n    'max_depth': {'x_label': 'Maximum Depth of Trees'},\n    'min_child_weight': {'x_label': 'Minimum Sum of Instance Weights in Child Nodes'},\n    'min_data_in_leaf': {'x_label': 'Minimum Data in Leaf'},\n    'min_split_gain': {'x_label': 'Minimum Sum of Instance Weights in Child Nodes'},\n    'num_leaves': {'x_label': 'Number of Leaves'},\n    'n_estimators': {'x_label': 'Number of Trees'},\n    'reg_alpha': {'x_label': 'L1 Regularization', 'log': True},\n    'reg_lambda': {'x_label': 'L2 Regularization','log': True},\n    'subsample': {'x_label': 'Fraction of Row Samples in Training Set'}\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:56:30.803447Z","iopub.execute_input":"2024-12-27T18:56:30.80384Z","iopub.status.idle":"2024-12-27T18:56:30.811132Z","shell.execute_reply.started":"2024-12-27T18:56:30.803811Z","shell.execute_reply":"2024-12-27T18:56:30.810135Z"}},"outputs":[],"execution_count":null}]}