{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# S4E12 HistGradientBoosting Optuna\n","metadata":{"papermill":{"duration":0.014799,"end_time":"2021-06-30T01:42:31.37554","exception":false,"start_time":"2021-06-30T01:42:31.360741","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport random\nimport optuna\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.ensemble import HistGradientBoostingRegressor","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":2.560292,"end_time":"2021-06-30T01:42:33.94908","exception":false,"start_time":"2021-06-30T01:42:31.388788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:56:47.406864Z","iopub.execute_input":"2024-12-04T07:56:47.407628Z","iopub.status.idle":"2024-12-04T07:56:52.434595Z","shell.execute_reply.started":"2024-12-04T07:56:47.407563Z","shell.execute_reply":"2024-12-04T07:56:52.43381Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"papermill":{"duration":1.678806,"end_time":"2021-06-30T01:42:35.642283","exception":false,"start_time":"2021-06-30T01:42:33.963477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:56:52.43619Z","iopub.execute_input":"2024-12-04T07:56:52.436756Z","iopub.status.idle":"2024-12-04T07:57:01.546946Z","shell.execute_reply.started":"2024-12-04T07:56:52.436726Z","shell.execute_reply":"2024-12-04T07:57:01.545851Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df_train, df_test):\n    combined_df = pd.concat([df_train, df_test])\n    for c in combined_df.columns:\n        if combined_df[c].dtype == 'object':\n            combined_df[c] = combined_df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(combined_df[c].values))\n            combined_df[c] = lbl.transform(combined_df[c].values)\n    return combined_df.iloc[:len(df_train)], combined_df.iloc[len(df_train):]\n\ntrain,test=labelencoder(train0,test0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T07:57:02.035768Z","iopub.status.idle":"2024-12-04T07:57:02.03611Z","shell.execute_reply.started":"2024-12-04T07:57:02.035941Z","shell.execute_reply":"2024-12-04T07:57:02.035958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = train['Premium Amount']\ndata = train.drop(['Premium Amount'],axis=1)\n\ncolumns=data.columns.to_list()\nprint(columns)","metadata":{"papermill":{"duration":0.087259,"end_time":"2021-06-30T01:42:35.884715","exception":false,"start_time":"2021-06-30T01:42:35.797456","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.037917Z","iopub.status.idle":"2024-12-04T07:57:02.038235Z","shell.execute_reply.started":"2024-12-04T07:57:02.038094Z","shell.execute_reply":"2024-12-04T07:57:02.03811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial,data=data,target=target):\n    \n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)\n    param =   {\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3, log=True),\n        \"max_iter\": trial.suggest_int(\"max_iter\", 50, 300),\n        \"max_leaf_nodes\": trial.suggest_int(\"max_leaf_nodes\", 10, 50),\n        \"max_bins\": trial.suggest_int(\"max_bins\", 64, 128),\n        \"min_samples_leaf\": trial.suggest_int(\"min_samples_leaf\", 5, 50),\n        \"l2_regularization\": trial.suggest_float(\"l2_regularization\", 0.0, 2.0),     \n        \n        #fixed part       \n        'early_stopping':True,        # Enable early stopping\n        'validation_fraction':0.2,    # Use 20% of training data for validation\n        'random_state':42,            # Reproducibility\n        'verbose':1                   # Show progress logs\n    }\n    model = HistGradientBoostingRegressor(**param)  \n\n    model.fit(train_x, np.log1p(train_y))\n    preds = np.expm1(model.predict(test_x))\n    \n    #model.fit(train_x,train_y)\n    #preds = model.predict(test_x)\n     \n    rmsle = np.sqrt(mean_squared_log_error(test_y, preds))   \n    #rmse = mean_squared_error(test_y, preds,squared=False)\n\n    return rmsle","metadata":{"papermill":{"duration":0.024903,"end_time":"2021-06-30T01:42:35.960585","exception":false,"start_time":"2021-06-30T01:42:35.935682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.039405Z","iopub.status.idle":"2024-12-04T07:57:02.039731Z","shell.execute_reply.started":"2024-12-04T07:57:02.039552Z","shell.execute_reply":"2024-12-04T07:57:02.039568Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=100)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)","metadata":{"papermill":{"duration":1081.471176,"end_time":"2021-06-30T02:00:37.446652","exception":false,"start_time":"2021-06-30T01:42:35.975476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.04085Z","iopub.status.idle":"2024-12-04T07:57:02.041123Z","shell.execute_reply.started":"2024-12-04T07:57:02.040983Z","shell.execute_reply":"2024-12-04T07:57:02.040996Z"},"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the scores from all trials\noptuna.visualization.plot_optimization_history(study)","metadata":{"papermill":{"duration":0.167578,"end_time":"2021-06-30T02:00:37.745069","exception":false,"start_time":"2021-06-30T02:00:37.577491","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.041939Z","iopub.status.idle":"2024-12-04T07:57:02.042208Z","shell.execute_reply.started":"2024-12-04T07:57:02.042072Z","shell.execute_reply":"2024-12-04T07:57:02.042092Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the evolution of the search\noptuna.visualization.plot_slice(study)","metadata":{"papermill":{"duration":0.309406,"end_time":"2021-06-30T02:00:38.218357","exception":false,"start_time":"2021-06-30T02:00:37.908951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.042989Z","iopub.status.idle":"2024-12-04T07:57:02.043277Z","shell.execute_reply.started":"2024-12-04T07:57:02.043124Z","shell.execute_reply":"2024-12-04T07:57:02.043138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize parameter importances.\noptuna.visualization.plot_param_importances(study)","metadata":{"papermill":{"duration":0.76143,"end_time":"2021-06-30T02:00:39.149399","exception":false,"start_time":"2021-06-30T02:00:38.387969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.044896Z","iopub.status.idle":"2024-12-04T07:57:02.045254Z","shell.execute_reply.started":"2024-12-04T07:57:02.045077Z","shell.execute_reply":"2024-12-04T07:57:02.045102Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict = {   \n    'early_stopping':True,        # Enable early stopping\n    'validation_fraction':0.2,    # Use 20% of training data for validation\n    'random_state':42,            # Reproducibility\n    'verbose':1                   # Show progress logs\n  }\nBest_trial.update(fix_dict)\n\nprint(Best_trial)","metadata":{"papermill":{"duration":0.04543,"end_time":"2021-06-30T02:00:39.31482","exception":false,"start_time":"2021-06-30T02:00:39.26939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.046548Z","iopub.status.idle":"2024-12-04T07:57:02.047074Z","shell.execute_reply.started":"2024-12-04T07:57:02.046807Z","shell.execute_reply":"2024-12-04T07:57:02.046834Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nprint(sample.shape)","metadata":{"papermill":{"duration":0.178934,"end_time":"2021-06-30T02:00:39.529731","exception":false,"start_time":"2021-06-30T02:00:39.350797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.049183Z","iopub.status.idle":"2024-12-04T07:57:02.049724Z","shell.execute_reply.started":"2024-12-04T07:57:02.049432Z","shell.execute_reply":"2024-12-04T07:57:02.049459Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = np.zeros((test.shape[0]))\nkf = KFold(n_splits=5, random_state=48, shuffle=True)\n\nfor trn_idx, test_idx in kf.split(train[columns], target):\n    X_tr, X_val = train[columns].iloc[trn_idx], train[columns].iloc[test_idx]\n    y_tr, y_val = target.iloc[trn_idx], target.iloc[test_idx]\n\n    model = HistGradientBoostingRegressor(**Best_trial)\n    model.fit(X_tr, np.log1p(y_tr))  \n\n    preds_fold = np.expm1(model.predict(test[columns])) \n    preds_fold = np.nan_to_num(preds_fold, nan=0, posinf=0, neginf=0)  \n    preds_fold = np.clip(preds_fold, 0, 1e10)\n    preds += preds_fold / kf.n_splits  \n\n    y_val_pred = np.expm1(model.predict(X_val))  \n    y_val_pred = np.nan_to_num(y_val_pred, nan=0, posinf=0, neginf=0)  \n    y_val_pred = np.clip(y_val_pred, 0, 1e10)\n    \n    y_val_actual = np.expm1(y_val)  \n    y_val_actual = np.nan_to_num(y_val_actual, nan=0, posinf=0, neginf=0) \n    \n    try:\n        rmsle = np.sqrt(mean_squared_log_error(y_val_actual, y_val_pred)) \n        print(f'RMSLE for fold: {rmsle}')\n    except ValueError as e:\n        print(f\"Error in fold: {e}\")","metadata":{"papermill":{"duration":150.161649,"end_time":"2021-06-30T02:03:09.726111","exception":false,"start_time":"2021-06-30T02:00:39.564462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.050774Z","iopub.status.idle":"2024-12-04T07:57:02.051231Z","shell.execute_reply.started":"2024-12-04T07:57:02.050997Z","shell.execute_reply":"2024-12-04T07:57:02.051021Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n    for trn_idx, test_idx in kf.split(train[columns],target):\n        X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]\n        y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]\n        model = HistGradientBoostingRegressor(**Best_trial)\n        model.fit(X_tr,y_tr)\n        preds+=model.predict(test[columns])/kf.n_splits   \n    \n        rmsle=np.sqrt(mean_squared_log_error(y_val, model.predict(X_val)))\n        print(rmsle) \n        \n        #rmse=mean_squared_error(y_val, model.predict(X_val),squared=False)\n        #print(rmse)\n        ","metadata":{"papermill":{"duration":150.161649,"end_time":"2021-06-30T02:03:09.726111","exception":false,"start_time":"2021-06-30T02:00:39.564462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.050774Z","iopub.status.idle":"2024-12-04T07:57:02.051231Z","shell.execute_reply.started":"2024-12-04T07:57:02.050997Z","shell.execute_reply":"2024-12-04T07:57:02.051021Z"}}},{"cell_type":"code","source":"subm = sample\nsubm['Premium Amount'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\nsubm","metadata":{"papermill":{"duration":1.598507,"end_time":"2021-06-30T02:03:11.447247","exception":false,"start_time":"2021-06-30T02:03:09.84874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-04T07:57:02.052649Z","iopub.status.idle":"2024-12-04T07:57:02.053168Z","shell.execute_reply.started":"2024-12-04T07:57:02.052874Z","shell.execute_reply":"2024-12-04T07:57:02.052899Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.037603,"end_time":"2021-06-30T02:03:11.600307","exception":false,"start_time":"2021-06-30T02:03:11.562704","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}