{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# S4E12 GradientBoostingRegressor Optuna\n","metadata":{"papermill":{"duration":0.014799,"end_time":"2021-06-30T01:42:31.37554","exception":false,"start_time":"2021-06-30T01:42:31.360741","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"GradientBoostingRegressor does not accept missing values encoded as NaN natively. ","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport random\nimport optuna\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import GradientBoostingRegressor ","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":2.560292,"end_time":"2021-06-30T01:42:33.94908","exception":false,"start_time":"2021-06-30T01:42:31.388788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:42.949113Z","iopub.execute_input":"2024-12-05T09:04:42.949498Z","iopub.status.idle":"2024-12-05T09:04:42.953964Z","shell.execute_reply.started":"2024-12-05T09:04:42.949446Z","shell.execute_reply":"2024-12-05T09:04:42.953145Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"papermill":{"duration":1.678806,"end_time":"2021-06-30T01:42:35.642283","exception":false,"start_time":"2021-06-30T01:42:33.963477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:42.955363Z","iopub.execute_input":"2024-12-05T09:04:42.955602Z","iopub.status.idle":"2024-12-05T09:04:48.493799Z","shell.execute_reply.started":"2024-12-05T09:04:42.955579Z","shell.execute_reply":"2024-12-05T09:04:48.493136Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train=train.fillna(0)\ntest=test.fillna(0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\ntrain = train.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:48.494886Z","iopub.execute_input":"2024-12-05T09:04:48.495246Z","iopub.status.idle":"2024-12-05T09:04:48.70163Z","shell.execute_reply.started":"2024-12-05T09:04:48.49521Z","shell.execute_reply":"2024-12-05T09:04:48.700771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = test.rename(columns = lambda x:re.sub('[^A-Za-z0-9]+', '_', x))\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:48.702562Z","iopub.execute_input":"2024-12-05T09:04:48.702806Z","iopub.status.idle":"2024-12-05T09:04:48.82521Z","shell.execute_reply.started":"2024-12-05T09:04:48.702782Z","shell.execute_reply":"2024-12-05T09:04:48.824323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.Policy_Start_Date=pd.to_datetime(train.Policy_Start_Date, infer_datetime_format=True)\ntrain.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:48.827637Z","iopub.execute_input":"2024-12-05T09:04:48.827927Z","iopub.status.idle":"2024-12-05T09:04:49.660388Z","shell.execute_reply.started":"2024-12-05T09:04:48.8279Z","shell.execute_reply":"2024-12-05T09:04:49.659476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.Policy_Start_Date=pd.to_datetime(test.Policy_Start_Date, infer_datetime_format=True)\ntest.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:49.661738Z","iopub.execute_input":"2024-12-05T09:04:49.662134Z","iopub.status.idle":"2024-12-05T09:04:50.208959Z","shell.execute_reply.started":"2024-12-05T09:04:49.662092Z","shell.execute_reply":"2024-12-05T09:04:50.208174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def day_part(hour):\n    if hour in [4,5]:\n        return 1\n    elif hour in [6,7]:\n        return 2\n    elif hour in [8,9,10]:\n        return 3\n    elif hour in [11,12,13]:\n        return 4\n    elif hour in [14,15,16]:\n        return 5\n    elif hour in [17, 18,19]:\n        return 6\n    elif hour in [20, 21, 22]:\n        return 7\n    elif hour in [23,0,1,2,3]:\n        return 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:50.209875Z","iopub.execute_input":"2024-12-05T09:04:50.21014Z","iopub.status.idle":"2024-12-05T09:04:50.215097Z","shell.execute_reply.started":"2024-12-05T09:04:50.210113Z","shell.execute_reply":"2024-12-05T09:04:50.214305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\ntrain['Policy_Start_Date:year'] = train['Policy_Start_Date'].dt.year\ntrain['Policy_Start_Date:month'] = train['Policy_Start_Date'].dt.month\ntrain['Policy_Start_Date:day'] = train['Policy_Start_Date'].dt.day\ntrain['Policy_Start_Date:day_of_week'] = train['Policy_Start_Date'].dt.day_of_week\ntrain['Policy_Start_Date:day_of_year'] = train['Policy_Start_Date'].dt.day_of_year\ntrain['Policy_Start_Date:is_year_start'] = train['Policy_Start_Date'].dt.is_year_start\ntrain['Policy_Start_Date:is_quarter_start'] = train['Policy_Start_Date'].dt.is_quarter_start\ntrain['Policy_Start_Date:is_year_end'] = train['Policy_Start_Date'].dt.is_year_end\ntrain['Policy_Start_Date:is_quarter_end'] = train['Policy_Start_Date'].dt.is_quarter_end\ntrain['Policy_Start_Date:is_month_start'] = train['Policy_Start_Date'].dt.is_month_start\ntrain['Policy_Start_Date:is_month_end'] = train['Policy_Start_Date'].dt.is_month_end\ntrain['Policy_Start_Date:is_weekend'] = np.where(train['Policy_Start_Date:day_of_week'].isin([5,6]), 1,0)\ntrain['Policy_Start_Date:hour'] = train['Policy_Start_Date'].dt.hour\ntrain['Policy_Start_Date:minute'] = train['Policy_Start_Date'].dt.minute\ntrain['Policy_Start_Date:second'] = train['Policy_Start_Date'].dt.second\ntrain['Policy_Start_Date:day_part'] = train['Policy_Start_Date:hour'].apply(day_part)\ntrain[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']]=train[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']].replace({True:1,False:0})\ntrain=train.drop(['Policy_Start_Date'],axis=1)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:50.216298Z","iopub.execute_input":"2024-12-05T09:04:50.216641Z","iopub.status.idle":"2024-12-05T09:04:53.339649Z","shell.execute_reply.started":"2024-12-05T09:04:50.216605Z","shell.execute_reply":"2024-12-05T09:04:53.338868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\ntest['Policy_Start_Date:year'] = test['Policy_Start_Date'].dt.year\ntest['Policy_Start_Date:month'] = test['Policy_Start_Date'].dt.month\ntest['Policy_Start_Date:day'] = test['Policy_Start_Date'].dt.day\ntest['Policy_Start_Date:day_of_week'] = test['Policy_Start_Date'].dt.day_of_week\ntest['Policy_Start_Date:day_of_year'] = test['Policy_Start_Date'].dt.day_of_year\ntest['Policy_Start_Date:is_year_start'] = test['Policy_Start_Date'].dt.is_year_start\ntest['Policy_Start_Date:is_quarter_start'] = test['Policy_Start_Date'].dt.is_quarter_start\ntest['Policy_Start_Date:is_year_end'] = test['Policy_Start_Date'].dt.is_year_end\ntest['Policy_Start_Date:is_quarter_end'] = test['Policy_Start_Date'].dt.is_quarter_end\ntest['Policy_Start_Date:is_month_start'] = test['Policy_Start_Date'].dt.is_month_start\ntest['Policy_Start_Date:is_month_end'] = test['Policy_Start_Date'].dt.is_month_end\ntest['Policy_Start_Date:is_weekend'] = np.where(test['Policy_Start_Date:day_of_week'].isin([5,6]), 1,0)\ntest['Policy_Start_Date:hour'] = test['Policy_Start_Date'].dt.hour\ntest['Policy_Start_Date:minute'] = test['Policy_Start_Date'].dt.minute\ntest['Policy_Start_Date:second'] = test['Policy_Start_Date'].dt.second\ntest['Policy_Start_Date:day_part'] = test['Policy_Start_Date:hour'].apply(day_part)\ntest[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']]=test[['Policy_Start_Date:is_month_end', 'Policy_Start_Date:is_month_start',\n       'Policy_Start_Date:is_quarter_end', 'Policy_Start_Date:is_quarter_start',\n       'Policy_Start_Date:is_year_end', 'Policy_Start_Date:is_year_start']].replace({True:1,False:0})\ntest=test.drop(['Policy_Start_Date'],axis=1)\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:53.341051Z","iopub.execute_input":"2024-12-05T09:04:53.341441Z","iopub.status.idle":"2024-12-05T09:04:55.406637Z","shell.execute_reply.started":"2024-12-05T09:04:53.341391Z","shell.execute_reply":"2024-12-05T09:04:55.405833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df_train, df_test):\n    combined_df = pd.concat([df_train, df_test])\n    for c in combined_df.columns:\n        if combined_df[c].dtype == 'object':\n            combined_df[c] = combined_df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(combined_df[c].values))\n            combined_df[c] = lbl.transform(combined_df[c].values)\n    return combined_df.iloc[:len(df_train)], combined_df.iloc[len(df_train):]\n\ntrain,test=labelencoder(train,test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T09:04:55.415537Z","iopub.status.idle":"2024-12-05T09:04:55.415829Z","shell.execute_reply.started":"2024-12-05T09:04:55.41569Z","shell.execute_reply":"2024-12-05T09:04:55.415705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = train['Premium_Amount']\ndata = train.drop(['Premium_Amount'],axis=1)\n\ncolumns=data.columns.to_list()\nprint(columns)","metadata":{"papermill":{"duration":0.087259,"end_time":"2021-06-30T01:42:35.884715","exception":false,"start_time":"2021-06-30T01:42:35.797456","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.416833Z","iopub.status.idle":"2024-12-05T09:04:55.417292Z","shell.execute_reply.started":"2024-12-05T09:04:55.41706Z","shell.execute_reply":"2024-12-05T09:04:55.417087Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial,data=data,target=target):\n    \n    train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)\n    param =   {\n        'n_estimators': trial.suggest_int('n_estimators', 50, 200),  \n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.2), \n        'max_depth': trial.suggest_int('max_depth', 3, 10),  \n        'min_samples_split': trial.suggest_int('min_samples_split', 2, 10),  \n        'subsample': trial.suggest_uniform('subsample', 0.7, 1.0), \n\n        'loss': 'squared_error',  \n        'criterion': 'friedman_mse',  \n        'random_state': 42,  \n    }\n    model = GradientBoostingRegressor(**param)      \n    model.fit(train_x,train_y)\n    preds = model.predict(test_x)\n    rmse = mean_squared_error(test_y, preds,squared=False)\n    \n    return rmse","metadata":{"papermill":{"duration":0.024903,"end_time":"2021-06-30T01:42:35.960585","exception":false,"start_time":"2021-06-30T01:42:35.935682","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.418814Z","iopub.status.idle":"2024-12-05T09:04:55.419228Z","shell.execute_reply.started":"2024-12-05T09:04:55.419042Z","shell.execute_reply":"2024-12-05T09:04:55.419062Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=10)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)","metadata":{"papermill":{"duration":1081.471176,"end_time":"2021-06-30T02:00:37.446652","exception":false,"start_time":"2021-06-30T01:42:35.975476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.420549Z","iopub.status.idle":"2024-12-05T09:04:55.420907Z","shell.execute_reply.started":"2024-12-05T09:04:55.420757Z","shell.execute_reply":"2024-12-05T09:04:55.420777Z"},"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the scores from all trials\noptuna.visualization.plot_optimization_history(study)","metadata":{"papermill":{"duration":0.167578,"end_time":"2021-06-30T02:00:37.745069","exception":false,"start_time":"2021-06-30T02:00:37.577491","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.422481Z","iopub.status.idle":"2024-12-05T09:04:55.42276Z","shell.execute_reply.started":"2024-12-05T09:04:55.422621Z","shell.execute_reply":"2024-12-05T09:04:55.422635Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# shows the evolution of the search\noptuna.visualization.plot_slice(study)","metadata":{"papermill":{"duration":0.309406,"end_time":"2021-06-30T02:00:38.218357","exception":false,"start_time":"2021-06-30T02:00:37.908951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.423808Z","iopub.status.idle":"2024-12-05T09:04:55.424112Z","shell.execute_reply.started":"2024-12-05T09:04:55.423941Z","shell.execute_reply":"2024-12-05T09:04:55.423955Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize parameter importances.\noptuna.visualization.plot_param_importances(study)","metadata":{"papermill":{"duration":0.76143,"end_time":"2021-06-30T02:00:39.149399","exception":false,"start_time":"2021-06-30T02:00:38.387969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.425378Z","iopub.status.idle":"2024-12-05T09:04:55.425681Z","shell.execute_reply.started":"2024-12-05T09:04:55.425532Z","shell.execute_reply":"2024-12-05T09:04:55.425548Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Best_trial=study.best_trial.params\n\nfix_dict = {   \n    'loss': 'squared_error',  \n    'criterion': 'friedman_mse',  \n    'random_state': 42, \n  }\nBest_trial.update(fix_dict)\n\nprint(Best_trial)","metadata":{"papermill":{"duration":0.04543,"end_time":"2021-06-30T02:00:39.31482","exception":false,"start_time":"2021-06-30T02:00:39.26939","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.426615Z","iopub.status.idle":"2024-12-05T09:04:55.426885Z","shell.execute_reply.started":"2024-12-05T09:04:55.426755Z","shell.execute_reply":"2024-12-05T09:04:55.426768Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nprint(sample.shape)","metadata":{"papermill":{"duration":0.178934,"end_time":"2021-06-30T02:00:39.529731","exception":false,"start_time":"2021-06-30T02:00:39.350797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.427827Z","iopub.status.idle":"2024-12-05T09:04:55.428155Z","shell.execute_reply.started":"2024-12-05T09:04:55.428003Z","shell.execute_reply":"2024-12-05T09:04:55.428026Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = np.zeros((sample.shape[0]))\nkf = KFold(n_splits=5,random_state=48,shuffle=True)\nfor trn_idx, test_idx in kf.split(train[columns],target):\n    X_tr,X_val=train[columns].iloc[trn_idx],train[columns].iloc[test_idx]\n    y_tr,y_val=target.iloc[trn_idx],target.iloc[test_idx]\n    model = GradientBoostingRegressor(**Best_trial)\n    model.fit(X_tr,y_tr)\n    preds+=model.predict(test[columns])/kf.n_splits   \n    rmse=mean_squared_error(y_val, model.predict(X_val),squared=False)\n    print(rmse)","metadata":{"papermill":{"duration":150.161649,"end_time":"2021-06-30T02:03:09.726111","exception":false,"start_time":"2021-06-30T02:00:39.564462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.430051Z","iopub.status.idle":"2024-12-05T09:04:55.430327Z","shell.execute_reply.started":"2024-12-05T09:04:55.430194Z","shell.execute_reply":"2024-12-05T09:04:55.430208Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subm = sample\nsubm['Premium Amount'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\nsubm","metadata":{"papermill":{"duration":1.598507,"end_time":"2021-06-30T02:03:11.447247","exception":false,"start_time":"2021-06-30T02:03:09.84874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-12-05T09:04:55.431502Z","iopub.status.idle":"2024-12-05T09:04:55.431946Z","shell.execute_reply.started":"2024-12-05T09:04:55.431718Z","shell.execute_reply":"2024-12-05T09:04:55.43174Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.037603,"end_time":"2021-06-30T02:03:11.600307","exception":false,"start_time":"2021-06-30T02:03:11.562704","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}