{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:51.203452Z","iopub.execute_input":"2024-12-28T15:20:51.203952Z","iopub.status.idle":"2024-12-28T15:20:51.212802Z","shell.execute_reply.started":"2024-12-28T15:20:51.203902Z","shell.execute_reply":"2024-12-28T15:20:51.211606Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nfrom scipy.signal import find_peaks\nfrom scipy.stats import skew\n\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, OneHotEncoder, LabelEncoder\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV\nfrom xgboost.sklearn import XGBRegressor\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nfrom lightgbm import early_stopping, log_evaluation\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_error, r2_score, make_scorer, mean_squared_log_error\nimport optuna\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"lightgbm\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:51.214195Z","iopub.execute_input":"2024-12-28T15:20:51.214529Z","iopub.status.idle":"2024-12-28T15:20:51.241412Z","shell.execute_reply.started":"2024-12-28T15:20:51.214502Z","shell.execute_reply":"2024-12-28T15:20:51.240331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:51.243322Z","iopub.execute_input":"2024-12-28T15:20:51.243666Z","iopub.status.idle":"2024-12-28T15:20:51.257472Z","shell.execute_reply.started":"2024-12-28T15:20:51.243638Z","shell.execute_reply":"2024-12-28T15:20:51.256123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/train.csv\")\ndf_test = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/test.csv\")\nsample_submission = pd.read_csv(r\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\ndf_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:51.259134Z","iopub.execute_input":"2024-12-28T15:20:51.259498Z","iopub.status.idle":"2024-12-28T15:20:58.648283Z","shell.execute_reply.started":"2024-12-28T15:20:51.25947Z","shell.execute_reply":"2024-12-28T15:20:58.647148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train.drop(columns = [\"id\"], axis = 1)\ndf_test = df_test.drop(columns = [\"id\"], axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:58.649354Z","iopub.execute_input":"2024-12-28T15:20:58.649726Z","iopub.status.idle":"2024-12-28T15:20:58.837003Z","shell.execute_reply.started":"2024-12-28T15:20:58.649682Z","shell.execute_reply":"2024-12-28T15:20:58.836153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET = \"Premium Amount\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:58.837793Z","iopub.execute_input":"2024-12-28T15:20:58.838035Z","iopub.status.idle":"2024-12-28T15:20:58.842029Z","shell.execute_reply.started":"2024-12-28T15:20:58.838012Z","shell.execute_reply":"2024-12-28T15:20:58.84113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Train Data Shape: {df_train.shape}\")\nprint(f\"Test Data Shape: {df_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:58.843189Z","iopub.execute_input":"2024-12-28T15:20:58.843556Z","iopub.status.idle":"2024-12-28T15:20:58.861811Z","shell.execute_reply.started":"2024-12-28T15:20:58.84352Z","shell.execute_reply":"2024-12-28T15:20:58.860688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:58.864428Z","iopub.execute_input":"2024-12-28T15:20:58.864689Z","iopub.status.idle":"2024-12-28T15:20:59.499408Z","shell.execute_reply.started":"2024-12-28T15:20:58.864666Z","shell.execute_reply":"2024-12-28T15:20:59.498231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.describe(include = 'object').T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:20:59.501068Z","iopub.execute_input":"2024-12-28T15:20:59.501363Z","iopub.status.idle":"2024-12-28T15:21:01.536569Z","shell.execute_reply.started":"2024-12-28T15:20:59.501333Z","shell.execute_reply":"2024-12-28T15:21:01.535599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.describe(exclude = 'object').T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:01.537462Z","iopub.execute_input":"2024-12-28T15:21:01.537757Z","iopub.status.idle":"2024-12-28T15:21:02.170707Z","shell.execute_reply.started":"2024-12-28T15:21:01.537717Z","shell.execute_reply":"2024-12-28T15:21:02.169653Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Timestamp Processing","metadata":{}},{"cell_type":"code","source":"df_train[\"Policy Start Date\"] = pd.to_datetime(df_train[\"Policy Start Date\"])\ndf_train[\"Policy Start Date\"] = df_train[\"Policy Start Date\"].astype(np.int64) / 10**9\n#df_train[\"Policy Start Date Year\"] = df_train[\"Policy Start Date\"].dt.year.astype(float)\n#df_train[\"Policy Start Date Month\"] = df_train[\"Policy Start Date\"].dt.month.astype(float)\n#df_train[\"Policy Start Date Day\"] = df_train[\"Policy Start Date\"].dt.day.astype(float)\n#df_train = df_train.drop(columns = [\"Policy Start Date\"], axis = 1)\n\ndf_test[\"Policy Start Date\"] = pd.to_datetime(df_test[\"Policy Start Date\"])\ndf_test[\"Policy Start Date\"] = df_test[\"Policy Start Date\"].astype(np.int64) / 10**9\n#df_test[\"Policy Start Date Year\"] = df_test[\"Policy Start Date\"].dt.year.astype(float)\n#df_test[\"Policy Start Date Month\"] = df_test[\"Policy Start Date\"].dt.month.astype(float)\n#df_test[\"Policy Start Date Day\"] = df_test[\"Policy Start Date\"].dt.day.astype(float)\n#df_test = df_test.drop(columns = [\"Policy Start Date\"], axis = 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:02.171624Z","iopub.execute_input":"2024-12-28T15:21:02.171961Z","iopub.status.idle":"2024-12-28T15:21:02.914381Z","shell.execute_reply.started":"2024-12-28T15:21:02.171933Z","shell.execute_reply":"2024-12-28T15:21:02.913126Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Null Ratio","metadata":{}},{"cell_type":"code","source":"df_train_missing_perc = pd.DataFrame(columns = [\"Feature\", \"Count\", \"Percentage\"])\n\ndf_train_missing_perc[\"Feature\"] = df_train.columns\ndf_train_missing_perc[\"Percentage\"] = ((df_train.isnull().sum() / len(df_train)) * 100).values\ndf_train_missing_perc[\"Count\"] = df_train.isnull().sum().values\ndf_train_missing_perc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:02.915506Z","iopub.execute_input":"2024-12-28T15:21:02.915857Z","iopub.status.idle":"2024-12-28T15:21:04.032927Z","shell.execute_reply.started":"2024-12-28T15:21:02.915811Z","shell.execute_reply":"2024-12-28T15:21:04.031874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_missing_perc = pd.DataFrame(columns = [\"Feature\", \"Count\", \"Percentage\"])\n\ndf_test_missing_perc[\"Feature\"] = df_test.columns\ndf_test_missing_perc[\"Percentage\"] = ((df_test.isnull().sum() / len(df_test)) * 100).values\ndf_test_missing_perc[\"Count\"] = df_test.isnull().sum().values\ndf_test_missing_perc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:04.034019Z","iopub.execute_input":"2024-12-28T15:21:04.034662Z","iopub.status.idle":"2024-12-28T15:21:04.78286Z","shell.execute_reply.started":"2024-12-28T15:21:04.034629Z","shell.execute_reply":"2024-12-28T15:21:04.781817Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Fill Empty Cells","metadata":{}},{"cell_type":"code","source":"def fill_empty_cells(df, col):\n  \"\"\"\n  Fills empty (NaN) categorical values in a specified column of a DataFrame.\n  The filling process is based on the most frequent values, selected until \n  their cumulative frequency exceeds a threshold (40%).\n\n  Input:\n      df: Original dataframe\n      col: Column name in dataset (feature)\n  \n  Return:\n      df: Filled dataframe\n  \"\"\"\n  filtered_values = df[col].value_counts().index.tolist()\n  filtered_values_ratio = df[col].value_counts(normalize = True).values.tolist()\n\n  total_value = 0\n  choosen_values = []\n  choosen_ratios = []\n\n  for index, value in enumerate(filtered_values_ratio):\n    total_value += value\n    choosen_values.append(filtered_values[index])\n    choosen_ratios.append(filtered_values_ratio[index])\n    if total_value >= 0.4:\n      break\n\n  choosen_ratios = np.array(choosen_ratios) / sum(choosen_ratios)\n\n  df[col] = df[col].fillna(np.random.choice(choosen_values, p = choosen_ratios))\n\n  return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:04.783861Z","iopub.execute_input":"2024-12-28T15:21:04.784107Z","iopub.status.idle":"2024-12-28T15:21:04.79033Z","shell.execute_reply.started":"2024-12-28T15:21:04.784085Z","shell.execute_reply":"2024-12-28T15:21:04.789152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_numeric = df_train.select_dtypes(include = [\"int32\" ,\"int64\", \"float64\"])\ndf_train_categorical = df_train.select_dtypes(exclude = [\"int32\" ,\"int64\", \"float64\"])\n\ndf_test_numeric = df_test.select_dtypes(include = [\"int32\" ,\"int64\", \"float64\"])\ndf_test_categorical = df_test.select_dtypes(exclude = [\"int32\" ,\"int64\", \"float64\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:04.791395Z","iopub.execute_input":"2024-12-28T15:21:04.791718Z","iopub.status.idle":"2024-12-28T15:21:05.741437Z","shell.execute_reply.started":"2024-12-28T15:21:04.791693Z","shell.execute_reply":"2024-12-28T15:21:05.740277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"{col: df_train_categorical[col].nunique() for col in df_train_categorical.columns}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:05.742358Z","iopub.execute_input":"2024-12-28T15:21:05.742639Z","iopub.status.idle":"2024-12-28T15:21:06.353528Z","shell.execute_reply.started":"2024-12-28T15:21:05.742615Z","shell.execute_reply":"2024-12-28T15:21:06.352365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in df_train.columns:\n  if col == TARGET:\n      continue\n  if col in df_train_numeric.columns:\n    # Option 1\n    df_train[col] = df_train[col].fillna(-1)\n    df_test[col] = df_test[col].fillna(-1)\n\n    # Option 2\n    #df_test[col] = df_test[col].fillna(df_test[col].median())\n    #df_train[col] = df_train[col].fillna(df_train[col].median())\n  else:\n    # Option 1\n    df_train[col] = df_train[col].fillna(\"Unknown\")\n    df_test[col] = df_test[col].fillna(\"Unknown\")\n\n    # Option 2\n    #fill_empty_cells(df_train, col)\n\n    # Option 3\n    #df_test[col] = df_test[col].fillna(df_test[col].mode()[0])\n    #df_train[col] = df_train[col].fillna(df_train[col].mode()[0])\n\n    \n\ndf_train_numeric = df_train.select_dtypes(include = [\"int32\", \"int64\", \"float64\"])\ndf_train_categorical = df_train.select_dtypes(exclude = [\"int32\" ,\"int64\", \"float64\"])\n\ndf_test_numeric = df_test.select_dtypes(include = [\"int32\", \"int64\", \"float64\"])\ndf_test_categorical = df_test.select_dtypes(exclude = [\"int32\" ,\"int64\", \"float64\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:06.354616Z","iopub.execute_input":"2024-12-28T15:21:06.355013Z","iopub.status.idle":"2024-12-28T15:21:08.612584Z","shell.execute_reply.started":"2024-12-28T15:21:06.354974Z","shell.execute_reply":"2024-12-28T15:21:08.611625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:08.613586Z","iopub.execute_input":"2024-12-28T15:21:08.613932Z","iopub.status.idle":"2024-12-28T15:21:09.177789Z","shell.execute_reply.started":"2024-12-28T15:21:08.613898Z","shell.execute_reply":"2024-12-28T15:21:09.176772Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Check Duplicates","metadata":{}},{"cell_type":"code","source":"train_duplicates = df_train.duplicated().sum()\nprint(f\"Number of duplicate rows in the training dataset: {train_duplicates}\")\n\ntest_duplicates = df_test.duplicated().sum()\nprint(f\"Number of duplicate rows in the test dataset: {test_duplicates}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:09.17893Z","iopub.execute_input":"2024-12-28T15:21:09.179294Z","iopub.status.idle":"2024-12-28T15:21:11.185752Z","shell.execute_reply.started":"2024-12-28T15:21:09.179256Z","shell.execute_reply":"2024-12-28T15:21:11.184683Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Encode","metadata":{}},{"cell_type":"code","source":"le_columns = [\"Education Level\", \"Location\", \"Policy Type\", \"Customer Feedback\", \"Exercise Frequency\", \"Property Type\"]\nonehot_columns = [\"Gender\", \"Marital Status\", \"Occupation\", \"Smoking Status\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:11.186777Z","iopub.execute_input":"2024-12-28T15:21:11.187145Z","iopub.status.idle":"2024-12-28T15:21:11.191824Z","shell.execute_reply.started":"2024-12-28T15:21:11.187105Z","shell.execute_reply":"2024-12-28T15:21:11.190652Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"education_order = [\"High School\", \"Bachelor's\", \"Master\", \"PhD\"]\nlocation_order = [\"Suburban\", \"Rural\", \"Urban\"]\npolicy_order = [\"Basic\", \"Comprehensive\", \"Premium\"]\ncustomer_order = [\"Poor\", \"Average\", \"Good\"]\nexercise_order = [\"Rarely\", \"Monthly\", \"Weekly\", \"Daily\"]\nproperty_order = [\"Condo\", \"Apartment\", \"House\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:11.193083Z","iopub.execute_input":"2024-12-28T15:21:11.193432Z","iopub.status.idle":"2024-12-28T15:21:11.210467Z","shell.execute_reply.started":"2024-12-28T15:21:11.193405Z","shell.execute_reply":"2024-12-28T15:21:11.20943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"le_dict = {}\nfor col in le_columns:\n  le = LabelEncoder()\n  if col == \"Education Level\":\n    le.fit(education_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  elif col == \"Location\":\n    le.fit(location_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  elif col == \"Policy Type\":\n    le.fit(policy_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  elif col == \"Customer Feedback\":\n    le.fit(customer_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  elif col == \"Exercise Frequency\":\n    le.fit(exercise_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  elif col == \"Property Type\":\n    le.fit(property_order)\n    df_train_categorical[col] = le.fit_transform(df_train_categorical[col])\n    le_dict[col] = le\n    df_test_categorical[col] = le.transform(df_test_categorical[col])\n  else:\n    print(\"Check\")\n\n\nencoder = OneHotEncoder(handle_unknown = \"ignore\", drop = None)\nencoder.fit(df_train_categorical[onehot_columns])\n\ndf_train_onehot_encoded = pd.DataFrame(\n    encoder.transform(df_train_categorical[onehot_columns]).toarray(),\n    columns=encoder.get_feature_names_out(onehot_columns),\n    index=df_train.index\n)\n\ndf_test_onehot_encoded = pd.DataFrame(\n    encoder.transform(df_test_categorical[onehot_columns]).toarray(),\n    columns=encoder.get_feature_names_out(onehot_columns),\n    index=df_test.index\n)\n\ndf_train_categorical = pd.concat([df_train_categorical.drop(columns=onehot_columns), df_train_onehot_encoded], axis=1)\ndf_test_categorical = pd.concat([df_test_categorical.drop(columns=onehot_columns), df_test_onehot_encoded], axis=1)\n\ndf_train_combined = pd.concat([df_train_numeric, df_train_categorical], axis = 1)\ndf_test_combined = pd.concat([df_test_numeric, df_test_categorical], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:11.211601Z","iopub.execute_input":"2024-12-28T15:21:11.211968Z","iopub.status.idle":"2024-12-28T15:21:15.927975Z","shell.execute_reply.started":"2024-12-28T15:21:11.21193Z","shell.execute_reply":"2024-12-28T15:21:15.927062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_combined.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:15.93171Z","iopub.execute_input":"2024-12-28T15:21:15.93202Z","iopub.status.idle":"2024-12-28T15:21:15.960285Z","shell.execute_reply.started":"2024-12-28T15:21:15.931995Z","shell.execute_reply":"2024-12-28T15:21:15.959324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_combined.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:15.961936Z","iopub.execute_input":"2024-12-28T15:21:15.962261Z","iopub.status.idle":"2024-12-28T15:21:15.999893Z","shell.execute_reply.started":"2024-12-28T15:21:15.962231Z","shell.execute_reply":"2024-12-28T15:21:15.99886Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Scale","metadata":{}},{"cell_type":"code","source":"numerical_columns = df_train_combined.select_dtypes(include=['float64']).columns\nnumerical_columns = numerical_columns[numerical_columns != TARGET]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:16.001098Z","iopub.execute_input":"2024-12-28T15:21:16.00143Z","iopub.status.idle":"2024-12-28T15:21:16.317313Z","shell.execute_reply.started":"2024-12-28T15:21:16.001403Z","shell.execute_reply":"2024-12-28T15:21:16.316426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:16.318241Z","iopub.execute_input":"2024-12-28T15:21:16.318546Z","iopub.status.idle":"2024-12-28T15:21:16.324784Z","shell.execute_reply.started":"2024-12-28T15:21:16.31852Z","shell.execute_reply":"2024-12-28T15:21:16.323595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop one-hot-columns\nnumerical_columns = numerical_columns.drop([\"Gender_Male\", \"Marital Status_Married\", \"Marital Status_Single\", \"Occupation_Self-Employed\", \"Occupation_Unemployed\", \"Smoking Status_Yes\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:16.325771Z","iopub.execute_input":"2024-12-28T15:21:16.326095Z","iopub.status.idle":"2024-12-28T15:21:16.340889Z","shell.execute_reply.started":"2024-12-28T15:21:16.326067Z","shell.execute_reply":"2024-12-28T15:21:16.339842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\n\ndf_train_combined[numerical_columns] = scaler.fit_transform(df_train_combined[numerical_columns])\ndf_test_combined[numerical_columns] = scaler.transform(df_test_combined[numerical_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:16.342081Z","iopub.execute_input":"2024-12-28T15:21:16.342495Z","iopub.status.idle":"2024-12-28T15:21:16.952775Z","shell.execute_reply.started":"2024-12-28T15:21:16.342462Z","shell.execute_reply":"2024-12-28T15:21:16.951503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Skewness","metadata":{}},{"cell_type":"code","source":"continuous_columns_train = ['Annual Income', 'Premium Amount']\ncontinuous_columns_test = ['Annual Income']\n\ntrain_skewness = df_train_combined[continuous_columns_train].apply(skew)\ntest_skewness = df_test_combined[continuous_columns_test].apply(skew)\n\nprint(\"Skewness for Training Dataset:\\n\")\ndisplay(train_skewness)\n\nprint(\"\\nSkewness for Test Dataset:\\n\")\ndisplay(test_skewness)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:16.953869Z","iopub.execute_input":"2024-12-28T15:21:16.954141Z","iopub.status.idle":"2024-12-28T15:21:17.039131Z","shell.execute_reply.started":"2024-12-28T15:21:16.954116Z","shell.execute_reply":"2024-12-28T15:21:17.03817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Log-transform skewed features\ndf_train_combined['Annual Income'] = np.log1p(df_train_combined['Annual Income'])\ndf_test_combined['Annual Income'] = np.log1p(df_test_combined['Annual Income'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.039873Z","iopub.execute_input":"2024-12-28T15:21:17.040152Z","iopub.status.idle":"2024-12-28T15:21:17.093041Z","shell.execute_reply.started":"2024-12-28T15:21:17.040126Z","shell.execute_reply":"2024-12-28T15:21:17.091843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = df_train_combined.drop([TARGET], axis=1)\ny_train = df_train_combined[TARGET]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.378078Z","iopub.execute_input":"2024-12-28T15:21:17.378401Z","iopub.status.idle":"2024-12-28T15:21:17.524686Z","shell.execute_reply.started":"2024-12-28T15:21:17.378372Z","shell.execute_reply":"2024-12-28T15:21:17.523867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train_log = np.log1p(y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.525507Z","iopub.execute_input":"2024-12-28T15:21:17.525788Z","iopub.status.idle":"2024-12-28T15:21:17.552646Z","shell.execute_reply.started":"2024-12-28T15:21:17.525762Z","shell.execute_reply":"2024-12-28T15:21:17.551823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.553529Z","iopub.execute_input":"2024-12-28T15:21:17.553829Z","iopub.status.idle":"2024-12-28T15:21:17.561458Z","shell.execute_reply.started":"2024-12-28T15:21:17.55379Z","shell.execute_reply":"2024-12-28T15:21:17.560149Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    param = {\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"boosting_type\": \"gbdt\",\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.2),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 1000),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 15),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 300),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 10, 100),\n        \"subsample\": trial.suggest_uniform(\"subsample\", 0.5, 1.0),\n        \"colsample_bytree\": trial.suggest_uniform(\"colsample_bytree\", 0.5, 1.0),\n        \"reg_alpha\": trial.suggest_loguniform(\"reg_alpha\", 1e-3, 10.0),\n        \"reg_lambda\": trial.suggest_loguniform(\"reg_lambda\", 1e-3, 10.0),\n        \"device\": \"cpu\",\n        \"verbose\": -1,\n    }\n\n    X_train_split, X_val, y_train_split, y_val = train_test_split(X_train, y_train_log, test_size=0.2, random_state=9)\n\n    model = lgb.LGBMRegressor(**param)\n    model.fit(\n        X_train_split, y_train_split,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        callbacks=[early_stopping(stopping_rounds=8, verbose=False)]\n    )\n\n    y_pred = model.predict(X_val)\n    rmsle = mean_squared_log_error(y_val, y_pred) ** 0.5\n    return rmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.562322Z","iopub.execute_input":"2024-12-28T15:21:17.562608Z","iopub.status.idle":"2024-12-28T15:21:17.575978Z","shell.execute_reply.started":"2024-12-28T15:21:17.562581Z","shell.execute_reply":"2024-12-28T15:21:17.575092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:21:17.577038Z","iopub.execute_input":"2024-12-28T15:21:17.577406Z","iopub.status.idle":"2024-12-28T15:33:08.807347Z","shell.execute_reply.started":"2024-12-28T15:21:17.577359Z","shell.execute_reply":"2024-12-28T15:33:08.8065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\nbest_params[\"objective\"] = \"regression\"\nbest_params[\"metric\"] = \"rmse\"\nbest_params[\"device\"] = \"cpu\"\n\n# Train the final model\nfinal_model = lgb.LGBMRegressor(**best_params)\nfinal_model.fit(X_train, y_train_log)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:33:08.808247Z","iopub.execute_input":"2024-12-28T15:33:08.808525Z","iopub.status.idle":"2024-12-28T15:33:25.373403Z","shell.execute_reply.started":"2024-12-28T15:33:08.808499Z","shell.execute_reply":"2024-12-28T15:33:25.372152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train_pred = final_model.predict(X_train)\ntrain_rmsle = mean_squared_log_error(y_train_log, y_train_pred) ** 0.5\nprint(\"RMSLE on Training Data:\", train_rmsle)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:33:25.374437Z","iopub.execute_input":"2024-12-28T15:33:25.374771Z","iopub.status.idle":"2024-12-28T15:33:35.402639Z","shell.execute_reply.started":"2024-12-28T15:33:25.374743Z","shell.execute_reply":"2024-12-28T15:33:35.401666Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"y_test_pred = np.expm1(final_model.predict(df_test_combined))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:39:02.62527Z","iopub.execute_input":"2024-12-28T15:39:02.625936Z","iopub.status.idle":"2024-12-28T15:39:09.340699Z","shell.execute_reply.started":"2024-12-28T15:39:02.62589Z","shell.execute_reply":"2024-12-28T15:39:09.339596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission[\"Premium Amount\"] = y_test_pred\nsample_submission.to_csv(\"submission.csv\", index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:39:09.341975Z","iopub.execute_input":"2024-12-28T15:39:09.342256Z","iopub.status.idle":"2024-12-28T15:39:10.926058Z","shell.execute_reply.started":"2024-12-28T15:39:09.342203Z","shell.execute_reply":"2024-12-28T15:39:10.925049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}