{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:29.21958Z","iopub.execute_input":"2024-12-26T02:54:29.219895Z","iopub.status.idle":"2024-12-26T02:54:29.537703Z","shell.execute_reply.started":"2024-12-26T02:54:29.219871Z","shell.execute_reply":"2024-12-26T02:54:29.536986Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Separate notebook for EDA","metadata":{}},{"cell_type":"code","source":"!pip install -U scikit-learn\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import root_mean_squared_log_error\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import cross_val_score, train_test_split, KFold, StratifiedKFold, GridSearchCV, RandomizedSearchCV\nfrom xgboost import XGBRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:29.538813Z","iopub.execute_input":"2024-12-26T02:54:29.539342Z","iopub.status.idle":"2024-12-26T02:54:42.57761Z","shell.execute_reply.started":"2024-12-26T02:54:29.539305Z","shell.execute_reply":"2024-12-26T02:54:42.576624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_train_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\nraw_test_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:42.579372Z","iopub.execute_input":"2024-12-26T02:54:42.580085Z","iopub.status.idle":"2024-12-26T02:54:51.274992Z","shell.execute_reply.started":"2024-12-26T02:54:42.580057Z","shell.execute_reply":"2024-12-26T02:54:51.274297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(raw_train_df.count())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:51.276301Z","iopub.execute_input":"2024-12-26T02:54:51.276557Z","iopub.status.idle":"2024-12-26T02:54:51.810803Z","shell.execute_reply.started":"2024-12-26T02:54:51.276536Z","shell.execute_reply":"2024-12-26T02:54:51.809913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Mean imputation\ndef mean_imputation(df, columns):\n    for i in columns:\n        df[i] = df[i].fillna(df[i].mean())\n    return df\n\ndef mode_imputation(df, columns):\n    for i in columns:\n        df[i] = df[i].fillna(df[i].value_counts().idxmax())\n    return df\n\ndef median_imputation(df, columns):\n    for i in columns:\n        df[i] = df[i].fillna(df[i].median())\n    return df\n\ndef impute_num_columns(df, columns):\n    for i in columns:\n        df[i] = df[i].fillna(-0.9)\n    return df\n\ndef impute_string_columns(df, columns):\n    for i in columns:\n        df[i] = df[i].fillna(\"Unknown\")\n    return df\n\ndef process_imputations(df):\n    df = df[['Age', 'Gender', 'Annual Income', 'Marital Status',\n       'Number of Dependents', 'Education Level', 'Health Score',\n       'Location', 'Policy Type', 'Vehicle Age',\n       'Credit Score', 'Insurance Duration',\n       'Customer Feedback', 'Smoking Status', 'Exercise Frequency',\n       'Property Type', 'Occupation', 'Policy Start Date']]\n    #Filling missing values with Unknown and -1\n    #num_columns = [\"Age\",\"Number of Dependents\", \"Annual Income\",\"Insurance Duration\",\"Health Score\",\"Vehicle Age\",\"Credit Score\"]\n    #df = impute_num_columns(df, num_columns)\n    df['Annual Income'] = np.log1p(df['Annual Income'])\n    #string_columns = ['Occupation','Marital Status', 'Education Level', 'Location', 'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type', 'Number of Dependents']\n    #df = impute_string_columns(df, string_columns)\n\n    #Finding cyclical and time series patterns\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['day_of_week'] = df['Policy Start Date'].dt.dayofweek\n    df['day_sin'] = np.sin(2.0 * np.pi * df['day_of_week'] / 7.0)\n    df['day_cos'] = np.cos(2.0 * np.pi * df['day_of_week'] / 7.0)\n    \n    df['day_of_year'] = df['Policy Start Date'].dt.dayofyear\n    df['year_day_sin'] = np.sin(2.0 * np.pi * df['day_of_year'] / 365.0)\n    df['year_day_cos'] = np.cos(2.0 * np.pi * df['day_of_year'] / 365.0)\n    \n    df['month_sin'] = np.sin(2.0 * np.pi * df['Month'] / 12.0)\n    df['month_cos'] = np.cos(2.0 * np.pi * df['Month'] / 12.0)\n\n    df[\"seconds_since_1970\"] = df['Policy Start Date'].astype(\"int64\") // 10**9\n\n    df['contract length'] = pd.cut(\n        df[\"Insurance Duration\"].fillna(99),  \n        bins=[-float('inf'), 1, 3, float('inf')],  \n        labels=[0, 1, 2]  \n    ).astype(int)\n\n    df['Group']=(df['Year']-2020)*48+df['Month']*4+df['Day']//7\n    \n    df.drop('Policy Start Date', axis=1, inplace=True)\n    print(df.columns)\n    return df\n\ndef one_hot_encoding_func(df, categorical_columns):\n    # Initialize OneHotEncoder\n    encoder = OneHotEncoder(sparse_output=False)\n    encoded_train_df = pd.DataFrame(df)\n    # Fit and transform the categorical columns\n    one_hot_encoded = encoder.fit_transform(df[categorical_columns])\n\n    \n    # Create a DataFrame with the encoded columns\n    one_hot_df = pd.DataFrame(one_hot_encoded, \n                              columns=encoder.get_feature_names_out(categorical_columns))\n    train_columns=encoder.get_feature_names_out(categorical_columns)\n    # Concatenate the one-hot encoded columns with the original DataFrame\n    encoded_train_df = pd.concat([encoded_train_df.drop(categorical_columns, axis=1), one_hot_df], axis=1)\n    \n    #print(f\"One-Hot Encoded Data using Scikit-Learn:\\n{encoded_train_df}\\n\")\n    return encoded_train_df\n\ndef encoding_func(df):\n    gender = {\"Male\":0, \"Female\":1}\n    education_level = {\"High School\":0, \"Bachelor's\":1, \"Master's\":2, \"PhD\":3}\n    policy = {'Basic':0, 'Comprehensive':1, 'Premium':2}\n    exercise = {'Rarely':0, 'Daily':1, 'Weekly':2, 'Monthly': 3}\n    feedback = {'Poor':0, 'Average':1, 'Good':2, \"Unknown\": 1}\n    smoking = {\"Yes\":1, \"No\":0}\n    print(df.head)\n    df['Gender'] = df['Gender'].map(gender)\n    df['Smoking Status'] = df['Smoking Status'].map(smoking)\n    df['Education Level'] = df['Education Level'].map(education_level)\n    df['Policy Type'] = df['Policy Type'].map(policy)\n    df['Exercise Frequency'] = df['Exercise Frequency'].map(exercise)\n    df['Customer Feedback'] = df['Customer Feedback'].map(feedback)\n    return df\n\n#Some new features\ndef add_new_features(df):\n    df['Income to Dependents Ratio'] = df['Annual Income'] / (df['Number of Dependents'].fillna(0) + 1)\n    df['Income_per_Dependent'] = df['Annual Income'] / (df['Number of Dependents'] + 1)\n    df['CreditScore_InsuranceDuration'] = df['Credit Score'] * df['Insurance Duration']\n    df['Health_Risk_Score'] = df['Smoking Status'].apply(lambda x: 1 if x == 'Smoker' else 0) + \\\n                                df['Exercise Frequency'].apply(lambda x: 1 if x == 'Low' else (0.5 if x == 'Medium' else 0)) + \\\n                                (100 - df['Health Score']) / 20\n    df['Credit_Health_Score'] = df['Credit Score'] * df['Health Score']\n    df['Health_Age_Interaction'] = df['Health Score'] * df['Age']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:51.811851Z","iopub.execute_input":"2024-12-26T02:54:51.812169Z","iopub.status.idle":"2024-12-26T02:54:51.827334Z","shell.execute_reply.started":"2024-12-26T02:54:51.812135Z","shell.execute_reply":"2024-12-26T02:54:51.826324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Preprocessing Raw trained data\n#Columns removed: 'id', 'Occupation', 'Policy Start Date'\nprint(raw_train_df.count())\nprocessed_train_df = pd.DataFrame(raw_train_df)\nprocessed_test_df = pd.DataFrame(raw_test_df)\n#OneHotEncoding\ncategorical_columns = ['Gender', 'Marital Status', 'Education Level', 'Location', 'Policy Type', 'Customer Feedback', 'Smoking Status', 'Exercise Frequency', 'Property Type', 'Occupation', 'day_of_week']\n\none_hot_column = ['Marital Status', 'Location','Occupation', 'Property Type']\npremium_amount = processed_train_df['Premium Amount']\nprocessed_train_df = process_imputations(processed_train_df)\nprocessed_train_df = add_new_features(processed_train_df)\nprocessed_train_df = encoding_func(processed_train_df)\nprocessed_train_df = one_hot_encoding_func(processed_train_df, one_hot_column)\nprocessed_test_df = process_imputations(processed_test_df)\nprocessed_test_df = add_new_features(processed_test_df)\nprocessed_test_df = encoding_func(processed_test_df)\nprocessed_test_df = one_hot_encoding_func(processed_test_df, one_hot_column)\nprint(processed_train_df.columns)\nprint(processed_test_df.columns)\nprocessed_train_df = processed_train_df.drop(['Marital Status_nan','Occupation_nan'], axis=1)\nprocessed_test_df = processed_test_df.drop(['Marital Status_nan','Occupation_nan'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:51.828314Z","iopub.execute_input":"2024-12-26T02:54:51.828622Z","iopub.status.idle":"2024-12-26T02:54:59.917174Z","shell.execute_reply.started":"2024-12-26T02:54:51.828592Z","shell.execute_reply":"2024-12-26T02:54:59.916222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_train_df = pd.DataFrame(processed_train_df)\nencoded_test_df = pd.DataFrame(processed_test_df)\nprint(encoded_train_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:59.918117Z","iopub.execute_input":"2024-12-26T02:54:59.918398Z","iopub.status.idle":"2024-12-26T02:54:59.923073Z","shell.execute_reply.started":"2024-12-26T02:54:59.918362Z","shell.execute_reply":"2024-12-26T02:54:59.922258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Train Test Split\nX = encoded_train_df\ny = np.log1p(premium_amount)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:54:59.925113Z","iopub.execute_input":"2024-12-26T02:54:59.925445Z","iopub.status.idle":"2024-12-26T02:55:00.446523Z","shell.execute_reply.started":"2024-12-26T02:54:59.925414Z","shell.execute_reply":"2024-12-26T02:55:00.445651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\n\nxgb_model = xgb.XGBRegressor(verbosity = 0, n_estimators = 1500, \n                             learning_rate=0.01, device=\"gpu\",\n                             objective = 'reg:squarederror', gamma=0.3)\nxgb_model.fit(X_train, y_train, eval_set=[(X_test, y_test)])\npredictions = xgb_model.predict(encoded_test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:55:00.447627Z","iopub.execute_input":"2024-12-26T02:55:00.447884Z","iopub.status.idle":"2024-12-26T02:56:12.539353Z","shell.execute_reply.started":"2024-12-26T02:55:00.447862Z","shell.execute_reply":"2024-12-26T02:56:12.538491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"n_splits=5\n# LightGBM Parameters\nlgb_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'learning_rate': 0.01,\n    'n_estimators': 1000,\n    'early_stopping_rounds': 60,\n    'random_state': 42\n}\n\n# CatBoost Parameters\ncb_params = {\n    'iterations':3000,\n    'learning_rate':0.03,\n    'depth':6,\n    'eval_metric':\"RMSE\",\n    'random_seed':42,\n    'l2_leaf_reg': 0.7,\n    'early_stopping_rounds': 100\n}\n\n# XGBRegressor Parameters\nxgb_params = {\n    'learning_rate': 0.012605165083674704, 'n_estimators': 709, 'max_depth': 8, 'min_child_weight': 67, 'reg_alpha': 0.1519202257366929, 'reg_lambda': 0.6133455919201877, 'gamma': 0.3643339318225859\n    }\n\n# Initialize predictions\nlgb_predictions = np.zeros(len(encoded_test_df))\ncb_predictions = np.zeros(len(encoded_test_df))\nxgb_predictions = np.zeros(len(encoded_test_df))\n\ndf_test = encoded_test_df\n\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\nX = encoded_train_df\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n    print(f\"Fold {fold + 1}\")\n    X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n    \n    # LightGBM\n    lgb_model = lgb.LGBMRegressor(**lgb_params)\n    lgb_model.fit(X_train, y_train, eval_set=[(X_val, y_val)])\n    lgb_val_pred = lgb_model.predict(X_val)\n    print(f\"LightGBM Fold {fold + 1} RMSLE: {root_mean_squared_log_error(y_val, lgb_val_pred):.4f}\")\n    lgb_predictions += lgb_model.predict(df_test) / n_splits\n\n    # CatBoost\n    cat_features = [X.columns.get_loc(col) for col in X.select_dtypes(include=['category', 'object']).columns]\n    cb_model = CatBoostRegressor(**cb_params)\n    cb_model.fit(X_train, y_train, eval_set=(X_val, y_val), cat_features=cat_features)\n    cb_val_pred = cb_model.predict(X_val)\n    print(f\"CatBoost Fold {fold + 1} RMSLE: {root_mean_squared_log_error(y_val, cb_val_pred):.4f}\")\n    cb_predictions += cb_model.predict(df_test) / n_splits\n\n    #XGBoost\n    xgb_model = XGBRegressor(**xgb_params)\n    xgb_model.fit(X_train, y_train, eval_set=[(X_val, y_val)])\n    xgb_val_pred = xgb_model.predict(X_val)\n    print(f\"XGBM Fold {fold + 1} RMSLE: {root_mean_squared_log_error(y_val, xgb_val_pred):.4f}\")\n    xgb_predictions += xgb_model.predict(df_test) / n_splits \n    \n\n# Print final RMSLE for both models\n#print(f\"LightGBM Final RMSLE: {root_mean_squared_log_error(y, lgb_model.predict(X)):.4f}\")\nprint(f\"CatBoost Final RMSLE: {root_mean_squared_log_error(y, cb_model.predict(X)):.4f}\")\n#print(f\"XGBoost Final RMSLE: {root_mean_squared_log_error(y, xgb_model.predict(X)):.4f}\")\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:56:12.540333Z","iopub.execute_input":"2024-12-26T02:56:12.540668Z","iopub.status.idle":"2024-12-26T02:56:12.548706Z","shell.execute_reply.started":"2024-12-26T02:56:12.540634Z","shell.execute_reply":"2024-12-26T02:56:12.547634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#predictions = (cb_predictions)\nfinal_test_df = pd.DataFrame(raw_test_df)\nfinal_test_df['Premium Amount'] = np.expm1(predictions)\n\nfinal_test_df[['id', 'Premium Amount']].to_csv('/kaggle/working/submissions2.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T02:56:12.549848Z","iopub.execute_input":"2024-12-26T02:56:12.550393Z","iopub.status.idle":"2024-12-26T02:56:13.551729Z","shell.execute_reply.started":"2024-12-26T02:56:12.550184Z","shell.execute_reply":"2024-12-26T02:56:13.551014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}