{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\nimport xgboost as xgb\nimport numpy as np\nimport pandas as pd\nfrom statistics import mean\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nimport optuna\nimport xgboost as xgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import mean_squared_log_error","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:39.134011Z","iopub.execute_input":"2024-12-23T16:37:39.134271Z","iopub.status.idle":"2024-12-23T16:37:40.101053Z","shell.execute_reply.started":"2024-12-23T16:37:39.134248Z","shell.execute_reply":"2024-12-23T16:37:40.097753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:40.105513Z","iopub.execute_input":"2024-12-23T16:37:40.106105Z","iopub.status.idle":"2024-12-23T16:37:46.283475Z","shell.execute_reply.started":"2024-12-23T16:37:40.106057Z","shell.execute_reply":"2024-12-23T16:37:46.28253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:46.285549Z","iopub.execute_input":"2024-12-23T16:37:46.285848Z","iopub.status.idle":"2024-12-23T16:37:46.310017Z","shell.execute_reply.started":"2024-12-23T16:37:46.285824Z","shell.execute_reply":"2024-12-23T16:37:46.309198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:46.311257Z","iopub.execute_input":"2024-12-23T16:37:46.311527Z","iopub.status.idle":"2024-12-23T16:37:46.331638Z","shell.execute_reply.started":"2024-12-23T16:37:46.311504Z","shell.execute_reply":"2024-12-23T16:37:46.33089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:46.332486Z","iopub.execute_input":"2024-12-23T16:37:46.332811Z","iopub.status.idle":"2024-12-23T16:37:46.88185Z","shell.execute_reply.started":"2024-12-23T16:37:46.332778Z","shell.execute_reply":"2024-12-23T16:37:46.881038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:46.882586Z","iopub.execute_input":"2024-12-23T16:37:46.882799Z","iopub.status.idle":"2024-12-23T16:37:47.258637Z","shell.execute_reply.started":"2024-12-23T16:37:46.88278Z","shell.execute_reply":"2024-12-23T16:37:47.257855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:47.259445Z","iopub.execute_input":"2024-12-23T16:37:47.259781Z","iopub.status.idle":"2024-12-23T16:37:47.855265Z","shell.execute_reply.started":"2024-12-23T16:37:47.259749Z","shell.execute_reply":"2024-12-23T16:37:47.854392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:47.85792Z","iopub.execute_input":"2024-12-23T16:37:47.858138Z","iopub.status.idle":"2024-12-23T16:37:48.20851Z","shell.execute_reply.started":"2024-12-23T16:37:47.85812Z","shell.execute_reply":"2024-12-23T16:37:48.207718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:48.209515Z","iopub.execute_input":"2024-12-23T16:37:48.20975Z","iopub.status.idle":"2024-12-23T16:37:48.750522Z","shell.execute_reply.started":"2024-12-23T16:37:48.209722Z","shell.execute_reply":"2024-12-23T16:37:48.749772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:48.751293Z","iopub.execute_input":"2024-12-23T16:37:48.751518Z","iopub.status.idle":"2024-12-23T16:37:49.107958Z","shell.execute_reply.started":"2024-12-23T16:37:48.7515Z","shell.execute_reply":"2024-12-23T16:37:49.107203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.hist('Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:49.108782Z","iopub.execute_input":"2024-12-23T16:37:49.109091Z","iopub.status.idle":"2024-12-23T16:37:49.427797Z","shell.execute_reply.started":"2024-12-23T16:37:49.10906Z","shell.execute_reply":"2024-12-23T16:37:49.426958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.hist('Annual Income')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:49.428551Z","iopub.execute_input":"2024-12-23T16:37:49.428773Z","iopub.status.idle":"2024-12-23T16:37:49.717175Z","shell.execute_reply.started":"2024-12-23T16:37:49.428753Z","shell.execute_reply":"2024-12-23T16:37:49.716319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def date_trans(df):\n    df['Policy Start Date']= pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['DayOfWeek'] = df['Policy Start Date'].dt.dayofweek\n    df.drop('Policy Start Date' , axis =1, inplace = True)\n    return df\n\ntrain_df = date_trans(train_df)\ntest_df = date_trans(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:49.717933Z","iopub.execute_input":"2024-12-23T16:37:49.71823Z","iopub.status.idle":"2024-12-23T16:37:50.927436Z","shell.execute_reply.started":"2024-12-23T16:37:49.718207Z","shell.execute_reply":"2024-12-23T16:37:50.926436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:50.928426Z","iopub.execute_input":"2024-12-23T16:37:50.928694Z","iopub.status.idle":"2024-12-23T16:37:50.932613Z","shell.execute_reply.started":"2024-12-23T16:37:50.928673Z","shell.execute_reply":"2024-12-23T16:37:50.931752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assume train and test are already loaded as DataFrames\nfor col in train_df.select_dtypes(include='object').columns:\n    train_df[col] = train_df[col].astype('category')\nfor col in test_df.select_dtypes(include='object').columns:\n    test_df[col] = test_df[col].astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:50.933505Z","iopub.execute_input":"2024-12-23T16:37:50.933797Z","iopub.status.idle":"2024-12-23T16:37:52.860087Z","shell.execute_reply.started":"2024-12-23T16:37:50.933747Z","shell.execute_reply":"2024-12-23T16:37:52.859428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error\nfrom xgboost import XGBRegressor\nfrom numpy import log1p, expm1\n\n# Set random seed\nnp.random.seed(42)\n\n# Assume train_df and test_df are your DataFrames\ndf = train_df\n\n# Features and target\nX = df.drop('Premium Amount', axis=1)\ny = df['Premium Amount']\n\n# Log transform the target\ny_log = np.log1p(y)\n\n# Stratify target by binning into discrete intervals\nbinner = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='uniform')\ny_binned_log = binner.fit_transform(y_log.values.reshape(-1, 1)).astype(int).ravel()\n\n# Stratified K-Fold\nn_splits = 5\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# Arrays to store predictions\noof_predictions = np.zeros(len(X))  # Out-of-fold predictions\ntest_predictions = np.zeros(len(X))  # Test predictions for each fold\n\n# Test set\nX_test = test_df\ntest_preds_per_fold = np.zeros((len(X_test), n_splits))  # Store test predictions per fold\nfold_oof_results = []  # To store ID, target, and OOF predictions\n\n# XGBoost Regressor parameters\nmodel_params = {'n_estimators': 7000,\n                'learning_rate': 0.0037894924041441663,\n                'max_depth': 9,\n                'min_child_weight': 7,\n                'gamma': 0.003166869962093635,\n                'subsample': 0.8491559357878403,\n                'colsample_bytree': 0.9931791435553496,\n                'reg_alpha': 8.030670352805062e-08,\n                'reg_lambda': 0.23939002451629704,\n                \"random_state\": 42,\n                'eval_metric': 'rmse',\n                \"objective\": \"reg:squarederror\",  # XGBoost objective for regression\n                \"tree_method\": \"hist\",  # Use GPU acceleration\n                \"device\": \"cuda\",\n                \"enable_categorical\": True  # Enable categorical handling if required\n               }\n\n# Cross-validation loop\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(X, y_binned_log)):\n    print(f\"Fold {fold + 1}/{n_splits}\")\n    \n    # Split data\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n    \n    # Model\n    model = XGBRegressor(**model_params, early_stopping_rounds=300)\n    model.fit(X_train, y_train,\n              eval_set=[(X_valid, y_valid)],\n              verbose=0)\n    \n    # Predictions (log transformed)\n    log_oof_preds = model.predict(X_valid)\n    \n    # Revert log transformation\n    oof_predictions[valid_idx] = np.expm1(log_oof_preds)\n\n    fold_rmsle = rmsle(y.iloc[valid_idx], oof_predictions[valid_idx])\n    print(f\"Fold {fold + 1} RMSLE: {fold_rmsle:.4f}\")\n    \n    # Store fold results\n    fold_result = pd.DataFrame({\n        'ID': X.index[valid_idx],\n        'Actual': y.iloc[valid_idx],\n        'OOF_Pred_XGB': oof_predictions[valid_idx],\n        'Fold': fold + 1\n    })\n    fold_oof_results.append(fold_result)\n    \n    # Test set predictions\n    log_test_preds = model.predict(X_test)\n    test_preds_per_fold[:, fold] = np.expm1(log_test_preds)\n\n# Combine fold results\noof_results_df = pd.concat(fold_oof_results, axis=0, ignore_index=True)\n\n# Average predictions on test data\nfinal_test_predictions = test_preds_per_fold.mean(axis=1)\n\n# Evaluate OOF predictions\noof_mse = mean_squared_error(y, oof_predictions)\noof_rmsle = np.sqrt(mean_squared_log_error(y, oof_predictions))\n\nprint(f\"OOF Mean Squared Error: {oof_mse:.4f}\")\nprint(f\"OOF Root Mean Squared Log Error: {oof_rmsle:.4f}\")\n\n# Output predictions\nprint(\"Final Test Predictions:\", final_test_predictions)\nprint(oof_results_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:37:52.8609Z","iopub.execute_input":"2024-12-23T16:37:52.861228Z","iopub.status.idle":"2024-12-23T16:43:25.491099Z","shell.execute_reply.started":"2024-12-23T16:37:52.861196Z","shell.execute_reply":"2024-12-23T16:43:25.490132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsub['Premium Amount'] = final_test_predictions\nsub.to_csv('submission.csv', index=False)\noof_results_df.to_csv('oof_xgb.csv',index = False)\nsub.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-23T16:43:25.491987Z","iopub.execute_input":"2024-12-23T16:43:25.492306Z","iopub.status.idle":"2024-12-23T16:43:29.743926Z","shell.execute_reply.started":"2024-12-23T16:43:25.492284Z","shell.execute_reply":"2024-12-23T16:43:29.743126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}