{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# IMPORT LIBRARIES\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom sklearn.metrics import make_scorer, mean_squared_log_error \nimport xgboost as xgb\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import plot_importance\nfrom catboost import CatBoostRegressor, Pool\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.impute import KNNImputer\nfrom lightgbm import LGBMRegressor\nfrom sklearn.preprocessing import OrdinalEncoder\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:41:02.392933Z","iopub.execute_input":"2024-12-01T18:41:02.393261Z","iopub.status.idle":"2024-12-01T18:41:02.39852Z","shell.execute_reply.started":"2024-12-01T18:41:02.393217Z","shell.execute_reply":"2024-12-01T18:41:02.39763Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# UPLOAD DATA","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsample_submission = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:41:02.688436Z","iopub.execute_input":"2024-12-01T18:41:02.688733Z","iopub.status.idle":"2024-12-01T18:41:07.311025Z","shell.execute_reply.started":"2024-12-01T18:41:02.688706Z","shell.execute_reply":"2024-12-01T18:41:07.31011Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DATA ANALYSIS","metadata":{}},{"cell_type":"code","source":"df_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:41:07.312919Z","iopub.execute_input":"2024-12-01T18:41:07.31329Z","iopub.status.idle":"2024-12-01T18:41:07.756303Z","shell.execute_reply.started":"2024-12-01T18:41:07.313232Z","shell.execute_reply":"2024-12-01T18:41:07.755323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:41:07.757656Z","iopub.execute_input":"2024-12-01T18:41:07.758015Z","iopub.status.idle":"2024-12-01T18:41:08.312268Z","shell.execute_reply.started":"2024-12-01T18:41:07.757976Z","shell.execute_reply":"2024-12-01T18:41:08.311397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:41:08.313895Z","iopub.execute_input":"2024-12-01T18:41:08.314168Z","iopub.status.idle":"2024-12-01T18:41:08.848506Z","shell.execute_reply.started":"2024-12-01T18:41:08.314141Z","shell.execute_reply":"2024-12-01T18:41:08.847642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# DATA PREPROCESSING","metadata":{}},{"cell_type":"code","source":"train = df_train.copy()\ntest = df_test.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:56:39.695346Z","iopub.execute_input":"2024-12-01T18:56:39.696012Z","iopub.status.idle":"2024-12-01T18:56:40.029743Z","shell.execute_reply.started":"2024-12-01T18:56:39.695979Z","shell.execute_reply":"2024-12-01T18:56:40.028776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**FILLING NULL VALUES**","metadata":{}},{"cell_type":"code","source":"def fill_important_nan_values(df,important_columns):\n    print(\"HERE1\")\n    params = {\n    \"random_state\" : 42,\n    \"verbose\" : -1,\n    \"device\":\"gpu\"\n}\n    dff = df.copy()\n    dff1 = df.copy()\n    dff2 = df.copy()\n\n    if \"Premium Amount\" in df.columns:\n        dff1.drop(columns = [\"Premium Amount\"])\n        dff2.drop(columns = [\"Premium Amount\"])\n\n    print(\"PREDICT SCORE IS STARTED\")\n    #PREDICT CREDIT SCORE\n    dff1['Annual Income'].fillna(dff1['Annual Income'].median(),inplace = True)\n    train_data = dff1[dff1['Credit Score'].notnull()]\n    test_data = dff1[dff1['Credit Score'].isnull()]\n    model = LGBMRegressor(**params)\n    X = train_data.drop(columns = [\"Credit Score\"])\n    y = train_data[\"Credit Score\"]\n    model.fit(X, y)\n    dff1.loc[test_data.index, 'Credit Score'] = model.predict(test_data[X.columns])\n    print(\"PREDICT ANNUAL INCOME IS STARTED\")\n    #PREDICT ANNUAL INCOME \n    dff2['Credit Score'].fillna(dff2['Credit Score'].median(), inplace=True)\n    train_data = dff2[dff2['Annual Income'].notnull()]\n    test_data = dff2[dff2['Annual Income'].isnull()]\n    model = LGBMRegressor(**params)\n    X = train_data.drop(columns=[\"Annual Income\"])\n    y = train_data[\"Annual Income\"]\n    model.fit(X, y)\n    dff2.loc[test_data.index, 'Annual Income'] = model.predict(test_data[X.columns])\n    \n    # MERGE COLUMNS BACK INTO dff\n    dff['Credit Score'] = dff1['Credit Score']\n    dff['Annual Income'] = dff2['Annual Income']\n\n    return dff\n            \ndef fill_null_values(df,important_columns):\n    \"\"\"\n    Fills missing (NaN) values in the DataFrame for non-important columns.\n    :param df: DataFrame to process\n    :return: DataFrame with filled missing values\n    \"\"\"\n        \n    for col in df.columns:\n        if col not in important_columns:\n            if df[col].isnull().sum() > 0:  # Check if the column has NaN values\n                if df[col].dtype in ['float64', 'int64']:  # Numerical columns\n                    # Check for skewness\n                    if df[col].skew() > 1 or df[col].skew() < -1:  # Highly skewed\n                        df[col] = df[col].fillna(df[col].median())  # Fill with median\n                    else:\n                        df[col] = df[col].fillna(df[col].mean())  # Fill with mean\n                else:  # Categorical columns\n                    df[col] = df[col].fillna(df[col].mode()[0])  # Fill with mode\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:48:44.273895Z","iopub.execute_input":"2024-12-01T18:48:44.274551Z","iopub.status.idle":"2024-12-01T18:48:44.28478Z","shell.execute_reply.started":"2024-12-01T18:48:44.274521Z","shell.execute_reply":"2024-12-01T18:48:44.283941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**CONVERTING CATEGORICAL TO NUMERICAL**","metadata":{}},{"cell_type":"code","source":"def preprocess_data(df):\n    \"\"\"\n    Converts and encodes data for the DataFrame.\n    :param df: DataFrame to process\n    :return: Preprocessed DataFrame\n    \"\"\"\n    important_columns = ['Annual Income', 'Credit Score']\n\n    \n    df = fill_null_values(df,important_columns) # fill null values other than important_columns\n    \n    df['Gender'] = df['Gender'].apply(lambda x: 1 if x == 'Female' else 0)\n\n    # Ordinal encoding for Marital Status\n    marital_status_encoder = OrdinalEncoder()\n    df['Marital Status'] = marital_status_encoder.fit_transform(df[['Marital Status']])\n\n    # Ordinal encoding for Education Level\n    education_level_order = ['High School', \"Bachelor's\", \"Master's\", 'PhD']\n    education_level_encoder = OrdinalEncoder(categories=[education_level_order])\n    df['Education Level'] = education_level_encoder.fit_transform(df[['Education Level']])\n\n    # Ordinal encoding for Occupation, Location, Policy Type\n    ordinal_encoder = OrdinalEncoder()\n    for col in ['Occupation', 'Location', 'Policy Type']:\n        df[col] = ordinal_encoder.fit_transform(df[[col]])\n    # Convert to pandas datetime\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    \n    # Encode datetime as numerical value (days since a reference date)\n    reference_date = pd.Timestamp('1970-01-01')  # UNIX epoch\n    df['numerical_date'] = (df['Policy Start Date'] - reference_date).dt.total_seconds() / (24 * 3600)  # Days since reference date\n    # Drop unnecessary columns\n    if 'Policy Start Date' in df.columns:\n        df.drop(columns=['Policy Start Date'], inplace=True)\n    if 'id' in df.columns:\n        df.drop(columns=['id'], inplace=True)\n\n    # Ordinal encoding for Customer Feedback\n    customer_feedback_order = [\"Poor\", \"Average\", \"Good\"]\n    customer_feedback_encoder = OrdinalEncoder(categories=[customer_feedback_order])\n    df['Customer Feedback'] = customer_feedback_encoder.fit_transform(df[['Customer Feedback']])\n\n    # Convert Smoking Status to binary\n    df['Smoking Status'] = df['Smoking Status'].apply(lambda x: 1 if x == 'Yes' else 0)\n\n    # Ordinal encoding for Exercise Frequency\n    exercise_frequency_order = [\"Rarely\", \"Monthly\", \"Weekly\", \"Daily\"]\n    exercise_frequency_encoder = OrdinalEncoder(categories=[exercise_frequency_order])\n    df['Exercise Frequency'] = exercise_frequency_encoder.fit_transform(df[['Exercise Frequency']])\n\n    # Ordinal encoding for Property Type\n    df['Property Type'] = ordinal_encoder.fit_transform(df[['Property Type']])\n    \n    df = fill_important_nan_values(df,important_columns)\n    \n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:48:45.591925Z","iopub.execute_input":"2024-12-01T18:48:45.592708Z","iopub.status.idle":"2024-12-01T18:48:45.601484Z","shell.execute_reply.started":"2024-12-01T18:48:45.592674Z","shell.execute_reply":"2024-12-01T18:48:45.600555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ids = test['id']\ntrain = preprocess_data(train)\ntest = preprocess_data(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:48:48.559779Z","iopub.execute_input":"2024-12-01T18:48:48.560109Z","iopub.status.idle":"2024-12-01T18:49:07.323049Z","shell.execute_reply.started":"2024-12-01T18:48:48.560081Z","shell.execute_reply":"2024-12-01T18:49:07.321857Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MODEL EVALUATION","metadata":{}},{"cell_type":"markdown","source":"*RMSLE*","metadata":{}},{"cell_type":"code","source":"\n# Define RMSLE Scorer\ndef rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\nrmsle_scorer = make_scorer(\n    lambda y_true, y_pred: np.sqrt(mean_squared_log_error(y_true, np.clip(y_pred, 0, None))), \n    greater_is_better=False\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:49:12.970947Z","iopub.execute_input":"2024-12-01T18:49:12.971297Z","iopub.status.idle":"2024-12-01T18:49:12.976336Z","shell.execute_reply.started":"2024-12-01T18:49:12.971264Z","shell.execute_reply":"2024-12-01T18:49:12.975271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = ['Marital Status',\n 'Education Level',\n 'Occupation',\n 'Location',\n 'Policy Type',\n 'Customer Feedback',\n 'Exercise Frequency',\n 'Property Type']\n\nxgb_params = {\n\"n_estimators\":2000,\n    \"learning_rate\":0.1,\n    \"max_depth\":5,\n    \"subsample\":0.8,\n    \"colsample_bytree\":0.8,\n    \"random_state\":42,\n    \"objective\":'reg:absoluteerror',\n    \"tree_method\":'hist',  # Use histogram-based method\n    \"device\":'cuda',       # Specify GPU usage\n}\n\n# Define CatBoost Parameters\ncb_params = {\n    'iterations': 1000,\n    'task_type': 'GPU',  # Enable GPU\n    'cat_features': cat_cols,  # Specify categorical features\n    'random_seed': 42,\n    'verbose': 100, \n    \"loss_function\":\"MAE\"\n}\nlgb_params = {\n    'objective': 'regression',          # Task type (e.g., regression, binary)\n    'metric': 'mae',                   # Evaluation metric (e.g., rmse, mae)\n    'boosting_type': 'gbdt',            # Gradient Boosting Decision Tree\n    'learning_rate': 0.05,              # Learning rate (reduce if more iterations)\n    'num_leaves': 31,                   # Maximum leaves in one tree\n    'max_depth': -1,                    # Unlimited depth (-1), or set it to a reasonable value\n    'min_data_in_leaf': 20,             # Minimum data points per leaf (reduce for smaller datasets)\n    'feature_fraction': 0.8,            # Fraction of features to use per iteration\n    'bagging_fraction': 0.8,            # Fraction of data to use for bagging\n    'bagging_freq': 5,                  # Perform bagging every 5 iterations\n    'lambda_l1': 0.1,                   # L1 regularization to prevent overfitting\n    'lambda_l2': 0.2,                   # L2 regularization to prevent overfitting\n    'max_bin': 255,                     # Maximum number of bins (increase for high-cardinality features)\n    'n_estimators': 1000,               # Total number of boosting iterations\n    'early_stopping_round': 50,         # Early stopping rounds for validation\n    'verbose': -1,\n    \"device\" : \"gpu\"\n}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:49:13.422637Z","iopub.execute_input":"2024-12-01T18:49:13.423449Z","iopub.status.idle":"2024-12-01T18:49:13.429917Z","shell.execute_reply.started":"2024-12-01T18:49:13.423413Z","shell.execute_reply":"2024-12-01T18:49:13.429087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DO_CV_XGB = False\nDO_CV_LGB = False\nDO_CV_CB = False\n\nmodel = None\n\nmode = \"xgb\" # CHOOSE MODE\n\nif \"cb\" in mode:\n    for col in cat_cols:\n        train[col] = train[col].fillna('Unknown').astype(str)\n        test[col] = test[col].fillna('Unknown').astype(str)\n    DO_CV_XGB = False\n    DO_CV_LGB = False\n    DO_CV_CB = True\n    \nelif \"xgb\" in mode:\n    DO_CV_XGB = True\n    DO_CV_LGB = False\n    DO_CV_CB = False\nelif \"lgb\" in mode:\n    DO_CV_XGB = False\n    DO_CV_LGB = True\n    DO_CV_CB = False\nelse:\n    raise \"model is not chosen\"\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:49:14.553087Z","iopub.execute_input":"2024-12-01T18:49:14.553807Z","iopub.status.idle":"2024-12-01T18:49:20.614534Z","shell.execute_reply.started":"2024-12-01T18:49:14.553774Z","shell.execute_reply":"2024-12-01T18:49:20.61381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#drop_columns = ['Age', 'Occupation', 'Marital Status', 'Policy Type', 'Location', 'Education Level', 'Vehicle Age', 'Insurance Duration', 'Smoking Status', 'Exercise Frequency', 'Property Type']\ndrop_columns = []\nX = train.drop(columns=['Premium Amount'] + drop_columns)  # Drop the target column from features\ny = train['Premium Amount']                # Target variable\nmodels = []  # Ensure this line has no non-breaking spaces\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:49:20.615937Z","iopub.execute_input":"2024-12-01T18:49:20.616204Z","iopub.status.idle":"2024-12-01T18:49:20.760862Z","shell.execute_reply.started":"2024-12-01T18:49:20.616179Z","shell.execute_reply":"2024-12-01T18:49:20.759658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DO_CV_XGB:\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n    fold_scores = []\n    \n    # Perform custom cross-validation\n    for fold, (train_idx, test_idx) in enumerate(kfold.split(X, y), 1):\n        # Split data\n        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n        \n        model = xgb.XGBRegressor(**xgb_params)\n\n        # Train the model\n        model.fit(X_train, y_train)\n        \n        # Make predictions\n        y_pred = np.clip(model.predict(X_test), 0, None)  # Ensure no negative predictions\n        \n        # Calculate RMSLE for the fold\n        fold_rmsle = rmsle(y_test, y_pred)\n        fold_scores.append(fold_rmsle)\n        # Print score for the current fold\n        print(f\"Fold {fold}: RMSLE = {fold_rmsle:.4f}\")\n\n        models.append(model)\n    \n    # Summary of results\n    print(f\"\\nMean RMSLE: {np.mean(fold_scores):.4f}\")\n    print(f\"Standard Deviation of RMSLE: {np.std(fold_scores):.4f}\")\n\n\n#RMSLE ==>  1.0717 (nan prediction in important_columns with lgb)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:49:20.762119Z","iopub.execute_input":"2024-12-01T18:49:20.762474Z","iopub.status.idle":"2024-12-01T18:49:20.770009Z","shell.execute_reply.started":"2024-12-01T18:49:20.762436Z","shell.execute_reply":"2024-12-01T18:49:20.768994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if DO_CV_CB:\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n\n    fold_scores = []\n    \n    for fold, (train_idx, test_idx) in enumerate(kfold.split(X, y), 1):\n        # Split data\n        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n        \n        # Initialize model with the current loss function\n        model = CatBoostRegressor(**cb_params)\n        \n        # Train the model\n        model.fit(X_train, y_train, cat_features=cat_cols, eval_set=(X_test, y_test), verbose=0)\n        \n        # Make predictions\n        y_pred = np.clip(model.predict(X_test), 0, None)  # Ensure no negative predictions\n        # Evaluate using RMSLE\n        fold_rmsle = rmsle(y_test, y_pred)\n        fold_scores.append(fold_rmsle)\n        \n        print(f\"  Fold {fold}: RMSLE = {fold_rmsle:.4f}\")\n\n        models.append(model)\n\n    mean_rmsle = np.mean(fold_scores)\n    std_rmsle = np.std(fold_scores)\n    print(f\"-> Mean RMSLE: {mean_rmsle:.4f}, Std Dev: {std_rmsle:.4f}\")\n\n#MAE -----------------> 1.1099\n#Huber:delta=1.0 -----> 1.0722","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:58:04.807694Z","iopub.execute_input":"2024-12-01T18:58:04.808426Z","iopub.status.idle":"2024-12-01T18:59:06.465078Z","shell.execute_reply.started":"2024-12-01T18:58:04.808396Z","shell.execute_reply":"2024-12-01T18:59:06.463931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nif DO_CV_LGB:\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n    fold_scores = []\n    \n    # Perform custom cross-validation\n    for fold, (train_idx, test_idx) in enumerate(kfold.split(X, y), 1):\n        # Split data\n        X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]\n        \n        # Prepare LightGBM datasets\n        train_data = lgb.Dataset(X_train, label=y_train)\n        valid_data = lgb.Dataset(X_test, label=y_test, reference=train_data)\n        \n        # Train the LightGBM model\n        model = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets=[train_data, valid_data]\n        )\n        \n        # Make predictions\n        y_pred = np.clip(model.predict(X_test), 0, None)  # Ensure no negative predictions\n        \n        # Calculate RMSLE for the fold\n        fold_rmsle = rmsle(y_test, y_pred)\n        fold_scores.append(fold_rmsle)\n        \n        # Print score for the current fold\n        print(f\"Fold {fold}: RMSLE = {fold_rmsle:.4f}\")\n\n        # Append the model to the list\n        models.append(model)\n    \n    # Summary of results\n    print(f\"\\nMean RMSLE: {np.mean(fold_scores):.4f}\")\n    print(f\"Standard Deviation of RMSLE: {np.std(fold_scores):.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:52:14.957531Z","iopub.execute_input":"2024-12-01T18:52:14.958123Z","iopub.status.idle":"2024-12-01T18:52:14.96517Z","shell.execute_reply.started":"2024-12-01T18:52:14.958081Z","shell.execute_reply":"2024-12-01T18:52:14.96419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#xgb -> Mean RMSLE: 1.0816 ( no drop columns)\n#xgb -> Mean RMSLE: 1.0797 (with drop columns)\n#xgb -> Mean RMSLE: 1.0717 (filling nan values with lgb)\n#cb --> Mean RMSLE: 1.0713 (filling nan values with lgb)(number of dependents'a overfit etmiş)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:52:14.966228Z","iopub.execute_input":"2024-12-01T18:52:14.966539Z","iopub.status.idle":"2024-12-01T18:52:14.977753Z","shell.execute_reply.started":"2024-12-01T18:52:14.966504Z","shell.execute_reply":"2024-12-01T18:52:14.976926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = models[0]\nmodel.fit(X, y) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:53:55.62205Z","iopub.execute_input":"2024-12-01T18:53:55.622397Z","iopub.status.idle":"2024-12-01T18:54:37.389832Z","shell.execute_reply.started":"2024-12-01T18:53:55.622368Z","shell.execute_reply":"2024-12-01T18:54:37.38904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DO_PLOT_IMPORTANCE = True\nif DO_PLOT_IMPORTANCE:\n    if DO_CV_XGB:\n        # XGBoost Feature Importance\n        plt.figure(figsize=(10, 8))\n        plot_importance(model, importance_type='weight')  # Top 10 features\n        plt.title(\"XGBoost Feature Importance\")\n        plt.show()\n    elif DO_CV_LGB:\n        # LightGBM Feature Importance\n        feature_importances = model.feature_importance(importance_type='split')  # Split-based importance\n        feature_names = X.columns\n        importance_df = pd.DataFrame({\n            'Feature': feature_names,\n            'Importance': feature_importances\n        }).sort_values(by='Importance', ascending=False)\n        \n        # Plot the top 10 feature importances\n        plt.figure(figsize=(10, 8))\n        plt.barh(importance_df['Feature'][:10][::-1], importance_df['Importance'][:10][::-1])\n        plt.title(\"LightGBM Feature Importance\")\n        plt.xlabel(\"Importance\")\n        plt.ylabel(\"Feature\")\n        plt.tight_layout()\n        plt.show()\n    elif DO_CV_CB:\n        # CatBoost Feature Importance\n        feature_importances = model.get_feature_importance(Pool(X, y, cat_features=cat_cols))\n        feature_names = X.columns\n        importance_df = pd.DataFrame({\n            'Feature': feature_names,\n            'Importance': feature_importances\n        }).sort_values(by='Importance', ascending=False)\n        \n        # Plot the top 10 feature importances\n        plt.figure(figsize=(10, 8))\n        plt.barh(importance_df['Feature'][:10][::-1], importance_df['Importance'][:10][::-1])\n        plt.title(\"CatBoost Feature Importance\")\n        plt.xlabel(\"Importance\")\n        plt.ylabel(\"Feature\")\n        plt.tight_layout()\n        plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:54:37.391363Z","iopub.execute_input":"2024-12-01T18:54:37.391633Z","iopub.status.idle":"2024-12-01T18:54:41.651607Z","shell.execute_reply.started":"2024-12-01T18:54:37.391608Z","shell.execute_reply":"2024-12-01T18:54:41.65076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure 'Premium Amount' is not in the test dataset\ntest_features = test.drop(columns=['Premium Amount'] + drop_columns, errors='ignore')  # Remove the target column if present\n\n# Make predictions using the trained model\ntest['Premium Amount'] = model.predict(test_features)\n\n# Prepare the submission file\nsubmission = pd.DataFrame({\n    'id': test_ids,  # Use the saved 'id' column\n    'Premium Amount': test['Premium Amount']\n})\n\n# Save the submission file\nsubmission_file_path = 'submission.csv'\nsubmission.to_csv(submission_file_path, index=False)\n\nprint(f\"Submission file created successfully and saved to: {submission_file_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T18:54:41.653103Z","iopub.execute_input":"2024-12-01T18:54:41.653839Z","iopub.status.idle":"2024-12-01T18:54:44.116631Z","shell.execute_reply.started":"2024-12-01T18:54:41.653791Z","shell.execute_reply":"2024-12-01T18:54:44.115751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}