{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport time\nimport contextlib\nimport optuna\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\n\nfrom itertools import chain\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import mean_squared_log_error, make_scorer\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, BayesianRidge\nfrom sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer, SimpleImputer\nfrom sklearn.feature_selection import SelectFromModel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:16.607568Z","iopub.execute_input":"2024-12-13T10:41:16.608024Z","iopub.status.idle":"2024-12-13T10:41:19.877974Z","shell.execute_reply.started":"2024-12-13T10:41:16.607986Z","shell.execute_reply":"2024-12-13T10:41:19.876558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 51\nDEBUG = False\n\nif DEBUG:\n    N_TRIALS = 3\n    N_SPLITS = 3\n\nelse:\n    N_TRIALS = 25\n    N_SPLITS = 5\n\n\ndef rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))\n    \n\ndef identify_columns(df):\n\n    cat_cols = list(df.select_dtypes(include=[\"category\", \"object\", \"bool\"]).columns)\n    num_cols = list(df.select_dtypes(include=[\"int64\", \"float64\"]).columns)\n    all_cols = cat_cols + num_cols\n    \n    return cat_cols, num_cols, all_cols\n\n\ndef create_preprocessor(df, SEED=42):\n    cat_cols, num_cols, _ = identify_columns(df)\n    \n    if not cat_cols and not num_cols:\n        raise ValueError(\"No columns identified for preprocessing!\")\n\n    preprocessor = ColumnTransformer(transformers=[\n        (\"cat\", Pipeline([\n            (\"imputer\", SimpleImputer(strategy=\"most_frequent\")),\n            (\"encoder\", OneHotEncoder(drop=\"if_binary\", sparse_output=True))  # Switch to sparse\n        ]), cat_cols),\n        (\"num\", Pipeline([\n            (\"imputer\", IterativeImputer(estimator=BayesianRidge(), random_state=SEED, min_value=0)),\n            (\"scaler\", StandardScaler())\n        ]), num_cols)\n    ], remainder=\"passthrough\")\n    \n    return preprocessor, cat_cols, num_cols\n\n\ndef create_pipeline(df, model, SEED=42):\n    preprocessor, cat_cols, num_cols = create_preprocessor(df, SEED)\n    \n    pipeline = Pipeline(steps=[\n        (\"preprocessor\", preprocessor),\n        (\"model\", model)\n    ])\n    return pipeline\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:19.87983Z","iopub.execute_input":"2024-12-13T10:41:19.880564Z","iopub.status.idle":"2024-12-13T10:41:19.891892Z","shell.execute_reply.started":"2024-12-13T10:41:19.880467Z","shell.execute_reply":"2024-12-13T10:41:19.890729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\", index_col=\"id\", parse_dates=[\"Policy Start Date\"])\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\", index_col=\"id\", parse_dates=[\"Policy Start Date\"])\nsubmit = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:19.893364Z","iopub.execute_input":"2024-12-13T10:41:19.893717Z","iopub.status.idle":"2024-12-13T10:41:31.549819Z","shell.execute_reply.started":"2024-12-13T10:41:19.893683Z","shell.execute_reply":"2024-12-13T10:41:31.548614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:31.55216Z","iopub.execute_input":"2024-12-13T10:41:31.552549Z","iopub.status.idle":"2024-12-13T10:41:31.584024Z","shell.execute_reply.started":"2024-12-13T10:41:31.552486Z","shell.execute_reply":"2024-12-13T10:41:31.582751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_summary(df):\n    describe = df.describe().transpose()\n    summary = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary[\"MissingValues\"] = df.isna().sum()\n    summary[\"UniqueValues\"] = df.nunique()\n    summary[\"Value_1\"] = df.iloc[0]\n    summary[\"Value_2\"] = df.iloc[1]\n    summary[\"Value_3\"] = df.iloc[2]\n    summary = pd.concat([summary, describe], axis=1)\n    \n    return summary\n\ncreate_summary(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:31.585518Z","iopub.execute_input":"2024-12-13T10:41:31.585976Z","iopub.status.idle":"2024-12-13T10:41:33.748735Z","shell.execute_reply.started":"2024-12-13T10:41:31.585927Z","shell.execute_reply":"2024-12-13T10:41:33.747414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:33.750113Z","iopub.execute_input":"2024-12-13T10:41:33.750418Z","iopub.status.idle":"2024-12-13T10:41:34.342002Z","shell.execute_reply.started":"2024-12-13T10:41:33.750389Z","shell.execute_reply":"2024-12-13T10:41:34.340906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Take samples from dataset\ntrain_sample = train.sample(frac=0.02, random_state=SEED)\n\nif not DEBUG:\n    \n    cols = 5\n    rows = int(np.ceil(len(train_sample.columns) / cols))\n    \n    fig,ax = plt.subplots(nrows=rows,ncols=cols,figsize=(20,15))\n    axes = chain.from_iterable(ax)\n    \n    plt.suptitle(\"Distributions of data\",size=24)\n    \n    for i,col in enumerate(train_sample.columns):\n        ax = next(axes)\n        if train_sample[col].dtype in ['float', 'int']:\n            sns.histplot(data=train_sample, x=col, ax=ax, kde=False)\n        else:\n            sns.countplot(data=train_sample, x=col, ax=ax)\n            if len(train_sample[col].unique()) > 10:\n                ax.tick_params(axis='x', rotation=90)\n    \n    fig.subplots_adjust(hspace=0.4, wspace=0.4)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T05:33:43.418772Z","iopub.execute_input":"2024-12-13T05:33:43.419076Z","iopub.status.idle":"2024-12-13T05:38:51.895495Z","shell.execute_reply.started":"2024-12-13T05:33:43.419047Z","shell.execute_reply":"2024-12-13T05:38:51.894227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature engineering\ndef feat_eng(df):\n\n    # Log-transform\n    df[\"Annual Income_log\"] = np.log1p(df[\"Annual Income\"])\n\n    # Create extra features\n    df[\"Age_Income_Ratio\"] = df[\"Annual Income\"] / df[\"Age\"]\n    df[\"Age_Credit_Ratio\"] = df[\"Credit Score\"] / df[\"Age\"]\n    df[\"Age_Health_Ratio\"] = df[\"Health Score\"] / df[\"Age\"]\n    \n    # Extract values from date-time\n    df[\"Policy Start Year\"] = df[\"Policy Start Date\"].dt.year\n    df[\"Policy Start Month\"] = df[\"Policy Start Date\"].dt.month\n    df[\"Policy Start Day\"] = df[\"Policy Start Date\"].dt.day\n    df[\"Policy Start Day_of_Year\"] = df[\"Policy Start Date\"].dt.dayofyear\n    df[\"Policy Start Quarter\"] = df[\"Policy Start Date\"].dt.quarter\n    df[\"Policy Start Day_of_Week\"] = df[\"Policy Start Date\"].dt.dayofweek\n    df[\"T - Days\"] = (df[\"Policy Start Date\"].max() - df[\"Policy Start Date\"]).dt.days\n\n    # Cyclic encoding\n    # Month (1-12)\n    df[\"Policy Start Month_sin\"] = np.sin(2 * np.pi * df[\"Policy Start Month\"] / 12)\n    df[\"Policy Start Month_cos\"] = np.cos(2 * np.pi * df[\"Policy Start Month\"] / 12)\n    \n    # Day of Week (0-6)\n    df[\"Policy Start Day_of_Week_sin\"] = np.sin(2 * np.pi * df[\"Policy Start Day_of_Week\"] / 7)\n    df[\"Policy Start Day_of_Week_cos\"] = np.cos(2 * np.pi * df[\"Policy Start Day_of_Week\"] / 7)\n\n    # Day of Month\n    df[\"Policy Start Day_of_Month_sin\"] = np.sin(2 * np.pi * df[\"Policy Start Day\"] / 31)\n    df[\"Policy Start Day_of_Month_cos\"] = np.cos(2 * np.pi * df[\"Policy Start Day\"] / 31)\n    \n    # Day of Year (1-365 or 1-366 for leap years)\n    df[\"Policy Start Day_of_Year_sin\"] = np.sin(2 * np.pi * df[\"Policy Start Day_of_Year\"] / 365)\n    df[\"Policy Start Day_of_Year_cos\"] = np.cos(2 * np.pi * df[\"Policy Start Day_of_Year\"] / 365)\n\n    # Original column no longer needed\n    df = df.drop([\"Policy Start Date\",\"Annual Income\"], axis=1)\n    \n    return df\n\ntrain = feat_eng(train)\ntest = feat_eng(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:34.343205Z","iopub.execute_input":"2024-12-13T10:41:34.343565Z","iopub.status.idle":"2024-12-13T10:41:35.834376Z","shell.execute_reply.started":"2024-12-13T10:41:34.343521Z","shell.execute_reply":"2024-12-13T10:41:35.833204Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:35.836417Z","iopub.execute_input":"2024-12-13T10:41:35.83679Z","iopub.status.idle":"2024-12-13T10:41:35.865058Z","shell.execute_reply.started":"2024-12-13T10:41:35.836757Z","shell.execute_reply":"2024-12-13T10:41:35.863925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.drop(\"Premium Amount\", axis=1)\ny = np.log1p(train[\"Premium Amount\"].values)\n\nX_test = test.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:36.09533Z","iopub.execute_input":"2024-12-13T10:41:36.096651Z","iopub.status.idle":"2024-12-13T10:41:36.629865Z","shell.execute_reply.started":"2024-12-13T10:41:36.096594Z","shell.execute_reply":"2024-12-13T10:41:36.628476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature Importance\nif not DEBUG:\n    \n    np.random.seed(SEED)\n    \n    # Copy X\n    X_feat = X.copy()\n    \n    preprocessor, cat_cols, num_cols = create_preprocessor(X, SEED)\n    \n    # Impute and scale\n    X_feat = preprocessor.fit_transform(X_feat)\n    y_feat = train[\"Premium Amount\"]\n    \n    # Get feature names after preprocessing\n    feature_names = preprocessor.get_feature_names_out()\n    \n    # SelectFromModel with RandomForestRegressor\n    rfr = SelectFromModel(RandomForestRegressor(n_estimators=10, random_state=SEED), threshold=0.01)\n    rfr.fit(X_feat, y_feat)\n    \n    # Feature importances\n    imp = rfr.estimator_.feature_importances_\n    \n    # Map feature importances to feature names\n    df = pd.DataFrame(imp, index=feature_names, columns=[\"Importance\"])\n    \n    # Plot feature importances\n    fig, ax = plt.subplots(figsize=(15, 10))\n    sorted_idx = imp.argsort()\n    ax.barh(df.index[sorted_idx], df[\"Importance\"][sorted_idx], height=0.8, facecolor=\"grey\", alpha=0.8, edgecolor=\"k\")\n    ax.set_xlabel(\"Importance score\")\n    ax.set_title(\"Feature Importance\")\n    plt.gca().invert_yaxis()\n    fig.tight_layout()\n    plt.show()\n\n    # Select features to keep based on threshold\n    to_keep = list(df.index[rfr.get_support()])\n    print(f\"Features suggested to keep: {to_keep}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T05:38:54.062139Z","iopub.execute_input":"2024-12-13T05:38:54.062508Z","iopub.status.idle":"2024-12-13T05:52:14.358811Z","shell.execute_reply.started":"2024-12-13T05:38:54.062472Z","shell.execute_reply":"2024-12-13T05:52:14.357591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rmsle_scorer = make_scorer(rmsle, greater_is_better=False)\ncv = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:43.531963Z","iopub.execute_input":"2024-12-13T10:41:43.532368Z","iopub.status.idle":"2024-12-13T10:41:43.537739Z","shell.execute_reply.started":"2024-12-13T10:41:43.532332Z","shell.execute_reply":"2024-12-13T10:41:43.536592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check various models\nif not DEBUG:\n    np.random.seed(SEED)\n    \n    models = {\n        \"Ridge\": Ridge(),\n        \"Lasso\": Lasso(),\n        \"ElasticNet\": ElasticNet(),\n        \"BayesianRidge\": BayesianRidge(),\n        \"LightGBM\": lgb.LGBMRegressor(verbose=0),\n        \"XGBoost\": xgb.XGBRegressor()\n    }\n    \n    if not DEBUG:\n        for model_name, model in models.items():\n            print(f\"Processing model: {model_name}\")\n            start_time = time.time()\n            pipeline = create_pipeline(X, model, SEED=SEED)\n            scores = -1 * cross_val_score(pipeline, X, y, cv=cv, scoring=rmsle_scorer)\n            avg_rmsle = np.mean(scores)\n            std_rmsle = scores.std()\n            end_time = time.time() - start_time\n            print(f\"{end_time:.0f} seconds - {model_name} - Average RMSLE: {avg_rmsle:.4f}, Std: {std_rmsle:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T10:41:45.806668Z","iopub.execute_input":"2024-12-13T10:41:45.807192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T01:07:19.461029Z","iopub.execute_input":"2024-12-12T01:07:19.461431Z","iopub.status.idle":"2024-12-12T01:07:20.990263Z","shell.execute_reply.started":"2024-12-12T01:07:19.4614Z","shell.execute_reply":"2024-12-12T01:07:20.988914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial, X_train, y_train, SEED=SEED):\n\n    objective = \"regression\"\n    learning_rate = trial.suggest_float(\"learning_rate\", 1e-3, 1e-1, log=True)\n    n_estimators = trial.suggest_int('n_estimators', 50, 500)\n    max_depth = trial.suggest_int('max_depth', 2, 32, log=True)\n    num_leaves = trial.suggest_int(\"num_leaves\", 2, 1024, log=True)\n    feature_fraction = trial.suggest_uniform(\"feature_fraction\", 0.6, 1.0)\n    bagging_fraction = trial.suggest_uniform(\"bagging_fraction\", 0.4, 1.0)\n    bagging_freq = trial.suggest_int(\"bagging_freq\", 5, 12)\n    lambda_l1 = trial.suggest_float('lambda_l1', 1e-8, 10.0, log=True)\n    lambda_l2 = trial.suggest_float('lambda_l2', 1e-8, 10.0, log=True)\n    random_state = SEED\n    \n    model = lgb.LGBMRegressor(objective=objective,learning_rate=learning_rate,n_estimators=n_estimators,\n                              max_depth=max_depth,num_leaves=num_leaves,feature_fraction=feature_fraction,\n                              bagging_fraction=bagging_fraction,bagging_freq=bagging_freq,reg_alpha=lambda_l1,\n                              reg_lambda=lambda_l2,random_state=random_state)\n\n    \n    pipeline = create_pipeline(X_train, model, SEED=SEED)\n    \n    cv_score = -1 * cross_val_score(pipeline, X_train, y_train, cv=cv, scoring=rmsle_scorer)\n    avg_rmsle = np.mean(cv_score)\n    \n    return avg_rmsle\n\n\nwith open(os.devnull, \"w\") as f, contextlib.redirect_stdout(f):\n    np.random.seed(SEED)\n    study = optuna.create_study(direction=\"minimize\")\n    study.optimize(lambda trial: objective(trial, X_train, y_train, SEED=SEED), n_trials=N_TRIALS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T01:12:01.723026Z","iopub.execute_input":"2024-12-12T01:12:01.723598Z","iopub.status.idle":"2024-12-12T04:28:09.93588Z","shell.execute_reply.started":"2024-12-12T01:12:01.723557Z","shell.execute_reply":"2024-12-12T04:28:09.934557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check model performance on validation set\nbest_params = study.best_params\n\nwith open(os.devnull, \"w\") as f, contextlib.redirect_stdout(f):\n    np.random.seed(SEED)\n    model = lgb.LGBMRegressor().set_params(**best_params)\n\n    pipeline = create_pipeline(X_train, model)\n    \n    best_model = pipeline.fit(X_train, y_train)\n    y_pred = best_model.predict(X_val)\n\nrmsle = rmsle(y_val, y_pred)\nprint(f\"RMSLE: {rmsle:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(os.devnull, \"w\") as f, contextlib.redirect_stdout(f):\n    model = lgb.LGBMRegressor().set_params(**best_params)\n\n    pipeline = create_pipeline(X, model)\n    \n    final_model = pipeline.fit(X, y)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = final_model.predict(X_test)\ntrue_results = np.expm1(results)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit[\"Premium Amount\"] = true_results","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.kdeplot(data=train_sample,x=\"Premium Amount\", label=\"True Data\")\nsns.kdeplot(data=submit, x=\"Premium Amount\", label=\"Predicted Data\")\nplt.legend()\nplt.show();","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit.to_csv(\"submission_BBG007.csv\",index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}