{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":11100710,"sourceType":"datasetVersion","datasetId":6919954}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":2430.171281,"end_time":"2025-03-17T08:06:52.205381","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-03-17T07:26:22.0341","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"2e9da04b","cell_type":"code","source":"!pip install hill-climbing","metadata":{},"outputs":[],"execution_count":null},{"id":"77bf0c32","cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.0053,"end_time":"2025-03-17T07:26:24.689395","exception":false,"start_time":"2025-03-17T07:26:24.684095","status":"completed"},"tags":[]}},{"id":"217fea99","cell_type":"code","source":"from hill_climbing import Climber, ClimberCV\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport joblib\nimport glob\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":3.144235,"end_time":"2025-03-17T07:26:27.838593","exception":false,"start_time":"2025-03-17T07:26:24.694358","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9018e969","cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/playground-series-s4e12/train.csv\"\n    test_path = \"/kaggle/input/playground-series-s4e12/test.csv\"\n    sample_sub_path = \"/kaggle/input/playground-series-s4e12/sample_submission.csv\"\n\n    oof_path = \"/kaggle/input/hill-climbing-example-datasets/regression\"\n    \n    target = \"Premium Amount\"\n    n_folds = 10\n    seed = 42","metadata":{"papermill":{"duration":0.0111,"end_time":"2025-03-17T07:26:27.854405","exception":false,"start_time":"2025-03-17T07:26:27.843305","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"326b80e5","cell_type":"markdown","source":"# Loading data and OOF files","metadata":{"papermill":{"duration":0.004356,"end_time":"2025-03-17T07:26:27.86353","exception":false,"start_time":"2025-03-17T07:26:27.859174","status":"completed"},"tags":[]}},{"id":"8cfa5d0c","cell_type":"code","source":"train = pd.read_csv(CFG.train_path, index_col=\"id\")\nX, y = train.drop(CFG.target, axis=1), np.log1p(train[CFG.target])","metadata":{"papermill":{"duration":0.051888,"end_time":"2025-03-17T07:26:27.919929","exception":false,"start_time":"2025-03-17T07:26:27.868041","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"14935bb5","cell_type":"code","source":"scores, oof_preds, test_preds = {}, {}, {}","metadata":{"papermill":{"duration":0.010486,"end_time":"2025-03-17T07:26:27.953002","exception":false,"start_time":"2025-03-17T07:26:27.942516","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"bb4b5a47","cell_type":"code","source":"paths = glob.glob(f\"{CFG.oof_path}/*\")\nfor path in paths:\n    files = glob.glob(f\"{path}/*\")\n    temp_oof_preds = joblib.load([file for file in files if \"oof_preds\" in file][0])\n    temp_test_preds = joblib.load([file for file in files if \"test_preds\" in file][0])\n    temp_oof_preds = np.log1p(temp_oof_preds)\n    temp_test_preds = np.log1p(temp_test_preds)\n    model_name = path.split(\"/\")[-1]\n        \n    temp_scores = []        \n    kf = KFold(n_splits=CFG.n_folds, random_state=CFG.seed, shuffle=True)\n    for _, val_idx in kf.split(X, y):\n        temp_scores.append(np.sqrt(mean_squared_error(y[val_idx], temp_oof_preds[val_idx])))\n    \n    scores[model_name] = temp_scores\n    oof_preds[model_name] = temp_oof_preds\n    test_preds[model_name] = temp_test_preds","metadata":{"papermill":{"duration":1.738768,"end_time":"2025-03-17T07:26:29.69646","exception":false,"start_time":"2025-03-17T07:26:27.957692","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"baa910f2","cell_type":"markdown","source":"# Hill climbing","metadata":{"papermill":{"duration":0.004847,"end_time":"2025-03-17T07:26:29.706533","exception":false,"start_time":"2025-03-17T07:26:29.701686","status":"completed"},"tags":[]}},{"id":"c066a695","cell_type":"code","source":"X = pd.DataFrame(oof_preds)\nX_test = pd.DataFrame(test_preds)","metadata":{"papermill":{"duration":0.023688,"end_time":"2025-03-17T07:26:29.797009","exception":false,"start_time":"2025-03-17T07:26:29.773321","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"16ea71d5","cell_type":"code","source":"X.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"242bc6f1","cell_type":"code","source":"def rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_error(y_true, y_pred))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"96abdb01","cell_type":"code","source":"climber = Climber(\n    objective=\"minimize\",\n    eval_metric=rmsle,\n    allow_negative_weights=True,\n    precision=0.001,\n    score_decimal_places=6\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"abd5ad70","cell_type":"code","source":"climber.fit(X, y)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"4b5dd8bc","cell_type":"code","source":"scores[\"climber\"] = [climber.best_score] * CFG.n_folds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"ffe2ee3f","cell_type":"code","source":"sns.set_style(\"whitegrid\", {'grid.linestyle': '--'})\nsns.set_context(\"notebook\", font_scale=1.2)\n\ndef add_annotations(ax, x, y):\n    for xi, yi in zip(x, y):\n        ax.annotate(\n            f'{yi:.6f}', (xi, yi),\n            textcoords=\"offset points\",\n            xytext=(0, 10),\n            ha='center',\n            va='bottom',\n            fontsize=8\n        )\n\npalette = sns.color_palette(\"deep\")\nfig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(15, 10), sharex=True)\n\nsns.lineplot(x=climber.history.model[1:], y=climber.history.coef[1:], color=palette[0], marker='x', ax=ax1, linewidth=2)\nadd_annotations(ax1, climber.history.model[1:], climber.history.coef[1:])\nax1.set_ylabel('Coefficient', fontsize=12)\nax1.spines[['top', 'right']].set_visible(False)\n\nsns.lineplot(x=climber.history.model[1:], y=climber.history.improvement[1:], color=palette[1], marker='o', ax=ax2, linewidth=2)\nadd_annotations(ax2, climber.history.model[1:], climber.history.improvement[1:])\nax2.set_ylabel('Improvement', fontsize=12)\nax2.spines[['top', 'right']].set_visible(False)\n\nsns.lineplot(x=climber.history.model[1:], y=climber.history.score[1:], color=palette[2], marker='*', ax=ax3, linewidth=2)\nadd_annotations(ax3, climber.history.model[1:], climber.history.score[1:])\nax3.set_ylabel('Score', fontsize=12)\nax3.set_xlabel('Model', fontsize=12)\nax3.spines[['top', 'right']].set_visible(False)\n\nplt.tight_layout()\nfig.suptitle('Hill Climbing History', y=1.02, fontsize=14, fontweight='bold')\n\nfor ax in [ax1, ax2, ax3]:\n    ax.tick_params(labelsize=8)\n    ax.yaxis.grid(True, linestyle='--', alpha=0.7)\n    ax.xaxis.grid(True, linestyle='--', alpha=0.7)\n    ymin, ymax = ax.get_ylim()\n    ax.set_ylim(ymin - (ymax-ymin)*0.1, ymax + (ymax-ymin)*0.1)\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"151f1ea3","cell_type":"markdown","source":"## Predicting on the test set","metadata":{}},{"id":"e351eed3","cell_type":"code","source":"preds = climber.predict(X_test)\npreds[:25]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"50082867","cell_type":"markdown","source":"# Hill climbing with cross-validation","metadata":{}},{"id":"d6d9ccb0","cell_type":"code","source":"climber_cv = ClimberCV(\n    objective=\"minimize\",\n    eval_metric=rmsle,\n    allow_negative_weights=True,\n    precision=0.001,\n    score_decimal_places=6,\n    cv=KFold(n_splits=CFG.n_folds, random_state=CFG.seed, shuffle=True),\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"2a84e6a5","cell_type":"code","source":"climber_cv.fit(X, y)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"40697a35","cell_type":"code","source":"scores[\"climber-cv\"] = climber_cv.fold_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"ab0d3197","cell_type":"code","source":"fold_1_history = climber_cv.history[climber_cv.history.fold == 1]\n\npalette = sns.color_palette(\"deep\")\nfig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(15, 10), sharex=True)\n\nsns.lineplot(x=fold_1_history.model[1:], y=fold_1_history.coef[1:], color=palette[0], marker='x', ax=ax1, linewidth=2)\nadd_annotations(ax1, fold_1_history.model[1:], fold_1_history.coef[1:])\nax1.set_ylabel('Coefficient', fontsize=12)\nax1.spines[['top', 'right']].set_visible(False)\n\nsns.lineplot(x=fold_1_history.model[1:], y=fold_1_history.train_score[1:], color=palette[1], marker='o', ax=ax2, linewidth=2)\nadd_annotations(ax2, fold_1_history.model[1:], fold_1_history.train_score[1:])\nax2.set_ylabel('Train Score', fontsize=12)\nax2.spines[['top', 'right']].set_visible(False)\n\nsns.lineplot(x=fold_1_history.model[1:], y=fold_1_history.val_score[1:], color=palette[2], marker='*', ax=ax3, linewidth=2)\nadd_annotations(ax3, fold_1_history.model[1:], fold_1_history.val_score[1:])\nax3.set_ylabel('Validation Score', fontsize=12)\nax3.set_xlabel('Model', fontsize=12)\nax3.spines[['top', 'right']].set_visible(False)\n\nplt.tight_layout()\nfig.suptitle('Hill Climbing History', y=1.02, fontsize=14, fontweight='bold')\n\nfor ax in [ax1, ax2, ax3]:\n    ax.tick_params(labelsize=10)\n    ax.yaxis.grid(True, linestyle='--', alpha=0.7)\n    ax.xaxis.grid(True, linestyle='--', alpha=0.7)\n    ymin, ymax = ax.get_ylim()\n    ax.set_ylim(ymin - (ymax-ymin)*0.1, ymax + (ymax-ymin)*0.1)\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"ab60ebaf","cell_type":"markdown","source":"## Predicting on the test set","metadata":{}},{"id":"2a6b1b73","cell_type":"code","source":"preds = climber_cv.predict(X_test)\npreds[:25]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"8b7d9f23","cell_type":"markdown","source":"# Results","metadata":{"papermill":{"duration":0.118824,"end_time":"2025-03-17T08:06:49.118491","exception":false,"start_time":"2025-03-17T08:06:48.999667","status":"completed"},"tags":[]}},{"id":"56936ee7","cell_type":"code","source":"sns.reset_defaults()\n\nscores = pd.DataFrame(scores)\nmean_scores = scores.mean().sort_values(ascending=True)\norder = scores.mean().sort_values(ascending=True).index.tolist()\n\nmin_score = mean_scores.min()\nmax_score = mean_scores.max()\npadding = (max_score - min_score) * 0.5\nlower_limit = min_score - padding\nupper_limit = max_score + padding\n\nfig, axs = plt.subplots(1, 2, figsize=(15, scores.shape[1] * 0.3))\n\nboxplot = sns.boxplot(data=scores, order=order, ax=axs[0], orient='h', color='grey')\naxs[0].set_title('Fold Score')\naxs[0].set_xlabel('')\naxs[0].set_ylabel('')\n\nbarplot = sns.barplot(x=mean_scores.values, y=mean_scores.index, ax=axs[1], color='grey')\naxs[1].set_title('Average Score')\naxs[1].set_xlabel('')\naxs[1].set_xlim(left=lower_limit, right=upper_limit)\naxs[1].set_ylabel('')\n\nfor i, (score, model) in enumerate(zip(mean_scores.values, mean_scores.index)):\n    color = 'cyan' if 'climber' in model.lower() else 'grey'\n    barplot.patches[i].set_facecolor(color)\n    boxplot.patches[i].set_facecolor(color)\n    barplot.text(score, i, round(score, 6), va='center')\n\nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":false,"papermill":{"duration":2.014537,"end_time":"2025-03-17T08:06:51.253329","exception":false,"start_time":"2025-03-17T08:06:49.238792","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}