{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ✨*Data Exploration*💰  \n\n> *Looking each feature of dataset*  \n\n\n---","metadata":{"_uuid":"0a40487a-ae74-41f9-8740-09fedb34323c","_cell_guid":"72826761-4ea8-4475-9f5b-5fbc459d12da","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## ✨*Preparation*💰  \n\n> *Import modules and read data*  \n\n---","metadata":{"_uuid":"8a6f9cec-8e7a-4bf6-8ed5-cc073877fbea","_cell_guid":"e94df9b7-0265-48bb-86a7-ae4d707dc4fe","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# import modules\nimport polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import train_test_split\n\n# import optuna.integration.lightgbm as lgb\nimport lightgbm as lgb\n\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport shap\n\nsns.set_theme(context=\"notebook\", style=\"whitegrid\", palette=\"Set2\")\nwarnings.simplefilter(action=\"ignore\", category=FutureWarning)\n\nimport time","metadata":{"_uuid":"547406ec-77b6-4108-b7ec-84caa9c86d68","_cell_guid":"73dba25e-b1a8-41cc-a50d-69948aff52cc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:08.285129Z","iopub.execute_input":"2024-12-07T22:47:08.285587Z","iopub.status.idle":"2024-12-07T22:47:18.953266Z","shell.execute_reply.started":"2024-12-07T22:47:08.285551Z","shell.execute_reply":"2024-12-07T22:47:18.951828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stop watch\nstart_time = time.time()\n\n# data reading\ndf = pl.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\").drop(\"id\")\n\ndf_obj = pl.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\ndf_obj_id = df_obj.select(\"id\")\ndf_obj = df_obj.drop(\"id\")","metadata":{"_uuid":"576dc47e-69f4-4cbd-8aec-87bd6903dba1","_cell_guid":"99d6e5bc-8c89-4a63-a429-889df415809d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2024-12-07T22:47:18.955384Z","iopub.execute_input":"2024-12-07T22:47:18.956086Z","iopub.status.idle":"2024-12-07T22:47:22.469931Z","shell.execute_reply.started":"2024-12-07T22:47:18.956046Z","shell.execute_reply":"2024-12-07T22:47:22.468752Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Data Handling class definition*💰  \n\n> *Class definition for handling features*  \n\n---","metadata":{"_uuid":"f65fb070-ef08-40a6-a0f1-91f6d812a29f","_cell_guid":"658070b6-a7ba-49df-a2d5-91c64f3d4494","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Data preprocessing class\nclass CustomTramsformer:\n    def __init__(self, target):\n        self.target = target\n        self.ordered_elements = {}\n        self.not_need_other = {}\n        self.feature_threshold = {}\n        self.scaler = StandardScaler()\n        self.encoder = OneHotEncoder(drop=\"first\")\n        self.ct = None\n        self.categories = None\n        self.ct_feature_names = None\n        self.num_features = None\n        self.cat_features = None\n\n    def reset_features(self, df: pl.DataFrame):\n        self.num_features = [\n            col\n            for col in df.columns\n            if df[col].dtype in [pl.Int64, pl.Float64, pl.Int32]\n        ]\n        self.cat_features = [col for col in df.columns if df[col].dtype in [pl.String]]\n\n    def set_threshold(\n        self, df1: pl.DataFrame, df2: pl.DataFrame, feature: str, val: int\n    ):\n        self.feature_threshold[feature] = val\n        df1 = self.set_categories_of_feature(df1, feature, val)\n        df2 = self.set_categories_of_feature(df2, feature)\n\n        if (\"other\" not in df1[feature].unique()) & (\"other\" not in df2[feature].unique()):\n            self.not_need_other[feature] = True\n        else:\n            self.not_need_other[feature] = False\n\n        if self.not_need_other[feature]:\n            self.ordered_elements[feature] = [\n                x for x in self.ordered_elements[feature] if x != \"other\"\n            ]\n        return df1, df2\n\n    def set_categories_of_feature(\n        self, df: pl.DataFrame, feature: str, n_items: int = 0\n    ):\n        if n_items != 0:\n            counts = df[feature].value_counts().sort(by=\"count\", descending=True)\n            self.ordered_elements[feature] = [\n                col for col, ct in counts.to_numpy() if ct >= n_items\n            ]\n\n        df = df.with_columns(\n            pl.when(pl.col(feature).is_in(self.ordered_elements[feature]))\n            .then(pl.col(feature))\n            .when(pl.col(feature).is_null())\n            .then(pl.col(feature))\n            .otherwise(pl.lit(\"other\"))\n            .alias(feature)\n        )\n\n        if feature not in self.not_need_other.keys():\n            self.not_need_other[feature] = False\n\n        if (not self.not_need_other[feature] and \"other\" not in self.ordered_elements[feature]):\n            self.ordered_elements[feature].append(\"other\")\n        return df\n\n    def set_ct(self):\n        num_wo_target = [val for val in self.num_features if val != self.target]\n        self.categories = [self.ordered_elements[col] for col in self.cat_features]\n        self.encoder = OneHotEncoder(drop=\"first\", categories=self.categories)\n\n        self.ct = ColumnTransformer(\n            [(\"scaler\", self.scaler, num_wo_target),\n             (\"encoder\", self.encoder, self.cat_features),\n            ],\n            remainder=\"passthrough\",\n        )\n\n    def ct_transform(self, df: pl.DataFrame):\n        data_scaled = self.ct.fit_transform(df.to_pandas(use_pyarrow_extension_array=True))\n        if self.categories is None:\n            self.categories = self.ct.named_transformers_[\"encoder\"].categories_\n        if self.ct_feature_names is None:\n            self.ct_feature_names = self.ct.get_feature_names_out().tolist()\n        return data_scaled\n\n    def pipeline(self, df: pl.DataFrame, set_categories: bool = False):\n        cols = self.cat_features\n        if set_categories:\n            n_threshes = []\n            for col in cols:\n                if col in self.feature_threshold.keys():\n                    val = self.feature_threshold[col]\n                else:\n                    val = 1\n                n_threshes.append(val)\n        else:\n            n_threshes = [0 for _ in range(len(cols))]\n\n        for col, thresh in zip(cols, n_threshes):\n            df = self.set_categories_of_feature(df, col, thresh)\n\n        if set_categories:\n            self.set_ct()\n        return df","metadata":{"_uuid":"2ad348b6-8b8d-4258-9613-93fbd61ceadf","_cell_guid":"a8d88eb5-4c7f-445d-a65f-1d0b530b5e51","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:22.472031Z","iopub.execute_input":"2024-12-07T22:47:22.472448Z","iopub.status.idle":"2024-12-07T22:47:22.492047Z","shell.execute_reply.started":"2024-12-07T22:47:22.472413Z","shell.execute_reply":"2024-12-07T22:47:22.490815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot cat features\ndef plot_cat(df1: pl.DataFrame, df2: pl.DataFrame, feature: str, order=None):\n\n    if order is None:\n        order = ut.ordered_elements[feature]\n\n    palette = dict(zip(order, sns.color_palette(\"Set2\")[: len(order)]))\n    _, ax = plt.subplots(1, 3, figsize=(8, 3), tight_layout=True)\n    plt.subplots_adjust(right=0.85)\n    sns.boxplot(\n        df1.to_pandas(), x=feature, y=ut.target, ax=ax[0], order=order, palette=palette,\n    )\n    sns.countplot(df1.to_pandas(), x=feature, ax=ax[1], order=order, palette=palette)\n    sns.countplot(df2.to_pandas(), x=feature, ax=ax[2], order=order, palette=palette)\n\n    for i, ylabel in enumerate([\"Premium\", \"train cnt.\", \"test cnt.\"]):\n        ax[i].set_ylabel(ylabel)\n        ax[i].set_xlabel(\"\")\n        ax[i].set_xticklabels(order, fontsize=11, rotation=90)\n    plt.suptitle(feature)\n    plt.show()\n\n\n# plot num features\ndef plot_num(df1: pl.DataFrame, df2: pl.DataFrame, feature: str):\n    if feature == ut.target:\n        return\n\n    min_val = min(df1[feature].min(), df2[feature].min())\n    max_val = max(df1[feature].max(), df2[feature].max())\n    bins = np.linspace(min_val, max_val, 31)\n\n    _, ax = plt.subplots(1, 3, figsize=(12, 3), tight_layout=True)\n    sns.scatterplot(df1.to_pandas(), x=feature, y=ut.target, alpha=0.005, ax=ax[0])\n    sns.histplot(df1.to_pandas(), x=feature, bins=bins, ax=ax[1])\n    sns.histplot(df2.to_pandas(), x=feature, bins=bins, ax=ax[2])\n    for i, ylabel in enumerate([\"Premium\", \"train cnt.\", \"test cnt.\"]):\n        ax[i].set_ylabel(ylabel)\n    plt.show()","metadata":{"_uuid":"9cea13bd-0ca8-48f5-999b-4391fdb4765f","_cell_guid":"b3a0e5ab-4953-458d-aaa5-ea2b05b363f0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:22.494683Z","iopub.execute_input":"2024-12-07T22:47:22.49515Z","iopub.status.idle":"2024-12-07T22:47:22.517286Z","shell.execute_reply.started":"2024-12-07T22:47:22.495089Z","shell.execute_reply":"2024-12-07T22:47:22.515902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Null Data Check*💰  \n\n> *There are many nulls. It's appropriate to replace null to other value. but null values at \"Financial Stress\" is only in train data, so they could be droped.*  \n\n---","metadata":{"_uuid":"998e2145-cfef-4419-a9f8-c90cd8cfb31f","_cell_guid":"c63350c7-28cf-411e-83f8-27a1cdb2c89d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def nullcheck(df: pl.DataFrame, title:str):\n    has_null = False\n    df_nullcheck = pl.DataFrame()\n    for col in df.columns:\n        nc = df[col].null_count()\n        if nc >= 1:\n            df_nullcheck = pl.concat(\n                [df_nullcheck, pl.DataFrame({\"feature\":col, \"nulls\":nc, \"nulls_ratio\": nc/len(df)})]\n            )\n            # print(f\"{col} has {nc} nulls. ({nc/len(df)*100:.1f} %)\")\n            has_null = True\n\n    if has_null:\n        colors = sns.color_palette('pastel')\n        plt.figure(figsize=(8,3))\n        ax = sns.barplot(df_nullcheck.to_pandas(), y=\"feature\", x=\"nulls\", palette=colors)\n        for container in ax.containers:\n            ax.bar_label(container, fmt='{:,.0f}', padding=3, fontsize=10)\n        plt.title(f\"Nulls check ({title})\")\n        plt.show()\n    else:\n        print(f\"dateframe {title} has no null.\")\n\n\nnullcheck(df, \"train\")\nnullcheck(df_obj, \"test\")","metadata":{"_uuid":"972b10be-50bc-46df-b71a-730d247826cc","_cell_guid":"d4e94eaa-0bc7-426c-b183-bb0678d5f9f4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:22.518882Z","iopub.execute_input":"2024-12-07T22:47:22.51934Z","iopub.status.idle":"2024-12-07T22:47:23.422331Z","shell.execute_reply.started":"2024-12-07T22:47:22.519282Z","shell.execute_reply":"2024-12-07T22:47:23.421173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Feature Overview and handling*💰  \n\n> *Looking each feature of dataset*  \n\n---","metadata":{"_uuid":"45c338e2-ea3e-458d-a5d8-e54ff4468883","_cell_guid":"720c410c-983c-413c-a10d-f965eb66614d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"ut = CustomTramsformer(\"Premium Amount\")\nut.reset_features(df)\ndf = ut.pipeline(df, True)","metadata":{"_uuid":"31f46a34-2a2d-48ec-af08-61c81029f7c3","_cell_guid":"3f380ad4-2d8f-4be4-a249-c8ac4482e3de","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:23.423813Z","iopub.execute_input":"2024-12-07T22:47:23.424249Z","iopub.status.idle":"2024-12-07T22:47:25.953844Z","shell.execute_reply.started":"2024-12-07T22:47:23.424201Z","shell.execute_reply":"2024-12-07T22:47:25.952356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Policy Start Date to year, month, day\ndef handle_policy_start(df: pl.DataFrame):\n    feature = \"Policy Start Date\"\n    df = df.with_columns(\n        pl.col(feature).str.extract(r\"^(\\d+)-\").cast(pl.Int32).alias(\"Start Year\"),\n        pl.col(feature).str.extract(r\"^\\d+-(\\d+)\").cast(pl.Int32).alias(\"Start Month\"),\n        pl.col(feature).str.extract(r\"^\\d+-\\d+-(\\d+) \").cast(pl.Int32).alias(\"Start Day\"),\n    ).drop(feature)\n    return df\n\ndf = handle_policy_start(df)\ndf_obj = handle_policy_start(df_obj)\nut.reset_features(df)","metadata":{"_uuid":"078dcab3-a806-40a3-82c1-3aa9ee8a59fc","_cell_guid":"49d49bdc-1a99-4638-bc3c-77f74b228dbf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:25.955458Z","iopub.execute_input":"2024-12-07T22:47:25.955971Z","iopub.status.idle":"2024-12-07T22:47:26.456805Z","shell.execute_reply.started":"2024-12-07T22:47:25.955921Z","shell.execute_reply":"2024-12-07T22:47:26.455701Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Target Value*💰  \n\n> *It seems to have three peaks.*  \n> *I concider to devide three categories, and make models for each peaks.*  \n\n---","metadata":{"_uuid":"cb3b11ef-a60e-4965-b373-490f9c021d68","_cell_guid":"7733b7ba-4786-4a9f-89f6-736a2b1f48b4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# target value\nprint(\"<< target value >>\")\n_, ax = plt.subplots(1, 1, figsize=(8, 3))\ndf_graph = df.with_columns(pl.when(pl.col(ut.target) < 500).then(pl.lit(\"-500\"))\n                           .when(pl.col(ut.target) < 2000).then(pl.lit(\"500-2000\"))\n                           .otherwise(pl.lit(\"2000-\")).alias(\"class\")\n                          )\nsns.histplot(df_graph.to_pandas(), x=ut.target, hue=\"class\", multiple=\"stack\", bins=50)\nplt.show()","metadata":{"_uuid":"930b004e-2f9b-45ab-92a5-327ebdbf685f","_cell_guid":"2be675ff-6bdd-4a23-a4bf-f821aa0f96bc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:26.45809Z","iopub.execute_input":"2024-12-07T22:47:26.458394Z","iopub.status.idle":"2024-12-07T22:47:29.667981Z","shell.execute_reply.started":"2024-12-07T22:47:26.458366Z","shell.execute_reply":"2024-12-07T22:47:29.666768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Categorical features*💰  \n\n---","metadata":{"_uuid":"95b0323b-c382-4626-b0d1-494aeb256bd6","_cell_guid":"269d7025-2d9d-444e-a42b-27a49f7ac4f9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Categorical features\nfeatures = ut.cat_features\n\n# Replace null value to mode\ndef null_to_mode(df: pl.DataFrame, feature: str):\n    df = df.with_columns(pl.col(feature).fill_null(pl.col(feature).mode().first()))\n    df = df.with_columns(pl.col(feature).fill_null(pl.lit(\"Unknown\")))\n    return df\n\n# boxplot and countplot of train & test data\nfor feature in features:\n    df = null_to_mode(df, feature)\n    df_obj = null_to_mode(df_obj, feature)\n    ut.reset_features(df)\n    ut.set_threshold(df, df_obj, feature, 1)\n    plot_cat(df, df_obj, feature)","metadata":{"_uuid":"37289ff0-7764-4a13-aad5-550cb8454450","_cell_guid":"9650f9dd-fce1-46d5-9a7b-73202a5e2f30","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:47:29.669507Z","iopub.execute_input":"2024-12-07T22:47:29.669972Z","iopub.status.idle":"2024-12-07T22:48:04.485223Z","shell.execute_reply.started":"2024-12-07T22:47:29.66992Z","shell.execute_reply":"2024-12-07T22:48:04.48407Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Numeric features*💰  \n\n---","metadata":{"_uuid":"865d14cb-4728-475c-aecc-eb54f07dac89","_cell_guid":"6c451307-91b0-4076-9f55-ba445098b626","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Numerical features\n# Replace null to mean value.\ndef handle_num_features(df: pl.DataFrame, feature: str):\n    df = df.with_columns(\n        pl.col(feature).fill_null(pl.col(feature).mean()).alias(feature)\n    )\n    return df\n\n\nfor feature in [val for val in ut.num_features if val != ut.target]:\n    df = handle_num_features(df, feature)\n    df_obj = handle_num_features(df_obj, feature)","metadata":{"_uuid":"321671ce-0385-428c-a2bf-285d271dadc3","_cell_guid":"9bc4aded-bd69-47c8-947f-d4d61d4c25d8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:48:04.489782Z","iopub.execute_input":"2024-12-07T22:48:04.490151Z","iopub.status.idle":"2024-12-07T22:48:04.659373Z","shell.execute_reply.started":"2024-12-07T22:48:04.490115Z","shell.execute_reply":"2024-12-07T22:48:04.658218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature Engineering\n# create some features\ndef create_feature(df: pl.DataFrame):\n    df = df.with_columns(\n        (pl.col(\"Annual Income\") * pl.col(\"Previous Claims\")).alias(\"Income x Claims\"),\n        (pl.col(\"Annual Income\") / pl.col(\"Health Score\")).alias(\"Income Health ratio\"),\n        (pl.col(\"Annual Income\") / pl.col(\"Credit Score\")).alias(\"Income Credit ratio\"),\n        (pl.col(\"Credit Score\") * pl.col(\"Health Score\")).alias(\"Credit x Health\"),\n        # (pl.col(\"Credit Score\") * pl.col(\"Previous Claims\")).alias(\"Credit x Claims\"),\n    )\n    return df\n\n\ndf = create_feature(df)\ndf_obj = create_feature(df_obj)\nut.reset_features(df)","metadata":{"_uuid":"3b233163-788e-4826-99ca-f527afb24222","_cell_guid":"95592959-6d89-4e66-9847-e10c05513ddb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:48:04.660728Z","iopub.execute_input":"2024-12-07T22:48:04.661095Z","iopub.status.idle":"2024-12-07T22:48:04.693103Z","shell.execute_reply.started":"2024-12-07T22:48:04.661052Z","shell.execute_reply":"2024-12-07T22:48:04.691894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#scatterplot and histgram of train & test data\nfor feature in [val for val in ut.num_features if val != ut.target]:\n    plot_num(df, df_obj, feature)","metadata":{"_uuid":"5ed3e5e5-e116-4157-9d99-6afbc3655dc8","_cell_guid":"0dbe71e4-ccf9-48d8-9db8-4a8c5c7497c9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:48:04.694555Z","iopub.execute_input":"2024-12-07T22:48:04.694895Z","iopub.status.idle":"2024-12-07T22:49:25.256106Z","shell.execute_reply.started":"2024-12-07T22:48:04.694864Z","shell.execute_reply":"2024-12-07T22:49:25.254792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Re: Null check\nnullcheck(df, \"train\")\nnullcheck(df_obj, \"test\")","metadata":{"_uuid":"a136f3f5-18f7-4053-b344-a96bc5c59a2c","_cell_guid":"06383db5-db63-4e60-8d3f-822296e4edbc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:49:25.257482Z","iopub.execute_input":"2024-12-07T22:49:25.257836Z","iopub.status.idle":"2024-12-07T22:49:25.263757Z","shell.execute_reply.started":"2024-12-07T22:49:25.257804Z","shell.execute_reply":"2024-12-07T22:49:25.262591Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Co-reration of numerical features*💰  \n\n> *Plotting heatmap*  \n\n---","metadata":{"_uuid":"18aecc78-c519-437e-8246-f75b20fb0d62","_cell_guid":"f120c9ee-04a7-4aa1-a95a-da465fd8a6ae","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 12), tight_layout=True)\nsns.heatmap(\n    df.select(ut.num_features).to_pandas().corr(),\n    cmap=\"coolwarm\", annot=True, fmt=\".2f\",\n    vmin=-1, vmax=1,\n    center=0, square=True,\n    linewidths=2, linecolor=\"white\",\n    cbar_kws={\"shrink\": 0.7},\n)\nplt.show()","metadata":{"_uuid":"dc9cd2d3-f859-4e4b-8633-0a48533c37f9","_cell_guid":"85287aa8-1782-4be1-9d0b-7b5c7c504b7f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:49:25.265232Z","iopub.execute_input":"2024-12-07T22:49:25.266002Z","iopub.status.idle":"2024-12-07T22:49:27.739992Z","shell.execute_reply.started":"2024-12-07T22:49:25.265954Z","shell.execute_reply":"2024-12-07T22:49:27.738634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ✨*Model creation and Prediction*💰  \n\n> *There are no extraordinary thing.*  \n\n---","metadata":{"_uuid":"0e205516-60c1-46ee-ad19-7e2581d6d4e4","_cell_guid":"4923995e-1c27-4a9b-bb24-09254b8df37e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## ✨*Create models to classify, and create prediction models of each category*💰  \n\n> *Using lightgbm binary auc*  \n\n---","metadata":{"_uuid":"0dffd8a4-dcf8-4109-9675-78b0fcb05bc2","_cell_guid":"91a802ce-0c7d-4756-9374-e8312f0ccb08","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"### ✨*Create models to classify*💰  \n\n---","metadata":{"_uuid":"bd3be450-bc2a-4670-9980-bd8a25df98e8","_cell_guid":"f5dadb6f-c89f-4a86-810d-b345819d0879","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# classify over 2000 & under 500\n\ndef train_classify(X_train, X_test, y_train, y_test, params):\n\n    train_data = lgb.Dataset(X_train, label=y_train)\n    test_data = lgb.Dataset(X_test, label=y_test)\n\n    model = lgb.train(\n        params,\n        train_data,\n        num_boost_round=10000,\n        valid_sets=[train_data, test_data],\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=500, verbose=True),\n            lgb.log_evaluation(250),\n        ],\n    )\n    return model\n\n\ndef plot_classify_result(y_test, pred_proba):\n\n    pred = np.where(pred_proba >= 0.5, 1, 0)\n\n    print(\"\\nClassification Report:\")\n    print(classification_report(y_test, pred))\n\n    cm = confusion_matrix(y_test, pred)\n    plt.figure(figsize=(2, 2))\n    sns.heatmap(\n        cm, annot=True, fmt=\"d\", cmap=\"Blues\", square=True,\n        xticklabels=[\"Negative\", \"Positive\"],\n        yticklabels=[\"Negative\", \"Positive\"],\n        cbar_kws={\"shrink\": 0.7},\n    )\n    plt.xlabel(\"Predicted Label\")\n    plt.ylabel(\"True Label\")\n    plt.show()\n\n\ndef premium_classify(df: pl.DataFrame, params: dict):\n    ut.pipeline(df, True)\n    X_c = ut.ct_transform(df.drop(ut.target))\n    y_c = df[ut.target].to_numpy()\n\n    Xc_train, Xc_test, yc_train, yc_test = train_test_split(\n        X_c, y_c, test_size=0.3, random_state=42\n    )\n\n    model = train_classify(Xc_train, Xc_test, yc_train, yc_test, params)\n    pred_test_proba = model.predict(Xc_test)\n    plot_classify_result(yc_test, pred_test_proba)\n\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T22:49:27.741404Z","iopub.execute_input":"2024-12-07T22:49:27.741776Z","iopub.status.idle":"2024-12-07T22:49:27.752162Z","shell.execute_reply.started":"2024-12-07T22:49:27.741743Z","shell.execute_reply":"2024-12-07T22:49:27.750939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preparation for classifying\ndf_over2000 = df.with_columns(\n    pl.when(pl.col(\"Premium Amount\") >= 2000)\n    .then(1).otherwise(0).cast(pl.Int32).alias(\"Premium Amount\")\n)\n\ndf_under500 = df.with_columns(\n    pl.when(pl.col(\"Premium Amount\") < 500)\n    .then(1).otherwise(0).cast(pl.Int32).alias(\"Premium Amount\")\n)\n\nparams_2000 = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"random_state\": 42,\n    \"verbose\": -1,\n    \"learning_rate\": 0.01,\n    \"feature_pre_filter\": False,\n    \"lambda_l1\": 0.0,\n    \"lambda_l2\": 0.0,\n    \"num_leaves\": 83,\n    \"feature_fraction\": 1.0,\n    \"bagging_fraction\": 0.8517449223941215,\n    \"bagging_freq\": 6,\n    \"min_child_samples\": 50,\n}\n\nparams_500 = {\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"random_state\": 42,\n    \"verbose\": -1,\n    \"learning_rate\": 0.02,\n    \"feature_pre_filter\": False,\n    \"lambda_l1\": 0.00011154117261448391,\n    \"lambda_l2\": 6.505134517633417e-05,\n    \"num_leaves\": 33,\n    \"feature_fraction\": 0.7,\n    \"bagging_fraction\": 0.7576770888561737,\n    \"bagging_freq\": 1,\n    \"min_child_samples\": 20,\n}\n\nprint(\"classifying whether Premium Amount is over 2000\")\nmodel_cls_over2000 = premium_classify(df_over2000, params_2000)\nprint(\"classifying whether Premium Amount under 500\")\nmode_cls_under500 = premium_classify(df_under500, params_500)","metadata":{"_uuid":"570b50cf-b26d-466b-9b51-b0a683e04e7e","_cell_guid":"773a5cd9-7be2-4451-a428-43bfce131cd0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T22:49:27.75388Z","iopub.execute_input":"2024-12-07T22:49:27.754385Z","iopub.status.idle":"2024-12-07T23:07:43.010514Z","shell.execute_reply.started":"2024-12-07T22:49:27.754333Z","shell.execute_reply":"2024-12-07T23:07:43.009188Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Create prediction models of each category*💰  \n\n---","metadata":{"_uuid":"0646557f-9c35-432a-86f3-ae9d6a9cabd1","_cell_guid":"45aebb47-ee58-47de-b56c-6bd99b16f088","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# train test split\nut.pipeline(df, True)\n\nmodels = []\ndf_tests = []\nX_trains, X_tests, y_trains, y_tests = [], [], [], []\n\nconditions = [\n    pl.col(\"Premium Amount\") < 500,\n    (pl.col(\"Premium Amount\") >= 500) & (pl.col(\"Premium Amount\") < 2000),\n    pl.col(\"Premium Amount\") >= 2000,\n]\n\nfor cond in conditions:\n    df_temp = df.filter(cond)\n    X = ut.ct_transform(df_temp.drop(ut.target))\n    y = df_temp[ut.target].to_numpy()\n\n    # Train test split\n    X_train, X_test, y_train, y_test, df_temp_train, df_temp_test = train_test_split(\n        X, y, df_temp, test_size=0.3, random_state=42\n    )\n    X_trains.append(X_train)\n    X_tests.append(X_test)\n    y_trains.append(y_train)\n    y_tests.append(y_test)\n    df_tests.append(df_temp_test)","metadata":{"_uuid":"972db809-e43c-4584-a254-ad9452626677","_cell_guid":"eabcbda1-ef0c-49a7-b4ca-e5263ce825b8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:07:43.011957Z","iopub.execute_input":"2024-12-07T23:07:43.012307Z","iopub.status.idle":"2024-12-07T23:07:54.433661Z","shell.execute_reply.started":"2024-12-07T23:07:43.012275Z","shell.execute_reply":"2024-12-07T23:07:54.432543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Classified prediction model\n# LightGBM\ndef train_lgb(X_train, X_test, y_train, y_test):\n    params_lgb = {\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"random_state\": 42,\n        \"verbose\": -1,\n        \"learning_rate\": 0.005,\n        \"max_depth\": -1,\n        \"lambda_l1\": 8.061284137884293,\n        \"lambda_l2\": 0.1808845183357011,\n        \"num_leaves\": 53,\n        \"feature_fraction\": 0.8,\n        \"min_child_samples\": 50,\n    }\n\n    train_data = lgb.Dataset(X_train, label=y_train)\n    test_data = lgb.Dataset(X_test, label=y_test)\n\n    model = lgb.train(\n        params_lgb,\n        train_data,\n        num_boost_round=10000,\n        valid_sets=[train_data, test_data],\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=500, verbose=True),\n            lgb.log_evaluation(250),\n        ],\n    )\n    return model\n\n\nfor i, cond in enumerate(conditions):\n    models.append(train_lgb(X_trains[i], X_tests[i], y_trains[i], y_tests[i]))","metadata":{"_uuid":"972db809-e43c-4584-a254-ad9452626677","_cell_guid":"eabcbda1-ef0c-49a7-b4ca-e5263ce825b8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:07:54.43504Z","iopub.execute_input":"2024-12-07T23:07:54.435396Z","iopub.status.idle":"2024-12-07T23:15:17.251439Z","shell.execute_reply.started":"2024-12-07T23:07:54.435361Z","shell.execute_reply":"2024-12-07T23:15:17.249763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def classify_amount_categories(df:pl.DataFrame, df_id:pl.DataFrame=None):\n\n    if df_id is None:\n        id = [i for i in range(len(df))]\n        df_id = pl.DataFrame(id, schema=[\"id\"])\n\n    if ut.target in df.columns:\n        df_t = df.select(ut.target)\n        df = df.drop(ut.target)\n    else:\n        df_t = pl.DataFrame()\n\n    X = ut.ct_transform(df)\n    \n    pred_over2000 = pl.DataFrame(model_cls_over2000.predict(X), schema=[\"o2000\"])\n    pred_under500 = pl.DataFrame(mode_cls_under500.predict(X), schema=[\"u500\"])\n    df_cls = pl.concat(\n        [df_id, df, df_t, pred_over2000, pred_under500], how=\"horizontal\"\n    )\n    \n    df_cls = df_cls.with_columns(\n        pl.when(pl.col(\"o2000\") >= 0.5).then(2)\n        .when(pl.col(\"u500\") >= 0.5).then(0)\n        .otherwise(1).alias(\"Premium class\")\n    ).drop([\"o2000\", \"u500\"])\n\n    dfs, ids = [], []\n    for i in range(3):\n        df_leaf = df_cls.filter(pl.col(\"Premium class\") == i)\n        ids.append(df_leaf.select(\"id\"))\n        dfs.append(df_leaf.drop([\"id\", \"Premium class\"]))       \n    return dfs, ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T23:15:17.252966Z","iopub.execute_input":"2024-12-07T23:15:17.253397Z","iopub.status.idle":"2024-12-07T23:15:17.265028Z","shell.execute_reply.started":"2024-12-07T23:15:17.25336Z","shell.execute_reply":"2024-12-07T23:15:17.263828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test pattern evaluation\n\ndf_sample = pl.concat(df_tests)\n\ndfs, _ = classify_amount_categories(df_sample)\n\nresult = pl.DataFrame()\nfor i in range(3):\n    X = ut.ct_transform(dfs[i].drop(ut.target))\n    y = dfs[i][ut.target].to_numpy()\n    pred = models[i].predict(X)\n    result = pl.concat([result, pl.DataFrame({\"true\":y, \"pred\":pred})])\n\nplt.figure(figsize=(5,5))\nsns.scatterplot(result.to_pandas(), x=\"true\", y=\"pred\", alpha=0.01)\nsns.lineplot(x=[0, result[\"true\"].max()], y=[0, result[\"true\"].max()], color=\"red\")\nplt.xlabel(\"True (Premium Amount)\")\nplt.ylabel(\"Prediction result\")\nplt.show()","metadata":{"_uuid":"972db809-e43c-4584-a254-ad9452626677","_cell_guid":"eabcbda1-ef0c-49a7-b4ca-e5263ce825b8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:15:17.266203Z","iopub.execute_input":"2024-12-07T23:15:17.266674Z","iopub.status.idle":"2024-12-07T23:17:36.689079Z","shell.execute_reply.started":"2024-12-07T23:15:17.266625Z","shell.execute_reply":"2024-12-07T23:17:36.687885Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Prediction of test data*💰  \n\n> *predict Premium Amount of test data and writing csv.*  \n\n---","metadata":{"_uuid":"90e67262-7c5a-4da3-9b7e-6ee97d78eb8d","_cell_guid":"3cac948e-ccef-42b2-9935-c4fe5d051d99","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"### ✨*Classify*💰  \n\n> *Classify test data into three categories*  \n\n---","metadata":{"_uuid":"3771dc4a-885e-4867-8666-eea6cd78a5d2","_cell_guid":"a3941349-ba89-48d8-bad5-a8abd2e60e04","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# category prediction\n\ndfs, ids = classify_amount_categories(df_obj, df_obj_id)","metadata":{"_uuid":"77973828-746c-4053-8dfb-dc1089a31b0a","_cell_guid":"d16e82b5-a1a4-418a-954b-d756f1cce913","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:17:36.690595Z","iopub.execute_input":"2024-12-07T23:17:36.691028Z","iopub.status.idle":"2024-12-07T23:19:42.288342Z","shell.execute_reply.started":"2024-12-07T23:17:36.690983Z","shell.execute_reply":"2024-12-07T23:19:42.287112Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Premium Amount Prediction*💰  \n\n> *Predict Premium Amount of test data*  \n\n---","metadata":{"_uuid":"124d4202-cdfe-4da8-baf7-513b24f9fc1a","_cell_guid":"79a35b31-fd8e-4c06-8c5c-c4a24bff574f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"preds = []\nfor i in range(3):\n    pred_obj_leaf = models[i].predict(ut.ct_transform(dfs[i]))\n    result_leaf = pl.concat(\n        [ids[i], pl.DataFrame(pred_obj_leaf, schema=[\"Premium Amount\"])],\n        how=\"horizontal\",\n    )\n    preds.append(result_leaf)","metadata":{"_uuid":"e4520de2-893a-4698-8c1e-1f5173468065","_cell_guid":"3de4489a-31bb-4bc9-b912-564ae3462e38","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:19:42.290005Z","iopub.execute_input":"2024-12-07T23:19:42.290468Z","iopub.status.idle":"2024-12-07T23:22:31.840921Z","shell.execute_reply.started":"2024-12-07T23:19:42.29042Z","shell.execute_reply":"2024-12-07T23:22:31.839771Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Making Submission data*💰  \n\n---","metadata":{"_uuid":"8db389b3-5d3b-40d7-a12b-6af7600ca7a4","_cell_guid":"d2b75208-8dcd-464b-8f24-3ac2749854d6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"result = pl.concat(preds).sort(by=\"id\", descending=False)\nresult.write_csv(\"submission.csv\")","metadata":{"_uuid":"acaad38b-9015-4b1c-90f5-b4954b7f2c23","_cell_guid":"efc55dd6-49ac-443e-8701-c455c0bde3f0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:22:31.842452Z","iopub.execute_input":"2024-12-07T23:22:31.842967Z","iopub.status.idle":"2024-12-07T23:22:31.99961Z","shell.execute_reply.started":"2024-12-07T23:22:31.842919Z","shell.execute_reply":"2024-12-07T23:22:31.998377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"end_time = time.time()\nelapsed = end_time - start_time\nprint(f\"elapsed time : {elapsed//3600}h {elapsed//60}min {elapsed%60:.1f}sec\")","metadata":{"_uuid":"4321b2fc-dfa2-4ac7-a521-821fc69dc446","_cell_guid":"5f5d65bd-f226-4695-9cb5-1753bee302d2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2024-12-07T23:22:32.001031Z","iopub.execute_input":"2024-12-07T23:22:32.001388Z","iopub.status.idle":"2024-12-07T23:22:32.009433Z","shell.execute_reply.started":"2024-12-07T23:22:32.001357Z","shell.execute_reply":"2024-12-07T23:22:32.008368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"d78c6faf-716b-489d-900a-f2c9892096e1","_cell_guid":"f3067a9b-7c09-4ce2-a694-5b0fe7689fb4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}