{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport sys \nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom xgboost import XGBRegressor\nimport xgboost as xgb, time\n\ntrain = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\",)\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\ncol_map = {}\nfor col in train.columns:\n  col_map[col] = '_'.join(col.split(' '))\ntrain = train.rename(columns=col_map)\ntest = test.rename(columns=col_map)\n\n## 处理日期变量\ndef pre_process(data):\n  data[\"Policy_Start_Date\"] = pd.to_datetime( data[\"Policy_Start_Date\"] )\n  data[\"year\"] = data[\"Policy_Start_Date\"].dt.year.astype(\"float32\")\n  data[\"month\"] = data[\"Policy_Start_Date\"].dt.month.astype(\"float32\")\n  data[\"day\"] = data[\"Policy_Start_Date\"].dt.day.astype(\"float32\")\n  data[\"dow\"] = data[\"Policy_Start_Date\"].dt.dayofweek.astype(\"float32\")\n  data[\"seconds\"] = (data[\"Policy_Start_Date\"].astype(\"int64\") // 10**9).astype(\"float32\")\n\n  return data\n\ntrain = pre_process(train)\ntest = pre_process(test)\ntrain = train.drop('Policy_Start_Date',axis=1)\ntest = test.drop('Policy_Start_Date',axis=1)\n\ntrain[\"target\"] = np.log1p( train[\"Premium_Amount\"])  ##np.log(train[\"Premium_Amount\"]+1)\ntrain.drop('Premium_Amount',axis=1,inplace=True)\n\n\n# 定义移除的列\ncolumns_to_remove = [\"id\", \"Policy Start Date\", \"Premium Amount\", \"target\"]\n\n# 从训练集和测试集中筛选出需要的特征列\nfeatures = [col for col in train.columns if col not in columns_to_remove]\n\n# 合并训练集和测试集\ncombined = pd.concat([train, test], axis=0, ignore_index=True)\n\n# 初始化分类特征和高基数特征列表\ncategorical_features = []\nhigh_cardinality_features = []\n\nprint(f\"THE {len(features)} BASIC features ARE:\")\n\n# 遍历特征列，处理数据类型和缺失值\nfor feature in features:\n    feature_type = \"numerical\"\n    unique_values_count = combined[feature].nunique()\n\n    # 处理分类特征\n    if combined[feature].dtype == \"object\":\n        categorical_features.append(feature)\n        combined[feature] = combined[feature].fillna(\"NAN\")\n        combined[feature], _ = combined[feature].factorize()\n        combined[feature] -= combined[feature].min()\n        feature_type = \"categorical\"\n\n    # 优化数据类型\n    if combined[feature].dtype == \"int64\":\n        combined[feature] = combined[feature].astype(\"int32\")\n    elif combined[feature].dtype == \"float64\":\n        combined[feature] = combined[feature].astype(\"float32\")\n\n    print(f\"{feature} ({feature_type}) with {unique_values_count} unique values\")\n\n    # 检测高基数特征\n    if unique_values_count >= 9:\n        high_cardinality_features.append(feature)\n\n# 分离训练集和测试集\ntrain = combined.iloc[:len(train)].copy()\ntest = combined.iloc[len(train):].reset_index(drop=True).copy()\n\nprint(\"\\nTHE FOLLOWING HAVE 9 OR MORE UNIQUE VALUES:\", high_cardinality_features)\n\nencoding_features = [\n    ['Annual_Income', 'Health_Score'], \n    ['Credit_Score', 'Health_Score'], \n    ['Customer_Feedback', 'Gender', 'Marital_Status', 'Occupation', 'Smoking_Status', 'year'],\n    ['Exercise_Frequency', 'Health_Score'],\n    ['Health_Score', 'Marital_Status'],\n    ['Education_Level', 'Gender', 'Health_Score'],\n    ['Health_Score', 'Occupation'],\n    ['Age', 'Health_Score'],\n    ['Health_Score', 'dow'],\n    ['Age', 'Exercise_Frequency', 'Location'],\n    ['Health_Score', 'Smoking_Status', 'month'],\n    ['Health_Score', 'Location', 'Policy_Type'],\n    ['Health_Score', 'Insurance_Duration'],\n    ['Health_Score', 'Number_of_Dependents'],\n    ['Customer_Feedback', 'Exercise_Frequency', 'Previous_Claims', 'Property_Type', 'dow'],\n    ['Customer_Feedback', 'Health_Score'],\n    ['Health_Score', 'Property_Type'],\n    ['Health_Score', 'day', 'seconds'],\n    ['Health_Score', 'year'],\n    ['Age', 'Gender', 'Insurance_Duration', 'year']\n]\n\n\ndef target_encode(train, valid, test, col, target=\"target\", kfold=5, smooth=20, agg=\"mean\"):\n    \"\"\"\n    对指定列进行目标编码（Target Encoding），并使用 k 折交叉验证来避免过拟合。\n\n    参数：\n        train (pd.DataFrame): 训练集。\n        valid (pd.DataFrame): 验证集。\n        test (pd.DataFrame): 测试集。\n        col (list): 需要进行目标编码的列名列表。\n        target (str): 目标变量列名，默认为 \"target\"。\n        kfold (int): 交叉验证的折数，默认为 5。\n        smooth (int): 平滑参数，用于防止过拟合，默认为 20。\n        agg (str): 聚合方法，可选值为 \"mean\"、\"median\"、\"min\"、\"max\"、\"nunique\"，默认为 \"mean\"。\n\n    返回：\n        tuple: 包含目标编码列的 (train, valid, test)。\n    \"\"\"\n    # 初始化 k 折列和目标编码列名\n    train['kfold'] = train.index % kfold\n    col_name = '_'.join(col)\n    target_encoded_col = f'TE_{agg.upper()}_{col_name}'\n    train[target_encoded_col] = 0.0\n\n    # 计算全局聚合值\n    if agg == \"mean\":\n        global_agg = train[target].mean()\n    elif agg == \"median\":\n        global_agg = train[target].median()\n    elif agg == \"min\":\n        global_agg = train[target].min()\n    elif agg == \"max\":\n        global_agg = train[target].max()\n    elif agg == \"nunique\":\n        global_agg = 0\n    else:\n        raise ValueError(\"无效的聚合方法。请选择 'mean'、'median'、'min'、'max' 或 'nunique'。\")\n\n    # 对训练集进行 k 折目标编码\n    for fold in range(kfold):\n        # 分割数据为训练折和验证折\n        df_train = train[train['kfold'] != fold]\n        df_valid = train[train['kfold'] == fold]\n\n        # 计算每个组合的聚合值\n        # 问ai\n        agg_values = df_train.groupby(col)[target].agg([agg, 'count']).reset_index()\n\n        agg_values.columns = col + [agg, 'count']\n\n        if agg == \"nunique\":\n            # 如果是 nunique，计算唯一值的比例\n            agg_values['TE_tmp'] = agg_values[agg] / agg_values['count']\n        else:\n            # 使用平滑公式计算目标编码值\n            agg_values['TE_tmp'] = ((agg_values[agg] * agg_values['count']) + (global_agg * smooth)) / (agg_values['count'] + smooth)\n\n        # 将目标编码值合并回验证折\n        # 问ai\n        df_valid = df_valid.merge(agg_values[col + ['TE_tmp']], on=col, how='left')\n        df_valid[target_encoded_col] = df_valid['TE_tmp'].fillna(global_agg)\n\n        # 更新训练集的目标编码列\n        train.loc[train['kfold'] == fold, target_encoded_col] = df_valid[target_encoded_col].values\n\n    # 删除 k 折列\n    train = train.drop('kfold', axis=1)\n    train[target_encoded_col] = train[target_encoded_col].astype(\"float32\")\n\n    # 计算全局目标编码值，用于验证集和测试集\n    agg_values = train.groupby(col)[target].agg([agg, 'count']).reset_index()\n    agg_values.columns = col + [agg, 'count']\n\n    if agg == \"nunique\":\n        agg_values['TE_tmp'] = agg_values[agg] / agg_values['count']\n    else:\n        agg_values['TE_tmp'] = ((agg_values[agg] * agg_values['count']) + (global_agg * smooth)) / (agg_values['count'] + smooth)\n\n    # 将目标编码值合并到验证集和测试集\n    valid = valid.merge(agg_values[col + ['TE_tmp']], on=col, how='left',suffixes=['','_'])\n    valid[target_encoded_col] = valid['TE_tmp'].fillna(global_agg).astype(\"float32\")\n\n    test = test.merge(agg_values[col + ['TE_tmp']], on=col, how='left',suffixes=['','_'])\n    test[target_encoded_col] = test['TE_tmp'].fillna(global_agg).astype(\"float32\")\n\n    return train, valid, test\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FOLDS = 5\nfrom sklearn.model_selection import KFold\nkf = KFold(n_splits=FOLDS, shuffle=True, random_state=42)\n\noof = np.zeros(len(train))\npred = np.zeros(len(test))\n\nfor i, (train_index, test_index) in enumerate(kf.split(train)):\n\n    print(\"#\"*25)\n    print(f\"### Fold {i+1}\")\n    print(\"#\"*25)\n    \n    x_train = train.loc[train_index,features+[\"target\"] ].copy()\n    y_train = train.loc[train_index,\"target\"]\n    x_valid = train.loc[test_index,features].copy()\n    y_valid = train.loc[test_index,\"target\"]\n    x_test = test[features].copy()\n\n    start = time.time()\n    print(f\"FEATURE ENGINEER {len(features)} COLUMNS and {len(encoding_features)} GROUPS: \",end=\"\")\n    for j,f in enumerate(features+encoding_features):\n\n        if j<len(features): c = [f]\n        else: c = f \n        print(f\"({j+1}){c}\",\", \",end=\"\")\n\n        # LOW CARDINALITY features - TARGET ENCODE MEAN AND MEDIAN\n        x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=20, agg=\"mean\")\n        x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"median\")\n\n        # HIGH CARDINALITY features - TE MIN, MAX, NUNIQUE and CE\n        if (j>=len(features)) | (c[0] in high_cardinality_features):\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"min\")\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"max\")\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"nunique\")\n    \n            # COUNT ENCODING (USING COMBINED TRAIN TEST)\n            tmp = combined.groupby(c)['target'].count()\n            nm = f\"CE_{'_'.join(c)}\"; tmp.name = nm\n            x_train = x_train.merge(tmp, on=c, how=\"left\")\n            x_valid = x_valid.merge(tmp, on=c, how=\"left\")\n            x_test = x_test.merge(tmp, on=c, how=\"left\")\n\n            x_train[nm] = x_train[nm].fillna(x_train[nm].mean())\n            x_valid[nm] = x_valid[nm].fillna(x_valid[nm].mean())\n            x_test[nm] = x_test[nm].fillna(x_test[nm].mean())\n\n            x_train[nm] = x_train[nm].astype(\"int32\")\n            x_valid[nm] = x_valid[nm].astype(\"int32\")\n            x_test[nm] = x_test[nm].astype(\"int32\")     \n    end = time.time()\n    elapsed = end-start\n    print(f\"Feature engineering took {elapsed:.1f} seconds\")\n    x_train = x_train.drop(\"target\",axis=1)\n\n    model = XGBRegressor(\n        device=\"cuda\",\n        max_depth=8, \n        colsample_bytree=0.9, \n        subsample=0.9, \n        n_estimators=2_000, \n        learning_rate=0.01, \n        early_stopping_rounds=25,  \n        eval_metric=\"rmse\",\n    )\n    model.fit(\n        x_train, y_train,\n        eval_set=[(x_valid[x_train.columns], y_valid)],   \n        verbose=100\n    )\n\n    # INFER OOF\n    oof[test_index] = model.predict(x_valid[x_train.columns])\n    # INFER TEST\n    pred += model.predict(x_test[x_train.columns])\n\n    m = np.sqrt(np.mean( (y_valid.to_numpy() - oof[test_index])**2.0 )) \n    print(f\" => Fold {i+1} RMSLE = {m:.5f}\")\n\n# COMPUTE AVERAGE TEST PREDS\npred /= FOLDS","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score_df = pd.DataFrame()\nfor name in ['train','valid']:\n  if name =='train':\n    x = x_train\n    y = y_train\n  else:\n    x = x_valid[x_train.columns]\n    y = y_valid\n  predictions = model.predict(x)\n  valid_score = {}\n  valid_score['MSE']= np.mean((y - predictions) ** 2)\n  valid_score['MAE']= np.mean(np.abs(y - predictions))\n  valid_score['RMSE']= np.sqrt(np.mean((y - predictions) ** 2))\n  df = pd.DataFrame(valid_score,index=[0])\n  df['Name'] = name\n  score_df = pd.concat([score_df,df],axis=0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\npredictions = model.predict(x_test[x_train.columns])\nsubmission_df['Premium Amount'] = np.exp(predictions)-1\nsubmission_df.to_csv('submission.csv', index=False)\nprint(submission_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}