{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\nThis notebook is an example of how ensembling can benefit your result. **Please make sure you checked out the previous notebooks to fully understand the task** of this competition and equipped a **full understanding of basic ML**.\n- [Titanic Top 11%| Starter II: Hyperparameter Tuning](https://www.kaggle.com/chienhsianghung/titanic-top-11-starter-ii-hyperparameter-tuning)\n- [TPS Apr.| Starter Pack: All Models](https://www.kaggle.com/chienhsianghung/tps-apr-starter-pack-all-models)\n- [Titanic Top 11%| Starter I: Models Comparing](https://www.kaggle.com/chienhsianghung/titanic-top-11-starter-i-models-comparing)\n\nOther than that, special thanks to [@quincyqiang](https://www.kaggle.com/quincyqiang) who made this amazing [notebook](https://www.kaggle.com/quincyqiang/tps-apr-2021-pseudo-labeling-voting-ensemble).","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport os\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold\n\nimport lightgbm as lgb\nimport catboost as ctb\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.tree import DecisionTreeClassifier, export_graphviz\n\nimport graphviz\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET = 'Survived'\n\nN_ESTIMATORS = 1000\nN_SPLITS = 10\nSEED = 42\nEARLY_STOPPING_ROUNDS = 100\nVERBOSE = 100\n\nALL_SIMPLE_AVERAGE = False","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\nset_seed(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/tabular-playground-series-apr-2021/train.csv')\ntest_df = pd.read_csv('../input/tabular-playground-series-apr-2021/test.csv')\nsubmission = pd.read_csv('../input/tabular-playground-series-apr-2021/sample_submission.csv')\ntest_df[TARGET] = pd.read_csv('../input/tps-apr-2021-label/pseudo_label.csv')[TARGET]\n\nall_df = pd.concat([train_df, test_df]).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filling Missing Values","metadata":{}},{"cell_type":"code","source":"all_df.Age = all_df.Age.fillna(all_df.Age.mean())\nall_df.Cabin = all_df.Cabin.fillna('X').map(lambda x: x[0].strip())\nall_df.Ticket = all_df.Ticket.fillna('X').map(lambda x: str(x).split()[0] if len(str(x).split()) > 1 else 'X')\n\nfare_map = all_df[['Fare', 'Pclass']].dropna().groupby('Pclass').median().to_dict()\nall_df.Fare = all_df.Fare.fillna(all_df.Pclass.map(fare_map['Fare']))\nall_df.Fare = np.log1p(all_df.Fare)\n\nall_df.Embarker = all_df.Embarked.fillna('X')\nall_df.Name = all_df.Name.map(lambda x: x.split(',')[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding","metadata":{}},{"cell_type":"code","source":"label_cols = ['Name', 'Ticket', 'Sex']\nonehot_cols = ['Cabin', 'Embarked']\nnumerical_cols = ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_encoder(c):\n    le = LabelEncoder()\n    return le.fit_transform(c)\n\nscaler = StandardScaler()\n\nonehot_encoded_df = pd.get_dummies(all_df[onehot_cols])\nlabel_encoded_df = all_df[label_cols].apply(label_encoder)\nnumerical_df = pd.DataFrame(scaler.fit_transform(all_df[numerical_cols]), columns=numerical_cols)\ntarget_df = all_df[TARGET]\n\nall_df = pd.concat([onehot_encoded_df, label_encoded_df, numerical_df, target_df], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM","metadata":{}},{"cell_type":"code","source":"params = {\n    'metric': 'binary_logloss',\n    'n_estimators': N_ESTIMATORS,\n    'objective': 'binary',\n    'random_state': SEED,\n    'learning_rate': 0.01,\n    'min_child_samples': 150,\n    'reg_alpha': 3e-5,\n    'reg_lambda': 9e-2,\n    'num_leaves': 20,\n    'max_depth': 16,\n    'colsample_bytree': 0.8,\n    'subsample': 0.8,\n    'subsample_freq': 2,\n    'max_bin': 240,\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_oof = np.zeros(train_df.shape[0])\nlgb_preds = np.zeros(test_df.shape[0])\nfeature_importances = pd.DataFrame()\n\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(all_df, all_df[TARGET])):\n    print(f\"===== FOLD {fold} =====\")\n    oof_idx = np.array([idx for idx in valid_idx if idx < train_df.shape[0]])\n    preds_idx = np.array([idx for idx in valid_idx if idx >= train_df.shape[0]])\n\n    X_train, y_train = all_df.iloc[train_idx].drop(TARGET, axis=1), all_df.iloc[train_idx][TARGET]\n    X_valid, y_valid = all_df.iloc[oof_idx].drop(TARGET, axis=1), all_df.iloc[oof_idx][TARGET]\n    X_test = all_df.iloc[preds_idx].drop(TARGET, axis=1)\n    \n    pre_model = lgb.LGBMRegressor(**params)\n    pre_model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train),(X_valid, y_valid)],\n        early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n        verbose=VERBOSE\n    )\n    \n    params2 = params.copy()\n    params2['learning_rate'] = params['learning_rate'] * 0.1\n    model = pre_model.set_params(**params2)    \n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train),(X_valid, y_valid)],\n        early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n        verbose=VERBOSE,\n    )\n    \n    fi_tmp = pd.DataFrame()\n    fi_tmp[\"feature\"] = all_df.columns.values[:-1]\n    fi_tmp[\"importance\"] = model.feature_importances_\n    fi_tmp[\"fold\"] = fold\n    fi_tmp[\"seed\"] = SEED\n    feature_importances = feature_importances.append(fi_tmp)\n    \n    lgb_oof[oof_idx] = model.predict(X_valid)\n    lgb_preds[preds_idx-train_df.shape[0]] = model.predict(X_test)\n    \n    acc_score = accuracy_score(y_valid, np.where(lgb_oof[oof_idx]>0.5, 1, 0))\n    print(f\"===== ACCURACY SCORE {acc_score:.6f} =====\\n\")\n    \nacc_score = accuracy_score(all_df[:train_df.shape[0]][TARGET], np.where(lgb_oof>0.5, 1, 0))\nprint(f\"===== ACCURACY SCORE {acc_score:.6f} =====\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just to get ideas to improve\norder = list(feature_importances.groupby(\"feature\").mean().sort_values(\"importance\", ascending=False).index)\nplt.figure(figsize=(10, 10))\nsns.barplot(x=\"importance\", y=\"feature\", data=feature_importances, order=order)\nplt.title(\"{} importance\".format(\"CatBoostClassifier\"))\nplt.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Catboost","metadata":{}},{"cell_type":"code","source":"params = {\n    'bootstrap_type': None, # None, 'Poisson', 'Bayesian', 'Gaussian'\n    'loss_function': 'Logloss',\n    'eval_metric': 'Logloss',\n    'random_seed': SEED,\n    'task_type': 'CPU', # GPU\n    'max_depth': 8,\n    'learning_rate': 0.01,\n    'n_estimators': N_ESTIMATORS,\n    'max_bin': 280,\n    'min_data_in_leaf': 64,\n    'l2_leaf_reg': 0.01,\n    'subsample': 0.8\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctb_oof = np.zeros(train_df.shape[0])\nctb_preds = np.zeros(test_df.shape[0])\nfeature_importances = pd.DataFrame()\n\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(all_df, all_df[TARGET])):\n    print(f\"===== FOLD {fold} =====\")\n    oof_idx = np.array([idx for idx in valid_idx if idx < train_df.shape[0]])\n    preds_idx = np.array([idx for idx in valid_idx if idx >= train_df.shape[0]])\n\n    X_train, y_train = all_df.iloc[train_idx].drop(TARGET, axis=1), all_df.iloc[train_idx][TARGET]\n    X_valid, y_valid = all_df.iloc[oof_idx].drop(TARGET, axis=1), all_df.iloc[oof_idx][TARGET]\n    X_test = all_df.iloc[preds_idx].drop(TARGET, axis=1)\n    \n    model = ctb.CatBoostClassifier(**params)\n    model.fit(X_train, y_train,\n              eval_set=[(X_valid, y_valid)],\n              use_best_model=True,\n              early_stopping_rounds=EARLY_STOPPING_ROUNDS,\n              verbose=VERBOSE\n              )\n    \n    fi_tmp = pd.DataFrame()\n    fi_tmp[\"feature\"] = X_test.columns.to_list()\n    fi_tmp[\"importance\"] = model.get_feature_importance()\n    fi_tmp[\"fold\"] = fold\n    fi_tmp[\"seed\"] = SEED\n    feature_importances = feature_importances.append(fi_tmp)\n    \n    ctb_oof[oof_idx] = model.predict(X_valid)\n    ctb_preds[preds_idx-train_df.shape[0]] = model.predict(X_test)\n    \n    acc_score = accuracy_score(y_valid, np.where(ctb_oof[oof_idx]>0.5, 1, 0))\n    print(f\"===== ACCURACY SCORE {acc_score:.6f} =====\\n\")\n    \nacc_score = accuracy_score(all_df[:train_df.shape[0]][TARGET], np.where(ctb_oof>0.5, 1, 0))\nprint(f\"===== ACCURACY SCORE {acc_score:.6f} =====\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just to get ideas to improve\norder = list(feature_importances.groupby(\"feature\").mean().sort_values(\"importance\", ascending=False).index)\nplt.figure(figsize=(10, 10))\nsns.barplot(x=\"importance\", y=\"feature\", data=feature_importances, order=order)\nplt.title(\"{} importance\".format(\"CatBoostClassifier\"))\nplt.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DecisionTreeModel","metadata":{}},{"cell_type":"code","source":"# Tuning the DecisionTreeClassifier by the GridSearchCV\nparameters = {\n    'max_depth': np.arange(2, 5, dtype=int),\n    'min_samples_leaf':  np.arange(2, 5, dtype=int)\n}\n\nclassifier = DecisionTreeClassifier(random_state=2021)\n\nmodel = GridSearchCV(\n    estimator=classifier,\n    param_grid=parameters,\n    scoring='accuracy',\n    cv=10,\n    n_jobs=-1)\nmodel.fit(X_train, y_train)\n\nbest_parameters = model.best_params_\nprint(best_parameters)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtm_oof = np.zeros(train_df.shape[0])\ndtm_preds = np.zeros(test_df.shape[0])\nfeature_importances = pd.DataFrame()\n\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(all_df, all_df[TARGET])):\n    print(f\"===== FOLD {fold} =====\")\n    oof_idx = np.array([idx for idx in valid_idx if idx < train_df.shape[0]])\n    preds_idx = np.array([idx for idx in valid_idx if idx >= train_df.shape[0]])\n\n    X_train, y_train = all_df.iloc[train_idx].drop(TARGET, axis=1), all_df.iloc[train_idx][TARGET]\n    X_valid, y_valid = all_df.iloc[oof_idx].drop(TARGET, axis=1), all_df.iloc[oof_idx][TARGET]\n    X_test = all_df.iloc[preds_idx].drop(TARGET, axis=1)\n    \n    model = DecisionTreeClassifier(\n        max_depth=best_parameters['max_depth'],\n        min_samples_leaf=best_parameters['min_samples_leaf'],\n        random_state=SEED\n    )\n    model.fit(X_train, y_train)\n    \n    dtm_oof[oof_idx] = model.predict(X_valid)\n    dtm_preds[preds_idx-train_df.shape[0]] = model.predict(X_test)\n    \n    acc_score = accuracy_score(y_valid, np.where(dtm_oof[oof_idx]>0.5, 1, 0))\n    print(f\"===== ACCURACY SCORE {acc_score:.6f} =====\\n\")\n    \nacc_score = accuracy_score(all_df[:train_df.shape[0]][TARGET], np.where(dtm_oof>0.5, 1, 0))\nprint(f\"===== ACCURACY SCORE {acc_score:.6f} =====\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot tree\ndot_data = export_graphviz(\n    model,\n    out_file=None,\n    feature_names=X_train.columns,\n    class_names=['0', '1'],\n    filled=True,\n    rounded=False,\n    special_characters=True,\n    precision=3\n)\ngraph = graphviz.Source(dot_data)\ngraph ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble\n\n|   | LB |\n| - | -- |\n| seed42 | 0.81646 |\n| seed1 | 0.81638 |\n| ASA42 | lose a chance to try |","metadata":{}},{"cell_type":"code","source":"if ALL_SIMPLE_AVERAGE:\n    submission[TARGET] = np.where(np.average((lgb_preds, ctb_preds, dtm_preds), axis=0) > 0.5, 1, 0)\n    submission.to_csv(\"ensembled_submission.csv\", index = False)\n    submission[TARGET].value_counts()\n    \nelse:\n    submission['submit_lgb'] = np.where(lgb_preds>0.5, 1, 0)\n    submission['submit_ctb'] = np.where(ctb_preds>0.5, 1, 0)\n    submission['submit_dtm'] = np.where(dtm_preds>0.5, 1, 0)\n    \n    submission[TARGET] = (submission[[col for col in submission.columns if col.startswith('submit_')]].sum(axis=1) >= 2).astype(int)\n    submission.drop([col for col in submission.columns if col.startswith('submit_')], axis=1, inplace=True)\n    submission.to_csv(\"ensembled_submission.csv\", index = False)\n    \n    submission[[col for col in submission.columns if col.startswith('submit_')]].sum(axis = 1).value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[TARGET].hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}