{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n","metadata":{}},{"cell_type":"markdown","source":"<div style=\"text-align: center;\">\n<h1> Please don't forget to vote if it helps you! ❤️ <h1/>\n</div>","metadata":{}},{"cell_type":"markdown","source":"# References\n\nThank @BigData RPG for the simple start lgbm https://www.kaggle.com/code/boydbigdatarpg/simple-start-with-tuned-lgbm","metadata":{}},{"cell_type":"markdown","source":"# LOGING VERSION\nVersion 1: Train, Test spliting 5 folds + XGBoost, LGBM and CatBoost + Optuna Tuning -- local 0.4901 LB: 0.03","metadata":{}},{"cell_type":"markdown","source":"# 1. Import Library","metadata":{}},{"cell_type":"code","source":"import os, gc\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport sys\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier\nimport optuna\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nfrom imblearn.over_sampling import ADASYN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-01T14:22:26.322353Z","iopub.execute_input":"2022-12-01T14:22:26.322971Z","iopub.status.idle":"2022-12-01T14:22:29.263008Z","shell.execute_reply.started":"2022-12-01T14:22:26.322841Z","shell.execute_reply":"2022-12-01T14:22:29.262095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.1 Parameter Setup","metadata":{}},{"cell_type":"code","source":"is_tune_params = True\nCATEGORICAL_COL = [\"view\", \"implant\",\"machine_id\", \"laterality\"]\nNUMERICAL_COL = [\"age\"]\nTARGET_COLS = [\"cancer\"]\nRANDOM_SEED = 42\n\ndef set_seed(seed=2022):\n    np.random.seed(seed)\n    #tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    #os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    \nset_seed(RANDOM_SEED)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.264448Z","iopub.execute_input":"2022-12-01T14:22:29.265511Z","iopub.status.idle":"2022-12-01T14:22:29.271719Z","shell.execute_reply.started":"2022-12-01T14:22:29.265475Z","shell.execute_reply":"2022-12-01T14:22:29.270735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/rsna-breast-cancer-detection/train.csv\")\ndf_test = pd.read_csv(\"../input/rsna-breast-cancer-detection/test.csv\").drop_duplicates(subset='prediction_id')\ndf_sub = pd.read_csv(\"../input/rsna-breast-cancer-detection/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.274244Z","iopub.execute_input":"2022-12-01T14:22:29.274553Z","iopub.status.idle":"2022-12-01T14:22:29.443257Z","shell.execute_reply.started":"2022-12-01T14:22:29.274516Z","shell.execute_reply":"2022-12-01T14:22:29.44221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FOLD = 5\nskf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=RANDOM_SEED)\nfor n, (train_index, val_index) in enumerate(skf.split(df_train, df_train[TARGET_COLS])):\n    df_train.loc[val_index, 'fold'] = int(n)\ndf_train['fold'] = df_train['fold'].astype(int)\ndf_train[\"age\"] = df_train[\"age\"].fillna(df_train[\"age\"].median())","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.445947Z","iopub.execute_input":"2022-12-01T14:22:29.447055Z","iopub.status.idle":"2022-12-01T14:22:29.483312Z","shell.execute_reply.started":"2022-12-01T14:22:29.446983Z","shell.execute_reply":"2022-12-01T14:22:29.482242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.485052Z","iopub.execute_input":"2022-12-01T14:22:29.485749Z","iopub.status.idle":"2022-12-01T14:22:29.508466Z","shell.execute_reply.started":"2022-12-01T14:22:29.485704Z","shell.execute_reply":"2022-12-01T14:22:29.507146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2 Check target Distribution ","metadata":{}},{"cell_type":"code","source":"df_train[TARGET_COLS].value_counts() * 100 / len(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.510101Z","iopub.execute_input":"2022-12-01T14:22:29.510657Z","iopub.status.idle":"2022-12-01T14:22:29.530008Z","shell.execute_reply.started":"2022-12-01T14:22:29.510614Z","shell.execute_reply":"2022-12-01T14:22:29.528599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3 Check class weight each class","metadata":{}},{"cell_type":"code","source":"len(df_train) / (2 * np.bincount(df_train['cancer'].values))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.531654Z","iopub.execute_input":"2022-12-01T14:22:29.532049Z","iopub.status.idle":"2022-12-01T14:22:29.540357Z","shell.execute_reply.started":"2022-12-01T14:22:29.532011Z","shell.execute_reply":"2022-12-01T14:22:29.539273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Prepare Dataset","metadata":{}},{"cell_type":"code","source":"# For now, let's only use the train dataset without consider the images. Working in process!","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.54173Z","iopub.execute_input":"2022-12-01T14:22:29.54207Z","iopub.status.idle":"2022-12-01T14:22:29.556322Z","shell.execute_reply.started":"2022-12-01T14:22:29.542039Z","shell.execute_reply":"2022-12-01T14:22:29.555452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Tuning Model\n- If you want to try tuning use this part and try varying range of parameters","metadata":{}},{"cell_type":"code","source":"for fold in range(N_FOLD):\n    print(f'\\n-----------FOLD {fold} ------------')\n    print('Data prepared.')\n    train_df = df_train[df_train['fold'] != fold].reset_index(drop=True)\n    valid_df = df_train[df_train['fold'] == fold].reset_index(drop=True)\n\n    dv = DictVectorizer(sparse=False)\n    ada = ADASYN(random_state=RANDOM_SEED)\n\n    train_dict = train_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    \n    X_train = dv.fit_transform(train_dict)\n    y_train = train_df[TARGET_COLS].values\n    X_res_train, y_res_train = ada.fit_resample(X_train, y_train)\n    \n\n    val_dict = valid_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_val = dv.transform(val_dict)\n    y_val = valid_df[TARGET_COLS].values\n\n\n    test_dict = df_test[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_test = dv.transform(test_dict)\n\n#     save_dict = f\"./dict_fold{fold}.pkl\"\n#     pickle.dump(dv, open(save_dict, 'wb'))\n\n    class_weight_arr = len(train_df) / (2 * np.bincount(train_df['cancer'].values))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:29.557615Z","iopub.execute_input":"2022-12-01T14:22:29.558216Z","iopub.status.idle":"2022-12-01T14:22:33.814968Z","shell.execute_reply.started":"2022-12-01T14:22:29.558183Z","shell.execute_reply":"2022-12-01T14:22:33.813801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective_LGBM(trial):\n    params = {\n        'metric': 'f1',\n        'random_state': 42,\n        'n_estimators': 300,\n        'learning_rate': 0.1,\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),\n        'max_depth': trial.suggest_categorical('max_depth', [10,20,100]),\n        'num_leaves' : trial.suggest_int('num_leaves', 1, 1000),\n        'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),\n        'cat_smooth' : trial.suggest_int('min_data_per_groups', 1, 100)\n    }\n    model = lgb.LGBMClassifier(**params, zero_as_missing=True)\n\n    model.fit(X_train, y_train)\n\n    y_va_pred = model.predict(X_val)\n    f1 = f1_score(y_val, y_va_pred, pos_label=1, average='macro')\n    \n    return f1\n\n\ndef objective_XGB(trial):\n    params = {\n        'max_depth': trial.suggest_int('max_depth', 1, 9),\n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 1.0),\n        'n_estimators': trial.suggest_int('n_estimators', 50, 500),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n        'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n        'subsample': trial.suggest_loguniform('subsample', 0.01, 1.0),\n        'colsample_bytree': trial.suggest_loguniform('colsample_bytree', 0.01, 1.0),\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 1.0),\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 1.0),\n        'eval_metric': 'mlogloss',\n        'use_label_encoder': False\n    }\n    \n    model = xgb.XGBClassifier(**params)\n\n    model.fit(X_train, y_train)\n\n    y_va_pred = model.predict(X_val)\n    f1 = f1_score(y_val, y_va_pred, pos_label=1, average='macro')\n    \n    return f1\n\ndef objective_Catboost(trial):\n    params = {\n        \"objective\": trial.suggest_categorical(\"objective\", [\"Logloss\", \"CrossEntropy\"]),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.01, 0.1),\n        \"depth\": trial.suggest_int(\"depth\", 1, 12),\n        \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"Ordered\", \"Plain\"]),\n        \"bootstrap_type\": trial.suggest_categorical(\n            \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\", \"MVS\"]\n        ),\n        \"used_ram_limit\": \"3gb\",\n    }\n    \n    if params[\"bootstrap_type\"] == \"Bayesian\":\n        params[\"bagging_temperature\"] = trial.suggest_float(\"bagging_temperature\", 0, 10)\n    elif params[\"bootstrap_type\"] == \"Bernoulli\":\n        params[\"subsample\"] = trial.suggest_float(\"subsample\", 0.1, 1)\n\n    model = CatBoostClassifier(**params)\n    \n    model.fit(X_train, y_train)\n\n    y_va_pred = model.predict(X_val)\n    f1 = f1_score(y_val, y_va_pred, pos_label=1, average='macro')\n    \n    return f1","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:33.819632Z","iopub.execute_input":"2022-12-01T14:22:33.819985Z","iopub.status.idle":"2022-12-01T14:22:33.836672Z","shell.execute_reply.started":"2022-12-01T14:22:33.819952Z","shell.execute_reply":"2022-12-01T14:22:33.835546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LGBM_params = {}\nXGB_params  = {}\nCatboost_params = {}","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:22:33.838199Z","iopub.execute_input":"2022-12-01T14:22:33.838896Z","iopub.status.idle":"2022-12-01T14:22:33.867347Z","shell.execute_reply.started":"2022-12-01T14:22:33.838862Z","shell.execute_reply":"2022-12-01T14:22:33.86636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_tune_params:\n    study_LGBM = optuna.create_study(\n        direction='maximize', \n        pruner=optuna.pruners.MedianPruner(n_warmup_steps=20),\n        study_name='RSNA')\n    study_LGBM.optimize(objective_LGBM, n_trials=20)\n    LGBM_params = study_LGBM.best_params\ngc.collect()","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:22:33.86861Z","iopub.execute_input":"2022-12-01T14:22:33.869341Z","iopub.status.idle":"2022-12-01T14:22:54.572431Z","shell.execute_reply.started":"2022-12-01T14:22:33.869308Z","shell.execute_reply":"2022-12-01T14:22:54.571356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_tune_params:\n    study_XBG = optuna.create_study(\n        direction='maximize', \n        pruner=optuna.pruners.MedianPruner(n_warmup_steps=20),\n        study_name='RSNA')\n    study_XBG.optimize(objective_XGB, n_trials=20)\n    XGB_params = study_XBG.best_params\n    \ngc.collect()","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:22:54.575894Z","iopub.execute_input":"2022-12-01T14:22:54.576281Z","iopub.status.idle":"2022-12-01T14:23:45.279504Z","shell.execute_reply.started":"2022-12-01T14:22:54.576249Z","shell.execute_reply":"2022-12-01T14:23:45.278306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_tune_params:\n    study_catboost = optuna.create_study(\n        direction='maximize', \n        pruner=optuna.pruners.MedianPruner(n_warmup_steps=20),\n        study_name='RSNA')\n    study_catboost.optimize(objective_Catboost, n_trials=20)\n    Catboost_params = study_catboost.best_params\ngc.collect()","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:23:45.280633Z","iopub.execute_input":"2022-12-01T14:23:45.280943Z","iopub.status.idle":"2022-12-01T14:28:01.764738Z","shell.execute_reply.started":"2022-12-01T14:23:45.280915Z","shell.execute_reply":"2022-12-01T14:28:01.76358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Select and tune best one\n- Part for getting running 1 fold, you can skip ","metadata":{}},{"cell_type":"code","source":"# if not is_tune_params:\n#     params_tuned = {\n#         'reg_alpha': 0.0028731193020013765,\n#         'reg_lambda': 0.04370710510459441,\n#         'colsample_bytree': 0.6,\n#         'subsample': 0.7,\n#         'max_depth': 20,\n#         'num_leaves': 594,\n#         'min_child_samples': 12,\n#         'min_data_per_groups': 65\n#     }\n#     tuned_model = lgb.LGBMClassifier(**params_tuned, zero_as_missing=True)\n#     tuned_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:28:01.766147Z","iopub.execute_input":"2022-12-01T14:28:01.766859Z","iopub.status.idle":"2022-12-01T14:28:01.772183Z","shell.execute_reply.started":"2022-12-01T14:28:01.766823Z","shell.execute_reply":"2022-12-01T14:28:01.771072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5 Folds Training and Inference\n- Prepare Train, Valid, and Test\n- Adding ADASYN processing","metadata":{}},{"cell_type":"code","source":"valid_f1_scores = []\npredictions = []\n\nfor fold in range(N_FOLD):\n    print(f'\\n-----------FOLD {fold} ------------')\n    print('Data prepared.')\n    train_df = df_train[df_train['fold'] != fold].reset_index(drop=True)\n    valid_df = df_train[df_train['fold'] == fold].reset_index(drop=True)\n\n    dv = DictVectorizer(sparse=False)\n    ada = ADASYN(random_state=RANDOM_SEED)\n\n    train_dict = train_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    \n    X_train = dv.fit_transform(train_dict)\n    y_train = train_df[TARGET_COLS].values\n    X_res_train, y_res_train = ada.fit_resample(X_train, y_train)\n    \n\n    val_dict = valid_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_val = dv.transform(val_dict)\n    y_val = valid_df[TARGET_COLS].values\n\n\n    test_dict = df_test[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_test = dv.transform(test_dict)\n\n#     save_dict = f\"./dict_fold{fold}.pkl\"\n#     pickle.dump(dv, open(save_dict, 'wb'))\n\n    class_weight_arr = len(train_df) / (2 * np.bincount(train_df['cancer'].values))\n\n    print('Model Training.')\n    ## FOR CLASS WEIGHTED TUNNING\n#     tuned_model = lgb.LGBMClassifier(**params_tuned, \n#                                     zero_as_missing=True, \n#                                     class_weight={0: class_weight_arr[0], \n#                                                   1: class_weight_arr[1]})\n\n    tuned_model = lgb.LGBMClassifier(**LGBM_params, zero_as_missing=True)\n    tuned_model.fit(X_res_train, y_res_train)\n\n    print('Model Inferencing.')\n    y_val_pred = tuned_model.predict(X_val)\n    y_test_pred = tuned_model.predict_proba(X_test)[:, 1]\n    \n    valid_f1_score = f1_score(y_val, y_val_pred, pos_label=1, average='macro')\n    print('Finished.')\n\n    valid_f1_scores.append(valid_f1_score)\n    predictions.append(y_test_pred)\n\n    del tuned_model, dv, ada\n    gc.collect()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:28:01.773849Z","iopub.execute_input":"2022-12-01T14:28:01.774232Z","iopub.status.idle":"2022-12-01T14:28:12.002903Z","shell.execute_reply.started":"2022-12-01T14:28:01.774201Z","shell.execute_reply":"2022-12-01T14:28:12.002094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(N_FOLD):\n    print(f'\\n-----------FOLD {fold} ------------')\n    print('Data prepared.')\n    train_df = df_train[df_train['fold'] != fold].reset_index(drop=True)\n    valid_df = df_train[df_train['fold'] == fold].reset_index(drop=True)\n\n    dv = DictVectorizer(sparse=False)\n    ada = ADASYN(random_state=RANDOM_SEED)\n\n    train_dict = train_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    \n    X_train = dv.fit_transform(train_dict)\n    y_train = train_df[TARGET_COLS].values\n    X_res_train, y_res_train = ada.fit_resample(X_train, y_train)\n    \n\n    val_dict = valid_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_val = dv.transform(val_dict)\n    y_val = valid_df[TARGET_COLS].values\n\n\n    test_dict = df_test[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_test = dv.transform(test_dict)\n\n#     save_dict = f\"./dict_fold{fold}.pkl\"\n#     pickle.dump(dv, open(save_dict, 'wb'))\n\n    class_weight_arr = len(train_df) / (2 * np.bincount(train_df['cancer'].values))\n\n    print('Model Training.')\n\n    xgb_model = xgb.XGBClassifier(**XGB_params)\n    xgb_model.fit(X_res_train, y_res_train)\n\n    print('Model Inferencing.')\n    y_val_pred = xgb_model.predict(X_val)\n    y_test_pred = xgb_model.predict_proba(X_test)[:, 1]\n    \n    valid_f1_score = f1_score(y_val, y_val_pred, pos_label=1, average='macro')\n    print('Finished.')\n\n    valid_f1_scores.append(valid_f1_score)\n    predictions.append(y_test_pred)\n\n    del xgb_model, dv, ada\n    gc.collect()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:28:12.004383Z","iopub.execute_input":"2022-12-01T14:28:12.005529Z","iopub.status.idle":"2022-12-01T14:28:36.111397Z","shell.execute_reply.started":"2022-12-01T14:28:12.00548Z","shell.execute_reply":"2022-12-01T14:28:36.110174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(N_FOLD):\n    print(f'\\n-----------FOLD {fold} ------------')\n    print('Data prepared.')\n    train_df = df_train[df_train['fold'] != fold].reset_index(drop=True)\n    valid_df = df_train[df_train['fold'] == fold].reset_index(drop=True)\n\n    dv = DictVectorizer(sparse=False)\n    ada = ADASYN(random_state=RANDOM_SEED)\n\n    train_dict = train_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    \n    X_train = dv.fit_transform(train_dict)\n    y_train = train_df[TARGET_COLS].values\n    X_res_train, y_res_train = ada.fit_resample(X_train, y_train)\n    \n\n    val_dict = valid_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_val = dv.transform(val_dict)\n    y_val = valid_df[TARGET_COLS].values\n\n\n    test_dict = df_test[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_test = dv.transform(test_dict)\n\n#     save_dict = f\"./dict_fold{fold}.pkl\"\n#     pickle.dump(dv, open(save_dict, 'wb'))\n\n    class_weight_arr = len(train_df) / (2 * np.bincount(train_df['cancer'].values))\n\n    print('Model Training.')\n\n    model_catboost = CatBoostClassifier(\n    iterations=5, \n    learning_rate=0.1, \n    #loss_function='CrossEntropy'\n)\n\n    model_catboost.fit(X_res_train, y_res_train)\n\n    print('Model Inferencing.')\n    y_val_pred = model_catboost.predict(X_val)\n    y_test_pred = model_catboost.predict_proba(X_test)[:, 1]\n    \n    valid_f1_score = f1_score(y_val, y_val_pred, pos_label=1, average='macro')\n    print('Finished.')\n\n    valid_f1_scores.append(valid_f1_score)\n    predictions.append(y_test_pred)\n\n    del model_catboost, dv, ada\n    gc.collect()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-01T14:28:36.112949Z","iopub.execute_input":"2022-12-01T14:28:36.114044Z","iopub.status.idle":"2022-12-01T14:28:43.280149Z","shell.execute_reply.started":"2022-12-01T14:28:36.113979Z","shell.execute_reply":"2022-12-01T14:28:43.279066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.1 Check Local CV","metadata":{}},{"cell_type":"code","source":"print(f'{len(valid_f1_scores)} Folds validation F1:\\n{valid_f1_scores}')\nprint(f'Local CV Average F1 score: {np.mean(valid_f1_scores)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:28:43.281385Z","iopub.execute_input":"2022-12-01T14:28:43.281795Z","iopub.status.idle":"2022-12-01T14:28:43.288527Z","shell.execute_reply.started":"2022-12-01T14:28:43.281751Z","shell.execute_reply":"2022-12-01T14:28:43.286879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2. Average Prediction","metadata":{}},{"cell_type":"code","source":"preds = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:28:43.290198Z","iopub.execute_input":"2022-12-01T14:28:43.290605Z","iopub.status.idle":"2022-12-01T14:28:43.30343Z","shell.execute_reply.started":"2022-12-01T14:28:43.290567Z","shell.execute_reply":"2022-12-01T14:28:43.302276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Submission","metadata":{}},{"cell_type":"code","source":"final_sub = pd.DataFrame()\nfinal_sub[\"prediction_id\"] = df_test['prediction_id']\nfinal_sub[\"cancer\"] = preds\nfinal_sub.to_csv('submission.csv', index=False)\nfinal_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T14:28:43.305052Z","iopub.execute_input":"2022-12-01T14:28:43.305355Z","iopub.status.idle":"2022-12-01T14:28:43.326467Z","shell.execute_reply.started":"2022-12-01T14:28:43.305326Z","shell.execute_reply":"2022-12-01T14:28:43.325364Z"},"trusted":true},"execution_count":null,"outputs":[]}]}