{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/code/anonamename/g2net-oof-weight-optimizer?scriptVersionId=75557143","metadata":{}},{"cell_type":"markdown","source":"## This notebook forked from [Optimise Blending Weights with Bonus :0](https://www.kaggle.com/gogo827jz/optimise-blending-weights-with-bonus-0) by [Yirun Zhang](https://www.kaggle.com/gogo827jz).","metadata":{}},{"cell_type":"code","source":"import glob\nimport datetime\nimport pandas as pd\nfrom time import time\nfrom pathlib import Path\nfrom pprint import pprint\n\nimport numpy as np\nfrom scipy.optimize import minimize, fsolve\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import log_loss\n\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\nimport seaborn as sns\nfrom matplotlib import pyplot\nfrom matplotlib.ticker import ScalarFormatter\n\nimport plotly.figure_factory as ff\nimport plotly.express as px\n\nsns.set_context(\"talk\")\nstyle.use('fivethirtyeight')\nsns.set(font_scale=2.3, font='IPAexGothic')\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:36.152622Z","iopub.execute_input":"2023-01-03T14:49:36.153048Z","iopub.status.idle":"2023-01-03T14:49:40.471877Z","shell.execute_reply.started":"2023-01-03T14:49:36.153009Z","shell.execute_reply":"2023-01-03T14:49:40.470659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# open submission csv files","metadata":{}},{"cell_type":"code","source":"!ls ../input/g2net2-csv","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:40.473752Z","iopub.execute_input":"2023-01-03T14:49:40.474215Z","iopub.status.idle":"2023-01-03T14:49:41.607258Z","shell.execute_reply.started":"2023-01-03T14:49:40.474178Z","shell.execute_reply":"2023-01-03T14:49:41.606181Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = pd.read_csv(\"../input/g2net-detecting-continuous-gravitational-waves/sample_submission.csv\")\n\n# subのcsvのパス\nSUB_DIR = \"../input/g2net2-csv\"\nsub_dict = {\n    #\"ex027_b4_ap\": SUB_DIR + \"/3090_kqi_ex027_fold01234_b4_ap_prob_mean_mixup03_submission.csv\",\n    #\"lb0685_owo_v5_sub_avg\": SUB_DIR + \"/lb0685_g2net2_oof_weight_optimizer_v5_sub_avg.csv\",\n    \n    #\"lb07_owo_v8_sub_opt_auc\": SUB_DIR + \"/lb0.7_g2net2_oof_weight_optimizer_v8_sub_ws_norm_opt_auc.csv\",  # lb0.7\n    #\"ex051_add_ex003_noise_01\": SUB_DIR + \"/3090_kqi_ex051_add_ex003_noise_01_gap_fold01234_b4_ap_submission.csv\",  # lb0.707\n    #\"ex057_add_ex003_noise_01\": SUB_DIR + \"/3090_kqi_ex057_npy_add_ex003_01_fold01234_b4_ap_submission.csv\",  # lb0.709\n    \n    #\"lb0748_public\": SUB_DIR + \"/g2net-large-kernel-inference/lb0748_submission.csv\",\n    #\"ex063_add_ex003_2-005-01_005iso-01\": SUB_DIR + \"/kqi_3090_ex063_npy_add_ex003_2-005-01_005iso-01_fold01234_b4_ap_noamp_submission.csv\",  # 0.739\n    #\"blend_add_ex003_01-005_005iso\": SUB_DIR + \"/lb0738_g2net2_oof_weight_optimizer_v9_sub_avg.csv\",  # 0.738\n    \n    #\"lb0748_public\": SUB_DIR + \"/g2net-large-kernel-inference/lb0748_submission.csv\",\n    #\"ex066_lb0743\": SUB_DIR + \"/3090_kqi_ex066_public_add_ex006-n1400-s800_fold01234_b4_ap_largekernel_p-robustscaler_ts-ma_submission.csv\",  # 0.743\n    #\"ex066_lb0744\": SUB_DIR + \"/3090_kqi_ex066_public_add_ex003_2-005-01_005iso-01_006_fold01234_b5_ap_ts-ma_p-robustscaler_submission.csv\",  # 0.744\n    #\"ex066_lb0747\": SUB_DIR + \"/3090_kqi_ex066_public_add_ex003_2-005-01_005iso-01_006_fold01234_b5_ap_largekernel_p-robustscaler_ts-ma_submission.csv\",  # 0.747\n    \n    #\"ex067_pseudo_lb0751\": SUB_DIR + \"/3090_kqi_ex067_pseudo_add_ex003_2-005-01_005iso-01_006_fold01234_b5_ap_submission.csv\",\n    \n    #\"sub_avg_lb0761\": SUB_DIR + \"/sub_avg_lb0761.csv\",\n    #\"sub_ws_norm_opt_auc_lb0755\": SUB_DIR + \"/sub_ws_norm_opt_auc_lb0755.csv\",\n    #\"ex073_b4_ap_lb0759\": SUB_DIR + \"/kqi_3090_ex073_pseudo_add_ex003_2-005-01_005iso-01_006_b4_ap_n_fold20_submission.csv\",\n    #\"ex073_b5_ap\": SUB_DIR + \"/3090_kqi_ex073_pseudo_add_ex003_2-005-01_005iso-01_006_b5_ap_n_fold20_submission.csv\",\n    \n    #\"lb0768_kqi_ex075_v2_n50\": SUB_DIR + \"/kqi_3090_ex075_v2_pseudo_multioutput_freq_b5_ap_freq_div_n50_submission.csv\",\n    #\"lb0765_ex081_lb0768pseudo\": SUB_DIR + \"/3090_kqi_ex081_pseudo_multioutput_freq_b5_ap_submission.csv\",\n    ##\"lb0763_ex078_10fold\": SUB_DIR + \"/kqi_3090_ex078_pseudo_multioutput_freq_b5_ap_10fold_submission.csv\",\n    ##\"lb0763_ex075_n25\": SUB_DIR + \"/kqi_3090_ex075_v2_pseudo_multioutput_freq_b5_ap_freq_div_n25_submission.csv\",\n    ##\"lb0762_ex075_mixup_use_pseudo_f\": SUB_DIR + \"/3090_kqi_ex075_v2_pseudo_multioutput_freq_b5_ap_mixup_use_pseudo_f_submission.csv\",\n    ##\"lb0761_ex075_20fold\": SUB_DIR + \"/kqi_3090_ex075_pseudo_multioutput_freq_b5_ap_20fold_submission.csv\",\n    #\"lb0761_g2net2_sub_avg_v12\": SUB_DIR + \"/lb0.761_g2net2_sub_avg_v12_sub_avg.csv\",\n    #\"lb0759_ex082_rsch\": SUB_DIR + \"/3090_kqi_ex082_pseudo_multioutput_freq_rsch_b5_submission.csv\",\n    #\"lb0759_ex080_freq_sample_mixup\": SUB_DIR + \"/3090_kqi_ex080_pseudo_multioutput_freq_b5_ap_submission.csv\",\n    ##\"lb0759_ex073_b4_ap_20fold\": SUB_DIR + \"/kqi_3090_ex073_pseudo_add_ex003_2-005-01_005iso-01_006_b4_ap_n_fold20_submission.csv\",\n    ##\"lb0758_ex077_add_ex008\": SUB_DIR + \"/3090_kqi_ex077_pseudo_b6_ap_Drop_add_Ex3-5-6-8_add_pseudo_EX7_1800_fold20_submission.csv\",\n    #\"lb0758_ex078_hstack_2ch\": SUB_DIR + \"/kqi_3090_ex078_pseudo_multioutput_freq_b5_ap_hstack_2ch_submission.csv\",\n    #\"ex078_vstack_2ch\": SUB_DIR + \"/kqi_3090_ex078_pseudo_multioutput_freq_b5_ap_vstack_2ch_submission.csv\",\n    ##\"lb0754_ex083_multioutput2\": SUB_DIR + \"/3090_kqi_ex083_pseudo_multioutput2_b5_submission.csv\",\n    #\"lb0754_ex088_gap_flg_ch05\": SUB_DIR + \"/3090_kqi_ex088_pseudo_multioutput_b5_gap_flg_ch05_submission.csv\",\n    #\"lb0753_ex075_gap_flg_ch00\": SUB_DIR + \"/3090_kqi_ex075_v5_gap_flg_ch00_submission.csv\",\n    ##\"lb0753_ex076_addemb\": SUB_DIR + \"/kqi_3090_ex076_pseudo_addemb_freq_b5_ap_submission.csv\",\n    \n    #\"lb0771_kqi_ex075_v2_n50_tta02\": SUB_DIR + \"/lb0.771_kqi_3090_ex075_v2_pseudo_multioutput_freq_b5_ap_freq_div_n50_submission_tta012.csv\",\n    #\"lb0767_kqi_ex075_v12\": SUB_DIR + \"/3090_kqi_ex075_v12_pseudo_kqi_ex075_v6_load_kqi_ex075_v11_ep50_submission.csv\",\n    #\"sub_avg_lb0771\": SUB_DIR + \"/sub_avg_lb0771.csv\",\n    #\"lb0772_kqi_ex075_v2_hvflip_lb0771_09501_sim0000_tta02\": SUB_DIR + \"/lb0.772_kqi_3090_ex075_v2_tta_pseudo_hvflip_lb0771_09501_sim0000_submission.csv\"\n    \n    #\"lb0769_3090_ex075_v21_zebra\": SUB_DIR + \"/3090_kqi_ex075_v21_zebra_mixup05_sample_mixup05_load_kqi_3090_ex075_v11_best_score_submission.csv\",\n    \n    #\"lb0773_kqi_ex075_v2\": SUB_DIR + \"/lb0.773_kqi_3090_ex075_v2_tta_pseudo_hvflip_lb0771_1000_sim0000_submission.csv\",\n    \n    #\"lb0774_kqi_ex075_v2\": SUB_DIR + \"/lb0.774_kqi_3090_ex075_v2_tta_v2_pseudo_hvflip_lb0771_1000_simall_100ep_submission.csv\",\n    \n    #\"lb0775_kqi_ex075_v2_stride12\": SUB_DIR + \"/lb0.775_kqi_3090_ex075_v2_tta_v3_pseudo_v3_hvflip_stride12_lb0771_1000_simall_100ep_submission_tta012.csv\",\n    \n    #\"lb0773_kqi_ex075_v2_no_norm\": SUB_DIR + \"/3090_kqi_ex075_v2_tta_v4_no_norm_hvflip_lb0771_pseudo_th_5_10_100ep_submission_tta012.csv\",\n    \n    #\"lb0775_kqi_ex075_v2_stride12_no_norm\": SUB_DIR + \"/kqi_3090_ex075_v2_tta_v4_no_norm_hvflip_stride12_lb0775_pseudo_th_5_10_100ep_submission_tta012.csv\",\n    \n    #\"lb0775_kqi_ex075_v2_norm_5_10\": SUB_DIR + \"/lb0.775_kqi_3090_ex075_v2_tta_v4_norm_hvflip_lb0771_pseudo_th_5_10_100ep_submission_tta012.csv\",\n    \n    #\"lb0776_kqi_ex075_v2_stride12_norm_5_10_addD_EX007\": SUB_DIR + \"/3090_kqi_ex075_v2_tta_v4_norm_stride12_lb0771_pseudo_th_5_10_100ep_addD_EX007_submission_tta012.csv\",\n    \n    #\"sub_rank_ver30_lb0778\": SUB_DIR + \"/sub_rank_ver30_lb0778.csv\",\n    #\"sub_avg_ver30_lb0778\": SUB_DIR + \"/sub_avg_ver30_lb0778.csv\",\n    \n    #\"lb0778_kqi_ex075_v2_stride12_norm_5_10_addD_EX007_2\": SUB_DIR + \"/lb0.778_3090_kqi_ex075_v2_tta_v4_norm_stride12_lb0771_pseudo_th_5_10_100ep_addD_EX007_2_submission_tta012.csv\",\n    \n    #\"sub_rank_ver32_lb0780\": SUB_DIR + \"/sub_rank_ver32_lb0780.csv\",\n    #\"sub_rank_ver34_lb0780\": SUB_DIR + \"/sub_rank_ver34_lb0780.csv\",\n    \n    #\"lb0.779_kqi_3090_ex075_GaussNoise_tta0128\": SUB_DIR + \"/lb0.779_kqi_3090_ex075_GaussNoise_submission_tta0128.csv\",\n    \n    #\"lb0.780_kqi_3090_ex075_GaussNoise_tta0128\": SUB_DIR + \"/lb0.780_kqi_3090_ex075_v2_stride12_norm_lb0771_pseudo_th_5_10_addD_EX007_3_GN_submission_tta012.csv\",\n    \n    \n    #\"GeM2f_stride12_norm\": SUB_DIR + \"/3090_kqi_ex075_v2_stride12_GeM2f_norm_lb0771_pseudo_th_5_10_addD_EX007_3_GN_submission_tta012.csv\",\n    #\"GeM2f_stride12_simall\": SUB_DIR + \"/kqi_3090_ex075_v2_stride12_GeM2f_norm_lb0771_pseudo_simall_addD_EX007_3_GN_submission_tta012.csv\",\n    #\"AndCh_stride12_norm\": SUB_DIR + \"/kqi_3090_ex075_v2_stride12_norm_lb0771_pseudo_th_5_10_addD_EX007_3_GN_AndCh_submission_tta012.csv\",\n    #\"stride12_norm\": SUB_DIR + \"/lb0.780_kqi_3090_ex075_v2_stride12_norm_lb0771_pseudo_th_5_10_addD_EX007_3_GN_submission_tta012.csv\",\n    #\"stride12_simall\": SUB_DIR + \"/kqi_3090_ex075_v2_stride12_lb0771_pseudo_simall_addD_EX007_3_GN_submission_tta012.csv\",\n    \n    \"sub_rank_ver35_lb0781\": SUB_DIR + \"/sub_rank_ver35_lb0781.csv\",\n    \"sub_rank_ver37_lb0.780\": SUB_DIR + \"/sub_rank_ver37_lb0.780.csv\",\n}\npprint(sub_dict)\n\nsubs = np.zeros((len(sub_dict), submit.shape[0], len(['target']) ))\nfor i, p in enumerate(sub_dict.keys()):\n    sub = pd.read_csv( list(sub_dict.values())[i] )\n    subs[i] = sub[['target']].values\ndisplay(pd.DataFrame(subs.reshape(subs.shape[0], 7975).T, columns=sub_dict.keys()))\nsubs","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:41.609531Z","iopub.execute_input":"2023-01-03T14:49:41.610218Z","iopub.status.idle":"2023-01-03T14:49:41.715061Z","shell.execute_reply.started":"2023-01-03T14:49:41.610171Z","shell.execute_reply":"2023-01-03T14:49:41.71393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 各subの相関係数確認\ntarget_columns  = ['target']\n\nsub_targets_scored = pd.read_csv( list(sub_dict.values())[0] )\n\nsub_v2 = np.zeros((len(sub_dict), submit.shape[0], len(target_columns) ))\n\nfor i, p in enumerate(sub_dict.keys()):\n    print(i,p)\n    tmp = pd.read_csv(sub_dict[p])\n    sub = sub_targets_scored.drop(columns=target_columns).merge(tmp[['id']+target_columns], on='id', how='left')\n    sub_v2[i,:,:] = sub[target_columns].values \n\ncorr = np.corrcoef(sub_v2.reshape(len(sub_dict), -1))\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(35, 30))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, cmap=cmap, annot=True, fmt='.3f',\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, ax=ax)\nax.set_ylim(corr.shape[0], 0)\nplt.yticks(rotation=0)\nplt.title(\"SUB correlation\")","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:41.717388Z","iopub.execute_input":"2023-01-03T14:49:41.717747Z","iopub.status.idle":"2023-01-03T14:49:42.478428Z","shell.execute_reply.started":"2023-01-03T14:49:41.717711Z","shell.execute_reply":"2023-01-03T14:49:42.476851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 単純平均","metadata":{}},{"cell_type":"code","source":"# 単純平均でsub計算\nsub_avg = np.zeros((submit.shape[0], ))\nfor i, s in enumerate(subs):\n    s_ws = s * 1.0 / len(subs)\n    sub_avg += s_ws.ravel()\n\n# ファイル出力\nsub_avg_df = pd.DataFrame({\"id\": submit.id, \"target\": sub_avg})\nsub_avg_df.to_csv(f'sub_avg.csv', index=False)\nprint(\"OUTPUT: sub_avg.csv\")\ndisplay(sub_avg_df)\n\nplt.hist(sub_avg_df[\"target\"], bins=100)\nplt.show()\nplt.clf()  # メモリ解放\nplt.close()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:42.479608Z","iopub.execute_input":"2023-01-03T14:49:42.479938Z","iopub.status.idle":"2023-01-03T14:49:42.864546Z","shell.execute_reply.started":"2023-01-03T14:49:42.479904Z","shell.execute_reply":"2023-01-03T14:49:42.863518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 重み平均でsub計算\n#ws = [0.748, 0.773]\n#sub_avg = np.zeros((submit.shape[0], ))\n#for s, w in zip(subs, ws):\n#    s_ws = s * w / sum(ws)\n#    sub_avg += s_ws.ravel()\n#\n## ファイル出力\n#sub_avg_df = pd.DataFrame({\"id\": submit.id, \"target\": sub_avg})\n#sub_avg_df.to_csv(f'sub_w_avg.csv', index=False)\n#print(\"OUTPUT: sub_w_avg.csv\")\n#display(sub_avg_df)\n#\n#plt.hist(sub_avg_df[\"target\"], bins=100)\n#plt.show()\n#plt.clf()  # メモリ解放\n#plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:42.865824Z","iopub.execute_input":"2023-01-03T14:49:42.866289Z","iopub.status.idle":"2023-01-03T14:49:42.870944Z","shell.execute_reply.started":"2023-01-03T14:49:42.866241Z","shell.execute_reply":"2023-01-03T14:49:42.870106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 各subの分布可視化（数が多いと時間かかる）\nfig = ff.create_distplot([s.ravel() for s in subs], list(sub_dict.keys()), bin_size=.2, show_hist=False, show_rug=False)\nfig.show()","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-01-03T14:49:42.872025Z","iopub.execute_input":"2023-01-03T14:49:42.872494Z","iopub.status.idle":"2023-01-03T14:49:43.338369Z","shell.execute_reply.started":"2023-01-03T14:49:42.872462Z","shell.execute_reply":"2023-01-03T14:49:43.337178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Rank Averaging from [tips: rank averaging](https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/205564) by [Tawara](https://www.kaggle.com/ttahara).\n\n# 順位平均で最適化","metadata":{}},{"cell_type":"code","source":"from scipy.stats import rankdata","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:43.340279Z","iopub.execute_input":"2023-01-03T14:49:43.3406Z","iopub.status.idle":"2023-01-03T14:49:43.346015Z","shell.execute_reply.started":"2023-01-03T14:49:43.340567Z","shell.execute_reply":"2023-01-03T14:49:43.344884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ********************************************************************\n# subを順位平均で最適化\n# ********************************************************************\npprint(sub_dict)\n\n# =====================\n# subの確信度集める\n# =====================\ntarget_columns = [\"target\"]\nsub_rank = np.zeros((len(sub_dict), submit.shape[0], len(target_columns) ))\nfor i in range(sub_rank.shape[0]):\n    sub = pd.read_csv( list(sub_dict.values())[i] )\n    sub = submit.drop(columns=target_columns).merge(sub[['id']+target_columns], on='id', how='left')\n    sub_rank[i] = sub[target_columns].values\n\n# =====================\n# 確信度を順位の平均に変換\n# =====================\nfor i in range(sub_rank.shape[0]):\n    for j in range(len(target_columns)):\n        sub_rank[i,:,j] = rankdata(sub_rank[i,:,j],method='average')\n\n# =====================\n# 出した順位の平均に合わせた確信度の値に変換（0-1に正規化）\n# =====================\nsub_rank = np.mean(sub_rank, axis=0)/np.max(sub_rank)\n#print(sub_rank)\n\n# ファイル出力\nsub_rank_df = pd.DataFrame({\"id\": submit.id, \"target\": sub_rank.ravel()})\nsub_rank_df.to_csv(f'sub_rank.csv', index=False)\nprint(\"OUTPUT: sub_rank.csv\")\ndisplay(sub_rank_df)\n\nplt.hist(sub_rank_df[\"target\"], bins=100)\nplt.show()\nplt.clf()  # メモリ解放\nplt.close()","metadata":{"execution":{"iopub.status.busy":"2023-01-03T14:49:43.347608Z","iopub.execute_input":"2023-01-03T14:49:43.348242Z","iopub.status.idle":"2023-01-03T14:49:43.779253Z","shell.execute_reply.started":"2023-01-03T14:49:43.348167Z","shell.execute_reply":"2023-01-03T14:49:43.778156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}