{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":41875,"databundleVersionId":5521661,"sourceType":"competition"},{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"},{"sourceId":5807722,"sourceType":"datasetVersion","datasetId":3207113},{"sourceId":13954718,"sourceType":"datasetVersion","datasetId":8894772},{"sourceId":5499219,"sourceType":"datasetVersion","datasetId":3167603}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport gc\n\n# ================= 文件路径配置 =================\n# CAFA5 (别人跑好的)\nC5_ESM_IDS_PATH = \"/kaggle/input/23468234/train_ids_esm2_t33_650M_UR50D.npy\"\nC5_ESM_EMB_PATH = \"/kaggle/input/23468234/train_embeds_esm2_t33_650M_UR50D.npy\"\nC5_T5_IDS_PATH  = \"/kaggle/input/t5embeds/train_ids.npy\"\nC5_T5_EMB_PATH  = \"/kaggle/input/t5embeds/train_embeds.npy\"\n\n# CAFA6 (你自己跑的，Unified)\nC6_IDS_PATH     = \"/kaggle/input/aaaaaaaaa/unified_ids.json\"\nC6_ESM_EMB_PATH = \"/kaggle/input/aaaaaaaaa/unified_esm.npy\"\nC6_T5_EMB_PATH  = \"/kaggle/input/aaaaaaaaa/unified_t5.npy\"\n\ndef load_ids(path, name):\n    print(f\"Loading {name} IDs...\")\n    if path.endswith('.json'):\n        with open(path) as f:\n            return np.array(json.load(f))\n    else:\n        return np.load(path, allow_pickle=True)\n\ndef check_compatibility(name, c5_emb_path, c6_emb_path, c5_ids, c6_ids):\n    print(f\"\\n⚡ 正在检查 {name} 模型的一致性...\")\n    \n    # 1. 检查形状和类型 (使用 mmap 模式，不占内存)\n    c5_emb = np.load(c5_emb_path, mmap_mode='r')\n    c6_emb = np.load(c6_emb_path, mmap_mode='r')\n    \n    print(f\"   [Shape Check]\")\n    print(f\"   CAFA5 {name}: {c5_emb.shape} | dtype: {c5_emb.dtype}\")\n    print(f\"   CAFA6 {name}: {c6_emb.shape} | dtype: {c6_emb.dtype}\")\n    \n    if c5_emb.shape[1] != c6_emb.shape[1]:\n        print(f\"❌ 严重警告：维度不匹配！CAFA5是 {c5_emb.shape[1]}维，你是 {c6_emb.shape[1]}维。无法合并！\")\n        return False\n    else:\n        print(f\"✅ 维度匹配 ({c5_emb.shape[1]}维)\")\n\n    # 2. 寻找公共蛋白进行“指纹对比”\n    print(f\"   [Value Consistency Check]\")\n    common_ids = np.intersect1d(c5_ids, c6_ids)\n    if len(common_ids) == 0:\n        print(\"⚠️ 警告：两个数据集中没有找到相同的蛋白 ID，无法验证提取逻辑是否一致。\")\n        return True # 无法验证，暂且放行\n    \n    # 随机取 3 个公共蛋白进行对比\n    sample_ids = common_ids[:3]\n    print(f\"   正在对比公共蛋白: {sample_ids}\")\n    \n    # 建立索引映射\n    c5_id_map = {pid: i for i, pid in enumerate(c5_ids)}\n    c6_id_map = {pid: i for i, pid in enumerate(c6_ids)}\n    \n    for pid in sample_ids:\n        idx5 = c5_id_map[pid]\n        idx6 = c6_id_map[pid]\n        \n        vec5 = c5_emb[idx5].astype(np.float32)\n        vec6 = c6_emb[idx6].astype(np.float32)\n        \n        # 计算余弦相似度\n        dot_product = np.dot(vec5, vec6)\n        norm5 = np.linalg.norm(vec5)\n        norm6 = np.linalg.norm(vec6)\n        similarity = dot_product / (norm5 * norm6 + 1e-8)\n        \n        print(f\"   ID: {pid} | Cosine Sim: {similarity:.5f}\")\n        \n        if similarity < 0.95:\n            print(\"❌ 警告：特征值差异巨大！提取逻辑可能不同（例如：一个是 Mean Pooling，一个是 CLS）。\")\n            print(\"   -> 建议不要合并，或者重新用同一套代码跑 CAFA5 的数据。\")\n            return False\n        \n    print(f\"✅ 通过！特征提取逻辑看似一致。\")\n    return True\n\n# ================= 主流程 =================\n\n# 1. 加载 ID\nids_c5_esm = load_ids(C5_ESM_IDS_PATH, \"CAFA5 ESM\")\nids_c5_t5  = load_ids(C5_T5_IDS_PATH, \"CAFA5 T5\")\nids_c6     = load_ids(C6_IDS_PATH, \"CAFA6 Unified\")\n\nprint(f\"\\nID 数量统计:\")\nprint(f\"CAFA5 ESM IDs: {len(ids_c5_esm)}\")\nprint(f\"CAFA5 T5  IDs: {len(ids_c5_t5)}\")\nprint(f\"CAFA6 Unified IDs: {len(ids_c6)}\")\n\n# 2. 验证 T5\nt5_ok = check_compatibility(\"T5\", C5_T5_EMB_PATH, C6_T5_EMB_PATH, ids_c5_t5, ids_c6)\n\n# 3. 验证 ESM\nesm_ok = check_compatibility(\"ESM\", C5_ESM_EMB_PATH, C6_ESM_EMB_PATH, ids_c5_esm, ids_c6)\n\nif t5_ok and esm_ok:\n    print(\"\\n🎉🎉🎉 完美！所有检查通过，可以安全合并！\")\nelse:\n    print(\"\\n🛑🛑🛑 检查失败，请根据上述错误修正你的特征提取方式。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T05:50:36.728655Z","iopub.execute_input":"2025-12-09T05:50:36.729014Z","iopub.status.idle":"2025-12-09T05:50:37.654347Z","shell.execute_reply.started":"2025-12-09T05:50:36.728989Z","shell.execute_reply":"2025-12-09T05:50:37.653443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# ================= 配置路径 =================\n# CAFA6 官方标签\nC6_TERMS_PATH = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\"\n# CAFA5 官方标签\nC5_TERMS_PATH = \"/kaggle/input/cafa-5-protein-function-prediction/Train/train_terms.tsv\"\n\n# 输出路径\nOUT_LABEL_PATH = \"combined_train_terms.tsv\"\n# ===========================================\n\ndef merge_labels():\n    print(\"正在合并 CAFA5 和 CAFA6 的标签...\")\n    \n    # 1. 读取 CAFA6 (高优先级)\n    print(\"Reading CAFA6 terms...\")\n    df_c6 = pd.read_csv(C6_TERMS_PATH, sep='\\t')\n    # CAFA6 列名通常是: EntryID, term, aspect\n    # 我们统一重命名方便处理\n    df_c6.rename(columns={'EntryID': 'Protein_ID', 'term': 'GO_Term', 'aspect': 'Aspect'}, inplace=True)\n    c6_ids = set(df_c6['Protein_ID'].unique())\n    print(f\"   - CAFA6 包含蛋白数: {len(c6_ids)}\")\n    \n    # 2. 读取 CAFA5 (低优先级)\n    print(\"Reading CAFA5 terms...\")\n    df_c5 = pd.read_csv(C5_TERMS_PATH, sep='\\t')\n    df_c5.rename(columns={'EntryID': 'Protein_ID', 'term': 'GO_Term', 'aspect': 'Aspect'}, inplace=True)\n    c5_ids = set(df_c5['Protein_ID'].unique())\n    print(f\"   - CAFA5 包含蛋白数: {len(c5_ids)}\")\n    \n    # 3. 筛选 CAFA5\n    # 逻辑：只保留那些【不在】CAFA6 中的蛋白的标签\n    # 如果一个蛋白在 CAFA6 里已经有了，我们就用 CAFA6 的新标签，扔掉 CAFA5 的旧标签\n    df_c5_unique = df_c5[~df_c5['Protein_ID'].isin(c6_ids)]\n    print(f\"   - CAFA5 独有蛋白数 (补充数据): {df_c5_unique['Protein_ID'].nunique()}\")\n    \n    # 4. 合并\n    df_final = pd.concat([df_c6, df_c5_unique], axis=0, ignore_index=True)\n    \n    # 5. 保存\n    print(f\"Saving combined labels to {OUT_LABEL_PATH}...\")\n    df_final.to_csv(OUT_LABEL_PATH, sep='\\t', index=False)\n    \n    print(\"✅ 标签合并完成！\")\n    print(f\"   总行数: {len(df_final)}\")\n    print(f\"   总唯一蛋白数: {df_final['Protein_ID'].nunique()}\")\n    print(f\"   (这就是我们训练集的最终大小)\")\n\nif __name__ == \"__main__\":\n    merge_labels()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T06:07:52.234713Z","iopub.execute_input":"2025-12-09T06:07:52.235242Z","iopub.status.idle":"2025-12-09T06:07:59.342632Z","shell.execute_reply.started":"2025-12-09T06:07:52.235211Z","shell.execute_reply":"2025-12-09T06:07:59.341614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport os\nimport gc\nfrom tqdm import tqdm\n\n# ================= 1. 配置输入路径 =================\n\n# CAFA5 特征 (来自 Kaggle Dataset)\nC5_IDS_PATH     = \"/kaggle/input/t5embeds/train_ids.npy\"\nC5_T5_PATH      = \"/kaggle/input/t5embeds/train_embeds.npy\"\n# 注意：CAFA5 的 ESM ID 文件\nC5_ESM_IDS_PATH = \"/kaggle/input/23468234/train_ids_esm2_t33_650M_UR50D.npy\" \nC5_ESM_PATH     = \"/kaggle/input/23468234/train_embeds_esm2_t33_650M_UR50D.npy\"\n\n# CAFA6 特征 (你的 Unified 数据)\nC6_IDS_PATH     = \"/kaggle/input/aaaaaaaaa/unified_ids.json\"\nC6_T5_PATH      = \"/kaggle/input/aaaaaaaaa/unified_t5.npy\"\nC6_ESM_PATH     = \"/kaggle/input/aaaaaaaaa/unified_esm.npy\"\n\n# 标签文件 (刚刚生成的合并标签)\nLABEL_PATH      = \"combined_train_terms.tsv\" \n\n# 输出目录\nOUT_DIR         = \"/kaggle/working/final_train_data\"\n\n# =================================================\n\ndef load_ids(path):\n    if path.endswith('.json'):\n        with open(path) as f: return np.array(json.load(f))\n    else:\n        return np.load(path, allow_pickle=True)\n\ndef main():\n    os.makedirs(OUT_DIR, exist_ok=True)\n    \n    # --- 1. 加载白名单 (Combined Labels) ---\n    print(\"📋 [Step 1] Loading Combined Labels...\")\n    if not os.path.exists(LABEL_PATH):\n        raise FileNotFoundError(\"请先运行上面的标签合并脚本！\")\n        \n    df_labels = pd.read_csv(LABEL_PATH, sep=\"\\t\", usecols=['Protein_ID'])\n    valid_ids_set = set(df_labels['Protein_ID'].unique())\n    print(f\"   - 目标训练集总蛋白数: {len(valid_ids_set)}\")\n    del df_labels\n    gc.collect()\n\n    # --- 2. 建立索引映射 ---\n    print(\"\\n🗺️ [Step 2] Building Merge Map (Priority: C6 > C5)...\")\n    \n    ids_c5_t5 = load_ids(C5_IDS_PATH)\n    ids_c5_esm = load_ids(C5_ESM_IDS_PATH)\n    ids_c6 = load_ids(C6_IDS_PATH)\n    \n    # 建立 C5 ESM 的映射，确保对齐\n    c5_esm_id_map = {pid: i for i, pid in enumerate(ids_c5_esm)}\n    \n    merge_map = {} # ProteinID -> (Source, IndexT5, IndexESM)\n\n    # > 遍历 CAFA5 (填补底库)\n    for idx_t5, pid in enumerate(ids_c5_t5):\n        if pid in valid_ids_set:\n            if pid in c5_esm_id_map:\n                idx_esm = c5_esm_id_map[pid]\n                merge_map[pid] = ('c5', idx_t5, idx_esm)\n\n    # > 遍历 CAFA6 (覆盖更新)\n    for idx, pid in enumerate(ids_c6):\n        if pid in valid_ids_set:\n            # Unified 意味着 T5 和 ESM 索引相同\n            merge_map[pid] = ('c6', idx, idx)\n    \n    final_ids_list = sorted(list(merge_map.keys()))\n    N = len(final_ids_list)\n    print(f\"   - 最终能匹配到特征的蛋白数: {N}\")\n    print(f\"   - (如果有丢失，说明有些蛋白有标签但没特征，通常很少)\")\n\n    # --- 3. 填充数据 ---\n    print(\"\\n🚚 [Step 3] Merging Embeddings...\")\n    \n    # 加载源数据\n    c5_t5_emb = np.load(C5_T5_PATH, mmap_mode='r')\n    c5_esm_emb = np.load(C5_ESM_PATH, mmap_mode='r')\n    c6_t5_emb = np.load(C6_T5_PATH, mmap_mode='r')\n    c6_esm_emb = np.load(C6_ESM_PATH, mmap_mode='r')\n    \n    # 创建结果数组\n    final_t5 = np.zeros((N, 1024), dtype=np.float32)\n    final_esm = np.zeros((N, 1280), dtype=np.float32)\n    \n    for i, pid in tqdm(enumerate(final_ids_list), total=N):\n        source, idx_t5, idx_esm = merge_map[pid]\n        \n        if source == 'c5':\n            final_t5[i] = c5_t5_emb[idx_t5]\n            final_esm[i] = c5_esm_emb[idx_esm]\n        else:\n            final_t5[i] = c6_t5_emb[idx_t5]\n            final_esm[i] = c6_esm_emb[idx_esm]\n\n    # --- 4. 保存 ---\n    print(\"\\n💾 [Step 4] Saving...\")\n    np.save(os.path.join(OUT_DIR, \"train_ids.npy\"), np.array(final_ids_list))\n    np.save(os.path.join(OUT_DIR, \"train_t5_embeds.npy\"), final_t5)\n    np.save(os.path.join(OUT_DIR, \"train_esm2_embeds.npy\"), final_esm)\n    \n    print(\"✅ 训练数据准备完毕！\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T06:08:45.907419Z","iopub.execute_input":"2025-12-09T06:08:45.907862Z","iopub.status.idle":"2025-12-09T06:09:56.069362Z","shell.execute_reply.started":"2025-12-09T06:08:45.907826Z","shell.execute_reply":"2025-12-09T06:09:56.068446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport json\nimport os\n\n# ================= 配置路径 =================\n# 1. 刚刚生成的最终数据\nFINAL_DIR = \"/kaggle/working/final_train_data\"\nFINAL_IDS_PATH = os.path.join(FINAL_DIR, \"train_ids.npy\")\nFINAL_T5_PATH  = os.path.join(FINAL_DIR, \"train_t5_embeds.npy\")\nFINAL_ESM_PATH = os.path.join(FINAL_DIR, \"train_esm2_embeds.npy\")\n\n# 2. 原始数据源 (用于比对)\n# CAFA5\nC5_IDS_PATH    = \"/kaggle/input/t5embeds/train_ids.npy\"\nC5_T5_PATH     = \"/kaggle/input/t5embeds/train_embeds.npy\"\n# CAFA6 (Unified)\nC6_IDS_PATH    = \"/kaggle/input/aaaaaaaaa/unified_ids.json\"\nC6_T5_PATH     = \"/kaggle/input/aaaaaaaaa/unified_t5.npy\"\n\n# ===========================================\n\ndef load_ids(path):\n    if path.endswith('.json'):\n        with open(path) as f: return np.array(json.load(f))\n    else:\n        return np.load(path, allow_pickle=True)\n\ndef check_vector_match(vec1, vec2, name):\n    # 允许微小的浮点误差\n    if np.allclose(vec1, vec2, atol=1e-5):\n        print(f\"   ✅ {name}: 完全一致！\")\n        return True\n    else:\n        diff = np.abs(vec1 - vec2).max()\n        print(f\"   ❌ {name}: 不匹配！最大差异: {diff}\")\n        return False\n\ndef verify_data():\n    print(\"🕵️‍♂️ 正在启动数据法医验证程序...\\n\")\n    \n    # --- 1. 加载最终数据 ---\n    print(\"1. 加载合并后的数据...\")\n    final_ids = np.load(FINAL_IDS_PATH, allow_pickle=True)\n    # 使用 mmap 模式，不占内存，读取快\n    final_t5 = np.load(FINAL_T5_PATH, mmap_mode='r')\n    final_esm = np.load(FINAL_ESM_PATH, mmap_mode='r')\n    \n    N = len(final_ids)\n    print(f\"   - 最终样本数: {N}\")\n    print(f\"   - T5 Shape: {final_t5.shape}\")\n    print(f\"   - ESM Shape: {final_esm.shape}\")\n    \n    if final_t5.shape[0] != N or final_esm.shape[0] != N:\n        raise ValueError(\"❌ 严重错误：ID数量与特征行数不匹配！\")\n\n    # --- 2. 零值检测 (防止空数据) ---\n    print(\"\\n2. 零值检测 (Zombie Check)...\")\n    # 随机抽 1000 个点检查，或者检查前 10 个\n    zero_count = 0\n    for i in range(min(100, N)):\n        if np.sum(np.abs(final_t5[i])) == 0:\n            print(f\"   ⚠️ 警告: 第 {i} 行 (ID: {final_ids[i]}) T5向量全为0！\")\n            zero_count += 1\n    \n    if zero_count == 0:\n        print(\"   ✅ 前100个样本无零向量。数据填充看起来正常。\")\n    \n    # --- 3. 逻辑验证 (核心) ---\n    print(\"\\n3. 来源优先级验证 (Source Logic Check)...\")\n    \n    # 加载原始 ID 用于查找\n    c5_ids = load_ids(C5_IDS_PATH)\n    c6_ids = load_ids(C6_IDS_PATH)\n    \n    # 建立 ID -> Index 映射\n    final_id_map = {pid: i for i, pid in enumerate(final_ids)}\n    c5_id_map = {pid: i for i, pid in enumerate(c5_ids)}\n    c6_id_map = {pid: i for i, pid in enumerate(c6_ids)}\n    \n    # 加载原始特征 (mmap)\n    c5_t5_origin = np.load(C5_T5_PATH, mmap_mode='r')\n    c6_t5_origin = np.load(C6_T5_PATH, mmap_mode='r')\n\n    # >> Case A: 测试 CAFA6 覆盖逻辑 (C6 > C5)\n    # 找一个两边都有的 ID\n    common_ids = np.intersect1d(c5_ids, c6_ids)\n    # 还要确保这个 ID 在最终列表里 (因为我们通过 label 过滤过)\n    common_in_final = [pid for pid in common_ids if pid in final_id_map]\n    \n    if common_in_final:\n        test_id = common_in_final[0]\n        print(f\"   [测试 A: 覆盖逻辑] ID: {test_id} (同时存在于 C5 和 C6)\")\n        \n        idx_final = final_id_map[test_id]\n        vec_final = final_t5[idx_final]\n        \n        idx_c6 = c6_id_map[test_id]\n        vec_c6 = c6_t5_origin[idx_c6]\n        \n        idx_c5 = c5_id_map[test_id]\n        vec_c5 = c5_t5_origin[idx_c5]\n        \n        # 验证：它应该等于 C6，不一定等于 C5 (虽然大概率一样，但我们要验证来源)\n        print(\"   -> 正在比对 Final vs CAFA6 (预期一致)...\")\n        match_c6 = check_vector_match(vec_final, vec_c6, \"Final vs C6\")\n        \n        if not match_c6:\n            print(\"   ❌ 致命错误：合并数据没有优先使用 CAFA6 的数据！\")\n    else:\n        print(\"   ⚠️ 没找到公共 ID，跳过覆盖测试。\")\n\n    # >> Case B: 测试 CAFA5 补全逻辑 (Only C5)\n    # 找一个只在 C5 有，且在最终列表里的 ID\n    only_c5_ids = np.setdiff1d(c5_ids, c6_ids)\n    only_c5_in_final = [pid for pid in only_c5_ids if pid in final_id_map]\n    \n    if only_c5_in_final:\n        test_id = only_c5_in_final[0]\n        print(f\"\\n   [测试 B: 补全逻辑] ID: {test_id} (只在 C5 存在)\")\n        \n        idx_final = final_id_map[test_id]\n        vec_final = final_t5[idx_final]\n        \n        idx_c5 = c5_id_map[test_id]\n        vec_c5 = c5_t5_origin[idx_c5]\n        \n        print(\"   -> 正在比对 Final vs CAFA5 (预期一致)...\")\n        match_c5 = check_vector_match(vec_final, vec_c5, \"Final vs C5\")\n        \n        if not match_c5:\n            print(\"   ❌ 致命错误：CAFA5 的数据没有正确写入！\")\n    else:\n        print(\"   ⚠️ 没找到仅属 CAFA5 的有效 ID (可能是 Label 过滤太严)，跳过此测试。\")\n\n    print(\"\\n------------------------------------------------\")\n    print(\"🎉 验证结束。如果没有红色❌出现，说明合并完美成功！\")\n\nif __name__ == \"__main__\":\n    verify_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T06:13:45.534562Z","iopub.execute_input":"2025-12-09T06:13:45.534903Z","iopub.status.idle":"2025-12-09T06:13:46.429211Z","shell.execute_reply.started":"2025-12-09T06:13:45.53488Z","shell.execute_reply":"2025-12-09T06:13:46.428241Z"}},"outputs":[],"execution_count":null}]}