{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\n\ninput_dir = Path(\"/kaggle/input\")\n\nprint(\"Các thư mục thật trong /kaggle/input:\")\nfor p in input_dir.iterdir():\n    print(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:47:22.808746Z","iopub.execute_input":"2026-07-02T03:47:22.80909Z","iopub.status.idle":"2026-07-02T03:47:22.814601Z","shell.execute_reply.started":"2026-07-02T03:47:22.809059Z","shell.execute_reply":"2026-07-02T03:47:22.813773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#tìm đường dẫn đúng của thư mục \nfrom pathlib import Path\n\nprint(\"Tất cả file CSV trong /kaggle/input:\")\nfor p in Path(\"/kaggle/input\").rglob(\"*.csv\"):\n    print(p)\n\nprint(\"\\nTất cả folder tên train trong /kaggle/input:\")\nfor p in Path(\"/kaggle/input\").rglob(\"train\"):\n    if p.is_dir():\n        print(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:47:32.325479Z","iopub.execute_input":"2026-07-02T03:47:32.326105Z","iopub.status.idle":"2026-07-02T03:47:49.975034Z","shell.execute_reply.started":"2026-07-02T03:47:32.326074Z","shell.execute_reply":"2026-07-02T03:47:49.974211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\n\n# File train.csv gốc của competition VinBigData\ntrain_csv_path = Path(\"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\n\n# File train_meta.csv của dataset ảnh resize 512x512\ntrain_meta_path = Path(\"/kaggle/input/datasets/vinbigdata-chest-xray-resized-png-512x512/train_meta.csv\")\n\n# Thư mục ảnh train resize 512x512\ntrain_img_dir = Path(\"/kaggle/input/datasets/xhlulu/vinbigdata/train\")\n\nprint(\"train.csv tồn tại:\", train_csv_path.exists())\nprint(\"train_meta.csv tồn tại:\", train_meta_path.exists())\nprint(\"train image folder tồn tại:\", train_img_dir.exists())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:49:35.407809Z","iopub.execute_input":"2026-07-02T03:49:35.408491Z","iopub.status.idle":"2026-07-02T03:49:35.41701Z","shell.execute_reply.started":"2026-07-02T03:49:35.408459Z","shell.execute_reply":"2026-07-02T03:49:35.41629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\n\ntrain_csv_path = Path(\"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\n\ndf = pd.read_csv(train_csv_path)\n\nlabel_stats = (\n    df.groupby([\"class_id\", \"class_name\"])\n    .agg(\n        so_annotation=(\"image_id\", \"count\"),\n        so_anh=(\"image_id\", \"nunique\")\n    )\n    .reset_index()\n    .sort_values(\"class_id\")\n)\n\nlabel_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:51:03.446728Z","iopub.execute_input":"2026-07-02T03:51:03.447347Z","iopub.status.idle":"2026-07-02T03:51:03.58946Z","shell.execute_reply.started":"2026-07-02T03:51:03.447316Z","shell.execute_reply":"2026-07-02T03:51:03.58856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for _, row in label_stats.iterrows():\n    print(\n        f\"ID {int(row['class_id']):2d} | \"\n        f\"{row['class_name']:<35} | \"\n        f\"Số annotation: {row['so_annotation']:5d} | \"\n        f\"Số ảnh: {row['so_anh']:5d}\"\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:51:27.380827Z","iopub.execute_input":"2026-07-02T03:51:27.381613Z","iopub.status.idle":"2026-07-02T03:51:27.387678Z","shell.execute_reply.started":"2026-07-02T03:51:27.38158Z","shell.execute_reply":"2026-07-02T03:51:27.386843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Tạo folder mới\nfrom pathlib import Path\n\nsave_dir = Path(\"/kaggle/working/vin512\")\nsave_dir.mkdir(parents=True, exist_ok=True)\n\nprint(\"Đã tạo folder:\", save_dir)\nprint(\"Folder tồn tại:\", save_dir.exists())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T06:53:28.279391Z","iopub.execute_input":"2026-07-02T06:53:28.279666Z","iopub.status.idle":"2026-07-02T06:53:28.289919Z","shell.execute_reply.started":"2026-07-02T06:53:28.279627Z","shell.execute_reply":"2026-07-02T06:53:28.288923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Xóa folder mới\nimport shutil\nfrom pathlib import Path\n\nfolder_path = Path(\"/kaggle/working/csv_outputs\")\n\nif folder_path.exists():\n    shutil.rmtree(folder_path)\n    print(\"Đã xóa folder:\", folder_path)\nelse:\n    print(\"Folder không tồn tại:\", folder_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:58:11.922444Z","iopub.execute_input":"2026-07-02T03:58:11.922776Z","iopub.status.idle":"2026-07-02T03:58:11.928519Z","shell.execute_reply.started":"2026-07-02T03:58:11.922745Z","shell.execute_reply":"2026-07-02T03:58:11.927706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Lưu ra file CSV\noutput_path = \"/kaggle/working/vin512/train1.csv\"\nlabel_stats.to_csv(output_path, index=False)\n\nprint(\"Đã tạo file CSV:\", output_path)\n\nlabel_stats","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T03:59:04.057982Z","iopub.execute_input":"2026-07-02T03:59:04.058884Z","iopub.status.idle":"2026-07-02T03:59:04.071333Z","shell.execute_reply.started":"2026-07-02T03:59:04.05885Z","shell.execute_reply":"2026-07-02T03:59:04.07049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gộp train.csv và train_meta.csv\n\nimport pandas as pd\nfrom pathlib import Path\nfrom IPython.display import FileLink\n\n# =========================\n# 1. Khai báo đường dẫn\n# =========================\n\ntrain_csv_path = Path(\"/kaggle/input/competitions/vinbigdata-chest-xray-abnormalities-detection/train.csv\")\ntrain_meta_path = Path(\"/kaggle/input/datasets/xhlulu/vinbigdata/train_meta.csv\")\n\n\n# Folder lưu kết quả trong /kaggle/working\nsave_dir = Path(\"/kaggle/working/vin512\")\nsave_dir.mkdir(parents=True, exist_ok=True)\n\noutput_path = save_dir / \"vinbigdata_train_merged_meta_512.csv\"\n\n# =========================\n# 2. Kiểm tra file tồn tại\n# =========================\n\nprint(\"train.csv:\", train_csv_path)\nprint(\"train.csv tồn tại:\", train_csv_path.exists())\n\nprint(\"train_meta.csv:\", train_meta_path)\nprint(\"train_meta.csv tồn tại:\", train_meta_path.exists())\n\nassert train_csv_path.exists(), \"Sai đường dẫn train.csv\"\nassert train_meta_path.exists(), \"Sai đường dẫn train_meta.csv\"\n\n# =========================\n# 3. Đọc 2 file CSV\n# =========================\n\ndf_train = pd.read_csv(train_csv_path)\ndf_meta = pd.read_csv(train_meta_path)\n\nprint(\"Các cột trong train.csv:\")\nprint(df_train.columns.tolist())\n\nprint(\"\\nCác cột trong train_meta.csv:\")\nprint(df_meta.columns.tolist())\n\nprint(\"\\nSố dòng train.csv:\", len(df_train))\nprint(\"Số ảnh unique trong train.csv:\", df_train[\"image_id\"].nunique())\n\nprint(\"Số dòng train_meta.csv:\", len(df_meta))\nprint(\"Số ảnh unique trong train_meta.csv:\", df_meta[\"image_id\"].nunique())\n\n# =========================\n# 4. Gộp 2 file theo image_id\n# Chỉ giữ ảnh có trong cả 2 file\n# =========================\n\ndf_merged = pd.merge(\n    df_train,\n    df_meta,\n    on=\"image_id\",\n    how=\"inner\"\n)\n\nprint(\"\\nSố dòng sau khi gộp:\", len(df_merged))\nprint(\"Số ảnh unique sau khi gộp:\", df_merged[\"image_id\"].nunique())\n\n# =========================\n# 5. Lưu file CSV mới\n# =========================\n\ndf_merged.to_csv(output_path, index=False)\n\nprint(\"\\nĐã lưu file tại:\", output_path)\n\ndisplay(df_merged.head())\n\nFileLink(str(output_path))","metadata":{"trusted":true,"execution":{"execution_failed":"2026-07-02T06:53:06.526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom pathlib import Path\nimport shutil\nimport random\nfrom tqdm import tqdm\nfrom IPython.display import FileLink\n\n# =========================\n# 1. Đường dẫn\n# =========================\n\n# File CSV đã gộp train.csv + train_meta.csv\ncsv_path = Path(\"/kaggle/working/vin512/vinbigdata_train_merged_meta_512.csv\")\n\n# Folder ảnh 512x512\nimg_dir = Path(\"/kaggle/input/datasets/xhlulu/vinbigdata/train\")\n\n# Folder output\nout_dir = Path(\"/kaggle/working/vin512_split_80_20_by_label\")\n\n# Xóa folder cũ nếu đã tồn tại để tránh lẫn ảnh từ lần chia trước\nif out_dir.exists():\n    shutil.rmtree(out_dir)\n\ntrain_img_dir = out_dir / \"train\"\ntest_img_dir = out_dir / \"test\"\n\ntrain_img_dir.mkdir(parents=True, exist_ok=True)\ntest_img_dir.mkdir(parents=True, exist_ok=True)\n\nprint(\"CSV tồn tại:\", csv_path.exists())\nprint(\"Folder ảnh tồn tại:\", img_dir.exists())\n\nassert csv_path.exists(), \"Sai đường dẫn file CSV\"\nassert img_dir.exists(), \"Sai đường dẫn folder ảnh\"\n\n# =========================\n# 2. Đọc CSV\n# =========================\n\ndf = pd.read_csv(csv_path)\n\ndf[\"class_id\"] = df[\"class_id\"].astype(int)  #df (dataframe), class_id(nguyên)\ndf[\"image_id\"] = df[\"image_id\"].astype(str)  #df (dataframe), image_id(chuỗi)\n\nprint(\"Số dòng annotation:\", len(df))\nprint(\"Số ảnh unique:\", df[\"image_id\"].nunique())\nprint(\"Số nhãn:\", df[\"class_id\"].nunique())\n\n# =========================\n# 3. Gom nhãn theo từng ảnh\n# =========================\n# 1 anhảnh có những nhãn nào\nimage_labels = (\n    df.groupby(\"image_id\")[\"class_id\"]\n    .apply(lambda x: set(x.astype(int)))\n    .to_dict()\n)   #\"image_001\": {0, 3},\n\nall_image_ids = list(image_labels.keys())\nall_class_ids = sorted(df[\"class_id\"].unique())  \n\n# 1 nhãn gồm những ảnh nào\nlabel_to_images = {}\n\nfor class_id in all_class_ids:\n    label_to_images[class_id] = [\n        image_id for image_id, labels in image_labels.items()\n        if class_id in labels\n    ]  #0: [\"img001\", \"img008\", \"img100\"],\n\n# Tổng số ảnh ccủa mỗi nhãn \nlabel_total = {\n    class_id: len(label_to_images[class_id])\n    for class_id in all_class_ids\n}  #  0: 716\n\n# Mục tiêu số ảnh test cho từng nhãn = 20%\nlabel_test_target = {\n    class_id: round(label_total[class_id] * 0.2)\n    for class_id in all_class_ids\n}\n\nprint(\"\\nMục tiêu test theo từng nhãn:\")\nfor class_id in all_class_ids:\n    print(\n        f\"class_id {class_id:2d} | \"\n        f\"total: {label_total[class_id]:5d} | \"\n        f\"test target: {label_test_target[class_id]:5d}\"\n    )\n\n# =========================\n# 4. Chia ngẫu nhiên có giữ tỷ lệ nhãn\n# =========================\n\nrandom.seed(42)\n\ntarget_test_size = round(len(all_image_ids) * 0.2)\n\ntest_ids = set()  #chứa các ảnh được chọn vào test\ntest_label_count = {class_id: 0 for class_id in all_class_ids} # số class trong test,đếm 1 class có bao nhiêu ảnh.\n\n# Ưu tiên nhãn ít ảnh trước\nclass_order = sorted(all_class_ids, key=lambda c: label_total[c])\n #trogn 14 class, mỗi class có bao nhiêu annotation vd: label_total[2]=500, nghĩa là class2 có 500 annotation\n\nfor class_id in class_order:\n    target = label_test_target[class_id]\n #mỗi class phải đạt target test=20% tổng\n\n    while test_label_count[class_id] < target:\n        candidates = [\n            image_id for image_id in label_to_images[class_id]\n            if image_id not in test_ids\n        ]   #ảnh được chọn trong candidates phải là ảnh thuộc class_id đang xét, và chưa có trong test\n \n        \n        if len(candidates) == 0:\n            break\n            #Nếu không có ảnh nào trùng với yêu cầu thì ngừng vòng lặp\n        \n\n        # Chọn ảnh giúp cải thiện nhiều nhãn đang thiếu test nhất\n        def score_image(image_id):\n            labels = image_labels[image_id]\n\n            improve = sum(\n                max(label_test_target[lb] - test_label_count[lb], 0)\n                for lb in labels\n            )\n\n            overshoot = sum(\n                max((test_label_count[lb] + 1) - label_test_target[lb], 0)\n                for lb in labels\n            )\n\n            return improve - overshoot + random.random() * 0.01\n\n        best_img = max(candidates, key=score_image)\n\n        test_ids.add(best_img)\n\n        for lb in image_labels[best_img]:\n            test_label_count[lb] += 1\n\n# Nếu test nhiều hơn 20%, thử bỏ bớt ảnh mà vẫn không làm thiếu nhãn\nwhile len(test_ids) > target_test_size:\n    removable = []\n\n    for image_id in test_ids:\n        labels = image_labels[image_id]\n\n        can_remove = True\n        for lb in labels:\n            if test_label_count[lb] - 1 < label_test_target[lb]:\n                can_remove = False\n                break\n\n        if can_remove:\n            removable.append(image_id)\n\n    if len(removable) == 0:\n        break\n\n    remove_img = random.choice(removable)\n    test_ids.remove(remove_img)\n\n    for lb in image_labels[remove_img]:\n        test_label_count[lb] -= 1\n\n# Nếu test ít hơn 20%, bốc thêm ngẫu nhiên\nremaining_ids = [img for img in all_image_ids if img not in test_ids]\nrandom.shuffle(remaining_ids)\n\nwhile len(test_ids) < target_test_size and len(remaining_ids) > 0:\n    image_id = remaining_ids.pop()\n    test_ids.add(image_id)\n\n    for lb in image_labels[image_id]:\n        test_label_count[lb] += 1\n\ntrain_ids = set(all_image_ids) - test_ids\n\nprint(\"\\nKết quả chia:\")\nprint(\"Số ảnh train:\", len(train_ids))\nprint(\"Số ảnh test:\", len(test_ids))\nprint(\"Tổng:\", len(train_ids) + len(test_ids))\n\n# Đảm bảo không có ảnh trùng train/test\noverlap = train_ids & test_ids\nprint(\"Số ảnh bị trùng train/test:\", len(overlap))\nassert len(overlap) == 0, \"Có ảnh bị trùng giữa train và test\"\n\n# =========================\n# 5. Tạo CSV train/test\n# =========================\n\ndf_train = df[df[\"image_id\"].isin(train_ids)].copy() \ndf_test = df[df[\"image_id\"].isin(test_ids)].copy()\n\ntrain_csv_path = out_dir / \"train_split.csv\"\ntest_csv_path = out_dir / \"test_split.csv\"\n\ndf_train.to_csv(train_csv_path, index=False)\ndf_test.to_csv(test_csv_path, index=False)\n\nprint(\"\\nĐã lưu CSV:\")\nprint(train_csv_path)\nprint(test_csv_path)\n\n# =========================\n# 6. Copy ảnh vào folder train/test\n# =========================\n\ndef find_image_path(image_id, folder):\n    for ext in [\".png\", \".jpg\", \".jpeg\", \".dcm\", \".dicom\"]:\n        p = folder / f\"{image_id}{ext}\"\n        if p.exists():\n            return p\n    return None\n\ndef copy_images(image_ids, dst_dir):\n    missing = []\n\n    for image_id in tqdm(image_ids):\n        src = find_image_path(image_id, img_dir)\n\n        if src is None:\n            missing.append(image_id)\n            continue\n\n        dst = dst_dir / src.name\n        shutil.copy2(src, dst)\n\n    return missing\n\nmissing_train = copy_images(train_ids, train_img_dir)\nmissing_test = copy_images(test_ids, test_img_dir)\n\nprint(\"\\nSố ảnh thiếu train:\", len(missing_train))\nprint(\"Số ảnh thiếu test:\", len(missing_test))\n\n# =========================\n# 7. Thống kê kiểm tra tỷ lệ từng nhãn\n# =========================\n\ndef label_stat(df_part, name):\n    stat = (\n        df_part.groupby([\"class_id\", \"class_name\"])\n        .agg(\n            **{\n                f\"annotation_{name}\": (\"image_id\", \"count\"),\n                f\"images_{name}\": (\"image_id\", \"nunique\")\n            }\n        )\n        .reset_index()\n    )\n    return stat\n\nstat_train = label_stat(df_train, \"train\")\nstat_test = label_stat(df_test, \"test\")\n\nstat_all = (\n    stat_train\n    .merge(stat_test, on=[\"class_id\", \"class_name\"], how=\"outer\")\n    .fillna(0)\n)\n\nstat_all[\"images_total\"] = stat_all[\"images_train\"] + stat_all[\"images_test\"]\nstat_all[\"test_ratio_by_image\"] = stat_all[\"images_test\"] / stat_all[\"images_total\"]\n\nstat_all[\"annotation_total\"] = stat_all[\"annotation_train\"] + stat_all[\"annotation_test\"]\nstat_all[\"test_ratio_by_annotation\"] = stat_all[\"annotation_test\"] / stat_all[\"annotation_total\"]\n\nstat_all = stat_all.sort_values(\"class_id\")\n\nstat_path = out_dir / \"label_distribution_train_test.csv\"\nstat_all.to_csv(stat_path, index=False)\n\nprint(\"\\nThống kê tỷ lệ từng nhãn:\")\ndisplay(stat_all)\n\nprint(\"\\nĐã lưu thống kê:\", stat_path)\n\n# =========================\n# 8. Kiểm tra số ảnh trong folder\n# =========================\n\ntrain_files = set(p.name for p in train_img_dir.glob(\"*\") if p.is_file())\ntest_files = set(p.name for p in test_img_dir.glob(\"*\") if p.is_file())\n\noverlap_files = train_files & test_files\n\nprint(\"\\nKiểm tra folder ảnh:\")\nprint(\"Số ảnh trong folder train:\", len(train_files))\nprint(\"Số ảnh trong folder test:\", len(test_files))\nprint(\"Số ảnh trùng giữa 2 folder:\", len(overlap_files))\n\n# Link tải CSV\ndisplay(FileLink(str(train_csv_path)))\ndisplay(FileLink(str(test_csv_path)))\ndisplay(FileLink(str(stat_path)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T04:39:21.386045Z","iopub.execute_input":"2026-07-02T04:39:21.386388Z","iopub.status.idle":"2026-07-02T04:42:41.502155Z","shell.execute_reply.started":"2026-07-02T04:39:21.38636Z","shell.execute_reply":"2026-07-02T04:42:41.501503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}