{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":13691806,"datasetId":8708483,"databundleVersionId":14434603},{"sourceType":"datasetVersion","sourceId":1799839,"datasetId":1069682,"databundleVersionId":1837296},{"sourceType":"datasetVersion","sourceId":1800777,"datasetId":1069787,"databundleVersionId":1838236}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:45:54.100438Z","iopub.execute_input":"2025-11-11T09:45:54.100709Z","iopub.status.idle":"2025-11-11T09:47:15.629242Z","shell.execute_reply.started":"2025-11-11T09:45:54.100691Z","shell.execute_reply":"2025-11-11T09:47:15.628496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -U numpy==1.26.4 scikit-learn==1.4.2 scipy==1.13.1 --force-reinstall","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:47:15.630806Z","iopub.execute_input":"2025-11-11T09:47:15.631354Z","iopub.status.idle":"2025-11-11T09:47:49.863557Z","shell.execute_reply.started":"2025-11-11T09:47:15.631331Z","shell.execute_reply":"2025-11-11T09:47:49.862821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport numpy as np\nimport shutil\nimport yaml\nimport matplotlib.pyplot as plt\nimport random\nimport cv2\nimport multiprocessing\nfrom tqdm import tqdm\nfrom glob import glob\nfrom sklearn import model_selection\nfrom skimage import exposure\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:47:49.864582Z","iopub.execute_input":"2025-11-11T09:47:49.864861Z","iopub.status.idle":"2025-11-11T09:47:51.371365Z","shell.execute_reply.started":"2025-11-11T09:47:49.864839Z","shell.execute_reply":"2025-11-11T09:47:51.370792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# size = 1024\n# TRAIN_LABELS_PATH = './vinbigdata/labels/train'\n# VAL_LABELS_PATH = './vinbigdata/labels/val'\n# TRAIN_IMAGES_PATH = './vinbigdata/images/train' #12000\n# VAL_IMAGES_PATH = './vinbigdata/images/val' #3000\n# External_DIR = f'../input/vinbigdata-{size}-image-dataset/vinbigdata/train' # 15000\n# os.makedirs(TRAIN_LABELS_PATH, exist_ok = True)\n# os.makedirs(VAL_LABELS_PATH, exist_ok = True)\n# os.makedirs(TRAIN_IMAGES_PATH, exist_ok = True)\n# os.makedirs(VAL_IMAGES_PATH, exist_ok = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:44:34.635636Z","iopub.execute_input":"2025-11-09T19:44:34.636099Z","iopub.status.idle":"2025-11-09T19:44:34.641621Z","shell.execute_reply.started":"2025-11-09T19:44:34.636074Z","shell.execute_reply":"2025-11-09T19:44:34.640988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# original_df = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\n# number_of_imageids = len(original_df['image_id'].values)\n# print(f'Total number of image_ids (train + validation) {number_of_imageids}')\n\n# number_of_images = len(os.listdir('../input/vinbigdata-chest-xray-abnormalities-detection/train'))\n# print(f'Total number of images (train + validation) {number_of_images}')\n\n# number_of_labels = len(os.listdir('../input/vinbigdata-yolo-labels-dataset/labels'))\n# print(f'Total number of labels (train + validation) {number_of_labels}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:44:52.890387Z","iopub.execute_input":"2025-11-09T19:44:52.890654Z","iopub.status.idle":"2025-11-09T19:44:53.308975Z","shell.execute_reply.started":"2025-11-09T19:44:52.890635Z","shell.execute_reply":"2025-11-09T19:44:53.30818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import cv2\n# import numpy as np\n# import pydicom\n# import multiprocessing\n# from tqdm import tqdm\n# from skimage import exposure\n\n# def dicom2array(path, voi_lut=True, fix_monochrome=True):\n#     dicom = pydicom.read_file(path)\n#     if voi_lut:\n#         data = apply_voi_lut(dicom.pixel_array, dicom)\n#     else:\n#         data = dicom.pixel_array\n#     if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n#         data = np.amax(data) - data\n#     data = data - np.min(data)\n#     data = data / np.max(data)\n#     data = (data * 255).astype(np.uint8)\n#     return data\n\n# def process_image(dicom_path_output_dir):\n#     dicom_path, output_dir = dicom_path_output_dir\n#     file_name = os.path.splitext(os.path.basename(dicom_path))[0]\n#     image_array = dicom2array(dicom_path)\n#     equalized_image = exposure.equalize_hist(image_array)\n#     equalized_image = (equalized_image * 255).astype(np.uint8)\n#     cv2.imwrite(os.path.join(output_dir, f\"{file_name}.jpeg\"), equalized_image)\n\n# def saving_image(output_dir, dicom_path_list):\n#     os.makedirs(output_dir, exist_ok=True)\n#     dicom_path_output_dir_list = [(path, output_dir) for path in dicom_path_list]\n\n#     # Use multiprocessing Pool for parallel processing\n#     with multiprocessing.Pool() as pool:\n#         list(tqdm(pool.imap(process_image, dicom_path_output_dir_list), total=len(dicom_path_list), desc=\"Processing Images\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:44:58.19249Z","iopub.execute_input":"2025-11-09T19:44:58.192973Z","iopub.status.idle":"2025-11-09T19:44:58.199917Z","shell.execute_reply.started":"2025-11-09T19:44:58.19295Z","shell.execute_reply":"2025-11-09T19:44:58.199239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df = pd.read_csv('/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\n# number_of_images = len(df['image_id'].values)\n# print(f'Total number of image ids (train + validation) {number_of_images}')\n\n# df = df[df.class_id!=14].reset_index(drop = True)\n# number_of_images = len(df['image_id'].values)\n# print(f'Total number of image ids after dropping normal images (train + validation) {number_of_images}')\n\n# df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:45:05.606355Z","iopub.execute_input":"2025-11-09T19:45:05.606603Z","iopub.status.idle":"2025-11-09T19:45:05.70747Z","shell.execute_reply.started":"2025-11-09T19:45:05.606586Z","shell.execute_reply":"2025-11-09T19:45:05.706823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df = df.drop(columns=['class_name', 'rad_id', 'x_min', 'x_max', 'y_min', 'y_max',  'class_id']) # we only need image ids, labels are pre-made\n# df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:45:07.944727Z","iopub.execute_input":"2025-11-09T19:45:07.945269Z","iopub.status.idle":"2025-11-09T19:45:07.954356Z","shell.execute_reply.started":"2025-11-09T19:45:07.945243Z","shell.execute_reply":"2025-11-09T19:45:07.953756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import pandas as pd\n# from sklearn.model_selection import GroupShuffleSplit\n# import os\n# from tqdm.notebook import tqdm\n# import shutil # Đảm bảo đã import shutil\n\n# # --- PHẦN 1: CODE CHIA \"SẠCH\" (GROUP SHUFFLE SPLIT) ---\n\n# # 1. Tải file train.csv GỐC\n# TRAIN_CSV_PATH = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv' \n# try:\n#     df_train_full = pd.read_csv(TRAIN_CSV_PATH)\n#     print(f\"Đã tải thành công file: {TRAIN_CSV_PATH}\")\n# except FileNotFoundError:\n#     print(f\"LỖI: Không tìm thấy file {TRAIN_CSV_PATH}.\")\n#     raise\n\n# # 2. Xác định Dữ liệu (X) và Nhóm (Groups)\n# X = df_train_full.index\n# groups = df_train_full['image_id'] # Dùng 'image_id' làm NHÓM\n# print(f\"Chuẩn bị chia {len(df_train_full)} bounding box...\")\n# print(f\"Dựa trên {len(groups.unique())} ảnh (nhóm) duy nhất.\")\n\n# # 3. Khởi tạo công cụ chia\n# gss = GroupShuffleSplit(n_splits=1, test_size=0.15, random_state=42)\n# train_idx, val_idx = next(gss.split(X, groups=groups))\n\n# # 4. Lấy danh sách image_id \"SẠCH\" (Không rò rỉ)\n# train_image_ids = df_train_full.iloc[train_idx]['image_id'].unique()\n# val_image_ids = df_train_full.iloc[val_idx]['image_id'].unique()\n# print(f\"Số ảnh Train: {len(train_image_ids)}\")\n# print(f\"Số ảnh Val: {len(val_image_ids)}\")\n\n\n# # --- PHẦN 2: CODE COPY FILE CỦA BẠN (PREPROCCESS_DATA) ---\n\n# # (Hàm preproccess_data của bạn giữ nguyên ở đây)\n# def preproccess_data(df, labels_path, images_path):\n#     # (Biến 'size' phải được định nghĩa ở cell trước đó, ví dụ: size = 1024)\n#     for img_id in tqdm(df.image_id.unique()):\n#         label_src = f\"../input/vinbigdata-yolo-labels-dataset/labels/{img_id}.txt\"\n#         image_src = f\"/kaggle/input/vinbigdata-{size}-image-dataset/vinbigdata/train/{img_id}.png\"\n#         if os.path.exists(label_src) and os.path.exists(image_src):\n#             shutil.copy(label_src, labels_path)\n#             shutil.copy(image_src, images_path)\n\n# # --- SỬA LỖI: Tạo DataFrame \"sạch\" ---\n# # (Biến train_image_ids và val_image_ids BÂY GIỜ đã tồn tại)\n# print(\"Đang tạo DataFrame 'sạch' từ danh sách ID...\")\n# df_train = pd.DataFrame({'image_id': train_image_ids})\n# df_valid = pd.DataFrame({'image_id': val_image_ids})\n# # --- HẾT PHẦN SỬA ---\n\n# # Bây giờ các biến df_train, df_valid đã tồn tại và \"sạch\"\n# preproccess_data(df_train, TRAIN_LABELS_PATH, TRAIN_IMAGES_PATH)\n# preproccess_data(df_valid, VAL_LABELS_PATH, VAL_IMAGES_PATH)\n\n# print(\"\\nSố lượng file sau khi copy:\")\n# print(\"Train Labels:\", len(os.listdir(TRAIN_LABELS_PATH)))\n# print(\"Train Images:\", len(os.listdir(TRAIN_IMAGES_PATH)))\n# print(\"Val Labels:\", len(os.listdir(VAL_LABELS_PATH)))\n# print(\"Val Images:\", len(os.listdir(VAL_IMAGES_PATH)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:46:55.671537Z","iopub.execute_input":"2025-11-09T19:46:55.672269Z","iopub.status.idle":"2025-11-09T19:50:44.311728Z","shell.execute_reply.started":"2025-11-09T19:46:55.672245Z","shell.execute_reply":"2025-11-09T19:50:44.310955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# classes = [\n#     'Aortic enlargement', 'Atelectasis', 'Calcification', 'Cardiomegaly',\n#     'Consolidation', 'ILD', 'Infiltration', 'Lung Opacity', 'Nodule/Mass',\n#     'Other lesion', 'Pleural effusion', 'Pleural thickening', 'Pneumothorax', 'Pulmonary fibrosis'\n# ]\n\n# data = dict(\n#     train='../vinbigdata/images/train',\n#     val='../vinbigdata/images/val',\n#     nc=14,\n#     names=classes\n# )\n\n# with open('/kaggle/working/vinbigdata.yaml', 'w') as outfile:\n#     yaml.dump(data, outfile, default_flow_style=False)\n\n# print(\"\\nCấu hình YAML:\")\n# print(open('/kaggle/working/vinbigdata.yaml').read())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T19:53:42.531062Z","iopub.execute_input":"2025-11-09T19:53:42.531669Z","iopub.status.idle":"2025-11-09T19:53:42.538668Z","shell.execute_reply.started":"2025-11-09T19:53:42.531644Z","shell.execute_reply":"2025-11-09T19:53:42.537773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\nold_runs_dir = '/kaggle/input/runs-epoch-51-yolov11/runs'\nnew_runs_dir = '/kaggle/working/runs'\n\nprint(f\"Chuẩn bị sao chép toàn bộ thư mục 'runs' cũ...\")\ntry:\n    if os.path.exists(old_runs_dir):\n        shutil.copytree(old_runs_dir, new_runs_dir)\n        print(f\"\\nSao chép thành công!\")\n        print(\"\\nCấu trúc thư mục mới trong /kaggle/working/runs/detect:\")\n        os.system(f\"ls -l {new_runs_dir}/detect/\")\n    else:\n        print(f\"\\n--- LỖI ---\")\n        print(f\"Không tìm thấy thư mục 'runs' cũ tại: {old_runs_dir}\")\nexcept FileExistsError:\n    print(f\"\\n--- CẢNH BÁO ---\")\n    print(f\"Thư mục {new_runs_dir} đã tồn tại. Bỏ qua bước sao chép.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:47:51.372671Z","iopub.execute_input":"2025-11-11T09:47:51.372992Z","iopub.status.idle":"2025-11-11T09:47:54.412937Z","shell.execute_reply.started":"2025-11-11T09:47:51.372975Z","shell.execute_reply":"2025-11-11T09:47:54.412182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"old_yaml_path = '/kaggle/input/runs-epoch-51-yolov11/vinbigdata.yaml'\n\nnew_yaml_path = '/kaggle/working/vinbigdata.yaml'\n\nprint(f\"\\nChuẩn bị sao chép file 'vinbigdata.yaml'...\")\nprint(f\"  Từ: {old_yaml_path}\")\nprint(f\"  Đến: {new_yaml_path}\")\n\ntry:\n    if os.path.exists(old_yaml_path):\n        shutil.copyfile(old_yaml_path, new_yaml_path)\n        print(f\"\\nSao chép 'vinbigdata.yaml' thành công!\")\n    else:\n        print(f\"\\n--- LỖI ---\")\n        print(f\"Không tìm thấy file .yaml cũ tại: {old_yaml_path}\")\n        print(\"Hãy kiểm tra lại đường dẫn.\")\n\nexcept FileExistsError:\n    print(f\"\\n--- CẢNH BÁO ---\")\n    print(f\"File {new_yaml_path} đã tồn tại. Bỏ qua bước sao chép .yaml.\")\n\nexcept Exception as e:\n    print(f\"\\n--- LỖI KHÁC KHI SAO CHÉP .yaml ---\")\n    print(e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:47:54.413614Z","iopub.execute_input":"2025-11-11T09:47:54.413854Z","iopub.status.idle":"2025-11-11T09:47:54.423354Z","shell.execute_reply.started":"2025-11-11T09:47:54.413836Z","shell.execute_reply":"2025-11-11T09:47:54.422617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\n# 1. Đường dẫn đến thư mục 'vinbigdata' trong output của version CŨ\nold_vinbigdata_dir = '/kaggle/input/runs-epoch-51-yolov11/vinbigdata/'\n\n# 2. Đường dẫn đến thư mục 'vinbigdata' MỚI trong /kaggle/working/\nnew_vinbigdata_dir = '/kaggle/working/vinbigdata'\n\n# --- Đã sửa lại câu thông báo ---\nprint(f\"Chuẩn bị sao chép toàn bộ thư mục 'vinbigdata' cũ...\") \nprint(f\"  Từ: {old_vinbigdata_dir}\")\nprint(f\"  Đến: {new_vinbigdata_dir}\")\n\n# 3. Dùng shutil.copytree để copy toàn bộ thư mục\ntry:\n    if os.path.exists(old_vinbigdata_dir):\n        shutil.copytree(old_vinbigdata_dir, new_vinbigdata_dir)\n        print(f\"\\nSao chép thành công!\")\n        \n        # --- Đã sửa lại lệnh kiểm tra ---\n        print(f\"\\nCấu trúc thư mục mới trong {new_vinbigdata_dir}:\")\n        os.system(f\"ls -l {new_vinbigdata_dir}\") # Liệt kê nội dung của vinbigdata\n        \n    else:\n        print(f\"\\n--- LỖI ---\")\n        print(f\"Không tìm thấy thư mục 'vinbigdata' cũ tại: {old_vinbigdata_dir}\")\n        print(\"Bạn đã '+ Add data' output của version cũ (notebook-yolov11-1024) chưa?\")\n        \nexcept FileExistsError:\n    print(f\"\\n--- CẢNH BÁO ---\")\n    print(f\"Thư mục {new_vinbigdata_dir} đã tồn tại. Bỏ qua bước sao chép.\")\n    print(\"Điều này ổn nếu bạn chạy lại cell.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:47:54.424158Z","iopub.execute_input":"2025-11-11T09:47:54.424401Z","iopub.status.idle":"2025-11-11T09:51:12.220029Z","shell.execute_reply.started":"2025-11-11T09:47:54.424378Z","shell.execute_reply":"2025-11-11T09:51:12.21924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ultralytics import YOLO\n\n# Load model\nmodel = YOLO('/kaggle/input/runs-epoch-51-yolov11/runs/detect/train/weights/best.pt')\n\n# Train YOLOv11\nmodel.train(\n    # --- Các tham số gốc của bạn ---\n    data='./vinbigdata.yaml',\n    imgsz=1024,\n    batch=16,\n    device=[0,1],\n    epochs=100,\n    patience=30,\n    cos_lr=True,\n    resume=True,\n\n    optimizer='SGD',  \n    lr0=0.0005,       \n    lrf=0.001,\n    \n    mixup=0.2,\n    cutmix=0.15,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T09:51:12.220833Z","iopub.execute_input":"2025-11-11T09:51:12.221094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from ultralytics import YOLO\n\n# # Load model\n# model = YOLO('yolo11l.pt')\n\n# # Train YOLOv11\n# model.train(\n#     # --- Các tham số gốc của bạn ---\n#     data='./vinbigdata.yaml',\n#     imgsz=1024,\n#     batch=16,\n#     device=[0,1],\n#     epochs=100,\n#     patience=30,\n#     cos_lr=True,\n#     resume=False,\n\n#     optimizer='SGD',  \n#     lr0=0.0005,       \n#     lrf=0.001,\n    \n#     mixup=0.2,\n#     cutmix=0.15,\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T20:06:31.11144Z","iopub.execute_input":"2025-11-09T20:06:31.11191Z"}},"outputs":[],"execution_count":null}]}