{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":24800,"databundleVersionId":1831594,"sourceType":"competition"},{"sourceId":1799839,"sourceType":"datasetVersion","datasetId":1069682},{"sourceId":1801996,"sourceType":"datasetVersion","datasetId":1069999},{"sourceId":1810939,"sourceType":"datasetVersion","datasetId":1075804},{"sourceId":2160905,"sourceType":"datasetVersion","datasetId":1297065},{"sourceId":13149551,"sourceType":"datasetVersion","datasetId":8331267},{"sourceId":13281014,"sourceType":"datasetVersion","datasetId":8416862},{"sourceId":13785408,"sourceType":"datasetVersion","datasetId":8775602},{"sourceId":13971963,"sourceType":"datasetVersion","datasetId":8858697}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q ultralytics","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:16:41.189827Z","iopub.execute_input":"2025-12-04T17:16:41.190772Z","iopub.status.idle":"2025-12-04T17:18:08.189842Z","shell.execute_reply.started":"2025-12-04T17:16:41.190731Z","shell.execute_reply":"2025-12-04T17:18:08.189123Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport os\nimport torch.nn\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom typing import Callable, Any\ntqdm.pandas()\n\nimport cv2\nimport shutil\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport pydicom\nimport yaml\nimport glob\nfrom PIL import Image \nfrom ultralytics import YOLO\nfrom PIL import Image\nfrom torch import nn, optim\nfrom torchvision import transforms, utils\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn.preprocessing import MultiLabelBinarizer \nfrom sklearn.model_selection import train_test_split\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom skimage import exposure","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:08.191294Z","iopub.execute_input":"2025-12-04T17:18:08.191555Z","iopub.status.idle":"2025-12-04T17:18:19.765755Z","shell.execute_reply.started":"2025-12-04T17:18:08.191528Z","shell.execute_reply":"2025-12-04T17:18:19.765197Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:19.766828Z","iopub.execute_input":"2025-12-04T17:18:19.767179Z","iopub.status.idle":"2025-12-04T17:18:19.856432Z","shell.execute_reply.started":"2025-12-04T17:18:19.76716Z","shell.execute_reply":"2025-12-04T17:18:19.855696Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ***Utils***\n\nimport multiprocessing\nfrom joblib import Parallel, delayed\nimport pydicom\nimport pydicom\nimport numpy as np\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef crop_with_mask_and_bboxes(\n    image: Image.Image,\n    mask: np.ndarray | None,\n    bbox_df: pd.DataFrame | None = None,\n    label_column: str = \"class_name\",\n    fallback_to_bboxes: bool = True,\n    resize_to: tuple[int, int] | None = None,\n ):\n    \"\"\"Crop an image using a mask and update bounding boxes provided via DataFrame.\n\n    If the mask has no foreground pixels and ``fallback_to_bboxes`` is True,\n    the crop will default to the tightest region covering the provided bounding boxes.\n    When ``resize_to`` is provided, the cropped image/mask are resized and\n    bounding boxes are scaled accordingly.\"\"\"\n    if resize_to is not None:\n        if not isinstance(resize_to, (tuple, list)) or len(resize_to) != 2:\n            raise ValueError(\"resize_to phải là tuple/list gồm (width, height).\")\n        if any(dim <= 0 for dim in resize_to):\n            raise ValueError(\"resize_to phải chứa các số nguyên dương.\")\n        target_width, target_height = int(resize_to[0]), int(resize_to[1])\n    else:\n        target_width = target_height = None\n\n    if mask is None:\n        mask_np = np.zeros((image.height, image.width), dtype=np.uint8)\n    else:\n        if mask.ndim == 3:\n            mask = np.argmax(mask, axis=0)\n        if mask.dtype != np.uint8:\n            mask = mask.astype(np.uint8)\n        mask_resized = Image.fromarray(mask).resize(image.size, resample=Image.NEAREST)\n        mask_np = np.array(mask_resized)\n\n    has_foreground = np.any(mask_np)\n\n    if not has_foreground and fallback_to_bboxes and bbox_df is not None and not bbox_df.empty:\n        required_cols = [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]\n        missing = [col for col in required_cols if col not in bbox_df.columns]\n        if missing:\n            raise KeyError(f\"Missing required columns in bbox DataFrame: {missing}\")\n        left = int(np.floor(bbox_df['x_min_new'].min()))\n        top = int(np.floor(bbox_df['y_min_new'].min()))\n        right = int(np.ceil(bbox_df['x_max_new'].max()))\n        bottom = int(np.ceil(bbox_df['y_max_new'].max()))\n        left = max(0, left)\n        top = max(0, top)\n        right = min(image.width, right)\n        bottom = min(image.height, bottom)\n        if left < right and top < bottom:\n            mask_np = np.zeros((image.height, image.width), dtype=np.uint8)\n            mask_np[top:bottom, left:right] = 1\n            has_foreground = True\n\n    if not has_foreground:\n        empty_cols = [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]\n        if label_column:\n            empty_cols.append(label_column)\n        return image, mask_np, pd.DataFrame(columns=empty_cols), (0, 0, image.size[0], image.size[1])\n\n    rows, cols = np.where(mask_np > 0)\n    top, bottom = rows.min(), rows.max() + 1\n    left, right = cols.min(), cols.max() + 1\n\n    cropped_img = np.array(image)[top:bottom, left:right]\n    cropped_mask = mask_np[top:bottom, left:right]\n\n    if bbox_df is None or bbox_df.empty:\n        empty_cols = [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]\n        if label_column:\n            empty_cols.append(label_column)\n        cropped_bbox_df = pd.DataFrame(columns=empty_cols)\n    else:\n        required_cols = [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]\n        missing = [col for col in required_cols if col not in bbox_df.columns]\n        if missing:\n            raise KeyError(f\"Missing required columns in bbox DataFrame: {missing}\")\n\n        records = []\n        for _, row in bbox_df.iterrows():\n            xmin, ymin, xmax, ymax = row[required_cols]\n            inter_left = max(xmin, left)\n            inter_top = max(ymin, top)\n            inter_right = min(xmax, right)\n            inter_bottom = min(ymax, bottom)\n            if inter_left < inter_right and inter_top < inter_bottom:\n                record = {\n                    \"x_min_new\": inter_left - left,\n                    \"y_min_new\": inter_top - top,\n                    \"x_max_new\": inter_right - left,\n                    \"y_max_new\": inter_bottom - top,\n                }\n                if label_column and label_column in bbox_df.columns:\n                    record[label_column] = row[label_column]\n                records.append(record)\n        if records:\n            cropped_bbox_df = pd.DataFrame(records)\n        else:\n            empty_cols = [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]\n            if label_column:\n                empty_cols.append(label_column)\n            cropped_bbox_df = pd.DataFrame(columns=empty_cols)\n\n    crop_width = cropped_img.shape[1]\n    crop_height = cropped_img.shape[0]\n    cropped_image_pil = Image.fromarray(cropped_img)\n\n    if resize_to is not None and (crop_width != target_width or crop_height != target_height):\n        scale_x_resize = target_width / crop_width\n        scale_y_resize = target_height / crop_height\n        cropped_image_resized = cropped_image_pil.resize((target_width, target_height), resample=Image.BILINEAR)\n        cropped_mask_resized = Image.fromarray(cropped_mask).resize((target_width, target_height), resample=Image.NEAREST)\n        cropped_mask = np.array(cropped_mask_resized)\n        if not cropped_bbox_df.empty:\n            cropped_bbox_df[['x_min_new', 'x_max_new']] = cropped_bbox_df[['x_min_new', 'x_max_new']].astype(float) * scale_x_resize\n            cropped_bbox_df[['y_min_new', 'y_max_new']] = cropped_bbox_df[['y_min_new', 'y_max_new']].astype(float) * scale_y_resize\n        cropped_image_pil = cropped_image_resized\n        crop_width, crop_height = target_width, target_height\n\n    if not cropped_bbox_df.empty:\n        for col in [\"x_min_new\", \"y_min_new\", \"x_max_new\", \"y_max_new\"]:\n            cropped_bbox_df[col] = cropped_bbox_df[col].astype(float)\n        if label_column and label_column in cropped_bbox_df.columns:\n            cropped_bbox_df[label_column] = cropped_bbox_df[label_column].astype(str)\n\n    return cropped_image_pil, cropped_mask, cropped_bbox_df, (left, top, right, bottom)\n\n\ndef batch_crop_images(\n    image_folder: str,\n    bbox_df: pd.DataFrame,\n    model: nn.Module,\n    transform: Callable[[Image.Image], torch.Tensor],\n    output_folder: str,\n    output_csv_path: str,\n    image_extensions: tuple[str, ...] = ('.png', '.jpg', '.jpeg'),\n    device: str | torch.device | None = None,\n    resize_to: tuple[int, int] | None = None,\n ) -> pd.DataFrame:\n    \"\"\"Run segmentation and crop every image in a folder, saving crops and updated bounding boxes.\n\n    Only images whose ``image_id`` appears in ``bbox_df`` are processed.\n    Bounding boxes are adjusted to match the cropped (and optionally resized) output.\"\"\"\n    if transform is None:\n        raise ValueError(\"Cần truyền vào tham số 'transform' để chuyển ảnh PIL sang tensor.\")\n\n    required_cols = {'image_id', 'class_name'}\n    missing_cols = required_cols - set(bbox_df.columns)\n    if missing_cols:\n        raise KeyError(f\"DataFrame bounding box phải chứa các cột: {sorted(required_cols)}. Thiếu: {sorted(missing_cols)}\")\n\n    bbox_df = bbox_df.copy()\n    bbox_df['image_id'] = bbox_df['image_id'].astype(str)\n    bbox_df['class_name'] = bbox_df['class_name'].astype(str).str.strip()\n\n    os.makedirs(output_folder, exist_ok=True)\n    image_files = sorted(\n        [f for f in os.listdir(image_folder) if f.lower().endswith(tuple(ext.lower() for ext in image_extensions))]\n    )\n    if not image_files:\n        raise FileNotFoundError(f'Không tìm thấy ảnh hợp lệ trong thư mục: {image_folder}')\n\n    target_ids = set(bbox_df['image_id'])\n    image_files = [f for f in image_files if os.path.splitext(f)[0] in target_ids]\n    if not image_files:\n        raise FileNotFoundError(\"Không có ảnh nào trong thư mục khớp với các 'image_id' trong DataFrame cung cấp.\")\n\n    model_device = next(model.parameters()).device\n    target_device = torch.device(device) if device is not None else model_device\n    was_training = model.training\n    model = model.to(target_device)\n    model.eval()\n\n    records: list[dict[str, Any]] = []\n\n    for filename in image_files:\n        image_id = os.path.splitext(filename)[0]\n        image_path = os.path.join(image_folder, filename)\n        image = Image.open(image_path).convert('L')\n        width, height = image.size\n\n        per_image_bboxes = bbox_df[bbox_df['image_id'] == image_id].copy()\n        if per_image_bboxes.empty:\n            continue\n\n        # Ensure normalized bbox columns exist; fall back to legacy names if needed.\n        norm_cols = ['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']\n        if not all(col in per_image_bboxes.columns for col in norm_cols):\n            legacy_cols = ['x_min', 'y_min', 'x_max', 'y_max']\n            if all(col in per_image_bboxes.columns for col in legacy_cols):\n                for legacy_col, norm_col in zip(legacy_cols, norm_cols):\n                    per_image_bboxes[norm_col] = per_image_bboxes[legacy_col]\n            else:\n                raise KeyError(\n                    \"Bounding boxes must contain either ['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new'] \"\n                    \"or ['x_min', 'y_min', 'x_max', 'y_max'] columns.\"\n                )\n\n        # Drop rows without valid class labels to avoid empty outputs.\n        per_image_bboxes['class_name'] = per_image_bboxes['class_name'].astype(str).str.strip()\n        per_image_bboxes = per_image_bboxes[per_image_bboxes['class_name'] != '']\n        if per_image_bboxes.empty:\n            continue\n\n        scale_x = width / 512.0\n        scale_y = height / 512.0\n        per_image_bboxes['x_min_new'] = per_image_bboxes['x_min_new'] * scale_x\n        per_image_bboxes['x_max_new'] = per_image_bboxes['x_max_new'] * scale_x\n        per_image_bboxes['y_min_new'] = per_image_bboxes['y_min_new'] * scale_y\n        per_image_bboxes['y_max_new'] = per_image_bboxes['y_max_new'] * scale_y\n        per_image_bboxes = per_image_bboxes.reset_index(drop=True)\n\n        input_tensor = transform(image)\n        if not isinstance(input_tensor, torch.Tensor):\n            raise TypeError(\"Tham số 'transform' phải trả về torch.Tensor.\")\n        if input_tensor.ndim == 3:\n            input_tensor = input_tensor.unsqueeze(0)\n        if input_tensor.ndim != 4:\n            raise ValueError(\"Tham số 'transform' phải trả về tensor dạng [C,H,W] hoặc [B,C,H,W].\")\n        input_tensor = input_tensor.to(target_device)\n\n        with torch.no_grad():\n            pred_mask = model(input_tensor)\n        pred_mask_np = pred_mask.squeeze().detach().cpu().numpy()\n\n        cropped_image, cropped_mask, cropped_bboxes_df, crop_bounds = crop_with_mask_and_bboxes(\n            image,\n            pred_mask_np,\n            per_image_bboxes,\n            label_column=\"class_name\",\n            fallback_to_bboxes=True,\n            resize_to=resize_to,\n        )\n\n        if cropped_bboxes_df is None:\n            continue\n        if not isinstance(cropped_bboxes_df, pd.DataFrame):\n            cropped_bboxes_df = pd.DataFrame(cropped_bboxes_df)\n\n        if cropped_bboxes_df.empty:\n            continue\n\n        cropped_bboxes_df['class_name'] = cropped_bboxes_df['class_name'].astype(str).str.strip()\n        cropped_bboxes_df = cropped_bboxes_df[cropped_bboxes_df['class_name'] != '']\n        if cropped_bboxes_df.empty:\n            continue\n\n        crop_filename = f\"{image_id}.png\"\n        crop_path = os.path.join(output_folder, crop_filename)\n        cropped_image.save(crop_path)\n\n        for _, row in cropped_bboxes_df.iterrows():\n            records.append({\n                'image_id': image_id,\n                'crop_path': crop_path,\n                'x_min': float(row['x_min_new']),\n                'y_min': float(row['y_min_new']),\n                'x_max': float(row['x_max_new']),\n                'y_max': float(row['y_max_new']),\n                'class_name': row['class_name'],\n            })\n\n    result_df = pd.DataFrame(records)\n    if not result_df.empty:\n        result_df.to_csv(output_csv_path, index=False)\n    else:\n        pd.DataFrame(columns=['image_id', 'crop_path', 'x_min', 'y_min', 'x_max', 'y_max', 'class_name']).to_csv(\n            output_csv_path, index=False\n        )\n\n    if was_training:\n        model.train()\n    model.to(model_device)\n\n    return result_df\n\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    # Đọc file DICOM\n    dicom = pydicom.dcmread(path)\n\n    # Áp dụng VOI LUT (windowing)\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array.astype(np.float32)\n\n    # MONOCHROME1 cần đảo ngược pixel\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.max(data) - data\n\n    # Chuẩn hóa về 0–1\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n\n    # Scale về 0–255\n    data = (data * 255).astype(np.uint8)\n\n    return data\n\ndef resize_boxes(row, h_resize, w_resize):\n    if pd.isna(row['x_min']):\n        return pd.Series([row['x_min'], row['y_min'], row['x_max'],  row['y_max']])\n\n    w_scale = w_resize/ row['width']\n    h_scale = h_resize/ row['height']\n\n    x_min_new = round(row['x_min'] * w_scale, 1)\n    x_max_new = round(row['x_max'] * w_scale, 1)\n    y_min_new = round(row['y_min'] * h_scale, 1)\n    y_max_new = round(row['y_max'] * h_scale, 1)\n\n    return pd.Series([x_min_new, y_min_new, x_max_new, y_max_new])\n\n# Hàm lấy tỉ lệ tương đối của bboxes\ndef normalize_bbox(df):\n    df['x_center'] = (df['x_min'] + df['x_max'])/2\n    df['y_center'] = (df['y_min'] + df['y_max'])/2\n    df['bbox_width'] = df['x_max'] - df['x_min']\n    df['bbox_height'] = df['y_max'] - df['y_min']\n\n    df['x_center_norm'] = df['x_center'] / df['width']\n    df['y_center_norm'] = df['y_center'] / df['height']\n    df['bbox_height_norm'] = df['bbox_height'] / df['height']\n    df['bbox_width_norm'] = df['bbox_width'] / df['width']\n    return df\n\n# Hàm ghi thông \ndef get_bbox(df, output_file):\n    with open(output_file, 'w') as f:\n        for _, row in df.iterrows():\n            class_id = row['class_id']\n            x_center, y_center = row['x_center_norm'], row['y_center_norm']\n            width, height = row[\"bbox_width_norm\"], row['bbox_height_norm']\n            f.write(f\"{class_id} {x_center} {y_center} {width} {height}\\n\")\n            \ndef plot_image_with_bounding_box(image, bounding_boxes, class_dict):\n    fig, ax = plt.subplots()\n    ax.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    \n    for box in bounding_boxes:\n        class_id, x, y, width, height = map(float, box.split())\n        image_width, image_height = image.shape[1], image.shape[0]\n        x1 = int((x - width / 2) * image_width)\n        y1 = int((y - height / 2) * image_height)\n        x2 = int((x + width / 2) * image_width)\n        y2 = int((y + height / 2) * image_height)\n        \n        # Choose random color for bounding box\n        color = [random.random() for _ in range(3)]\n        \n        rect = plt.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, edgecolor=color, facecolor='none')\n        ax.add_patch(rect)\n        \n        # Add label text using class label from dictionary\n        label_text = class_dict[int(class_id)]\n        ax.text(x1, y1, label_text, color='white', verticalalignment='top', bbox={'color': color, 'pad': 0})\n    \n    plt.show()\n\ndef main(image_path, bounding_box_path, class_dict):\n    # Read image\n    print(type(image_path))\n    image = cv2.imread(image_path)\n    \n    # Read bounding boxes\n    with open(bounding_box_path, 'r') as file:\n        bounding_boxes = file.readlines()\n    \n    # Plot image with bounding boxes\n    plot_image_with_bounding_box(image, bounding_boxes, class_dict)\n    \n# Hàm plot hình ảnh\ndef plot_img(imgs, cols=4, size=7, title=\"\", cmap='gray', is_rgb=True, img_size=(500, 500)):\n    rows = len(imgs) // cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n# Hàm cân bằng histogram cho ảnh\ndef hist_equalize(img_path_with_output):\n    img_path, output_dir = img_path_with_output\n    filename = os.path.splitext(os.path.basename(img_path))[0]\n    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    equalize_img = exposure.equalize_hist(img)\n    equalize_img = (equalize_img * 255).astype(np.uint8)\n    cv2.imwrite(os.path.join(output_dir, f'{filename}.png'), equalize_img)\n\n# Hàm lưu \ndef save_img(img_path_list, output_dir, n_jobs = -1):\n    os.makedirs(output_dir, exist_ok=True)\n    img_output_list = [(path, output_dir) for path in img_path_list]\n\n    Parallel(n_jobs = n_jobs)(\n        delayed(hist_equalize)(args) for args in tqdm(img_output_list, desc=\"Histogram Equalizing\")\n    )\n\n\ndef visualize_original_vs_crop(image_id, train_df, crop_df, image_folder, figsize=(16, 8)):\n    \"\"\"\n    Hiển thị ảnh gốc và ảnh crop cạnh nhau với bounding boxes.\n    \n    Args:\n        image_id: ID của ảnh cần hiển thị\n        train_df: DataFrame chứa bbox của ảnh gốc (train512_merge_df)\n        crop_df: DataFrame chứa bbox của ảnh crop\n        image_folder: Thư mục chứa ảnh gốc\n        figsize: Kích thước figure\n    \"\"\"\n    fig, axes = plt.subplots(1, 2, figsize=figsize)\n    \n    # ===== Ảnh gốc =====\n    ax_orig = axes[0]\n    orig_path = os.path.join(image_folder, f'{image_id}.png')\n    \n    if not os.path.exists(orig_path):\n        print(f\"Không tìm thấy ảnh: {orig_path}\")\n        return\n    \n    orig_img = Image.open(orig_path).convert(\"RGB\")\n    ax_orig.imshow(orig_img)\n    ax_orig.set_title(f\"Original Image: {image_id}\", fontsize=14, fontweight='bold')\n    ax_orig.axis(\"off\")\n    \n    # Vẽ bbox trên ảnh gốc\n    orig_bboxes = train_df[train_df['image_id'] == image_id]\n    for _, row in orig_bboxes.iterrows():\n        if pd.notna(row['x_min_new']):\n            x_min, x_max = row['x_min_new'], row['x_max_new']\n            y_min, y_max = row['y_min_new'], row['y_max_new']\n            width, height = x_max - x_min, y_max - y_min\n            \n            rect = patches.Rectangle(\n                (x_min, y_min), width, height,\n                linewidth=2, edgecolor=\"red\", facecolor=\"none\"\n            )\n            ax_orig.add_patch(rect)\n            ax_orig.text(x_min, y_min - 5, row[\"class_name\"], \n                        color=\"yellow\", fontsize=10, fontweight='bold',\n                        bbox=dict(facecolor=\"black\", alpha=0.7, pad=2))\n    \n    # ===== Ảnh crop =====\n    ax_crop = axes[1]\n    crop_data = crop_df[crop_df['image_id'] == image_id]\n    \n    if crop_data.empty:\n        ax_crop.text(0.5, 0.5, \"No crop available\", \n                    ha='center', va='center', fontsize=14)\n        ax_crop.axis(\"off\")\n        plt.tight_layout()\n        plt.show()\n        return\n    \n    crop_path = crop_data['crop_path'].iloc[0]\n    crop_img = Image.open(crop_path).convert(\"RGB\")\n    ax_crop.imshow(crop_img)\n    ax_crop.set_title(f\"Cropped Image: {image_id}\", fontsize=14, fontweight='bold')\n    ax_crop.axis(\"off\")\n    \n    # Vẽ bbox trên ảnh crop\n    for _, row in crop_data.iterrows():\n        x_min, x_max = row['x_min'], row['x_max']\n        y_min, y_max = row['y_min'], row['y_max']\n        width, height = x_max - x_min, y_max - y_min\n        \n        rect = patches.Rectangle(\n            (x_min, y_min), width, height,\n            linewidth=2, edgecolor=\"lime\", facecolor=\"none\"\n        )\n        ax_crop.add_patch(rect)\n        ax_crop.text(x_min, y_min - 5, row[\"class_name\"], \n                    color=\"yellow\", fontsize=10, fontweight='bold',\n                    bbox=dict(facecolor=\"green\", alpha=0.7, pad=2))\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # In thông tin\n    print(f\"\\n{'='*60}\")\n    print(f\"Image ID: {image_id}\")\n    print(f\"Original size: {orig_img.size}\")\n    print(f\"Cropped size: {crop_img.size}\")\n    print(f\"Number of bboxes (original): {len(orig_bboxes)}\")\n    print(f\"Number of bboxes (crop): {len(crop_data)}\")\n    print(f\"{'='*60}\\n\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:19.858598Z","iopub.execute_input":"2025-12-04T17:18:19.858929Z","iopub.status.idle":"2025-12-04T17:18:19.915597Z","shell.execute_reply.started":"2025-12-04T17:18:19.858902Z","shell.execute_reply":"2025-12-04T17:18:19.914905Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***1. Phân tích data***","metadata":{}},{"cell_type":"markdown","source":"## **Loading data**","metadata":{}},{"cell_type":"code","source":"sourse_path = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train/\"\nbboxes_path = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv\"\ndicom_path = \"/kaggle/input/vinbigdata/train_meta.csv\"\n\ns_csv = pd.read_csv(bboxes_path)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:19.916382Z","iopub.execute_input":"2025-12-04T17:18:19.916595Z","iopub.status.idle":"2025-12-04T17:18:20.07107Z","shell.execute_reply.started":"2025-12-04T17:18:19.916578Z","shell.execute_reply":"2025-12-04T17:18:20.070368Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Columns trong bộ competition: \", s_csv.columns)\n# print(\"Columns trong bộ resized: \", p_df.columns)\nprint(\"Số lượng values trong bộ competition:\" , len(s_csv))\n# print(\"Số lượng values trong bộ resized:\" , len(p_df))\nprint(\"Số lượng ảnh trong bộ competition: \", s_csv['image_id'].nunique())","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:20.071807Z","iopub.execute_input":"2025-12-04T17:18:20.07204Z","iopub.status.idle":"2025-12-04T17:18:20.095595Z","shell.execute_reply.started":"2025-12-04T17:18:20.072022Z","shell.execute_reply":"2025-12-04T17:18:20.094921Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check class id của từng bệnh\nid_csv = s_csv[['class_name', 'class_id']].drop_duplicates().reset_index(drop=True)\nid_csv.sort_values(by='class_id')","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:20.096409Z","iopub.execute_input":"2025-12-04T17:18:20.096684Z","iopub.status.idle":"2025-12-04T17:18:20.140757Z","shell.execute_reply.started":"2025-12-04T17:18:20.096663Z","shell.execute_reply":"2025-12-04T17:18:20.140217Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***2. Preprocessing***","metadata":{}},{"cell_type":"code","source":"train512_dir = '/kaggle/input/vinbigdata/train'\n# train256_dir = '/kaggle/input/vinbigdata-chest-xray-resized-png-256x256/train'\n\npath512_list = glob.glob(train512_dir + '/*')\n# path256_list = glob.glob(train256_dir + '/*')","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:20.141436Z","iopub.execute_input":"2025-12-04T17:18:20.141636Z","iopub.status.idle":"2025-12-04T17:18:20.777465Z","shell.execute_reply.started":"2025-12-04T17:18:20.141619Z","shell.execute_reply":"2025-12-04T17:18:20.776855Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add thêm path của từng ảnh vô trong file csv (512x512)\np512_csv = s_csv.copy()\np512_csv['image_path'] = p512_csv['image_id'].progress_apply(lambda x: next(filter(lambda y: x in y, path512_list), None))\np512_csv['image_path'].head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:18:20.778248Z","iopub.execute_input":"2025-12-04T17:18:20.778457Z","iopub.status.idle":"2025-12-04T17:19:22.344998Z","shell.execute_reply.started":"2025-12-04T17:18:20.778441Z","shell.execute_reply":"2025-12-04T17:19:22.344252Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(p512_csv)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:22.347731Z","iopub.execute_input":"2025-12-04T17:19:22.347975Z","iopub.status.idle":"2025-12-04T17:19:22.352606Z","shell.execute_reply.started":"2025-12-04T17:19:22.347958Z","shell.execute_reply":"2025-12-04T17:19:22.351932Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lung_diseases = { \"Atelectasis\" : 0,\n                 \"Consolidation\" : 1, \n                 \"ILD\" : 2, \n                 \"Infiltration\" : 3, \n                 \"Lung Opacity\" : 4, \n                 \"Nodule/Mass\" : 5, \n                 \"Pleural effusion\" : 6, \n                 \"Pleural thickening\" : 7, \n                 \"Pneumothorax\" : 8, \n                 \"Pulmonary fibrosis\" : 9 \n                } \ntrain512_df = p512_csv[p512_csv['class_name'].isin(lung_diseases)].reset_index(drop=True) \n# Tạo cột class_id mới theo dict\ntrain512_df[\"class_id\"] = train512_df[\"class_name\"].map(lung_diseases)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:22.353308Z","iopub.execute_input":"2025-12-04T17:19:22.35358Z","iopub.status.idle":"2025-12-04T17:19:22.392282Z","shell.execute_reply.started":"2025-12-04T17:19:22.353559Z","shell.execute_reply":"2025-12-04T17:19:22.391587Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_df['class_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:22.393039Z","iopub.execute_input":"2025-12-04T17:19:22.393486Z","iopub.status.idle":"2025-12-04T17:19:22.40139Z","shell.execute_reply.started":"2025-12-04T17:19:22.393454Z","shell.execute_reply":"2025-12-04T17:19:22.400739Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ***Segmentation model***","metadata":{}},{"cell_type":"code","source":"import torch\n\n# Định nghĩa lại Block và UNet\nclass Block(torch.nn.Module):\n    def __init__(self, in_channels, mid_channel, out_channels, batch_norm=False):\n        super().__init__()\n        self.conv1 = torch.nn.Conv2d(in_channels, mid_channel, kernel_size=3, padding=1)\n        self.conv2 = torch.nn.Conv2d(mid_channel, out_channels, kernel_size=3, padding=1)\n        self.batch_norm = batch_norm\n        if batch_norm:\n            self.bn1 = torch.nn.BatchNorm2d(mid_channel)\n            self.bn2 = torch.nn.BatchNorm2d(out_channels)\n    def forward(self, x):\n        x = self.conv1(x)\n        if self.batch_norm:\n            x = self.bn1(x)\n        x = torch.nn.functional.relu(x, inplace=True)\n        x = self.conv2(x)\n        if self.batch_norm:\n            x = self.bn2(x)\n        out = torch.nn.functional.relu(x, inplace=True)\n        return out\n\nclass UNet(torch.nn.Module):\n    def up(self, x, size):\n        return torch.nn.functional.interpolate(x, size=size, mode=self.upscale_mode)\n    def down(self, x):\n        return torch.nn.functional.max_pool2d(x, kernel_size=2)\n    def __init__(self, in_channels, out_channels, batch_norm=False, upscale_mode=\"nearest\"):\n        super().__init__()\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.batch_norm = batch_norm\n        self.upscale_mode = upscale_mode\n        self.enc1 = Block(in_channels, 64, 64, batch_norm)\n        self.enc2 = Block(64, 128, 128, batch_norm)\n        self.enc3 = Block(128, 256, 256, batch_norm)\n        self.enc4 = Block(256, 512, 512, batch_norm)\n        self.center = Block(512, 1024, 512, batch_norm)\n        self.dec4 = Block(1024, 512, 256, batch_norm)\n        self.dec3 = Block(512, 256, 128, batch_norm)\n        self.dec2 = Block(256, 128, 64, batch_norm)\n        self.dec1 = Block(128, 64, 64, batch_norm)\n        self.out = torch.nn.Conv2d(64, out_channels, kernel_size=1)\n    def forward(self, x):\n        enc1 = self.enc1(x)\n        enc2 = self.enc2(self.down(enc1))\n        enc3 = self.enc3(self.down(enc2))\n        enc4 = self.enc4(self.down(enc3))\n        center = self.center(self.down(enc4))\n        dec4 = self.dec4(torch.cat([self.up(center, enc4.size()[-2:]), enc4], 1))\n        dec3 = self.dec3(torch.cat([self.up(dec4, enc3.size()[-2:]), enc3], 1))\n        dec2 = self.dec2(torch.cat([self.up(dec3, enc2.size()[-2:]), enc2], 1))\n        dec1 = self.dec1(torch.cat([self.up(dec2, enc1.size()[-2:]), enc1], 1))\n        out = self.out(dec1)\n        return out","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:22.402149Z","iopub.execute_input":"2025-12-04T17:19:22.402414Z","iopub.status.idle":"2025-12-04T17:19:22.41771Z","shell.execute_reply.started":"2025-12-04T17:19:22.402386Z","shell.execute_reply":"2025-12-04T17:19:22.417205Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load pre-trained weights UNet đúng kiến trúc và file weight đã lưu\nsegment_model = UNet(in_channels=1, out_channels=2, batch_norm=False)\nsegment_model.load_state_dict(torch.load('/kaggle/input/yolo-12-small-weight/unet-2v.pt', map_location='cpu'))\nsegment_model.eval()  # Đặt mô hình về chế độ đánh giá","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:22.418483Z","iopub.execute_input":"2025-12-04T17:19:22.418739Z","iopub.status.idle":"2025-12-04T17:19:23.761692Z","shell.execute_reply.started":"2025-12-04T17:19:22.418717Z","shell.execute_reply":"2025-12-04T17:19:23.761025Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### *Ảnh gốc*","metadata":{}},{"cell_type":"code","source":"# Plot một vài ảnh để kiểm tra 512x512\nimgs512 = list(set(train512_df['image_path']))\ntest512_list = imgs512[:4]\n\nimg512_list = [cv2.imread(img) for img in test512_list]\nplot_img(img512_list)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:23.762557Z","iopub.execute_input":"2025-12-04T17:19:23.762847Z","iopub.status.idle":"2025-12-04T17:19:24.789833Z","shell.execute_reply.started":"2025-12-04T17:19:23.762819Z","shell.execute_reply":"2025-12-04T17:19:24.789094Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Lưu ảnh đã được xử lý*","metadata":{}},{"cell_type":"code","source":"# # Lấy list path của mỗi ảnh 512x512 và 256x256\nimg512_path_list = list(set(train512_df['image_path'].tolist()))\n# img256_path_list = list(set(train256_df['image_path'].tolist()))\nprint(\"512x512: \", len(img512_path_list))\n# print(\"256x256: \",len(img256_path_list))","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:24.790683Z","iopub.execute_input":"2025-12-04T17:19:24.790921Z","iopub.status.idle":"2025-12-04T17:19:24.796599Z","shell.execute_reply.started":"2025-12-04T17:19:24.790902Z","shell.execute_reply":"2025-12-04T17:19:24.795848Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Lấy size gốc các ảnh được dùng trong training*","metadata":{}},{"cell_type":"code","source":"img_size_path = '/kaggle/input/vinbigdata/train_meta.csv'\nimg_size_df = pd.read_csv(img_size_path)\nimg_size_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:24.797468Z","iopub.execute_input":"2025-12-04T17:19:24.797677Z","iopub.status.idle":"2025-12-04T17:19:24.836026Z","shell.execute_reply.started":"2025-12-04T17:19:24.797661Z","shell.execute_reply":"2025-12-04T17:19:24.835411Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_merge_df = train512_df.merge(img_size_df, on='image_id', how='left')\ntrain512_merge_df = train512_merge_df[['image_id', 'image_path', 'class_name', 'class_id', 'rad_id', 'x_min', 'y_min','x_max','y_max', 'dim0', 'dim1']]\ntrain512_merge_df = train512_merge_df.rename(columns = {\n    'dim0': 'height',\n    'dim1': 'width'\n})\ntrain512_merge_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:24.836704Z","iopub.execute_input":"2025-12-04T17:19:24.836988Z","iopub.status.idle":"2025-12-04T17:19:24.868078Z","shell.execute_reply.started":"2025-12-04T17:19:24.836956Z","shell.execute_reply":"2025-12-04T17:19:24.867343Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Resize lại bounding box cho đúng với ảnh resize*","metadata":{}},{"cell_type":"code","source":"# train1024_merge_df[['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']] = train1024_merge_df.progress_apply(resize_boxes, axis=1, result_type='expand', args=(1024, 1024))\ntrain512_merge_df[['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']] = train512_merge_df.progress_apply(resize_boxes, axis=1, result_type='expand', args=(512, 512))\n# train256_merge_df[['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']] = train256_merge_df.progress_apply(resize_boxes, axis=1, result_type='expand', args=(256, 256))","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:24.868923Z","iopub.execute_input":"2025-12-04T17:19:24.869353Z","iopub.status.idle":"2025-12-04T17:19:27.564343Z","shell.execute_reply.started":"2025-12-04T17:19:24.869331Z","shell.execute_reply":"2025-12-04T17:19:27.563608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ***Visualize ảnh XRAY với bounding box đã được resize***","metadata":{}},{"cell_type":"code","source":"import random\nnum_images = 9\n\nlist_images = train512_merge_df['image_id'].tolist()\nsample_images = random.sample(list(list_images), num_images)\nsample_images","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:27.565084Z","iopub.execute_input":"2025-12-04T17:19:27.565334Z","iopub.status.idle":"2025-12-04T17:19:27.571601Z","shell.execute_reply.started":"2025-12-04T17:19:27.565307Z","shell.execute_reply":"2025-12-04T17:19:27.570718Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dicom_dir = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train\"     # thư mục chứa DICOM\nsample_images_dicom = [i + \".dicom\" for i in sample_images]\n\nfull_paths = [os.path.join(dicom_dir, f) for f in sample_images_dicom]","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:27.572444Z","iopub.execute_input":"2025-12-04T17:19:27.572775Z","iopub.status.idle":"2025-12-04T17:19:27.588952Z","shell.execute_reply.started":"2025-12-04T17:19:27.572753Z","shell.execute_reply":"2025-12-04T17:19:27.588434Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"list_dicom = [dicom2array(i) for i in full_paths]\nlen(list_dicom)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:27.589635Z","iopub.execute_input":"2025-12-04T17:19:27.589908Z","iopub.status.idle":"2025-12-04T17:19:33.543549Z","shell.execute_reply.started":"2025-12-04T17:19:27.589885Z","shell.execute_reply":"2025-12-04T17:19:33.542839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"filenames = [os.path.basename(p).split('.')[0] for p in full_paths]\nprint(filenames)\nfiltered = s_csv[s_csv[\"image_id\"].isin(filenames)]","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:33.544348Z","iopub.execute_input":"2025-12-04T17:19:33.544578Z","iopub.status.idle":"2025-12-04T17:19:33.5567Z","shell.execute_reply.started":"2025-12-04T17:19:33.544559Z","shell.execute_reply":"2025-12-04T17:19:33.55606Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport os\n\ndef show_multiple_dicoms_with_boxes(dicom_paths, df, cols=3):\n    rows = math.ceil(len(dicom_paths) / cols)\n\n    fig, axes = plt.subplots(rows, cols, figsize=(cols * 6, rows * 6))\n\n    # Nếu chỉ có 1 hàng -> giữ axes dạng list\n    if rows == 1:\n        axes = [axes]\n\n    # Flatten axes để dễ xử lý\n    axes = np.array(axes).reshape(-1)\n\n    for idx, dicom_path in enumerate(dicom_paths):\n        ax = axes[idx]\n        filename = os.path.basename(dicom_path).split('.')[0]\n\n        # Lọc bbox\n        boxes = df[df[\"image_id\"] == filename]\n        # Load ảnh\n        img = dicom2array(dicom_path)\n\n        ax.imshow(img, cmap=\"gray\")\n        ax.set_title(filename)\n        ax.axis(\"off\")\n\n        # Vẽ bbox\n        for _, row in boxes.iterrows():\n            x1, y1, x2, y2 = row[\"x_min\"], row[\"y_min\"], row[\"x_max\"], row[\"y_max\"]\n            class_name = row.get(\"class_name\", \"\")\n\n            rect = patches.Rectangle(\n                (x1, y1), \n                x2 - x1, \n                y2 - y1, \n                linewidth=2, \n                edgecolor='red', \n                facecolor='none'\n            )\n            ax.add_patch(rect)\n            ax.text(x1, y1 - 5, class_name, color=\"yellow\", fontsize=10, backgroundcolor=\"black\")\n\n    # Nếu số ảnh không chia hết cho cols → tắt các ô rỗng\n    for j in range(idx + 1, len(axes)):\n        axes[j].axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:33.55744Z","iopub.execute_input":"2025-12-04T17:19:33.557613Z","iopub.status.idle":"2025-12-04T17:19:33.574377Z","shell.execute_reply.started":"2025-12-04T17:19:33.557599Z","shell.execute_reply":"2025-12-04T17:19:33.573795Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_multiple_dicoms_with_boxes(full_paths, train512_merge_df, cols=3)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:33.575045Z","iopub.execute_input":"2025-12-04T17:19:33.575288Z","iopub.status.idle":"2025-12-04T17:19:43.502927Z","shell.execute_reply.started":"2025-12-04T17:19:33.575271Z","shell.execute_reply":"2025-12-04T17:19:43.502103Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_folder = \"/kaggle/input/vinbigdata/train\"\n\nfig, axes = plt.subplots(3, 3, figsize=(15, 15))\naxes = axes.flatten()\n\nfor ax, img_id in zip(axes, sample_images):\n    img_path = os.path.join(image_folder, f'{img_id}.png')\n\n    if not os.path.exists(img_path):\n        print(\"Không tìm thấy ảnh trong thư mục\")\n        continue\n\n    image = Image.open(img_path).convert(\"RGB\")\n    ax.imshow(image)\n    ax.axis(\"off\")\n    ax.set_title(img_id[:10])\n\n    org_width, org_height = image.size\n\n    bboxes = train512_merge_df[train512_merge_df['image_id'] == img_id]\n\n    for _, row in bboxes.iterrows():\n        if pd.notna(row['x_min']):\n            x_min, x_max, y_min, y_max = row['x_min_new'],row['x_max_new'],row['y_min_new'],row['y_max_new']\n            width, height = x_max - x_min, y_max - y_min\n\n            rect = patches.Rectangle(\n                (x_min, y_min), width, height,\n                linewidth=2, edgecolor=\"red\", facecolor=\"none\"\n            )\n\n            ax.add_patch(rect)\n            ax.text(x_min, y_min - 5, row[\"class_name\"], color=\"yellow\", fontsize=8,\n                    bbox=dict(facecolor=\"black\", alpha=0.5, pad=1))\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:43.504166Z","iopub.execute_input":"2025-12-04T17:19:43.504519Z","iopub.status.idle":"2025-12-04T17:19:45.567181Z","shell.execute_reply.started":"2025-12-04T17:19:43.504485Z","shell.execute_reply":"2025-12-04T17:19:45.566297Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***Crop và lưu ảnh crop với bounding box***","metadata":{}},{"cell_type":"markdown","source":"### ***Merge overlapping bounding boxes trước khi crop***","metadata":{}},{"cell_type":"markdown","source":"**Tại sao cần merge bounding boxes?**\n\nMedical imaging datasets thường có nhiều radiologists annotate cùng 1 ảnh → overlapping boxes cho cùng 1 lesion.\n\n**Strategy:**\n1. Group boxes theo `image_id` và `class_name`\n2. Tính IoU (Intersection over Union) giữa mọi cặp boxes\n3. Nếu IoU > 0.3 → merge thành 1 box lớn hơn bao quanh cả 2\n4. Lặp lại cho đến khi không còn boxes nào merge được\n\n**Benefits:**\n- ✅ Giảm redundancy (nhiều boxes cho 1 lesion)\n- ✅ Cleaner training data\n- ✅ Better bbox quality\n- ✅ Faster training (ít boxes hơn)","metadata":{}},{"cell_type":"code","source":"# 🔧 MERGE OVERLAPPING BOUNDING BOXES\n# Merge các bbox cùng class có IoU > 0.3 để giảm redundancy\n\ndef calculate_iou(box1, box2):\n    \"\"\"\n    Tính IoU (Intersection over Union) giữa 2 bounding boxes.\n    \n    Args:\n        box1, box2: dict với keys ['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']\n    \n    Returns:\n        float: IoU score (0-1)\n    \"\"\"\n    x1_min, y1_min = box1['x_min_new'], box1['y_min_new']\n    x1_max, y1_max = box1['x_max_new'], box1['y_max_new']\n    \n    x2_min, y2_min = box2['x_min_new'], box2['y_min_new']\n    x2_max, y2_max = box2['x_max_new'], box2['y_max_new']\n    \n    # Tính intersection\n    inter_x_min = max(x1_min, x2_min)\n    inter_y_min = max(y1_min, y2_min)\n    inter_x_max = min(x1_max, x2_max)\n    inter_y_max = min(y1_max, y2_max)\n    \n    if inter_x_min >= inter_x_max or inter_y_min >= inter_y_max:\n        return 0.0\n    \n    inter_area = (inter_x_max - inter_x_min) * (inter_y_max - inter_y_min)\n    \n    # Tính union\n    box1_area = (x1_max - x1_min) * (y1_max - y1_min)\n    box2_area = (x2_max - x2_min) * (y2_max - y2_min)\n    union_area = box1_area + box2_area - inter_area\n    \n    if union_area == 0:\n        return 0.0\n    \n    iou = inter_area / union_area\n    return iou\n\n\ndef merge_boxes(box1, box2):\n    \"\"\"\n    Merge 2 bounding boxes thành 1 box bao quanh cả 2.\n    \n    Args:\n        box1, box2: dict với bbox coordinates\n    \n    Returns:\n        dict: Merged bounding box\n    \"\"\"\n    merged = box1.copy()\n    merged['x_min_new'] = min(box1['x_min_new'], box2['x_min_new'])\n    merged['y_min_new'] = min(box1['y_min_new'], box2['y_min_new'])\n    merged['x_max_new'] = max(box1['x_max_new'], box2['x_max_new'])\n    merged['y_max_new'] = max(box1['y_max_new'], box2['y_max_new'])\n    return merged\n\n\ndef merge_overlapping_boxes(df, iou_threshold=0.3):\n    \"\"\"\n    Merge các bounding boxes cùng class có IoU > threshold.\n    \n    Args:\n        df: DataFrame chứa bounding boxes với columns ['image_id', 'class_name', 'x_min_new', ...]\n        iou_threshold: IoU threshold để merge (default 0.3)\n    \n    Returns:\n        DataFrame: DataFrame sau khi merge\n    \"\"\"\n    if df.empty:\n        return df\n    \n    print(f\"📊 Before merging: {len(df)} bounding boxes\")\n    \n    # Columns cần giữ lại\n    required_cols = ['image_id', 'image_path', 'class_name', 'class_id', 'rad_id', \n                     'x_min', 'y_min', 'x_max', 'y_max', 'height', 'width',\n                     'x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']\n    \n    # Filter columns tồn tại\n    keep_cols = [col for col in required_cols if col in df.columns]\n    \n    merged_records = []\n    processed_indices = set()\n    \n    # Group by image_id và class_name\n    for (img_id, class_name), group in tqdm(df.groupby(['image_id', 'class_name']), \n                                             desc=\"Merging overlapping boxes\"):\n        group_indices = group.index.tolist()\n        \n        # Skip nếu chỉ có 1 box\n        if len(group_indices) == 1:\n            merged_records.append(group.iloc[0].to_dict())\n            processed_indices.update(group_indices)\n            continue\n        \n        # Convert to list of dicts cho dễ xử lý\n        boxes = group.to_dict('records')\n        box_indices = group_indices.copy()\n        merged_flags = [False] * len(boxes)\n        \n        # Merge boxes có IoU > threshold\n        for i in range(len(boxes)):\n            if merged_flags[i]:\n                continue\n            \n            current_box = boxes[i]\n            boxes_to_merge = [i]\n            \n            # Tìm tất cả boxes overlap với current box\n            for j in range(i + 1, len(boxes)):\n                if merged_flags[j]:\n                    continue\n                \n                iou = calculate_iou(current_box, boxes[j])\n                \n                if iou > iou_threshold:\n                    boxes_to_merge.append(j)\n                    merged_flags[j] = True\n            \n            # Merge tất cả boxes found\n            if len(boxes_to_merge) > 1:\n                # Merge iteratively\n                merged_box = boxes[boxes_to_merge[0]]\n                for idx in boxes_to_merge[1:]:\n                    merged_box = merge_boxes(merged_box, boxes[idx])\n                merged_records.append(merged_box)\n                processed_indices.update([box_indices[idx] for idx in boxes_to_merge])\n            else:\n                # Không merge, giữ nguyên\n                merged_records.append(current_box)\n                processed_indices.add(box_indices[i])\n    \n    # Tạo DataFrame mới\n    merged_df = pd.DataFrame(merged_records)\n    \n    # Đảm bảo columns order\n    final_cols = [col for col in keep_cols if col in merged_df.columns]\n    merged_df = merged_df[final_cols]\n    \n    print(f\"✅ After merging: {len(merged_df)} bounding boxes\")\n    print(f\"📉 Reduced: {len(df) - len(merged_df)} boxes ({(len(df) - len(merged_df)) / len(df) * 100:.1f}%)\")\n    \n    return merged_df\n\n\n# 🚀 Apply merge to train512_merge_df\nprint(\"=\"*80)\nprint(\"🔧 MERGING OVERLAPPING BOUNDING BOXES\")\nprint(\"=\"*80)\nprint(f\"IoU Threshold: 0.3\")\nprint(f\"Strategy: Merge boxes with same class and IoU > 0.3\")\nprint()\n\ntrain512_merge_df_original = train512_merge_df.copy()\ntrain512_merge_df = merge_overlapping_boxes(train512_merge_df, iou_threshold=0.3)\n\nprint(\"\\n📊 Merge Statistics by Class:\")\nfor class_name in sorted(train512_merge_df['class_name'].unique()):\n    original_count = len(train512_merge_df_original[train512_merge_df_original['class_name'] == class_name])\n    merged_count = len(train512_merge_df[train512_merge_df['class_name'] == class_name])\n    reduction = original_count - merged_count\n    print(f\"  {class_name:25s}: {original_count:5d} → {merged_count:5d} (-{reduction:4d}, {reduction/original_count*100:5.1f}%)\")\n\nprint(\"\\n💡 Benefits of merging:\")\nprint(\"  ✓ Reduced redundant overlapping boxes\")\nprint(\"  ✓ Cleaner training data\")\nprint(\"  ✓ Better bbox quality\")\nprint(\"  ✓ Faster training (fewer boxes)\")\nprint(\"=\"*80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:45.568148Z","iopub.execute_input":"2025-12-04T17:19:45.568392Z","iopub.status.idle":"2025-12-04T17:19:50.389906Z","shell.execute_reply.started":"2025-12-04T17:19:45.568374Z","shell.execute_reply":"2025-12-04T17:19:50.389289Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***Visualize kết quả merge***","metadata":{}},{"cell_type":"code","source":"# 📊 Visualize Before/After Merge cho một vài sample images\n# Chọn images có nhiều boxes để thấy rõ effect\n\ndef visualize_merge_comparison(image_id, df_before, df_after, image_folder, figsize=(16, 8)):\n    \"\"\"\n    Hiển thị comparison giữa bboxes trước và sau merge.\n    \"\"\"\n    fig, axes = plt.subplots(1, 2, figsize=figsize)\n    \n    # Load image\n    img_path = os.path.join(image_folder, f'{image_id}.png')\n    if not os.path.exists(img_path):\n        print(f\"⚠️ Image not found: {img_path}\")\n        return\n    \n    image = Image.open(img_path).convert(\"RGB\")\n    \n    # === BEFORE MERGE ===\n    ax_before = axes[0]\n    ax_before.imshow(image)\n    ax_before.set_title(f\"Before Merge: {image_id[:15]}\", fontsize=14, fontweight='bold')\n    ax_before.axis(\"off\")\n    \n    boxes_before = df_before[df_before['image_id'] == image_id]\n    for _, row in boxes_before.iterrows():\n        if pd.notna(row['x_min_new']):\n            x_min, x_max = row['x_min_new'], row['x_max_new']\n            y_min, y_max = row['y_min_new'], row['y_max_new']\n            width, height = x_max - x_min, y_max - y_min\n            \n            rect = patches.Rectangle(\n                (x_min, y_min), width, height,\n                linewidth=2, edgecolor=\"red\", facecolor=\"none\", alpha=0.7\n            )\n            ax_before.add_patch(rect)\n            ax_before.text(x_min, y_min - 5, row[\"class_name\"], \n                          color=\"yellow\", fontsize=8, fontweight='bold',\n                          bbox=dict(facecolor=\"red\", alpha=0.7, pad=2))\n    \n    # === AFTER MERGE ===\n    ax_after = axes[1]\n    ax_after.imshow(image)\n    ax_after.set_title(f\"After Merge: {image_id[:15]}\", fontsize=14, fontweight='bold')\n    ax_after.axis(\"off\")\n    \n    boxes_after = df_after[df_after['image_id'] == image_id]\n    for _, row in boxes_after.iterrows():\n        if pd.notna(row['x_min_new']):\n            x_min, x_max = row['x_min_new'], row['x_max_new']\n            y_min, y_max = row['y_min_new'], row['y_max_new']\n            width, height = x_max - x_min, y_max - y_min\n            \n            rect = patches.Rectangle(\n                (x_min, y_min), width, height,\n                linewidth=2, edgecolor=\"lime\", facecolor=\"none\", alpha=0.7\n            )\n            ax_after.add_patch(rect)\n            ax_after.text(x_min, y_min - 5, row[\"class_name\"], \n                         color=\"yellow\", fontsize=8, fontweight='bold',\n                         bbox=dict(facecolor=\"green\", alpha=0.7, pad=2))\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # Print statistics\n    print(f\"\\n{'='*60}\")\n    print(f\"Image ID: {image_id}\")\n    print(f\"Boxes before merge: {len(boxes_before)}\")\n    print(f\"Boxes after merge:  {len(boxes_after)}\")\n    print(f\"Reduction:          {len(boxes_before) - len(boxes_after)} boxes\")\n    print(f\"{'='*60}\\n\")\n\n\n# Find images với nhiều boxes để visualize\nbox_counts = train512_merge_df_original.groupby('image_id').size()\nimages_with_many_boxes = box_counts[box_counts >= 3].sort_values(ascending=False).head(5).index.tolist()\n\nprint(\"🔍 Visualizing merge results for images with multiple boxes...\")\nprint(f\"Selected {min(3, len(images_with_many_boxes))} images to visualize\\n\")\n\nfor img_id in images_with_many_boxes[:3]:  # Visualize top 3\n    visualize_merge_comparison(\n        image_id=img_id,\n        df_before=train512_merge_df_original,\n        df_after=train512_merge_df,\n        image_folder=image_folder\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:50.394299Z","iopub.execute_input":"2025-12-04T17:19:50.394566Z","iopub.status.idle":"2025-12-04T17:19:53.162552Z","shell.execute_reply.started":"2025-12-04T17:19:50.394546Z","shell.execute_reply":"2025-12-04T17:19:53.161906Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***Tiếp tục với cropping (sử dụng merged boxes)***","metadata":{}},{"cell_type":"code","source":"image_folder = '/kaggle/input/vinbigdata/train'\noutput_folder = '/kaggle/working/crop_images'\noutput_csv_path = '/kaggle/working/crop_bboxes.csv'\n\nos.makedirs(\"/kaggle/working/crop_images\", exist_ok=True)\n\ntransform = transforms.Compose([\n    # transforms.Resize((256, 256)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.5], [0.5])\n])\nnew_df = batch_crop_images(\n    image_folder = image_folder,\n    bbox_df = train512_merge_df,\n    model = segment_model,\n    output_folder = output_folder,\n    output_csv_path = output_csv_path,\n    transform = transform,\n    device = device\n)\nprint(\"Crop hoàn tất!!!\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.220236Z","iopub.status.idle":"2025-12-04T17:19:53.220491Z","shell.execute_reply.started":"2025-12-04T17:19:53.220381Z","shell.execute_reply":"2025-12-04T17:19:53.220392Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"crop_df = pd.read_csv('/kaggle/working/crop_bboxes.csv')\ncrop_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.221944Z","iopub.status.idle":"2025-12-04T17:19:53.222274Z","shell.execute_reply.started":"2025-12-04T17:19:53.2221Z","shell.execute_reply":"2025-12-04T17:19:53.222115Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for img_id in sample_images:\n    visualize_original_vs_crop(\n        image_id=img_id,\n        train_df=train512_merge_df,\n        crop_df=crop_df,\n        image_folder=image_folder\n    )","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.224063Z","iopub.status.idle":"2025-12-04T17:19:53.224343Z","shell.execute_reply.started":"2025-12-04T17:19:53.224223Z","shell.execute_reply":"2025-12-04T17:19:53.224238Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Lấy tỉ lệ tương đối của bounding box*","metadata":{}},{"cell_type":"code","source":"# train1024_yolo = normalize_bbox(train1024_merge_df)\ntrain512_yolo = normalize_bbox(train512_merge_df)\n# train256_yolo = normalize_bbox(train256_merge_df)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.22531Z","iopub.status.idle":"2025-12-04T17:19:53.225532Z","shell.execute_reply.started":"2025-12-04T17:19:53.225425Z","shell.execute_reply":"2025-12-04T17:19:53.225435Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_yolo.to_csv('train512.csv', index=False, encoding='utf-8-sig')","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.22811Z","iopub.status.idle":"2025-12-04T17:19:53.2284Z","shell.execute_reply.started":"2025-12-04T17:19:53.228288Z","shell.execute_reply":"2025-12-04T17:19:53.228299Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_yolo.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.22913Z","iopub.status.idle":"2025-12-04T17:19:53.229362Z","shell.execute_reply.started":"2025-12-04T17:19:53.229255Z","shell.execute_reply":"2025-12-04T17:19:53.229264Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_yolo.to_csv(\"/kaggle/working/bboxes.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.230981Z","iopub.status.idle":"2025-12-04T17:19:53.231251Z","shell.execute_reply.started":"2025-12-04T17:19:53.231138Z","shell.execute_reply":"2025-12-04T17:19:53.231153Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(glob.glob(output_folder + '/*.png'))","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.231917Z","iopub.status.idle":"2025-12-04T17:19:53.232227Z","shell.execute_reply.started":"2025-12-04T17:19:53.232068Z","shell.execute_reply":"2025-12-04T17:19:53.232083Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"crop_df = pd.read_csv('/kaggle/working/crop_bboxes.csv')\ncrop_df['class_name'] = crop_df['class_name'].astype(str).str.strip()\ncrop_df = crop_df[crop_df['class_name'] != ''].copy()\ncrop_df['image_id'] = crop_df['image_id'].astype(str)\ncrop_df['class_id'] = crop_df['class_name'].map(lung_diseases)\n\nmissing_classes = crop_df['class_id'].isna()\nif missing_classes.any():\n    missing_labels = crop_df.loc[missing_classes, 'class_name'].unique()\n    raise ValueError(f\"Unknown class labels found in crop_df: {missing_labels}\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.233334Z","iopub.status.idle":"2025-12-04T17:19:53.233625Z","shell.execute_reply.started":"2025-12-04T17:19:53.233476Z","shell.execute_reply":"2025-12-04T17:19:53.233489Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def write_yolo_labels_from_crops(df: pd.DataFrame, label_dir: str, class_mapping: dict[str, int] | None = None):\n    \"\"\"Write YOLO txt files that stay aligned with the cropped images.\"\"\"\n    os.makedirs(label_dir, exist_ok=True)\n\n    df = df.copy()\n    if 'class_id' not in df.columns:\n        if class_mapping is None:\n            raise ValueError(\"Provide 'class_mapping' when 'class_id' column is missing.\")\n        df['class_id'] = df['class_name'].map(class_mapping)\n\n    df = df.dropna(subset=['x_min', 'y_min', 'x_max', 'y_max'])\n    df['image_id'] = df['image_id'].astype(str)\n\n    for image_id, group in tqdm(df.groupby('image_id'), desc='Writing YOLO labels'):\n        crop_path = group['crop_path'].iloc[0]\n        with Image.open(crop_path) as img:\n            width, height = img.size\n        if width == 0 or height == 0:\n            continue  # skip corrupt crops\n\n        lines: list[str] = []\n        for _, row in group.iterrows():\n            x_center = ((row['x_min'] + row['x_max']) / 2) / width\n            y_center = ((row['y_min'] + row['y_max']) / 2) / height\n            bbox_width = (row['x_max'] - row['x_min']) / width\n            bbox_height = (row['y_max'] - row['y_min']) / height\n            lines.append(\n                f\"{int(row['class_id'])} {x_center:.6f} {y_center:.6f} {bbox_width:.6f} {bbox_height:.6f}\"\n            )\n\n        label_path = os.path.join(label_dir, f\"{image_id}.txt\")\n        with open(label_path, 'w', encoding='utf-8') as f:\n            f.write('\\n'.join(lines))\n","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.234674Z","iopub.status.idle":"2025-12-04T17:19:53.234927Z","shell.execute_reply.started":"2025-12-04T17:19:53.234786Z","shell.execute_reply":"2025-12-04T17:19:53.234795Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"crop_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.236965Z","iopub.status.idle":"2025-12-04T17:19:53.23724Z","shell.execute_reply.started":"2025-12-04T17:19:53.237107Z","shell.execute_reply":"2025-12-04T17:19:53.237132Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(crop_df['image_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.238051Z","iopub.status.idle":"2025-12-04T17:19:53.238353Z","shell.execute_reply.started":"2025-12-04T17:19:53.238204Z","shell.execute_reply":"2025-12-04T17:19:53.238217Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Tạo file lưu bounding box*","metadata":{}},{"cell_type":"code","source":"p512_img_org_list = list(set(crop_df['crop_path']))","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.239092Z","iopub.status.idle":"2025-12-04T17:19:53.23931Z","shell.execute_reply.started":"2025-12-04T17:19:53.239207Z","shell.execute_reply":"2025-12-04T17:19:53.239216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_label_dir = \"/kaggle/working/chest_detection/labels\"\nif os.path.exists(img_label_dir):\n    shutil.rmtree(img_label_dir)\nos.makedirs(img_label_dir, exist_ok=True)\n\nwrite_yolo_labels_from_crops(crop_df, img_label_dir, lung_diseases)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.24076Z","iopub.status.idle":"2025-12-04T17:19:53.241072Z","shell.execute_reply.started":"2025-12-04T17:19:53.240914Z","shell.execute_reply":"2025-12-04T17:19:53.24093Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"crop_img_id = sorted(crop_df['image_id'].unique().tolist())","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.241831Z","iopub.status.idle":"2025-12-04T17:19:53.242127Z","shell.execute_reply.started":"2025-12-04T17:19:53.242007Z","shell.execute_reply":"2025-12-04T17:19:53.242016Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***✅ FIX: Apply CLAHE preprocessing (moved here after crop_img_id is defined)***","metadata":{}},{"cell_type":"code","source":"# 🔧 FIX: CLAHE được di chuyển xuống đây sau khi crop_img_id đã được định nghĩa\n\ndef apply_clahe_to_images_fixed(input_folder, output_folder, clip_limit=2.0, tile_grid_size=(8, 8)):\n    \"\"\"\n    Apply CLAHE preprocessing to enhance X-ray image contrast.\n    Có thể tăng mAP lên 3-5% cho medical imaging.\n    \"\"\"\n    os.makedirs(output_folder, exist_ok=True)\n    \n    # Create CLAHE object\n    clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=tile_grid_size)\n    \n    image_files = glob.glob(os.path.join(input_folder, '*.png'))\n    print(f\"🔄 Applying CLAHE to {len(image_files)} images...\")\n    \n    for img_path in tqdm(image_files, desc=\"CLAHE Processing\"):\n        # Read image\n        img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n        \n        if img is None:\n            print(f\"⚠️ Warning: Could not read {img_path}\")\n            continue\n        \n        # Apply CLAHE\n        clahe_img = clahe.apply(img)\n        \n        # Save enhanced image\n        filename = os.path.basename(img_path)\n        output_path = os.path.join(output_folder, filename)\n        cv2.imwrite(output_path, clahe_img)\n    \n    print(f\"✅ CLAHE preprocessing completed! Images saved to: {output_folder}\")\n\n# Apply CLAHE to cropped images\nclahe_output_folder = '/kaggle/working/crop_images_clahe'\napply_clahe_to_images_fixed(output_folder, clahe_output_folder, clip_limit=2.0, tile_grid_size=(8, 8))\n\n# Visualize comparison (now crop_img_id is already defined)\nif len(crop_img_id) > 0:\n    sample_id = crop_img_id[0]\n    original_path = os.path.join(output_folder, f'{sample_id}.png')\n    clahe_path = os.path.join(clahe_output_folder, f'{sample_id}.png')\n    \n    if os.path.exists(original_path) and os.path.exists(clahe_path):\n        original_img = cv2.imread(original_path, cv2.IMREAD_GRAYSCALE)\n        clahe_img = cv2.imread(clahe_path, cv2.IMREAD_GRAYSCALE)\n        \n        fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n        axes[0].imshow(original_img, cmap='gray')\n        axes[0].set_title('Original Image', fontsize=14, fontweight='bold')\n        axes[0].axis('off')\n        \n        axes[1].imshow(clahe_img, cmap='gray')\n        axes[1].set_title('CLAHE Enhanced', fontsize=14, fontweight='bold')\n        axes[1].axis('off')\n        \n        plt.tight_layout()\n        plt.show()\n    else:\n        print(f\"⚠️ Sample images not found for visualization\")\nelse:\n    print(\"⚠️ crop_img_id is empty\")\n\nprint(\"\\n💡 CLAHE Benefits:\")\nprint(\"  ✓ Enhanced local contrast\")\nprint(\"  ✓ Better edge detection\")\nprint(\"  ✓ Improved small lesion visibility\")\nprint(\"  ✓ Expected mAP gain: +3-5%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.242954Z","iopub.status.idle":"2025-12-04T17:19:53.243266Z","shell.execute_reply.started":"2025-12-04T17:19:53.24311Z","shell.execute_reply":"2025-12-04T17:19:53.243124Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Load file txt lên xem thử*","metadata":{}},{"cell_type":"code","source":"list_label = glob.glob(img_label_dir + \"/*\")\n\nwith open(list_label[0]) as f:\n    file = f.read()\n    print(file)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.244107Z","iopub.status.idle":"2025-12-04T17:19:53.244414Z","shell.execute_reply.started":"2025-12-04T17:19:53.244261Z","shell.execute_reply":"2025-12-04T17:19:53.244275Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***ADVANCED: Apply CLAHE preprocessing để enhance contrast***","metadata":{}},{"cell_type":"code","source":"# 🚀 ADVANCED PREPROCESSING: CLAHE (Contrast Limited Adaptive Histogram Equalization)\n# CLAHE cải thiện contrast local trong X-ray images → tăng mAP đáng kể\n\ndef apply_clahe_to_images(input_folder, output_folder, clip_limit=2.0, tile_grid_size=(8, 8)):\n    \"\"\"\n    Apply CLAHE preprocessing to enhance X-ray image contrast.\n    Có thể tăng mAP lên 3-5% cho medical imaging.\n    \"\"\"\n    os.makedirs(output_folder, exist_ok=True)\n    \n    # Create CLAHE object\n    clahe = cv2.createCLAHE(clipLimit=clip_limit, tileGridSize=tile_grid_size)\n    \n    image_files = glob.glob(os.path.join(input_folder, '*.png'))\n    print(f\"🔄 Applying CLAHE to {len(image_files)} images...\")\n    \n    for img_path in tqdm(image_files, desc=\"CLAHE Processing\"):\n        # Read image\n        img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n        \n        # Apply CLAHE\n        clahe_img = clahe.apply(img)\n        \n        # Save enhanced image\n        filename = os.path.basename(img_path)\n        output_path = os.path.join(output_folder, filename)\n        cv2.imwrite(output_path, clahe_img)\n    \n    print(f\"✅ CLAHE preprocessing completed! Images saved to: {output_folder}\")\n\n# Apply CLAHE to cropped images\nclahe_output_folder = '/kaggle/working/crop_images_clahe'\napply_clahe_to_images(output_folder, clahe_output_folder, clip_limit=2.0, tile_grid_size=(8, 8))\n\n# Visualize comparison\nsample_id = crop_img_id[0]\noriginal_img = cv2.imread(os.path.join(output_folder, f'{sample_id}.png'), cv2.IMREAD_GRAYSCALE)\nclahe_img = cv2.imread(os.path.join(clahe_output_folder, f'{sample_id}.png'), cv2.IMREAD_GRAYSCALE)\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 6))\naxes[0].imshow(original_img, cmap='gray')\naxes[0].set_title('Original Image', fontsize=14, fontweight='bold')\naxes[0].axis('off')\n\naxes[1].imshow(clahe_img, cmap='gray')\naxes[1].set_title('CLAHE Enhanced', fontsize=14, fontweight='bold')\naxes[1].axis('off')\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n💡 CLAHE Benefits:\")\nprint(\"  ✓ Enhanced local contrast\")\nprint(\"  ✓ Better edge detection\")\nprint(\"  ✓ Improved small lesion visibility\")\nprint(\"  ✓ Expected mAP gain: +3-5%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.245649Z","iopub.status.idle":"2025-12-04T17:19:53.246024Z","shell.execute_reply.started":"2025-12-04T17:19:53.245798Z","shell.execute_reply":"2025-12-04T17:19:53.245813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(list_label)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.248753Z","iopub.status.idle":"2025-12-04T17:19:53.249068Z","shell.execute_reply.started":"2025-12-04T17:19:53.248887Z","shell.execute_reply":"2025-12-04T17:19:53.248902Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_order = sorted(lung_diseases.items(), key=lambda item: item[1])\nclass_dict = {idx: name for name, idx in class_order}\nclass_names_ordered = [name for name, _ in class_order]\nclass_dict","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.25034Z","iopub.status.idle":"2025-12-04T17:19:53.250742Z","shell.execute_reply.started":"2025-12-04T17:19:53.250566Z","shell.execute_reply":"2025-12-04T17:19:53.250582Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"saved_path = \"/kaggle/working/chest_detection/p512x512_imgs\"","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.251516Z","iopub.status.idle":"2025-12-04T17:19:53.251817Z","shell.execute_reply.started":"2025-12-04T17:19:53.25167Z","shell.execute_reply":"2025-12-04T17:19:53.251682Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ***4. Model Implementation***","metadata":{}},{"cell_type":"markdown","source":"## *Chia dữ liệu thành tập train và validate*","metadata":{}},{"cell_type":"code","source":"# ⚠️ DEPRECATED: Simple split - này sẽ bị OVERRIDE bởi Stratified Split bên dưới\n# Keeping for reference only\n\nprint(f\"Total unique images: {len(crop_img_id)}\")\nprint(\"⚠️ Note: Simple split below will be OVERRIDDEN by Stratified Split\")\nprint(\"=\" * 80)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.253357Z","iopub.status.idle":"2025-12-04T17:19:53.253619Z","shell.execute_reply.started":"2025-12-04T17:19:53.253474Z","shell.execute_reply":"2025-12-04T17:19:53.253482Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***ADVANCED: Stratified Split để đảm bảo class balance***","metadata":{}},{"cell_type":"code","source":"# 🎯 STRATIFIED SPLIT: Đảm bảo mỗi class được phân bố đồng đều trong train/val\n# Điều này quan trọng cho imbalanced medical datasets\n\nfrom collections import Counter\n\n# Tạo label cho mỗi image (dominant class)\nimage_labels = {}\nfor image_id in crop_img_id:\n    image_classes = crop_df[crop_df['image_id'] == image_id]['class_name'].tolist()\n    # Lấy class xuất hiện nhiều nhất\n    if image_classes:\n        dominant_class = Counter(image_classes).most_common(1)[0][0]\n        image_labels[image_id] = dominant_class\n\n# Convert to lists for stratification\nimage_ids = list(image_labels.keys())\nlabels = [image_labels[img_id] for img_id in image_ids]\n\n# Stratified split\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\nsplitter = StratifiedShuffleSplit(n_splits=1, test_size=0.15, random_state=42)\ntrain_idx, val_idx = next(splitter.split(image_ids, labels))\n\ntrain_image_ids_stratified = [image_ids[i] for i in train_idx]\nval_image_ids_stratified = [image_ids[i] for i in val_idx]\n\nprint(\"📊 Stratified Split Results:\")\nprint(\"=\"*80)\nprint(f\"Train images: {len(train_image_ids_stratified)}\")\nprint(f\"Val images:   {len(val_image_ids_stratified)}\")\n\n# Verify class distribution\nprint(\"\\n📈 Class Distribution in Train Set:\")\ntrain_labels = [image_labels[img_id] for img_id in train_image_ids_stratified]\ntrain_dist = Counter(train_labels)\nfor class_name, count in sorted(train_dist.items()):\n    print(f\"  {class_name:25s}: {count:4d} ({count/len(train_labels)*100:.1f}%)\")\n\nprint(\"\\n📈 Class Distribution in Val Set:\")\nval_labels = [image_labels[img_id] for img_id in val_image_ids_stratified]\nval_dist = Counter(val_labels)\nfor class_name, count in sorted(val_dist.items()):\n    print(f\"  {class_name:25s}: {count:4d} ({count/len(val_labels)*100:.1f}%)\")\n\nprint(\"\\n✅ Stratified split ensures balanced class representation!\")\n\n# ✅ CRITICAL FIX: Override variables để các cells sau dùng stratified split\ntrain_image_ids = train_image_ids_stratified\nval_image_ids = val_image_ids_stratified\n\nprint(\"\\n🔧 FIXED: Overriding simple split with stratified split\")\nprint(f\"  ✅ train_image_ids now points to stratified split ({len(train_image_ids)} images)\")\nprint(f\"  ✅ val_image_ids now points to stratified split ({len(val_image_ids)} images)\")\nprint(\"=\"*80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.253994Z","iopub.status.idle":"2025-12-04T17:19:53.254292Z","shell.execute_reply.started":"2025-12-04T17:19:53.25411Z","shell.execute_reply":"2025-12-04T17:19:53.254142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Phân tích Class Distribution để xử lý imbalance*","metadata":{}},{"cell_type":"code","source":"# Phân tích class distribution\nclass_distribution = crop_df['class_name'].value_counts()\nprint(\"Class Distribution:\")\nprint(class_distribution)\nprint(\"\\n\" + \"=\"*60)\n\n# Tính class weights để xử lý imbalance (inverse frequency)\ntotal_samples = len(crop_df)\nclass_weights = {}\nfor class_name, count in class_distribution.items():\n    class_id = lung_diseases[class_name]\n    weight = total_samples / (len(lung_diseases) * count)\n    class_weights[class_id] = round(weight, 3)\n\nprint(\"\\nClass Weights (for loss balancing):\")\nfor class_name, class_id in sorted(lung_diseases.items(), key=lambda x: x[1]):\n    print(f\"{class_name:25s} (ID {class_id}): {class_weights[class_id]:.3f}\")\n\n# Visualize distribution\nplt.figure(figsize=(12, 6))\nclass_distribution.plot(kind='bar', color='steelblue')\nplt.title('Class Distribution in Dataset', fontsize=14, fontweight='bold')\nplt.xlabel('Disease Class')\nplt.ylabel('Number of Instances')\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y', alpha=0.3)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.255343Z","iopub.status.idle":"2025-12-04T17:19:53.255568Z","shell.execute_reply.started":"2025-12-04T17:19:53.255465Z","shell.execute_reply":"2025-12-04T17:19:53.255475Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tạo folder chia dữ liệu thành images và labels \nos.makedirs(\"/kaggle/working/dataset/train/images\", exist_ok=True)\nos.makedirs(\"/kaggle/working/dataset/val/images\", exist_ok=True)\nos.makedirs(\"/kaggle/working/dataset/train/labels\", exist_ok=True)\nos.makedirs(\"/kaggle/working/dataset/val/labels\", exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.256773Z","iopub.status.idle":"2025-12-04T17:19:53.257108Z","shell.execute_reply.started":"2025-12-04T17:19:53.256965Z","shell.execute_reply":"2025-12-04T17:19:53.256981Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***Create DataFrames with CLAHE images và stratified split***","metadata":{}},{"cell_type":"code","source":"# ✅ FIXED: Create DataFrames với CLAHE images và stratified split\n# train_image_ids và val_image_ids đã được override bởi stratified version ở cell trước\n\n# Verify variables are correct\nprint(\"🔍 Verifying variables:\")\nprint(f\"  train_image_ids length: {len(train_image_ids)}\")\nprint(f\"  val_image_ids length: {len(val_image_ids)}\")\nprint(f\"  CLAHE folder exists: {os.path.exists(clahe_output_folder)}\")\nprint(f\"  CLAHE images count: {len(glob.glob(os.path.join(clahe_output_folder, '*.png')))}\")\nprint()\n\n# Create DataFrames với CLAHE images\ntrain_labels = [os.path.join(img_label_dir, f\"{img_id}.txt\") for img_id in train_image_ids]\ntrain_images = [os.path.join(clahe_output_folder, f\"{img_id}.png\") for img_id in train_image_ids]\nyolo_df_train = pd.DataFrame({'label_path': train_labels, 'image_path': train_images})\n\nval_labels = [os.path.join(img_label_dir, f\"{img_id}.txt\") for img_id in val_image_ids]\nval_images = [os.path.join(clahe_output_folder, f\"{img_id}.png\") for img_id in val_image_ids]\nyolo_df_val = pd.DataFrame({'label_path': val_labels, 'image_path': val_images})\n\nprint(\"✅ DataFrames created successfully!\")\nprint(f\"  📁 Image source: {clahe_output_folder} (CLAHE enhanced)\")\nprint(f\"  📊 Split method: Stratified (balanced classes)\")\nprint(f\"  📈 Train samples: {len(yolo_df_train)}\")\nprint(f\"  📉 Val samples:   {len(yolo_df_val)}\")\n\n# Verify no data leakage\ntrain_ids_set = set(train_image_ids)\nval_ids_set = set(val_image_ids)\noverlap = train_ids_set.intersection(val_ids_set)\nprint(f\"\\n🔒 Data leakage check: {len(overlap)} overlapping images\")\nif len(overlap) == 0:\n    print(\"  ✅ PASSED - No data leakage!\")\nelse:\n    print(f\"  ❌ FAILED - Found {len(overlap)} overlapping images!\")\n\n# Verify files exist\ntrain_missing = sum(1 for p in train_images if not os.path.exists(p))\nval_missing = sum(1 for p in val_images if not os.path.exists(p))\nif train_missing + val_missing == 0:\n    print(f\"\\n✅ All image files exist!\")\nelse:\n    print(f\"\\n⚠️ Warning: {train_missing} train + {val_missing} val images missing\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.258898Z","iopub.status.idle":"2025-12-04T17:19:53.259209Z","shell.execute_reply.started":"2025-12-04T17:19:53.259055Z","shell.execute_reply":"2025-12-04T17:19:53.259069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"yolo_df_train.head()","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.26026Z","iopub.status.idle":"2025-12-04T17:19:53.260528Z","shell.execute_reply.started":"2025-12-04T17:19:53.26042Z","shell.execute_reply":"2025-12-04T17:19:53.26043Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Copying train images\nprint(\"COPYING TRAIN IMAGES :-->\", \"-\"*50)\nfor img_path in yolo_df_train['image_path']:\n    shutil.copy(img_path, \"/kaggle/working/dataset/train/images\")\n    \n# Copying validation images\nprint(\"COPYING VALID IMAGES :-->\", \"-\"*50)\nfor img_path in yolo_df_val['image_path']:\n    shutil.copy(img_path, \"/kaggle/working/dataset/val/images\")\n\n# Copying train labels\nprint(\"COPYING TRAIN LABELS :-->\", \"-\"*50)\nfor label_path in yolo_df_train['label_path']:\n    shutil.copy(label_path, \"/kaggle/working/dataset/train/labels\")\n\n# Copying validation labels\nprint(\"COPYING VALID LABELS :-->\", \"-\"*50)\nfor label_path in yolo_df_val['label_path']:\n    shutil.copy(label_path, \"/kaggle/working/dataset/val/labels\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.261615Z","iopub.status.idle":"2025-12-04T17:19:53.261838Z","shell.execute_reply.started":"2025-12-04T17:19:53.261741Z","shell.execute_reply":"2025-12-04T17:19:53.261749Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_label_count = len(glob.glob('/kaggle/working/dataset/train/labels/*'))\nprint(\"Number of train labels:\", train_label_count)\n\ntrain_image_count = len(glob.glob('/kaggle/working/dataset/train/images/*'))\nprint(\"Number of train images:\", train_image_count)\n\nvalid_label_count = len(glob.glob('/kaggle/working/dataset/val/labels/*'))\nprint(\"Number of valid labels:\", valid_label_count)\n\nvalid_image_count = len(glob.glob('/kaggle/working/dataset/val/images/*'))\nprint(\"Number of valid images:\", valid_image_count)\n\n# Verify counts match\nassert train_label_count == train_image_count == len(yolo_df_train), \"Train label/image count mismatch!\"\nassert valid_label_count == valid_image_count == len(yolo_df_val), \"Val label/image count mismatch!\"\nprint(\"\\n✅ All counts verified successfully!\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.262961Z","iopub.status.idle":"2025-12-04T17:19:53.263261Z","shell.execute_reply.started":"2025-12-04T17:19:53.263127Z","shell.execute_reply":"2025-12-04T17:19:53.263142Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"move_dir = '/kaggle/working/dataset'\ngoal_dir = '/kaggle/working/yolo/dataset'\n\nshutil.move(move_dir, goal_dir)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.264009Z","iopub.status.idle":"2025-12-04T17:19:53.264221Z","shell.execute_reply.started":"2025-12-04T17:19:53.264122Z","shell.execute_reply":"2025-12-04T17:19:53.264131Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"yaml_dir = \"/kaggle/working/yolo/dataset\"\n\ndata = {\n    'names': class_names_ordered,\n    'nc': len(class_names_ordered),\n\n    'train': '/kaggle/working/yolo/dataset/train/images/',\n    'val': '/kaggle/working/yolo/dataset/val/images/'\n}\n\nwith open(yaml_dir+'/data.yaml', 'w') as file:\n    yaml.dump(data, file)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.264855Z","iopub.status.idle":"2025-12-04T17:19:53.265127Z","shell.execute_reply.started":"2025-12-04T17:19:53.265015Z","shell.execute_reply":"2025-12-04T17:19:53.265027Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Cấu hình Hyperparameters tối ưu cho Medical Imaging*","metadata":{}},{"cell_type":"code","source":"# Tạo file hyperparameters tối ưu\nhyp_config = {\n    # Learning rate settings - quan trọng cho medical imaging\n    'lr0': 0.001,              # Initial learning rate (giảm từ 0.01 xuống 0.001 cho stable training)\n    'lrf': 0.01,               # Final learning rate (lr0 * lrf) \n    'momentum': 0.937,         # SGD momentum/Adam beta1\n    'weight_decay': 0.0005,    # Optimizer weight decay 5e-4\n    \n    # Loss weights - cân bằng giữa các loss components\n    'box': 7.5,                # Box loss gain (tăng từ 0.05 lên 7.5 - bbox quan trọng)\n    'cls': 0.5,                # Class loss gain (giảm nhẹ vì có 10 classes)\n    'dfl': 1.5,                # DFL loss gain\n    \n    # Augmentation settings - quan trọng cho generalization\n    'hsv_h': 0.015,            # Image HSV-Hue augmentation (giữ nhẹ cho medical)\n    'hsv_s': 0.7,              # Image HSV-Saturation augmentation\n    'hsv_v': 0.4,              # Image HSV-Value augmentation\n    'degrees': 10.0,           # Image rotation (+/- deg) - giảm từ 180\n    'translate': 0.1,          # Image translation (+/- fraction)\n    'scale': 0.5,              # Image scale (+/- gain) - giảm từ 0.9\n    'shear': 0.0,              # Image shear (+/- deg) - tắt shear\n    'perspective': 0.0,        # Image perspective (+/- fraction) - tắt perspective\n    'flipud': 0.0,             # Image flip up-down (probability) - không flip medical images\n    'fliplr': 0.5,             # Image flip left-right (probability)\n    'mosaic': 1.0,             # Mosaic augmentation (probability)\n    'mixup': 0.1,              # MixUp augmentation (probability) - thêm mixup\n    'copy_paste': 0.1,         # Copy-paste augmentation (probability) - thêm copy-paste\n    \n    # Other settings\n    'warmup_epochs': 3.0,      # Warmup epochs (fractions ok)\n    'warmup_momentum': 0.8,    # Warmup initial momentum\n    'warmup_bias_lr': 0.1,     # Warmup initial bias lr\n    'close_mosaic': 10,        # Disable mosaic in final N epochs\n}\n\n# Lưu hyperparameters\nimport yaml\nhyp_path = '/kaggle/working/yolo/hyp_custom.yaml'\nos.makedirs(os.path.dirname(hyp_path), exist_ok=True)\n\nwith open(hyp_path, 'w') as f:\n    yaml.dump(hyp_config, f, default_flow_style=False)\n\nprint(\"✅ Custom hyperparameters saved to:\", hyp_path)\nprint(\"\\n📊 Key optimizations:\")\nprint(\"  - Lower initial LR (0.001) for stable convergence\")\nprint(\"  - Higher box loss weight (7.5) - bbox accuracy crucial\")\nprint(\"  - Medical-safe augmentation (no flip-ud, limited rotation)\")\nprint(\"  - Added mixup + copy-paste for better generalization\")\nprint(\"  - Warmup strategy for stable start\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.265906Z","iopub.status.idle":"2025-12-04T17:19:53.266149Z","shell.execute_reply.started":"2025-12-04T17:19:53.266045Z","shell.execute_reply":"2025-12-04T17:19:53.266055Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***ULTRA OPTIMIZATION: Hyperparameters cho mAP50 = 0.5***","metadata":{}},{"cell_type":"code","source":"# 🎯 ULTRA OPTIMIZED HYPERPARAMETERS - Target: mAP50 >= 0.5\n# Aggressive settings cho maximum performance\n\nhyp_ultra = {\n    # Learning rate - aggressive cho faster convergence\n    'lr0': 0.002,              # Tăng từ 0.001 (faster learning)\n    'lrf': 0.001,              # Lower final LR\n    'momentum': 0.95,          # Higher momentum\n    'weight_decay': 0.0001,    # Giảm weight decay (less regularization)\n    \n    # Loss weights - CRITICAL for mAP50\n    'box': 10.0,               # ⬆️ Tăng mạnh box loss (từ 7.5)\n    'cls': 1.0,                # ⬆️ Tăng class loss (từ 0.5)\n    'dfl': 2.0,                # ⬆️ Tăng DFL loss (từ 1.5)\n    \n    # Augmentation - aggressive mix\n    'hsv_h': 0.02,             # Tăng hue variation\n    'hsv_s': 0.8,              # Tăng saturation\n    'hsv_v': 0.5,              # Tăng value\n    'degrees': 15.0,           # ⬆️ Tăng rotation (từ 10°)\n    'translate': 0.15,         # ⬆️ Tăng translation\n    'scale': 0.7,              # ⬆️ Tăng scale variation\n    'shear': 2.0,              # ✅ BẬT shear (từ 0)\n    'perspective': 0.0001,     # ✅ BẬT perspective nhẹ\n    'flipud': 0.0,             # Giữ 0 (medical safe)\n    'fliplr': 0.5,             # Keep\n    'mosaic': 1.0,             # Max mosaic\n    'mixup': 0.15,             # ⬆️ Tăng mixup (từ 0.1)\n    'copy_paste': 0.15,        # ⬆️ Tăng copy-paste\n    \n    # Advanced augmentations\n    'erasing': 0.4,            # ✅ Random erasing (new)\n    \n    # Training dynamics\n    'warmup_epochs': 5.0,      # ⬆️ Longer warmup (từ 3)\n    'warmup_momentum': 0.9,    # Higher warmup momentum\n    'warmup_bias_lr': 0.15,    # Higher warmup bias LR\n    'close_mosaic': 15,        # ⬆️ Close mosaic later (từ 10)\n}\n\n# Save\nwith open('/kaggle/working/yolo/hyp_ultra.yaml', 'w') as f:\n    yaml.dump(hyp_ultra, f, default_flow_style=False)\n\nprint(\"🚀 ULTRA OPTIMIZED HYPERPARAMETERS\")\nprint(\"=\"*80)\nprint(\"🎯 Target: mAP50 >= 0.5 (50%)\")\nprint(\"\\n📊 Key Changes for Higher mAP50:\")\nprint(\"  ⬆️ Box Loss: 7.5 → 10.0  (better localization)\")\nprint(\"  ⬆️ Class Loss: 0.5 → 1.0  (better classification)\")\nprint(\"  ⬆️ DFL Loss: 1.5 → 2.0    (better box quality)\")\nprint(\"  ⬆️ Rotation: 10° → 15°    (more diversity)\")\nprint(\"  ⬆️ MixUp: 0.1 → 0.15      (better generalization)\")\nprint(\"  ✅ Shear: 0 → 2.0         (geometric variation)\")\nprint(\"  ✅ Random Erasing: 0.4    (robustness)\")\nprint(\"=\"*80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.267343Z","iopub.status.idle":"2025-12-04T17:19:53.267571Z","shell.execute_reply.started":"2025-12-04T17:19:53.26746Z","shell.execute_reply":"2025-12-04T17:19:53.267469Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Gọi model để thực hiện training*","metadata":{}},{"cell_type":"code","source":"from ultralytics import YOLO\n\n# ✅ OPTIMIZATION 1: Dùng YOLO12s thay vì yolo12n (nano quá nhỏ cho medical imaging)\n# YOLOv12s có capacity tốt hơn để học features phức tạp của X-ray images\nmodel = YOLO('yolo12m.pt')  # Small model - cân bằng giữa accuracy và speed\n\n# Display model information\nprint(\"=\"*80)\nprint(\"MODEL ARCHITECTURE INFO:\")\nprint(\"=\"*80)\nmodel.info()\nprint(\"\\n\")\n\n# ✅ OPTIMIZATION 2: Training với hyperparameters tối ưu\nprint(\"=\"*80)\nprint(\"STARTING OPTIMIZED TRAINING...\")\nprint(\"=\"*80)\n\nresults = model.train(\n    # Data config\n    data=yaml_dir + '/data.yaml',\n    \n    # Model config  \n    imgsz=512,                    # Input image size\n    \n    # Training config\n    epochs=150,                   # Tăng từ 100 -> 150 epochs\n    batch=16,                     # Batch size (auto-adjust nếu GPU nhỏ)\n    device=0,                     # GPU device\n    \n    # Optimization\n    optimizer='AdamW',            # AdamW tốt hơn SGD cho medical imaging\n    lr0=0.001,                    # Initial learning rate (thấp hơn cho stable)\n    lrf=0.01,                     # Final learning rate factor\n    momentum=0.937,               # Momentum\n    weight_decay=0.0005,          # Weight decay for regularization\n    \n    # Augmentation (medical-safe)\n    degrees=10.0,                 # Rotation ±10° (không quá mạnh)\n    translate=0.1,                # Translation\n    scale=0.5,                    # Scaling\n    flipud=0.0,                   # Không flip up-down (giữ orientation)\n    fliplr=0.5,                   # Flip left-right\n    mosaic=1.0,                   # Mosaic augmentation\n    mixup=0.1,                    # MixUp augmentation\n    copy_paste=0.1,               # Copy-paste augmentation\n    \n    # Loss weights\n    box=7.5,                      # Box loss weight (cao - bbox quan trọng)\n    cls=0.5,                      # Class loss weight\n    dfl=1.5,                      # DFL loss weight\n    \n    # Callbacks & monitoring\n    patience=20,                  # Early stopping patience (tăng từ 10)\n    save=True,                    # Save checkpoints\n    save_period=10,               # Save every N epochs\n    val=True,                     # Validate during training\n    plots=True,                   # Save training plots\n    \n    # Performance\n    workers=4,                    # Dataloader workers (tăng từ 4)\n    cache=False,                  # Cache images (tắt nếu RAM ít)\n    amp=True,                     # Automatic Mixed Precision\n    \n    # Project organization\n    project='/kaggle/working/runs',\n    name='yolo12s_optimized',\n    exist_ok=True,\n    \n    # Advanced\n    close_mosaic=10,              # Disable mosaic 10 epochs trước khi kết thúc\n    resume=False,                 # Resume from last checkpoint\n    verbose=True,                 # Verbose output\n)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"TRAINING COMPLETED!\")\nprint(\"=\"*80)\n\n# ✅ OPTIMIZATION 3: Validation với detailed metrics\nprint(\"\\n\" + \"=\"*80)\nprint(\"FINAL VALIDATION RESULTS:\")\nprint(\"=\"*80)\n\nmetrics = model.val()\n\nprint(f\"\\n📊 Overall Metrics:\")\nprint(f\"  mAP50-95: {metrics.box.map:.4f}\")\nprint(f\"  mAP50:    {metrics.box.map50:.4f}\")\nprint(f\"  mAP75:    {metrics.box.map75:.4f}\")\n\nprint(f\"\\n📊 Per-Class mAP50-95:\")\nfor i, (class_name, class_id) in enumerate(sorted(lung_diseases.items(), key=lambda x: x[1])):\n    if i < len(metrics.box.maps):\n        print(f\"  {class_name:25s}: {metrics.box.maps[i]:.4f}\")\n\n# Lưu kết quả\nresults_summary = {\n    'mAP50_95': float(metrics.box.map),\n    'mAP50': float(metrics.box.map50),\n    'mAP75': float(metrics.box.map75),\n    'per_class_map': {name: float(metrics.box.maps[i]) \n                      for i, (name, _) in enumerate(sorted(lung_diseases.items(), key=lambda x: x[1]))}\n}\n\nimport json\nwith open('/kaggle/working/final_metrics.json', 'w') as f:\n    json.dump(results_summary, f, indent=2)\n\nprint(\"\\n✅ Metrics saved to: /kaggle/working/final_metrics.json\")\nprint(\"✅ Best model saved to:\", model.trainer.best)\nprint(\"=\"*80)","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.268941Z","iopub.status.idle":"2025-12-04T17:19:53.269183Z","shell.execute_reply.started":"2025-12-04T17:19:53.269062Z","shell.execute_reply":"2025-12-04T17:19:53.269074Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Post-Training Analysis & Ensemble (Optional)*","metadata":{}},{"cell_type":"code","source":"# Phân tích lỗi trên validation set để cải thiện thêm\nfrom ultralytics import YOLO\nimport glob\n\n# Load best model\nbest_model_path = glob.glob('/kaggle/working/runs/yolo12s_optimized*/weights/best.pt')\nif best_model_path:\n    best_model = YOLO(best_model_path[0])\n    print(f\"✅ Loaded best model from: {best_model_path[0]}\")\n    \n    # Predict on validation set để phân tích\n    val_img_dir = '/kaggle/working/yolo/dataset/val/images'\n    val_results = best_model.predict(\n        source=val_img_dir,\n        save=True,\n        save_txt=True,\n        conf=0.25,  # Confidence threshold\n        iou=0.45,   # NMS IOU threshold\n        project='/kaggle/working/analysis',\n        name='val_predictions'\n    )\n    \n    print(\"\\n📊 Prediction Analysis:\")\n    print(f\"  Total images: {len(val_results)}\")\n    print(f\"  Predictions saved to: /kaggle/working/analysis/val_predictions\")\n    \n    # Thống kê confidence scores\n    all_confs = []\n    for result in val_results:\n        if len(result.boxes) > 0:\n            all_confs.extend(result.boxes.conf.cpu().numpy().tolist())\n    \n    if all_confs:\n        print(f\"\\n📈 Confidence Statistics:\")\n        print(f\"  Mean: {np.mean(all_confs):.3f}\")\n        print(f\"  Median: {np.median(all_confs):.3f}\")\n        print(f\"  Std: {np.std(all_confs):.3f}\")\n        print(f\"  Min: {np.min(all_confs):.3f}\")\n        print(f\"  Max: {np.max(all_confs):.3f}\")\nelse:\n    print(\"⚠️ No trained model found. Please train first.\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.270616Z","iopub.status.idle":"2025-12-04T17:19:53.270918Z","shell.execute_reply.started":"2025-12-04T17:19:53.270746Z","shell.execute_reply":"2025-12-04T17:19:53.27076Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## *Advanced: Test Time Augmentation (TTA) để tăng accuracy*","metadata":{}},{"cell_type":"code","source":"# Test Time Augmentation để boost performance khi inference\n# TTA có thể tăng mAP lên 1-3%\n\nif best_model_path:\n    best_model = YOLO(best_model_path[0])\n    \n    print(\"🚀 Running Validation with Test Time Augmentation (TTA)...\")\n    print(\"=\"*80)\n    \n    # Validate với TTA enabled\n    tta_metrics = best_model.val(\n        data=yaml_dir + '/data.yaml',\n        augment=True,  # Enable TTA\n        conf=0.001,    # Lower confidence để detect nhiều hơn\n        iou=0.6,       # Higher IOU cho stricter NMS\n        device=0\n    )\n    \n    print(\"\\n📊 Comparison: Normal vs TTA\")\n    print(\"=\"*80)\n    print(f\"Normal mAP50-95: {metrics.box.map:.4f}\")\n    print(f\"TTA mAP50-95:    {tta_metrics.box.map:.4f}\")\n    print(f\"Improvement:     {(tta_metrics.box.map - metrics.box.map)*100:+.2f}%\")\n    print()\n    print(f\"Normal mAP50: {metrics.box.map50:.4f}\")\n    print(f\"TTA mAP50:    {tta_metrics.box.map50:.4f}\")\n    print(f\"Improvement:  {(tta_metrics.box.map50 - metrics.box.map50)*100:+.2f}%\")\n    print(\"=\"*80)\n    \n    print(\"\\n💡 TIP: Sử dụng augment=True khi inference để tăng accuracy!\")\nelse:\n    print(\"⚠️ Train model trước khi chạy TTA\")","metadata":{"execution":{"iopub.status.busy":"2025-12-04T17:19:53.272234Z","iopub.status.idle":"2025-12-04T17:19:53.272487Z","shell.execute_reply.started":"2025-12-04T17:19:53.272384Z","shell.execute_reply":"2025-12-04T17:19:53.272393Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ***BONUS: Weighted Boxes Fusion (WBF) cho Ensemble***","metadata":{}},{"cell_type":"code","source":"# 🎯 ADVANCED ENSEMBLE: Train nhiều models và ensemble để đạt mAP50 >= 0.5\n# Nếu single model chưa đủ, ensemble có thể boost thêm 5-10%\n\nprint(\"🚀 ENSEMBLE STRATEGY FOR mAP50 >= 0.5\")\nprint(\"=\"*80)\nprint(\"\\n📋 Recommendation:\")\nprint(\"  1. Train YOLO12m với config hiện tại\")\nprint(\"  2. Train YOLO12l (large) với same config\")\nprint(\"  3. Train YOLO12x (xlarge) nếu GPU đủ mạnh\")\nprint(\"  4. Ensemble 3 models với Weighted Boxes Fusion\")\nprint(\"\\n💡 Expected Results:\")\nprint(\"  - Single YOLO12m: mAP50 ~ 0.42-0.48\")\nprint(\"  - Ensemble (m+l+x): mAP50 ~ 0.50-0.55 ✅\")\nprint(\"=\"*80)\n\n# Code để train thêm YOLO12l (uncomment để chạy)\ntrain_additional_models = \"\"\"\n# Train YOLO12l\nmodel_l = YOLO('yolo12l.pt')\nresults_l = model_l.train(\n    data=yaml_dir + '/data.yaml',\n    imgsz=640,\n    epochs=200,\n    batch=8,  # Giảm batch vì model lớn hơn\n    # ... (same params as YOLO12m)\n    name='yolo12l_ultra_optimized',\n)\n\n# Train YOLO12x (nếu GPU đủ mạnh)\nmodel_x = YOLO('yolo12x.pt')\nresults_x = model_x.train(\n    data=yaml_dir + '/data.yaml',\n    imgsz=640,\n    epochs=200,\n    batch=4,  # Batch nhỏ hơn cho model lớn\n    # ... (same params)\n    name='yolo12x_ultra_optimized',\n)\n\"\"\"\n\nprint(\"\\n📝 Code to train additional models saved in variable 'train_additional_models'\")\nprint(\"⚠️ Chỉ train thêm models nếu single YOLO12m chưa đạt mAP50 >= 0.5\")\nprint(\"\\n🔍 Sau khi train xong, sử dụng ensemble_models() để combine predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-04T17:19:53.273338Z","iopub.status.idle":"2025-12-04T17:19:53.273559Z","shell.execute_reply.started":"2025-12-04T17:19:53.273453Z","shell.execute_reply":"2025-12-04T17:19:53.273462Z"}},"outputs":[],"execution_count":null}]}