{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":24800,"databundleVersionId":1831594,"sourceType":"competition"},{"sourceId":1801996,"sourceType":"datasetVersion","datasetId":1069999},{"sourceId":14604334,"sourceType":"datasetVersion","datasetId":9328565}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q ultralytics --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:34:15.140551Z","iopub.execute_input":"2026-01-30T11:34:15.140739Z","iopub.status.idle":"2026-01-30T11:36:49.986435Z","shell.execute_reply.started":"2026-01-30T11:34:15.140722Z","shell.execute_reply":"2026-01-30T11:36:49.985648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom typing import Callable, Any\nimport multiprocessing\nimport math\n\nimport torch\nfrom torch import nn, optim\nfrom torchvision import transforms, utils\nfrom torch.utils.data import DataLoader, Dataset\ntqdm.pandas()\n\nimport cv2\nimport shutil\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport pydicom\nimport yaml\nimport glob\n\nfrom PIL import Image \n# from ultralytics import YOLO\n\nfrom sklearn.preprocessing import MultiLabelBinarizer \nfrom sklearn.model_selection import train_test_split\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom skimage import exposure\n\nfrom joblib import Parallel, delayed\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport random","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:36:49.987527Z","iopub.execute_input":"2026-01-30T11:36:49.987794Z","iopub.status.idle":"2026-01-30T11:37:02.327017Z","shell.execute_reply.started":"2026-01-30T11:36:49.987765Z","shell.execute_reply":"2026-01-30T11:37:02.32626Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:37:02.328955Z","iopub.execute_input":"2026-01-30T11:37:02.329368Z","iopub.status.idle":"2026-01-30T11:37:02.59339Z","shell.execute_reply.started":"2026-01-30T11:37:02.329349Z","shell.execute_reply":"2026-01-30T11:37:02.592661Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Relevant terms\n\n- DICOM: The image is originally in .dicom format, storing not only the image data but high-bit depth information (up to 16-bit) and metadata (patient info, settings,..)\n\n- VOI LUT (Volume of Interest Look-Up Table): is a mathematical transformation used in medical imaging to make the relevant parts of 16-bit DICOM image visible to human eye (or ML model)","metadata":{}},{"cell_type":"code","source":"# Each dicom file is a container with pixel data, patient info and extraction metadata\nsample_path = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train/000434271f63a053c4128a0ba6352c7f.dicom'\n\ndicom = pydicom.dcmread(sample_path)\nprint(dicom)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:02.594134Z","iopub.execute_input":"2026-01-30T11:37:02.594462Z","iopub.status.idle":"2026-01-30T11:37:02.898882Z","shell.execute_reply.started":"2026-01-30T11:37:02.59443Z","shell.execute_reply":"2026-01-30T11:37:02.898033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(dicom.pixel_array.shape) #(2 dim for original resolution)\n\ndicom.pixel_array","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:02.899724Z","iopub.execute_input":"2026-01-30T11:37:02.900007Z","iopub.status.idle":"2026-01-30T11:37:02.915359Z","shell.execute_reply.started":"2026-01-30T11:37:02.89998Z","shell.execute_reply":"2026-01-30T11:37:02.914583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### apply_voi_lut applies a transformation (windowing) to focus on a specific shade of gray that matters for anatomical details\n# -> intensify better gray areas\n\napply_voi_lut(dicom.pixel_array, dicom)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:02.916187Z","iopub.execute_input":"2026-01-30T11:37:02.916828Z","iopub.status.idle":"2026-01-30T11:37:03.047616Z","shell.execute_reply.started":"2026-01-30T11:37:02.916806Z","shell.execute_reply":"2026-01-30T11:37:03.046958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dicom2image(path, voi_lut=True, fix_monochrome=True):\n    # Read the file\n    dicom = pydicom.dcmread(path)\n\n    # VOI LUT\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array.astype(np.float32)\n\n    # pixel inversion black <-> white \n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\": #monochrome1 0-white, 1-black, need to invert\n        data = np.max(data) - data\n\n    # normalize to 0-1\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n\n    # Scale to 0–255 and change type to Unasigned 8-bit Int\n    data = (data * 255).astype(np.uint8)\n\n    return data","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:37:03.048445Z","iopub.execute_input":"2026-01-30T11:37:03.048728Z","iopub.status.idle":"2026-01-30T11:37:03.05438Z","shell.execute_reply.started":"2026-01-30T11:37:03.048708Z","shell.execute_reply":"2026-01-30T11:37:03.053772Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resize_boxes(row, h_resize, w_resize):\n\n    # For 0-finding rows, don't do anything\n    if pd.isna(row['x_min']):\n        return pd.Series([row['x_min'], row['y_min'], row['x_max'],  row['y_max']])\n\n    w_scale = w_resize/ row['width']\n    h_scale = h_resize/ row['height']\n\n    x_min_new = round(row['x_min'] * w_scale, 1)\n    x_max_new = round(row['x_max'] * w_scale, 1)\n    y_min_new = round(row['y_min'] * h_scale, 1)\n    y_max_new = round(row['y_max'] * h_scale, 1)\n\n    return pd.Series([x_min_new, y_min_new, x_max_new, y_max_new])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.05513Z","iopub.execute_input":"2026-01-30T11:37:03.055423Z","iopub.status.idle":"2026-01-30T11:37:03.071664Z","shell.execute_reply.started":"2026-01-30T11:37:03.055404Z","shell.execute_reply":"2026-01-30T11:37:03.071076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# change box from [x max, y max, x min, y min] to normalized Center & size\ndef normalize_bbox(df):\n    df['x_center'] = (df['x_min'] + df['x_max'])/2\n    df['y_center'] = (df['y_min'] + df['y_max'])/2\n    df['bbox_width'] = df['x_max'] - df['x_min']\n    df['bbox_height'] = df['y_max'] - df['y_min']\n\n    df['x_center_norm'] = df['x_center'] / df['width']\n    df['y_center_norm'] = df['y_center'] / df['height']\n    df['bbox_height_norm'] = df['bbox_height'] / df['height']\n    df['bbox_width_norm'] = df['bbox_width'] / df['width']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.074396Z","iopub.execute_input":"2026-01-30T11:37:03.074605Z","iopub.status.idle":"2026-01-30T11:37:03.086056Z","shell.execute_reply.started":"2026-01-30T11:37:03.074588Z","shell.execute_reply":"2026-01-30T11:37:03.085491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# save normalized Center & size bbox to file\ndef get_bbox(df, output_file):\n    with open(output_file, 'w') as f:\n        for _, row in df.iterrows():\n            class_id = row['class_id']\n            x_center, y_center = row['x_center_norm'], row['y_center_norm']\n            width, height = row[\"bbox_width_norm\"], row['bbox_height_norm']\n            f.write(f\"{class_id} {x_center} {y_center} {width} {height}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.086856Z","iopub.execute_input":"2026-01-30T11:37:03.087385Z","iopub.status.idle":"2026-01-30T11:37:03.101413Z","shell.execute_reply.started":"2026-01-30T11:37:03.087357Z","shell.execute_reply":"2026-01-30T11:37:03.100874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_image_with_bounding_box(image, bounding_boxes, class_dict):\n    fig, ax = plt.subplots()\n    ax.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))\n    \n    for box in bounding_boxes:\n        class_id, x, y, width, height = map(float, box.split())\n        image_width, image_height = image.shape[1], image.shape[0]\n        x1 = int((x - width / 2) * image_width)\n        y1 = int((y - height / 2) * image_height)\n        x2 = int((x + width / 2) * image_width)\n        y2 = int((y + height / 2) * image_height)\n        \n        # Choose random color for bounding box\n        color = [random.random() for _ in range(3)]\n        \n        rect = plt.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, edgecolor=color, facecolor='none')\n        ax.add_patch(rect)\n        \n        # Add label text using class label from dictionary\n        label_text = class_dict[int(class_id)]\n        ax.text(x1, y1, label_text, color='white', verticalalignment='top', bbox={'color': color, 'pad': 0})\n    \n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.102075Z","iopub.execute_input":"2026-01-30T11:37:03.102324Z","iopub.status.idle":"2026-01-30T11:37:03.121676Z","shell.execute_reply.started":"2026-01-30T11:37:03.102307Z","shell.execute_reply":"2026-01-30T11:37:03.12095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main_plot(image_path, bbox_path, class_dict):\n    print(type(image_path))\n    image = cv2.imread(image_path)\n    \n    # Read bboxes\n    with open(bbox_path, 'r') as file:\n        bounding_boxes = file.readlines()\n    \n    # Plot image with bboxes\n    plot_image_with_bounding_box(image, bounding_boxes, class_dict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.122497Z","iopub.execute_input":"2026-01-30T11:37:03.123279Z","iopub.status.idle":"2026-01-30T11:37:03.136569Z","shell.execute_reply.started":"2026-01-30T11:37:03.123255Z","shell.execute_reply":"2026-01-30T11:37:03.13602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot image\ndef plot_img(imgs, cols=4, size=7, title=\"\", cmap='gray', is_rgb=True, img_size=(500, 500)):\n    rows = len(imgs) // cols + 1\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None:\n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.137173Z","iopub.execute_input":"2026-01-30T11:37:03.137432Z","iopub.status.idle":"2026-01-30T11:37:03.150373Z","shell.execute_reply.started":"2026-01-30T11:37:03.13741Z","shell.execute_reply":"2026-01-30T11:37:03.149772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normalized histogram, increase contrast \ndef hist_equalize(img_path_output):\n    img_path, output_dir = img_path_output\n    filename = os.path.splitext(os.path.basename(img_path))[0]\n    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    equalize_img = exposure.equalize_hist(img)\n    equalize_img = (equalize_img * 255).astype(np.uint8)\n    cv2.imwrite(os.path.join(output_dir, f'{filename}.png'), equalize_img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.151361Z","iopub.execute_input":"2026-01-30T11:37:03.151542Z","iopub.status.idle":"2026-01-30T11:37:03.177171Z","shell.execute_reply.started":"2026-01-30T11:37:03.151519Z","shell.execute_reply":"2026-01-30T11:37:03.176468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save img\ndef save_img(img_path_list, output_dir, n_jobs = -1):\n    os.makedirs(output_dir, exist_ok=True)\n    img_output_list = [(path, output_dir) for path in img_path_list]\n\n    Parallel(n_jobs = n_jobs)(\n        delayed(hist_equalize)(args) for args in tqdm(img_output_list, desc=\"Histogram Equalizing\")\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:37:03.177979Z","iopub.execute_input":"2026-01-30T11:37:03.178256Z","iopub.status.idle":"2026-01-30T11:37:03.193938Z","shell.execute_reply.started":"2026-01-30T11:37:03.178209Z","shell.execute_reply":"2026-01-30T11:37:03.193186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sourse_path = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train/\"\nbboxes_path = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv\"\ndicom_path = \"/kaggle/input/d/xhlulu/vinbigdata/train_meta.csv\"\n\nbbox_df = pd.read_csv(bboxes_path)","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:37:03.19465Z","iopub.execute_input":"2026-01-30T11:37:03.194886Z","iopub.status.idle":"2026-01-30T11:37:03.343708Z","shell.execute_reply.started":"2026-01-30T11:37:03.19487Z","shell.execute_reply":"2026-01-30T11:37:03.34309Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_label = bbox_df[['class_name', 'class_id']].drop_duplicates().reset_index(drop=True)\nclass_label.sort_values(by='class_id')","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:37:03.34453Z","iopub.execute_input":"2026-01-30T11:37:03.344847Z","iopub.status.idle":"2026-01-30T11:37:03.394655Z","shell.execute_reply.started":"2026-01-30T11:37:03.344794Z","shell.execute_reply":"2026-01-30T11:37:03.394014Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_dir = '/kaggle/input/d/xhlulu/vinbigdata/train'\npath512_list = glob.glob(train512_dir + '/*')\n\nimage_512 = bbox_df.copy()\nimage_512['image_path'] = image_512['image_id'].progress_apply(lambda x: next(filter(lambda y: x in y, path512_list), None))\n\n# Drop rows with \"No Finding\" labels\ntrain512_df = image_512[image_512['class_name']!='No finding'].reset_index(drop=True) \n\n# List of image paths\nimg512_path_list = list(set(train512_df['image_path'].tolist()))","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:37:03.395396Z","iopub.execute_input":"2026-01-30T11:37:03.395652Z","iopub.status.idle":"2026-01-30T11:38:01.102392Z","shell.execute_reply.started":"2026-01-30T11:37:03.395634Z","shell.execute_reply":"2026-01-30T11:38:01.101711Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train512_df.head(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:01.103196Z","iopub.execute_input":"2026-01-30T11:38:01.103494Z","iopub.status.idle":"2026-01-30T11:38:01.11396Z","shell.execute_reply.started":"2026-01-30T11:38:01.103468Z","shell.execute_reply":"2026-01-30T11:38:01.113152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Image Id with each resolution\n\nimg_size_path = '/kaggle/input/d/xhlulu/vinbigdata/train_meta.csv'\nimg_size_df = pd.read_csv(img_size_path)\nimg_size_df.head()","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:01.114881Z","iopub.execute_input":"2026-01-30T11:38:01.115151Z","iopub.status.idle":"2026-01-30T11:38:01.163482Z","shell.execute_reply.started":"2026-01-30T11:38:01.115119Z","shell.execute_reply":"2026-01-30T11:38:01.162834Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train512_df.merge(img_size_df, on='image_id', how='left')\ntrain = train[['image_id', 'image_path', 'class_name', 'class_id', 'rad_id', 'x_min', 'y_min','x_max','y_max', 'dim0', 'dim1']]\ntrain = train.rename(columns = {\n    'dim0': 'height',\n    'dim1': 'width'\n})\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:01.164248Z","iopub.execute_input":"2026-01-30T11:38:01.164489Z","iopub.status.idle":"2026-01-30T11:38:01.202425Z","shell.execute_reply.started":"2026-01-30T11:38:01.164462Z","shell.execute_reply":"2026-01-30T11:38:01.201678Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Rescale bbox coordinate to match with the 512x512 resolution image\n\ntrain[['x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']] = train.progress_apply(resize_boxes, axis=1, result_type='expand', args=(512, 512))","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:01.203162Z","iopub.execute_input":"2026-01-30T11:38:01.20344Z","iopub.status.idle":"2026-01-30T11:38:05.97028Z","shell.execute_reply.started":"2026-01-30T11:38:01.203409Z","shell.execute_reply":"2026-01-30T11:38:05.968407Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:05.971505Z","iopub.execute_input":"2026-01-30T11:38:05.97177Z","iopub.status.idle":"2026-01-30T11:38:05.993751Z","shell.execute_reply.started":"2026-01-30T11:38:05.971749Z","shell.execute_reply":"2026-01-30T11:38:05.992881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:05.994529Z","iopub.execute_input":"2026-01-30T11:38:05.994813Z","iopub.status.idle":"2026-01-30T11:38:06.015158Z","shell.execute_reply.started":"2026-01-30T11:38:05.994796Z","shell.execute_reply":"2026-01-30T11:38:06.014467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random\n\nnum_images = 9\n\nlist_images = train['image_id'].tolist()\nsample_images = random.sample(list(list_images), num_images)\n\ndicom_dir = \"/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train\" \n\nsample_images_dicom = [i + \".dicom\" for i in sample_images]\n\nfull_paths = [os.path.join(dicom_dir, f) for f in sample_images_dicom]\n\nlist_dicom = [dicom2image(i) for i in full_paths]\n\nfilenames = [os.path.basename(p).split('.')[0] for p in full_paths]\n\nfiltered = bbox_df[bbox_df[\"image_id\"].isin(filenames)]","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:06.015936Z","iopub.execute_input":"2026-01-30T11:38:06.016434Z","iopub.status.idle":"2026-01-30T11:38:18.334979Z","shell.execute_reply.started":"2026-01-30T11:38:06.016409Z","shell.execute_reply":"2026-01-30T11:38:18.33424Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"full_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:18.335858Z","iopub.execute_input":"2026-01-30T11:38:18.336245Z","iopub.status.idle":"2026-01-30T11:38:18.341003Z","shell.execute_reply.started":"2026-01-30T11:38:18.336205Z","shell.execute_reply":"2026-01-30T11:38:18.34041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_images_with_boxes(dicom_paths, df, cols=3):\n    rows = math.ceil(len(dicom_paths) / cols)\n\n    fig, axes = plt.subplots(rows, cols, figsize=(cols * 6, rows * 6))\n\n    if rows == 1:\n        axes = [axes]\n\n    # Flatten axes \n    axes = np.array(axes).reshape(-1)\n\n    for idx, dicom_path in enumerate(dicom_paths):\n        ax = axes[idx]\n        filename = os.path.basename(dicom_path).split('.')[0]\n\n        boxes = df[df[\"image_id\"] == filename]\n        img = dicom2image(dicom_path)\n\n        ax.imshow(img, cmap=\"gray\")\n        ax.set_title(filename)\n        ax.axis(\"off\")\n\n        for _, row in boxes.iterrows():\n            x1, y1, x2, y2 = row[\"x_min\"], row[\"y_min\"], row[\"x_max\"], row[\"y_max\"]\n            class_name = row.get(\"class_name\", \"\")\n\n            rect = patches.Rectangle(\n                (x1, y1), \n                x2 - x1, \n                y2 - y1, \n                linewidth=2, \n                edgecolor='red', \n                facecolor='none'\n            )\n            ax.add_patch(rect)\n            ax.text(x1, y1 - 5, class_name, color=\"yellow\", fontsize=10, backgroundcolor=\"black\")\n\n    for j in range(idx + 1, len(axes)):\n        axes[j].axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:18.345329Z","iopub.execute_input":"2026-01-30T11:38:18.345607Z","iopub.status.idle":"2026-01-30T11:38:18.361699Z","shell.execute_reply.started":"2026-01-30T11:38:18.345589Z","shell.execute_reply":"2026-01-30T11:38:18.360851Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_images_with_boxes(full_paths, train, cols=3)","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:18.362916Z","iopub.execute_input":"2026-01-30T11:38:18.363206Z","iopub.status.idle":"2026-01-30T11:38:35.023564Z","shell.execute_reply.started":"2026-01-30T11:38:18.363186Z","shell.execute_reply":"2026-01-30T11:38:35.022677Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Merge overlapping bounding boxes","metadata":{}},{"cell_type":"markdown","source":"IoU >= 0.3 -> merge","metadata":{}},{"cell_type":"code","source":"def calculate_iou(box1, box2):\n\n    x1_min, y1_min = box1['x_min_new'], box1['y_min_new']\n    x1_max, y1_max = box1['x_max_new'], box1['y_max_new']\n    \n    x2_min, y2_min = box2['x_min_new'], box2['y_min_new']\n    x2_max, y2_max = box2['x_max_new'], box2['y_max_new']\n    \n    # Intersection\n    inter_x_min = max(x1_min, x2_min)\n    inter_y_min = max(y1_min, y2_min)\n    inter_x_max = min(x1_max, x2_max)\n    inter_y_max = min(y1_max, y2_max)\n    \n    if inter_x_min >= inter_x_max or inter_y_min >= inter_y_max:\n        return 0.0\n    \n    inter_area = (inter_x_max - inter_x_min) * (inter_y_max - inter_y_min)\n    \n    # Union\n    box1_area = (x1_max - x1_min) * (y1_max - y1_min)\n    box2_area = (x2_max - x2_min) * (y2_max - y2_min)\n    union_area = box1_area + box2_area - inter_area\n    \n    if union_area == 0:\n        return 0.0\n    \n    iou = inter_area / union_area\n    return iou","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:35.025803Z","iopub.execute_input":"2026-01-30T11:38:35.02624Z","iopub.status.idle":"2026-01-30T11:38:35.035181Z","shell.execute_reply.started":"2026-01-30T11:38:35.026195Z","shell.execute_reply":"2026-01-30T11:38:35.034267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_boxes(box1, box2):\n    merged = box1.copy()\n    merged['x_min_new'] = min(box1['x_min_new'], box2['x_min_new'])\n    merged['y_min_new'] = min(box1['y_min_new'], box2['y_min_new'])\n    merged['x_max_new'] = max(box1['x_max_new'], box2['x_max_new'])\n    merged['y_max_new'] = max(box1['y_max_new'], box2['y_max_new'])\n    return merged","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:35.036244Z","iopub.execute_input":"2026-01-30T11:38:35.037042Z","iopub.status.idle":"2026-01-30T11:38:35.058429Z","shell.execute_reply.started":"2026-01-30T11:38:35.037Z","shell.execute_reply":"2026-01-30T11:38:35.057536Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def merge_overlapping_boxes(df, iou_threshold=0.3):\n\n    if df.empty:\n        return df\n    \n    required_cols = ['image_id', 'image_path', 'class_name', 'class_id', 'rad_id', \n                     'x_min', 'y_min', 'x_max', 'y_max', 'height', 'width',\n                     'x_min_new', 'y_min_new', 'x_max_new', 'y_max_new']\n\n    keep_cols = [col for col in required_cols if col in df.columns]\n    \n    merged_records = []\n    processed_indices = set()\n\n    for (img_id, class_name), group in tqdm(df.groupby(['image_id', 'class_name']), \n                                             desc=\"Merging overlapping boxes\"):\n        group_indices = group.index.tolist()\n\n        if len(group_indices) == 1:\n            merged_records.append(group.iloc[0].to_dict())\n            processed_indices.update(group_indices)\n            continue\n        \n        # Convert to list of dicts \n        boxes = group.to_dict('records')\n        box_indices = group_indices.copy()\n        merged_flags = [False] * len(boxes)\n        \n        # Merge boxes with IoU > threshold\n        for i in range(len(boxes)):\n            if merged_flags[i]:\n                continue\n            \n            current_box = boxes[i]\n            boxes_to_merge = [i]\n\n            for j in range(i + 1, len(boxes)):\n                if merged_flags[j]:\n                    continue\n                \n                iou = calculate_iou(current_box, boxes[j])\n                \n                if iou > iou_threshold:\n                    boxes_to_merge.append(j)\n                    merged_flags[j] = True\n\n            if len(boxes_to_merge) > 1:\n                # Merge iteratively\n                merged_box = boxes[boxes_to_merge[0]]\n                for idx in boxes_to_merge[1:]:\n                    merged_box = merge_boxes(merged_box, boxes[idx])\n                merged_records.append(merged_box)\n                processed_indices.update([box_indices[idx] for idx in boxes_to_merge])\n            else:\n                merged_records.append(current_box)\n                processed_indices.add(box_indices[i])\n\n    merged_df = pd.DataFrame(merged_records)\n\n    final_cols = [col for col in keep_cols if col in merged_df.columns]\n    merged_df = merged_df[final_cols]\n\n    return merged_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:35.059439Z","iopub.execute_input":"2026-01-30T11:38:35.059731Z","iopub.status.idle":"2026-01-30T11:38:35.072906Z","shell.execute_reply.started":"2026-01-30T11:38:35.059704Z","shell.execute_reply":"2026-01-30T11:38:35.072261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_original = train.copy()\ntrain = merge_overlapping_boxes(train, iou_threshold=0.3)","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:35.073684Z","iopub.execute_input":"2026-01-30T11:38:35.073905Z","iopub.status.idle":"2026-01-30T11:38:44.437332Z","shell.execute_reply.started":"2026-01-30T11:38:35.073887Z","shell.execute_reply":"2026-01-30T11:38:44.436643Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_yolo = normalize_bbox(train)\ntrain_yolo.to_csv('train_yolo.csv', index=False, encoding='utf-8-sig')","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:44.43812Z","iopub.execute_input":"2026-01-30T11:38:44.438376Z","iopub.status.idle":"2026-01-30T11:38:44.935614Z","shell.execute_reply.started":"2026-01-30T11:38:44.438356Z","shell.execute_reply":"2026-01-30T11:38:44.934755Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try_df = pd.read_csv('/kaggle/input/d/xhlulu/vinbigdata/train_meta.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:44.936553Z","iopub.execute_input":"2026-01-30T11:38:44.936855Z","iopub.status.idle":"2026-01-30T11:38:44.955735Z","shell.execute_reply.started":"2026-01-30T11:38:44.936829Z","shell.execute_reply":"2026-01-30T11:38:44.955003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def write_yolo_labels(df: pd.DataFrame, label_dir: str):\n\n    os.makedirs(label_dir, exist_ok=True)\n\n    df = df.copy()\n\n    df['image_id'] = df['image_id'].astype(str)\n    \n    for image_id, group in tqdm(df.groupby('image_id'), desc='Writing YOLO labels'):\n        lines: list[str] = []\n        for _, row in group.iterrows():\n            # Assuming normalized columns exist\n            if pd.isna(row['x_center_norm']): continue\n            \n            lines.append(\n                f\"{int(row['class_id'])} {row['x_center_norm']:.6f} {row['y_center_norm']:.6f} {row['bbox_width_norm']:.6f} {row['bbox_height_norm']:.6f}\"\n            )\n\n        label_path = os.path.join(label_dir, f\"{image_id}.txt\")\n        with open(label_path, 'w', encoding='utf-8') as f:\n            f.write('\\n'.join(lines))","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:44.956544Z","iopub.execute_input":"2026-01-30T11:38:44.957641Z","iopub.status.idle":"2026-01-30T11:38:44.962789Z","shell.execute_reply.started":"2026-01-30T11:38:44.957613Z","shell.execute_reply":"2026-01-30T11:38:44.962251Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define label directory\nimg_label_dir = \"/kaggle/working/chest_detection/labels\"\n\n# Generate labels\nprint(f\"Generating YOLO labels in {img_label_dir}...\")\nwrite_yolo_labels(train_yolo, img_label_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:44.963445Z","iopub.execute_input":"2026-01-30T11:38:44.963631Z","iopub.status.idle":"2026-01-30T11:38:47.315294Z","shell.execute_reply.started":"2026-01-30T11:38:44.963615Z","shell.execute_reply":"2026-01-30T11:38:47.314525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"list_label = glob.glob(img_label_dir + \"/*\")\n\nwith open(list_label[0]) as f:\n    file = f.read()\n    print(file)\n\n## Each line contains the Class-x_center-y_center-height-width of the bbox","metadata":{"execution":{"iopub.status.busy":"2026-01-30T11:38:47.31617Z","iopub.execute_input":"2026-01-30T11:38:47.317067Z","iopub.status.idle":"2026-01-30T11:38:47.330385Z","shell.execute_reply.started":"2026-01-30T11:38:47.317045Z","shell.execute_reply":"2026-01-30T11:38:47.32965Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_yolo","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:47.331252Z","iopub.execute_input":"2026-01-30T11:38:47.331471Z","iopub.status.idle":"2026-01-30T11:38:47.371053Z","shell.execute_reply.started":"2026-01-30T11:38:47.331453Z","shell.execute_reply":"2026-01-30T11:38:47.370178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clahe_output_folder = \"/kaggle/working/clahe_images\"\nos.makedirs(clahe_output_folder, exist_ok=True)\n\nunique_image_paths = train_yolo['image_path'].unique().tolist()\n\nprint(f\"Applying CLAHE to {len(unique_image_paths)} images...\")\nprint(f\"Source images example: {unique_image_paths[0]}\")\nprint(f\"Output folder: {clahe_output_folder}\")\n\n# Run CLAHE in parallel\nsave_img(unique_image_paths, clahe_output_folder)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:38:47.371957Z","iopub.execute_input":"2026-01-30T11:38:47.372209Z","iopub.status.idle":"2026-01-30T11:39:36.215542Z","shell.execute_reply.started":"2026-01-30T11:38:47.37218Z","shell.execute_reply":"2026-01-30T11:39:36.214946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# original image\n# img = cv2.imread('/kaggle/input/d/xhlulu/vinbigdata/train/' + sample)\n# plt.imshow(img)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:39:36.216306Z","iopub.execute_input":"2026-01-30T11:39:36.216571Z","iopub.status.idle":"2026-01-30T11:39:36.301095Z","shell.execute_reply.started":"2026-01-30T11:39:36.216546Z","shell.execute_reply":"2026-01-30T11:39:36.300078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Clahe image\n\n# sample = os.listdir('/kaggle/working/clahe_images')[0]\n# img = cv2.imread('/kaggle/working/clahe_images/' + sample)\n# plt.imshow(img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:39:36.301588Z","iopub.status.idle":"2026-01-30T11:39:36.301817Z","shell.execute_reply.started":"2026-01-30T11:39:36.301706Z","shell.execute_reply":"2026-01-30T11:39:36.301716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nimport os\n\nfolder_to_zip = \"/kaggle/working/clahe_images\"\nzip_path = \"/kaggle/working/clahe_images_archive.zip\"\n\n# Remove old zip if exists\nif os.path.exists(zip_path):\n    os.remove(zip_path)\n\n# Create zip\nshutil.make_archive(\n    base_name=zip_path.replace(\".zip\", \"\"),\n    format=\"zip\",\n    root_dir=folder_to_zip\n)\n\nprint(\"Zip created at:\", zip_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-30T11:56:36.318958Z","iopub.execute_input":"2026-01-30T11:56:36.319324Z","iopub.status.idle":"2026-01-30T11:56:59.592466Z","shell.execute_reply.started":"2026-01-30T11:56:36.3193Z","shell.execute_reply":"2026-01-30T11:56:59.59156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -r /kaggle/working/chest_detection\n!ls /kaggle/working/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-04T01:59:50.731918Z","iopub.execute_input":"2026-02-04T01:59:50.732103Z","iopub.status.idle":"2026-02-04T01:59:50.85269Z","shell.execute_reply.started":"2026-02-04T01:59:50.732085Z","shell.execute_reply":"2026-02-04T01:59:50.851866Z"}},"outputs":[],"execution_count":null}]}