{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":24800,"datasetId":1042002,"databundleVersionId":1831594},{"sourceType":"datasetVersion","sourceId":1950595,"datasetId":1164135,"databundleVersionId":1989350}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport torchvision\nfrom torchvision.models.detection import fasterrcnn_resnet50_fpn, FastRCNNPredictor\nfrom torch.utils.data import DataLoader, Dataset\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torchmetrics.detection.mean_ap import MeanAveragePrecision\nimport logging\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Cấu hình logging\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n\n# Định nghĩa đường dẫn\nDIR_INPUT = '/kaggle/input/vinbigdata-chest-xray-original-png'\nDIR_TRAIN = f'{DIR_INPUT}/train'\nDIR_TEST = f'{DIR_INPUT}/test'\nCSV_PATH = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Hàm chuyển class_id thành tên\ndef label_to_name(class_id):\n    class_id = int(class_id) - 1\n    names = [\"Aortic enlargement\", \"Atelectasis\", \"Calcification\", \"Cardiomegaly\", \"Consolidation\",\n             \"ILD\", \"Infiltration\", \"Lung Opacity\", \"Nodule/Mass\", \"Other lesion\",\n             \"Pleural effusion\", \"Pleural thickening\", \"Pneumothorax\", \"Pulmonary fibrosis\"]\n    return names[class_id] if 0 <= class_id < 14 else \"No finding\"\n\n# --- EDA ---\ndef perform_eda(train_df, image_dir):\n    logging.info(\"Starting EDA\")\n    print(\"Kích thước dataframe:\", train_df.shape)\n    print(\"Số lượng ảnh duy nhất:\", train_df['image_id'].nunique())\n    print(\"Số lượng class:\", train_df['class_id'].nunique())\n    \n    plt.figure(figsize=(12, 6))\n    sns.countplot(data=train_df, x='class_id', order=range(15))\n    plt.title(\"Phân bố class\")\n    plt.xticks(ticks=range(15), labels=[label_to_name(i+1) for i in range(15)], rotation=90)\n    plt.show()\n    \n    train_df['area'] = (train_df['x_max'] - train_df['x_min']) * (train_df['y_max'] - train_df['y_min'])\n    plt.figure(figsize=(10, 6))\n    sns.histplot(train_df['area'], bins=50, kde=True)\n    plt.title(\"Phân bố diện tích bounding box\")\n    plt.show()\n    \n    sample_id = train_df['image_id'].unique()[0]\n    sample_records = train_df[train_df['image_id'] == sample_id]\n    image = np.array(Image.open(f\"{image_dir}/{sample_id}.png\").convert('L'))\n    plt.figure(figsize=(8, 8))\n    plt.imshow(image, cmap='gray')\n    for _, row in sample_records.iterrows():\n        box = [row['x_min'], row['y_min'], row['x_max'], row['y_max']]\n        plt.gca().add_patch(plt.Rectangle((box[0], box[1]), box[2]-box[0], box[3]-box[1], \n                                          edgecolor='red', facecolor='none', lw=2))\n        plt.text(box[0], box[1], label_to_name(row['class_id']), color='red', fontsize=12)\n    plt.title(f\"Sample: {sample_id}\")\n    plt.axis('off')\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Chuẩn bị dữ liệu ---\ntrain_df = pd.read_csv(CSV_PATH)\ntrain_df.fillna(0, inplace=True)\ntrain_df.loc[train_df[\"class_id\"] == 14, ['x_max', 'y_max']] = 1.0\ntrain_df[\"class_id\"] = train_df[\"class_id\"] + 1\ntrain_df.loc[train_df[\"class_id\"] == 15, [\"class_id\"]] = 0\n\nimage_ids = train_df['image_id'].unique()\nvalid_ids = image_ids[-10000:]\ntrain_ids = image_ids[:-10000]\nvalid_df = train_df[train_df['image_id'].isin(valid_ids)]\ntrain_df = train_df[train_df['image_id'].isin(train_ids)]\n\ntrain_df['area'] = (train_df['x_max'] - train_df['x_min']) * (train_df['y_max'] - train_df['y_min'])\nvalid_df['area'] = (valid_df['x_max'] - valid_df['x_min']) * (valid_df['y_max'] - valid_df['y_min'])\ntrain_df = train_df[train_df['area'] > 1]\nvalid_df = valid_df[valid_df['area'] > 1]\ntrain_df = train_df[(train_df['class_id'] > 0) & (train_df['class_id'] < 15)]\nvalid_df = valid_df[(valid_df['class_id'] > 0) & (valid_df['class_id'] < 15)]\ntrain_df = train_df.drop(['area'], axis=1)\nvalid_df = valid_df.drop(['area'], axis=1)\n\nperform_eda(train_df, DIR_TRAIN)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Dataset ---\nclass VinBigDataset(Dataset):\n    def __init__(self, dataframe, image_dir, transforms=None, mode='train'):\n        self.image_ids = dataframe['image_id'].unique()\n        self.df = dataframe\n        self.image_dir = image_dir\n        self.transforms = transforms\n        self.mode = mode\n\n    def __getitem__(self, index):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_id'] == image_id].reset_index(drop=True)\n        image = np.array(Image.open(f\"{self.image_dir}/{image_id}.png\").convert('RGB')).astype(np.float32)\n\n        if self.mode == 'train':\n            boxes = records[['x_min', 'y_min', 'x_max', 'y_max']].values\n            labels = torch.tensor(records[\"class_id\"].values, dtype=torch.int64)\n            area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n            target = {\n                'boxes': torch.tensor(boxes, dtype=torch.float32),\n                'labels': labels,\n                'image_id': torch.tensor([index]),\n                'area': torch.tensor(area, dtype=torch.float32),\n                'iscrowd': torch.zeros(len(records), dtype=torch.int64)\n            }\n            if self.transforms:\n                sample = self.transforms(image=image, bboxes=target['boxes'], labels=labels)\n                image, target['boxes'] = sample['image'], torch.tensor(sample['bboxes'], dtype=torch.float32)\n            return image, target, image_id\n        else:\n            if self.transforms:\n                sample = self.transforms(image=image)\n                image = sample['image']\n            return image, image_id\n\n    def __len__(self):\n        return len(self.image_ids)\n\n# --- Data Augmentation ---\ndef get_train_transform():\n    return A.Compose([\n        A.Flip(p=0.5),\n        A.ShiftScaleRotate(scale_limit=0.1, rotate_limit=45, p=0.25),\n        A.LongestMaxSize(max_size=800, p=1.0),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_test_transform():\n    return A.Compose([\n        A.LongestMaxSize(max_size=800, p=1.0),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2(p=1.0)\n    ])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- DataLoader ---\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\ntrain_dataset = VinBigDataset(train_df, DIR_TRAIN, get_train_transform(), mode='train')\nvalid_dataset = VinBigDataset(valid_df, DIR_TRAIN, get_valid_test_transform(), mode='train')\ntest_df = pd.read_csv('/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/sample_submission.csv')\ntest_dataset = VinBigDataset(test_df, DIR_TEST, get_valid_test_transform(), mode='test')\n\ndef collate_fn(batch):\n    return tuple(zip(*batch))\n\n# Tăng batch_size cho 2 GPU (16 thay vì 8)\ntrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=2, collate_fn=collate_fn)\nvalid_loader = DataLoader(valid_dataset, batch_size=16, shuffle=False, num_workers=2, collate_fn=collate_fn)\ntest_loader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=2, collate_fn=collate_fn)\n\n# --- Model ---\nmodel = fasterrcnn_resnet50_fpn(pretrained=True, min_size=600)\nnum_classes = 15\nin_features = model.roi_heads.box_predictor.cls_score.in_features\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)\n\n# Sử dụng DataParallel để chạy trên 2 GPU\nif torch.cuda.device_count() > 1:\n    logging.info(f\"Using {torch.cuda.device_count()} GPUs!\")\n    model = nn.DataParallel(model)\nmodel.to(device)\n\n# --- Training ---\noptimizer = torch.optim.SGD(model.parameters(), lr=0.005, momentum=0.9, weight_decay=0.0005)\nlr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=2)\nscaler = GradScaler()\nnum_epochs = 1\nbest_map = 0.0\n\ndef train_epoch(model, loader, device, optimizer, scaler):\n    model.train()\n    total_loss = 0\n    for images, targets, _ in tqdm(loader, desc=\"Training\"):\n        images = list(img.to(device) for img in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        optimizer.zero_grad()\n        with autocast():\n            loss_dict = model(images, targets)\n            losses = sum(loss for loss in loss_dict.values())\n        scaler.scale(losses).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        total_loss += losses.item()\n    return total_loss / len(loader)\n\ndef evaluate(model, loader, device):\n    model.eval()\n    total_loss = 0\n    metric = MeanAveragePrecision()\n    with torch.no_grad():\n        for images, targets, _ in tqdm(loader, desc=\"Evaluating\"):\n            images = list(img.to(device) for img in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n            loss_dict = model(images, targets)\n            losses = sum(loss for loss in loss_dict.values())\n            total_loss += losses.item()\n            outputs = model(images)\n            metric.update([dict(boxes=o['boxes'].cpu(), scores=o['scores'].cpu(), labels=o['labels'].cpu()) for o in outputs],\n                          [dict(boxes=t['boxes'].cpu(), labels=t['labels'].cpu()) for t in targets])\n    map_score = metric.compute()['map'].item()\n    return total_loss / len(loader), map_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Huấn luyện\nfor epoch in range(num_epochs):\n    train_loss = train_epoch(model, train_loader, device, optimizer, scaler)\n    valid_loss, valid_map = evaluate(model, valid_loader, device)\n    lr_scheduler.step(valid_loss)\n    logging.info(f\"Epoch {epoch+1}/{num_epochs} - Train Loss: {train_loss:.4f}, Valid Loss: {valid_loss:.4f}, Valid mAP: {valid_map:.4f}\")\n    if valid_map > best_map:\n        best_map = valid_map\n        torch.save(model.module.state_dict(), 'faster_rcnn_best.pth')  # Lưu state_dict của model gốc\n        logging.info(\"Saved best model\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Visualization ---\ndef plot_image_with_boxes(img, boxes, labels, title=\"\"):\n    img = img.permute(1, 2, 0).cpu().numpy()\n    img = (img * np.array([0.229, 0.224, 0.225]) + np.array([0.485, 0.456, 0.406])) * 255\n    img = img.astype(np.uint8)\n    fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n    for box, label in zip(boxes, labels):\n        box = box.cpu().numpy().astype(np.int32)\n        cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (255, 0, 0), 2)\n        cv2.putText(img, label_to_name(label), (box[0], box[1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 0, 0), 2)\n    ax.imshow(img)\n    ax.set_title(title)\n    ax.axis('off')\n    plt.show()\n\nimages, targets, image_ids = next(iter(train_loader))\nimages = list(img.to(device) for img in images)\ntargets = [{k: v.to(device) for k, v in t.items()} for t in targets]\nfor i in range(3):\n    plot_image_with_boxes(images[i], targets[i]['boxes'], targets[i]['labels'], f\"Train Sample {i+1}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Inference và Submission ---\ndef format_prediction_string(labels, boxes, scores):\n    pred_strings = []\n    for label, score, box in zip(labels, scores, boxes):\n        pred_strings.append(f\"{label} {score:.4f} {box[0]} {box[1]} {box[2]} {box[3]}\")\n    return \" \".join(pred_strings)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_state_dict(torch.load('faster_rcnn_best.pth'))  # Tải lại mô hình gốc\nmodel = nn.DataParallel(model)  # Bọc lại DataParallel cho inference\nmodel.to(device)\nmodel.eval()\nresults = []\nwith torch.no_grad():\n    for images, image_ids in tqdm(test_loader, desc=\"Predicting\"):\n        images = list(img.to(device) for img in images)\n        outputs = model(images)\n        for img_id, output in zip(image_ids, outputs):\n            scores = output['scores'].cpu().numpy()\n            labels = output['labels'].cpu().numpy()\n            boxes = output['boxes'].cpu().numpy()\n            mask = scores > 0.5\n            pred_string = format_prediction_string(labels[mask], boxes[mask], scores[mask])\n            results.append({'image_id': img_id, 'PredictionString': pred_string})\n\nsubmission_df = pd.DataFrame(results)\nsubmission_df.to_csv('submission.csv', index=False)\nlogging.info(\"Submission file created: submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}