{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":52254,"databundleVersionId":9674523,"sourceType":"competition"},{"sourceId":6211844,"sourceType":"datasetVersion","datasetId":3567114}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hi I tried convert Keras team's code to Pytorch including all training pipeline as well.","metadata":{}},{"cell_type":"code","source":"# Cài đặt thư viện iterative-stratification\n!pip install iterative-stratification\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:13:23.235751Z","iopub.execute_input":"2024-10-05T05:13:23.236639Z","iopub.status.idle":"2024-10-05T05:15:33.016532Z","shell.execute_reply.started":"2024-10-05T05:13:23.236583Z","shell.execute_reply":"2024-10-05T05:15:33.015474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install iterstrat==0.2.0\n#pip install --upgrade pip\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:15:33.018677Z","iopub.execute_input":"2024-10-05T05:15:33.019018Z","iopub.status.idle":"2024-10-05T05:17:42.467971Z","shell.execute_reply.started":"2024-10-05T05:15:33.018982Z","shell.execute_reply":"2024-10-05T05:17:42.467038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom PIL import Image\nimport torch.optim as optim\nfrom torch import nn\n\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score, accuracy_score","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:42.469499Z","iopub.execute_input":"2024-10-05T05:17:42.469894Z","iopub.status.idle":"2024-10-05T05:17:48.364353Z","shell.execute_reply.started":"2024-10-05T05:17:42.469857Z","shell.execute_reply":"2024-10-05T05:17:48.363547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    SEED = 42\n    IMAGE_SIZE = [256, 256]\n    BATCH_SIZE = 32\n    EPOCHS = 200\n    TARGET_COLS  = [\n        \"bowel_injury\", \"extravasation_injury\",\n        \"kidney_healthy\", \"kidney_low\", \"kidney_high\",\n        \"liver_healthy\", \"liver_low\", \"liver_high\",\n        \"spleen_healthy\", \"spleen_low\", \"spleen_high\",\n    ]\n\nconfig = Config()\nprint(f\"Số lượng nhãn mục tiêu: {len(Config.TARGET_COLS)}\")\ntorch.manual_seed(Config.SEED)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.367186Z","iopub.execute_input":"2024-10-05T05:17:48.367994Z","iopub.status.idle":"2024-10-05T05:17:48.384273Z","shell.execute_reply.started":"2024-10-05T05:17:48.367947Z","shell.execute_reply":"2024-10-05T05:17:48.383413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"# Đường dẫn cơ bản đến dữ liệu\nBASE_PATH = f\"/kaggle/input/rsna-atd-512x512-png-v2-dataset\"\n\n# Tải dữ liệu train\ndataframe = pd.read_csv(f\"{BASE_PATH}/train.csv\")\ndataframe[\"image_path\"] = (\n    f\"{BASE_PATH}/train_images/\"\n    + dataframe.patient_id.astype(str)\n    + \"/\"\n    + dataframe.series_id.astype(str)\n    + \"/\"\n    + dataframe.instance_number.astype(str) + \".png\"\n)\ndataframe = dataframe.drop_duplicates()\n\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-05T05:17:48.385527Z","iopub.execute_input":"2024-10-05T05:17:48.385898Z","iopub.status.idle":"2024-10-05T05:17:48.51327Z","shell.execute_reply.started":"2024-10-05T05:17:48.385858Z","shell.execute_reply":"2024-10-05T05:17:48.512519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_paths= [\"/kaggle/input/rsna-atd-512x512-png-v2-dataset/test_images/48843/62825/30.png\",\n                \"/kaggle/input/rsna-atd-512x512-png-v2-dataset/test_images/50046/24574/30.png\",\n                \"/kaggle/input/rsna-atd-512x512-png-v2-dataset/test_images/63706/39279/30.png\"\n               ]\nid_list = [48843, 50046, 63706]","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.514309Z","iopub.execute_input":"2024-10-05T05:17:48.51459Z","iopub.status.idle":"2024-10-05T05:17:48.519013Z","shell.execute_reply.started":"2024-10-05T05:17:48.514559Z","shell.execute_reply":"2024-10-05T05:17:48.518096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\ndataframe = pd.read_csv(f\"{BASE_PATH}/train.csv\")\ndataframe[\"image_path\"] = f\"{BASE_PATH}/train_images\"\\\n                    + \"/\" + dataframe.patient_id.astype(str)\\\n                    + \"/\" + dataframe.series_id.astype(str)\\\n                    + \"/\" + dataframe.instance_number.astype(str) +\".png\"\ndataframe = dataframe.drop_duplicates()\n\ndataframe.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.520352Z","iopub.execute_input":"2024-10-05T05:17:48.520994Z","iopub.status.idle":"2024-10-05T05:17:48.630823Z","shell.execute_reply.started":"2024-10-05T05:17:48.520952Z","shell.execute_reply":"2024-10-05T05:17:48.629953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, paths, labels, transform=None):\n        self.paths = paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, idx):\n        image = Image.open(self.paths[idx]).convert('RGB')\n        label = torch.tensor(self.labels[idx], dtype=torch.float32)\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, label\n\n# Định nghĩa các chuyển đổi ảnh, bao gồm augmentation\ntransform = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.RandomResizedCrop(256),   # Cắt ngẫu nhiên và thay đổi kích thước\n    transforms.RandomHorizontalFlip(),    # Lật ngang ngẫu nhiên\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1),  # Biến đổi màu sắc\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.632055Z","iopub.execute_input":"2024-10-05T05:17:48.63242Z","iopub.status.idle":"2024-10-05T05:17:48.641652Z","shell.execute_reply.started":"2024-10-05T05:17:48.632369Z","shell.execute_reply":"2024-10-05T05:17:48.640749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SimpleCNN(nn.Module):\n    def __init__(self, num_classes=11):\n        super(SimpleCNN, self).__init__()\n        \n        self.conv_layers = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n        )\n        \n        self.fc_layers = nn.Sequential(\n            nn.Linear(64 * 64 * 64, 128),\n            nn.ReLU(),\n            nn.Linear(128, num_classes)\n        )\n        \n    def forward(self, x):\n        x = self.conv_layers(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc_layers(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.643128Z","iopub.execute_input":"2024-10-05T05:17:48.643538Z","iopub.status.idle":"2024-10-05T05:17:48.653221Z","shell.execute_reply.started":"2024-10-05T05:17:48.643494Z","shell.execute_reply":"2024-10-05T05:17:48.65237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Số lượng folds\nK = 4  # Bạn có thể chọn 5 hoặc 10 tùy ý\n\n# Khởi tạo K-Fold splitter\nkf = KFold(n_splits=K, shuffle=True, random_state=config.SEED)\n\n# Chuẩn bị dữ liệu\nX = dataframe.image_path.values\ny = dataframe[config.TARGET_COLS].values\n\n# Khởi tạo danh sách để lưu trữ các chỉ số và metric của từng fold\nfold_train_losses = []\nfold_val_losses = []\nfold_val_accuracies = []\nfold_val_f1_scores = []\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.657226Z","iopub.execute_input":"2024-10-05T05:17:48.657541Z","iopub.status.idle":"2024-10-05T05:17:48.668646Z","shell.execute_reply.started":"2024-10-05T05:17:48.657491Z","shell.execute_reply":"2024-10-05T05:17:48.66793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Chia dữ liệu cho từng fold\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n    X_train, X_val = X[train_idx], X[val_idx]\n    y_train, y_val = y[train_idx], y[val_idx]\n\n    # Tạo dataset và dataloader cho fold hiện tại\n    train_dataset = CustomDataset(paths=X_train, labels=y_train, transform=transform)\n    val_dataset = CustomDataset(paths=X_val, labels=y_val, transform=transform)\n\n    train_dataloader = DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=4, pin_memory=True)\n    val_dataloader = DataLoader(val_dataset, batch_size=16, shuffle=False, num_workers=4, pin_memory=True)\n\n    # Huấn luyện và đánh giá cho fold hiện tại\n    for epoch in range(config.EPOCHS):\n        # Vòng lặp huấn luyện và validation tương tự như trước\n        ...\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.669661Z","iopub.execute_input":"2024-10-05T05:17:48.669994Z","iopub.status.idle":"2024-10-05T05:17:48.685759Z","shell.execute_reply.started":"2024-10-05T05:17:48.669962Z","shell.execute_reply":"2024-10-05T05:17:48.684959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Kiểm tra nếu có GPU thì sử dụng, nếu không sẽ sử dụng CPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.686839Z","iopub.execute_input":"2024-10-05T05:17:48.687827Z","iopub.status.idle":"2024-10-05T05:17:48.753321Z","shell.execute_reply.started":"2024-10-05T05:17:48.687783Z","shell.execute_reply":"2024-10-05T05:17:48.752404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SimpleCNN(num_classes=len(config.TARGET_COLS)).to(device)\ncriterion = nn.BCEWithLogitsLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config.EPOCHS)\nscaler = torch.amp.GradScaler(\"cuda\")  # Sử dụng cho mixed precision\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:48.754871Z","iopub.execute_input":"2024-10-05T05:17:48.755264Z","iopub.status.idle":"2024-10-05T05:17:49.304953Z","shell.execute_reply.started":"2024-10-05T05:17:48.755221Z","shell.execute_reply":"2024-10-05T05:17:49.303969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accumulation_steps = 4  # ví dụ: cập nhật sau mỗi 4 batch\nfor i, (images, labels) in enumerate(train_dataloader):\n    images, labels = images.to(device), labels.to(device)\n    with torch.amp.autocast(device_type=device.type):\n        outputs = model(images)\n        loss = criterion(outputs, labels) / accumulation_steps\n    scaler.scale(loss).backward()\n\n    if (i + 1) % accumulation_steps == 0:\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()  # reset lại gradient\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:17:49.306185Z","iopub.execute_input":"2024-10-05T05:17:49.306502Z","iopub.status.idle":"2024-10-05T05:18:46.501688Z","shell.execute_reply.started":"2024-10-05T05:17:49.306468Z","shell.execute_reply":"2024-10-05T05:18:46.500638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.utils.checkpoint as checkpoint\n\ndef forward(self, x):\n    x = checkpoint.checkpoint(self.conv_layers, x)\n    x = x.view(x.size(0), -1)\n    x = checkpoint.checkpoint(self.fc_layers, x)\n    return x\nif (epoch + 1) % 10 == 0:  # Lưu sau mỗi 10 epoch\n    torch.save(model.state_dict(), f\"model_checkpoint_epoch_{epoch+1}.pth\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:18:46.503124Z","iopub.execute_input":"2024-10-05T05:18:46.503437Z","iopub.status.idle":"2024-10-05T05:18:46.770158Z","shell.execute_reply.started":"2024-10-05T05:18:46.503393Z","shell.execute_reply":"2024-10-05T05:18:46.769144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# torch dataloader with augmentation","metadata":{}},{"cell_type":"code","source":"# ...\n\n# Vòng lặp huấn luyện cho mỗi fold\nfor epoch in range(config.EPOCHS):\n    model.train()\n    running_loss = 0.0\n    optimizer.zero_grad()  # Reset optimizer\n\n    # Training loop\n    for i, (images, labels) in enumerate(train_dataloader):\n        images, labels = images.to(device), labels.to(device)\n\n        # Sử dụng cú pháp mới của autocast\n        with torch.amp.autocast(device_type=device.type):\n            outputs = model(images)\n            loss = criterion(outputs, labels) / accumulation_steps  # Chia loss theo accumulation steps\n        \n        scaler.scale(loss).backward()  # Tích lũy gradient\n        \n        # Thực hiện bước optimizer mỗi accumulation_steps batch\n        if (i + 1) % accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()  # Reset lại gradient\n\n        running_loss += loss.item() * images.size(0)  # Không nhân với accumulation_steps ở đây\n\n    # Lưu lại train loss\n    epoch_train_loss = running_loss / len(train_dataloader.dataset)\n    fold_train_losses.append(epoch_train_loss)\n\n    # Cập nhật learning rate\n    scheduler.step()\n\n    # Validation loop\n    model.eval()\n    val_loss, correct, total = 0.0, 0, 0\n    all_preds, all_targets = [], []\n\n    with torch.no_grad():\n        for images, labels in val_dataloader:\n            images, labels = images.to(device), labels.to(device)\n\n            # Sử dụng cú pháp mới của autocast trong validation\n            with torch.amp.autocast(device_type=device.type):\n                outputs = model(images)\n                loss = criterion(outputs, labels)\n\n            val_loss += loss.item() * images.size(0)\n\n            preds = torch.sigmoid(outputs) > 0.5\n            correct += (preds.int() == labels.int()).sum().item()\n            total += labels.numel()\n\n            all_preds.append(preds.cpu().numpy())\n            all_targets.append(labels.cpu().numpy())\n\n    # Lưu lại validation loss và accuracy\n    epoch_val_loss = val_loss / len(val_dataloader.dataset)\n    fold_val_losses.append(epoch_val_loss)\n\n    epoch_val_accuracy = 100.0 * correct / total\n    fold_val_accuracies.append(epoch_val_accuracy)\n\n    # Tính F1 score\n    all_preds_np = np.vstack(all_preds)\n    all_targets_np = np.vstack(all_targets)\n    epoch_f1 = f1_score(all_targets_np, all_preds_np, average='macro')\n    fold_val_f1_scores.append(epoch_f1)\n\n    print(f\"Epoch [{epoch + 1}/{config.EPOCHS}] - Train Loss: {epoch_train_loss:.4f} - Val Loss: {epoch_val_loss:.4f} - Val Acc: {epoch_val_accuracy:.2f}% - Val F1: {epoch_f1:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T05:18:46.771517Z","iopub.execute_input":"2024-10-05T05:18:46.771843Z","iopub.status.idle":"2024-10-05T08:47:02.059655Z","shell.execute_reply.started":"2024-10-05T05:18:46.77181Z","shell.execute_reply":"2024-10-05T08:47:02.058483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lưu checkpoint mỗi fold\ntorch.save(model.state_dict(), f\"model_fold_{fold}_final.pth\")\ntorch.save(model.state_dict(), f\"model_fold_{fold}_epoch_{epoch}.pth\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T08:47:02.06119Z","iopub.execute_input":"2024-10-05T08:47:02.061538Z","iopub.status.idle":"2024-10-05T08:47:02.589018Z","shell.execute_reply.started":"2024-10-05T08:47:02.061502Z","shell.execute_reply":"2024-10-05T08:47:02.588221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tính trung bình các metric qua các fold\navg_train_loss = np.mean([np.mean(fold_train_loss) for fold_train_loss in fold_train_losses])\navg_val_loss = np.mean([np.mean(fold_val_loss) for fold_val_loss in fold_val_losses])\navg_val_accuracy = np.mean([np.mean(fold_val_acc) for fold_val_acc in fold_val_accuracies])\navg_val_f1_score = np.mean([np.mean(fold_val_f1) for fold_val_f1 in fold_val_f1_scores])\n\nprint(f\"Average Train Loss: {avg_train_loss:.4f}\")\nprint(f\"Average Val Loss: {avg_val_loss:.4f}\")\nprint(f\"Average Val Accuracy: {avg_val_accuracy:.2f}%\")\nprint(f\"Average Val F1 Score: {avg_val_f1_score:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T08:47:02.590114Z","iopub.execute_input":"2024-10-05T08:47:02.590412Z","iopub.status.idle":"2024-10-05T08:47:02.606579Z","shell.execute_reply.started":"2024-10-05T08:47:02.590379Z","shell.execute_reply":"2024-10-05T08:47:02.605747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting Dataframe to dataloader","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Số epoch\nepochs = range(1, config.EPOCHS + 1)\n\n# Vẽ đồ thị accuracy qua các epoch\nplt.plot(epochs, fold_val_accuracies, label='Validation Accuracy')\n\n# Tùy chỉnh đồ thị\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy (%)')\nplt.title('Validation Accuracy over Epochs')\nplt.legend()\n\n# Hiển thị đồ thị\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T08:47:02.60775Z","iopub.execute_input":"2024-10-05T08:47:02.608064Z","iopub.status.idle":"2024-10-05T08:47:02.927722Z","shell.execute_reply.started":"2024-10-05T08:47:02.608033Z","shell.execute_reply":"2024-10-05T08:47:02.926864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Giả sử bạn muốn lưu trữ mô hình có F1 Score tốt nhất trong mỗi fold\nfor fold in range(K):\n    best_epoch = np.argmax(fold_val_f1_scores[fold])\n    best_model_path = f\"best_model_fold_{fold+1}_epoch_{best_epoch+1}.pth\"\n    torch.save(model.state_dict(), best_model_path)\n    print(f\"Đã lưu mô hình tốt nhất cho Fold {fold+1} tại epoch {best_epoch+1}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-05T08:47:02.928939Z","iopub.execute_input":"2024-10-05T08:47:02.929306Z","iopub.status.idle":"2024-10-05T08:47:03.970064Z","shell.execute_reply.started":"2024-10-05T08:47:02.929265Z","shell.execute_reply":"2024-10-05T08:47:03.969182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv(\"/kaggle/input/rsna-atd-512x512-png-v2-dataset/sample_submission.csv\")\ndf_sub","metadata":{"execution":{"iopub.status.busy":"2024-10-05T08:47:03.971304Z","iopub.execute_input":"2024-10-05T08:47:03.971622Z","iopub.status.idle":"2024-10-05T08:47:03.997236Z","shell.execute_reply.started":"2024-10-05T08:47:03.971587Z","shell.execute_reply":"2024-10-05T08:47:03.996388Z"},"trusted":true},"execution_count":null,"outputs":[]}]}