{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm\n!pip install -q albumentations==1.2.1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T15:18:32.909653Z","iopub.execute_input":"2023-02-12T15:18:32.910075Z","iopub.status.idle":"2023-02-12T15:18:56.240237Z","shell.execute_reply.started":"2023-02-12T15:18:32.909962Z","shell.execute_reply":"2023-02-12T15:18:56.238948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader,Dataset\nfrom sklearn.metrics import roc_auc_score\nimport timm\nimport os\nimport tqdm\nfrom matplotlib.pyplot import figure\nimport cv2\nfrom torch.cuda.amp import GradScaler, autocast\nimport torch\nimport albumentations as A\nfrom torch.utils.tensorboard import SummaryWriter\nimport torch.nn as nn\nimport gc\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:18:56.242933Z","iopub.execute_input":"2023-02-12T15:18:56.244701Z","iopub.status.idle":"2023-02-12T15:19:01.241132Z","shell.execute_reply.started":"2023-02-12T15:18:56.244655Z","shell.execute_reply":"2023-02-12T15:19:01.24002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Output","metadata":{}},{"cell_type":"code","source":"OUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:23:39.893947Z","iopub.execute_input":"2023-02-12T15:23:39.89458Z","iopub.status.idle":"2023-02-12T15:23:39.899616Z","shell.execute_reply.started":"2023-02-12T15:23:39.89454Z","shell.execute_reply":"2023-02-12T15:23:39.898624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class cfg:\n    img_size=512\n#     criterion='LabelSmoothing'#'FocalLoss'\n    smoothing=0.05\n    target_size=1\n    batch_size=8\n    weight_decay=1e-06\n    lr=1e-04\n    backbone='convnext_large'#'seresnext50_32x4d'    'vit_base_patch16_384'\n    device='cuda' if torch.cuda.is_available() else 'cpu'\n    img_path='/kaggle/input/rsna-bcd-roi-1024x-png-dataset/train_images'\n    OUTPUT_DIR= './'\n    weights=None\n    epochs=10\n    clf_threshold=0.2","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:48:05.56301Z","iopub.execute_input":"2023-02-12T15:48:05.563409Z","iopub.status.idle":"2023-02-12T15:48:05.569581Z","shell.execute_reply.started":"2023-02-12T15:48:05.563375Z","shell.execute_reply":"2023-02-12T15:48:05.568542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class net(nn.Module):\n    def __init__(self,backbone,out_channel=1,pretrained=True,not_freeze=False):\n        super(net,self).__init__()\n        self.model=timm.create_model(backbone,pretrained=True)\n        if backbone=='seresnext50_32x4d':\n            in_features=self.model.fc.in_features\n            self.model.fc=nn.Linear(in_features,out_channel)\n        elif backbone=='convnext_large':\n            if not_freeze:\n                in_features=self.model.head.fc.in_features\n                self.model.head.fc=nn.Linear(in_features,out_channel)\n        elif backbone=='vit_base_patch16_384':\n            in_features=self.model.head.in_features\n#             self.model.head=nn.Linear(in_features,out_channel)\n    def forward(self,x):\n        x=self.model(x)\n        return x\ndebug_mode=True\nif debug_mode:  \n    model=net(cfg.backbone,pretrained=True)\n#     model = torch.nn.DataParallel(model)\n    model.to(cfg.device)\n    x=torch.rand(1,3,384,384).to(cfg.device)\n    output=model(x)\n    print(output.size())\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:50:30.96497Z","iopub.execute_input":"2023-02-12T15:50:30.965372Z","iopub.status.idle":"2023-02-12T15:50:36.565533Z","shell.execute_reply.started":"2023-02-12T15:50:30.965337Z","shell.execute_reply":"2023-02-12T15:50:36.564462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Call dataset","metadata":{}},{"cell_type":"code","source":"class RSNADataset(Dataset):\n    def __init__(self, df, img_path,choose_data,transform):\n        self.df = df\n        self.img_path = img_path\n        self.choose_data=choose_data\n        self.transform=transform\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        patient_id, image_id, cancer = self.df.iloc[idx][['patient_id', 'image_id', 'cancer']]\n        if self.choose_data==1:\n            file = os.path.join(self.img_path, f'{patient_id}_{image_id}.png')\n            file = os.path.join(self.img_path, f'{patient_id}/{image_id}.png')\n        elif self.choose_data==2:\n        # dataset 2 :\n            file=f'/kaggle/input/rsna-cut-off-empty-space-from-images/{patient_id}/{image_id}.png'\n        #data set 3\n        elif self.choose_data==3:\n            file=f'/kaggle/input/rsna-bcd-roi-1024x-png-dataset/train_images/{patient_id}/{image_id}.png'\n                \n        file = cv2.imread(file, cv2.COLOR_BGR2GRAY)\n#         image=cv2.resize(file,(512,512))\n        image=cv2.resize(file,(1024,512))\n\n#         file=cv2.resize(file,(1024,512))\n        if self.transform=='train':\n            image = self.transform(image=image)['image']\n        file = image.astype(np.float32) / 255 #.transpose(2, 0, 1)\n        file=torch.from_numpy(file)\n        X=torch.stack([file,file,file],dim=0).float()\n#         X = torch.tensor(file[np.newaxis]).to(device)\n        y = torch.tensor([cancer]).float()\n        return X,y,patient_id\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:44:40.095098Z","iopub.execute_input":"2023-02-12T15:44:40.09611Z","iopub.status.idle":"2023-02-12T15:44:40.107664Z","shell.execute_reply.started":"2023-02-12T15:44:40.096071Z","shell.execute_reply":"2023-02-12T15:44:40.106561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# augmentation","metadata":{}},{"cell_type":"code","source":"transforms_train = A.Compose([\n#     A.Resize(image_size, image_size),\n#     A.HorizontalFlip(p=0.5),\n#     A.VerticalFlip(p=0.5),\n#     A.Transpose(p=0.5),\n    A.RandomBrightness(limit=0.1, p=0.7),\n#     A.ShiftScaleRotate(shift_limit=0.3, scale_limit=0.3, rotate_limit=45, border_mode=4, p=0.7),\n    A.CenterCrop(always_apply=False, p=1.0, height=cfg.img_size, width=cfg.img_size)\n])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:32:46.003257Z","iopub.execute_input":"2023-02-12T15:32:46.003621Z","iopub.status.idle":"2023-02-12T15:32:46.008902Z","shell.execute_reply.started":"2023-02-12T15:32:46.003589Z","shell.execute_reply":"2023-02-12T15:32:46.007879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_trainloader(df_train):\n    train_dataset = RSNADataset(df_train,cfg.img_path,choose_data=3,transform=transforms_train)\n    train_dataloader = DataLoader(train_dataset,cfg.batch_size,shuffle=True)\n    return train_dataloader\ndef get_validloader(df_valid):\n    val_dataset = RSNADataset(df_valid,cfg.img_path,choose_data=3,transform=None)\n    val_dataloader = DataLoader(val_dataset,cfg.batch_size,shuffle=False)\n    return val_dataloader\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:32:48.820513Z","iopub.execute_input":"2023-02-12T15:32:48.820882Z","iopub.status.idle":"2023-02-12T15:32:48.827061Z","shell.execute_reply.started":"2023-02-12T15:32:48.82085Z","shell.execute_reply":"2023-02-12T15:32:48.826008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug_data=False\npreprocess=True\nif preprocess:\n    import pandas as pd\n    from sklearn.model_selection import train_test_split\n    df=pd.read_csv('/kaggle/input/rsna-bcd-roi-1024x-png-dataset/train.csv')\n    counts = df['cancer'].value_counts()\n    df['weights'] = df['cancer'].apply(lambda x: 1/counts[x])\n    train_df, val_df = train_test_split(df, test_size=0.25, stratify=df['cancer'])\nif debug_data:\n    val_=val_df.iloc[:10]\n    train_=train_df.iloc[:10]\n    train_.shape,val_.shape\n    train_dataloader=get_trainloader(train_)\n    val_dataloader=get_validloader(val_)\n    print(\"train: \", len(train_dataloader), \" val: \", len(val_dataloader))    \nelse:\n    df_val=val_df.iloc[:600]\n    df_train=train_df.iloc[:600]\n    df_train.shape,df_val.shape\n    train_dataloader=get_trainloader(df_train)\n    val_dataloader=get_validloader(df_val)\n    print(\"train: \", len(train_dataloader), \" val: \", len(val_dataloader))  \n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:32:51.730075Z","iopub.execute_input":"2023-02-12T15:32:51.730796Z","iopub.status.idle":"2023-02-12T15:32:52.126815Z","shell.execute_reply.started":"2023-02-12T15:32:51.730756Z","shell.execute_reply":"2023-02-12T15:32:52.12562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train and test function","metadata":{}},{"cell_type":"code","source":"def run_train(\n    model,\n    train_dataloader,\n    optimizer,\n    scheduler,\n    scaler,\n    writer,\n    epoch,\n    iteration,\n    step,\n    loss_function):\n    \n    model.train()\n    losses = []\n  \n    all_outputs, all_labels = [], []\n    for img,label,ids in train_dataloader:\n        inputs, labels = img.to(cfg.device), label.to(cfg.device)\n        iteration += 1\n\n        step += cfg.batch_size\n        torch.set_grad_enabled(True)\n        with autocast():\n            outputs = model(inputs)\n#             loss = loss_function(outputs, labels)\n            loss=bce_focal(outputs,labels,gamma=2)\n        losses.append(loss.item())\n\n        outputs = torch.sigmoid(outputs)\n        labels = labels\n\n#         all_outputs.extend(outputs)\n#         all_labels.extend(labels)\n        all_outputs.append(outputs)\n        all_labels.append(labels)\n           \n\n        scaler.scale(loss).backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 10.0)\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n        scheduler.step()\n        print(f\"loss: {np.mean(losses)} lr: {scheduler.get_last_lr()}\")\n        \n    all_labels= torch.cat(all_labels).squeeze().detach().cpu().numpy().astype(np.int8)\n    all_outputs= torch.cat(all_outputs).squeeze().detach().cpu().numpy()\n    score = pfbeta(all_labels.astype(np.float32), all_outputs, 0.5)\n    print(f'label{all_labels} and prediction{all_outputs}')\n#     print(f'label{all_labels} and prediction{all_outputs}')\n#     auc = roc_auc_score(all_labels, all_outputs)\n#     all_labels= torch.cat(all_labels).squeeze().detach().cpu().numpy()\n#     all_outputs= torch.cat(all_outputs).squeeze().detach().cpu().numpy().astype(np.int8)\n#     print(f'label{all_labels} and prediction{all_outputs}')\n    try:\n        auc = roc_auc_score(all_labels,all_outputs)\n        print(\"Train F1: \", score, \"AUC: \", auc)\n\n    except ValueError:\n        pass\n    print(\"Train F1: \", score)\n    return np.mean(losses)\n\n    \ndef run_eval(model, val_dataloader, writer, epoch, loss_function):\n\n    model.eval()\n#     torch.set_grad_enabled(False)\n    tr_it = iter(val_dataloader)\n    all_labels = []\n    all_outputs = []\n    all_ids = []\n    val_loss=[]\n    with torch.no_grad():\n        for img,label,ids in val_dataloader:\n            inputs, labels = img.to(cfg.device), label.to(cfg.device)\n            outputs = model(inputs)\n            outputs = torch.sigmoid(outputs)\n            labels = labels\n#             loss = loss_function(outputs, labels)\n            loss=bce_focal(outputs,labels,gamma=2)\n            val_loss.append(loss.item())\n            print(f\"val_loss: {np.mean(val_loss)}\")\n            all_labels.append(labels)\n            all_outputs.append(outputs)\n\n        all_labels= torch.cat(all_labels).squeeze().detach().cpu().numpy().astype(np.int8)\n        all_outputs= torch.cat(all_outputs).squeeze().detach().cpu().numpy()\n        print(f'label{all_labels} and prediction{all_outputs}')\n    \n        score = pfbeta(all_labels.astype(np.float32), all_outputs, 0.5)\n    try:\n        auc = roc_auc_score(all_labels, all_outputs)\n        print(\"Val F1: \", score, \"AUC: \", auc)\n    except ValueError:\n        pass\n        \n    all_outputs = (np.array(all_outputs) > cfg.clf_threshold).astype(np.int8).tolist()\n    try:\n        bin_score = pfbeta(all_labels, all_outputs, 0.5)\n        print(\"Val F1: \", score, \"Val Bin F1: \", bin_score, \"AUC: \", auc)\n\n    except:\n        bin_score = 0.0\n#     print(\"Val F1: \", score, \"Val Bin F1: \", bin_score)#, \"AUC: \", auc)\n    writer.add_scalar(\"F1\", bin_score, epoch)\n    return np.mean(val_loss),score","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:31.369379Z","iopub.execute_input":"2023-02-12T15:36:31.369746Z","iopub.status.idle":"2023-02-12T15:36:31.387807Z","shell.execute_reply.started":"2023-02-12T15:36:31.369713Z","shell.execute_reply":"2023-02-12T15:36:31.386492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metric","metadata":{}},{"cell_type":"code","source":"# # numpy\n# def pfbeta(labels, preds, beta=1):\n#     eps = 1e-5\n#     preds = preds.clip(0, 1)\n#     y_true_count = labels.sum()\n#     ctp = preds[labels==1].sum()\n#     cfp = preds[labels==0].sum()\n#     beta_squared = beta * beta\n#     c_precision = ctp / (ctp + cfp + eps)\n#     c_recall = ctp / (y_true_count + eps)\n#     if (c_precision > 0 and c_recall > 0):\n#         result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + eps)\n#         return result\n#     else:\n#         return 0.0\ndef pfbeta(labels, predictions, beta):\n    eps=1e-05\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n#         print(f'predict ne {prediction}---label từng id ne {labels[idx]}')\n        \n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n#             print(f'true possitive :{ctp}')\n        else:\n            cfp += prediction\n#             print(f'false possitive:{ctp}')\n\n    beta_squared = beta * beta\n    if ctp + cfp == 0:\n        return 0\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / (y_true_count+eps)\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:26.401639Z","iopub.execute_input":"2023-02-12T15:36:26.402032Z","iopub.status.idle":"2023-02-12T15:36:26.41197Z","shell.execute_reply.started":"2023-02-12T15:36:26.401999Z","shell.execute_reply":"2023-02-12T15:36:26.410341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://github.com/ga642381/ML2021-Spring/blob/main/HW01/HW01.ipynb    \ndef plot_learning_curve(loss_record, title=''):\n    ''' Plot learning curve of your DNN (train & dev loss) '''\n    total_steps = len(loss_record['train'])\n    x_1 = range(total_steps)\n    x_2 = x_1[::len(loss_record['train']) // len(loss_record['valid'])]\n    figure(figsize=(6, 4))\n    plt.plot(x_1, loss_record['train'], c='tab:red', label='train')\n    plt.plot(x_2, loss_record['valid'], c='tab:cyan', label='valid')\n    plt.ylim(0.0, 1.)\n    plt.xlabel('Training steps')\n    plt.ylabel('BCE loss')\n    plt.title('Learning curve of {}'.format(title))\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:23.792381Z","iopub.execute_input":"2023-02-12T15:36:23.792738Z","iopub.status.idle":"2023-02-12T15:36:23.800067Z","shell.execute_reply.started":"2023-02-12T15:36:23.792707Z","shell.execute_reply":"2023-02-12T15:36:23.798911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop","metadata":{}},{"cell_type":"code","source":"def bce_focal(preds, targets,gamma=2):\n    criterion = nn.BCEWithLogitsLoss(reduction='none')\n    bce_loss = criterion(preds, targets)\n    probas = torch.sigmoid(preds)\n    loss = torch.where(targets <= 0.5, (1. - probas)**gamma * bce_loss, probas**gamma * bce_loss)\n    loss = loss.mean()\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:21.781685Z","iopub.execute_input":"2023-02-12T15:36:21.782086Z","iopub.status.idle":"2023-02-12T15:36:21.7888Z","shell.execute_reply.started":"2023-02-12T15:36:21.782055Z","shell.execute_reply":"2023-02-12T15:36:21.787584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_checkpoint(model, optimizer, epoch, scheduler=None, scaler=None):\n    checkpoint = {\n        \"model\": model.state_dict(),\n        \"optimizer\": optimizer.state_dict(),\n        \"epoch\": epoch,\n    }\n    if scheduler is not None:\n        checkpoint[\"scheduler\"] = scheduler.state_dict()\n\n    if scaler is not None:\n        checkpoint[\"scaler\"] = scaler.state_dict()\n    return checkpoint","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:36.406517Z","iopub.execute_input":"2023-02-12T15:36:36.406902Z","iopub.status.idle":"2023-02-12T15:36:36.413686Z","shell.execute_reply.started":"2023-02-12T15:36:36.40687Z","shell.execute_reply":"2023-02-12T15:36:36.412363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = torch.cuda.amp.GradScaler()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:36:38.378313Z","iopub.execute_input":"2023-02-12T15:36:38.378687Z","iopub.status.idle":"2023-02-12T15:36:38.384158Z","shell.execute_reply.started":"2023-02-12T15:36:38.378655Z","shell.execute_reply":"2023-02-12T15:36:38.382939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=net(cfg.backbone,pretrained=True)\nmodel.to(cfg.device)\nfor name, param in model.named_parameters():\n    if \"fc\" in name: # Example: fc layer in the resnet18 model\n        param.requires_grad = False\nmodel.head.fc=nn.Linear(1536,cfg.target_size)\n# model = torch.nn.DataParallel(model)\nmodel\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:50:04.732281Z","iopub.execute_input":"2023-02-12T15:50:04.732655Z","iopub.status.idle":"2023-02-12T15:50:09.954485Z","shell.execute_reply.started":"2023-02-12T15:50:04.732624Z","shell.execute_reply":"2023-02-12T15:50:09.951963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run():\n    train_dataloader=get_trainloader(df_train)\n    val_dataloader=get_validloader(df_val)\n    loss_record={'train':[],'valid':[]}\n    # set model\n    model=net(cfg.backbone,pretrained=True)\n#     model = torch.nn.DataParallel(model)\n    for param in model.parameters():\n        param.requires_grad = False\n    model.fc=nn.Linear(1000,cfg.target_size)\n    model = torch.nn.DataParallel(model)\n    model.to(cfg.device)\n    \n    if cfg.weights is not None:\n        model.load_state_dict(\n            torch.load(os.path.join(f\"{cfg.OUTPUT_DIR}\", cfg.weights))[\n                \"model\"])\n        print(f\"weights from: {cfg.weights} are loaded.\")\n    # set optimizer, lr scheduler\n    optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n#     scheduler = torch.optim.lr_scheduler.OneCycleLR(\n#         optimizer,\n#         max_lr=cfg.lr,\n#         epochs=cfg.epochs,\n#         steps_per_epoch=int(len(train_dataset) / cfg.batch_size),\n#         pct_start=0.1,\n#         anneal_strategy=\"cos\",\n#         div_factor=cfg.lr_div,\n#         final_div_factor=cfg.lr_final_div,\n#     )\n    lambda1 = lambda epoch : cfg.epochs/10\n    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda1)\n    # set loss\n    loss_function = torch.nn.BCEWithLogitsLoss().to(cfg.device)\n\n    # set other tools\n    scaler\n    writer = SummaryWriter(str(cfg.OUTPUT_DIR + f\"/train/\"))\n\n    # train and val loop\n    step = 0\n    i = 0\n    best_metric = 0.0\n    optimizer.zero_grad()\n    print(\"start from: \", best_metric)\n    for epoch in range(cfg.epochs):\n        print(\"EPOCH:\", epoch)\n        gc.collect()\n        train_loss=run_train(\n            model=model,\n            train_dataloader=train_dataloader,\n            optimizer=optimizer,\n            scheduler=scheduler,\n            scaler=scaler,\n            writer=writer,\n            epoch=epoch,\n            iteration=i,\n            step=step,\n            loss_function=loss_function,\n        )\n\n        val_loss,val_metric = run_eval(\n            model=model,\n            val_dataloader=val_dataloader,\n            writer=writer,\n            epoch=epoch,\n            loss_function=loss_function\n        )\n        loss_record['train'].append(train_loss)\n        loss_record['valid'].append(val_loss)\n\n        if val_metric > best_metric:\n            print(f\"SAVING CHECKPOINT: val_metric {best_metric:.5} -> {val_metric:.5}\")\n            best_metric = val_metric\n            checkpoint = create_checkpoint(\n                model,\n                optimizer,\n                epoch,\n                scheduler=scheduler,\n                scaler=scaler,\n            )\n            torch.save(\n                checkpoint,\n                f\"{OUTPUT_DIR}/checkpoint_best_metric.pth\",\n            )\n        plot_learning_curve(loss_record, title='train/valid curved')\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:44:58.464616Z","iopub.execute_input":"2023-02-12T15:44:58.464978Z","iopub.status.idle":"2023-02-12T15:44:58.479597Z","shell.execute_reply.started":"2023-02-12T15:44:58.464946Z","shell.execute_reply":"2023-02-12T15:44:58.478474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T15:45:02.258746Z","iopub.execute_input":"2023-02-12T15:45:02.259132Z","iopub.status.idle":"2023-02-12T15:45:07.808739Z","shell.execute_reply.started":"2023-02-12T15:45:02.259093Z","shell.execute_reply":"2023-02-12T15:45:07.807162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_true = np.array([0, 0, 1, 1,1])\n# y_scores = np.array([0.1, 0.4, 0.35, 0.85,0.57])\n# roc_auc_score(y_true, y_scores)\n\n# list_pred=[]\n# list_gt=[]\n# y1=torch.rand(8,1)\n# y2=torch.rand(2,1)\n# list_pred.append(y1)\n# list_pred.append(y2)\n\n# y3=torch.tensor([0,1,1,0,1,0,1,0])\n# y4=torch.tensor([0,1])\n# list_gt.append(y3)\n# list_gt.append(y4)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T17:24:31.538858Z","iopub.execute_input":"2023-02-11T17:24:31.539477Z","iopub.status.idle":"2023-02-11T17:24:31.543873Z","shell.execute_reply.started":"2023-02-11T17:24:31.539438Z","shell.execute_reply":"2023-02-11T17:24:31.542952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_true=torch.cat(list_gt).squeeze().detach().numpy()\n# y_pred=torch.cat(list_pred).squeeze().detach().numpy()\n# # print(np.array(y_true))\n# auc = roc_auc_score(y_true,y_pred)\n# print(auc)\n# y_true,y_pred\n# list_pred,list_gt","metadata":{"execution":{"iopub.status.busy":"2023-02-11T17:24:31.545323Z","iopub.execute_input":"2023-02-11T17:24:31.545662Z","iopub.status.idle":"2023-02-11T17:24:31.574602Z","shell.execute_reply.started":"2023-02-11T17:24:31.545628Z","shell.execute_reply":"2023-02-11T17:24:31.573736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_gt=list()\n# all_pred=list()\n# prediction1=np.array([0.5],dtype=np.float32)\n# prediction2=np.array([0.6],dtype=np.float32)\n# gt=np.array([0],dtype=np.float32)\n# gt1=np.array([1],dtype=np.float32)\n# all_gt.append(gt)\n# all_gt.append(gt1)\n\n# all_pred.append(prediction1)\n# all_pred.append(prediction2)\n# # all_pred=np.concatenate(all_pred,0)\n# # all_gt=np.concatenate(all_gt,0)\n\n# auc = roc_auc_score(np.array(all_gt).ravel(),np.array(all_pred).ravel())\n# print(auc)\n# print(np.array(all_gt).ravel())\n# print(np.array(all_pred).ravel())\n","metadata":{"execution":{"iopub.status.busy":"2023-02-11T17:24:31.575927Z","iopub.execute_input":"2023-02-11T17:24:31.57628Z","iopub.status.idle":"2023-02-11T17:24:31.594805Z","shell.execute_reply.started":"2023-02-11T17:24:31.576231Z","shell.execute_reply":"2023-02-11T17:24:31.593866Z"},"trusted":true},"execution_count":null,"outputs":[]}]}