{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a pytorch training notebook. You can expect 0.25+ pF1 CV and 0.34 pF1 LB.\n\nI've tried models like seresnext50-32x4d and efficientnets. Efficientnets tend to perform much better.\n\nNote that the training time for 1 epoch is >1hr, perhaps someone could look into this and make it faster enough though I'm using the 2 T4.","metadata":{}},{"cell_type":"markdown","source":"Check out my other notebooks:\n\nTrain: https://www.kaggle.com/chaitanyagiri/rsna-breast-cancer-training-with-2-t4\n\nInfer: https://www.kaggle.com/chaitanyagiri/rsna-breast-cancer-inference-with-2-t4","metadata":{}},{"cell_type":"markdown","source":"References:\n\nhttps://www.kaggle.com/code/vslaykovsky/train-pytorch-aux-targets-weighted-loss-thres\n\nhttps://www.kaggle.com/code/theoviel/rsna-breast-baseline-faster-inference-with-dali\n\nhttps://www.kaggle.com/code/vslaykovsky/infer-pytorch-aux-targets-weighted-loss-thres","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport cv2\nimport glob\nimport json\nimport shutil\nimport random\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master/')\nimport timm\nfrom timm.data import create_transform\nfrom timm import create_model, list_models\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.functional import binary_cross_entropy_with_logits, cross_entropy\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:59:10.351999Z","iopub.execute_input":"2022-12-27T21:59:10.352351Z","iopub.status.idle":"2022-12-27T21:59:12.759704Z","shell.execute_reply.started":"2022-12-27T21:59:10.352321Z","shell.execute_reply":"2022-12-27T21:59:12.758377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n    \"\"\"\n    Parameters used for training\n    \"\"\"\n    seed = 42\n    verbose = 1\n    save_weights = True\n    \n    img_size = (1024, 512)\n    batch_size = 16\n    epochs = 5\n    use_fp16 = True\n    n_folds = 5\n    train_folds = [0, 1]\n    \n    weight_decay = 0.024\n    one_cycle_max_lr = 4e-4  # 8e-4\n    \n\n    # Model\n    model_name = \"efficientnet_b2\"\n    pretrained_weights = None\n    num_classes = 1\n    n_channels = 3\n    \n    pos_target_weight = 20\n    target = \"cancer\"\n    tta = True\n\nSAVE_FOLDER = \"./\"\nos.environ['CUDA_VISIBLE_DEVICES'] = \"0,1\"\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.153621Z","iopub.status.idle":"2022-12-27T21:51:36.153973Z","shell.execute_reply.started":"2022-12-27T21:51:36.153791Z","shell.execute_reply":"2022-12-27T21:51:36.153807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print('> SEEDING DONE')\n    \nset_seed(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.15511Z","iopub.status.idle":"2022-12-27T21:51:36.155452Z","shell.execute_reply.started":"2022-12-27T21:51:36.155269Z","shell.execute_reply":"2022-12-27T21:51:36.155285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Prep","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\nsample = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.156556Z","iopub.status.idle":"2022-12-27T21:51:36.156874Z","shell.execute_reply.started":"2022-12-27T21:51:36.15671Z","shell.execute_reply":"2022-12-27T21:51:36.156725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_PATH = f\"/kaggle/input/rsna-cut-off-empty-space-from-images/\"\ntrain[\"path\"] = IMG_PATH + train[\"patient_id\"].astype(str) + \"/\" + train[\"image_id\"].astype(str) + \".png\"","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.157858Z","iopub.status.idle":"2022-12-27T21:51:36.158225Z","shell.execute_reply.started":"2022-12-27T21:51:36.158039Z","shell.execute_reply":"2022-12-27T21:51:36.158057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(CFG.n_folds)\ntrain[\"fold\"] = -1\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(train, train[\"cancer\"])):\n    train.loc[val_idx, \"fold\"] = fold\n    \ntrain.groupby(\"fold\")[\"cancer\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.159607Z","iopub.status.idle":"2022-12-27T21:51:36.159983Z","shell.execute_reply.started":"2022-12-27T21:51:36.159786Z","shell.execute_reply":"2022-12-27T21:51:36.159804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class BreastCancerDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.paths = df['path'].values\n        self.transforms = transforms\n        self.targets = df['cancer'].values\n\n    def __len__(self):\n        return len(self.paths)\n\n    def __getitem__(self, idx):\n        \"\"\"\n        Item accessor\n\n        Args:\n            idx (int): Index.\n\n        Returns:\n            np array [H x W x C]: Image.\n            torch tensor [1]: Label.\n            torch tensor [1]: Sample weight.\n        \"\"\" \n        try:\n            image = np.asarray(Image.open(self.paths[idx]).convert('RGB'))\n        except Exception as ex:\n            print(self.paths[idx], ex)\n            return None\n        \n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n\n        if CFG.target in self.df.columns:\n            target = torch.as_tensor(self.df.iloc[idx].cancer)\n            return image, target\n\n        return image","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.161117Z","iopub.status.idle":"2022-12-27T21:51:36.161447Z","shell.execute_reply.started":"2022-12-27T21:51:36.161269Z","shell.execute_reply":"2022-12-27T21:51:36.161285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation","metadata":{}},{"cell_type":"code","source":"def transformer(stage):\n    if stage == \"train\":\n        return A.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.Rotate(limit=5),\n            A.augmentations.crops.RandomResizedCrop(height=CFG.img_size[0], width=CFG.img_size[1], scale=(0.8, 1), ratio=(0.45, 0.55)),\n            A.Normalize(),\n            A.pytorch.transforms.ToTensorV2()\n        ])\n    else:\n        return A.Compose([\n                A.Resize(CFG.img_size[0], CFG.img_size[1]),\n                A.Normalize(),\n                A.pytorch.transforms.ToTensorV2()\n            ])","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.162645Z","iopub.status.idle":"2022-12-27T21:51:36.162972Z","shell.execute_reply.started":"2022-12-27T21:51:36.162805Z","shell.execute_reply":"2022-12-27T21:51:36.162821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"def plot_df(df):\n    fig,ax = plt.subplots(2,2,figsize=(20,10))\n    \n    ax[0, 0].plot(df['train_loss'])\n    ax[0, 0].plot(df['valid_loss'])\n    ax[0, 0].legend()\n    ax[0, 0].set_title('Loss')\n    \n    ax[1, 0].plot(df['pF1'])\n    ax[1, 0].legend()\n    ax[1, 0].set_title('pF1')\n    \n    ax[1, 1].plot(df['thres'])\n    ax[1, 1].legend()\n    ax[1, 1].set_title('Threshold')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.16415Z","iopub.status.idle":"2022-12-27T21:51:36.164481Z","shell.execute_reply.started":"2022-12-27T21:51:36.164316Z","shell.execute_reply":"2022-12-27T21:51:36.164331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\", strict=True):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n    state_dict = torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\")\n\n    try:\n        model.load_state_dict(state_dict[\"model\"], strict=strict)\n    except BaseException:\n        try:\n            del state_dict['logits.weight'], state_dict['logits.bias']\n            model.load_state_dict(state_dict, strict=strict)\n        except BaseException:\n            del state_dict['encoder.conv_stem.weight']\n            model.load_state_dict(state_dict, strict=strict)\n\n    if verbose:\n        print(f\"\\n -> Loading encoder weights from {os.path.join(cp_folder,filename)}\\n\")\n\n    return model, state_dict[\"threshold\"], state_dict[\"model_type\"]","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.165342Z","iopub.status.idle":"2022-12-27T21:51:36.165688Z","shell.execute_reply.started":"2022-12-27T21:51:36.165517Z","shell.execute_reply":"2022-12-27T21:51:36.165534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_model(name, model, thres, model_type=CFG.model_name):\n    torch.save({'model': model.state_dict(), 'threshold': thres, 'model_type': model_type}, f'{name}')","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.166733Z","iopub.status.idle":"2022-12-27T21:51:36.167056Z","shell.execute_reply.started":"2022-12-27T21:51:36.166886Z","shell.execute_reply":"2022-12-27T21:51:36.166901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def define_model(\n    name,\n    num_classes=1,\n    num_classes_aux=0,\n    n_channels=3,\n    pretrained_weights=\"\",\n    pretrained=True,\n):\n    \"\"\"\n    Loads a pretrained model & builds the architecture.\n    Supports timm models.\n\n    Args:\n        name (str): Model name\n        num_classes (int, optional): Number of classes. Defaults to 1.\n        num_classes_aux (int, optional): Number of aux classes. Defaults to 0.\n        n_channels (int, optional): Number of image channels. Defaults to 3.\n        pretrained_weights (str, optional): Path to pretrained encoder weights. Defaults to ''.\n        pretrained (bool, optional): Whether to load timm pretrained weights.\n\n    Returns:\n        torch model -- Pretrained model.\n    \"\"\"\n    # Load pretrained model\n    encoder = create_model(CFG.model_name, pretrained=True, num_classes=num_classes, drop_rate=0.)\n    encoder.name = name\n\n    # Tile Model\n    model = BreastCancerModel(\n        encoder,\n        num_classes=num_classes,\n        num_classes_aux=num_classes_aux,\n        n_channels=n_channels,\n    )\n\n    if pretrained_weights:\n        model = load_model_weights(model, pretrained_weights, verbose=1, strict=False)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.168207Z","iopub.status.idle":"2022-12-27T21:51:36.168591Z","shell.execute_reply.started":"2022-12-27T21:51:36.1684Z","shell.execute_reply":"2022-12-27T21:51:36.168418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta(labels, predictions, beta=1.):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / max(y_true_count, 1)  # avoid / 0\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0\n\ndef optimal_f1(labels, predictions):\n    thres = np.linspace(0, 1, 101)\n    f1s = [pfbeta(labels, predictions > thr) for thr in thres]\n    idx = np.argmax(f1s)\n    return f1s[idx], thres[idx]","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.169636Z","iopub.status.idle":"2022-12-27T21:51:36.169998Z","shell.execute_reply.started":"2022-12-27T21:51:36.169811Z","shell.execute_reply":"2022-12-27T21:51:36.169829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class BreastCancerModel(nn.Module):\n    def __init__(\n        self,\n        model,\n        num_classes=1,\n        num_classes_aux=0,\n        n_channels=3,\n    ):\n        \"\"\"\n        Constructor.\n\n        Args:\n            encoder (timm model): Encoder.\n            num_classes (int, optional): Number of classes. Defaults to 1.\n            num_classes_aux (int, optional): Number of aux classes. Defaults to 0.\n            n_channels (int, optional): Number of image channels. Defaults to 3.\n        \"\"\"\n        super().__init__()\n\n        self.model = model\n        self.backbone_dim = self.model(torch.randn(1, 3, CFG.img_size[0], CFG.img_size[1])).shape[-1]\n\n        self.num_classes = num_classes\n        self.n_channels = n_channels\n\n        self.logits = nn.Linear(self.backbone_dim, num_classes)\n        \n        self._update_num_channels()\n\n    def _update_num_channels(self):\n        if self.n_channels != 3:\n            for n, m in self.model.named_modules():\n                if n:\n                    # print(\"Replacing\", n)\n                    old_conv = getattr(self.model, n)\n                    new_conv = nn.Conv2d(\n                        self.n_channels,\n                        old_conv.out_channels,\n                        kernel_size=old_conv.kernel_size,\n                        stride=old_conv.stride,\n                        padding=old_conv.padding,\n                        bias=old_conv.bias is not None,\n                    )\n                    setattr(self.model, n, new_conv)\n                    break\n\n    def forward(self, x, return_fts=False):\n        \"\"\"\n        Forward function.\n\n        Args:\n            x (torch tensor [batch_size x n_channels x h x w]): Input batch.\n\n        Returns:\n            torch tensor [batch_size x num_classes]: logits.\n            torch tensor [batch_size x num_classes_aux]: logits aux.\n        \"\"\"\n        x = self.model(x)\n        logits = self.logits(x).squeeze()\n\n        return logits","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.171124Z","iopub.status.idle":"2022-12-27T21:51:36.171455Z","shell.execute_reply.started":"2022-12-27T21:51:36.17128Z","shell.execute_reply":"2022-12-27T21:51:36.171295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_weight_decay(model, weight_decay=1e-5, skip_list=()):\n    decay = []\n    no_decay = []\n    for name, param in model.named_parameters():\n        if not param.requires_grad:\n            continue\n        if len(param.shape) == 1 or np.any([v in name.lower()  for v in skip_list]):\n            # print(name, 'no decay')\n            no_decay.append(param)\n        else:\n            # print(name, 'decay')\n            decay.append(param)\n    return [\n        {'params': no_decay, 'weight_decay': 0.},\n        {'params': decay, 'weight_decay': weight_decay}]","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.172486Z","iopub.status.idle":"2022-12-27T21:51:36.172803Z","shell.execute_reply.started":"2022-12-27T21:51:36.172641Z","shell.execute_reply":"2022-12-27T21:51:36.172657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_optimizer_and_scheduler(model, dataloader, optim=\"adam\"):\n    if optim == \"adamw\":\n         optimizer = torch.optim.AdamW(\n             add_weight_decay(model,\n                              weight_decay=CFG.weight_decay,\n                              skip_list=['bias']),\n             lr=CFG.one_cycle_max_lr,\n             betas=(0.9, 0.999),\n             weight_decay=CFG.weight_decay)\n    else:\n        optimizer = torch.optim.Adam(model.parameters())\n    \n    scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1,\n                                              max_lr=CFG.one_cycle_max_lr, epochs=CFG.epochs, steps_per_epoch=len(dataloader))\n    return optimizer, scheduler","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.173929Z","iopub.status.idle":"2022-12-27T21:51:36.174253Z","shell.execute_reply.started":"2022-12-27T21:51:36.174089Z","shell.execute_reply":"2022-12-27T21:51:36.174105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(dataloader, model, scheduler, optimizer, scaler, epoch):\n    model.train()\n    total_loss = 0\n    pbar = tqdm(dataloader, desc=f\"Train: Epoch {epoch + 1}\", total=len(dataloader), mininterval=5)\n    \n    for img, target in pbar:\n        optimizer.zero_grad()\n        img = img.to(device)\n\n        # Using mixed precision training\n        with autocast():\n            outputs = model(img)\n            loss = binary_cross_entropy_with_logits(\n                outputs,\n                target.to(float).to(device),\n                pos_weight=torch.tensor([CFG.pos_target_weight]).to(device)\n            )\n            \n            if np.isinf(loss.item()) or np.isnan(loss.item()):\n                print(f'Bad loss, skipping the batch {batch_idx}')\n                del loss, outputs\n                gc_collect()\n                continue\n        \n        # scaler is needed to prevent \"gradient underflow\"\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        if scheduler is not None:\n            scheduler.step()\n        scaler.update()\n        \n        lr = scheduler.get_last_lr()[0] if scheduler else CFG.one_cycle_max_lr\n        loss = loss.item()\n        \n        pbar.set_postfix({\"loss\": loss, \"lr\": lr})\n        total_loss += loss\n    \n    total_loss /= len(dataloader)\n    gc.collect()\n    torch.cuda.empty_cache()\n    return total_loss","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.175394Z","iopub.status.idle":"2022-12-27T21:51:36.175825Z","shell.execute_reply.started":"2022-12-27T21:51:36.175657Z","shell.execute_reply":"2022-12-27T21:51:36.175674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_one_epoch(dataloader, model, epoch):\n    model.eval()\n    pred_cancer = []\n    with torch.no_grad():\n        total_loss = 0\n        targets = []\n        pbar = tqdm(dataloader, desc=f'Eval: {epoch + 1}', total=len(dataloader), mininterval=5)\n\n        for img, target in pbar:\n            with autocast(enabled=True):\n                img = img.to(device)\n\n                outputs = model(img)\n                if CFG.tta:\n                    outputs2 = model(torch.flip(img, dims=[-1])) # horizontal mirror\n                    outputs = (outputs + outputs2) / 2\n\n                loss = binary_cross_entropy_with_logits(\n                            outputs, \n                            target.to(float).to(device),\n                            pos_weight=torch.tensor([CFG.pos_target_weight]).to(device)\n                        ).item()\n                \n                pbar.set_postfix({\"loss\": loss})\n                \n                pred_cancer.append(torch.sigmoid(outputs))\n                total_loss += loss\n                targets.append(target.cpu().numpy())\n             \n    targets = np.concatenate(targets)\n    pred = torch.concat(pred_cancer).cpu().numpy()\n    pf1, thres = optimal_f1(targets, pred)\n\n    total_loss /= len(dataloader)\n    gc.collect()\n    torch.cuda.empty_cache()\n    return total_loss, pf1, thres, pred","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.176828Z","iopub.status.idle":"2022-12-27T21:51:36.17715Z","shell.execute_reply.started":"2022-12-27T21:51:36.176981Z","shell.execute_reply":"2022-12-27T21:51:36.176996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fnc(train_dataloader, valid_dataloader, model, fold, optimizer, scheduler):\n    train_losses = []\n    valid_losses = []\n    valid_scores = []\n    thresholds   = []\n    \n    scaler = GradScaler()\n    best_loss = 999\n    best_score = -1\n    for epoch in range(CFG.epochs):\n        train_loss = train_one_epoch(train_dataloader, model, scheduler, optimizer, scaler, epoch)\n        valid_loss, valid_score, thres, pred = valid_one_epoch(valid_dataloader, model, epoch)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_scores.append(valid_score)\n        thresholds.append(thres)\n        \n        if valid_score > best_score:\n            best_score = valid_score\n            save_model(f\"fold{fold}_best_score.pth\", model, thres)\n            print(\"New Best Score\")\n        \n        if valid_loss < best_loss:\n            best_loss = valid_loss\n            save_model(f\"fold{fold}_best_loss.pth\", model, thres)\n            print(\"New Best Loss\")\n        print()\n        \n        print(f\"-------- Epoch {epoch + 1} --------\")\n        print(\"Train Loss: \", train_loss)\n        print(\"Valid Loss: \", valid_loss)\n        print(\"pF1: \", valid_score)\n        print(\"Best Score: \", best_score)\n        print(\"Best Loss: \", best_loss)\n        print()\n        \n    column_names = ['train_loss','valid_loss', 'pF1', 'thres']\n    df = pd.DataFrame(np.stack([train_losses, valid_losses, valid_scores, thresholds],\n                               axis=1),columns=column_names)\n    display(df)\n    plot_df(df)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.178436Z","iopub.status.idle":"2022-12-27T21:51:36.178806Z","shell.execute_reply.started":"2022-12-27T21:51:36.178614Z","shell.execute_reply":"2022-12-27T21:51:36.178631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(CFG.n_folds):\n    if fold not in CFG.train_folds: continue\n        \n    print(\"*\"*10, f\"Fold: {fold}\", \"*\"*10)\n    train_df = train[train[\"fold\"] != fold]\n    valid_df = train[train[\"fold\"] == fold]\n    \n    train_dataset = BreastCancerDataset(train_df, transformer(\"train\"))\n    valid_dataset = BreastCancerDataset(valid_df, transformer(\"valid\"))\n    \n    train_dataloader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, pin_memory=True)\n    valid_dataloader = DataLoader(valid_dataset, batch_size=CFG.batch_size * 2, shuffle=False, pin_memory=True)\n\n    model = define_model(CFG.model_name).to(device)\n    model = nn.DataParallel(model)\n    optimizer, scheduler = get_optimizer_and_scheduler(model, train_dataloader, \"adamw\")\n    \n    train_fnc(train_dataloader, valid_dataloader, model, fold, optimizer, scheduler)","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.179825Z","iopub.status.idle":"2022-12-27T21:51:36.180204Z","shell.execute_reply.started":"2022-12-27T21:51:36.180003Z","shell.execute_reply":"2022-12-27T21:51:36.18002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_predictions(models, train):\n    train_predictions = []\n    pbar = tqdm(enumerate(models), total=len(models), desc='Folds')\n    for fold, model in pbar:\n        if model is not None:\n            eval_dataset = BreastCancerDataset(train.query('fold == @fold'), transformer(\"valid\"))\n            eval_dataloader = DataLoader(eval_dataset, batch_size=CFG.batch_size, shuffle=False)\n            \n            eval_loss, pF1, thres, pred = valid_one_epoch(eval_dataloader, model, -1)\n            \n            pbar.set_description(f'Eval fold:{fold} pF1:{pF1:.02f}')\n            pred_df = pd.DataFrame(data=pred,\n                                          columns=['cancer_pred_proba'])\n            pred_df['cancer_pred'] = pred_df.cancer_pred_proba > thres\n\n            df = pd.concat(\n                [train.query('fold == @fold').reset_index(drop=True), pred_df],\n                axis=1\n            ).sort_values(['patient_id', 'image_id'])\n            train_predictions.append(df)\n    train_predictions = pd.concat(train_predictions)\n    return train_predictions","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.181132Z","iopub.status.idle":"2022-12-27T21:51:36.181466Z","shell.execute_reply.started":"2022-12-27T21:51:36.181287Z","shell.execute_reply":"2022-12-27T21:51:36.181319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Result","metadata":{}},{"cell_type":"code","source":"print(\"-\"*15, \" BEST LOSS \", \"-\"*15)\nmodels_path = glob.glob(f\"./*best_loss.pth\")","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.182467Z","iopub.status.idle":"2022-12-27T21:51:36.18278Z","shell.execute_reply.started":"2022-12-27T21:51:36.18262Z","shell.execute_reply":"2022-12-27T21:51:36.182635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [load_model_weights(define_model(CFG.model_name).to(device), model)[0] for model in models_path]\npred_df = gen_predictions(models, train)\npred_df.to_csv('train_predictions.csv', index=False)\n!head train_predictions.csv","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.183974Z","iopub.status.idle":"2022-12-27T21:51:36.184298Z","shell.execute_reply.started":"2022-12-27T21:51:36.184136Z","shell.execute_reply":"2022-12-27T21:51:36.184151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.read_csv('train_predictions.csv')\nprint('F1 CV score (multiple thresholds):', f1_score(pred_df.cancer, pred_df.cancer_pred))    \npred_df = pred_df.groupby(['patient_id', 'laterality']).agg(\n    cancer_max=('cancer_pred_proba', 'max'), cancer_mean=('cancer_pred_proba', 'mean'), cancer=('cancer', 'max')\n)\nprint('pF1 CV score. Mean aggregation, single threshold:', optimal_f1(pred_df.cancer.values, pred_df.cancer_mean.values))\nprint('pF1 CV score. Max aggregation, single threshold:', optimal_f1(pred_df.cancer.values, pred_df.cancer_max.values))","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.185604Z","iopub.status.idle":"2022-12-27T21:51:36.185923Z","shell.execute_reply.started":"2022-12-27T21:51:36.185758Z","shell.execute_reply":"2022-12-27T21:51:36.185773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"-\"*15, \" BEST SCORE \", \"-\"*15)\nmodels_path = glob.glob(f\"./*best_score.pth\")","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.186823Z","iopub.status.idle":"2022-12-27T21:51:36.187147Z","shell.execute_reply.started":"2022-12-27T21:51:36.186978Z","shell.execute_reply":"2022-12-27T21:51:36.186993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [load_model_weights(define_model(CFG.model_name).to(device), model)[0] for model in models_path]\npred_df = gen_predictions(models, train)\npred_df.to_csv('train_predictions.csv', index=False)\n!head train_predictions.csv","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.188096Z","iopub.status.idle":"2022-12-27T21:51:36.188487Z","shell.execute_reply.started":"2022-12-27T21:51:36.188276Z","shell.execute_reply":"2022-12-27T21:51:36.188294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.read_csv('train_predictions.csv')\nprint('F1 CV score (multiple thresholds):', f1_score(pred_df.cancer, pred_df.cancer_pred))    \npred_df = pred_df.groupby(['patient_id', 'laterality']).agg(\n    cancer_max=('cancer_pred_proba', 'max'), cancer_mean=('cancer_pred_proba', 'mean'), cancer=('cancer', 'max')\n)\nprint('pF1 CV score. Mean aggregation, single threshold:', optimal_f1(pred_df.cancer.values, pred_df.cancer_mean.values))\nprint('pF1 CV score. Max aggregation, single threshold:', optimal_f1(pred_df.cancer.values, pred_df.cancer_max.values))","metadata":{"execution":{"iopub.status.busy":"2022-12-27T21:51:36.189692Z","iopub.status.idle":"2022-12-27T21:51:36.19007Z","shell.execute_reply.started":"2022-12-27T21:51:36.189872Z","shell.execute_reply":"2022-12-27T21:51:36.18989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}