{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\nimport gc\nimport pickle\nimport time\nimport random\nimport math\nimport shutil\nimport argparse\nfrom argparse import Namespace\nfrom PIL import Image\nfrom numpy import asarray\nfrom torchvision import transforms\n\nimport yaml\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport numpy as np\nimport pandas as pd\nimport sklearn\nfrom sklearn.model_selection import KFold, StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score, auc, precision_recall_curve\nimport timm\nfrom timm.scheduler import CosineLRScheduler\n#from albumentations  import *\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n\nprint('torch version:', torch.__version__)\nprint('timm version:', timm.__version__)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install iterative-stratification\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nprint('done')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config :\n    DEVICE = 'cuda:0'\n    seed = 10\n    INPUT_BASE = '/kaggle/input/rsna-breast-cancer-detection'\n    OUTPUT_BASE = '/kaggle/working/'\n    \n    windowing = False\n    batch_size = 8\n    name = 'tf_efficientnetv2_s'\n    lr = 5.0e-5\n    epochs = 3\n    epochs_warmup = 0\n    num_cycles = 0.5\n    VER = '084'\n    n_folds = 4\n    apex = True\n    MEAN = 0.3089279\n    STD = 0.25053555408335154\n    # parameters for elastic deformation\n    print_freq = 100\n    a = 10\n    s = 15\n        \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = Config.DEVICE if torch.cuda.is_available() else 'cpu'\nprint('device:', device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Setting\ndef seed_everything(seed: int):    \n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\nseed_everything(Config.seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset\n# Breast Level\nclass MammoDataset(Dataset):\n    def __init__(self, df, Config, train=True, tfms=None, windowing=False):\n        self.df = df\n        self.train = train\n        self.tfms = tfms\n        self.Config = Config\n        self.INPUT_BASE = Path(Config.INPUT_BASE)\n        self.windowing = windowing\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        data = self.df.iloc[idx]\n        img_id = f\"{data['image_id']}.png\"\n        input_path = '/kaggle/input/preprocessed-images-rsna/output'\n        path = str(Path(input_path).joinpath(\"preprocessed_images_RSNA\", str(data['patient_id']), \n                                             img_id))\n        #path = str(self.INPUT_BASE.joinpath(\"train_images\", str(data['patient_id']), img_id))\n        #img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n        img = Image.open(path).convert('RGB')\n        img = img.resize((1024,912))\n        #img = asarray(img)\n        #print(img.shape)\n       \n        if self.tfms:\n            #augmented = self.tfms(image=img)\n            img = self.tfms(img).to(torch.float32)\n            #img = augmented['image']\n        #img = img.astype('float32')\n\n        if self.windowing:\n            # sigmoid windowing\n            img /= 255\n            img = img * (data['max'] - data['min']) + data['min']\n            if data['rev'] == 1:\n                img = data['rev_max'] - img\n            img = data['y_range'] / (1 + np.exp(-4 * (img - data['center']) / data['width']))\n            if data['rev'] == 1:\n                img = np.amax(img) - img\n\n        #img = img.astype('float32')\n        #minimum_value = img.min()\n        #maximum_value = img.max()\n        \n        #imgs -= img.min()\n        #imgs /= img.min()\n        \n        #img = torch.tensor((img - self.Config.MEAN)/self.Config.STD, dtype=torch.float32)\n        \n        if self.train:\n            return img, torch.tensor(data['cancer'], dtype=torch.long)\n        else:\n            return img\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Augmentations\n'''\ndef get_aug(p=1.0, a=10, s=10):\n    return Compose([\n        HorizontalFlip(),\n        VerticalFlip(),\n        # RandomRotate90(),\n#         ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, p=0.8, \n#                          border_mode=cv2.BORDER_REFLECT)\n        # OneOf([Affine(rotate=20, translate_percent=0.1, scale=[0.8,1.2], shear=20)])\n        Affine(rotate=20, translate_percent=0.1, scale=[0.8,1.2], shear=20),\n        ElasticTransform(alpha=a, sigma=s)\n    ], p=p)\n'''\n\n\ndef get_aug(p=1.0, a=10, s=10):\n    \n    augmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomVerticalFlip(0.5),\n    #transforms.RandomPerspective(),\n    #transforms.RandomRotation((0, 90)),\n    #transforms.RandomAutocontrast(),\n    #transforms.RandomAffine(degrees=(0, 180), scale=(0.8, 1.2)),\n    #transforms.ElasticTransform(),   \n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n    transforms.Normalize(mean = [0.1338, 0.1338, 0.1338],\n                         std = [0.2068, 0.2068, 0.2068])    \n])\n    return augmentator\n\n\ndef get_aug_valid():\n    augmentator = transforms.Compose([ \n    transforms.ToTensor(), # return it as a tensor\n    transforms.Normalize(mean = [0.1336, 0.1336, 0.1336],\n                         std = [0.2082, 0.2082, 0.2082])      \n])\n    return augmentator\n\n\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model \ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (\n            self.__class__.__name__\n            + \"(\"\n            + \"p=\"\n            + \"{:.4f}\".format(self.p.data.tolist()[0])\n            + \", \"\n            + \"eps=\"\n            + str(self.eps)\n            + \")\"\n        )\n    \nclass MammoModel(nn.Module):\n    def __init__(self, name, *, pretrained=False, in_chans=3, p=3, p_trainable=False, eps=1e-6):\n        super().__init__()\n        model = timm.create_model(name, pretrained=pretrained, in_chans=in_chans)\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n        \n        self.fc = nn.Linear(n_features, 1) # cancer\n        self.model = model\n\n        self.pool = nn.Sequential(\n            GeM(p=p, eps=eps, p_trainable=p_trainable),\n            nn.Flatten())\n    \n    def forward(self, x):\n        # x = self.model(x)\n        x = self.model.forward_features(x)\n        x = self.pool(x)\n        logits = self.fc(x)\n        return logits   \n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train function\ndef train_fn(fold, train_loader, model, criterion, optimizer, epoch, device):\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled=Config.apex)\n    losses = AverageMeter()\n    start = end = time.time()\n    for step, (inputs, labels) in enumerate(train_loader):\n        inputs = inputs.to(device)\n        labels = labels.float().to(device)\n        batch_size = labels.size(0)\n        with torch.cuda.amp.autocast(enabled=Config.apex):\n            y_preds = model(inputs)\n        loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))\n        losses.update(loss.item(), batch_size)\n        scaler.scale(loss).backward()\n        # grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), Config.max_grad_norm)\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n        # batch scheduler\n        # scheduler.step()\n        end = time.time()\n        if step % Config.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'LR: {lr:.8f}'\n                  .format(epoch+1, step, len(train_loader), \n                          remain=timeSince(start, float(step+1)/len(train_loader)),\n                          loss=losses,\n                          lr=optimizer.param_groups[0]['lr']))\n    return losses.avg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation function\ndef valid_fn(valid_loader, model, criterion, device):\n    losses = AverageMeter()\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (inputs, labels) in enumerate(valid_loader):\n        inputs = inputs.to(device)\n        labels = labels.float().to(device)\n        batch_size = labels.size(0)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))\n        losses.update(loss.item(), batch_size)\n        preds.append(y_preds.squeeze(1).sigmoid().to('cpu').numpy())\n        end = time.time()\n        if step % Config.print_freq == 0 or step == (len(valid_loader) - 1):\n            print('EVAL: [{0}/{1}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(step, len(valid_loader),\n                          loss=losses,\n                          remain=timeSince(start, float(step+1)/len(valid_loader))))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/sohier/probabilistic-f-score\ndef pfbeta(labels, predictions, beta):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n            # cfp += 1 - prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta_binarized(labels, predictions):\n    positives = predictions[labels == 1]\n    scores = []\n    for th in positives:\n        binarized = (predictions >= th).astype('int')\n        score = pfbeta(labels, binarized, 1)\n        scores.append(score)\n    return np.max(scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pr_auc(y_true, y_pred):\n    precision, recall, _ = precision_recall_curve(y_true, y_pred)\n    score = auc(recall, precision)\n    return score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_custom_folds(train_df, Config):\n    train_df_all = train_df.copy()\n\n    # count images per prediction_id\n    train_df['prediction_id'] = train_df['patient_id'].astype(str).str.cat(train_df['laterality'], sep='_')\n    train_df['prediction_id'] = train_df['patient_id'].astype(str).str.cat(train_df['laterality'], sep='_')\n    num_count = train_df[['prediction_id', 'image_id']].groupby('prediction_id').count().reset_index()\n    count_map = {pred_id: img_id for pred_id, img_id in zip(num_count['prediction_id'].values, num_count['image_id'].values)}\n    train_df['count'] = train_df['prediction_id'].map(count_map)\n\n    # group by patient_id and stratify by age, implant, machine_id, cancer, biopsy, BIRADS, density, and count\n    train_df = train_df.groupby('prediction_id').first().reset_index()\n    dummy = train_df[['patient_id', 'age', 'implant', 'machine_id']].groupby('patient_id').first()\n    machine2int = {machine_id: n for n, machine_id in enumerate(train_df[['machine_id', 'cancer']].groupby('machine_id').mean().sort_values('cancer').index.values)}\n    dummy['machine_id'] = dummy['machine_id'].apply(lambda x: machine2int[x])\n    dummy2 = train_df[['patient_id', 'cancer', 'biopsy', 'count']].groupby('patient_id').mean()\n    dummy3 = train_df[['patient_id', 'BIRADS']].groupby('patient_id').min().fillna(-1)\n    dummy4 = train_df[['patient_id', 'density']].groupby('patient_id').max().fillna('E')\n    dummy4['density'] = dummy4['density'].map({'E': -1, 'D': 0, 'C': 1, 'B': 2, 'A': 3})\n    dummy = pd.concat([dummy, dummy2, dummy3, dummy4], axis=1)\n    dummy['age'] = dummy['age'].fillna(dummy['age'].mean())\n    dummy['fold'] = -1\n    mskf = MultilabelStratifiedKFold(n_splits=Config.n_folds, shuffle=True, random_state=Config.seed)\n    for fold, (trn_ind, val_ind) in enumerate(mskf.split(dummy, dummy.values)):\n        dummy.iloc[val_ind, -1] = fold\n    dummy = dummy.reset_index()\n    dummy['patient_id'] = dummy['patient_id'].astype('int')\n    fold_map = {patient_id: fold for patient_id, fold in zip(dummy['patient_id'].values, dummy['fold'].values)}\n    \n    # show some stat regarding each fold\n    train_df = train_df.merge(dummy[['patient_id','fold']], on='patient_id', how='left')\n    for fold in range(Config.n_folds):\n        trn_ind = train_df[train_df['fold'] != fold].index\n        val_ind = train_df[train_df['fold'] == fold].index\n        print(f'=========== Fold {fold} ===========')\n        print(f'Train {len(trn_ind)}', end='')\n        _, counts = np.unique(train_df.loc[trn_ind, 'cancer'].values, return_counts=True)\n        print(f'        (positive {counts[1]}, negative {counts[0]})')\n        print(f'Validation {len(val_ind)}', end='')\n        _, counts = np.unique(train_df.loc[val_ind, 'cancer'].values, return_counts=True)\n        print(f'    (positive {counts[1]}, negative {counts[0]})')\n    \n    # concat 'fold' column on train_df_all\n    train_df_all['fold'] = train_df_all['patient_id'].map(fold_map)\n    return train_df_all\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop (using k-folds)\ndef train_loop(folds, fold):\n    \n    print(f'================== fold: {fold} training ======================')\n    train_folds = folds[folds['fold'] != fold].reset_index(drop=True)\n    valid_folds = folds[folds['fold'] == fold].reset_index(drop=True)\n    \n    \n    train_dataset = MammoDataset(train_folds, Config, tfms = get_aug(a=Config.a, s=Config.s), windowing=Config.windowing)\n    valid_dataset = MammoDataset(valid_folds, Config, tfms = get_aug(a=Config.a, s=Config.s))\n        \n    \n    train_loader = DataLoader(train_dataset,\n                             batch_size=Config.batch_size,\n                             shuffle=True,\n                             num_workers=2, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset,\n                             batch_size=Config.batch_size,\n                             shuffle=False,\n                             num_workers=2, pin_memory=True, drop_last=False)\n    \n    model = MammoModel(Config.name, pretrained=True)\n    model.to(device)\n    \n    optimizer = Adam(model.parameters(), lr=Config.lr)\n    scheduler = get_cosine_schedule_with_warmup(\n                optimizer, num_warmup_steps=Config.epochs_warmup, num_training_steps=Config.epochs, \n        num_cycles=Config.num_cycles\n            )\n    \n\n    criterion = nn.BCEWithLogitsLoss(reduction='mean')\n    \n    best_score = 0.\n    best_aucroc = 0.\n    best_prauc = 0.\n    for epoch in range(Config.epochs):\n        start_time = time.time()\n        # train\n        avg_loss = train_fn(fold, train_loader, model, criterion, optimizer, epoch, device)\n        scheduler.step()\n        # eval\n        avg_val_loss, predictions = valid_fn(valid_loader, model, criterion, device)\n        valid_folds['prediction'] = predictions\n        valid_agg = valid_folds[['patient_id', 'laterality', 'cancer', 'prediction', 'fold']].groupby(['patient_id', 'laterality']).mean()\n        score = pfbeta_binarized(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        prauc = pr_auc(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        aucroc = roc_auc_score(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        \n        elapsed = time.time() - start_time\n        \n        print(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        print(f'Epoch {epoch+1} - pF Score: {score:.4f}, PR-AUC Score: {prauc:.4f}, AUC-ROC Score: {aucroc:.4f}')\n        \n        if best_prauc < prauc:\n            best_prauc = prauc\n            # torch.save({'model': model.state_dict(),\n            #             'predictions': predictions},\n            #             OUTPUT_BASE.joinpath(\"models\", f\"{model_base_name}_seed_{config.seed}_fold{fold}_best_prauc_ver{config.VER}.pth\"))\n            \n        if best_aucroc < aucroc:\n            best_aucroc = aucroc\n            # torch.save({'model': model.state_dict(),\n            #             'predictions': predictions},\n            #             OUTPUT_BASE.joinpath(\"models\", f\"{model_base_name}_seed_{config.seed}_fold{fold}_best_aucroc_ver{config.VER}.pth\"))\n            \n        if best_score < score:\n            best_score = score\n            print(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(),\n                        'predictions': predictions},\n                        OUTPUT_BASE.joinpath(f\"{model_base_name}_seed_{Config.seed}_fold{fold}.pth\"))\n        \n            #torch.save({'model': model.state_dict(),\n            #            'predictions': predictions},\n            #            OUTPUT_BASE.joinpath(\"models\", f\"{model_base_name}_seed_{Config.seed}_fold{fold}.pth\"))\n        \n    predictions = torch.load(OUTPUT_BASE.joinpath(f'{model_base_name}_seed_{Config.seed}_fold{fold}.pth'), map_location='cpu')['predictions']\n    valid_folds['prediction'] = predictions\n    print(f'[Fold{fold}] Best pF Score: {best_score}, PR-AUC Score: {best_prauc}, AUC-ROC Score: {best_aucroc:.4f}')\n    torch.cuda.empty_cache()\n    gc.collect()\n    return valid_folds\n","metadata":{}},{"cell_type":"code","source":"# Train loop (without k-folds)\n\ndef train_loop_No_kfolds(df, fold = None):\n    \n    train_df, valid_df = sklearn.model_selection.train_test_split(df, \n                                                                  test_size=0.15, \n                                                                  shuffle=True, \n                                                                  stratify=df['cancer'].values)\n    \n    \n    train_dataset = MammoDataset(train_df, Config, tfms = get_aug(a=Config.a, s=Config.s), windowing=Config.windowing)\n    valid_dataset = MammoDataset(valid_df, Config, tfms = get_aug(a=Config.a, s=Config.s))\n    \n    train_loader = DataLoader(train_dataset,\n                             batch_size=Config.batch_size,\n                             shuffle=True,\n                             num_workers=2, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset,\n                             batch_size=Config.batch_size,\n                             shuffle=False,\n                             num_workers=2, pin_memory=True, drop_last=False)\n    \n    model = MammoModel(Config.name, pretrained=True)\n    model.to(device)\n    \n    optimizer = Adam(model.parameters(), lr=Config.lr)\n    scheduler = get_cosine_schedule_with_warmup(\n                optimizer, num_warmup_steps=Config.epochs_warmup, num_training_steps=Config.epochs, \n        num_cycles=Config.num_cycles\n            )\n    \n\n    criterion = nn.BCEWithLogitsLoss(reduction='mean')\n    \n    best_score = 0.\n    best_aucroc = 0.\n    best_prauc = 0.\n    for epoch in range(Config.epochs):\n        start_time = time.time()\n        # train\n        avg_loss = train_fn(fold, train_loader, model, criterion, optimizer, epoch, device)\n        scheduler.step()\n        # eval\n        avg_val_loss, predictions = valid_fn(valid_loader, model, criterion, device)\n        valid_df['prediction'] = predictions\n        \n        valid_agg = valid_df\n        \n        '''\n        valid_agg = valid_df[['patient_id',\n                                 'laterality', \n                                 'cancer', \n                                 'prediction']].groupby(['patient_id', 'laterality']).mean()\n        '''\n        \n        score = pfbeta_binarized(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        prauc = pr_auc(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        aucroc = roc_auc_score(valid_agg['cancer'].values, valid_agg['prediction'].values)\n        \n        elapsed = time.time() - start_time\n        \n        print(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        print(f'Epoch {epoch+1} - pF Score: {score:.4f}, PR-AUC Score: {prauc:.4f}, AUC-ROC Score: {aucroc:.4f}')\n        \n        if best_prauc < prauc:\n            best_prauc = prauc\n            # torch.save({'model': model.state_dict(),\n            #             'predictions': predictions},\n            #             OUTPUT_BASE.joinpath(\"models\", f\"{model_base_name}_seed_{config.seed}_fold{fold}_best_prauc_ver{config.VER}.pth\"))\n            \n        if best_aucroc < aucroc:\n            best_aucroc = aucroc\n            # torch.save({'model': model.state_dict(),\n            #             'predictions': predictions},\n            #             OUTPUT_BASE.joinpath(\"models\", f\"{model_base_name}_seed_{config.seed}_fold{fold}_best_aucroc_ver{config.VER}.pth\"))\n            \n        if best_score < score:\n            best_score = score\n            print(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(),\n                        'predictions': predictions},\n                        OUTPUT_BASE.joinpath(f\"{model_base_name}_seed_{Config.seed}.pth\"))\n        \n    predictions = torch.load(OUTPUT_BASE.joinpath(f'{model_base_name}_seed_{Config.seed}.pth'), map_location='cpu')['predictions']\n    valid_df['prediction'] = predictions\n    print(f'Best pF Score: {best_score}, PR-AUC Score: {best_prauc}, AUC-ROC Score: {best_aucroc:.4f}')\n    torch.cuda.empty_cache()\n    gc.collect()\n    return valid_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###############################\n##### Train using k-folds #####\n###############################\n\nINPUT_BASE = Path(Config.INPUT_BASE)\n#OUTPUT_BASE = Path(Config.OUTPUT_BASE, exist_ok=True)\nOUTPUT_BASE = Path('/kaggle/working/models/', exist_ok=True)\nos.makedirs(OUTPUT_BASE, exist_ok = True) \n \n\nmodel_base_name = 'efficientv2' if  'efficientv2' in Config.name else 'efficientv5'\n\n### Load Train\ntrain_df = pd.read_csv(INPUT_BASE.joinpath('train.csv'))\n\n# cv splitting, grouped by patient_id and stratified by age, implant, machine_id, cancer, biopsy, BIRADS, density, and the num of images\n# get_custom_folds returns train_df with 'fold' column\ntrain_df = get_custom_folds(train_df, Config)\n\noof_df = pd.DataFrame()\nfor fold in range(Config.n_folds):\n    seed_everything(Config.seed)\n    _oof_df = train_loop(train_df, fold)\n    oof_df = pd.concat([oof_df, _oof_df])\noof_df = oof_df.reset_index(drop=True)\noof_df_agg = oof_df[['patient_id', 'laterality', 'cancer', 'prediction', 'fold']].groupby(['patient_id', 'laterality']).mean()\n\nprint('================ CV ================')\n\nscore = pfbeta_binarized(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\nprauc = pr_auc(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\naucroc = roc_auc_score(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\nprint(f'Score: {score}, PR-AUC: {prauc}, AUC-ROC: {aucroc}')\noof_df.to_pickle(OUTPUT_BASE.joinpath('preds', f'oof_df_ver{Config.VER}_seed{Config.seed}.pkl'))\n","metadata":{}},{"cell_type":"markdown","source":"import torch\nfrom pathlib import Path\nimport os\nfold = 1\nseed = 42\n\nOUTPUT_BASE = Path('/kaggle/working/models/', exist_ok=True)\n\nos.makedirs(OUTPUT_BASE, exist_ok = True) \n\ntorch.save({'predictions': 'predictions'},\n            OUTPUT_BASE.joinpath(f\"{'efficientv2'}_seed_{seed}_fold{fold}.pth\"))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T16:16:38.299927Z","iopub.execute_input":"2023-06-14T16:16:38.300617Z","iopub.status.idle":"2023-06-14T16:16:38.309584Z","shell.execute_reply.started":"2023-06-14T16:16:38.300582Z","shell.execute_reply":"2023-06-14T16:16:38.307796Z"}}},{"cell_type":"code","source":"\n#################################\n##### Train without k-folds #####\n#################################\n\nINPUT_BASE = Path(Config.INPUT_BASE)\n#OUTPUT_BASE = Path(Config.OUTPUT_BASE)\nOUTPUT_BASE = Path('/kaggle/working/models/', exist_ok=True)\nos.makedirs(OUTPUT_BASE, exist_ok = True) \n\nmodel_base_name = 'efficientv2' if  'efficientv2' in Config.name else 'efficientv5'\n\n### Load Train\ntrain_df = pd.read_csv(INPUT_BASE.joinpath('train.csv'))\n\n# cv splitting, grouped by patient_id and stratified by age, implant, machine_id, cancer, biopsy, BIRADS, density, and the num of images\n\nseed_everything(Config.seed)\noof_df = train_loop_No_kfolds(train_df, fold=None)\n\noof_df_agg = oof_df\n\n'''\noof_df_agg = oof_df[['patient_id', \n                       'laterality', \n                       'cancer', \n                       'prediction']].groupby(['patient_id', 'laterality']).mean()     \n'''\n             \nprint('================ CV ================')\n\nscore = pfbeta_binarized(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\nprauc = pr_auc(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\naucroc = roc_auc_score(oof_df_agg['cancer'].values, oof_df_agg['prediction'].values)\nprint(f'Score: {score}, PR-AUC: {prauc}, AUC-ROC: {aucroc}')\n\noutput_path = Path('/kaggle/working/models/preds', exist_ok=True)\nos.makedirs(output_path, exist_ok = True) \noof_df.to_pickle(output_path.joinpath(f'oof_df_ver{Config.VER}_seed{Config.seed}.pkl'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\noutput_path = Path('/kaggle/working/models/preds', exist_ok=True)\nos.makedirs(output_path, exist_ok = True) \noutput_path.joinpath(f'oof_df_ver{Config.VER}_seed{Config.seed}.pkl')\n'''\n#oof_df.to_pickle(OUTPUT_BASE.joinpath(f'oof_df_ver{Config.VER}_seed{Config.seed}.pkl'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In this notebook version, I used RGB images and so on, I changed the line of images normalization \n# by adding normalization as transformation in the augmentation part. The line that does normalization \n# in the Dataset class is valid for one channel (grayscale) images.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Should the normalization be applied on validation and test data?\n# In this version I applied it on both the training and validation data.\n\n# see : https://discuss.pytorch.org/t/finding-mean-and-std-for-each-of-the-train-val-and-test-dataloader-to-use-for-normalize-in-data-transform/145974/2\n\n\n'''\nDefinitely you should normalize your data. You normalize the data for the following aims:\nFor having different features in same scale, which is for accelerating learning process.\nFor caring different features fairly without caring the scale.\nAfter training, your learning algorithm has learnt to deal with the data in scaled form, so you have \nto normalize your test data with the normalizing parameters used for training data.\nreference : https://datascience.stackexchange.com/questions/27615/should-we-apply-normalization-to-test-data-as-well#:~:text=Definitely%20you%20should%20normalize%20your,fairly%20without%20caring%20the%20scale.\n\n'''\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In this notebook, we don't use k-folds.\n# In this version, we don't use aggregation.","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}