{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Please upvote if useful 🔥🔥🔥\n## Author: Rishi Chandra  @sapthrishi007\n\n# This is NOT an alternative to YOLO type object detection\n\n# This is simple image classification using resnet with soft attention (CBAM) pretrained on imagenet and demonstrates the following:\n\n## 1. Grad-Cam visualization of the final CNN layer feature map and localizing the bounding box on the image which mostly activates this feature map.\n\n## 2. Using this bounding box for the submission. Obviously this is not a replacement to yolo since the model does not trains on the input bounding boxes of the training set.\n\n## 3. Soft attention CBAM: Convolutional Block Attention Module \n\nCBAM paper ref: https://arxiv.org/pdf/1807.06521.pdf\n\nCBAM resnet github: https://github.com/Jongchan/attention-module\n\n## 4. A reusable custom Trainer with plots a variety of metrics. Trains it through Progressive Resizing of image, and checkoints best model for a variety of metrics. You can reuse this trainer for your projects too!!!\n\n## 5. Resume training from last epoch, model and optimizer state, by just specifying the last checkpoint !  Also adjust no. of cycles and plot ur lr curve and see.\n\n## 6. Searches for an optimum probability threshold, instead of default 0.5, for image classification. The trainer is for scenario of MultiLabelBinaryClassification, and can easily be tweaked for any scenario.\n\n## 7. Uses sum of DiceLoss and BCEWithLogitsLoss. DiceLoss to take care of class sparcity, class unbalance. And class weighted BCEWithLogitsLoss for class imbalance and smoother gradients.\n\n## 8. Just specify your model path in torch.load(...) statements and infer from your model\n\n\nToday is Holi, Wish you Happy Holi !!!"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/pred_probs.pt\n!ls ../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth\n!cp ../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth ./\n!cp ../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth ./cbam_resnet50_fold0_start.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/vinbigdata-chest-xray-resized-png-1024x1024/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install GPUtil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import sys\n# sys.path.append ('/kaggle/input/pytorch-images-seresnet')\nimport os\nimport gc\nimport time\nimport math\nimport random\nimport datetime\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nfrom   torch.nn import init\nfrom   torch.nn import CrossEntropyLoss, MSELoss\nfrom   torch.nn.modules.loss import _WeightedLoss\nimport torch.nn.functional as F\nfrom   torch.nn import Parameter\nfrom   torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\nfrom   transformers import AdamW, get_cosine_schedule_with_warmup\nfrom   torch.cuda.amp import autocast, GradScaler\n\nfrom   fastai.losses import LabelSmoothingCrossEntropy\n\n# from warmup_scheduler import GradualWarmupScheduler\nfrom   sklearn import preprocessing\nfrom   sklearn.metrics import accuracy_score\nfrom   sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom   sklearn.metrics import accuracy_score, precision_recall_fscore_support, matthews_corrcoef, roc_auc_score\nfrom   sklearn.metrics import mean_squared_error, mean_absolute_error, explained_variance_score, r2_score\n\n# import timm\nimport albumentations as A\nfrom   albumentations import *\nfrom   albumentations.pytorch import ToTensorV2\nfrom   albumentations.core.transforms_interface import DualTransform\nfrom   albumentations.augmentations import functional as AF\nimport cv2\n\nfrom   tqdm import tqdm\nfrom   pprint import pprint\nfrom   functools import partial\nimport matplotlib.pyplot as plt\n# from GPUtil import showUtilization as gpu_usage\nfrom   numba import cuda\nimport warnings\nwarnings.filterwarnings (\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    device       = torch.device ('cuda' if torch.cuda.is_available () else 'cpu')\n    num_workers  = 8\n    model_name   = 'cbam_resnet50'        # ['deit_base_patch16_224', 'vit_base_patch16_384', 'resnext50_32x4d', 'tf_efficientnet_b7_ns']\n    size         = 640                    # [64, 128, 224, 384, 512, 640, 720]\n    isTrain      = True\n    isFreeze     = True\n    lr           = 1e-3\n    epochs       = 6\n    warmup_steps = 0.50                    # if float: these many epochs are with frozen model at the beginning, if int = actual steps\n    lr_num_cycles= 2.5\n    epochsNx     = 1\n    criterion    = 'dice_bce'              # ['dice_bce', CrossEntropyLoss', 'BCEWithLogitsLoss', 'SmoothBCEwithLogits']\n    weight_decay = 1e-6\n    max_grad_norm= 1000.0\n    seed         = 42\n    n_fold       = 10\n    train_fold   = [0]                      # [0, 1, 2, 3, 4]\n    print_every  = 100\n    adam_epsilon = 1e-8\n    train_batch_size = 12\n    eval_batch_size  = 112\n    img_ext          = '.png'\n    img_col          = \"image_id\"\n    raw_label_cols   = 'class_id'\n    label_cols       = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]\n    target_size      = len (label_cols)\n    model_infer_path_prefix = \".\"\n    model_train_path_prefix = \".\"\n    train_path       = '../input/vinbigdata-chest-xray-resized-png-1024x1024/train'\n    train_csv        = '../input/vinbigdata-chest-xray-abnormalities-detection/train.csv'\n    test_path        = '../input/vinbigdata-chest-xray-resized-png-1024x1024/test'\n    output_dir       = './results'        # output directory    \n    eval_steps       = 0.5                # if float: these many epochs are with frozen model at the beginning, if int = actual steps \n    max_steps        = 0\n    MODEL            = None\n    thresholds       = [0.5, 0.01, 0.001, 0.1, 0.01, 0.001, 0.01, 0.001, 0.001, 0.0001, 0.1, 0.1, 0.01, 0.1]\n    \n    IMG_MEAN         = [0.485, 0.456, 0.406] #Mean for normalization Transform cassava = [0.4303, 0.4967, 0.3134] imgnet = [0.485, 0.456, 0.406]\n    IMG_STD          = [0.229, 0.224, 0.225] #STD for normalization Transform cassava = [0.2142, 0.2191, 0.1954] imgnet = [0.229, 0.224, 0.225]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def init_logger (log_file=CFG.output_dir+'train.log'):\n    \n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger (__name__)\n    logger.setLevel (INFO)\n    handler1 = StreamHandler ()\n    handler1.setFormatter (Formatter (\"%(message)s\"))\n    handler2 = FileHandler (filename=log_file)\n    handler2.setFormatter (Formatter (\"%(message)s\"))\n    logger.addHandler (handler1)\n    logger.addHandler (handler2)\n    return logger","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything (seed):\n    \n    random.seed (seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed (seed)\n    torch.manual_seed (seed)\n    torch.cuda.manual_seed (seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_train_csv ():\n    \n    df    = pd.read_csv (CFG.train_csv).reset_index (drop=True)\n    cols  = [CFG.img_col] + [CFG.raw_label_cols]               #;print (df[CFG.raw_label_cols])\n    df    = df[cols]                                             #;print ('df.shape =', df.shape, df.iloc[0,1])\n    no_df = df.loc[df[CFG.raw_label_cols] == 14]\n    df    = df.loc[df[CFG.raw_label_cols] != 14]                 #;print ('df.shape =', df.shape, df[CFG.raw_label_cols]!=14)\n    df    = df.append (no_df.sample (n=500, random_state=CFG.seed))\n    del no_df\n    \n    dg       = df.groupby (CFG.img_col)\n    df_dicts = []\n    for k in dg:\n        # print (k[0], list(k[1]['class_id']))\n        row = {CFG.img_col:k[0]}\n        for cat in range(14):\n            row[cat] = 0\n        for cat in list(k[1]['class_id']):\n            if cat != 14:\n                row[cat] = 1\n        df_dicts.append (row)\n    df = pd.DataFrame.from_dict (df_dicts)\n    # df.head ()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"junk = preprocess_train_csv ()\njunk.head ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del junk; gc.collect ()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dual Cutout implementations\nclass CutoutV2 (A.DualTransform):\n    def __init__(\n        self,\n        num_holes=8,\n        max_h_size=8,\n        max_w_size=8,\n        fill_value=0,\n        always_apply=False,\n        p=0.5,\n    ):\n        super(CutoutV2, self).__init__(always_apply, p)\n        self.num_holes = num_holes\n        self.max_h_size = max_h_size\n        self.max_w_size = max_w_size\n        self.fill_value = fill_value\n\n    def apply(self, image, fill_value=0, holes=(), **params):\n        return A.functional.cutout(image, holes, fill_value)\n\n    def get_params_dependent_on_targets(self, params):\n        img = params[\"image\"]\n        height, width = img.shape[:2]\n\n        holes = []\n        for _n in range(self.num_holes):\n            y = random.randint(0, height)\n            x = random.randint(0, width)\n\n            y1 = np.clip(y - self.max_h_size // 2, 0, height)\n            y2 = np.clip(y1 + self.max_h_size, 0, height)\n            x1 = np.clip(x - self.max_w_size // 2, 0, width)\n            x2 = np.clip(x1 + self.max_w_size, 0, width)\n            holes.append((x1, y1, x2, y2))\n\n        return {\"holes\": holes}\n\n    @property\n    def targets_as_params(self):\n        return [\"image\"]\n\n    def get_transform_init_args_names(self):\n        return (\"num_holes\", \"max_h_size\", \"max_w_size\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NOTE: We don't normalize here since it all gets dark\n# if advprop:           # for models using advprop pretrained weights\n#     normalize = transforms.Lambda(lambda img: img * 2.0 - 1.0)\n# else:\n#     normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n\ndef get_transforms (data='train'):\n    \n    light_transforms = A.Compose ([\n        A.Resize (CFG.size, CFG.size),\n        A.HorizontalFlip (p=0.5),\n        A.HueSaturationValue (hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n        A.RandomBrightnessContrast (brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n        A.augmentations.transforms.RGBShift (r_shift_limit=20, g_shift_limit=20, b_shift_limit=20, always_apply=False, p=0.5),\n        A.augmentations.transforms.ChannelDropout (channel_drop_range=(1, 1), fill_value=0, always_apply=False, p=0.5),\n        A.augmentations.transforms.GridDistortion (num_steps=5, distort_limit=0.3, interpolation=1, border_mode=4, value=None, mask_value=None, always_apply=False, p=0.5),\n        A.CoarseDropout(p=0.5),\n        A.Cutout (max_h_size=int(CFG.size * 0.2), max_w_size=int(CFG.size * 0.2), num_holes=2, p=0.75),\n        # A.Normalize (),\n        # ToTensorV2 (p=1.0),\n    ])\n\n    heavy_transforms = A.Compose ([\n        A.RandomResizedCrop(CFG.size, CFG.size, scale=(0.9, 1), p=1), \n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(p=0.5),\n        A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=0.7),\n        A.RandomBrightnessContrast(brightness_limit=(-0.2,0.2), contrast_limit=(-0.2, 0.2), p=0.7),\n        A.CLAHE(clip_limit=(1,4), p=0.5),\n        A.OneOf([\n           A.OpticalDistortion(distort_limit=1.0),\n           A.GridDistortion(num_steps=5, distort_limit=1.),\n           A.ElasticTransform(alpha=3),\n        ], p=0.2),\n        A.OneOf([\n           A.GaussNoise(var_limit=[10, 50]),\n           A.GaussianBlur(),\n           A.MotionBlur(),\n           A.MedianBlur(),\n        ], p=0.2),\n        A.Resize(CFG.size, CFG.size),\n        A.OneOf([\n          JpegCompression(),\n          Downscale(scale_min=0.1, scale_max=0.15),\n        ], p=0.2),\n        IAAPiecewiseAffine(p=0.2),\n        IAASharpen(p=0.2),\n        CutoutV2 (max_h_size=int(CFG.size * 0.4), max_w_size=int(CFG.size * 0.4), num_holes=1, p=0.75),\n        # A.Normalize(),\n        # ToTensorV2 (p=1.0),\n    ])\n\n    train_transform = A.OneOf ([\n        light_transforms,\n        heavy_transforms,\n    ], p=1)\n\n    valid_transforms = A.Compose ([\n        A.Resize (CFG.size, CFG.size),\n        # A.Normalize (),\n        # ToTensorV2 (p=1.0),\n    ])\n    \n        \n    if 'train' in data:\n        return light_transforms # train_transform, heavy_transforms\n    elif 'valid' in data:\n        return valid_transforms\n    return valid_transforms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImgDataset (Dataset):\n    \n    def __init__(self, df, img_file_colname=CFG.img_col, label_cols=CFG.label_cols, \n                 transform=get_transforms(), img_dir=CFG.train_path, img_ext=CFG.img_ext):\n        \n        super ().__init__()\n        self.df               = df.reset_index (drop=True)\n        self.img_ext          = CFG.img_ext\n        self.img_dir          = img_dir\n        self.label_cols       = label_cols\n        self.img_file_colname = img_file_colname\n        self.transform        = transform\n        return\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        \n        file_name = self.df[self.img_file_colname][idx].replace (self.img_ext, '') + self.img_ext\n        file_path = f'{self.img_dir}/{file_name}'\n        image     = cv2.imread (file_path)[:, :, ::-1]                         #;print (image.shape)\n        image     = self.transform (image=image)['image']\n        image     = image.astype (np.float32).transpose (2, 0, 1) / 255.0\n        image     = torch.tensor (image).float ()\n        if len (self.label_cols) > 0:\n            \n            label = torch.tensor (self.df.loc[idx, self.label_cols]).float () # long ()\n            return image, label\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getFolds ():\n    \n    train_folds_df = preprocess_train_csv () # pd.read_csv (CFG.train_csv)\n    label = train_folds_df[CFG.label_cols]\n    if len (CFG.label_cols) > 1:\n        label = train_folds_df[CFG.label_cols[0]]\n        \n    skf = StratifiedKFold (n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\n    for n, (train_index, val_index) in enumerate (skf.split (train_folds_df, label)):\n        train_folds_df.loc[val_index, 'fold'] = int (n)\n    train_folds_df['fold'] = train_folds_df['fold'].astype (int)\n    # print (train_folds_df.groupby (['fold', label]).size ())\n        \n    return train_folds_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_imgs (dataset_show):\n    \n    from pylab import rcParams\n    rcParams['figure.figsize'] = 20,10\n    for i in range (2):\n        f, axarr = plt.subplots (1,5)\n        for p in range (5):\n            idx = np.random.randint (0, len (dataset_show))\n            img, label = dataset_show[idx]                         # ;print (img.size()) ;print (label)\n            # img = img.byte ()\n            axarr[p].imshow (img.permute(1, 2, 0))\n            axarr[p].set_title (idx)\n    return\n\nTR_DATASET = ImgDataset (getFolds (), transform=get_transforms())\nplot_imgs (TR_DATASET)\ndel TR_DATASET\ngc.collect ()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> # Loss Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# this works for only 1 label, not multi-label target.\nclass SmoothBCEwLogits (_WeightedLoss):\n    \n    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    @staticmethod\n    def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):\n        assert 0 <= smoothing < 1\n        with torch.no_grad():\n            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n        return targets\n\n    def forward(self, inputs, targets):\n        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),\n            self.smoothing)\n        loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)\n\n        if  self.reduction == 'sum':\n            loss = loss.sum()\n        elif  self.reduction == 'mean':\n            loss = loss.mean()\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DiceBCELoss (nn.Module):\n    \n    def __init__(self, class_wt=None, size_average=True):\n        super(DiceBCELoss, self).__init__()\n        self.bceWithLogitsLoss = nn.BCEWithLogitsLoss (pos_weight=class_wt)\n\n    def forward(self, logits, targets, smooth=1):\n        \n        #flatten label and prediction tensors\n        logits_flat  = logits.view(-1)                   #;print (\"logits.shape =\", logits.shape)\n        targets_flat = targets.view(-1)                  #;print (\"targets.shape =\", targets.shape)\n        #comment out if your model contains a sigmoid or equivalent activation layer\n        inputs = F.sigmoid (logits_flat)                 #;print (\"inputs.shape =\", inputs.shape)\n        \n        intersection = (inputs * targets_flat).sum()                            \n        dice_loss = 1 - (2.*intersection + smooth)/(inputs.sum() + targets_flat.sum() + smooth)  \n        BCE = self.bceWithLogitsLoss (logits, targets)\n        Dice_BCE = BCE + dice_loss\n        \n        return Dice_BCE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_criterion (class_wt):\n    \n    if CFG.criterion=='CrossEntropyLoss':\n        criterion = LabelSmoothingCrossEntropy ()\n    elif CFG.criterion=='SmoothBCEwithLogits':\n        criterion = SmoothBCEwLogits (smoothing=CFG.smoothing)\n    elif CFG.criterion=='BCEWithLogitsLoss':\n        criterion =  nn.BCEWithLogitsLoss (pos_weight=class_wt)\n    elif CFG.criterion=='dice_bce':\n        criterion = DiceBCELoss (class_wt=class_wt)\n    return criterion","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CBAM\nhttps://github.com/Jongchan/attention-module"},{"metadata":{"trusted":true},"cell_type":"code","source":"class BasicConv(nn.Module):\n    def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1, groups=1, relu=True, bn=True, bias=False):\n        super(BasicConv, self).__init__()\n        self.out_channels = out_planes\n        self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)\n        self.bn = nn.BatchNorm2d(out_planes,eps=1e-5, momentum=0.01, affine=True) if bn else None\n        self.relu = nn.ReLU() if relu else None\n\n    def forward(self, x):\n        x = self.conv(x)\n        if self.bn is not None:\n            x = self.bn(x)\n        if self.relu is not None:\n            x = self.relu(x)\n        return x\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size(0), -1)\n\nclass ChannelGate(nn.Module):\n    def __init__(self, gate_channels, reduction_ratio=16, pool_types=['avg', 'max']):\n        super(ChannelGate, self).__init__()\n        self.gate_channels = gate_channels\n        self.mlp = nn.Sequential(\n            Flatten(),\n            nn.Linear(gate_channels, gate_channels // reduction_ratio),\n            nn.ReLU(),\n            nn.Linear(gate_channels // reduction_ratio, gate_channels)\n            )\n        self.pool_types = pool_types\n    def forward(self, x):\n        channel_att_sum = None\n        for pool_type in self.pool_types:\n            if pool_type=='avg':\n                avg_pool = F.avg_pool2d( x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))\n                channel_att_raw = self.mlp( avg_pool )\n            elif pool_type=='max':\n                max_pool = F.max_pool2d( x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))\n                channel_att_raw = self.mlp( max_pool )\n            elif pool_type=='lp':\n                lp_pool = F.lp_pool2d( x, 2, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))\n                channel_att_raw = self.mlp( lp_pool )\n            elif pool_type=='lse':\n                # LSE pool only\n                lse_pool = logsumexp_2d(x)\n                channel_att_raw = self.mlp( lse_pool )\n\n            if channel_att_sum is None:\n                channel_att_sum = channel_att_raw\n            else:\n                channel_att_sum = channel_att_sum + channel_att_raw\n\n        scale = F.sigmoid( channel_att_sum ).unsqueeze(2).unsqueeze(3).expand_as(x)\n        return x * scale\n\ndef logsumexp_2d(tensor):\n    tensor_flatten = tensor.view(tensor.size(0), tensor.size(1), -1)\n    s, _ = torch.max(tensor_flatten, dim=2, keepdim=True)\n    outputs = s + (tensor_flatten - s).exp().sum(dim=2, keepdim=True).log()\n    return outputs\n\nclass ChannelPool(nn.Module):\n    def forward(self, x):\n        return torch.cat( (torch.max(x,1)[0].unsqueeze(1), torch.mean(x,1).unsqueeze(1)), dim=1 )\n\nclass SpatialGate(nn.Module):\n    def __init__(self):\n        super(SpatialGate, self).__init__()\n        kernel_size = 7\n        self.compress = ChannelPool()\n        self.spatial = BasicConv(2, 1, kernel_size, stride=1, padding=(kernel_size-1) // 2, relu=False)\n    def forward(self, x):\n        x_compress = self.compress(x)\n        x_out = self.spatial(x_compress)\n        scale = F.sigmoid(x_out) # broadcasting\n        return x * scale\n\nclass CBAM(nn.Module):\n    def __init__(self, gate_channels, reduction_ratio=16, pool_types=['avg', 'max'], no_spatial=False):\n        super(CBAM, self).__init__()\n        self.ChannelGate = ChannelGate(gate_channels, reduction_ratio, pool_types)\n        self.no_spatial=no_spatial\n        if not no_spatial:\n            self.SpatialGate = SpatialGate()\n    def forward(self, x):\n        x_out = self.ChannelGate(x)\n        if not self.no_spatial:\n            x_out = self.SpatialGate(x_out)\n        return x_out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ResNet50 Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv3x3(in_planes, out_planes, stride=1):\n    \"3x3 convolution with padding\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=1, bias=False)\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, use_cbam=False):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n        if use_cbam:\n            self.cbam = CBAM( planes * 4, 16 )\n        else:\n            self.cbam = None\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        if not self.cbam is None:\n            out = self.cbam(out)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\nclass ResNet(nn.Module):\n    def __init__(self, block, layers,  network_type, num_classes, att_type=None):\n        self.inplanes = 64\n        super(ResNet, self).__init__()\n        self.network_type = network_type\n        # different model config between ImageNet and CIFAR \n        if network_type == \"ImageNet\":\n            self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)\n            self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n            self.avgpool = nn.AvgPool2d(7)\n        else:\n            self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n\n        if att_type=='BAM':\n            self.bam1 = BAM(64*block.expansion)\n            self.bam2 = BAM(128*block.expansion)\n            self.bam3 = BAM(256*block.expansion)\n        else:\n            self.bam1, self.bam2, self.bam3 = None, None, None\n\n        self.layer1 = self._make_layer(block, 64,  layers[0], att_type=att_type)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, att_type=att_type)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, att_type=att_type)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2, att_type=att_type)\n\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        init.kaiming_normal(self.fc.weight)\n        for key in self.state_dict():\n            if key.split('.')[-1]==\"weight\":\n                if \"conv\" in key:\n                    init.kaiming_normal(self.state_dict()[key], mode='fan_out')\n                if \"bn\" in key:\n                    if \"SpatialGate\" in key:\n                        self.state_dict()[key][...] = 0\n                    else:\n                        self.state_dict()[key][...] = 1\n            elif key.split(\".\")[-1]=='bias':\n                self.state_dict()[key][...] = 0\n        \n        self.pooling = nn.AdaptiveAvgPool2d (1)\n        return\n\n    def _make_layer(self, block, planes, blocks, stride=1, att_type=None):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, use_cbam=att_type=='CBAM'))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, use_cbam=att_type=='CBAM'))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        if self.network_type == \"ImageNet\":\n            x = self.maxpool(x)\n\n        x = self.layer1(x)\n        if not self.bam1 is None:\n            x = self.bam1(x)\n\n        x = self.layer2(x)\n        if not self.bam2 is None:\n            x = self.bam2(x)\n\n        x = self.layer3(x)\n        if not self.bam3 is None:\n            x = self.bam3(x)\n\n        x = self.layer4(x)\n        \n        \n        if self.network_type == \"ImageNet\":\n            x  = self.pooling (x)\n        else:\n            x = F.avg_pool2d(x, 4)\n        x = x.view (x.size (0), -1)\n        x = self.fc (x)\n        return x\n    \n    def freeze (self):\n        # To freeze the residual layers\n        for param in self.parameters ():\n            param.requires_grad = False\n\n        for param in self.fc.parameters ():\n            param.requires_grad = True\n        return\n    \n    def unfreeze (self):\n        # Unfreeze all layers\n        for param in self.parameters ():\n            param.requires_grad = True\n        for param in self.fc.parameters ():\n            param.requires_grad = True\n        return\n\ndef getResNet (network_type=\"ImageNet\", depth=50, num_classes=CFG.target_size, att_type=\"CBAM\"):\n\n    assert network_type in [\"ImageNet\", \"CIFAR10\", \"CIFAR100\"], \"network type should be ImageNet or CIFAR10 / CIFAR100\"\n    assert depth in [50, 101], 'network depth should be 50 or 101'\n    \n    if depth == 50:\n        model = ResNet(Bottleneck, [3, 4, 6, 3], network_type, num_classes, att_type)\n\n    elif depth == 101:\n        model = ResNet(Bottleneck, [3, 4, 23, 3], network_type, num_classes, att_type)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_state_dict (sd):\n    \n    import re\n    state_dict = dict ()\n    for k in sd:\n        k2 = re.sub (r'^module\\.', '', k)\n        # print (k2)\n        state_dict[k2] = sd[k]\n    return state_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_state (model_path, model):\n    \n    state_dict = None\n    try:  # single GPU model_file\n        state_dict = torch.load (model_path, map_location=torch.device ('cpu')) # ['state_dict']\n        # print (state_dict)\n        # state_dict = fix_state_dict (state_dict)\n        model.load_state_dict (state_dict) # (torch.load (model_path, map_location=torch.device ('cpu')), strict=True)\n    except:  # multi GPU model_file\n        state_dict = torch.load (model_path, map_location=torch.device ('cpu'))\n        state_dict = {k[7:] if k.startswith ('module.') else k: state_dict[k] for k in state_dict.keys ()}\n        model.load_state_dict (state_dict)\n    return state_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getModel (fold, isTrain=True):\n    \n    model = getResNet (\"ImageNet\", 50, CFG.target_size, \"CBAM\")\n    if isTrain:\n        \n        # TODO: _infer_ to _train_,   _end to _maxacc\n        model_path = f'{CFG.model_train_path_prefix}/{CFG.model_name}_fold{fold}_start.pth'\n        print (\"loading\", model_path)\n        load_state (model_path, model)\n        \n        # replace the imagenet head with Vin head\n        # n_features = model.fc.in_features\n        # model.model.global_pool = nn.Identity ()\n        # model.fc = nn.Identity ()\n        # model.pooling = nn.AdaptiveAvgPool2d (1)\n        # model.fc_new = nn.Linear (n_features, CFG.target_size)\n    else:\n        \n        # model_path = f'{CFG.model_infer_path_prefix}/{CFG.model_name}_fold{fold}_maxacc.pth'\n        # load_state (model_path, model)\n        pass\n    \n    # torch.save (model.state_dict (), model_path)\n    if CFG.isFreeze:        \n        model.freeze ()\n    else:\n        model.unfreeze ()\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trainer Helpers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_time (elapsed):\n    '''\n    Takes a time in seconds and returns a string hh:mm:ss\n    '''\n    \n    # Round to the nearest second.\n    elapsed_rounded = int(round((elapsed)))\n    \n    # Format as hh:mm:ss\n    return str (datetime.timedelta (seconds=elapsed_rounded))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid (x):  \n    return np.exp (-np.logaddexp (0, -x))\n\ndef compute_metrics (labels, pred_pr, threshold=None):\n    \n    preds   = pred_pr.argmax (-1)\n    if threshold is not None:\n        preds = pred_pr[:,1] >= threshold\n    print ('labels.shape=', labels.shape, 'preds.shape=', preds.shape, '#P+ve=', np.sum (preds), '#A+ve=', np.sum (labels))   #TODO: comment this\n    precision, recall, f1, _ = precision_recall_fscore_support (labels, preds, average='binary')\n    acc     = accuracy_score (labels, preds)\n    mcc     = matthews_corrcoef (labels, preds)   # matthews correlation coefficient\n    auc     = -1\n    try:\n        auc = roc_auc_score (labels, pred_pr[:, 1])\n    except:\n        pass\n    metrics = {\n        'mcc'      : mcc,\n        'accuracy' : acc,\n        'f1'       : f1,\n        'precision': precision,\n        'recall'   : recall,\n        'auc'      : auc\n    }\n    return metrics\n\ndef find_best_thresh (label, pred_p):\n    \n    scores = []\n    thresholds = [1e-5, 1e-4, 1e-3, 1e-2, 0.1, 0.5]\n    for threshold in thresholds:\n        \n        score = compute_metrics (label, pred_p, threshold)\n        score['threshold'] = threshold\n        scores.append (score)\n        \n    df = pd.DataFrame (scores)            #;print (df)\n    # df.set_index('threshold').plot.bar (rot=0, title='threshold vs scores', figsize=(80,10), fontsize=14)\n    df = df.loc[df.f1 == df.f1.max ()]\n    df = df.iloc[0, :]\n    return df.to_dict ()\n\ndef compute_multilabel_binary_metrics (labels, pred_pr, isThreshAdjust=False):\n    \n    # pred_pr = sigmoid (logits)    \n    metrics = []\n    n_class = labels.shape[1]\n    thresholds = []\n    for i in range (n_class):\n        \n        label  = labels[:, i]\n        prob1  = pred_pr[:, i]\n        prob0  = 1 - prob1\n        pred_p = np.hstack ((prob0.reshape ((-1, 1)), prob1.reshape ((-1, 1))))\n        scores = None\n        if isThreshAdjust:\n            scores = find_best_thresh (label, pred_p)\n            thresholds.append (scores['threshold'])\n            print (f\"for class-{i+1}/{n_class} maxF1 thresh={scores['threshold']}\")\n        else:\n            scores = compute_metrics (label, pred_p)\n        metrics.append (scores)\n        \n    # Now Avg over each classes\n    metrics_df = pd.DataFrame (metrics)  \n    auc = list (metrics_df['auc'].values)\n    auc = np.mean ([a for a in auc if a >= 0])\n    metrics_df.drop (columns=['auc'], inplace=True)\n    metrics_df = metrics_df.mean ()\n    metrics_df['auc'] = auc\n    # print (metrics_df)                                       #TODO: comment this\n    if isThreshAdjust:\n        print (\"class wise thresholds =\", thresholds)\n        CFG.thresholds = thresholds\n    return metrics_df.to_dict ()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Trainer"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyTrainer:\n    \n    def __init__(self, fold, model, train_dataset, eval_dataset, criterion, \n                 compute_metrics=compute_multilabel_binary_metrics, checkpoint_path=None, isResume=False):\n        \n        self.fold             = fold\n        self.start_epoch      = 0\n        self.model            = model\n        # load checkpoint\n        if checkpoint_path is not None:\n            if isResume:\n                self.start_epoch = self.load_checkpoint (checkpoint_path, isResume=True) + 1\n            else:\n                self.load_checkpoint (checkpoint_path, isResume=False)\n        self.model            = self.model.to (CFG.device)\n        if CFG.isFreeze:\n            self.model.freeze ()\n        else:\n            self.model.unfreeze ()\n        CFG.MODEL             = self.model\n        self.train_dataset    = train_dataset\n        self.eval_dataset     = eval_dataset\n        self.criterion        = criterion\n        self.compute_metrics  = compute_metrics\n        self.isTrained        = False\n        self.device           = CFG.device\n        self.optimizer        = AdamW (self.model.parameters (), lr=CFG.lr, eps=CFG.adam_epsilon, weight_decay=CFG.weight_decay)\n        self.epochs           = CFG.epochs\n        self.set_dataLoaders ()\n        self.training_stats   = []\n        self.modelFile        = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}.pth\"\n        if eval_dataset is not None:\n            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_val_loss.pth\"\n            self.maxAucModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_val_auc.pth\"\n        else:\n            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_tr_loss.pth\"\n            self.maxAucModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_tr_auc.pth\"\n        \n        self.min_val_loss         = 9999\n        self.min_train_loss       = 9999\n        self.max_val_auc          = -1    \n        return\n    \n    def set_dataLoaders (self):\n        # Create the DataLoaders for our training and validation sets.\n        \n        if isinstance (self.train_dataset, torch.utils.data.IterableDataset):\n            train_sampler = None\n        else:\n            train_sampler = RandomSampler (self.train_dataset)           # Better use RandomSampler\n        train_dataloader  = DataLoader (\n                    self.train_dataset,                                  # The training samples.\n                    sampler     = train_sampler,                           \n                    batch_size  = CFG.train_batch_size,\n                    num_workers = CFG.num_workers,\n                    pin_memory  = True\n        )\n        # train_dataloader  = DataLoader (self.train_dataset, batch_size=CFG.train_batch_size) # TODO: comment this\n        validation_dataloader = None\n        if self.eval_dataset:\n            validation_dataloader = DataLoader (\n                        self.eval_dataset, \n                        sampler     = SequentialSampler (self.eval_dataset),\n                        batch_size  = CFG.eval_batch_size,\n                        num_workers = CFG.num_workers,\n                        pin_memory  = False\n            )\n            # validation_dataloader  = DataLoader (self.eval_dataset, batch_size=CFG.eval_batch_size) # TODO: comment this\n        \n        if type (CFG.warmup_steps) is float:\n            CFG.warmup_steps = int (CFG.warmup_steps * len (train_dataloader))\n        # Total number of training steps is [number of batches] x [number of epochs]\n        num_training_steps = len (train_dataloader) * self.epochs        \n        lr_scheduler = get_cosine_schedule_with_warmup (self.optimizer, num_cycles=CFG.lr_num_cycles,\n                        num_warmup_steps=CFG.warmup_steps, num_training_steps=num_training_steps)\n        \n        if type (CFG.eval_steps) is float:\n            CFG.eval_steps = int (CFG.eval_steps * len (train_dataloader))\n        self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps=train_dataloader, validation_dataloader, lr_scheduler, num_training_steps\n        return\n            \n    def test_iterate_dataloader (self):\n        \n        for step, batch in enumerate (self.train_dataloader):\n            print (step)\n            print (batch)\n            break\n        return\n    \n    def save_checkpoint (self, epoch, path):\n        \n        checkpoint = {\n            'epoch'               : epoch,\n            'model_state_dict'    : self.model.state_dict (),\n            'optimizer_state_dict': self.optimizer.state_dict (),\n            'lr_sched_state_dict' : self.lr_scheduler.state_dict (),\n            'training_stats'      : self.training_stats,\n            'max_val_auc'         : self.max_val_auc,\n            'min_train_loss'      : self.min_train_loss,\n            'min_val_loss'        : self.min_val_loss,\n        }\n        torch.save (checkpoint, path)\n        gc.collect (); torch.cuda.empty_cache ()\n        print (\"saved checkpoint\", path)\n        return\n    \n    def load_checkpoint (self, path, isResume=False):\n        \n        epoch      = 0\n        checkpoint = torch.load (path, map_location=torch.device ('cpu'))\n        self.model.load_state_dict (checkpoint['model_state_dict'])\n        if isResume:\n            \n            self.optimizer.load_state_dict (checkpoint['optimizer_state_dict'])\n            self.lr_scheduler.load_state_dict (checkpoint['lr_sched_state_dict'])\n            epoch = checkpoint['epoch']\n            self.training_stats  = checkpoint['training_stats']\n            self.min_val_loss    = checkpoint['min_val_loss']\n            self.min_train_loss  = checkpoint['min_train_loss']\n            self.max_val_auc     = checkpoint['max_val_auc']\n            print (\"Loaded model, optimizer, and lr_scheduler from -\", path)\n        else:\n            print (\"Loaded model from -\", path)\n            \n        self.model.train ()\n        return epoch\n    \n    def train (self):\n        \n        seed_everything (seed=CFG.seed)\n        step             = 0\n        total_t0         = time.time ()\n        scaler           = GradScaler()\n        for epoch_i in range (self.start_epoch, self.epochs):\n            \n            avg_epoch_train_loss   = 0\n            total_epoch_train_loss = 0\n            print('======== Epoch {:} / {:} ========'.format (epoch_i + 1, self.epochs))\n            t0 = time.time ()\n            self.model.train ()\n            for stp, batch in tqdm (enumerate (self.train_dataloader), total=len(self.train_dataloader)):\n                \n                # Print Stats\n                # if step % CFG.print_every == 0:\n                #     elapsed = format_time (time.time() - t0)\n                #     print ('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format (step, len (self.train_dataloader), elapsed))                \n                if ((CFG.max_steps > 0 and CFG.max_steps < step) or \n                    (CFG.eval_steps>0 and stp==CFG.eval_steps)): # or step==0):   # TODO: rm this comment\n                    \n                    self.save_checkpoint (epoch_i, self.modelFile)\n                    training_time = format_time (time.time () - t0)            \n                    if self.validation_dataloader:\n                        \n                        avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n                        # save this model if the eval loss decreases from the minimum so far\n                        checkpoint_epoch = epoch_i\n                        if stp==CFG.eval_steps:\n                            # don't count this epoch in the checkpoint since this epoch \n                            # has not completed. Hence, checkpoint at prev epoch\n                            checkpoint_epoch = epoch_i-1\n                        if avg_val_loss < self.min_val_loss:                             \n                            self.min_val_loss = avg_val_loss\n                            self.save_checkpoint (checkpoint_epoch, self.minLossModelFile)\n                        if avg_val_auc > self.max_val_auc:\n                            self.max_val_auc = avg_val_auc\n                            self.save_checkpoint (checkpoint_epoch, self.maxAucModelFile)\n                    if CFG.max_steps > 0 and CFG.max_steps < step:\n                        \n                        print (\"\")\n                        print (\"Training complete!\")\n                        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n                        self.isTrained = True\n                        self.model.cpu ()\n                        self.model.eval ()\n                        self.save_checkpoint (epoch_i, self.modelFile)\n                        try:\n                            torch.cuda.empty_cache ()\n                            self.plot_train_stats (self.training_stats)\n                        except:\n                            pass\n                        return pd.DataFrame (self.training_stats)\n                \n                ########################################################\n                # Train\n                ########################################################\n                # self.model.zero_grad ()                        \n                images = batch[0].to (self.device)\n                labels = batch[1].to (self.device)\n                with autocast():\n                    \n                    logits = self.model (images)\n                    loss   = self.criterion (logits, labels)\n                    torch.nn.utils.clip_grad_norm_ (self.model.parameters (), CFG.max_grad_norm)\n                    scaler.scale (loss).backward ()\n                    scaler.step (self.optimizer)\n                    scaler.update ()\n                    self.optimizer.zero_grad ()\n                    self.lr_scheduler.step ()\n                    \n                total_epoch_train_loss += loss.cpu ().item ()\n                avg_epoch_train_loss    = total_epoch_train_loss / (stp+1)\n                step += 1\n            # all steps of an epoch end\n            \n            # Measure how long this epoch took.\n            training_time = format_time (time.time () - t0)            \n            print (\"  Average training loss: {0:.4f}\".format (avg_epoch_train_loss))\n            print (\"  Training epcoh took: {:}\".format (training_time))            \n            if self.validation_dataloader:    \n                \n                avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n                # save this epoch's model if the eval loss decreases from the minimum so far\n                if avg_val_loss < self.min_val_loss:                    \n                    self.min_val_loss = avg_val_loss\n                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n                if avg_val_auc > self.max_val_auc:\n                    self.max_val_auc = avg_val_auc\n                    self.save_checkpoint (epoch_i, self.maxAucModelFile)\n            else:                \n                training_stats.append ({\n                    'epoch'         : epoch_i + 1,\n                    'training_loss' : avg_epoch_train_loss,\n                    'training_time' : training_time,\n                })\n                if avg_train_loss < self.min_train_loss:                     \n                    self.min_train_loss = avg_train_loss\n                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n            self.save_checkpoint (epoch_i, self.modelFile)\n            # 1 epoch end\n        # all epochs end\n        \n        # just get the best class thresholds at the end\n        if self.validation_dataloader:\n            print ('At training end, threshold Adjustment (last row of the train summary DF)')\n            print (self.evaluate (epoch_i, avg_epoch_train_loss, training_time, isThreshAdjust=True))\n            print ('<: avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time')\n        \n        print (\"***** Training complete! *****\")\n        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n        self.isTrained = True\n        self.model.cpu ()\n        self.model.eval ()\n        try:\n            torch.cuda.empty_cache ()\n            self.plot_train_stats (self.training_stats)\n        except:\n            pass\n        return pd.DataFrame (self.training_stats)\n    \n    \n    def evaluate (self, epoch_i, avg_train_loss=999, training_time=999, isThreshAdjust=False):\n        \n        t0           = time.time ()\n        all_labels   = []\n        all_pred_prs = []\n        # Put the model in evaluation mode--the dropout layers behave differently\n        # during evaluation.\n        self.model.eval ()\n\n        # Tracking variables \n        total_eval_mcc       = 0\n        total_eval_f1        = 0\n        total_eval_precision = 0\n        total_eval_recall    = 0\n        total_eval_auc       = 0\n        total_eval_accuracy  = 0\n        total_eval_loss      = 0\n        nb_eval_steps        = 0\n\n        # Evaluate data for one epoch\n        for batch in self.validation_dataloader:\n            with torch.no_grad ():\n                \n                images   = batch[0].to (self.device)\n                labels   = batch[1].to (self.device)\n                logits   = self.model (images)\n                loss     = self.criterion (logits, labels).cpu ().detach ()\n                labels   = labels.cpu ().numpy ()\n                pred_prs = torch.sigmoid (logits.cpu ()).detach ().numpy ()\n                \n            total_eval_loss += loss.item ()            \n            # Calculate the accuracy for this batch of test sentences, and\n            # accumulate it over all batches.\n            all_labels.append (labels)\n            all_pred_prs.append (pred_prs)\n        # all iterations in an epoch end\n        \n        all_labels   = np.vstack (all_labels)\n        all_pred_prs = np.vstack (all_pred_prs)\n        metrics      = self.compute_metrics (all_labels, all_pred_prs, isThreshAdjust)\n        # Report the final accuracy for this validation run.\n        avg_val_f1   = metrics['f1']\n        # print (\"  F1: {0:.4f}\".format (avg_val_f1))\n        avg_val_mcc  = metrics['mcc']\n        # print (\"  MCC: {0:.4f}\".format (avg_val_mcc))\n        avg_val_precision = metrics['precision']\n        # print (\"  Precision: {0:.4f}\".format (avg_val_precision))\n        avg_val_recall = metrics['recall']\n        # print (\"  Recall: {0:.4f}\".format (avg_val_recall))\n        avg_val_auc = metrics['auc']\n        # print (\"  AUC: {0:.4f}\".format (avg_val_auc))\n        avg_val_accuracy = metrics['accuracy']\n        # print (\"  Accuracy: {0:.4f}\".format (avg_val_accuracy))\n        # Calculate the average loss over all of the batches.\n        avg_val_loss = total_eval_loss / len (self.validation_dataloader)\n        # Measure how long the validation run took.\n        validation_time = format_time (time.time () - t0)\n        # print (\"  Validation Loss: {0:.4f}\".format (avg_val_loss))\n        # print (\"  Validation took: {:}\".format (validation_time))\n        \n        self.training_stats.append ({\n                'epoch'         : epoch_i + 1,\n                'training_loss' : avg_train_loss,\n                'eval_loss'     : avg_val_loss,\n                'eval_f1'       : avg_val_f1,\n                'eval_mcc'      : avg_val_mcc, \n                'eval_precision': avg_val_precision,\n                'eval_recall'   : avg_val_recall,\n                'eval_auc'      : avg_val_auc, \n                'eval_accuracy' : avg_val_accuracy,\n                'training_time' : training_time,\n                'eval_time'     : validation_time                   \n        })\n        self.model.train ()\n        print (\"Validation took {:} (h:mm:ss)\".format (format_time (time.time () - t0)))\n        return avg_val_loss, avg_val_f1, avg_val_mcc, avg_val_auc, avg_val_precision, avg_val_recall, avg_val_accuracy, validation_time\n    \n    \n    def plot_train_stats (self, training_stats):\n        \"\"\"\n        Draw Classification Report curve\n        \"\"\"\n        \n        mccs   = accuracies = f1_scores = precisions = recalls = auc = losses = epochs = -1\n        epochs = len (training_stats)\n        if 'eval_mcc' in training_stats[0]:\n            mccs       = [e['eval_mcc'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=mccs,       label='val_mcc')\n        if 'eval_accuracy' in training_stats[0]:\n            accuracies = [e['eval_accuracy'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')\n        if 'eval_f1' in training_stats[0]:\n            f1_scores  = [e['eval_f1'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=f1_scores,  label='val_f1') \n        if 'eval_precision' in training_stats[0]:\n            precisions = [e['eval_precision'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=precisions, label='val_precision')\n        if 'eval_recall' in training_stats[0]:\n            recalls    = [e['eval_recall'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=recalls,    label='val_recall')\n        if 'eval_auc' in training_stats[0]:\n            auc        = [e['eval_auc'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=mccs,       label='val_auc')\n        if 'eval_loss' in training_stats[0]:\n            losses     = [e['eval_loss'] for e in training_stats]\n        if 'training_loss'  in training_stats[0]:\n            tr_losses  = [e['training_loss'] for e in training_stats]\n            sns.lineplot (x=np.arange(1, epochs + 1), y=tr_losses,  label='tr_losses')\n            \n        plt.show ()\n        print ('mccs       :', mccs)\n        print ('accuracies :', accuracies)\n        print ('precisions :', precisions)\n        print ('recalls    :', recalls)\n        print ('f1_scores  :', f1_scores)\n        print ('auc        :', auc)\n        print ('losses     :', losses)\n        print ('tr_losses  :', tr_losses)\n        \n        \n        plt.plot (recalls, precisions, marker='.', label='Prcision-Recall Curve')\n        # axis labels\n        plt.xlabel ('Recall')\n        plt.ylabel ('Precision')\n        # show the legend\n        plt.legend ()\n        # show the plot\n        plt.show ()\n        return\n    \n    \n    def get_model (self):\n        \n        if self.isTrained:\n            return self.model.eval ()\n        return None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def free_gpu_cache ():\n    \n    # print(\"Initial GPU Usage\")\n    # gpu_usage()                             \n\n    torch.cuda.empty_cache()\n\n    # cuda.select_device(0)\n    # cuda.close()\n    # cuda.select_device(0)\n\n    # print(\"GPU Usage after emptying the cache\")\n    # gpu_usage()\n    return\n\n# free_gpu_cache()           ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fold_loop (fold, train_df, checkpoint_path=None):\n\n    print (f\"========== fold: {fold} training ==========\")\n\n    trn_idx        = train_df[train_df['fold'] != fold].index\n    val_idx        = train_df[train_df['fold'] == fold].index\n    train_folds_df = train_df.loc[trn_idx].reset_index (drop=True)\n    valid_folds_df = train_df.loc[val_idx].reset_index (drop=True) \n    class_wt       = 1 / (1 + np.array (train_df[CFG.label_cols].sum (axis=0).values))\n    class_wt       = torch.tensor (class_wt / np.sum (class_wt)).to (CFG.device)   ;print ('class_wt =', class_wt)\n    class_wt       = class_wt.to (CFG.device)\n    \n    criterion      = get_criterion (class_wt=class_wt)\n    model          = CFG.MODEL\n    if model is None and checkpoint_path is None:\n        print (\"CFG.MODEL is None\")\n        model      = getModel (fold, isTrain=True)\n        model      = model.float()\n    elif model is not None and checkpoint_path is not None:\n        checkpoint_path = None\n    elif model is None and checkpoint_path is not None:\n        print (\"CFG.MODEL is None\")\n        model      = getModel (fold, isTrain=False)\n        model      = model.float()\n        \n    train_dataset  = ImgDataset (train_folds_df, transform=get_transforms ('train'))\n    valid_dataset  = ImgDataset (valid_folds_df, transform=get_transforms ('valid'))    \n    trainer        = MyTrainer (\n        fold            = fold,\n        model           = model,\n        train_dataset   = train_dataset,\n        eval_dataset    = valid_dataset,\n        criterion       = criterion,\n        compute_metrics = compute_multilabel_binary_metrics,\n        checkpoint_path = checkpoint_path\n    )\n    metrics = trainer.train ()\n    return metrics\n    \n    # To plot lr uncomment this\n    # lrs = []\n    # for i in range (CFG.epochs*len (trainer.train_dataloader)):\n    #     trainer.lr_scheduler.step ()\n    #     lrs.append (trainer.optimizer.param_groups[0][\"lr\"])\n    # print (lrs)\n    # plt.plot (lrs)\n    # plt.show ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_main (checkpoint_path=None):\n    \n    print (f\"========== train_main() ==========\")\n    train_df = getFolds ()\n    if CFG.isTrain:        \n        valid_scores_df = pd.DataFrame ()\n        for fold in range (CFG.n_fold):\n            if fold in CFG.train_fold:\n                \n                valid_scores_fold_df = train_fold_loop (fold, train_df, checkpoint_path)\n                # valid_scores_fold = np.array (valid_scores_fold).reshape ((1, -1))\n                valid_scores_df = valid_scores_df.append (valid_scores_fold_df)\n                \n        print (f\"========== CV ==========\")\n        # print (valid_scores_df)\n        # valid_scores = np.vstack (valid_scores)\n        # valid_scores = np.mean (valid_scores, axis=0)\n        valid_scores = valid_scores_df.iloc[-1, :]  #.mean ()\n        print (\"CV Scores :-\");  print (valid_scores)\n    return valid_scores_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Single config training"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"!mkdir -p /kaggle/working/Output/\n!touch /kaggle/working/Output/train.log\ngc.collect ()\nmodel_names = timm.list_models (pretrained=True)\nmodel_names = timm.list_models ('*resnet*', pretrained=True)\npprint (model_names)\nLOGGER = init_logger ()\nseed_everything (seed=CFG.seed)"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"valid_scores_df = train_main ()\n\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# To train, uncomment these"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.warmup_steps = 0.5\nCFG.eval_steps   = 0.5\nCFG.num_workers  = 8\nCFG.size = 128         # [128, 224, 384, 512, 640, 1024]\nCFG.train_batch_size = 1024 * 3\nCFG.eval_batch_size  = 1024 * 3\nCFG.freeze = True\nCFG.epochs = 2\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.warmup_steps = 0.5\nCFG.eval_steps   = 0.5\nCFG.num_workers = 8\nCFG.size = 128\nCFG.train_batch_size = 108\nCFG.eval_batch_size  = 108\nCFG.isFreeze = False\nCFG.epochs = 5\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df = train_main ()\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 4\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 224\nCFG.train_batch_size = 888\nCFG.eval_batch_size  = 888\nCFG.isFreeze         = True\nCFG.epochs           = 2\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_224_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 224\nCFG.train_batch_size = 78\nCFG.eval_batch_size  = 78\nCFG.isFreeze         = False\nCFG.epochs           = 5\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main ()\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 4\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 384\nCFG.train_batch_size = 228\nCFG.eval_batch_size  = 228\nCFG.isFreeze         = True\nCFG.epochs           = 2\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_384_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 384\nCFG.train_batch_size = 21\nCFG.eval_batch_size  = 21\nCFG.isFreeze         = False\nCFG.epochs           = 4\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main ()\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 512\nCFG.train_batch_size = 80\nCFG.eval_batch_size  = 80\nCFG.isFreeze         = True\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-cbamresnet50-384-fold0/cbam_resnet50_384_fold0.pth\") \nvalid_scores_df"},{"metadata":{},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 512\nCFG.train_batch_size = 17\nCFG.eval_batch_size  = 17\nCFG.isFreeze         = False\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_384_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 4\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 640\nCFG.train_batch_size = 62\nCFG.eval_batch_size  = 62\nCFG.isFreeze         = True\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () #(checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_640_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 640\nCFG.train_batch_size = 9\nCFG.eval_batch_size  = 9\nCFG.isFreeze         = False\nCFG.epochs           = 6\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main ()\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"gc.collect (); torch.cuda.empty_cache ()\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0.5\nCFG.eval_steps       = 0.5\nCFG.size             = 1024       # grayscale\nCFG.train_batch_size = 14\nCFG.eval_batch_size  = 14\nCFG.isFreeze         = True\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_640_fold0.pth\")\nvalid_scores_df"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"torch.cuda.empty_cache ()\nCFG.lr               = 5e-5\nCFG.lr_num_cycles    = 0.5\nCFG.num_workers      = 8\nCFG.warmup_steps     = 0 # 0.5\nCFG.eval_steps       = 0 # 0.5\nCFG.size             = 1024\nCFG.train_batch_size = 4\nCFG.eval_batch_size  = 4\nCFG.isFreeze         = False\nCFG.epochs           = 1\nprint (f\"***** Training for size={CFG.size} freeze={CFG.isFreeze} *****\")\nvalid_scores_df      = train_main () # (checkpoint_path=\"../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/cbam_resnet50_1024_fold0.pth\")\nvalid_scores_df"},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch-gradcam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gradcam import GradCAM, GradCAMpp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = getModel (0, False)\npath  = \"./cbam_resnet50_fold0_start.pth\"\nstate_dict = torch.load (path, map_location=torch.device ('cpu'))['model_state_dict']\nmodel.load_state_dict (state_dict)\ncheckpoint = {'model_state_dict' : model.state_dict ()}\ntorch.save (checkpoint, path)\npath  = \"./cbam_resnet50_1024_fold0.pth\"\ntorch.save (checkpoint, path)\nmodel.unfreeze ()\nmodel = model.to (CFG.device).eval ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_ids = [f.replace ('.png', '') for f in os.listdir (CFG.test_path)]\ndf = pd.DataFrame ()\ndf[CFG.img_col] = img_ids\ndf.head ()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_example_gradcam (filename, transform=get_transforms('valid')):\n    \n    from torchvision import transforms\n    from gradcam.utils import visualize_cam\n    from torchvision.utils import make_grid, save_image\n    \n    # Specify your file path here\n    img_path = f\"{CFG.test_path}/{filename}\"\n    image    = cv2.imread (img_path)[:, :, ::-1]                         #;print (image.shape)\n    image    = transform (image=image)['image']\n    image    = image.astype (np.float32).transpose (2, 0, 1) / 255.0\n    image    = torch.tensor (image).float ().to (CFG.device)\n    \n    normed_torch_img = image.unsqueeze(0)\n    images = []\n    layer_name = 'layer4'\n    gradcam_config = dict (model_type='resnet', arch=model, layer_name='layer4')\n    gradcam, gradcam_pp = [cls.from_config (**gradcam_config) for cls in (GradCAM, GradCAMpp)]\n    mask, _ = gradcam (normed_torch_img)\n    mask    = mask.cpu ()\n    image   = image.cpu ()\n    heatmap, result = visualize_cam (mask, image)\n    mask_pp, _ = gradcam_pp (normed_torch_img)\n    heatmap_pp, result_pp = visualize_cam (mask_pp, image)\n    images.extend ([image.cpu(), heatmap, heatmap_pp, result, result_pp])\n    grid_image = make_grid (images, nrow=1)\n    return transforms.ToPILImage () (grid_image)\n\ndisplay_example_gradcam (filename='002a34c58c5b758217ed1f584ccbcfe9.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predict_string (pred_probs, img_ids, thresh=CFG.thresholds):\n    \n    pred_strs = []\n    transform = get_transforms ('valid')\n    gradcam_config = dict (model_type='resnet', arch=model, layer_name='layer4')\n    gradcam, gradcam_pp = [cls.from_config (**gradcam_config) for cls in (GradCAM, GradCAMpp)]\n    for img_idx in range (pred_probs.shape[0]):\n        \n        pred_prs = pred_probs[img_idx]\n        pred_bin = (pred_prs >= thresh) + 0\n        if np.sum (pred_bin) == 0:\n            pred_strs.append (\"14 1 0 0 1 1\")\n            continue\n            \n        pred_str = ''\n        for class_id in range (len (pred_bin)):\n            if pred_bin[class_id] == 0:\n                continue\n            \n            # get pr\n            pred_pr = pred_prs[class_id]\n            # get image\n            file_name  = img_ids[img_idx] + CFG.img_ext\n            file_path  = f'{CFG.test_path}/{file_name}'\n            image      = cv2.imread (file_path)[:, :, ::-1]                         #;print (image.shape)\n            image      = transform (image=image)['image']\n            image      = image.astype (np.float32).transpose (2, 0, 1) / 255.0\n            image      = torch.tensor (image).unsqueeze (0).float ().to (CFG.device)\n            mask, _    = gradcam (image, class_id)\n            # mask, _  = gradcam_pp (image, class_id)\n            mask       = mask.squeeze (0).squeeze (0).detach ().cpu ().numpy ()\n            th         = np.percentile (mask, 90)     ;print (\"th =\", th)\n            mask       = ((mask >= th) + 0).astype (np.uint8)  ;print (\"max min (mask) =\", np.max (mask), np.min (mask))\n            contours,_ = cv2.findContours (mask.copy (), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n            x, y, w, h = CFG.size//2, CFG.size//2, CFG.size//4, CFG.size//4\n            if len (contours) > 0:\n                x, y, w, h = cv2.boundingRect (contours[0])\n            pred_str  += f\"{class_id} 0.75 {x} {y} {x+w} {y+h} \"\n            del mask, contours\n            gc.collect ()\n            \n        pred_strs.append (pred_str)\n        # break   \n    # print ('pred_strs =', pred_strs)  \n    return pred_strs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# To infer uncomment this"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"CFG.size = 1024\nCFG.eval_batch_size = 4\n\nds = ImgDataset (df, img_file_colname=CFG.img_col, label_cols=[], transform=get_transforms ('valid'), \n                 img_dir=CFG.test_path, img_ext=CFG.img_ext)\ndl = DataLoader (ds, batch_size=CFG.eval_batch_size)\n\npred_probs = []\nwith torch.no_grad ():\n    for stp, batch in tqdm (enumerate (dl), total=len (dl)):\n        \n        batch = batch.to (CFG.device)\n        probs = torch.sigmoid (model (batch)).detach ().cpu ().numpy ()\n        pred_probs.append (probs)\n        # break\n\npred_probs = np.vstack (pred_probs)\n\n### pred_probs = torch.load ('../input/vin-manual-torch-trainer-fp16-cbamresnet50-gradcam/pred_probs.pt')\n\ntorch.save (pred_probs, 'pred_probs.pt')\npred_strs = get_predict_string (pred_probs, img_ids)\ndf['PredictionString'] = pred_strs\ndf.to_csv ('submission.csv', index=False)\ndf.head ()"},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Done !')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}