{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/panda-cwd/')\nsys.path.append('../input/pandamixupbaselineb0seutao/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# drop efficientnet dependency\n!echo \"class EfficientNet: pass\" > efficientnet_pytorch.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom copy import copy, deepcopy\nfrom pprint import pprint\nimport random\n\nimport skimage.io as io\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets\nfrom torchvision import transforms\nimport torchvision.models as models\nimport torch.utils.data as D\nfrom torchvision import transforms as T\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau\ntry:\n    from apex import amp\n    USE_APEX = True\nexcept:\n    USE_APEX = False\n    \nfrom albumentations import *\nfrom albumentations.pytorch import ToTensor, ToTensorV2\nfrom kuma_utils.nn.training import TorchTrainer\nfrom kuma_utils.nn.logger import Logger\nfrom kuma_utils.nn.snapshot import *\nfrom kuma_utils.metrics import *\n\n# from configs import *\nfrom panda_models import *\nfrom transforms import *\nfrom metrics import sigmoid, OptimizedRounder\nfrom datasets import PandaDataset\nfrom utils import analyse_results","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import collections\nfrom collections import defaultdict, Counter\nfrom functools import partial\nimport math\n\n########################################################################\n############### HELPERS FUNCTIONS FOR MODEL ARCHITECTURE ###############\n########################################################################\nUSE_PRETRAINED = False\n\n# Parameters for the entire model (stem, all blocks, and head)\nGlobalParams = collections.namedtuple('GlobalParams', [\n    'batch_norm_momentum', 'batch_norm_epsilon', 'dropout_rate',\n    'num_classes', 'width_coefficient', 'depth_coefficient',\n    'depth_divisor', 'min_depth', 'drop_connect_rate', 'image_size'])\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple('BlockArgs', [\n    'kernel_size', 'num_repeat', 'input_filters', 'output_filters',\n    'expand_ratio', 'id_skip', 'stride', 'se_ratio'])\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training: return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand([batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device)\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\n\ndef get_width_and_height_from_size(x):\n    \"\"\" Obtains width and height from a int or tuple \"\"\"\n    if isinstance(x, int): return x, x\n    if isinstance(x, list) or isinstance(x, tuple): return x\n    else: raise TypeError()\n\n\ndef calculate_output_image_size(input_image_size, stride):\n    \"\"\" Calculates the output image size when using Conv2dSamePadding with a stride. \n        Necessary for static padding. Thanks to mannatsingh for pointing this out. \"\"\"\n    if input_image_size is None: return None\n    image_height, image_width = get_width_and_height_from_size(input_image_size)\n    stride = stride if isinstance(stride, int) else stride[0]\n    image_height = int(math.ceil(image_height / stride))\n    image_width = int(math.ceil(image_width / stride))\n    return [image_height, image_width]\n\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True):\n        super().__init__(in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2])\n        return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n\n    def __init__(self, in_channels, out_channels, kernel_size, image_size=None, **kwargs):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = (image_size, image_size) if isinstance(image_size, int) else image_size\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d((pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2))\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups)\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self, ):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n\n########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        'efficientnet-b0': (1.0, 1.0, 224, 0.2),\n        'efficientnet-b1': (1.0, 1.1, 240, 0.2),\n        'efficientnet-b2': (1.1, 1.2, 260, 0.3),\n        'efficientnet-b3': (1.2, 1.4, 300, 0.3),\n        'efficientnet-b4': (1.4, 1.8, 380, 0.4),\n        'efficientnet-b5': (1.6, 2.2, 456, 0.4),\n        'efficientnet-b6': (1.8, 2.6, 528, 0.5),\n        'efficientnet-b7': (2.0, 3.1, 600, 0.5),\n        'efficientnet-b8': (2.2, 3.6, 672, 0.5),\n        'efficientnet-l2': (4.3, 5.3, 800, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split('_')\n        options = {}\n        for op in ops:\n            splits = re.split(r'(\\d.*)', op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (('s' in options and len(options['s']) == 1) or\n                (len(options['s']) == 2 and options['s'][0] == options['s'][1]))\n\n        return BlockArgs(\n            kernel_size=int(options['k']),\n            num_repeat=int(options['r']),\n            input_filters=int(options['i']),\n            output_filters=int(options['o']),\n            expand_ratio=int(options['e']),\n            id_skip=('noskip' not in block_string),\n            se_ratio=float(options['se']) if 'se' in options else None,\n            stride=[int(options['s'][0])])\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            'r%d' % block.num_repeat,\n            'k%d' % block.kernel_size,\n            's%d%d' % (block.strides[0], block.strides[1]),\n            'e%s' % block.expand_ratio,\n            'i%d' % block.input_filters,\n            'o%d' % block.output_filters\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append('se%s' % block.se_ratio)\n        if block.id_skip is False:\n            args.append('noskip')\n        return '_'.join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(width_coefficient=None, depth_coefficient=None, dropout_rate=0.2,\n                 drop_connect_rate=0.2, image_size=None, num_classes=1000):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        'r1_k3_s11_e1_i32_o16_se0.25', 'r2_k3_s22_e6_i16_o24_se0.25',\n        'r2_k5_s22_e6_i24_o40_se0.25', 'r3_k3_s22_e6_i40_o80_se0.25',\n        'r3_k5_s11_e6_i80_o112_se0.25', 'r4_k5_s22_e6_i112_o192_se0.25',\n        'r1_k3_s11_e6_i192_o320_se0.25',\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith('efficientnet'):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s)\n    else:\n        raise NotImplementedError('model name is not pre-defined: %s' % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\n    'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth',\n    'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth',\n    'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth',\n    'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth',\n    'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth',\n    'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth',\n    'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth',\n    'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth',\n}\n\n\nurl_map_advprop = {\n    'efficientnet-b0': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b0-b64d5a18.pth',\n    'efficientnet-b1': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b1-0f3ce85a.pth',\n    'efficientnet-b2': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b2-6e9d97e5.pth',\n    'efficientnet-b3': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b3-cdd7c0f4.pth',\n    'efficientnet-b4': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b4-44fb3a87.pth',\n    'efficientnet-b5': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b5-86493f6b.pth',\n    'efficientnet-b6': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b6-ac80338e.pth',\n    'efficientnet-b7': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b7-4652b6dd.pth',\n    'efficientnet-b8': 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/adv-efficientnet-b8-22a8fe65.pth',\n}\n\n\ndef load_pretrained_weights(model, model_name, load_fc=True, advprop=False):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    # AutoAugment or Advprop (different preprocessing)\n    url_map_ = url_map_advprop if advprop else url_map\n    state_dict = model_zoo.load_url(url_map_[model_name])\n    if load_fc:\n        model.load_state_dict(state_dict)\n    else:\n        state_dict.pop('_fc.weight')\n        state_dict.pop('_fc.bias')\n        res = model.load_state_dict(state_dict, strict=False)\n        assert set(res.missing_keys) == set(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'\n    print('Loaded pretrained weights for {}'.format(model_name))\n    \n    \nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params, image_size=None):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1)\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = self._block_args.input_filters * self._block_args.expand_ratio  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            Conv2d = get_same_padding_conv2d(image_size=image_size)\n            self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)\n            self._bn0 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n            # image_size = calculate_output_image_size(image_size, 1) <-- this would do nothing\n        \n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._depthwise_conv = Conv2d(\n            in_channels=oup, out_channels=oup, groups=oup,  # groups makes it depthwise\n            kernel_size=k, stride=s, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=oup, momentum=self._bn_mom, eps=self._bn_eps)\n        image_size = calculate_output_image_size(image_size, s)\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            Conv2d = get_same_padding_conv2d(image_size=(1,1))\n            num_squeezed_channels = max(1, int(self._block_args.input_filters * self._block_args.se_ratio))\n            self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)\n            self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)\n        self._bn2 = nn.BatchNorm2d(num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps)\n        self._swish = MemoryEfficientSwish()\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = self._swish(self._bn0(self._expand_conv(inputs)))\n        x = self._swish(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters\n        if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n\n\nclass EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), 'blocks_args should be a list'\n        assert len(blocks_args) > 0, 'block args must be greater than 0'\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Get stem static or dynamic convolution depending on image size\n        image_size = global_params.image_size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Stem\n        in_channels = 3  # rgb\n        out_channels = round_filters(32, self._global_params)  # number of output channels\n        self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n        self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n        image_size = calculate_output_image_size(image_size, 2)\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(block_args.input_filters, self._global_params),\n                output_filters=round_filters(block_args.output_filters, self._global_params),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params)\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))\n            image_size = calculate_output_image_size(image_size, block_args.stride)\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))\n                # image_size = calculate_output_image_size(image_size, block_args.stride)  # ?\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        Conv2d = get_same_padding_conv2d(image_size=image_size)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)\n\n        # Final linear layer\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self._dropout = nn.Dropout(self._global_params.dropout_rate)\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n        self._swish = MemoryEfficientSwish()\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        for block in self._blocks:\n            block.set_swish(memory_efficient)\n\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        bs = inputs.size(0)\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = self._avg_pooling(x)\n        x = x.view(bs, -1)\n        x = self._dropout(x)\n        x = self._fc(x)\n        return x\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return cls(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(cls, model_name, advprop=False, num_classes=1000, in_channels=3):\n        model = cls.from_name(model_name, override_params={'num_classes': num_classes})\n        load_pretrained_weights(model, model_name, load_fc=(num_classes == 1000), advprop=advprop)\n        model._change_in_channels(in_channels)\n        return model\n    \n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name):\n        \"\"\" Validates model name. \"\"\" \n        valid_models = ['efficientnet-b'+str(i) for i in range(9)]\n        if model_name not in valid_models:\n            raise ValueError('model_name should be one of: ' + ', '.join(valid_models))\n\n    def _change_in_channels(model, in_channels):\n        if in_channels != 3:\n            Conv2d = get_same_padding_conv2d(image_size = model._global_params.image_size)\n            out_channels = round_filters(32, model._global_params)\n            model._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)\n\n            \nclass FeatureEfficientNet(EfficientNet):\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        bs = inputs.size(0)\n        # Convolution layers\n        x = self.extract_features(inputs)\n        return x\n    \n    \nimport skimage.io\nclass PandaDataset(D.Dataset):\n    def __init__(self, images, labels, insts=None, img_size=2, transform=None, bin_label=False,\n                 root_path='', istest=False, return_index=True,\n                 use_cache=False, mixup=False, mixup_alpha=1.0, separate_image=False, cat_insts=False):\n        assert len(images) == len(labels)\n        self.images = images\n        self.labels = labels\n        self.insts = insts\n        self.img_size = img_size\n        self.transform = transform\n        self.bin_label = bin_label\n        self.root = root_path\n        self.istest = istest\n        self.return_index = return_index\n        self.use_cache = use_cache\n        self.cache = {}\n        self.mixup = mixup\n        self.mixup_alpha = mixup_alpha\n        self.separate_image = separate_image\n        self.cat_insts = cat_insts\n\n    def __len__(self):\n        return len(self.images)\n\n    def _load_data(self, idx):\n        cache_loaded = False\n        if idx in self.cache.keys():\n            image = self.cache[idx]\n            cache_loaded = True\n        else:\n            if self.istest:\n                fpath = str(self.root/'test_images'/f'{self.images[idx]}.tiff')\n            else:\n                fpath = str(self.root/'train_images'/f'{self.images[idx]}.tiff')\n            image = skimage.io.MultiImage(fpath)[self.img_size]\n            if self.use_cache and not self.separate_image:\n                self.cache[idx] = image\n\n        if self.transform:\n            if self.separate_image:\n                if cache_loaded:\n                    pass\n                else:\n                    assert 'tile' in self.transform.keys()\n                    image = self.transform['tile'](image=image)['image']  # N x 3 x W x H\n                    if self.use_cache:\n                        self.cache[idx] = image\n                output = []\n                for tile in image:\n                    output.append(self.transform['augmentation'](image=tile)['image'])\n                output = torch.stack(output)\n            else:\n                output = self.transform(image=image)['image']\n\n        label = self.labels[idx]\n\n        if self.insts is not None and self.cat_insts:\n            output = torch.flatten(output)\n            insts = self.insts[idx]\n            if insts == 'karolinska':\n                insts = torch.tensor([0.0])\n            elif insts == 'radboud':\n                insts = torch.tensor([1.0])\n            output = torch.cat((output, insts))\n            \n        return output, label\n\n    def __getitem__(self, idx):\n        image, label = self._load_data(idx)\n\n        if self.mixup:\n            idx2 = np.random.randint(0, len(self.images))\n            lam = np.random.beta(self.mixup_alpha, self.mixup_alpha)\n            image2, label2 = self._load_data(idx2)\n            # image = lam * image + (1 - lam) * image2\n            image = torch.cat([torch.Tensor([lam]), image.view(-1), image2.view(-1)])\n            label = lam * label + (1 - lam) * label2\n\n        if self.bin_label:  # 2: [1, 1, 0, 0, 0] / 3: [1, 1, 1, 0, 0]\n            if self.mixup:\n                label_dec = label - label_int\n                label2 = torch.zeros(5)\n                label2[:label_int] = 1.0\n                if label_int < 5:\n                    label2[label_int] = label_dec\n            else: \n                label2 = torch.zeros(5)\n                label2[:label] = 1\n        else:\n            label2 = label\n\n        if self.return_index:\n            return image, label2, idx\n        else:\n            return image, label2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    #print(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\n\ndef print_config(cfg):\n    items = [\n        'name', \n        # general\n        'patch_size', 'patch_dim',\n        'resume', 'img_size', 'batch_size', 'lr', 'epochs', 'CV', 'seed',\n        # dataset\n        'use_cache', 'separate_image', 'return_index', 'bin_label', 'mixup',\n        # \n        'model', 'criterion', 'metric', 'log_metrics', 'stopper', 'event', 'transform',\n    ]\n    print(f'\\n----- Config -----')\n    for key in items:\n        try:\n            value = eval(f'cfg.{key}')\n            print(f'{key}: {value}')\n        except:\n            print(f'{key}: ERROR')\n    print(f'----- Config -----\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport os\nimport cv2\nimport numpy as np\n\nfrom torch.utils.data import Dataset, DataLoader, Subset\n\nMAX_GRIDS = 200\n\n\n# full_folder = r'../input/prostate-cancer-grade-assessment/train_images'\n# full_folder_png = r'./png'\n# if os.path.exists(full_folder_png):\n#     os.makedirs(full_folder_png)\n\n### Task-specific utilities\ndef encode_gleason(gleason):\n    if gleason == 'negative':\n        a = b = 0\n    else:\n        a, b = gleason.split('+')\n        a, b = int(a), int(b)\n        if a >= 3:\n            a -= 1\n        else:\n            a = 1\n        if b >= 3:\n            b -= 1\n        else:\n            b = 1\n    a = [1 for _ in range(a)] + [0 for _ in range(4 - a)]\n    b = [1 for _ in range(b)] + [0 for _ in range(4 - b)]\n    \n    return a + b\n\ndef Rotate(img, degrees=45):\n \n    (h, w) = img.shape[:2]\n    M = cv2.getRotationMatrix2D((w // 2, h // 2), degrees, 1.0)\n\n    M_ = M[:, :2].T\n    corners = np.array([[w // 2, w // 2], [h // 2, -h // 2]])\n    new_corners = M_ @ corners\n    del_y = max(np.abs(new_corners[0])) - w // 2\n    del_x = max(np.abs(new_corners[1])) - h // 2\n    pad_x, pad_y = int(max(del_x, 0)), int(max(del_y, 0))\n    img = np.pad(img, ((pad_x, pad_x), (pad_y, pad_y), (0, 0)), constant_values=255)\n\n    (h, w) = img.shape[:2]\n    M = cv2.getRotationMatrix2D((w // 2, h // 2), degrees, 1.0)\n    rotated = cv2.warpAffine(img, M, (w, h), cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,\n                             borderValue=(255, 255, 255))\n    return rotated\n\n\n\ndef get_img(png_path, mode = 0, tile_size = 224, patch_dim = 8):\n  \n    \n    if '.png' in png_path:\n        img_ = cv2.imread(png_path)\n    else:\n        img_ = io.MultiImage(png_path)[1]\n        \n    img = cv2.resize(img_, (img_.shape[1], img_.shape[0]), cv2.INTER_AREA)\n\n    mean = 1 - torch.tensor([.485, .456, .406]).float().unsqueeze(1).unsqueeze(1).unsqueeze(0)\n    std = torch.tensor([.229, .224, .225]).float().unsqueeze(1).unsqueeze(1).unsqueeze(0)\n\n  \n    if mode % 2 == 1:\n        img = np.pad(img, ((tile_size // 2, tile_size // 2),\n                           (tile_size // 2, tile_size // 2),\n                           (0, 0)), constant_values=255)\n\n    degrees = (mode % 3) * 30\n    img = Rotate(img, degrees)\n\n    size = tile_size\n\n    if img.shape[0] % size == 0:\n        pad_h = 0\n    else:\n        pad_h = (img.shape[0] // size + 1) * size - img.shape[0]\n\n    if img.shape[1] % size == 0:\n        pad_w = 0\n    else:\n        pad_w = (img.shape[1] // size + 1) * size - img.shape[1]\n\n    img = np.pad(img, ((pad_h, 0), (pad_w, 0), (0, 0)), constant_values=255)\n\n    h_, w_ = img.shape[0] // size, img.shape[1] // size\n    imgs = img.reshape(h_, size, w_, size, 3).transpose(0, 2, 1, 3, 4).reshape(-1, size, size, 3)\n    brightness = imgs.reshape(imgs.shape[0], -1).mean(1)\n    indices = np.argsort(brightness)\n    num_valid = np.array(brightness < 250).astype(np.uint8).sum()\n    indices = indices[:num_valid]\n    img = np.concatenate(imgs[indices], axis=1)\n\n    imgs = np.split(img, img.shape[1] // tile_size, axis=1)\n    drop = (np.random.rand(len(imgs)) < 0.0).astype(np.float32)\n    brightness = np.array([tile.mean() for tile in imgs])\n    brightness = drop * 255 + (1 - drop) * brightness\n    indices = np.argsort(brightness)[:patch_dim * patch_dim]\n    imgs = [imgs[i].astype(np.uint8) for i in indices]\n\n    if len(imgs) < patch_dim * patch_dim:\n        deficit = patch_dim * patch_dim - len(imgs)\n        imgs = np.concatenate([imgs + [np.ones([tile_size , tile_size, 3]) * 255 for _ in range(deficit)]], axis=1)\n\n    img = 255 - np.stack([img for img in imgs], axis=0)\n    img = torch.from_numpy(img).permute(0, 3, 1, 2).float() / 255.\n    img = (img - mean) / std\n    return img\n\n# class PandaDataset_valid(Dataset):\n#     def __init__(self, df):\n#         self.names = df.image_id.values\n#         self.labels = df.isup_grade.values\n#         self.df = df\n\n#     def __getitem__(self, idx):\n#         try:\n#             img = get_img(os.path.join(full_folder, self.names[idx] + '.tiff'))\n            \n#             label = self.labels[idx]\n#             isup = [1 for _ in range(label)] + [0 for _ in range(5 - label)]\n#             gleason = encode_gleason(self.df.gleason_score.values[idx])\n#             label = torch.tensor(isup + gleason)\n\n#             return img , label\n#         except:\n# #           print(idx, 'error')\n#             new_idx = np.random.randint(len(self.labels))\n#             return self.__getitem__(new_idx)\n\n#     def __len__(self):\n#         return len(self.labels)\n    \nclass PandaDataset_Inference(Dataset):\n    def __init__(self, df, tta_mode, folder,tile_size = 224, patch_dim = 8):\n        self.names = df.image_id.values\n        self.df = df\n        self.tta_mode = tta_mode \n        self.folder = folder\n        self.tile_size = tile_size\n        self.patch_dim = patch_dim\n\n    def __getitem__(self, idx):\n#         png = os.path.join(full_folder_png, self.names[idx] + '.png')\n\n        tta_mode = random.randint(0,7)\n        img = get_img(os.path.join(self.folder, self.names[idx] + '.tiff'), tta_mode, self.tile_size, self.patch_dim)\n\n        return img,img\n\n    def __len__(self):\n        return len(self.names)\n    \n# img = get_img(os.path.join(full_folder, '12625a6ae522d7d2168049db06b4a86d.tiff'))\n# print(img.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# dirty code for to_mish, only for b0\nfrom utils_enet import Swish, MemoryEfficientSwish\nfrom activation import Mish\nfrom activation import Swish as Swish_timm\n\ndef to_mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, (nn.ReLU, Swish, MemoryEfficientSwish, Swish_timm)) or child_name == '_swish':\n            print(child_name)\n            setattr(model, child_name, Mish())\n        else:\n            to_mish(child)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class Classification:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 8\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = False\n    cat_insts = False\n    \n    ### Model: Base\n    model = PatchPoolModel2(\n        base_model = senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),\n#         base_model = FeatureEfficientNet.from_name('efficientnet-b2'),\n#         base_model = xception_mod(in_channel=3, num_classes=1000, pretrained=USE_PRETRAINED),\n#         base_model = densenet_mod(torchvision.models.densenet121, pretrained=USE_PRETRAINED),\n        patch_total=patch_dim**2, num_classes=6\n    )\n    \n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n        ### Transform: Concated image\n#         'test': Compose([\n#             HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n#             MakePatches(patch_size, patch_dim,\n#                         criterion='blue_ratio', always_apply=True),\n#             Normalize([0.910, 0.819, 0.878],\n#                       [0.363, 0.499, 0.404], always_apply=True),\n#             ToTensor()\n#         ]),\n    }\n    \n\nclass Classification2:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 8\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = False\n    cat_insts = False\n    \n    ### Model: Base\n    model = PatchPoolModel2(\n        base_model = xception_mod(in_channel=3, num_classes=1000, pretrained=USE_PRETRAINED),\n        patch_total=patch_dim**2, num_classes=6\n    )\n    \n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n\n\nclass Regression:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 12\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = False\n    cat_insts = False\n    \n    ### Model: Base\n    model = PatchPoolModel2(\n        base_model=senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),\n        patch_total=patch_dim**2, num_classes=1\n    )\n    \n    transform = {\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n    \n    \nclass OrdinalRegressionBear:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 6\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = PatchPoolModel2(\n        base_model = senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),\n        patch_total=patch_dim**2, num_classes=5\n    )\n\n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n    \nclass OrdinalRegressionBear101:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 6\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = PatchPoolModel2(\n        base_model = senet_mod(se_resnext101_32x4d, pretrained=USE_PRETRAINED),\n        patch_total=patch_dim**2, num_classes=5\n    )\n\n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n\nclass OrdinalRegression0:\n    \n    # General\n    img_size = 1\n    patch_size = 256\n    patch_dim = 6\n    batch_size = 6\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = PatchPoolModel2(\n        base_model=FeatureEfficientNet.from_name('efficientnet-b0'),\n        patch_total=patch_dim**2, num_classes=5\n    )\n    \n    to_mish(model)\n\n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n    \nclass OrdinalRegression1:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 8\n    batch_size = 6\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = PatchPoolModel2(\n        base_model=FeatureEfficientNet.from_name('efficientnet-b0'),\n        patch_total=patch_dim**2, num_classes=5\n    )\n    \n    to_mish(model)\n\n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n    \nclass OrdinalRegression2:\n    \n    # General\n    img_size = 1\n    patch_size = 192\n    patch_dim = 8\n    batch_size = 6\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = PatchPoolModel2(\n        base_model=FeatureEfficientNet.from_name('efficientnet-b0'),\n        patch_total=patch_dim**2, num_classes=5\n    )\n    \n    to_mish(model)\n\n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n    \n\nclass MixOrdinalRegression:\n    \n    # General\n    img_size = 1\n    patch_size = 224\n    patch_dim = 7\n    batch_size = 8\n\n    # Dataset\n    separate_image = True\n    return_index = False\n    bin_label = True\n    cat_insts = False\n    \n    ### Model: Bin label\n    model = MixPatchPoolModel(\n        # base_model=senet_mod(se_resnext50_32x4d, pretrained=USE_PRETRAINED),\n        # base_model=resnest_mod(torchvision.models.resnet34, pretrained=USE_PRETRAINED),\n        base_model=FeatureEfficientNet.from_name('efficientnet-b0'),\n        patch_size=patch_size, patch_total=patch_dim**2, num_classes=5,\n    )\n    \n    transform = {\n        ### Transform: Base\n        'test': {\n            'tile': Compose([\n                MakePatches(patch_size, patch_dim, concat=False, always_apply=True)\n            ]),\n            'augmentation': Compose([\n                ShiftScaleRotate(scale_limit=0.0625, rotate_limit=15, p=0.5),\n                HorizontalFlip(p=0.5), VerticalFlip(p=0.5),\n                Normalize([0.910, 0.819, 0.878],\n                          [0.363, 0.499, 0.404], always_apply=True),\n                ToTensor()\n            ])\n        },\n    }\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_setting = [\n#      {\n#         'config': OrdinalRegressionBear,\n#         'snapshot_path': [\n#               '../input/pandabest-model-reproduction/fold0.pt',\n#              '../input/pandabest-model-reproduction/fold1.pt',\n#              '../input/pandabest-model-reproduction/fold2.pt',\n#              '../input/pandabest-model-reproduction/fold3.pt',\n#              '../input/pandabest-model-reproduction/fold4.pt',\n            \n#             ],\n#         'oof_path':  [\n#             '../input/pandabest-model-reproduction/oof0.npy',\n#             '../input/pandabest-model-reproduction/oof1.npy',\n#             '../input/pandabest-model-reproduction/oof2.npy',\n#             '../input/pandabest-model-reproduction/oof3.npy',\n#             '../input/pandabest-model-reproduction/oof4.npy',\n#         ],\n#     }\n#     ,\n    \n#     {\n#         'config': OrdinalRegressionBear,\n#         'snapshot_path': [\n#             '../input/panda-ckpts-seutao/fold0.pt',\n#             '../input/panda-ckpts-seutao/fold1.pt',\n#             '../input/panda-ckpts-seutao/fold2.pt',\n#             '../input/panda-ckpts-seutao/fold3.pt',\n#             '../input/panda-ckpts-seutao/fold4.pt',\n            \n#             ],\n#         'oof_path':  [\n#             '../input/panda-ckpts-seutao/oof0.npy',\n#             '../input/panda-ckpts-seutao/oof1.npy',\n#             '../input/panda-ckpts-seutao/oof2.npy',\n#             '../input/panda-ckpts-seutao/oof3.npy',\n#             '../input/panda-ckpts-seutao/oof4.npy',\n#         ],\n#     }\n#     ,\n#         {\n#         'config': OrdinalRegressionBear101,\n#         'snapshot_path': [\n#             '../input/panda-ckpts-seutao/se101_fold0.pt',\n#             '../input/panda-ckpts-seutao/se101_fold1.pt',\n#             '../input/panda-ckpts-seutao/se101_fold2.pt',\n#             '../input/panda-ckpts-seutao/se101_fold3.pt',\n#             '../input/panda-ckpts-seutao/se101_fold4.pt',\n            \n#             ],\n#         'oof_path':  [\n#             '../input/panda-ckpts-seutao/se101_oof.npy',\n#         ],\n#     }\n# ]\n\nsubmission_setting_ = [\n#     {\n#         'config': OrdinalRegression0,\n#         'snapshot_path': [\n#             '../input/pandamixupbaselineb0seutao/mixup_baseline/fold_0_45.pt',\n#             '../input/pandamixupbaselineb0seutao/mixup_baseline/fold_1_38.pt',\n#             '../input/pandamixupbaselineb0seutao/mixup_baseline/fold_2_49.pt',\n#             '../input/pandamixupbaselineb0seutao/mixup_baseline/fold_3_48.pt',\n#             '../input/pandamixupbaselineb0seutao/mixup_baseline/fold_4_46.pt'\n#             ],\n#         'oof_path':  [\n#             '../input/pandamixupbaselineb0seutao/fold_0_45_oof.npy',\n#             '../input/pandamixupbaselineb0seutao/fold_1_38_oof.npy',\n#             '../input/pandamixupbaselineb0seutao/fold_2_49_oof.npy',\n#             '../input/pandamixupbaselineb0seutao/fold_3_48_oof.npy',\n#             '../input/pandamixupbaselineb0seutao/fold_4_46_oof.npy',\n#         ],\n#     },\n    \n        {\n        'config': OrdinalRegression1,\n        'snapshot_path': [\n            '../input/bigalphamixup/fold_0_46.pt',\n            '../input/bigalphamixup/fold_1_45.pt',\n            '../input/bigalphamixup/fold_2_40.pt',\n            '../input/bigalphamixup/fold_3_45.pt',\n            '../input/bigalphamixup/fold_4_40.pt'\n            ],\n        'oof_path':  [\n            '../input/bigalphamixup/mixup_large_alpha_oof_224_-1.npy',\n        ],\n        },\n    \n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_PATH = Path('../input/prostate-cancer-grade-assessment/')\nUSE_PRETRAINED = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import  random\n\ndef simple_inference_(snapshots, df, cfg, tta, folder):\n    \n    print(cfg.patch_size, cfg.patch_dim)\n    images = df.image_id.values\n    insts = df.data_provider.values\n    labels = np.zeros(len(df), dtype=int)\n    \n    predictions = np.zeros((tta, len(snapshots), len(df)), dtype=np.float16)\n    model = deepcopy(cfg.model)\n    model.to(device)\n\n    ds = PandaDataset_Inference(df, None, folder, cfg.patch_size, cfg.patch_dim)\n    loader = D.DataLoader(ds, batch_size=cfg.batch_size, shuffle=False, num_workers=2)\n    \n    with torch.no_grad():\n        for tta_i, tta_fold in enumerate(range(tta)):\n            \n            pred_fold = []    \n            for idx, (inputs, _) in enumerate(tqdm(loader, total=len(loader), desc=f'tta{tta_i}')):\n                pred_batch = np.zeros((len(snapshots), len(inputs)), dtype=np.float16)\n                \n                for fold_i, fold_snapshot in enumerate(snapshots):\n                    model.load_state_dict(torch.jit.load(fold_snapshot, map_location='cpu').state_dict())\n                    model.eval()\n                    inputs = inputs.to(device)\n                    outputs = model(inputs)\n                    if outputs.shape[1] == 6: # Classification\n                        outputs = F.softmax(outputs, dim=1).cpu().detach().numpy()\n                        outputs = np.dot(outputs, np.arange(6))\n                    elif outputs.shape[1] == 7: # DACClassification\n                        outputs = F.softmax(outputs[:, :6], dim=1).cpu().detach().numpy()\n                        outputs = np.dot(outputs, np.arange(6))\n                    elif outputs.shape[1] == 5: # OrdinalRegression\n                        outputs = outputs.sigmoid().sum(1).cpu().detach().numpy()\n                    elif outputs.shape[1] == 1: # Regression\n                        outputs = outputs.reshape(-1).cpu().detach().numpy()\n                    pred_batch[fold_i] = outputs\n                pred_fold.append(pred_batch)\n                \n            predictions[tta_i] = np.concatenate(pred_fold, axis=1)\n                \n    return predictions\n\ndef simple_inference(snapshots, df, cfg, tta=1, folder=None):\n    images = df.image_id.values\n    insts = df.data_provider.values\n    labels = np.zeros(len(df), dtype=int)\n    ds = PandaDataset(\n        images=images, labels=labels, insts=insts,\n        img_size=cfg.img_size, transform=cfg.transform['test'],\n        return_index=cfg.return_index, bin_label=cfg.bin_label,\n        separate_image=cfg.separate_image, cat_insts=cfg.cat_insts,\n        root_path=INPUT_PATH, istest=IS_TEST)\n    loader = D.DataLoader(ds, batch_size=cfg.batch_size, shuffle=False, num_workers=2)\n    predictions = np.zeros((tta, len(snapshots), len(df)), dtype=np.float16)\n    model = deepcopy(cfg.model)\n    model.to(device)\n    \n    with torch.no_grad():\n        for tta_i, tta_fold in enumerate(range(tta)):\n            pred_fold = []    \n            for idx, (inputs, _) in enumerate(tqdm(loader, total=len(loader), desc=f'tta{tta_i}')):\n                pred_batch = np.zeros((len(snapshots), len(inputs)), dtype=np.float16)\n                for fold_i, fold_snapshot in enumerate(snapshots):\n                    load_snapshots_to_model(fold_snapshot, model=model)\n                    model.eval()\n                    inputs = inputs.to(device)\n                    outputs = model(inputs)\n                    if outputs.shape[1] == 6: # Classification\n                        outputs = F.softmax(outputs, dim=1).cpu().detach().numpy()\n                        outputs = np.dot(outputs, np.arange(6))\n                    elif outputs.shape[1] == 7: # DACClassification\n                        outputs = F.softmax(outputs[:, :6], dim=1).cpu().detach().numpy()\n                        outputs = np.dot(outputs, np.arange(6))\n                    elif outputs.shape[1] == 5: # OrdinalRegression\n                        outputs = outputs.sigmoid().sum(1).cpu().detach().numpy()\n                    elif outputs.shape[1] == 1: # Regression\n                        outputs = outputs.reshape(-1).cpu().detach().numpy()\n                    pred_batch[fold_i] = outputs\n                pred_fold.append(pred_batch)\n            predictions[tta_i] = np.concatenate(pred_fold, axis=1)\n                \n    return predictions\n    \n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(INPUT_PATH/'train.csv')\ntest_df = pd.read_csv(INPUT_PATH/'test.csv')\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nTTA = 2\n\nif (INPUT_PATH/'test_images').exists():\n    ### Inference\n    print('Inference')\n    target = test_df\n    IS_TEST = True\n    folder = INPUT_PATH/'test_images'\nelse:\n    print('Debug')\n    target = train_df.head(10)\n    IS_TEST = False\n    TTA = 1\n    folder = INPUT_PATH/'train_images'\n\nseed_everything(2020)\ntarget.head()\n\n# all_predictions = []\n# for c in submission_setting:\n#     all_predictions.append(simple_inference(c['snapshot_path'], target, c['config'], TTA, folder))\n# all_predictions = np.concatenate(all_predictions)\n\nall_predictions_ = []\nfor c in submission_setting_:\n    all_predictions_.append(simple_inference_(c['snapshot_path'], target, c['config'], TTA, folder))\nall_predictions = np.concatenate(all_predictions_)\n\nprint(all_predictions.shape)\n# print(all_predictions_.shape)\n# all_predictions = np.concatenate([all_predictions, all_predictions_])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(all_predictions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_setting","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for cfg in submission_setting:\n#     print_config(cfg['config'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy.stats as stats\n\ndef logits_to_predictions(pred):\n    return np.dot(F.softmax(torch.Tensor(pred), 1).numpy(), np.arange(6))\n\ndef soft_vote(pred):\n    return np.clip(np.mean(pred, axis=0).round(), 0, 5).astype(int)\n\ndef hard_vote(pred):\n    return np.clip(stats.mode(pred.round().astype(int), axis=0)[0][0], 0, 5).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.zeros(len(target), dtype=int)\nprint(all_predictions)\npredictions = soft_vote(all_predictions.mean(1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_ = target.isup_grade.values\nqwk = QWK(6)\nprint(qwk(labels_, predictions))\nprint(predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimize threshold for QWK","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.stats import mode\nOPTIMIZE_QWK = True\nHARD_VOTE = False\nlimit_loss = 1 - 1/12\n\nmean_predictions = all_predictions.mean(1)\nhard_predictions = np.zeros_like(mean_predictions)\nprint(mean_predictions.shape)\nprint(hard_predictions.shape)\n\n\nif OPTIMIZE_QWK:\n\n    oof_all_list = []\n    for ic, c in enumerate(submission_setting_):\n        oof_list = []\n        print('============================================')\n        for npy in c['oof_path']:\n            print(npy)\n            oof = np.load(npy)\n            oof_list.append(oof)\n        oof_all_list.append(np.sum(oof_list, axis=0))\n    \n    all_coefs = []\n    for i_oof, oof_fold in enumerate(oof_all_list):\n            print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')\n            print(len(oof_all_list))\n\n            mask_fold = oof_fold.sum(axis=1) != 0\n            print(mask_fold.shape)\n            print(np.sum(mask_fold))\n            output_shape = oof_fold.shape\n            if output_shape[0] == 9555:\n                _train = pd.read_csv('../input/mixup-baseline-b0-224-8-train2/train2.csv')\n                _train = _train.query('certain_flag2 == 1')\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n            elif output_shape[0] == 8493:\n                _train = pd.read_csv('../input/panda-seed-gacha/gatcha_train_nr20.csv')\n                _train = _train.query('certain_flag2 == 1')\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n            elif output_shape[0] == 10086:\n                _train = pd.read_csv('../input/panda-seed-gacha/gatcha_train_nr5.csv')\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n            elif output_shape[0] == 9024:\n                _train = pd.read_csv('../input/panda-seed-gacha/gatcha_train_nr15.csv')\n                _train = _train.query('certain_flag2 == 1')\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n                labels = df.isup_grade.values\n                insts = df.data_provider.values\n            elif output_shape[0] == 10615:\n                _train = pd.read_csv('../input/panda-seed-gacha/train.csv')\n                _train = _train[_train.image_id.values != '3790f55cad63053e956fb73027179707'].reset_index(drop=True)\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n            else:\n                _train = pd.read_csv('../input/panda-seed-gacha/train.csv')\n                labels = _train.isup_grade.values\n                insts = _train.data_provider.values\n                \n\n            if output_shape[1] == 5: # Ordinal Regression\n                oof_fold = torch.from_numpy(oof_fold[mask_fold]).float()\n                label_fold = torch.zeros_like(oof_fold)\n                for i, t in enumerate(labels[mask_fold]):\n                    if t == 0:\n                        continue\n                    label_fold[i, 0:t] = 1\n                criterion = nn.BCEWithLogitsLoss(reduction='none')\n                loss_values = criterion(oof_fold, label_fold).mean(1)\n\n                _, certainty_mask = loss_values.topk(round(len(oof_fold)*limit_loss), largest=False)\n                oof_to_use = oof_fold[certainty_mask].sigmoid().sum(1).numpy()\n                label_to_use = label_fold[certainty_mask].sum(1).numpy()\n                analyse_results(oof_to_use.round(), label_to_use)\n                \n            optR = OptimizedRounder()\n            optR.fit(oof_to_use, label_to_use)\n            coefficients = optR.coefficients()\n            print(coefficients)\n            all_coefs.append(coefficients)\n            hard_predictions[i_oof] = optR.predict(mean_predictions[i_oof], coefficients)\n            analyse_results(optR.predict(oof_to_use, coefficients), label_to_use)\n            \n    all_coefs = np.vstack(all_coefs)\n    print(all_coefs)\n    print(all_coefs.mean(0))\n    soft_predictions = optR.predict(mean_predictions.mean(0), all_coefs.mean(0))\n\n    if HARD_VOTE:\n        print('HARD_VOTE')\n        print(hard_predictions)\n        optimized_predictions = mode(hard_predictions)[0][0]\n        print(optimized_predictions)\n    else:\n        print('SOFT_PRED')\n        print(mean_predictions)\n        print(soft_predictions)\n        optimized_predictions = soft_predictions\n        \nelse:\n    print(all_predictions.mean(0).mean(0))\n    optimized_predictions = all_predictions.mean(0).mean(0).round()\n    print(optimized_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(INPUT_PATH/'sample_submission.csv')\nsubmission['isup_grade'] = optimized_predictions.astype(int)\nsubmission.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if (INPUT_PATH/'test_images').exists():\n    if submission['isup_grade'].sum() > 0:\n        submission.to_csv('submission.csv', index=False)\n    else:\n        pass\nelse:\n    submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}