{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport pandas as pd\nimport os\nimport torchvision.transforms as tr\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nimport pydicom\nimport glob\nimport collections\nfrom datetime import datetime\nfrom skimage import measure\nfrom skimage.measure import block_reduce\nfrom matplotlib import pyplot as plt\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nfrom skimage.transform import resize\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.autograd import Variable\nimport seaborn as sns\nfrom collections import defaultdict","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-18T02:24:32.801121Z","iopub.status.idle":"2023-01-18T02:24:32.803243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:09.133865Z","iopub.execute_input":"2023-01-18T01:35:09.134211Z","iopub.status.idle":"2023-01-18T01:35:09.138333Z","shell.execute_reply.started":"2023-01-18T01:35:09.13418Z","shell.execute_reply":"2023-01-18T01:35:09.137097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:10.130442Z","iopub.execute_input":"2023-01-18T01:35:10.130767Z","iopub.status.idle":"2023-01-18T01:35:10.301697Z","shell.execute_reply.started":"2023-01-18T01:35:10.130735Z","shell.execute_reply":"2023-01-18T01:35:10.300972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_file = os.listdir('../input/vinbigdata-chest-xray-abnormalities-detection/train')\ninput_files = []\nfor ip in input_file:\n    input_files.append(ip.split('.')[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:11.260591Z","iopub.execute_input":"2023-01-18T01:35:11.260945Z","iopub.status.idle":"2023-01-18T01:35:11.926703Z","shell.execute_reply.started":"2023-01-18T01:35:11.26091Z","shell.execute_reply":"2023-01-18T01:35:11.925754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(input_files)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:12.954628Z","iopub.execute_input":"2023-01-18T01:35:12.954985Z","iopub.status.idle":"2023-01-18T01:35:12.960984Z","shell.execute_reply.started":"2023-01-18T01:35:12.954953Z","shell.execute_reply":"2023-01-18T01:35:12.960062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['image_id']==input_files[3]].sort_values(by=['class_id'])","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:14.000343Z","iopub.execute_input":"2023-01-18T01:35:14.000671Z","iopub.status.idle":"2023-01-18T01:35:14.022194Z","shell.execute_reply.started":"2023-01-18T01:35:14.000639Z","shell.execute_reply":"2023-01-18T01:35:14.021202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class traindataset(torch.utils.data.Dataset):\n\n    def __init__(self, df, file_list, transform):\n        super().__init__()\n        \n        self.file_list = file_list\n        self.df = df\n    \n    def __len__(self) :\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        \n        img_id = self.file_list[idx]\n        df = self.df\n        s = 224\n        N = 15\n        d_f = df[df['image_id']==img_id]\n        dff = d_f.sort_values(by=['class_id'])\n        class_id = dff['class_id'].values.tolist()\n        img_pxl = pydicom.read_file('../input/vinbigdata-chest-xray-abnormalities-detection/train/'+img_id+'.dicom').pixel_array\n        img_res = resize(img_pxl,(s,s),anti_aliasing=True)\n        img_np = img_res.astype(np.float32())\n        img_tr = torch.from_numpy(img_np)\n        x_ = s/img_pxl.shape[1]\n        y_ = s/img_pxl.shape[0]\n        xmin = [x*x_ for x in dff['x_min'].values.tolist()]\n        ymin = [y*y_ for y in dff['y_min'].values.tolist()]\n        xmax = [x1*x_ for x1 in dff['x_max'].values.tolist()]\n        ymax = [y1*y_ for y1 in dff['y_max'].values.tolist()]\n        #bbox = []\n        #for z in range(len(xmin)):\n        #    bbox.append([xmin[z],ymin[z],xmax[z],ymax[z]])\n        mask = np.zeros((N,s,s))\n        for k,m in enumerate(class_id):\n            if m != 14:\n                x1,x2,y1,y2 = int(xmin[k]),int(xmax[k]),int(ymin[k]),int(ymax[k])\n                mask[m,y1:y2,x1:x2] = 1\n        mask_numpy = mask.astype(np.float32())\n        mask_tensor = torch.from_numpy(mask_numpy)\n        #mask_tensor = np.transpose(mask_tensor, (2,0,1))\n        return img_tr,mask_tensor","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:14.900873Z","iopub.execute_input":"2023-01-18T01:35:14.901244Z","iopub.status.idle":"2023-01-18T01:35:14.912825Z","shell.execute_reply.started":"2023-01-18T01:35:14.901214Z","shell.execute_reply":"2023-01-18T01:35:14.911798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traindata = traindataset(file_list = input_files,df =df,transform = None)\ndata_loader = torch.utils.data.DataLoader(traindata, batch_size=1, shuffle=True, num_workers=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:15.840398Z","iopub.execute_input":"2023-01-18T01:35:15.840738Z","iopub.status.idle":"2023-01-18T01:35:15.84564Z","shell.execute_reply.started":"2023-01-18T01:35:15.840707Z","shell.execute_reply":"2023-01-18T01:35:15.844396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.patches import Rectangle\nfor k,kk in enumerate(traindata):\n    plt.imshow(kk[0])\n    for org in kk[1][3]:\n        plt.gca().add_patch(Rectangle((org[0], org[1]), (org[2]-org[0]), (org[3]-org[1]),linewidth=1,edgecolor='b',facecolor='none'))\n    plt.show()\n    \n    if k ==3:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:35:16.72651Z","iopub.execute_input":"2023-01-18T01:35:16.726867Z","iopub.status.idle":"2023-01-18T01:35:27.288494Z","shell.execute_reply.started":"2023-01-18T01:35:16.726834Z","shell.execute_reply":"2023-01-18T01:35:27.287535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nimport re\n\n\nnfnet_params = {\n    'F0': {\n        'width': [256, 512, 1536, 1536], 'depth': [1, 2, 6, 3], 'drop_rate': 0.2},\n    'F1': {\n        'width': [256, 512, 1536, 1536], 'depth': [2, 4, 12, 6], 'drop_rate': 0.3},\n    'F2': {\n        'width': [256, 512, 1536, 1536], 'depth': [3, 6, 18, 9], 'drop_rate': 0.4},\n    'F3': {\n        'width': [256, 512, 1536, 1536], 'depth': [4, 8, 24, 12], 'drop_rate': 0.4},\n    'F4': {\n        'width': [256, 512, 1536, 1536], 'depth': [5, 10, 30, 15], 'drop_rate': 0.5},\n    'F5': {\n        'width': [256, 512, 1536, 1536], 'depth': [6, 12, 36, 18], 'drop_rate': 0.5},\n    'F6': {\n        'width': [256, 512, 1536, 1536], 'depth': [7, 14, 42, 21], 'drop_rate': 0.5},\n    'F7': {\n        'width': [256, 512, 1536, 1536], 'depth': [8, 16, 48, 24], 'drop_rate': 0.5},\n}\n\n# These extra constant values ensure that the activations\n# are variance preserving\nclass VPGELU(nn.Module):\n    def forward(self, input: torch.Tensor) -> torch.Tensor:\n        return F.gelu(input) * 1.7015043497085571\n\nclass VPReLU(nn.Module):\n    def forward(self, input: torch.Tensor) -> torch.Tensor:\n        return F.relu(input, inplace=True) * 1.7139588594436646\n\nactivations_dict = {\n    'gelu': VPGELU(),\n    'relu': VPReLU()\n}\n\nclass NFNet(nn.Module):\n    def __init__(self, num_classes:int, variant:str='F0', stochdepth_rate:float=None, \n        alpha:float=0.2, se_ratio:float=0.5, activation:str='gelu'):\n        super(NFNet, self).__init__()\n\n        if not variant in nfnet_params:\n            raise RuntimeError(f\"Variant {variant} does not exist and could not be loaded.\")\n\n        block_params = nfnet_params[variant]\n\n        self.activation = activations_dict[activation]\n        self.drop_rate = block_params['drop_rate']\n        self.num_classes = num_classes\n\n        self.stem = Stem(activation=activation)\n\n        num_blocks, index = sum(block_params['depth']), 0\n\n        blocks = []\n        expected_std = 1.0\n        in_channels = block_params['width'][0] // 2\n\n        block_args = zip(\n            block_params['width'],\n            block_params['depth'],\n            [0.5] * 4, # bottleneck pattern\n            [128] * 4, # group pattern. Original groups [128] * 4\n            [1, 2, 2, 2] # stride pattern\n        )\n\n        for (block_width, stage_depth, expand_ratio, group_size, stride) in block_args:\n            for block_index in range(stage_depth):\n                beta = 1. / expected_std\n\n                block_sd_rate = stochdepth_rate * index / num_blocks\n                out_channels = block_width\n\n                blocks.append(NFBlock(\n                    in_channels=in_channels, \n                    out_channels=out_channels,\n                    stride=stride if block_index == 0 else 1,\n                    alpha=alpha,\n                    beta=beta,\n                    se_ratio=se_ratio,\n                    group_size=group_size,\n                    stochdepth_rate=block_sd_rate,\n                    activation=activation))\n\n                in_channels = out_channels\n                index += 1\n\n                if block_index == 0:\n                    expected_std = 1.0\n                \n                expected_std = (expected_std **2 + alpha**2)**0.5\n\n        self.body = nn.Sequential(*blocks)\n\n        final_conv_channels = 2*in_channels\n        self.final_conv = WSConv2D(in_channels=out_channels, out_channels=final_conv_channels, kernel_size=1)\n        self.pool = nn.AvgPool2d(1)\n        \n        if self.drop_rate > 0.:\n            self.dropout = nn.Dropout(self.drop_rate)\n\n        self.linear = nn.Linear(final_conv_channels, self.num_classes)\n        nn.init.normal_(self.linear.weight, 0, 0.01)\n\n    def forward(self, x):\n        out = self.stem(x)\n        out = self.body(out)\n        out = self.activation(self.final_conv(out))\n        pool = torch.mean(out, dim=(2,3))\n\n        if self.training and self.drop_rate > 0.:\n            pool = self.dropout(pool)\n\n        return self.linear(pool)\n\n    def exclude_from_weight_decay(self, name:str) -> bool:\n        # Regex to find layer names like\n        # \"stem.6.bias\", \"stem.6.gain\", \"body.0.skip_gain\", \n        # \"body.0.conv0.bias\", \"body.0.conv0.gain\"\n        regex = re.compile('stem.*(bias|gain)|conv.*(bias|gain)|skip_gain')\n        return len(regex.findall(name)) > 0\n\n    def exclude_from_clipping(self, name: str) -> bool:\n        # Last layer should not be clipped\n        return name.startswith('linear')\n\nclass Stem(nn.Module):\n    def __init__(self, activation:str='gelu'):\n        super(Stem, self).__init__()\n        \n        self.activation = activations_dict[activation]\n        self.conv0 = WSConv2D(in_channels=3, out_channels=16, kernel_size=3, stride=2)\n        self.conv1 = WSConv2D(in_channels=16, out_channels=32, kernel_size=3, stride=1)\n        self.conv2 = WSConv2D(in_channels=32, out_channels=64, kernel_size=3, stride=1)\n        self.conv3 = WSConv2D(in_channels=64, out_channels=128, kernel_size=3, stride=2)\n    \n    def forward(self, x):\n        out = self.activation(self.conv0(x))\n        out = self.activation(self.conv1(out))\n        out = self.activation(self.conv2(out))\n        out = self.conv3(out)\n        return out\n\nclass NFBlock(nn.Module):\n    def __init__(self, in_channels:int, out_channels:int, expansion:float=0.5, \n        se_ratio:float=0.5, stride:int=1, beta:float=1.0, alpha:float=0.2, \n        group_size:int=1, stochdepth_rate:float=None, activation:str='gelu'):\n\n        super(NFBlock, self).__init__()\n\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.expansion = expansion\n        self.se_ratio = se_ratio\n        self.activation = activations_dict[activation]\n        self.beta, self.alpha = beta, alpha\n        self.group_size = group_size\n        \n        width = int(self.out_channels * expansion)\n        self.groups = width // group_size\n        self.width = group_size * self.groups\n        self.stride = stride\n\n        self.conv0 = WSConv2D(in_channels=self.in_channels, out_channels=self.width, kernel_size=1)\n        self.conv1 = WSConv2D(in_channels=self.width, out_channels=self.width, kernel_size=3, stride=stride, padding=1, groups=self.groups)\n        self.conv1b = WSConv2D(in_channels=self.width, out_channels=self.width, kernel_size=3, stride=1, padding=1, groups=self.groups)\n        self.conv2 = WSConv2D(in_channels=self.width, out_channels=self.out_channels, kernel_size=1)\n        \n        self.use_projection = self.stride > 1 or self.in_channels != self.out_channels\n        if self.use_projection:\n            if stride > 1:\n                self.shortcut_avg_pool = nn.AvgPool2d(kernel_size=2, stride=2, padding=0 if self.in_channels==1536 else 1)\n            self.conv_shortcut = WSConv2D(self.in_channels, self.out_channels, kernel_size=1)\n            \n        self.squeeze_excite = SqueezeExcite(self.out_channels, self.out_channels, se_ratio=self.se_ratio, activation=activation)\n        self.skip_gain = nn.Parameter(torch.zeros(()))\n\n        self.use_stochdepth = stochdepth_rate is not None and stochdepth_rate > 0. and stochdepth_rate < 1.\n        if self.use_stochdepth:\n            self.stoch_depth = StochDepth(stochdepth_rate)\n\n    def forward(self, x):\n        out = self.activation(x) * self.beta\n\n        if self.stride > 1:\n            shortcut = self.shortcut_avg_pool(out)\n            shortcut = self.conv_shortcut(shortcut)\n        elif self.use_projection:\n            shortcut = self.conv_shortcut(out)\n        else:\n            shortcut = x\n\n        out = self.activation(self.conv0(out))\n        out = self.activation(self.conv1(out))\n        out = self.activation(self.conv1b(out))\n        out = self.conv2(out)\n        out = (self.squeeze_excite(out)*2) * out\n\n        if self.use_stochdepth:\n            out = self.stoch_depth(out)\n\n        return out * self.alpha * self.skip_gain + shortcut\n\n# Implementation mostly from https://arxiv.org/abs/2101.08692\n# Implemented changes from https://arxiv.org/abs/2102.06171 and\n#  https://github.com/deepmind/deepmind-research/tree/master/nfnets\nclass WSConv2D(nn.Conv2d):\n    def __init__(self, in_channels: int, out_channels: int, kernel_size, stride = 1, padding = 0,\n        dilation = 1, groups: int = 1, bias: bool = True, padding_mode: str = 'zeros'):\n\n        super(WSConv2D, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias, padding_mode)\n        \n        nn.init.xavier_normal_(self.weight)\n        self.gain = nn.Parameter(torch.ones(self.out_channels, 1, 1, 1))\n        self.register_buffer('eps', torch.tensor(1e-4, requires_grad=False), persistent=False)\n        self.register_buffer('fan_in', torch.tensor(np.prod(self.weight.shape[1:]), requires_grad=False).type_as(self.weight), persistent=False)\n\n    def standardized_weights(self):\n        # Original code: HWCN\n        mean = torch.mean(self.weight, axis=[1,2,3], keepdims=True)\n        var = torch.var(self.weight, axis=[1,2,3], keepdims=True)\n        scale = torch.rsqrt(torch.maximum(var * self.fan_in, self.eps))\n        return (self.weight - mean) * scale * self.gain\n        \n    def forward(self, x):\n        return F.conv2d(\n            input=x,\n            weight=self.standardized_weights(),\n            bias=self.bias,\n            stride=self.stride,\n            padding=self.padding,\n            dilation=self.dilation,\n            groups=self.groups\n        )\n\nclass SqueezeExcite(nn.Module):\n    def __init__(self, in_channels:int, out_channels:int, se_ratio:float=0.5, activation:str='gelu'):\n        super(SqueezeExcite, self).__init__()\n\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.se_ratio = se_ratio\n\n        self.hidden_channels = max(1, int(self.in_channels * self.se_ratio))\n        \n        self.activation = activations_dict[activation]\n        self.linear = nn.Linear(self.in_channels, self.hidden_channels)\n        self.linear_1 = nn.Linear(self.hidden_channels, self.out_channels)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        out = torch.mean(x, (2,3))\n        out = self.linear_1(self.activation(self.linear(out)))\n        out = self.sigmoid(out)\n\n        b,c,_,_ = x.size()\n        return out.view(b,c,1,1).expand_as(x)\n\nclass StochDepth(nn.Module):\n    def __init__(self, stochdepth_rate:float):\n        super(StochDepth, self).__init__()\n\n        self.drop_rate = stochdepth_rate\n\n    def forward(self, x):\n        if not self.training:\n            return x\n\n        batch_size = x.shape[0]\n        rand_tensor = torch.rand(batch_size, 1, 1, 1).type_as(x).to(x.device)\n        keep_prob = 1 - self.drop_rate\n        binary_tensor = torch.floor(rand_tensor + keep_prob)\n        \n        return x * binary_tensor","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:36:25.881512Z","iopub.execute_input":"2023-01-18T01:36:25.881902Z","iopub.status.idle":"2023-01-18T01:36:26.054893Z","shell.execute_reply.started":"2023-01-18T01:36:25.88185Z","shell.execute_reply":"2023-01-18T01:36:26.053859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.optim import Optimizer\n\n\n# Compute norm depending on the shape of x\ndef unitwise_norm(x):\n    if (len(torch.squeeze(x).shape)) <= 1: # Scalars, vectors\n        axis = 0\n        keepdims = False\n    elif len(x.shape) in [2,3]: # Linear layers\n        # Original code: IO\n        # Pytorch: OI\n        axis = 1\n        keepdims = True\n    elif len(x.shape) == 4: # Conv kernels\n        # Original code: HWIO\n        # Pytorch: OIHW\n        axis = [1, 2, 3]\n        keepdims = True\n    else:\n        raise ValueError(f'Got a parameter with len(shape) not in [1, 2, 3, 4]! {x}')\n\n    return torch.sqrt(torch.sum(torch.square(x), axis=axis, keepdim=keepdims))\n\n\n# This is a copy of the pytorch SGD implementation\n# enhanced with gradient clipping\nclass SGD_AGC(Optimizer):\n    def __init__(self, named_params, lr:float, momentum=0, dampening=0,\n                 weight_decay=0, nesterov=False, clipping:float=None, eps:float=1e-3):\n        if lr < 0.0:\n            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n        if momentum < 0.0:\n            raise ValueError(\"Invalid momentum value: {}\".format(momentum))\n        if weight_decay < 0.0:\n            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n\n        defaults = dict(lr=lr, momentum=momentum, dampening=dampening,\n                        weight_decay=weight_decay, nesterov=nesterov,\n                        # Extra defaults\n                        clipping=clipping,\n                        eps=eps\n                        )\n\n        if nesterov and (momentum <= 0 or dampening != 0):\n            raise ValueError(\"Nesterov momentum requires a momentum and zero dampening\")\n\n        # Put params in list so each one gets its own group\n        params = []\n        for name, param in named_params:\n            params.append({'params': param, 'name': name})\n\n        super(SGD_AGC, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(SGD_AGC, self).__setstate__(state)\n        for group in self.param_groups:\n            group.setdefault('nesterov', False)\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        loss = None\n        if closure is not None:\n            with torch.enable_grad():\n                loss = closure()\n\n        for group in self.param_groups:\n            weight_decay = group['weight_decay']\n            momentum = group['momentum']\n            dampening = group['dampening']\n            nesterov = group['nesterov']\n\n            # Extra values for clipping\n            clipping = group['clipping']\n            eps = group['eps']\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                d_p = p.grad\n\n                # =========================\n                # Gradient clipping\n                if clipping is not None:\n                    param_norm = torch.maximum(unitwise_norm(p), torch.tensor(eps).to(p.device))\n                    grad_norm = unitwise_norm(d_p)\n                    max_norm = param_norm * group['clipping']\n\n                    trigger_mask = grad_norm > max_norm\n                    clipped_grad = p.grad * (max_norm / torch.maximum(grad_norm, torch.tensor(1e-6).to(p.device)))\n                    d_p = torch.where(trigger_mask, clipped_grad, d_p)\n                # =========================\n\n                if weight_decay != 0:\n                    d_p = d_p.add(p, alpha=weight_decay)\n                if momentum != 0:\n                    param_state = self.state[p]\n                    if 'momentum_buffer' not in param_state:\n                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()\n                    else:\n                        buf = param_state['momentum_buffer']\n                        buf.mul_(momentum).add_(d_p, alpha=1 - dampening)\n                    if nesterov:\n                        d_p = d_p.add(buf, alpha=momentum)\n                    else:\n                        d_p = buf\n\n                p.add_(d_p, alpha=-group['lr'])\n\n        return loss","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:37:29.043578Z","iopub.execute_input":"2023-01-18T01:37:29.043941Z","iopub.status.idle":"2023-01-18T01:37:29.061721Z","shell.execute_reply.started":"2023-01-18T01:37:29.043904Z","shell.execute_reply":"2023-01-18T01:37:29.060616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision\nimport torchvision.transforms as transforms","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:59:04.703234Z","iopub.execute_input":"2023-01-18T01:59:04.703565Z","iopub.status.idle":"2023-01-18T01:59:04.70956Z","shell.execute_reply.started":"2023-01-18T01:59:04.703534Z","shell.execute_reply":"2023-01-18T01:59:04.708638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = NFNet(num_classes=15, variant='F1', stochdepth_rate=0.25, alpha=0.2, se_ratio=0.5, activation='gelu').cuda()\n\n# Loss and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = SGD_AGC(named_params=model.named_parameters(), lr=0.001, momentum=0.9, clipping=0.1, weight_decay=5e-4, nesterov=True)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)\ncalc = nn.MSELoss()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:59:05.781326Z","iopub.execute_input":"2023-01-18T01:59:05.781662Z","iopub.status.idle":"2023-01-18T01:59:08.080585Z","shell.execute_reply.started":"2023-01-18T01:59:05.781632Z","shell.execute_reply":"2023-01-18T01:59:08.079271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dice_loss(pred, target, smooth = 1.):\n    pred = pred.contiguous()\n    target = target.contiguous()    \n\n    intersection = (pred * target).sum(dim=2).sum(dim=2)\n    \n    loss = (1 - ((2. * intersection + smooth) / (pred.sum(dim=2).sum(dim=2) + target.sum(dim=2).sum(dim=2) + smooth)))\n    \n    return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:59:09.540347Z","iopub.execute_input":"2023-01-18T01:59:09.540661Z","iopub.status.idle":"2023-01-18T01:59:09.546161Z","shell.execute_reply.started":"2023-01-18T01:59:09.540633Z","shell.execute_reply":"2023-01-18T01:59:09.545275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_loss(pred, target, metrics, bce_weight=0.5):\n    bce = F.binary_cross_entropy_with_logits(pred, target)\n        \n    pred = F.sigmoid(pred)\n    dice = dice_loss(pred, target)\n    \n    loss = bce * bce_weight + dice * (1 - bce_weight)\n    \n    metrics['bce'] += bce.data.cpu().numpy() * target.size(0)\n    metrics['dice'] += dice.data.cpu().numpy() * target.size(0)\n    metrics['loss'] += loss.data.cpu().numpy() * target.size(0)\n    \n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:59:10.570418Z","iopub.execute_input":"2023-01-18T01:59:10.570827Z","iopub.status.idle":"2023-01-18T01:59:10.577256Z","shell.execute_reply.started":"2023-01-18T01:59:10.570785Z","shell.execute_reply":"2023-01-18T01:59:10.576147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = defaultdict(float)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:59:11.442271Z","iopub.execute_input":"2023-01-18T01:59:11.442592Z","iopub.status.idle":"2023-01-18T01:59:11.447245Z","shell.execute_reply.started":"2023-01-18T01:59:11.442563Z","shell.execute_reply":"2023-01-18T01:59:11.446132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 1\nsteps = 0\nprint_every = 750\ntrain_losses, train_accuracy = [], []\n#model.load_state_dict(torch.load('./final_model.pth'))\nfor epoch in range(epochs):\n    model.train()\n    size = 0\n    running_loss = 0\n    acc = 0\n    for a,(image_train, y_train) in enumerate(data_loader):\n        steps += 1\n        image_train, y_train = image_train.unsqueeze(0).cuda(), y_train.cuda()\n        image_train = Variable(image_train,requires_grad=True)\n        image_train=  image_train.detach().cpu().numpy()\n        image_train = np.dstack([image_train]*3)\n        image_train = image_train.reshape(-1,3,224,224)\n        image_train = torch.from_numpy(image_train).cuda()\n        y_train = y_train.detach().cpu().numpy()[0,:,1,1]\n        y_train =torch.from_numpy(y_train).cuda()\n        optimizer.zero_grad()\n        y_predtrain = model.forward(image_train)\n        #y_train=y_train.type(torch.LongTensor)\n        #loss = calc_loss(y_predtrain, y_train,metrics)\n        loss = calc(y_predtrain, y_train)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n        ps = torch.exp(y_predtrain)\n        top_p = torch.max(y_predtrain, 1)\n        top_class = torch.argmax(y_predtrain,dim = 1)\n        equals = top_class == y_train\n        print(torch.mean(equals.type(torch.FloatTensor)).item())\n        acc += torch.mean(equals.type(torch.FloatTensor)).item()\n        size += image_train.shape[0]\n        model.eval()\n        print(f\"Epoch {epoch+1}/{epochs}.. \"\n              f\"Train loss: {running_loss/print_every:.3f}.. \"\n              f\"Train accuracy: {acc/len(data_loader):.3f}\")\n    #torch.save(model.state_dict(),'./'+str(epoch)+'model.pth')\n    #print('model saved')\n    torch.save(model.state_dict(),'./'+str(epoch)+'unet_model.pth')\n    train_losses.append(float(running_loss)/float(size))\n    train_accuracy.append(float(acc)/float(size))\n    print('train_losses',epoch,train_losses)\n    print('train_accuracy',epoch,train_accuracy)\ntorch.save(model.state_dict(),'./final_unet_model.pth')\nprint('model saved')\nprint('train_losses',epoch,train_losses)\nprint('train_accuracy',epoch,train_accuracy)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T02:06:18.463049Z","iopub.execute_input":"2023-01-18T02:06:18.463407Z","iopub.status.idle":"2023-01-18T02:08:07.522859Z","shell.execute_reply.started":"2023-01-18T02:06:18.463373Z","shell.execute_reply":"2023-01-18T02:08:07.5194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_pxl = pydicom.read_file('../input/vinbigdata-chest-xray-abnormalities-detection/train/000d68e42b71d3eac10ccc077aba07c1.dicom').pixel_array\nimg_res = resize(img_pxl,(1024,1024),anti_aliasing=True)\nimg_np = img_res.astype(np.float32())\nimg_tr = torch.from_numpy(img_np).unsqueeze(0).unsqueeze(0).cuda()\n\nimg_tr = Variable(img_tr,requires_grad=True)\nimg_tr=  img_tr.detach().cpu().numpy()\nimg_tr = np.dstack([img_tr]*3)\nimg_tr = img_tr.reshape(-1,3,1024,1024)\nimg_tr = torch.from_numpy(img_tr).cuda()\noutput = model(img_tr)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T01:49:58.770348Z","iopub.status.idle":"2023-01-18T01:49:58.77111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = output.cpu().detach().squeeze()\nprint(predict.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T13:01:25.822729Z","iopub.execute_input":"2023-01-14T13:01:25.823104Z","iopub.status.idle":"2023-01-14T13:01:25.83022Z","shell.execute_reply.started":"2023-01-14T13:01:25.823071Z","shell.execute_reply":"2023-01-14T13:01:25.829009Z"},"trusted":true},"execution_count":null,"outputs":[]}]}