{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"_____\n**Credit:** \n- Parts of this notebook are based on the [great](https://www.kaggle.com/code/realneuralnetwork/coatnet-strip-ai-inference) [notebooks](https://www.kaggle.com/code/realneuralnetwork/cnn-strip-ai-inference) by [Kabir Ivan\n](https://www.kaggle.com/realneuralnetwork)\n- originally copy of CoAtNet + EfficientNet ensemble notebook, edited to train a discriminator and use it for constructing the ensemble using Dempster-Shafer rules\n_____\n","metadata":{}},{"cell_type":"markdown","source":"# [Train + Infer] CoAtNet + EfficientNet\n\nThis notebook contains steps and code to train CoAtNet and EfficientNet-B4 for the Mayo Clinic - STRIP AI competition.\nIn this scenario we train CoAtNet + EfficinetNet-B4, then use these models for inference","metadata":{}},{"cell_type":"markdown","source":"#### Setup","metadata":{}},{"cell_type":"code","source":"#!pip install einops\n!pip install einops --no-index --find-links=file:///kaggle/input/dkm-dependeicies/wheels/\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:11:57.693352Z","iopub.execute_input":"2022-08-16T12:11:57.694349Z","iopub.status.idle":"2022-08-16T12:12:09.528949Z","shell.execute_reply.started":"2022-08-16T12:11:57.694201Z","shell.execute_reply":"2022-08-16T12:12:09.527811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/einops')\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nfrom efficientnet_pytorch import EfficientNet\n# !pip install --upgrade efficientnet-pytorch","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-16T12:12:09.532538Z","iopub.execute_input":"2022-08-16T12:12:09.532921Z","iopub.status.idle":"2022-08-16T12:12:10.95881Z","shell.execute_reply.started":"2022-08-16T12:12:09.532891Z","shell.execute_reply":"2022-08-16T12:12:10.957912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CoAtNet: Marrying Convolution and Attention for All Data Sizes\n\n[Paper](https://arxiv.org/pdf/2106.04803.pdf)\n\nThe CoAtNet paper attempts to effectively combine the strengths from both convolutional and transformers architectures, they present CoAtNets(pronounced \"coat\" nets), a family of hybrid models built from two key insights: \n- Depthwise Convolution and self-Attention can be naturally unified via simple relative attention\n- Vertically stacking convolution layers and attention layers in a principled way is surprisingly effective in improving generalization, capacity and efficiency.\n\n![](https://i.ibb.co/Sd6wj7D/Selection-998.png)","metadata":{}},{"cell_type":"markdown","source":"### Imports, Seed, Data loading","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport torch\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch.nn as nn\nimport seaborn as sns\nfrom random import randint\nfrom einops import rearrange\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom torch.optim import lr_scheduler\nfrom einops.layers.torch import Rearrange\nfrom efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nimport warnings; warnings.filterwarnings(\"ignore\")\ngc.enable()\n\ntrain_models = False\ndef seed_everything(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    os.environ['PYTHONHASHSEED'] = str(seed_value)    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = True\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nseed = 42\nseed_everything(seed)\n\n\ndebug = False\ngenerate_new = False\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 1000)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":3.063995,"end_time":"2022-07-08T14:24:41.045696","exception":false,"start_time":"2022-07-08T14:24:37.981701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-16T12:12:10.960086Z","iopub.execute_input":"2022-08-16T12:12:10.961055Z","iopub.status.idle":"2022-08-16T12:12:11.876255Z","shell.execute_reply.started":"2022-08-16T12:12:10.961015Z","shell.execute_reply":"2022-08-16T12:12:11.87512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport datetime\ntry:\n    os.system('mkdir model-dataset')\nexcept:\n    pass\nos.system('cp -r ../input/model-mover1/* model-dataset')\nos.environ['KAGGLE_CONFIG_DIR'] = '/kaggle/working/model-dataset'\nos.system(\"chmod 600 /kaggle/working/model-dataset/kaggle.json\") \n\ndef get_time_str():    \n    curr_time = datetime.datetime.now()\n    time = datetime.datetime.strftime(curr_time,'%m_%d_%H_%M_%S')\n    return time\n\ndef sync():\n    # Initiate\n    os.system('kaggle datasets init -p model-dataset')\n    \n    # Edit the dataset-metadata.json\n    with open(\"./model-dataset/dataset-metadata.json\", \"r\",encoding='utf-8') as jsonFile:\n        data = json.load(jsonFile)\n    data[\"title\"] = \"model-mover1\"\n    data['id'] = \"pbkowalski/model-mover1\"\n    with open(\"./model-dataset/dataset-metadata.json\", \"w\") as jsonFile:\n        json.dump(data, jsonFile,ensure_ascii=False)\n    # Upload your folder as a new version of the dataset    \n    exit = os.system('kaggle datasets version -m ' + get_time_str() + ' -p model-dataset -d')\n    \n    if(exit == 0):\n        print(\"Uploaded!\")\n    else:\n        print(\"Failed to upload!, exit code \", exit)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:11.877947Z","iopub.execute_input":"2022-08-16T12:12:11.878345Z","iopub.status.idle":"2022-08-16T12:12:14.91841Z","shell.execute_reply.started":"2022-08-16T12:12:11.878307Z","shell.execute_reply":"2022-08-16T12:12:14.9174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = ['model.pth', 'efficientnet_model.pth']\nfor model in model_names:\n    os.system('cp ../input/model-mover1/'+model + ' .')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:14.923118Z","iopub.execute_input":"2022-08-16T12:12:14.923412Z","iopub.status.idle":"2022-08-16T12:12:15.376494Z","shell.execute_reply.started":"2022-08-16T12:12:14.923386Z","shell.execute_reply":"2022-08-16T12:12:15.375301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_count = max(train_df.label.value_counts())\nfor label in train_df.label.unique():\n    df = train_df.loc[train_df.label == label]\n    while(train_df.label.value_counts()[label] < max_count):\n        train_df = pd.concat([train_df, df.head(max_count - train_df.label.value_counts()[label])], axis = 0)\n        \nif(generate_new):\n    os.mkdir(\"./train/\")\n    os.mkdir(\"./test/\")\n    for i in tqdm(range(test_df.shape[0])):\n        img_id = test_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./test/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n    for i in tqdm(range(train_df.shape[0])):\n        img_id = train_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[0] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./train/{img_id}.jpg\", img)\n        del img\n        gc.collect()","metadata":{"papermill":{"duration":69.477711,"end_time":"2022-07-08T14:25:50.554416","exception":false,"start_time":"2022-07-08T14:24:41.076705","status":"completed"},"tags":[],"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-16T12:12:15.378709Z","iopub.execute_input":"2022-08-16T12:12:15.379087Z","iopub.status.idle":"2022-08-16T12:12:15.407839Z","shell.execute_reply.started":"2022-08-16T12:12:15.37905Z","shell.execute_reply":"2022-08-16T12:12:15.406798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Class","metadata":{}},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns    \n    def __len__(self): return len(self.df)    \n    def __getitem__(self, index):\n        if(generate_new): paths = [\"./test/\", \"./train/\"]\n        else: paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        label = None\n        if(self.train): label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id                   ","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-16T12:12:15.409455Z","iopub.execute_input":"2022-08-16T12:12:15.410178Z","iopub.status.idle":"2022-08-16T12:12:15.419617Z","shell.execute_reply.started":"2022-08-16T12:12:15.410139Z","shell.execute_reply":"2022-08-16T12:12:15.418571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoAtNet Training","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:15.421292Z","iopub.execute_input":"2022-08-16T12:12:15.422198Z","iopub.status.idle":"2022-08-16T12:12:15.428689Z","shell.execute_reply.started":"2022-08-16T12:12:15.422159Z","shell.execute_reply":"2022-08-16T12:12:15.427331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CoAtNet Architecture","metadata":{}},{"cell_type":"code","source":"def conv_3x3_bn(inp, oup, image_size, downsample=False):\n    stride = 1 if downsample == False else 2\n    return nn.Sequential(\n        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),\n        nn.BatchNorm2d(oup),\n        nn.GELU()\n    )\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn, norm):\n        super().__init__()\n        self.norm = norm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass SE(nn.Module):\n    def __init__(self, inp, oup, expansion=0.25):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(oup, int(inp * expansion), bias=False),\n            nn.GELU(),\n            nn.Linear(int(inp * expansion), oup, bias=False),\n            nn.Sigmoid())\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout=0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout))\n    def forward(self, x):\n        return self.net(x)\n\n\nclass MBConv(nn.Module):\n    def __init__(self, inp, oup, image_size, downsample=False, expansion=4):\n        super().__init__()\n        self.downsample = downsample\n        stride = 1 if self.downsample == False else 2\n        hidden_dim = int(inp * expansion)\n\n        if self.downsample:\n            self.pool = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n\n        if expansion == 1:\n            self.conv = nn.Sequential(\n                nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup))\n        else:\n            self.conv = nn.Sequential(\n                nn.Conv2d(inp, hidden_dim, 1, stride, 0, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),                \n                nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1, groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                SE(inp, hidden_dim),\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup))\n        \n        self.conv = PreNorm(inp, self.conv, nn.BatchNorm2d)\n\n    def forward(self, x):\n        if self.downsample: return self.proj(self.pool(x)) + self.conv(x)\n        else: return x + self.conv(x)\n\nclass Attention(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, dropout=0.):\n        super().__init__()\n        inner_dim = dim_head * heads\n        project_out = not (heads == 1 and dim_head == inp)\n        self.ih, self.iw = image_size\n        self.heads = heads\n        self.scale = dim_head ** -0.5       \n        self.relative_bias_table = nn.Parameter(\n            torch.zeros((2 * self.ih - 1) * (2 * self.iw - 1), heads))\n        coords = torch.meshgrid((torch.arange(self.ih), torch.arange(self.iw)))\n        coords = torch.flatten(torch.stack(coords), 1)\n        relative_coords = coords[:, :, None] - coords[:, None, :]\n        relative_coords[0] += self.ih - 1\n        relative_coords[1] += self.iw - 1\n        relative_coords[0] *= 2 * self.iw - 1\n        relative_coords = rearrange(relative_coords, 'c h w -> h w c')\n        relative_index = relative_coords.sum(-1).flatten().unsqueeze(1)\n        self.register_buffer(\"relative_index\", relative_index)\n        self.attend = nn.Softmax(dim=-1)\n        self.to_qkv = nn.Linear(inp, inner_dim * 3, bias=False)\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, oup),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        qkv = self.to_qkv(x).chunk(3, dim=-1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=self.heads), qkv)\n        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n        relative_bias = self.relative_bias_table.gather(0, self.relative_index.repeat(1, self.heads))\n        relative_bias = rearrange(relative_bias, '(h w) c -> 1 c h w', h=self.ih*self.iw, w=self.ih*self.iw)\n        dots = dots + relative_bias\n        attn = self.attend(dots)\n        out = torch.matmul(attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        out = self.to_out(out)\n        return out\n\nclass Transformer(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, downsample=False, dropout=0.):\n        super().__init__()\n        hidden_dim = int(inp * 4)\n        self.ih, self.iw = image_size\n        self.downsample = downsample\n        if self.downsample:\n            self.pool1 = nn.MaxPool2d(3, 2, 1)\n            self.pool2 = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n        self.attn = Attention(inp, oup, image_size, heads, dim_head, dropout)\n        self.ff = FeedForward(oup, hidden_dim, dropout)\n        self.attn = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(inp, self.attn, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw))\n        self.ff = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(oup, self.ff, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw))\n\n    def forward(self, x):\n        if self.downsample: x = self.proj(self.pool1(x)) + self.attn(self.pool2(x))\n        else: x = x + self.attn(x)\n        x = x + self.ff(x)\n        return x\n\n\nclass CoAtNet(nn.Module):\n    def __init__(self, image_size, in_channels, num_blocks, channels, num_classes=1000, block_types=['C', 'C', 'T', 'T']):\n        super().__init__()\n        ih, iw = image_size\n        block = {'C': MBConv, 'T': Transformer}\n\n        self.s0 = self._make_layer(\n            conv_3x3_bn, in_channels, channels[0], num_blocks[0], (ih // 2, iw // 2))\n        self.s1 = self._make_layer(\n            block[block_types[0]], channels[0], channels[1], num_blocks[1], (ih // 4, iw // 4))\n        self.s2 = self._make_layer(\n            block[block_types[1]], channels[1], channels[2], num_blocks[2], (ih // 8, iw // 8))\n        self.s3 = self._make_layer(\n            block[block_types[2]], channels[2], channels[3], num_blocks[3], (ih // 16, iw // 16))\n        self.s4 = self._make_layer(\n            block[block_types[3]], channels[3], channels[4], num_blocks[4], (ih // 32, iw // 32))\n\n        self.pool = nn.AvgPool2d(ih // 32, 1)\n        self.fc = nn.Linear(channels[-1], num_classes, bias=False)\n\n    def forward(self, x):\n        x = self.s0(x)\n        x = self.s1(x)\n        x = self.s2(x)\n        x = self.s3(x)\n        x = self.s4(x)\n\n        x = self.pool(x).view(-1, x.shape[1])\n        x = self.fc(x)\n        return x\n\n    def _make_layer(self, block, inp, oup, depth, image_size):\n        layers = nn.ModuleList([])\n        for i in range(depth):\n            if i == 0:\n                layers.append(block(inp, oup, image_size, downsample=True))\n            else:\n                layers.append(block(oup, oup, image_size))\n        return nn.Sequential(*layers)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-16T12:12:15.430655Z","iopub.execute_input":"2022-08-16T12:12:15.431232Z","iopub.status.idle":"2022-08-16T12:12:15.58225Z","shell.execute_reply.started":"2022-08-16T12:12:15.43119Z","shell.execute_reply":"2022-08-16T12:12:15.581147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, model_name):\n    best_acc = 0.0\n    for epoch in range(num_epochs):\n        model.cuda()       \n        for phase in ['train', 'val']:\n            if phase == 'train': model.train()\n            else: model.eval()\n            epoch_loss = 0.0\n            epoch_acc = 0\n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().long()\n                optimizer.zero_grad()                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)                    \n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')    \n        if epoch_acc > best_acc:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n            traced.save(model_name)\n            best_acc = epoch_acc","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:15.583907Z","iopub.execute_input":"2022-08-16T12:12:15.584268Z","iopub.status.idle":"2022-08-16T12:12:15.596238Z","shell.execute_reply.started":"2022-08-16T12:12:15.584233Z","shell.execute_reply":"2022-08-16T12:12:15.595295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Architecture Definition & Running the training","metadata":{}},{"cell_type":"code","source":"train, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\nbatch_size = 1\ntrain_loader = DataLoader(ImgDataset(train), batch_size=batch_size, shuffle=False, num_workers=1)\nval_loader = DataLoader(ImgDataset(val), batch_size=batch_size, shuffle=False, num_workers=1)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-16T12:12:15.597879Z","iopub.execute_input":"2022-08-16T12:12:15.598782Z","iopub.status.idle":"2022-08-16T12:12:15.612147Z","shell.execute_reply.started":"2022-08-16T12:12:15.598729Z","shell.execute_reply":"2022-08-16T12:12:15.611135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#\nif train_models:\n    num_blocks = [2, 2, 12, 28, 2]\n    channels = [64, 64, 128, 256, 512]\n    dataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\n    try:\n        model = torch.jit.load('model.pth')\n    except:\n        model = CoAtNet((512, 512), 3, num_blocks, channels, num_classes=2)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n    train_model(model, dataloaders_dict, criterion, optimizer, 2, 'model.pth')\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\n    train_model(model, dataloaders_dict, criterion, optimizer, 5, 'model.pth')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:15.614728Z","iopub.execute_input":"2022-08-16T12:12:15.615097Z","iopub.status.idle":"2022-08-16T12:12:15.621911Z","shell.execute_reply.started":"2022-08-16T12:12:15.615064Z","shell.execute_reply":"2022-08-16T12:12:15.620593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        #try:\n        images = item[0].cuda().float()\n        ids.append(patient_id)\n        output = model(images)\n        outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        #except:\n            #ids.append(patient_id)\n            #outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids       \n    \nmodel = torch.jit.load('model.pth')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:15.623309Z","iopub.execute_input":"2022-08-16T12:12:15.62463Z","iopub.status.idle":"2022-08-16T12:12:16.827308Z","shell.execute_reply.started":"2022-08-16T12:12:15.624575Z","shell.execute_reply":"2022-08-16T12:12:16.826328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoAtNet Inference","metadata":{}},{"cell_type":"markdown","source":"**Setups**","metadata":{}},{"cell_type":"code","source":"!mkdir /root/.cache\n!mkdir /root/.cache/torch\n!mkdir /root/.cache/torch/hub\n!mkdir /root/.cache/torch/hub/checkpoints\n!cp -r ../input/torchhub-efficientnet-b4/nvidia_efficientnet-b4_210412.pth /root/.cache/torch/hub/checkpoints/","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-16T12:12:16.832473Z","iopub.execute_input":"2022-08-16T12:12:16.833024Z","iopub.status.idle":"2022-08-16T12:12:22.089801Z","shell.execute_reply.started":"2022-08-16T12:12:16.832994Z","shell.execute_reply":"2022-08-16T12:12:22.088608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport efficientnet_pytorch\nfrom efficientnet_pytorch import EfficientNet\n\n\ndebug = False\ngenerate_new = False\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 1000)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\n            \n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-16T12:12:22.09276Z","iopub.execute_input":"2022-08-16T12:12:22.093434Z","iopub.status.idle":"2022-08-16T12:12:22.109139Z","shell.execute_reply.started":"2022-08-16T12:12:22.093399Z","shell.execute_reply":"2022-08-16T12:12:22.108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Running the training","metadata":{}},{"cell_type":"code","source":"#model = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b4\")\ntry:\n    model = torch.jit.load('efficientnet_model.pth')\nexcept:\n    model = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b4\")\n\ncheckpoint = torch.load('../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth')\nmodel.load_state_dict(checkpoint)\n\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\n#batch_size = 4\ntrain_loader = DataLoader(ImgDataset(train), batch_size=batch_size, shuffle=False, num_workers=1)\nval_loader = DataLoader(ImgDataset(val), batch_size=batch_size, shuffle=False, num_workers=1)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\ncriterion = nn.CrossEntropyLoss()\n#\nif train_models:\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n    train_model(model, dataloaders_dict, criterion, optimizer, 2, 'efficientnet_model.pth')\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\n    train_model(model, dataloaders_dict, criterion, optimizer, 5, 'efficientnet_model.pth')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:22.110453Z","iopub.execute_input":"2022-08-16T12:12:22.110897Z","iopub.status.idle":"2022-08-16T12:12:23.854703Z","shell.execute_reply.started":"2022-08-16T12:12:22.110863Z","shell.execute_reply":"2022-08-16T12:12:23.853756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if train_models:\n    model_names = ['model.pth', 'efficientnet_model.pth']\n    for model_name in model_names:\n        model = torch.jit.load(model_name)\n        traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n        traced.save('model-dataset/'+ model_name)\n    sync()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:23.856214Z","iopub.execute_input":"2022-08-16T12:12:23.856736Z","iopub.status.idle":"2022-08-16T12:12:23.863616Z","shell.execute_reply.started":"2022-08-16T12:12:23.856695Z","shell.execute_reply":"2022-08-16T12:12:23.862433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loader = DataLoader(ImgDataset(val), batch_size=1, shuffle=False, num_workers=1)\nmodel1 = torch.jit.load('model.pth')\nmodel2 = torch.jit.load('efficientnet_model.pth')\nans1, ids1 = predict(model1, val_loader)\nans2, ids2 = predict(model2, val_loader)\nlabels = [x[1][0].item() for x in val_loader]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:12:23.86536Z","iopub.execute_input":"2022-08-16T12:12:23.8659Z","iopub.status.idle":"2022-08-16T12:13:16.716785Z","shell.execute_reply.started":"2022-08-16T12:12:23.865863Z","shell.execute_reply":"2022-08-16T12:13:16.715267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_probs = [x[0] if labels[id] == 0 else x[1] for id, x in enumerate((ans1 - ans2))]","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.718667Z","iopub.execute_input":"2022-08-16T12:13:16.719059Z","iopub.status.idle":"2022-08-16T12:13:16.725798Z","shell.execute_reply.started":"2022-08-16T12:13:16.719021Z","shell.execute_reply":"2022-08-16T12:13:16.724642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc1 = sum([(x[0] > x[1] and labels[id] == 0) or (x[0] < x[1] and labels[id] == 1) for id, x in enumerate(ans1)])/ len(ans1)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.727657Z","iopub.execute_input":"2022-08-16T12:13:16.728468Z","iopub.status.idle":"2022-08-16T12:13:16.73634Z","shell.execute_reply.started":"2022-08-16T12:13:16.728415Z","shell.execute_reply":"2022-08-16T12:13:16.735348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc2 = sum([(x[0] > x[1] and labels[id] == 0) or (x[0] < x[1] and labels[id] == 1) for id, x in enumerate(ans2)])/ len(ans2)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.738152Z","iopub.execute_input":"2022-08-16T12:13:16.738705Z","iopub.status.idle":"2022-08-16T12:13:16.746239Z","shell.execute_reply.started":"2022-08-16T12:13:16.738645Z","shell.execute_reply":"2022-08-16T12:13:16.745245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc2","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.748072Z","iopub.execute_input":"2022-08-16T12:13:16.748571Z","iopub.status.idle":"2022-08-16T12:13:16.75839Z","shell.execute_reply.started":"2022-08-16T12:13:16.748536Z","shell.execute_reply":"2022-08-16T12:13:16.757286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len([x for x in label_probs if x < 0])/len(label_probs)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.760363Z","iopub.execute_input":"2022-08-16T12:13:16.760736Z","iopub.status.idle":"2022-08-16T12:13:16.768971Z","shell.execute_reply.started":"2022-08-16T12:13:16.760699Z","shell.execute_reply":"2022-08-16T12:13:16.767749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val2 = val.copy()\nval2['label'] = label_probs\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.770568Z","iopub.execute_input":"2022-08-16T12:13:16.771343Z","iopub.status.idle":"2022-08-16T12:13:16.780691Z","shell.execute_reply.started":"2022-08-16T12:13:16.771306Z","shell.execute_reply":"2022-08-16T12:13:16.779711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NewImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns    \n    def __len__(self): return len(self.df)    \n    def __getitem__(self, index):\n        if(generate_new): paths = [\"./test/\", \"./train/\"]\n        else: paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        label = self.df.iloc[index].label\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id                   ","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.782251Z","iopub.execute_input":"2022-08-16T12:13:16.782968Z","iopub.status.idle":"2022-08-16T12:13:16.792169Z","shell.execute_reply.started":"2022-08-16T12:13:16.782933Z","shell.execute_reply":"2022-08-16T12:13:16.791233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use the earlier architectures for assessing performance","metadata":{}},{"cell_type":"code","source":"class shiftSigmoid(nn.Module):\n    def forward(self, input: torch.Tensor) -> torch.Tensor:\n        return 2*torch.sigmoid(input)-1","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.793651Z","iopub.execute_input":"2022-08-16T12:13:16.794191Z","iopub.status.idle":"2022-08-16T12:13:16.801571Z","shell.execute_reply.started":"2022-08-16T12:13:16.794155Z","shell.execute_reply":"2022-08-16T12:13:16.800627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\nclass EfficientNetRegModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        #self.input = nn.Conv2d(1,3,kernel_size=3,padding=1,stride=2)\n        #self.bn = nn.BatchNorm2d(3)\n        #self.model = EfficientNet.from_pretrained('efficientnet_b4')\n        #model = EfficientNet.from_pretrained('efficientnet-b4')\n        self.model = EfficientNet.from_name(\"efficientnet-b4\", {'drop_connect_rate':0.6, 'dropout_rate': 0.6})\n        checkpoint = torch.load('../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth')\n        self.model.load_state_dict(checkpoint)\n        self.model._fc = nn.Linear(1792, 500, bias=True)\n        self.dropout = nn.Dropout(p=0.7)\n        self.output = nn.Linear(500, 1)\n        self.outsig = shiftSigmoid()\n    \n    def forward(self, x):\n        x = self.model(x)\n        x = self.dropout(x)\n        x = self.output(x)\n        return self.outsig(x)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.803245Z","iopub.execute_input":"2022-08-16T12:13:16.803706Z","iopub.status.idle":"2022-08-16T12:13:16.813549Z","shell.execute_reply.started":"2022-08-16T12:13:16.803673Z","shell.execute_reply":"2022-08-16T12:13:16.812326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model_reg(model, dataloaders_dict, criterion, optimizer, num_epochs, model_name):\n    best_loss = None\n    for epoch in range(num_epochs):\n        model.cuda()       \n        for phase in ['train', 'val']:\n            if phase == 'train': model.train()\n            else: model.eval()\n            epoch_loss = 0.0\n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().float()\n                optimizer.zero_grad()                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                    epoch_loss += loss.item() * len(output)\n                    #print(loss.item())\n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f}')    \n        if best_loss is None or epoch_loss < best_loss:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n            traced.save(model_name)\n            best_loss = epoch_loss\n            \ndef predict_reg(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    ids = []\n    with torch.no_grad():\n        for item in tqdm(dataloader, leave=False):\n            patient_id = item[2][0]\n            #try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(output.cpu())\n            #except:\n                #ids.append(patient_id)\n                #outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids       \n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.815685Z","iopub.execute_input":"2022-08-16T12:13:16.815952Z","iopub.status.idle":"2022-08-16T12:13:16.831284Z","shell.execute_reply.started":"2022-08-16T12:13:16.815927Z","shell.execute_reply":"2022-08-16T12:13:16.830105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install GPUtil\n\n# import torch\n# from GPUtil import showUtilization as gpu_usage\n# from numba import cuda\n\n# def free_gpu_cache():\n#     print(\"Initial GPU Usage\")\n#     gpu_usage()                             \n\n#     torch.cuda.empty_cache()\n\n\n\n#     print(\"GPU Usage after emptying the cache\")\n#     gpu_usage()\n\n# free_gpu_cache()   ","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.834551Z","iopub.execute_input":"2022-08-16T12:13:16.834999Z","iopub.status.idle":"2022-08-16T12:13:16.840984Z","shell.execute_reply.started":"2022-08-16T12:13:16.834964Z","shell.execute_reply":"2022-08-16T12:13:16.839945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#custom loss function\nclass cosloss(nn.Module):\n  def __init__(self):\n    super(cosloss, self).__init__();\n\n  def forward(self, predictions, target):\n    #loss_value =  torch.abs( (target - predictions) / target) \n    loss_value =  torch.abs( (target - predictions) ) \n#     target_abs = 0 if target <0 else 1\n#     predictions01 = (predictions+1) /2 \n#     loss2 = -target_abs * torch.log(predictions01) - (1 - target_abs) * torch.log(1 - predictions01)\n    return  loss_value\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.842765Z","iopub.execute_input":"2022-08-16T12:13:16.843341Z","iopub.status.idle":"2022-08-16T12:13:16.850017Z","shell.execute_reply.started":"2022-08-16T12:13:16.843146Z","shell.execute_reply":"2022-08-16T12:13:16.848944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#custom loss function\nclass jointloss(nn.Module):\n  def __init__(self):\n    super(jointloss, self).__init__();\n\n  def forward(self, predictions, target):\n    #loss_value =  torch.abs( (target - predictions) / target) \n    loss_value =  torch.abs( (target - predictions) )\n    target_abs = torch.Tensor([0]) if target <0 else torch.Tensor([1])\n    predictions01 = (predictions+1) /2 \n    target_abs = target_abs.to(device='cuda')\n    loss2 = torch.nn.functional.soft_margin_loss(predictions01, target_abs)\n    return loss2\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.851979Z","iopub.execute_input":"2022-08-16T12:13:16.852523Z","iopub.status.idle":"2022-08-16T12:13:16.860961Z","shell.execute_reply.started":"2022-08-16T12:13:16.852417Z","shell.execute_reply":"2022-08-16T12:13:16.859872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model3 = EfficientNetRegModel()\n\ntrain_reg, val_reg = train_test_split(val2, test_size=0.1, random_state=42, stratify = val2.label>0)\n\ntrain_loaderreg = DataLoader(NewImgDataset(train_reg), batch_size=batch_size, shuffle=False, num_workers=1)\nval_loaderreg = DataLoader(NewImgDataset(val_reg), batch_size=batch_size, shuffle=False, num_workers=1)\ndataloaders_dictreg = {\"train\": train_loaderreg, \"val\": val_loaderreg}\n#criterion = nn.CrossEntropyLoss()\n#criterion = cosloss()\nregloss = cosloss()\nsmlLoss = jointloss()\nfor lr in [1e-4, 1e-5]:\n    for ep in range(3):\n        for loss in [smlLoss, regloss]:\n            if loss == regloss:\n                print (\"regloss:\")\n            else:\n                print(\"smlloss:\")\n            optimizer = torch.optim.AdamW(model3.parameters(), lr = lr)\n            train_model_reg(model3, dataloaders_dictreg, loss, optimizer, 1, 'model_reg.pth')\n            \n#train_model_reg(model3, dataloaders_dictreg, smlLoss, torch.optim.AdamW(model3.parameters(), lr = 1e-5) , 2, 'model_reg.pth')\n# print('sml')\n# train_model_reg(model3, dataloaders_dictreg, smlLoss, torch.optim.AdamW(model3.parameters(), lr = 1e-4) , 2, 'model_reg.pth')\n# train_model_reg(model3, dataloaders_dictreg, smlLoss, torch.optim.AdamW(model3.parameters(), lr = 1e-5) , 2, 'model_reg.pth')\n# print('reg')\n# train_model_reg(model3, dataloaders_dictreg, regloss, torch.optim.AdamW(model3.parameters(), lr = 1e-4) , 3, 'model_reg.pth')\n# print('sml')\n# train_model_reg(model3, dataloaders_dictreg, smlLoss, torch.optim.AdamW(model3.parameters(), lr = 1e-5) , 2, 'model_reg.pth')\n# print('reg')\n# train_model_reg(model3, dataloaders_dictreg, regloss, torch.optim.AdamW(model3.parameters(), lr = 1e-5) , 2, 'model_reg.pth')\n\n# optimizer = torch.optim.AdamW(model3.parameters(), lr=1e-4)\n# train_model_reg(model3, dataloaders_dictreg, criterion, optimizer, 6, 'model_reg.pth')\n# optimizer = torch.optim.AdamW(model3.parameters(), lr=1e-5)\n# train_model_reg(model3, dataloaders_dictreg, criterion, optimizer, 4, 'model_reg.pth')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T12:13:16.862909Z","iopub.execute_input":"2022-08-16T12:13:16.863293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ansreg, idsreg = predict_reg(model3, val_loaderreg)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_loader_reg = DataLoader(NewImgDataset(val2), batch_size=batch_size, shuffle=False, num_workers=1)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ansregfull, idsregfull = predict_reg(model3, full_loader_reg)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"foo1 = {x: label_probs[i] for i, x in enumerate(ids1) if x in idsreg}\nfoo2 = {x: ansreg[i] for i, x in enumerate(idsreg)}\n[(foo1[k], foo2[k].item()) for k in foo1.keys()] \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean([abs((foo1[k] - foo2[k].item())) for k in foo1.keys()])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ans1, ids1 = predict(model1, val_loader)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"factor = 1\nfoo = np.transpose( np.transpose(ans1)* (acc1 + factor * ansregfull))\nbar = np.transpose( np.transpose(ans2)* (acc1 - factor * ansregfull))\njoint_mass = np.multiply(foo, bar) + np.multiply(foo, np.transpose(np.array([1 - np.sum(bar,axis=1)]))) + np.multiply(bar, np.transpose(np.array([1 - np.sum(foo,axis=1)])))\nbetp = joint_mass / np.transpose(np.array([np.sum(joint_mass,axis=1)]))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc3 = sum([(x[0] > x[1] and labels[id] == 0) or (x[0] < x[1] and labels[id] == 1) for id, x in enumerate(betp)])/ len(betp)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference","metadata":{}},{"cell_type":"code","source":"# gc.enable()\n# debug = False\n# test_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\n# dirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\n\n# try:\n#     os.mkdir(\"../test/\")\n# except:\n#     pass\n# for i in tqdm(range(test_df.shape[0])):\n#     img_id = test_df.iloc[i].image_id\n#     try:\n#         sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n#     except:\n#         sz = 1000000000\n#     if(sz > 8e8):\n#         img = np.zeros((512,512,3), np.uint8)\n#     else:\n#         try:\n#             img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n#         except:\n#             img = np.zeros((512,512,3), np.uint8)\n#     cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n#     del img\n#     gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \n    \nclass ImgDataset_nolabel(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            print('doopa')\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            print('doopa')\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id\n       \n\n# def predict(model, dataloader):\n#     model.cuda()\n#     model.eval()\n#     dataloader = dataloader\n#     outputs = []\n#     s = nn.Softmax(dim=1)\n#     ids = []\n#     for item in tqdm(dataloader, leave=False):\n#         patient_id = item[2][0]\n#         try:\n#             images = item[0].cuda().float()\n#             ids.append(patient_id)\n#             output = model(images)\n#             outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n#         except:\n#             ids.append(patient_id)\n#             outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n#     return np.array(outputs), ids\n\n\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset_nolabel(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\nans_test_1, id_test_1 = predict(model1, test_loader)\nans_test_2, id_test_2 = predict(model2, test_loader)\n\n# regtest_loader = DataLoader(\n#     ImgDataset_nolabel(test_df), \n#     batch_size=batch_size, \n#     shuffle=False, \n#     num_workers=1\n# )\nans_test_reg, id_test_reg = predict_reg(model3, test_loader)\n\n# prob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\n# submission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\n# submission.CE = prob.CE.to_list()\n# submission.LAA = prob.LAA.to_list()\n# submission.to_csv(\"submission_efficientnet.csv\", index = False)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_loader = DataLoader(ImgDataset(val), batch_size=1, shuffle=False, num_workers=1)\n# model1 = torch.jit.load('model.pth')\n# #model2 = torch.jit.load('efficientnet_model.pth')\n# ans1, ids1 = predict(model1, val_loader)\n\nans_test_reg, id_test_reg = predict_reg(model3, test_loader)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble\n\nDempster-Shafer like ensemble (discounting based on accuracy and results of model3, combination is simplified as it needs to be converted to BetP anyway)","metadata":{}},{"cell_type":"code","source":"foo = np.transpose( np.transpose(ans_test_1)* (acc1 + ans_test_reg))\nbar = np.transpose( np.transpose(ans_test_2)* (acc1 - ans_test_reg))\njoint_mass = np.multiply(foo, bar) + np.multiply(foo, np.transpose(np.array([1 - np.sum(bar,axis=1)]))) + np.multiply(bar, np.transpose(np.array([1 - np.sum(foo,axis=1)])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"betp = joint_mass / np.transpose(np.array([np.sum(joint_mass,axis=1)]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"betp[:,0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob = pd.DataFrame({\"CE\" : betp[:,0], \"LAA\" : betp[:,1], \"id\" : id_test_1}).groupby(\"id\").mean()\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\nsubmission.to_csv(\"submission.csv\", index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}