{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"sz = 384\nbs = 4\nnfolds = 10\nSEED = 2022\nN = 16 #number of tiles per image\nEPOCHS = 10\nN_tile = 64","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-26T12:17:34.437329Z","iopub.execute_input":"2022-09-26T12:17:34.438499Z","iopub.status.idle":"2022-09-26T12:17:34.448498Z","shell.execute_reply.started":"2022-09-26T12:17:34.438415Z","shell.execute_reply":"2022-09-26T12:17:34.447554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:34.45868Z","iopub.execute_input":"2022-09-26T12:17:34.458953Z","iopub.status.idle":"2022-09-26T12:17:34.464371Z","shell.execute_reply.started":"2022-09-26T12:17:34.458922Z","shell.execute_reply":"2022-09-26T12:17:34.463307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport zipfile\nimport torch\n\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nimport cv2\n# from efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nfrom torchvision import models\nimport torchvision\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom fastai.vision import *\nfrom fastai.layers import AdaptiveConcatPool2d, Flatten, Mish\nimport timm","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:34.465854Z","iopub.execute_input":"2022-09-26T12:17:34.466454Z","iopub.status.idle":"2022-09-26T12:17:36.57675Z","shell.execute_reply.started":"2022-09-26T12:17:34.466406Z","shell.execute_reply":"2022-09-26T12:17:36.575481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nscaler = torch.cuda.amp.GradScaler()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:36.580588Z","iopub.execute_input":"2022-09-26T12:17:36.582324Z","iopub.status.idle":"2022-09-26T12:17:36.637763Z","shell.execute_reply.started":"2022-09-26T12:17:36.582276Z","shell.execute_reply":"2022-09-26T12:17:36.636579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttentionSoftMax(torch.nn.Module):\n    def __init__(self, in_features=3, out_features = None):\n        super(AttentionSoftMax, self).__init__()\n        self.otherdim = 'b'\n        if out_features is None:\n            out_features = in_features\n\n        self.layer_linear_tr = nn.Linear(in_features, out_features) \n        self.activation = nn.LeakyReLU() \n        self.layer_linear_query= nn.Linear(out_features, 1)\n\n    def forward(self, x):\n        keys = self.layer_linear_tr(x)\n        keys = self.activation(keys)\n\n        attention_map_raw = self.layer_linear_query(keys)[...,0]\n        attention_map = nn.Softmax(dim=-1)(attention_map_raw)\n\n        result = torch.einsum(f'{self.otherdim}i, {self.otherdim}ij->{self.otherdim}j', attention_map, x)\n        return result\n\nclass Model (nn.Module):\n    def __init__(self, arch='tf_efficientnetv2_s', n=2, pre=True, enc_out_feat=1280): \n        super().__init__()\n        m = timm.create_model(model_name=arch, pretrained = pre, num_classes = 0)\n        self.enc = nn.Sequential(*list(m.children())[:-1])\n        self.enc_out_feat = enc_out_feat\n        self.head = nn.Sequential(AttentionSoftMax(enc_out_feat), nn.Dropout(0.5), nn.Linear(enc_out_feat,n))\n\n    def forward(self, x):\n#         print(x.shape)\n        bs, instance_num, c, w, h = x.shape \n        x = x.view(bs*instance_num,c,w,h)\n\n        # print(x.shape)\n        #x: bs*instance_num x cx wxh\n\n        x = self.enc(x)\n        #x: bs instance_num x enc_out_feat\n\n        x = x.view(bs,instance_num, self.enc_out_feat).contiguous()\n\n        #x: bs x instance_num x enc_out_feat\n        x = self.head(x)\n        #x: bs x n\n\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:36.638914Z","iopub.execute_input":"2022-09-26T12:17:36.639748Z","iopub.status.idle":"2022-09-26T12:17:36.657434Z","shell.execute_reply.started":"2022-09-26T12:17:36.63971Z","shell.execute_reply":"2022-09-26T12:17:36.656372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #PyTorch\n# class DiceBCELoss(nn.Module):\n#     def __init__(self, weight=None, size_average=True):\n#         super(DiceBCELoss, self).__init__()\n        \n#     def onehot_onezero(y_true):\n#         res = []\n#         for i in y_true:\n#             if i == 0:\n#                 res.append(torch.tensor([1., 0.]))\n#             else:\n#                 res.append(torch.tensor([0., 1.]))\n\n#         return torch.stack(res)\n\n#     def forward(self, inputs, targets, smooth=1):\n        \n#         #comment out if your model contains a sigmoid or equivalent activation layer\n#         inputs = F.sigmoid(inputs)  \n        \n#         targets = DiceBCELoss.onehot_onezero(targets).to(device)\n#         #flatten label and prediction tensors\n#         inputs = inputs.view(-1)\n#         targets = targets.view(-1)\n        \n#         intersection = (inputs * targets).sum()                            \n#         dice_loss = 1 - (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)  \n#         BCE = torch.nn.BCELoss()(inputs, targets)\n#         Dice_BCE = BCE + dice_loss\n        \n#         return Dice_BCE","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:36.661974Z","iopub.execute_input":"2022-09-26T12:17:36.662319Z","iopub.status.idle":"2022-09-26T12:17:36.670996Z","shell.execute_reply.started":"2022-09-26T12:17:36.66229Z","shell.execute_reply":"2022-09-26T12:17:36.669785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLossWithOneHot(nn.Module):\n    def __init__(self, gamma=0, eps=1e-7):\n        super(FocalLossWithOneHot, self).__init__()\n        self.gamma = gamma\n        self.eps = eps\n\n    def forward(self, input, target):\n        y = torch.nn.functional.one_hot(target.to(torch.int64), num_classes=2)\n\n        logit = torch.nn.functional.softmax(input, dim=-1)\n        logit = logit.clamp(self.eps, 1. - self.eps)\n\n        loss = -1 * y * torch.log(logit) # cross entropy\n        loss = loss * (1 - logit) ** self.gamma # focal loss\n\n        return loss.sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:36.672576Z","iopub.execute_input":"2022-09-26T12:17:36.6736Z","iopub.status.idle":"2022-09-26T12:17:36.686589Z","shell.execute_reply.started":"2022-09-26T12:17:36.673549Z","shell.execute_reply":"2022-09-26T12:17:36.685596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model()\nmodel = model.to(device)\n\n# クロスエントロピー損失関数使用\n# loss_fn = nn.BCEWithLogitsLoss().cuda()\nloss_fn = nn.CrossEntropyLoss().cuda()\n# loss_fn = nn.CrossEntropyLoss(label_smoothing=0.1).cuda()\n\n# loss_fn = FocalLossWithOneHot(gamma=2).cuda()\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0001)\n\n# optimizer = SGD(model, 0.1)\nscheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n# 前処理\ntransform = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((sz, sz)),\n#     torchvision.transforms.RandAugment(),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.RandomVerticalFlip(),\n    torchvision.transforms.RandomRotation(45),\n#     torchvision.transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n])\n\ntransform_val = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((sz, sz)),\n#     torchvision.transforms.RandAugment(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:36.691638Z","iopub.execute_input":"2022-09-26T12:17:36.692324Z","iopub.status.idle":"2022-09-26T12:17:49.545166Z","shell.execute_reply.started":"2022-09-26T12:17:36.692288Z","shell.execute_reply":"2022-09-26T12:17:49.544025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.546584Z","iopub.execute_input":"2022-09-26T12:17:49.546956Z","iopub.status.idle":"2022-09-26T12:17:49.552065Z","shell.execute_reply.started":"2022-09-26T12:17:49.546919Z","shell.execute_reply":"2022-09-26T12:17:49.551114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from torchinfo import summary\n# summary(model=model, input_size=(bs, N, 3, sz, sz))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.553477Z","iopub.execute_input":"2022-09-26T12:17:49.554317Z","iopub.status.idle":"2022-09-26T12:17:49.563213Z","shell.execute_reply.started":"2022-09-26T12:17:49.55428Z","shell.execute_reply":"2022-09-26T12:17:49.562268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This block is reffered from https://www.kaggle.com/code/yasufuminakama/mayo-train-images-size-1024-n-16-1/notebook\ntrain = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\n# train = train[train[\"image_id\"] != \"2c3c06_0\"]\n\ntrain['image_dir'] = ''\n\ntrain.loc[:100,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_1/'\ntrain.loc[100:200,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_2/'\ntrain.loc[200:300,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_3/'\ntrain.loc[300:400,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_4/'\ntrain.loc[400:500,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_5/'\ntrain.loc[500:600,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_6/'\ntrain.loc[600:700,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_7/'\ntrain.loc[700:,'image_dir'] = '../input/mayo-tiled-16-384x384/train_images_noscale/train_images/train_images_8/'\n# train.loc[:100,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_1/'\n# train.loc[100:200,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_2/'\n# train.loc[200:300,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_3/'\n# train.loc[300:400,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_4/'\n# train.loc[400:500,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_5/'\n# train.loc[500:600,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_6/'\n# train.loc[600:700,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_7/'\n# train.loc[700:,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_8/'\n\ntarget_mapper = {\"CE\": 0, \"LAA\": 1}\n\ntrain[\"target\"] = train[\"label\"].map(lambda x: target_mapper[x])\n\nsplits = StratifiedKFold(n_splits=nfolds, random_state=SEED, shuffle=True)\nsplits = list(splits.split(train,train.center_id))\nfolds_splits = np.zeros(len(train)).astype(np.int)\nfor i in range(nfolds): folds_splits[splits[i][1]] = i\ntrain['split'] = folds_splits\n\nclass TrainDataset(Dataset):\n    def __init__(self, cfg, df, transform=None, aug=True):\n        self.cfg = cfg\n        self.image_ids = df['image_id'].values\n        self.image_dirs = df['image_dir'].values\n#         self.image_path = df[\"path\"].values\n        self.labels = df['target'].values\n        self.transform = transform\n        self.aug = aug\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        image_dir = self.image_dirs[idx]\n        images = []\n#         img_indexes = random.sample(list(range(0, N_tile)), N)\n        img_indexes = list(range(0, N))\n\n        for i in img_indexes:\n            path = image_dir + image_id + f'_{i}.jpg'\n            image = cv2.imread(path)\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            image = self.transform(image)\n            images.append(image)\n        images = torch.stack(images, dim=0)\n        \n#         if self.aug:\n#             images = torchvision.transforms.RandAugment()(images)\n            \n        label = torch.tensor(self.labels[idx]).long()\n        return images, label","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.564892Z","iopub.execute_input":"2022-09-26T12:17:49.565541Z","iopub.status.idle":"2022-09-26T12:17:49.595566Z","shell.execute_reply.started":"2022-09-26T12:17:49.565506Z","shell.execute_reply":"2022-09-26T12:17:49.594511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, device, train_loader, val_loader, optimizer, scheduler, epoch, loss_fn):\n    running_loss = 0.\n    last_loss = 0.\n    val_loss = 0.\n    model.train(True)\n\n    # Here, we use enumerate(training_loader) instead of\n    # iter(training_loader) so that we can track the batch\n    # index and do some intra-epoch reporting\n    for i, (data, target) in enumerate(train_loader):\n        # Every data instance is an input + label pair\n        inputs, labels = data.to(device), target.to(device)\n        # Zero your gradients for every batch!\n        optimizer.zero_grad()\n\n        # Make predictions for this batch\n        with torch.cuda.amp.autocast():\n            outputs = model(inputs)\n#             loss = loss_fn(outputs, labels)\n\n#             loss = loss_fn(torch.squeeze(outputs), labels.float())\n            loss = loss_fn(outputs, labels)\n\n#         if len(outputs) != bs:\n#             print(outputs)\n\n        # Compute the loss and its gradients\n        \n        scaler.scale(loss).backward()\n        \n        scaler.step(optimizer)\n        scaler.update()\n\n        # Gather data and report\n        running_loss += loss.item()\n        if i % 10 == 9:\n            last_loss = running_loss / 10 # loss per batch\n            print('  batch {} loss: {}'.format(i + 1, last_loss))\n            tb_x = epoch * len(train_loader) + i + 1\n            running_loss = 0.\n#     scheduler.step()\n    \n#     model.requires_grad_(False)\n    model.eval()\n    with torch.no_grad():\n        for j, (data, target) in enumerate(val_loader):\n            inputs, labels = data.to(device), target.to(device)\n            with torch.cuda.amp.autocast():\n                outputs = model(inputs)\n#                 loss = loss_fn(torch.squeeze(outputs), labels.float())\n                loss = loss_fn(outputs, labels)\n\n            val_loss += loss.item()\n    \n    return val_loss/j","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.597096Z","iopub.execute_input":"2022-09-26T12:17:49.597737Z","iopub.status.idle":"2022-09-26T12:17:49.608105Z","shell.execute_reply.started":"2022-09-26T12:17:49.597699Z","shell.execute_reply":"2022-09-26T12:17:49.607168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i, one_batch in enumerate(training_loader):\n#     x = one_batch[0]\n#     print(x.shape)\n#     y = one_batch[1]\n#     x = [x for x in x]\n#     shape = x[0].shape\n#     print(torch.stack(x,1).view(-1,shape[1],shape[2],shape[3]).shape)\n#     if i==5:\n#         break","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.613782Z","iopub.execute_input":"2022-09-26T12:17:49.614114Z","iopub.status.idle":"2022-09-26T12:17:49.620806Z","shell.execute_reply.started":"2022-09-26T12:17:49.614087Z","shell.execute_reply":"2022-09-26T12:17:49.619712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.621857Z","iopub.execute_input":"2022-09-26T12:17:49.623003Z","iopub.status.idle":"2022-09-26T12:17:49.634447Z","shell.execute_reply.started":"2022-09-26T12:17:49.622967Z","shell.execute_reply":"2022-09-26T12:17:49.633518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(nfolds):\n    training_set = TrainDataset(None,train[train[\"split\"]!=fold], transform=transform, aug=True)\n    training_loader = torch.utils.data.DataLoader(training_set, shuffle=True, num_workers=2, batch_size=bs, drop_last=True)\n    val_set = TrainDataset(None,train[train[\"split\"]==fold], transform=transform_val, aug=False)\n    val_loader = torch.utils.data.DataLoader(val_set, shuffle=True, num_workers=2, batch_size=bs, drop_last=True)\n\n    epoch_number=0\n    avg_loss = [999,]\n    for epoch in range(EPOCHS):\n        print('EPOCH {}:'.format(epoch_number + 1))\n\n        epoch_loss = train_one_epoch(model=model, device=device, train_loader=training_loader, val_loader=val_loader, optimizer=optimizer, scheduler=scheduler, epoch=epoch_number, loss_fn=loss_fn)\n        epoch_number += 1\n        print(\"EPOCH \", str(epoch+1), \"val loss: \", epoch_loss)\n        \n        if min(avg_loss) > epoch_loss:\n            print(\"save model concat-tile-pooling-384-effv2-fold{}.pth\".format(str(fold)))\n            torch.save(model.state_dict(), \"concat-tile-pooling-384-effv2-fold{}.pth\".format(str(fold)))\n        \n        avg_loss.append(epoch_loss)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:17:49.63587Z","iopub.execute_input":"2022-09-26T12:17:49.636302Z","iopub.status.idle":"2022-09-26T12:53:08.024595Z","shell.execute_reply.started":"2022-09-26T12:17:49.636267Z","shell.execute_reply":"2022-09-26T12:53:08.022966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for input, label in training_loader:\n#     break","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:53:08.026414Z","iopub.status.idle":"2022-09-26T12:53:08.027181Z","shell.execute_reply.started":"2022-09-26T12:53:08.026926Z","shell.execute_reply":"2022-09-26T12:53:08.026954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:53:08.028588Z","iopub.status.idle":"2022-09-26T12:53:08.029331Z","shell.execute_reply.started":"2022-09-26T12:53:08.029074Z","shell.execute_reply":"2022-09-26T12:53:08.029099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.save(model.state_dict(), \"concat-tile-pooling-model.pth\")","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:53:08.030648Z","iopub.status.idle":"2022-09-26T12:53:08.031401Z","shell.execute_reply.started":"2022-09-26T12:53:08.031128Z","shell.execute_reply":"2022-09-26T12:53:08.031153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/models\n!mv /kaggle/working/*.pth /kaggle/working/models/","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:53:08.032717Z","iopub.status.idle":"2022-09-26T12:53:08.033447Z","shell.execute_reply.started":"2022-09-26T12:53:08.03317Z","shell.execute_reply":"2022-09-26T12:53:08.033193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -cvf /kaggle/working/models.tar /kaggle/working/models/","metadata":{"execution":{"iopub.status.busy":"2022-09-26T12:53:08.03479Z","iopub.status.idle":"2022-09-26T12:53:08.035519Z","shell.execute_reply.started":"2022-09-26T12:53:08.035245Z","shell.execute_reply":"2022-09-26T12:53:08.03527Z"},"trusted":true},"execution_count":null,"outputs":[]}]}