{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Baseline SEResNeXt50 Classification Model + 5fold Training and Inference\n\nThanks to [@xhlulu](https://www.kaggle.com/xhlulu) for the 512x512 image dataset which can be found [here](https://www.kaggle.com/xhlulu/panda-resized-train-data-512x512)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install pytorchcv\n!pip install ../input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet","execution_count":null,"outputs":[]},{"metadata":{"id":"uzJo_QdJrJnS","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom tqdm import tqdm,trange\nfrom sklearn.model_selection import train_test_split\nimport sklearn.metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nfrom numba import jit \n\n# Kindly stolen from: https://www.kaggle.com/c/prostate-cancer-grade-assessment/discussion/145105\n@jit\ndef qwk3(a1, a2, max_rat):\n    assert(len(a1) == len(a2))\n    a1 = np.asarray(a1, dtype=int)\n    a2 = np.asarray(a2, dtype=int)\n\n    hist1 = np.zeros((max_rat + 1, ))\n    hist2 = np.zeros((max_rat + 1, ))\n\n    o = 0\n    for k in range(a1.shape[0]):\n        i, j = a1[k], a2[k]\n        hist1[i] += 1\n        hist2[j] += 1\n        o +=  (i - j) * (i - j)\n\n    e = 0\n    for i in range(max_rat + 1):\n        for j in range(max_rat + 1):\n            e += hist1[i] * hist2[j] * (i - j) * (i - j)\n\n    e = e / a1.shape[0]\n\n    return 1 - o / e","execution_count":null,"outputs":[]},{"metadata":{"id":"Mzm9cE0FrfrU","trusted":true},"cell_type":"code","source":"IMAGE_PATH = '../input/panda-resized-train-data-512x512/train_images/train_images/'\nnum_classes = 6\n\nfrom sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold\nmskf = StratifiedKFold(n_splits=5, random_state=12)\n\ntrain_df2 = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')\n# train_df2 = train_df2.sample(frac=1).reset_index(drop=True)\ntrain_df2 = train_df2.drop(['gleason_score'], axis=1)\nX, y = train_df2.values[:,0:2], train_df2[['isup_grade']].values[:,0]\n\ntrain_df2['fold'] = -1\nfor fld, (_, test_idx) in enumerate(mskf.split(X, y)):\n    train_df2.iloc[test_idx, -1] = fld","execution_count":null,"outputs":[]},{"metadata":{"id":"sWixylOGuvzT"},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"id":"p97FNaLVuesJ","trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\n\nclass ImageDataset(Dataset):\n    def __init__(self, dataframe, root_dir, folds, transform=None):\n        self.df = dataframe[dataframe.fold.isin(folds).reset_index(drop=True)]\n        self.root_dir = root_dir\n        self.transform = transform\n        self.folds = folds\n\n        self.paths = self.df.image_id.values\n        self.labels = self.df.values[:,2]\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.paths[idx]\n        img_path = f'{self.root_dir}{img_name}.png'\n        \n        img = cv2.imread(img_path)\n        # img = cv2.resize(img, (224, 130), interpolation=cv2.INTER_AREA)\n        \n        if self.transform is not None:\n            img = self.transform(image=img)['image']\n        \n        img = np.rollaxis(img, -1, 0)\n        \n        labels = np.array(self.labels[idx]).astype(np.long)\n        return [img, labels]","execution_count":null,"outputs":[]},{"metadata":{"id":"4uMauTcMxLk7"},"cell_type":"markdown","source":"## Model"},{"metadata":{"id":"WhrbPWmXxHrE","trusted":true},"cell_type":"code","source":"from pytorchcv.model_provider import get_model\n\nclass Head(torch.nn.Module):\n    def __init__(self, in_f, out_f):\n        super(Head, self).__init__()\n    \n        self.f = nn.Flatten()\n        self.d = nn.Dropout(0.25)\n        self.o = nn.Linear(in_f, out_f)\n\n    def forward(self, x):\n        x = self.f(x)\n        x = self.d(x)\n\n        out = self.o(x)\n        return out\n\nclass FCN(torch.nn.Module):\n    def __init__(self, base, in_f):\n        super(FCN, self).__init__()\n        self.base = base\n        self.h1 = Head(in_f, num_classes)\n  \n    def forward(self, x):\n        x = self.base(x)\n        return self.h1(x)\n\ndef create_model():\n    model = get_model(\"seresnext50_32x4d\", pretrained=False)\n    model.load_state_dict(torch.load('../input/seresnext50-32x4d-pretrained/seresnext50_32x4d-0521-b0ce2520.pth'))\n    model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n    model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n    model = FCN(model, 2048)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"id":"CQCJ9XDyxmFE"},"cell_type":"markdown","source":"## Train funcs"},{"metadata":{"id":"7oQAFCx6xnnS","trusted":true},"cell_type":"code","source":"\n\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=0, alpha=None, size_average=True):\n        super(FocalLoss, self).__init__()\n        self.gamma = gamma\n        self.alpha = alpha\n        if isinstance(alpha,(float,int)): self.alpha = torch.Tensor([alpha,1-alpha])\n        if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)\n        self.size_average = size_average\n\n    def forward(self, input, target):\n        if input.dim()>2:\n            input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W\n            input = input.transpose(1,2)    # N,C,H*W => N,H*W,C\n            input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C\n        target = target.view(-1,1)\n\n        logpt = F.log_softmax(input)\n        logpt = logpt.gather(1,target)\n        logpt = logpt.view(-1)\n        pt = Variable(logpt.data.exp())\n\n        if self.alpha is not None:\n            if self.alpha.type()!=input.data.type():\n                self.alpha = self.alpha.type_as(input.data)\n            at = self.alpha.gather(0,target.data.view(-1))\n            logpt = logpt * Variable(at)\n\n        loss = -1 * (1-pt)**self.gamma * logpt\n        if self.size_average: return loss.mean()\n        else: return loss.sum()\n        \ndef cal_loss(pred, gold, smoothing):\n    ''' Calculate cross entropy loss, apply label smoothing if needed. '''\n\n    gold = gold.contiguous().view(-1)\n#     Constants.PAD  = -1\n    if smoothing:\n        eps = 0.1\n        n_class = pred.size(1)\n\n        one_hot = torch.zeros_like(pred).scatter(1, gold.view(-1, 1), 1)\n        one_hot = one_hot * (1 - eps) + (1 - one_hot) * eps / (n_class - 1)\n        log_prb = F.log_softmax(pred, dim=1)\n\n        non_pad_mask = gold.ne(-1)\n        loss = -(one_hot * log_prb).sum(dim=1)\n        loss = loss.masked_select(non_pad_mask).sum()  # average later\n    else:\n        loss = F.cross_entropy(pred, gold, ignore_index=-1, reduction='sum')\n\n    return loss\n\n\ndef criterion1(pred1, targets):\n    l1 = FocalLoss(gamma=2)(pred1, targets)\n    \n    return l1\n\ndef criterion1_smloss(pred1, targets):\n    l1 = cal_loss(pred1, targets, 1)\n    return l1\n\ndef criterion1_orig(pred1, targets):\n    l1 = F.cross_entropy(pred1, targets)\n    return l1\n\ndef train_model(epoch, optimizer, scheduler=None, history=None):\n    model.train()\n    total_loss = 0\n    \n    t = tqdm(train_loader)\n    for batch_idx, (img_batch, y_batch) in enumerate(t):\n        img_batch = img_batch.cuda().float()\n        y_batch = y_batch.cuda()\n        \n        optimizer.zero_grad()\n        \n        output1 = model(img_batch)\n        loss = criterion1(output1, y_batch)\n\n        total_loss += loss.data.cpu().numpy()\n        t.set_description(f'Epoch {epoch+1}/{n_epochs}, LR: %6f, Loss: %.4f'%(optimizer.state_dict()['param_groups'][0]['lr'],total_loss/(batch_idx+1)))\n\n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n            history.loc[epoch + batch_idx / len(train_loader), 'lr'] = optimizer.state_dict()['param_groups'][0]['lr']\n        \n        loss.backward()\n        optimizer.step()\n        if scheduler is not None:\n            scheduler.step()\n\ndef evaluate_model(epoch, scheduler=None, history=None):\n    model.eval()\n    loss = 0\n    \n    preds_1 = []\n    tars_1 = []\n    with torch.no_grad():\n        t = tqdm(val_loader)\n        for img_batch, y_batch in t:\n            img_batch = img_batch.cuda().float()\n            y_batch = y_batch.cuda()\n\n            o1 = model(img_batch)\n\n            l1 = criterion1(o1, y_batch)\n            loss += l1\n\n            for j in range(len(o1)):\n                preds_1.append(torch.argmax(F.softmax(o1[j]), -1))\n            for i in y_batch:\n                tars_1.append(i.data.cpu().numpy())\n    \n    preds_1 = [p.data.cpu().numpy() for p in preds_1]\n    preds_1 = np.array(preds_1).T.reshape(-1)\n\n    acc = sklearn.metrics.recall_score(tars_1, preds_1, average='macro')\n    final_score = qwk3(tars_1, preds_1, 5)\n    \n    loss /= len(val_loader)\n    \n    if history is not None:\n        history.loc[epoch, 'val_loss'] = loss.cpu().numpy()\n        history.loc[epoch, 'acc'] = acc\n        history.loc[epoch, 'qwk'] = final_score\n    \n    if scheduler is not None:\n        scheduler.step(final_score)\n\n    print(f'Dev loss: %.4f, QWK: {final_score}, Acc: {acc}'%(loss))\n    \n    return loss, final_score","execution_count":null,"outputs":[]},{"metadata":{"id":"q474NOtTyd4t"},"cell_type":"markdown","source":"## Augmentation"},{"metadata":{"id":"spHI34d9ydOW","trusted":true},"cell_type":"code","source":"import albumentations as A\n\ntrain_transform = A.Compose([\n                             A.Normalize(always_apply=True)\n])\nval_transform = A.Compose([\n                           A.Normalize(always_apply=True)\n])\n\nfold = 0\nfolds = [0,1,2,3,4]\ntrain_dataset = ImageDataset(train_df2, IMAGE_PATH, folds=[i for i in folds if i != fold], transform=train_transform)\nval_dataset = ImageDataset(train_df2, IMAGE_PATH, folds=[fold], transform=val_transform)","execution_count":null,"outputs":[]},{"metadata":{"id":"sDju5Ixnytuz","outputId":"2eab9872-2a96-47be-cd27-2aabec75f5fd","trusted":true},"cell_type":"code","source":"nrow, ncol = 3, 6\nfig, axes = plt.subplots(nrow, ncol, figsize=(20, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_dataset[i]\n    ax.imshow(image[0])\n    ax.set_title(f'label: {label}')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"id":"_AfB7fE4zl1N"},"cell_type":"markdown","source":"## 5fold Training"},{"metadata":{"id":"icPt9ftIzdCC","outputId":"0d66ab55-7394-41b7-bb56-b68ad81c243a","trusted":true},"cell_type":"code","source":"import gc\n\nfolds = [0,1,2,3,4]\n\nvalidations = []\n\nfor fold in range(5):\n\n    \n    history = pd.DataFrame()\n    history2 = pd.DataFrame()\n\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    best = 0\n    best2 = 1e10\n    n_epochs = 12\n    early_epoch = 0\n\n    train_dataset = ImageDataset(train_df2, IMAGE_PATH, folds=[i for i in folds if i != fold], transform=train_transform)\n    val_dataset = ImageDataset(train_df2, IMAGE_PATH, folds=[fold], transform=val_transform)\n    \n    BATCH_SIZE = 16\n    \n    train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\n    val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n    \n    model = create_model()\n    model = model.cuda()\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, mode='max', factor=0.75, verbose=True, min_lr=1e-5)\n\n    for epoch in range(n_epochs-early_epoch):\n        epoch += early_epoch\n        torch.cuda.empty_cache()\n        gc.collect()\n\n        train_model(epoch, optimizer, scheduler=None, history=history)\n\n        loss, kaggle = evaluate_model(epoch, scheduler=scheduler, history=history2)\n\n        if kaggle > best:\n            best = kaggle\n            print(f'Saving best model... (qwk)')\n            torch.save(model.state_dict(), f'model-fld{fold+1}.pth')\n        \n    print()\n    validations.append(best)\n\nvalidations = np.array(validations)\nfor i,val in enumerate(validations):\n    print(f'Fold {i+1}: {val}')\nprint(f'5fold CV: {np.mean(validations)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5fold Inference"},{"metadata":{"id":"fW80PAOSG2QG","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm\nfrom timeit import default_timer as timer\nimport skimage.io\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import *\n\nif True:\n    DATA_DIR = '/kaggle/input/prostate-cancer-grade-assessment/'\n    SUBMISSION_CSV_FILE = 'submission.csv'\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Use this to test inference\ntrain = pd.read_csv(f'{DATA_DIR}train.csv')[:1000]\n# submission = train\n\nsubmission = pd.read_csv(f'{DATA_DIR}sample_submission.csv')\n\nWIDTH = 512\nHEIGHT = 512\n\n#### net #########################################################################\n\ndef do_predict(net, inputs):\n    def logit_to_probability(logit):\n        probability=[]\n        for l in logit:\n            p = F.softmax(l)\n            probability.append(p)\n        return probability\n    \n    num_ensemble = len(net)\n    for i in range(num_ensemble):\n        net[i].eval()\n\n    probability=[0,0,0,0]\n    for i in range(num_ensemble):\n        logit = net[i](inputs)\n        prob = logit_to_probability(logit)\n        probability = [p+q for p,q in zip(probability,prob)]\n    \n    #----\n    probability = [p/num_ensemble for p in probability]\n    predict = [torch.argmax(p,-1) for p in probability]\n    predict = [p.data.cpu().numpy() for p in predict]\n    predict = np.array(predict).T\n    predict = predict.reshape(-1)\n\n    return predict\n\n## load net -----------------------------------\nnet = []\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld1.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld2.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld3.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld4.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld5.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\n#------------------------------------------\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport albumentations as A\n\ntest_transform = A.Compose([\n                           A.Normalize(always_apply=True)\n])\n\nclass ImageDataset(Dataset):\n    def __init__(self, dataframe, root_dir, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n\n        self.paths = self.df.image_id.values\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.paths[idx]\n        file_path = f'{self.root_dir}{img_name}.tiff'\n        \n        image = skimage.io.MultiImage(file_path)\n        image = cv2.resize(image[-1], (WIDTH, HEIGHT))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform is not None:\n            img = self.transform(image=image)['image']\n        \n        img = np.rollaxis(img, -1, 0)\n        \n        return img\n#---------------------------------------------\n\ndef run_make_submission_csv():\n    target=[]\n    batch_size= 4\n\n    if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):\n    # Use below lines to test inference\n#     if True:\n#         test_dataset = ImageDataset(train, f'{DATA_DIR}train_images/', test_transform)\n        test_dataset = ImageDataset(submission, f'{DATA_DIR}test_images/', test_transform)\n        test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n        \n        t = tqdm(test_loader)\n        with torch.no_grad():\n            for b, image_batch in enumerate(t):\n                image_batch = image_batch.cuda().float()\n                predict = do_predict(net, image_batch)\n                target.append(predict)\n        print('')\n    #---------\n    else:\n        target = [[1],[1],[1]]\n    target = np.concatenate(target)\n\n    submission['isup_grade'] = target\n    submission['isup_grade'] = submission['isup_grade'].astype(int)\n    submission.to_csv(SUBMISSION_CSV_FILE, index=False)\n    print(submission.head())\n\nif __name__ == '__main__':\n    run_make_submission_csv()\n\n    print('\\nsucess!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## And thats it! Thanks for reading and make sure to upvote if you found this kernal helpful!\n\nThings that you can experiment with:\n- Change amount of epochs\n- Change optimizer/scheduler\n- Add basic Augmentations (SSR, Cutout, etc.)\n- Add complex Augmentations (Mixup, Cutmix, etc.)\n- Change Backbone (Resnet, EfficientNet, etc.)\n- Change model head (add another linear layer, add batchnormalization, change dropout, etc.)\n- Change image size (256x256, 128x128, etc.)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset[0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset[0].shape)","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"PANDA.ipynb","provenance":[],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":4}