{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Baseline SEResNeXt50 Classification Model + 5fold Training and Inference\n\nThanks to [@xhlulu](https://www.kaggle.com/xhlulu) for the 512x512 image dataset which can be found [here](https://www.kaggle.com/xhlulu/panda-resized-train-data-512x512)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Install pytorchcv\n!pip install ../input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet","execution_count":null,"outputs":[]},{"metadata":{"id":"uzJo_QdJrJnS","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom tqdm import tqdm,trange\nfrom sklearn.model_selection import train_test_split\nimport sklearn.metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.metrics import cohen_kappa_score\n\ndef quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')\n\nclass config:\n    IMAGE_WIDTH = 512\n    IMAGE_HEIGHT = 515\n    epochs = 10\n    batch_size = 16\n    num_classes = 6\n    IMAGE_PATH = '../input/panda-resized-train-data-512x512/train_images/train_images/'\n    lr = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"id":"Mzm9cE0FrfrU","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedShuffleSplit, StratifiedKFold\nmskf = StratifiedKFold(n_splits=5, random_state=12)\n\ntrain_df2 = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')\ntrain_df2 = train_df2.drop(['gleason_score'], axis=1)\nX, y = train_df2.values[:,0:2], train_df2[['isup_grade']].values[:,0]\n\ntrain_df2['fold'] = -1\nfor fld, (_, test_idx) in enumerate(mskf.split(X, y)):\n    train_df2.iloc[test_idx, -1] = fld","execution_count":null,"outputs":[]},{"metadata":{"id":"sWixylOGuvzT"},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"id":"p97FNaLVuesJ","trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\n\nclass ImageDataset(Dataset):\n    def __init__(self, dataframe, root_dir, folds, transform=None):\n        self.df = dataframe[dataframe.fold.isin(folds).reset_index(drop=True)]\n        self.root_dir = root_dir\n        self.transform = transform\n        self.folds = folds\n\n        self.paths = self.df.image_id.values\n        self.labels = self.df.values[:,2]\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.paths[idx]\n        img_path = f'{self.root_dir}{img_name}.png'\n        \n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (config.IMAGE_WIDTH, config.IMAGE_HEIGHT), interpolation=cv2.INTER_AREA)\n        img = (255 - img).astype(np.float32) / 255.\n\n        if self.transform is not None:\n          img = self.transform(image=img)['image']\n        \n        img = np.rollaxis(img, -1, 0)\n        \n        labels = np.array(self.labels[idx]).astype(np.long)\n        return [img, labels]","execution_count":null,"outputs":[]},{"metadata":{"id":"4uMauTcMxLk7"},"cell_type":"markdown","source":"# Model"},{"metadata":{"id":"WhrbPWmXxHrE","trusted":true},"cell_type":"code","source":"from pytorchcv.model_provider import get_model\n\nclass Head(torch.nn.Module):\n  def __init__(self, in_f, out_f, dropout):\n    super(Head, self).__init__()\n    \n    self.f = nn.Flatten()\n    self.d = nn.Dropout(0.25)\n    self.dropout = dropout\n    self.o = nn.Linear(in_f, out_f)\n\n  def forward(self, x):\n    x = self.f(x)\n    if self.dropout:\n      x = self.d(x)\n\n    out = self.o(x)\n    return out\n\nclass FCN(torch.nn.Module):\n  def __init__(self, base, in_f, num_classes, dropout=True):\n    super(FCN, self).__init__()\n    self.base = base\n    self.h1 = Head(in_f, num_classes, dropout)\n  \n  def forward(self, x):\n    x = self.base(x)\n    return self.h1(x)\n\ndef create_model():\n    model = get_model(\"seresnext50_32x4d\", pretrained=False)\n    model.load_state_dict(torch.load('../input/seresnext50-32x4d-pretrained/seresnext50_32x4d-0521-b0ce2520.pth'))\n    model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n    model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))\n    model = FCN(model, 2048, config.num_classes, dropout=True)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixup"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ref: https://www.kaggle.com/c/bengaliai-cv19/discussion/128637\n\ndef mixup(data, targets1, alpha):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_targets1 = targets1[indices]\n\n    lam = np.random.beta(alpha, alpha)\n    data = data * lam + shuffled_data * (1 - lam)\n    targets = [targets1, shuffled_targets1, lam]\n\n    return data, targets\n\ndef mixup_criterion(preds1, targets):\n    targets1, targets2, lam = targets[0], targets[1], targets[2]\n    criterion = nn.CrossEntropyLoss(reduction='mean')\n    return lam * criterion(preds1, targets1) + (1 - lam) * criterion(preds1, targets2)","execution_count":null,"outputs":[]},{"metadata":{"id":"CQCJ9XDyxmFE"},"cell_type":"markdown","source":"# Train funcs"},{"metadata":{"id":"7oQAFCx6xnnS","trusted":true},"cell_type":"code","source":"def criterion1(pred1, targets):\n  l1 = F.cross_entropy(pred1, targets)\n  return l1\n\ndef train_model(epoch, optimizer, scheduler=None, history=None):\n    model.train()\n    total_loss = 0\n    \n    t = tqdm(train_loader)\n    for batch_idx, (img_batch, y_batch) in enumerate(t):\n        img_batch = img_batch.cuda().float()\n        y_batch = y_batch.cuda()\n        \n        optimizer.zero_grad()\n        \n        rand = np.random.rand()\n        if rand < 0.5:\n            images, targets = mixup(img_batch, y_batch, 0.4)\n            output1 = model(images)\n            loss = mixup_criterion(output1, targets)\n        else:\n            output1 = model(img_batch)\n            loss = criterion1(output1, y_batch)\n\n        total_loss += loss.data.cpu().numpy()\n        t.set_description(f'Epoch {epoch+1}/{n_epochs}, LR: %6f, Loss: %.4f'%(optimizer.state_dict()['param_groups'][0]['lr'],total_loss/(batch_idx+1)))\n\n        if history is not None:\n          history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n          history.loc[epoch + batch_idx / len(train_loader), 'lr'] = optimizer.state_dict()['param_groups'][0]['lr']\n        \n        loss.backward()\n        optimizer.step()\n        if scheduler is not None:\n          scheduler.step()\n\ndef evaluate_model(epoch, scheduler=None, history=None):\n    model.eval()\n    loss = 0\n    \n    preds_1 = []\n    tars_1 = []\n    with torch.no_grad():\n        t = tqdm(val_loader)\n        for img_batch, y_batch in t:\n            img_batch = img_batch.cuda().float()\n            y_batch = y_batch.cuda()\n\n            o1 = model(img_batch)\n\n            l1 = criterion1(o1, y_batch)\n            loss += l1\n\n            for j in range(len(o1)):\n              preds_1.append(torch.argmax(F.softmax(o1[j]), -1))\n            for i in y_batch:\n              tars_1.append(i.data.cpu().numpy())\n    \n    preds_1 = [p.data.cpu().numpy() for p in preds_1]\n    preds_1 = np.array(preds_1).T.reshape(-1)\n\n    acc = sklearn.metrics.recall_score(tars_1, preds_1, average='macro')\n    final_score = quadratic_weighted_kappa(tars_1, preds_1)\n    \n    loss /= len(val_loader)\n    \n    if history is not None:\n      history.loc[epoch, 'val_loss'] = loss.cpu().numpy()\n      history.loc[epoch, 'acc'] = acc\n      history.loc[epoch, 'qwk'] = final_score\n    \n    if scheduler is not None:\n      scheduler.step(final_score)\n\n    print(f'Dev loss: %.4f, QWK: {final_score}, Acc: {acc}'%(loss))\n    \n    return loss, final_score","execution_count":null,"outputs":[]},{"metadata":{"id":"q474NOtTyd4t"},"cell_type":"markdown","source":"# Augmentation"},{"metadata":{"id":"spHI34d9ydOW","trusted":true},"cell_type":"code","source":"import albumentations as A\n\ntrain_transform = A.Compose([\n                             A.CoarseDropout(max_holes=4, max_height=64, max_width=64, p=0.9),\n                             A.OneOf([\n                              A.ShiftScaleRotate(scale_limit=.15, rotate_limit=20, border_mode=cv2.BORDER_CONSTANT, p=1.0),\n                              A.IAAAffine(shear=20, mode='constant', p=1.0),\n                              A.IAAPerspective(p=1.0),\n                             ], p=.9),\n                             A.HorizontalFlip(p=0.5),\n                             A.VerticalFlip(p=0.5),\n])\n\nfold = 0\nfolds = [0,1,2,3,4]\ntrain_dataset = ImageDataset(train_df2, config.IMAGE_PATH, folds=[i for i in folds if i != fold], transform=train_transform)","execution_count":null,"outputs":[]},{"metadata":{"id":"sDju5Ixnytuz","outputId":"2eab9872-2a96-47be-cd27-2aabec75f5fd","trusted":true},"cell_type":"code","source":"nrow, ncol = 3, 6\nfig, axes = plt.subplots(nrow, ncol, figsize=(20, 8))\naxes = axes.flatten()\nfor i, ax in enumerate(axes):\n    image, label = train_dataset[i]\n    ax.imshow(image[0])\n    ax.set_title(f'label: {label}')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"id":"_AfB7fE4zl1N"},"cell_type":"markdown","source":"# 5fold Training"},{"metadata":{"id":"icPt9ftIzdCC","outputId":"0d66ab55-7394-41b7-bb56-b68ad81c243a","trusted":true},"cell_type":"code","source":"import gc\n\nfolds = [0,1,2,3,4]\n\nvalidations = []\n\nfor fold in range(5):\n    print(f'Train Fold {fold+1}')\n    \n    history = pd.DataFrame()\n    history2 = pd.DataFrame()\n\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    best = 0\n    best2 = 1e10\n    n_epochs = config.epochs\n    \n    train_dataset = ImageDataset(train_df2, config.IMAGE_PATH, folds=[i for i in folds if i != fold], transform=train_transform)\n    val_dataset = ImageDataset(train_df2, config.IMAGE_PATH, folds=[fold], transform=None)\n    \n    BATCH_SIZE = 16\n    \n    train_loader = DataLoader(dataset=train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=4)\n    val_loader = DataLoader(dataset=val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=0)\n    \n    model = create_model()\n    model = model.cuda()\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=5, mode='max', factor=0.75, verbose=True, min_lr=1e-5)\n\n    for epoch in range(n_epochs):\n        torch.cuda.empty_cache()\n        gc.collect()\n\n        train_model(epoch, optimizer, scheduler=None, history=history)\n\n        loss, kaggle = evaluate_model(epoch, scheduler=scheduler, history=history2)\n\n        if kaggle > best:\n          best = kaggle\n          print(f'Saving best model... (qwk)')\n          torch.save(model.state_dict(), f'model-fld{fold+1}.pth')\n        \n    print()\n    validations.append(best)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validations = np.array(validations)\nfor i,val in enumerate(validations):\n    print(f'Fold {i+1}: {val}')\nprint(f'5fold CV: {np.mean(validations)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5fold Inference"},{"metadata":{"id":"fW80PAOSG2QG","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom tqdm import tqdm\nfrom timeit import default_timer as timer\nimport skimage.io\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data.dataset import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.sampler import *\n\nif True:\n    DATA_DIR = '/kaggle/input/prostate-cancer-grade-assessment/'\n    SUBMISSION_CSV_FILE = 'submission.csv'\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Use this to test inference\ntrain = pd.read_csv(f'{DATA_DIR}train.csv')[:1000]\n# submission = train\n\nsubmission = pd.read_csv(f'{DATA_DIR}sample_submission.csv')\n\nWIDTH = 512\nHEIGHT = 512\n\n#### net #########################################################################\n\ndef do_predict(net, inputs):\n    def logit_to_probability(logit):\n        probability=[]\n        for l in logit:\n            p = F.softmax(l)\n            probability.append(p)\n        return probability\n    \n    num_ensemble = len(net)\n    for i in range(num_ensemble):\n        net[i].eval()\n\n    probability=[0,0,0,0]\n    for i in range(num_ensemble):\n        logit = net[i](inputs)\n        prob = logit_to_probability(logit)\n        probability = [p+q for p,q in zip(probability,prob)]\n    \n    #----\n    probability = [p/num_ensemble for p in probability]\n    predict = [torch.argmax(p,-1) for p in probability]\n    predict = [p.data.cpu().numpy() for p in predict]\n    predict = np.array(predict).T\n    predict = predict.reshape(-1)\n\n    return predict\n\n## load net -----------------------------------\nnet = []\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld1.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld2.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld3.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld4.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\nmodel = create_model()\nmodel = model.cuda()\nstate = torch.load('model-fld5.pth') # .\nmodel.load_state_dict(state)\nnet.append(model)\n\n#------------------------------------------\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\n\nclass ImageDataset(Dataset):\n    def __init__(self, dataframe, root_dir, transform=None):\n        self.df = dataframe\n        self.root_dir = root_dir\n        self.transform = transform\n\n        self.paths = self.df.image_id.values\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_name = self.paths[idx]\n        file_path = f'{self.root_dir}{img_name}.tiff'\n        \n        image = skimage.io.MultiImage(file_path)\n        image = cv2.resize(image[-1], (WIDTH, HEIGHT), interpolation=cv2.INTER_AREA)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = (255 - image).astype(np.float32) / 255.\n        \n        if self.transform is not None:\n          image = self.transform(image=image)['image']\n        \n        image = np.rollaxis(image, -1, 0)\n        \n        return image\n#---------------------------------------------\n\ndef run_make_submission_csv():\n    target=[]\n    batch_size= 4\n\n    if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):\n    # Use below lines to test inference\n#     if True:\n#         test_dataset = ImageDataset(train, f'{DATA_DIR}train_images/', None)\n        test_dataset = ImageDataset(submission, f'{DATA_DIR}test_images/', None)\n        test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n        \n        t = tqdm(test_loader)\n        with torch.no_grad():\n            for b, image_batch in enumerate(t):\n                image_batch = image_batch.cuda().float()\n                predict = do_predict(net, image_batch)\n                target.append(predict)\n        print('')\n    #---------\n    else:\n        target = [[1],[1],[1]]\n    target = np.concatenate(target)\n\n    submission['isup_grade'] = target\n    submission['isup_grade'] = submission['isup_grade'].astype(int)\n    submission.to_csv(SUBMISSION_CSV_FILE, index=False)\n    print(submission.head())\n\nif __name__ == '__main__':\n    run_make_submission_csv()\n\n    print('\\nsucess!')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## And thats it! Thanks for reading and make sure to upvote if you found this kernal helpful!\n\nThings that you can experiment with:\n- Change amount of epochs\n- Change optimizer/scheduler\n- Add basic Augmentations (SSR, Cutout, etc.)\n- Add complex Augmentations (Mixup, Cutmix, etc.)\n- Change Backbone (Resnet, EfficientNet, etc.)\n- Change model head (add another linear layer, add batchnormalization, change dropout, etc.)\n- Change image size (256x256, 128x128, etc.)"}],"metadata":{"colab":{"name":"PANDA.ipynb","provenance":[],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":4}