{"cells":[{"metadata":{},"cell_type":"markdown","source":"**TRAINING KERNEL**\n\nThis is a solid starting point for any kaggler in this competition. \nThis base allows you to\n* understand how pytorch and computer vision works\n* be competitive in the competition\n* gain experience by trying easy to implement different customizations \n\nThere are a lot of knobs to tweek in order to personalize it:\n* a lot of augmentation techniques\n* choose image size\n* choose the desired pretrained model\n* customize the arhitecture by add more dense layers or any other layer types\n* change number of folds or the spliting\n\nThe inference kernel after you train the models is https://www.kaggle.com/vladvdv/pytorch-inference-multiple-models-and-folds\n\nFor the input data I use https://www.kaggle.com/dhananjay3/panda2 which is the Panda dataset images at level 2 uploaded by @Dhananjay Raut\n\nVersion 4 update: added LS (label smoothing)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null # no output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport cv2\nimport albumentations as A\nimport torch\nfrom skimage.transform import AffineTransform, warp\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport torch.nn.functional as F\nfrom torch.utils.data.dataloader import DataLoader\nimport torch.nn as nn\nimport pretrainedmodels\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.model_selection import StratifiedKFold\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define paths\n\nBASE_PATH=\"/kaggle/input/\"\nTRAIN_IMG_DIR = BASE_PATH+\"panda2/train_images/\"\nTRAIN_MASK_DIR = BASE_PATH+\"panda2/train_label_masks/\"\ntrain = pd.read_csv(BASE_PATH+\"/prostate-cancer-grade-assessment/train.csv\").set_index(\"image_id\")\ntest = pd.read_csv(BASE_PATH+\"/prostate-cancer-grade-assessment/test.csv\")\nsample_submission = pd.read_csv(BASE_PATH+\"/prostate-cancer-grade-assessment/sample_submission.csv\")\n\ntrain.head()\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dataset class used both for training and validation, difference consist in passing different transform objects\nclass PandaDataset:\n    def __init__(self, df, transform=None):\n        self.image_ids=df.index.values\n        self.isup_grade = df.isup_grade.values\n        self.transform=transform\n        self.df=df\n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, index):\n        image = cv2.imread(f\"{TRAIN_IMG_DIR}{self.image_ids[index]}.png\")\n        label = self.isup_grade[index]\n        image = (255 - image).astype(np.float32) / 255.\n        if self.transform:\n            image,_ = self.transform([image,label])\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n\n        return torch.tensor(image, dtype=torch.float), torch.tensor(self.isup_grade[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This cells are the implementation for multiple augmentation techniques\n#There are implemented:\n# * affine transforms (scale, rotation, shear, translation)\n# * flip (horizontal, vertical)\n# * resize\n# * blur (blur, median blur, gaussian blur, motion blur)\n# * noise (GaussNoise, MultiplicativeNoise)\n# * distort \n# * RandomBrightnessContrast\n# * cutout (CoarseDropout)\n#Not all techniques are tested, if there are problems with one of them let me know \n\ndef Aug_affine(img, prob):\n    if (np.random.uniform()>=prob):\n        return img\n    # --- scale ---\n    min_scale = 0.7\n    max_scale = 1.3\n    sx = np.random.uniform(min_scale, max_scale)\n    sy = np.random.uniform(min_scale, max_scale)\n\n    # --- rotation ---\n    max_rot_angle = 20\n    rot_angle = np.random.uniform(-max_rot_angle, max_rot_angle) * np.pi / 180.\n\n    # --- shear ---\n    max_shear_angle = 7\n    shear_angle = np.random.uniform(-max_shear_angle, max_shear_angle) * np.pi / 180.\n\n    # --- translation ---\n    max_translation = 20\n    tx = np.random.randint(-max_translation, max_translation)\n    ty = np.random.randint(-max_translation, max_translation)\n\n    tform = AffineTransform(scale=(sx, sy), rotation=rot_angle, shear=shear_angle,\n                            translation=(tx, ty))\n    transformed_image = warp(img, tform)\n\n    return transformed_image    \n\n\n\ndef Aug_flip(img,prob):\n    if (np.random.uniform()>=prob):\n        return img\n    r = np.random.uniform()\n    if r < 0.5:\n        img = apply_aug(A.HorizontalFlip(p=1), img)\n    else:\n         img = apply_aug(A.VerticalFlip(p=1), img)\n\n        \n    return img    \n    \ndef apply_aug(aug, image):\n    return aug(image=image)['image']\n\n\ndef Aug_resize(img,size):\n    return cv2.resize(img, size, interpolation=cv2.INTER_AREA )\n\n\ndef Aug_blur(img, prob):\n    if (np.random.uniform()>=prob):\n        return img\n    r = np.random.uniform()\n    if r < 0.25:\n        img = apply_aug(A.Blur(p=1), img)\n    elif r < 0.5:\n        img = apply_aug(A.MedianBlur(blur_limit=5, p=1), img)\n    elif r < 0.75:\n        img = apply_aug(A.GaussianBlur(p=1), img)\n    else:\n        img = apply_aug(A.MotionBlur(p=1), img)    \n        \n    return img\n\n\ndef Aug_noise(img, prob):\n    if (np.random.uniform()>=prob):\n        return img\n    r = np.random.uniform()\n    if r < 0.50:\n        img = apply_aug(A.GaussNoise(var_limit=5. / 255., p=1), img)\n    else:\n        img = apply_aug(A.MultiplicativeNoise(p=1), img)    \n    return img\n        \ndef Aug_distort(img, prob):\n    img = apply_aug(A.GridDistortion(p=prob), img)    \n    return img\n    \ndef Aug_brightness(img, prob):\n    img = apply_aug(A.RandomBrightnessContrast(p=prob), img)     \n    return img\n\ndef Aug_coarseDropout(img,prob):\n    img = apply_aug(A.CoarseDropout(max_holes=4, max_height=30, max_width=30, p=prob), img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#augmentation manager class\nclass Transform:\n    def __init__(self, size=(512, 512),\n                 normalize=True, train=True,\n                 blurProb=0, noiseProb=0, distortionProb=0, \n                 elasticDistortionProb=0., randomBrightnessProb=0,\n                 affineTransformProb=0, coarseDropout=0, flipProb=0):\n        self.size=size\n        self.normalize=normalize\n        self.train=train\n        self.blurProb=blurProb\n        self.noiseProb=noiseProb\n        self.distortionProb=distortionProb\n        self.randomBrightnessProb=randomBrightnessProb\n        self.affineTransformProb=affineTransformProb\n        self.coarseDropout=coarseDropout\n        self.flipProb=flipProb\n        \n    def __call__(self, example):\n        if self.train:\n            x, y = example\n        else:\n            x = example[0]\n\n        # --- Augmentation ---\n        x = Aug_affine(x.astype(np.float32), prob=self.affineTransformProb)\n  \n        x = Aug_resize(x.astype(np.float32), size=self.size)\n\n        x = Aug_blur(x.astype(np.float32), prob=self.blurProb)\n\n        x = Aug_flip(x.astype(np.float32),  prob=self.flipProb)\n        \n        x = Aug_noise(x.astype(np.float32), prob=self.noiseProb)\n        \n        x = Aug_distort(x.astype(np.float32), prob=self.distortionProb)\n        \n        x = Aug_brightness(x.astype(np.float32), prob=self.randomBrightnessProb)\n        \n        x = Aug_coarseDropout(x.astype(np.float32),prob=self.coarseDropout)\n\n        # normalizing\n        x = (x.astype(np.float32) - 0.0692) / 0.2051\n        \n        if self.train:\n            return x, y\n        else:\n            return x, None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#label smoothing\n\ndef onehot_encoding(label, n_classes):\n    return torch.zeros(label.size(0), n_classes).to(label.device).scatter_(\n        1, label.view(-1, 1), 1)\ndef cross_entropy_loss(input, target, reduction):\n    logp = F.log_softmax(input, dim=1)\n    loss = torch.sum(-logp * target, dim=1)\n    if reduction == 'none':\n        return loss\n    elif reduction == 'mean':\n        return loss.mean()\n    elif reduction == 'sum':\n        return loss.sum()\n    else:\n        raise ValueError(\n            '`reduction` must be one of \\'none\\', \\'mean\\', or \\'sum\\'.')\n        \ndef label_smoothing_criterion(epsilon=0.1, reduction='mean'):\n    def _label_smoothing_criterion(preds, targets):\n        n_classes = preds.size(1)\n        device = preds.device\n\n        onehot = onehot_encoding(targets, n_classes).float().to(device)\n        targets = onehot * (1 - epsilon) + torch.ones_like(onehot).to(\n            device) * epsilon / n_classes\n        loss = cross_entropy_loss(preds, targets, reduction)\n        if reduction == 'none':\n            return loss\n        elif reduction == 'mean':\n            return loss.mean()\n        elif reduction == 'sum':\n            return loss.sum()\n        else:\n            raise ValueError(\n                '`reduction` must be one of \\'none\\', \\'mean\\', or \\'sum\\'.')\n\n    return _label_smoothing_criterion","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# QWK metric function (competition metric)\ndef qwk3(a1, a2, max_rat):\n    assert(len(a1) == len(a2))\n    a1 = np.asarray(a1, dtype=int)\n    a2 = np.asarray(a2, dtype=int)\n\n    hist1 = np.zeros((max_rat + 1, ))\n    hist2 = np.zeros((max_rat + 1, ))\n\n    o = 0\n    for k in range(a1.shape[0]):\n        i, j = a1[k], a2[k]\n        hist1[i] += 1\n        hist2[j] += 1\n        o +=  (i - j) * (i - j)\n\n    e = 0\n    for i in range(max_rat + 1):\n        for j in range(max_rat + 1):\n            e += hist1[i] * hist2[j] * (i - j) * (i - j)\n\n    e = e / a1.shape[0]\n\n    return 1 - o / e","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class of the model arhitecture\n#you can play with this class a lot(change model arhitecture, add more linear layers, etc)\n\nclass ModelArhitecture(nn.Module):\n    def __init__(self,pretrainedModelArhitecture='se_resnet50', pretrainedModelWeights='imagenet'):\n        super(ModelArhitecture, self).__init__()\n        self.base_model = pretrainedmodels.__dict__[pretrainedModelArhitecture](pretrained=pretrainedModelWeights).to(device)\n        self.final1_1 = nn.Linear(in_features=524288, out_features=6, bias=True).to(device)\n                \n    def forward(self,x):\n        self.do_pooling=False\n        h=self.base_model.features(x)\n#        print (h.shape)\n        if self.do_pooling:\n            h = torch.sum(h, dim=(-1, -2))\n        else:\n            bs, ch, height, width = h.shape\n            h = h.view(bs, ch*height*width)\n\n        h1=self.final1_1(h)\n\n        \n        return h1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#class for the general algorithm (takes as input the model, trains and output prediction and statistics about training)\nclass PandaAlgorithm(nn.Module):\n    \n    def __init__(self, model, fold):        \n        super(PandaAlgorithm, self).__init__()\n        self.model = model\n        self.lossTrain=[]\n        self.lossVal=[]\n        self.qwkTrain=[]\n        self.qwkVal=[]\n        self.fold=fold\n        self.criterion=label_smoothing_criterion()\n        \n    def forward(self,x, optimizer,y=None, phase='train'):\n\n        inputs = x.to(device)\n        labels = y.to(device)\n\n        outputs =self.model(inputs)\n        loss = self.criterion(outputs, labels)                   \n\n        if phase == 'train':\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n        return [loss.data.cpu().numpy(), outputs, labels]\n    \n    def train(self,dataloaders,num_epochs=2):\n        self.dataloaders=dataloaders\n        optimizer = torch.optim.Adam(self.model.parameters(), lr=0.001)\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.8, patience=5, min_lr=1e-10)\n        BestQwkScore=0\n        for epoch in range(num_epochs):\n           \n            print('Epoch {}/{}'.format(epoch+1, num_epochs))\n            print('-' * 10)     \n            for phase in ['train', 'validation']:\n                epoch_metrics={'CEloss':0}\n                print (phase)\n                outputList=[]\n                labelList =[]\n                if phase == 'train':\n                    self.model.train()\n                else:\n                    self.model.eval()\n                for inputs, labels in tqdm(self.dataloaders[phase]):   \n                    MSELossPerBatch,outputsNP,labels2=self.forward(inputs,optimizer,labels,phase)\n                    \n                    for i in range(len(labels2)):\n                        labelList.append(labels2[i].item())\n                    for j in range(len(outputsNP)):\n                        outputList.append(torch.argmax(F.softmax(outputsNP[j])).item())\n                    epoch_metrics['CEloss']+=MSELossPerBatch\n\n\n                loss,qwkScore=self.TrainingStats(epoch_metrics,phase, outputList, labelList)\n                \n                if phase=='validation':\n                    scheduler.step(loss)\n                    print ('learning rate:',scheduler.optimizer.param_groups[0]['lr']) \n                    \n            if (qwkScore>BestQwkScore):\n                BestQwkScore=qwkScore\n                torch.save(self.model.state_dict(), 'model_fold'+str(self.fold)+\"_epoch\"+str(epoch)+'_Qwk'+str(BestQwkScore)+'_v3Beta.pth') \n                    \n                \n        return self.model\n    \n    def TrainingStats(self,epoch_metrics,phase, outputList, labels):\n\n        epoch_metrics['CEloss']=epoch_metrics['CEloss']/ len(self.dataloaders[phase])\n\n        labelsNP=np.array(labels)\n\n        outputsNP=np.array(outputList)\n        qwkScore=qwk3(labelsNP,outputsNP,6)\n\n        if phase == 'train':\n            self.lossTrain.append(epoch_metrics['CEloss'])\n            self.qwkTrain.append(qwkScore)\n#            uncomment plot functions if you are running on your local machine and want to plot loss and qwk for train and valid            \n#            plt.plot(self.lossTrain,color='blue')\n#            plt.show()\n\n#            plt.plot(self.qwkTrain,color='red')\n#            plt.show()\n            \n            \n        else:\n            \n            self.lossVal.append(epoch_metrics['CEloss'])  \n            self.qwkVal.append(qwkScore)\n                              \n#            plt.plot(self.lossVal,color='blue')\n#            plt.show()                    \n\n#            plt.plot(self.qwkVal,color='red')\n#            plt.show()            \n        print (epoch_metrics)\n        print (\"qwkScore: \",qwkScore)\n                           \n        \n        return epoch_metrics['CEloss'], qwkScore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nseed=41\nrandom.seed(seed)\nos.environ['PYTHONHASHSEED'] = str(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\n\n#here I defined 5 folds splits keeping the histogram of labels constant (StratifiedKFold)\n#change the batch_size, num_epochs according to you needs (my advice is to use minim 40 epochs)\nmskf = StratifiedKFold(n_splits=5, random_state=seed)\nmskf.get_n_splits(train, train.isup_grade.values)\n\n\nnum_epochs=1 #don't forget to change this (I suggest >50 epochs)\nfold=0\nfor train_index, test_index in mskf.split(train, train.isup_grade.values):\n    print(\"fold:\", fold)\n    print (\"Train data: \",len(train_index))\n    print (\"Valid data: \",len(test_index))\n\n    \n    \n    trainData = train.iloc[train_index]\n    validData = train.iloc[test_index]\n    \n    #here is where you change the augmentation probabilities and features\n    transformTrain=Transform(noiseProb=0.1, affineTransformProb=0.3, flipProb= 0.2)\n    #for validation we don't augment\n    transformValid=Transform(train=False)\n    \n    \n    \n    trainDataset = PandaDataset(trainData,transform=transformTrain)\n    validDataset = PandaDataset(validData,transform=transformValid)\n    out = trainDataset.__getitem__(100)\n    out\n    \n    \n    print('train_dataset', len(trainDataset), 'valid_dataset', len(validDataset))\n    \n    image_datasets={'train': trainDataset,\n                    'validation':validDataset}\n    batch_size=4\n    train_loader = DataLoader(trainDataset, batch_size=batch_size, shuffle=True)\n    valid_loader = DataLoader(validDataset, batch_size=batch_size, shuffle=False)\n    \n    \n    dataloaders = {\n        'train':train_loader,\n        'validation':valid_loader \n        }\n            \n    # select running device        \n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    \n     \n    fullModelArhitecture=ModelArhitecture()                \n    clf=PandaAlgorithm(model=fullModelArhitecture, fold=fold)\n    model=clf.train(dataloaders,num_epochs=num_epochs)\n    fold+=1\n    torch.cuda.empty_cache()\n    gc.collect()\n    del model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Good luck to everyone"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}