{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os, sys\nsys.path = ['../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master', ] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Basic Python and Machine learning libraries\nimport random, time, cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom PIL import Image\nfrom scipy import stats\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils import class_weight\nfrom IPython.display import display\nfrom tqdm.notebook import tqdm\n\n#Pytorch and Albumentations(Data Augmentation Library)\nimport torch\nimport albumentations\nfrom albumentations.pytorch import ToTensorV2\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    \n    DEBUG = False\n    # change for continue training model\n    CONTINUE_TRAIN = True\n    # last epoch for model\n    last_epoch = 10 if CONTINUE_TRAIN else 0\n    # last lr for model\n    last_lr = 2e-5\n    pwd = '/kaggle/working/'\n    #change to choose coresponding dataset. VALID: 16, 36\n    n_tiles = 36\n    assert n_tiles == 16 or n_tiles == 36\n    data_dir = '../input/panda-dataset-medium-16-256-256/' if n_tiles == 16 else '../input/panda-dataset-medium-36-256-256/'\n    train_img_dir = os.path.join(data_dir, 'train_images')\n    test_img_dir = os.path.join(data_dir, 'test_images')\n    backbone = 'efficientnet-b1'\n    #Add to config understandable definition of target size for BCE or CCE\n    SUM_PREDICTION = False\n    out_dim = 5 if SUM_PREDICTION else 6\n    n_images_to_plot = 16\n    n_folds = 2 if DEBUG else 5\n    image_size = 256\n    batch_size = 2\n    num_workers = 4\n    num_epochs = 2 if DEBUG else 9\n    lr = 3e-4 if not CONTINUE_TRAIN else last_lr\n    SEED = 2020\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    #Image-net standard mean and std\n    mean = [0.485, 0.456, 0.406]\n    std = [0.229, 0.224, 0.225]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Config.device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(Config.SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(Config.data_dir+'train.csv')\ntrain_df = train_df.sample(100).reset_index(drop=True) if Config.DEBUG else train_df\ndisplay(train_df.head())\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not Config.DEBUG:\n    sample_to_drop = train_df[(train_df['isup_grade'] == 2) & (train_df['gleason_score'] == '4+3')].index\n    train_df.drop(sample_to_drop, inplace=True)\n    train_df.reset_index(inplace=True)\n    print(len(train_df))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDA_Dataset(Dataset):\n    def __init__(self,\n                 df,\n                 image_size,\n                 n_tiles,\n                 rand=False,\n                 tile_transform=None,\n                 img_transform=None\n                ):\n\n        self.df = df.reset_index(drop=True)\n        self.image_size = image_size\n        self.n_tiles = n_tiles\n        self.rand = rand\n        self.tile_transform = tile_transform\n        self.img_transform = img_transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        tiles_path = os.path.join(Config.train_img_dir, self.df['image_id'].values[index]) + '_'\n        if Config.SUM_PREDICTION:\n            label = np.zeros(5).astype(np.float32)\n            label[:self.df['isup_grade'].values[index]] = 1.\n        else:\n            label = self.df['isup_grade'].values[index]\n        \n        if self.rand:\n            idxes = np.random.choice(list(range(self.n_tiles)), self.n_tiles, replace=False)\n        else:\n            idxes = list(range(self.n_tiles))\n            \n        n_row_tiles = int(np.sqrt(self.n_tiles))\n        images = np.zeros((self.image_size * n_row_tiles, self.image_size * n_row_tiles, 3))\n        for h in range(n_row_tiles):\n            for w in range(n_row_tiles):\n                i = h * n_row_tiles + w\n                tile_i = Image.open(tiles_path + str(i) + '.png')\n                tile_i = np.array(tile_i)\n                # TRANSFORM FIX\n                if self.tile_transform is not None:\n                    tile_i = self.tile_transform(image=tile_i)['image']\n                h1 = h * self.image_size\n                w1 = w * self.image_size\n                images[h1:h1+self.image_size, w1:w1+self.image_size] = tile_i\n        \n        if self.img_transform is not None:\n            images = images.astype(np.float32)\n            images = self.img_transform(image=images)['image']\n        else:\n            images = images.astype(np.float32)\n            images /= 255\n            \n        images = images.transpose(2, 0, 1)\n        \n        return torch.tensor(images), torch.tensor(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The below code will plot down some images for you, given a list of images\ndef plot_images(images):\n\n    n_images = len(images)\n\n    rows = int(np.sqrt(n_images))\n    cols = int(np.sqrt(n_images))\n\n    fig = plt.figure(figsize=(20, 20))\n    for i in range(rows*cols):\n        ax = fig.add_subplot(rows, cols, i+1)\n        ax.set_title('ISUP: '+str(images[i][1]))\n        ax.imshow(images[i][0].transpose(0, 1).transpose(1,2).squeeze())\n        ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = PANDA_Dataset(train_df, Config.image_size, Config.n_tiles, False, None)\nimages = [(image, label) for image, label in [train_data[i] for i in range(Config.n_images_to_plot)]] \nplot_images(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(Config.n_folds, shuffle=True, random_state=Config.SEED)\ntrain_df['fold'] = -1\nfor i, (tr_idx, val_idx) in enumerate(skf.split(train_df, train_df['isup_grade'])):\n    train_df.loc[val_idx, 'fold'] = i\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['data_provider', 'gleason_score'], inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tile_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n])\n\ntrain_img_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True)\n])\n\ntest_tile_transforms = None\n\ntest_img_transforms = albumentations.Compose([\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n    'efficientnet-b1': '../input/efnetb1fold2epoch10/efficientnet-b1_fold_2_epoch_10.pt' if Config.CONTINUE_TRAIN else '../input/efficientnet-pytorch/efficientnet-b1-dbc7070a.pth'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BasicModel(nn.Module):\n    def __init__(self, backbone, continue_training=False, out_dim=6):\n        super(BasicModel, self).__init__()\n        self.enet = EfficientNet.from_name(backbone)\n        if not continue_training:\n            self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n        \n        self.fc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n    \n    def forward(self, x):\n        x = self.enet(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # FOR EXTRACTING IMAGE SIZE FROM ENETS\n# EfficientNet.get_image_size('efficientnet-b0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs\n\n# model = BasicModel(Config.backbone, Config.out_dim).to(Config.device)\n# print(f'The model has {count_parameters(model):,} trainable parameters')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Training Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    \n    for (data, target) in tqdm(iterator):\n        \n        data = data.to(device)\n        target = target.to(device)\n        \n        optimizer.zero_grad()\n        y_pred = model(data)\n        loss = criterion(y_pred, target)\n        \n        loss.backward()\n        optimizer.step()\n        \n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        \n    return epoch_loss/len(iterator)\n\ndef evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    preds_list = []\n    targets_list = []\n    model.eval()\n    \n    with torch.no_grad():\n        \n        for (data, target) in tqdm(iterator):\n        \n            data = data.to(device)\n            target = target.to(device)\n\n            logits = model(data)\n            loss = criterion(logits, target)\n            pred = torch.argmax(logits, dim=1)\n            \n            preds_list.append(pred)\n            targets_list.append(target)\n\n            loss_np = loss.detach().cpu().numpy()\n            epoch_loss += loss_np\n\n    preds_list = torch.cat(preds_list).cpu().numpy()\n    targets_list = torch.cat(targets_list).cpu().numpy()\n    \n    metric = metrics.cohen_kappa_score(preds_list, targets_list, weights='quadratic')\n    \n    return epoch_loss/len(iterator), metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, scheduler, loss_criterion, device, n_epochs, last_epoch, fold):\n    \"\"\" Fits a dataset to model\"\"\"\n    #Setting best validation loss to infinity :p\n    best_valid_metric = -1.\n    \n    train_losses = []\n    valid_losses = []\n    valid_metric_scores = []\n    \n    #Let's loop through our data\n    for epoch in range(n_epochs):\n    \n        start_time = time.time()\n        \n        print(f'Epoch: {epoch+last_epoch+1:02} | Training:')\n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        print(f'Epoch: {epoch+last_epoch+1:02} | Validating:')\n        valid_loss, valid_metric_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        scheduler.step()\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_metric_scores.append(valid_metric_score)\n\n        #Let's keep updating our model, so that we save only the best one at the end\n        if valid_metric_score > best_valid_metric:\n            print('Score Increased ({:.6f} --> {:.6f}).  Saving model ...'.format(best_valid_metric, valid_metric_score))\n            best_valid_metric = valid_metric_score\n            torch.save(model.state_dict(), f'{model_name}_fold_{fold}_epoch_{epoch+last_epoch+1}.pt')\n    \n        end_time = time.time()\n\n        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n        \n        content = f'Epoch: {epoch+last_epoch+1:02}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}  | Epoch Time: {epoch_mins}m {epoch_secs}s' + '\\n' + f'\\tTrain Loss: {train_loss:.3f}' + '\\n' + f'\\t Val. Loss: {valid_loss:.3f} |  Val. Metric Score: {valid_metric_score:.3f}'\n        with open(f'training_log.txt', 'a') as appender:\n            appender.write(content + '\\n\\n')\n            \n        #Printing and returning some important statistics\n        print(content)\n        \n    return pd.DataFrame({f'{model_name}_fold_{fold}_Training_Loss':train_losses,  \n                        f'{model_name}_fold_{fold}_Validation_Loss':valid_losses, \n                        f'{model_name}_fold_{fold}_Valid_Metric_Score':valid_metric_scores})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This will simply plot the training statistics we returned\ndef plot_training_statistics(train_stats, model_name, fold):\n    \n    fig, axes = plt.subplots(2, figsize=(15,15))\n    axes[0].plot(train_stats[f'{model_name}_fold_{fold}_Training_Loss'], label=f'{model_name}_fold_{fold}_Training_Loss')\n    axes[0].plot(train_stats[f'{model_name}_fold_{fold}_Validation_Loss'], label=f'{model_name}_fold_{fold}_Validation_Loss')\n    axes[1].plot(train_stats[f'{model_name}_fold_{fold}_Valid_Metric_Score'], label=f'{model_name}_fold_{fold}_Valid_Metric_Score')\n    \n    axes[0].set_xlabel(\"Number of Epochs\"), axes[0].set_ylabel(\"Loss\")\n    axes[1].set_xlabel(\"Number of Epochs\"), axes[1].set_ylabel(\"Score on Metric\")\n    \n    axes[0].legend(), axes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training with K-Fold CV"},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 2\n#Make Train and Valid DataFrame from fold\ntrain_df_fold = train_df[train_df['fold'] != fold]\nvalid_df_fold = train_df[train_df['fold'] == fold]\n\n#compute Clas weights\nclass_weights = class_weight.compute_class_weight('balanced', np.unique(train_df_fold['isup_grade']), train_df_fold['isup_grade'])\nprint(class_weights)\nclass_weights = torch.Tensor(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Fitting on Fold {fold+1}\")\n#Build and load Dataset\ntrain_data = PANDA_Dataset(train_df_fold, Config.image_size, Config.n_tiles, False, train_tile_transforms, train_img_transforms)\nvalid_data = PANDA_Dataset(valid_df_fold, Config.image_size, Config.n_tiles, False, test_tile_transforms, test_img_transforms)\ntrain_iterator = DataLoader(train_data, shuffle=True, batch_size=Config.batch_size, num_workers=Config.num_workers)\nvalid_iterator = DataLoader(valid_data, batch_size=Config.batch_size, num_workers=Config.num_workers)\n    \n#Initialize model, loss and optimizer\nmodel = BasicModel(Config.backbone, Config.CONTINUE_TRAIN, Config.out_dim).to(Config.device)\nif Config.CONTINUE_TRAIN:\n    model.load_state_dict(torch.load(pretrained_model[Config.backbone], map_location=Config.device))\nloss_criterion = None\nif Config.SUM_PREDICTION:\n    loss_criterion = nn.BCEWithLogitsLoss().to(Config.device)\nelse:\n    loss_criterion = nn.CrossEntropyLoss(weight=class_weights).to(Config.device)\n\noptimizer=optim.Adam(model.parameters(), lr=Config.lr)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, Config.num_epochs)\n    \n#Fit the model and visualize the training curves\ntrain_stats = fit_model(model, 'efficientnet-b1', train_iterator, valid_iterator, \n                    optimizer, scheduler, loss_criterion, Config.device, Config.num_epochs, Config.last_epoch, fold)\nplot_training_statistics(train_stats, 'efficientnet-b1', fold)\n    \n#Just making sure that the output looks neat\nprint('\\n')\nprint('-------------------------------------------------------')\nprint('\\n')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}