{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys\n# sys.path = ['../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master', ] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Basic Python and Machine learning libraries\nimport random, time, cv2\nimport pandas as pd\nimport numpy as np\nfrom matplotlib.colors import from_levels_and_colors\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom PIL import Image\nfrom scipy import stats\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom IPython.display import display\nfrom tqdm.notebook import tqdm\n\n#Pytorch and Albumentations(Data Augmentation Library)\nimport torch\nimport albumentations\nfrom albumentations.pytorch import ToTensorV2\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    DEBUG = False\n    pwd = '/kaggle/working/'\n    data_dir = '../input/panda-dataset-medium-256-64-64/'\n    train_img_dir = os.path.join(data_dir, 'train_images')\n    train_mask_dir = os.path.join(data_dir, 'train_label_masks')\n    test_img_dir = os.path.join(data_dir, 'test_images')\n    orig_masks_dir = '../input/prostate-cancer-grade-assessment/train_label_masks'\n    backbone = 'efficientnet-b0'\n#     SUM_PREDICTION = True\n    n_images_to_plot = 16\n    n_folds = 2 if DEBUG else 5\n    image_size = 64\n    tile_size = 64\n    n_tiles = 256\n    out_dim = 5\n    batch_size = 512\n    num_workers = 4\n    num_epochs = 2 if DEBUG else 4\n    lr = 1e-4\n    t_0 = 2\n    SEED = 713\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    #Image-net standard mean and std\n    mean = [0.485, 0.456, 0.406]\n    std = [0.229, 0.224, 0.225]\n    cmap, norm = from_levels_and_colors([1, 2, 3, 4, 5], ['black', 'gray', 'green', 'yellow', 'orange', 'red'], 'both')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(Config.device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(Config.SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(Config.data_dir+'train.csv')\n\nmasks = os.listdir(Config.orig_masks_dir)\nmasks_df = pd.Series(masks).to_frame()\nmasks_df.columns = ['mask_file_name']\nmasks_df['image_id'] = masks_df.mask_file_name.apply(lambda x: x.split('_')[0])\ntrain_df = pd.merge(train_df, masks_df, on='image_id', how='outer')\ndel masks_df\nprint(f\"There are {len(train_df[train_df.mask_file_name.isna()])} images without a mask.\")\n\n## removing items where image mask is null\ntrain_df = train_df[~train_df.mask_file_name.isna()]\n\nprint(len(train_df))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_df[train_df['gleason_score']=='0+0']['isup_grade']))\nprint(len(train_df[train_df['gleason_score']=='negative']['isup_grade']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# trunc_train_df = train_df.copy()\ntrain_df['gleason_score'] = train_df['gleason_score'].apply(lambda x: \"0+0\" if x==\"negative\" else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_df[train_df['gleason_score']=='0+0']['isup_grade']))\nprint(len(train_df[train_df['gleason_score']=='negative']['isup_grade']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not Config.DEBUG:\n    sample_to_drop = train_df[(train_df['isup_grade'] == 2) & (train_df['gleason_score'] == '4+3')].index\n    train_df.drop(sample_to_drop, inplace=True)\n    print(len(train_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(train_df[train_df.data_provider=='karolinska'].query('isup_grade != 0').index, inplace=True)\nprint(len(train_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.sample(6).reset_index(drop=True) if Config.DEBUG else train_df.reset_index(drop=True)\nprint(len(train_df))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDA_Dataset(Dataset):\n    def __init__(self,\n                 df,\n                 n_tiles,\n                 transform=None,\n                ):\n\n        self.df = df.reset_index(drop=True)\n        self.n_tiles = n_tiles\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0] * self.n_tiles\n\n    def __getitem__(self, index):\n        img_tile_path = os.path.join(Config.train_img_dir, self.df['image_id'].values[index // self.n_tiles]) + '_'\n        mask_tile_path = os.path.join(Config.train_mask_dir, self.df['image_id'].values[index // self.n_tiles]) + '_mask_'\n        \n        img_tile = Image.open(img_tile_path + str(index % self.n_tiles) + '.png')\n        img_tile = np.array(img_tile)\n        \n        mask_tile = Image.open(mask_tile_path + str(index % self.n_tiles) + '.png')\n        mask_tile = np.array(mask_tile)\n        \n        (values,counts) = np.unique(mask_tile, return_counts=True)\n        val_ind = np.argmax(counts)\n        label = values[val_ind]\n        if self.df['data_provider'].values[index // self.n_tiles] == 'radboud':\n            if label > 1:\n                label -= 1\n        img_tile = img_tile.astype(np.float32)\n#         print(masks.dtype)\n#         masks = masks.astype(np.float32)\n        img_tile /= 255\n#         masks = np.where((masks == 2), 1, np.where((masks > 2), 2, masks))\n        \n        if self.transform is not None:\n            transformed = self.transform(image=img_tile, mask=mask_tile)\n            img_tile = transformed['image']\n#             mask_tile = transformed['mask']\n            \n        img_tile = img_tile.transpose(2, 0, 1)\n        return torch.tensor(img_tile), torch.tensor(label, dtype=torch.long)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDA_Dataset_test(Dataset):\n    def __init__(self,\n                 df,\n                 n_tiles,\n                 transform=None,\n                ):\n\n        self.df = df.reset_index(drop=True)\n        self.n_tiles = n_tiles\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        img_tile_path = os.path.join(Config.train_img_dir, self.df['image_id'].values[index]) + '_'\n        mask_tile_path = os.path.join(Config.train_mask_dir, self.df['image_id'].values[index]) + '_mask_'\n        isup_label = self.df['isup_grade'].values[index]\n        \n        imgs = np.zeros((self.n_tiles, 64, 64, 3))\n        masks_label = np.zeros(self.n_tiles, dtype=np.uint8)\n        for i in range(self.n_tiles):\n            img_tile = Image.open(img_tile_path + str(i) + '.png')\n            img_tile = np.array(img_tile)\n\n            mask_tile = Image.open(mask_tile_path + str(i) + '.png')\n            mask_tile = np.array(mask_tile)\n            \n            (values,counts) = np.unique(mask_tile, return_counts=True)\n            val_ind = np.argmax(counts)\n            label = values[val_ind]\n            if self.df['data_provider'].values[index] == 'radboud':\n                if label > 1:\n                    label -= 1\n                \n            if self.transform is not None:\n                transformed = self.transform(image=img_tile)\n                img_tile = transformed['image']\n            imgs[i] = img_tile\n            masks_label[i] = label\n            \n        \n        imgs = imgs.astype(np.float32)\n        imgs /= 255\n        imgs = imgs.transpose(0, 3, 1, 2)\n        \n        return torch.tensor(imgs), torch.tensor(masks_label, dtype=torch.long), isup_label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.reset_index(drop=True, inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(Config.n_folds, shuffle=True, random_state=Config.SEED)\ntrain_df['fold'] = -1\nfor i, (tr_idx, val_idx) in enumerate(skf.split(train_df, train_df['isup_grade'])):\n    train_df.loc[val_idx, 'fold'] = i\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_data = PANDA_Dataset(trunc_train_df, Config.n_tiles, None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_model = {\n    'efficientnet-b0': '../input/model-4ep/efficientnet-b0_fold_1.pt'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BasicModel(nn.Module):\n    def __init__(self, backbone, out_dim=5):\n        super(BasicModel, self).__init__()\n        self.enet = EfficientNet.from_pretrained('efficientnet-b0', num_classes=out_dim)\n#         self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n    \n    def forward(self, x):\n        x = self.enet(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs\n\n# model = BasicModel(Config.backbone, Config.out_dim).to(Config.device)\n# print(f'The model has {count_parameters(model):,} trainable parameters')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, scheduler, epoch, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    iter_len = len(iterator)\n    for (data, target) in tqdm(iterator):\n        \n        data = data.to(device)\n        target = target.to(device)\n        optimizer.zero_grad()\n        y_pred = model(data)\n        loss = criterion(y_pred, target)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        \n    return epoch_loss/len(iterator)\n\n# ДОПИСАТь\ndef evaluate(model, iterator, criterion, device):\n    \n    epoch_loss  = 0\n    preds_list = []\n    targets_list = []\n    model.eval()\n    \n    with torch.no_grad():\n        \n        for (data, target, isup_target) in tqdm(iterator):\n            data = data.to(device).squeeze()\n            target = target.to(device).squeeze()\n            logits = model(data)\n            loss = criterion(logits, target)\n            \n            pred = torch.argmax(logits, dim=1)\n            pred = pred.cpu().numpy() #pred - 256х1 np array\n            # По pred вычисление isup -- > pred\n            values = np.unique(pred)\n            result = sorted(values, key = list(values).count, reverse = True)\n            result = result[:2]\n            \n            isup_pred = 0\n#             запарсить гавно\n            if len(result) == 1:\n                result.append(result[0])\n            \n            if result[0] == 0 or result[0] == 1:\n                isup_pred = 0\n            \n            if result[0] == 2:\n                if result[1] == 0 or result[1] == 1 or result[1] == 2:\n                    isup_pred = 1\n                elif result[1] == 3:\n                    isup_pred = 2\n                elif result[1] == 4:\n                    isup_pred = 4\n                    \n            if result[0] == 3:\n                if result[1] == 0 or result[1] == 1 or result[1] == 3:\n                    isup_pred = 4\n                elif result[1] == 2:\n                    isup_pred = 3\n                elif result[1] == 4:\n                    isup_pred = 5\n            \n            if result[0] == 4:\n                if result[1] == 2:\n                    isup_pred = 4\n                else:\n                    isup_pred = 5\n                \n            preds_list.append(isup_pred)\n            targets_list.append(isup_target)\n\n            loss = loss.detach().cpu().numpy()\n\n            epoch_loss += loss\n\n    metric = metrics.cohen_kappa_score(preds_list, targets_list, weights='quadratic')\n    \n    return epoch_loss/len(iterator), metric\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, scheduler, loss_criterion, device, n_epochs, fold):\n    \"\"\" Fits a dataset to model\"\"\"\n    #Setting best validation loss to infinity :p\n    best_valid_metric = -1.\n    \n    train_losses = []\n    valid_losses = []\n    valid_metric_scores = []\n    \n    #Let's loop through our data\n    for epoch in range(n_epochs):\n    \n        start_time = time.time()\n        \n        print(f'Epoch: {epoch+1:02} | Training:')\n        train_loss = train(model, train_iterator, optimizer, scheduler, epoch, loss_criterion, device)\n        print(f'Epoch: {epoch+1:02} | Validating:')\n        valid_loss, valid_metric_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        scheduler.step()\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_metric_scores.append(valid_metric_score)\n\n        #Let's keep updating our model, so that we save only the best one at the end\n        if valid_metric_score > best_valid_metric:\n            best_valid_metric = valid_metric_score\n            torch.save(model.state_dict(), f'{model_name}_fold_{fold}.pt')\n    \n        end_time = time.time()\n\n        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n        \n        #Printing and returning some important statistics\n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'\\tTrain Loss: {train_loss:.3f}')\n        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Metric Score: {valid_metric_score:.3f}')\n        \n    return pd.DataFrame({f'{model_name}_fold_{fold}_Training_Loss':train_losses,  \n                        f'{model_name}_fold_{fold}_Validation_Loss':valid_losses, \n                        f'{model_name}_fold_{fold}_Valid_Metric_Score':valid_metric_scores})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#This will simply plot the training statistics we returned\ndef plot_training_statistics(train_stats, model_name, fold):\n    \n    fig, axes = plt.subplots(2, figsize=(15,15))\n    axes[0].plot(train_stats[f'{model_name}_fold_{fold}_Training_Loss'], label=f'{model_name}_fold_{fold}_Training_Loss')\n    axes[0].plot(train_stats[f'{model_name}_fold_{fold}_Validation_Loss'], label=f'{model_name}_fold_{fold}_Validation_Loss')\n    axes[1].plot(train_stats[f'{model_name}_fold_{fold}_Valid_Metric_Score'], label=f'{model_name}_fold_{fold}_Valid_Metric_Score')\n    \n    axes[0].set_xlabel(\"Number of Epochs\"), axes[0].set_ylabel(\"Loss\")\n    axes[1].set_xlabel(\"Number of Epochs\"), axes[1].set_ylabel(\"Score on Metric\")\n    \n    axes[0].legend(), axes[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True)\n])\ntest_transforms = albumentations.Compose([\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fold = 1\nprint(f\"Fitting on Fold {fold+1}\")\n#Make Train and Valid DataFrame from fold\ntrain_df_fold = train_df[train_df['fold'] != fold]\nvalid_df_fold = train_df[train_df['fold'] == fold]\n    \n#Build and load Dataset\ntrain_data = PANDA_Dataset(train_df_fold, Config.n_tiles, train_transforms)\nvalid_data = PANDA_Dataset_test(valid_df_fold, Config.n_tiles, test_transforms)\ntrain_iterator = DataLoader(train_data, shuffle=True, batch_size=Config.batch_size, num_workers=Config.num_workers)\nvalid_iterator = DataLoader(valid_data, batch_size=1, num_workers=Config.num_workers)\n    \n#Initialize model, loss and optimizer\nmodel = BasicModel(Config.backbone, Config.out_dim).to(Config.device)\n# model.load_state_dict(torch.load(pretrained_model[Config.backbone], map_location=Config.device))\nloss_criterion = nn.CrossEntropyLoss().to(Config.device)\noptimizer=optim.Adam(model.parameters(), lr=Config.lr)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, Config.num_epochs)\n    \n#Fit the model and visualize the training curves\ntrain_stats = fit_model(model, 'efficientnet-b0', train_iterator, valid_iterator, \n                    optimizer, scheduler, loss_criterion, Config.device, Config.num_epochs, fold)\nplot_training_statistics(train_stats, 'efficientnet-b0', fold)\n    \n#Just making sure that the output looks neat\nprint('\\n')\nprint('-------------------------------------------------------')\nprint('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('hello hell!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}