{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":18647,"databundleVersionId":1126921,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# I Just wanted my plots to look neat hence fixing some plot styles\nplt.style.use('seaborn-whitegrid')\nplt.rcParams['lines.linewidth'] = 2\nplt.rcParams['font.sans-serif'] = 'Arial'\nplt.rcParams['text.color'] = 'black'\nplt.rcParams['axes.labelcolor']= 'black'\nplt.rcParams['xtick.color'] = 'black'\nplt.rcParams['ytick.color'] = 'black'\nplt.rcParams['font.size'] = 12","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    \n    DEBUG = False\n    tqdm_bar = False\n    pwd = '/kaggle/working/'\n    data_dir = '../input/prostate-cancer-grade-assessment/'\n    train_img_dir = os.path.join(data_dir, 'train_images')\n    test_img_dir = os.path.join(data_dir, 'test_images')\n    num_images_to_plot = 16\n    num_folds = 2 if DEBUG else 5\n    height = 512\n    width = 512\n    out_dim = 5\n    batch_size = 16\n    num_workers = 4\n    num_epochs = 2 if DEBUG else 10\n    num_warmup_epochs = 1\n    warmup_factor = 10\n    SEED = 713\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n    #Image-net standard mean and std\n    mean = [0, 0, 0]\n    std = [255, 255, 255]\n    pretrainied_models = {\n        'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'\n    }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fixing seed so that our results are always reproducible\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(Config.SEED)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(Config.data_dir+'train.csv')\ntrain_df = train_df.sample(100).reset_index(drop=True) if Config.DEBUG else train_df\ndisplay(train_df.head())\nlen(train_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile(img, sz=128, N=16, tile_transforms=None):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                 constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    img = img[idxs]\n    if tile_transforms is not None:\n        for i in range(N):\n            augmented = tile_transforms(image=img[i])\n            img[i] = augmented[\"image\"]\n            \n    return img","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Build_Dataset(Dataset):\n    '''Builds Dataset to be fed to Neural Network\n       :param df: train_df or test_df\n       :param resize: tuple, eg(256, 256)\n       :param mode: string train or test \n       :param: augmentations: Image augmentations\n    '''\n    #Here I am defining my constructor\n    def __init__(self, df, resize=None, mode='train', augmentations=None, tile_transforms=None):\n        self.df = df\n        self.resize = resize\n        self.mode = mode\n        self.augmentations = augmentations\n        self.tile_transforms = tile_transforms\n      \n    # This method returns the total length of the dataset\n    def __len__(self):\n        return len(self.df)\n    \n    # This method returns (image, label) at index idx\n    def __getitem__(self, idx):\n        if self.mode == 'train':\n            img_path = os.path.join(Config.train_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)[1]\n            image = tile(image, sz=128, N=16, tile_transforms=self.tile_transforms)\n            isup_grade = self.df['isup_grade'].values[idx]\n            \n        if self.mode == 'test':\n            img_path = os.path.join(Config.test_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)[1]\n            image = tile(image, sz=256, N=16, tile_transforms=self.tile_transforms)\n            isup_grade = -1\n            \n        images = []\n        for i in range(4):\n            images.append(cv2.vconcat([image[i*4+j] for j in range(4)]))\n                \n        image = cv2.hconcat(images)\n        label = np.zeros(5, dtype=np.float32)\n        label[:isup_grade] = 1\n        if self.resize is not None:\n            image = cv2.resize(image, (self.resize[0], self.resize[1]))\n            \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = np.array(image)\n        image = 255-image\n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            image = augmented[\"image\"]\n            \n        return image, label\n            ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The below code will plot down some images for you, given a list of images\ndef plot_images(images):\n\n    n_images = len(images)\n\n    rows = int(np.sqrt(n_images))\n    cols = int(np.sqrt(n_images))\n\n    fig = plt.figure(figsize=(20,10))\n    for i in range(rows*cols):\n        ax = fig.add_subplot(rows, cols, i+1)\n        ax.set_title('ISUP: '+str(images[i][1]))\n        ax.imshow(np.array(images[i][0]))\n        ax.axis('off')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tile_transforms = train_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n])\ntrain_data = Build_Dataset(train_df, mode='train', tile_transforms=tile_transforms)\nimages = [(image, label) for image, label in [train_data[i] for i in range(Config.num_images_to_plot)]] \nplot_images(images)\n# train_data[0][0].shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(train_df.isup_grade)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(Config.num_folds, shuffle=True, random_state=Config.SEED)\ntrain_df['fold'] = -1\nfor i, (tr_idx, val_idx) in enumerate(skf.split(train_df, train_df['isup_grade'])):\n    train_df.loc[val_idx, 'fold'] = i\ntrain_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop(columns=['data_provider', 'gleason_score'], inplace=True)\ntrain_df.head()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Defining train and test transforms\ntrain_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True),\n    albumentations.pytorch.ToTensorV2(),\n])\ntest_transforms = albumentations.Compose([\n    albumentations.Normalize(mean=Config.mean, std=Config.std, always_apply=True),\n    ToTensorV2(),\n])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class enetv2(nn.Module):\n    def __init__(self, backbone, out_dim):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(Config.pretrainied_models[backbone]))\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n    \n    def extract(self, x):\n        return self.enet(x)\n    \n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#These are just 2 helpful functions I like using\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs\n\nmodel = enetv2('efficientnet-b0', Config.out_dim).to(Config.device)\nprint(f'The model has {count_parameters(model):,} trainable parameters')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    bar = tqdm(iterator) if Config.tqdm_bar else iterator\n    \n    for (x, y) in bar:\n        \n        x = x.to(device, dtype=torch.float)\n        y = y.to(device, dtype=torch.float)\n        optimizer.zero_grad()\n        y_pred = model(x)\n        loss = criterion(y_pred, y)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        \n        if Config.tqdm_bar:\n            bar.set_description('Training loss: %.5f' % (loss_np))\n        \n    return epoch_loss/len(iterator)\n\ndef evaluate(model, iterator, scheduler, criterion, device):\n\n    model.eval()\n    val_loss = []\n    LOGITS = []\n    PREDS = []\n    TARGETS = []\n    bar = tqdm(iterator) if Config.tqdm_bar else iterator\n\n    with torch.no_grad():\n        for (x, y) in bar:\n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.float)\n            logits = model(x)\n\n            loss = criterion(logits, y)\n\n            pred = logits.sigmoid().sum(1).detach().round()\n            LOGITS.append(logits)\n            PREDS.append(pred)\n            TARGETS.append(y.sum(1))\n\n            val_loss.append(loss.detach().cpu().numpy())\n            if Config.tqdm_bar:\n                bar.set_description('Validation loss: %.5f' % (loss))\n                \n        val_loss = np.mean(val_loss)\n\n    LOGITS = torch.cat(LOGITS).cpu().numpy()\n    PREDS = torch.cat(PREDS).cpu().numpy()\n    TARGETS = torch.cat(TARGETS).cpu().numpy()\n    \n    qwk = metrics.cohen_kappa_score(PREDS, TARGETS, weights='quadratic')\n    scheduler.step(val_loss)\n#     qwk_k = cohen_kappa_score(PREDS[df_valid['data_provider'] == 'karolinska'], df_valid[df_valid['data_provider'] == 'karolinska'].isup_grade.values, weights='quadratic')\n#     qwk_r = cohen_kappa_score(PREDS[df_valid['data_provider'] == 'radboud'], df_valid[df_valid['data_provider'] == 'radboud'].isup_grade.values, weights='quadratic')\n#     print('qwk', qwk, 'qwk_k', qwk_k, 'qwk_r', qwk_r)\n\n    return val_loss, qwk","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, scheduler, loss_criterion, device, num_epochs, fold):\n    \"\"\" Fits a dataset to model\"\"\"\n    #Setting best validation loss to infinity :p\n    best_valid_metric_score = -1*float('inf')\n    \n    train_losses = []\n    valid_losses = []\n    valid_metric_scores = []\n    \n    #Let's loop through our data\n    for epoch in range(num_epochs):\n    \n        start_time = time.time()\n    \n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        valid_loss, valid_metric_score = evaluate(model, valid_iterator, scheduler, loss_criterion, device)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_metric_scores.append(valid_metric_score)\n\n        #Let's keep updating our model, so that we save only the best one at the end\n        if valid_metric_score > best_valid_metric_score:\n            best_valid_metric_score = valid_metric_score\n            torch.save(model.state_dict(), f'{model_name}_fold_{fold}.pt')\n    \n        end_time = time.time()\n\n        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n        \n        #Printing and returning some important statistics\n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'\\tTrain Loss: {train_loss:.3f}')\n        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Metric Score: {valid_metric_score:.3f}')\n        \n    return pd.DataFrame({f'{model_name}_fold_{fold}_Training_Loss':train_losses,  \n                        f'{model_name}_fold_{fold}_Validation_Loss':valid_losses, \n                        f'{model_name}_fold_{fold}_Valid_Metric_Score':valid_metric_scores})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"complete_statistics = pd.DataFrame()\n\nfor fold in range(1):\n    print(f\"Fitting on Fold {fold}\")\n    #Make Train and Valid DataFrame from fold\n    train_df_fold = train_df[train_df['fold'] != fold]\n    valid_df_fold = train_df[train_df['fold'] == fold]\n    \n    #Build and load Dataset\n    train_data = Build_Dataset(train_df_fold, mode='train', augmentations=train_transforms)\n    valid_data = Build_Dataset(valid_df_fold, mode='train', augmentations=test_transforms)\n    train_iterator = DataLoader(train_data, shuffle=True, batch_size=Config.batch_size, num_workers=Config.num_workers)\n    valid_iterator = DataLoader(valid_data, batch_size=Config.batch_size, num_workers=Config.num_workers)\n    \n    #Initialize model, loss and optimizer\n    model = enetv2('efficientnet-b0', out_dim=Config.out_dim).to(Config.device)\n    loss_criterion = nn.BCEWithLogitsLoss().to(Config.device)\n    optimizer=optim.Adam(model.parameters(), lr=1e-4/Config.warmup_factor, betas=(0.9, 0.999))\n    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, Config.num_epochs-Config.num_warmup_epochs)\n    scheduler = GradualWarmupScheduler(optimizer, multiplier=Config.warmup_factor, total_epoch=Config.num_warmup_epochs, after_scheduler=scheduler_cosine)\n    \n    #Fit the model and visualize the training curves\n    train_stats = fit_model(model, 'efficientnet-b0', train_iterator, valid_iterator, optimizer,\n                            scheduler, loss_criterion, Config.device, Config.num_epochs, fold)\n    complete_statistics = pd.concat([complete_statistics, train_stats], axis=1)\n    \n    #Just making sure that the output looks neat\n    print('\\n')\n    print('-------------------------------------------------------')\n    print('\\n')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"complete_statistics.head(Config.num_epochs)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_fold_models = [Config.pwd+model for model in os.listdir(Config.pwd) if 'efficientnet-b0' in model]\nk_fold_models","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This bit of code is gonna fetch predictions for a single model\ndef get_predictions(model, iterator, device):\n    \n    preds = []\n    preds = np.array(preds)\n    model.eval()\n    \n    with torch.no_grad():\n        \n        for (x, y) in iterator:\n        \n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.long)\n            y_pred = model(x)\n            preds = np.append(preds, np.argmax(y_pred.detach().cpu().numpy(), axis = 1))\n            \n    return preds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(Config.data_dir+'test.csv')\nsample = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')\ntest_df.drop(columns=['data_provider'], inplace=True)\n\ndef submit(sample):\n    if os.path.exists(Config.test_img_dir):\n        test_data = Build_Dataset(test_df, resize=(Config.height, Config.width), \n                                  mode='test', augmentations=test_transforms)\n        test_iterator = DataLoader(test_data, batch_size=Config.batch_size, num_workers=Config.num_workers)\n        \n        #Lets Ensemble all of our models and predict the most frequent(ie Voting ensemble)\n        final_preds = []\n        for fold in range(Config.num_folds):\n            model = enetv2('efficientnet-b0', out_dim=Config.out_dim).to(Config.device)\n            model.load_state_dict(torch.load(k_fold_models[fold], map_location=lambda storage, loc: storage))\n            model.to(Config.device)\n            preds = get_predictions(model, test_iterator, Config.device)\n            final_preds.append(preds)\n            \n        final_preds = np.array(final_preds)\n        final_preds = stats.mode(final_preds)\n        sample['isup_grade'] = final_preds[0][0]\n    return sample\n\nsubmission = submit(sample)\nsubmission['isup_grade'] = submission['isup_grade'].astype(int)\nsubmission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}