{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport torch\nimport pytorch_lightning as pl\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks import Timer\nfrom torch import nn\nimport os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torchvision\nimport torch.nn.functional as F\nimport timm\nfrom timm.models.layers import LayerNorm2d, BatchNormAct2d, get_norm_act_layer\nimport timm.models.convnext as ConvN\nimport glob\nfrom pathlib import Path\nfrom torchvision import transforms\nimport openslide\nfrom tqdm.notebook import tqdm\nimport pickle\n\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nprint(f'Using {device} for inference')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-05T10:23:49.525101Z","iopub.execute_input":"2023-06-05T10:23:49.525714Z","iopub.status.idle":"2023-06-05T10:24:04.339046Z","shell.execute_reply.started":"2023-06-05T10:23:49.525676Z","shell.execute_reply":"2023-06-05T10:24:04.338024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data module for individual tiles","metadata":{}},{"cell_type":"code","source":"def RandomGroupKFold_split(groups, n,df, seed=None): \n    #modified from https://stackoverflow.com/a/54254277/11808186\n    # now taken from https://stats.stackexchange.com/questions/418254/k-fold-cv-scheme-stratifying-response-and-considering-groups and adapted to work with varying size of groups\n    \"\"\"\n    Takes a dataframe with groups (from groups) and places them randomly in n folds, fills these with individual samples without groups.\n    Random analogous of sklearn.model_selection.GroupKFold.split.\n\n    :return: list of (train, test) indices based on df index\n    \"\"\"\n    groups = pd.Series(groups)\n    remaining_samples = df['image_id'][df['group_id'].isnull()]\n    \n    n_samples = len(groups)+len(remaining_samples)\n    ix = df.index.copy()\n    unique = np.unique(groups)\n    np.random.RandomState(seed).shuffle(unique) # shuffling group ids so that subarrays are rando\n    result = []\n    target_size=np.round(n_samples/n) # test target size \n    \n    samples_left = np.full_like(remaining_samples, True)\n    test_full = []\n    for split in np.array_split(unique, n): # splitting unique groups in even subarrays \n#         mask = groups.isin(split)\n        mask = df['group_id'].isin(split) # put these in test \n\n        n_missing = int(target_size-sum(mask))\n        sample_from = remaining_samples[samples_left] # only use samples which have not been used for test \n        if n_missing <= len(sample_from):\n            chosen = sample_from.sample(n=n_missing, replace=False, random_state=seed) # chose n_missing non-used \n            samples_left = np.logical_not(remaining_samples.isin(chosen)) & samples_left # remember which have been used before and add the newly used ones \n        else:\n            chosen = sample_from\n            \n        mask = mask | df.isin(chosen).image_id\n#         mask = mask.append(remaining_samples.isin(chosen)) # filling up mask with individual samples \n        \n        train, test = ix[~mask], ix[mask] # select train and test for this split \n\n        result.append((train, test))\n    \n    return result\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:07.377497Z","iopub.execute_input":"2023-06-05T10:24:07.378014Z","iopub.status.idle":"2023-06-05T10:24:07.394106Z","shell.execute_reply.started":"2023-06-05T10:24:07.377962Z","shell.execute_reply":"2023-06-05T10:24:07.392564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/duplicate-imgids-imghash-thres-092csv/duplicate_imgids_imghash_thres_092.csv') # all images with their group\npaths = sorted(glob.glob('/kaggle/input/prostate-cancer-grade-assessment/train_images/*.tiff'))\n\nimgids = [Path(p).stem.split('_')[0] for p in paths] # all imgids\nunique_samples = [id for id in imgids if id not in list(df['image_id'])] # all imgids which are not in a group\n\ndf2 = pd.DataFrame({'image_id':unique_samples}) # make dataframe with non-group images\ndf2 = df2.reindex(df.columns.tolist(), axis=1)\n\ndf = pd.concat([df, df2]).reset_index(drop=True) # join the dfs \n\ntrain_df = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\n\ndf = train_df.join(other=df.set_index('image_id'), on='image_id', how='left')\n\n# remove sus data from dft\nsussy_df = pd.read_csv('/kaggle/input/panda-analysis/PANDA_Suspicious_Slides.csv')\ndf = df[~df.image_id.isin(sussy_df.image_id)] #remove sus images\ngroups= df['group_id'].dropna(inplace=False) # getting individual group ids \n\n\nfolds = RandomGroupKFold_split(groups=groups, n=10, df=df,seed=42) ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:07.883211Z","iopub.execute_input":"2023-06-05T10:24:07.88594Z","iopub.status.idle":"2023-06-05T10:24:13.100482Z","shell.execute_reply.started":"2023-06-05T10:24:07.885901Z","shell.execute_reply":"2023-06-05T10:24:13.099495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_folds = False\n\nif check_folds:\n    \n    print(len(df))\n    \n    train_d = []\n    test_d = []\n    for fold, (train, test) in enumerate(folds): \n        df.loc[train,:].to_csv(f'train_{fold}.csv')\n        df.loc[test,:].to_csv(f'test_{fold}.csv')\n        train_d += list(df.loc[train,:]['image_id'])\n        test_d += list(df.loc[test,:]['image_id'])\n    print(len(np.unique(train_d)))\n    print(len(np.unique(test_d))) # checking if all or almost all are included (depending on rounding some might be left out)\n\n    print(np.unique(df.group_id[df.index.isin(folds[1][1])])) # checking if a given fold has low group id --> these are big, should avoid this if this is the only validation they are since they are similar ","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:13.10264Z","iopub.execute_input":"2023-06-05T10:24:13.103294Z","iopub.status.idle":"2023-06-05T10:24:13.111776Z","shell.execute_reply.started":"2023-06-05T10:24:13.103259Z","shell.execute_reply":"2023-06-05T10:24:13.110914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Tiler():\n    def __init__(self, N_tiles= 36, tile_size = 2**8, level=1):\n        self.N_tiles = N_tiles\n        self.tile_size = tile_size\n        self.level = level\n        \n    # This function takes an openslide object and returns the top left coordinates of N tiles (of a given size) with the most tissue pixels. \n    # Note: slide.level_dimensions[level] = (width,height).\n    # Note: padding is done to the right and bottom, this is to keep it simple while having at most 1 tile in memory at a time.\n    def _get_tile_locations_from_slide(self, slide):\n        tiles = []\n        required_padding = False\n        xlocs, ylocs = np.arange(0, slide.level_dimensions[self.level][0], self.tile_size), np.arange(0, slide.level_dimensions[self.level][1], self.tile_size) # Get the coordinates of the top left corners of the tiles.\n        for x_i, xloc in enumerate(xlocs):\n            for y_i, yloc in enumerate(ylocs):\n                region = np.copy(slide.read_region((xloc*(4**self.level),yloc*(4**self.level)), self.level, (self.tile_size,self.tile_size))) # The position is wrt. level 0, so must convert to level 0 coordinates by multiplying by the downsampling factor.\n                region_arr = np.asarray(region)[:,:,:3] # Ignore the alpha channel\n                if xloc+self.tile_size > slide.level_dimensions[self.level][0] or yloc+self.tile_size > slide.level_dimensions[self.level][1]: # if the tile goes out of bounds\n                    region_arr[region_arr==0] = 255\n                    required_padding = True\n                pixel_sum = region_arr.sum()\n                tiles.append({'xloc': xloc, 'yloc': yloc, 'pixel_sum': pixel_sum, 'required_padding': required_padding}) # store top left corner location and the tile's pixel_sum\n                required_padding = False\n        sorted_tiles = sorted(tiles, key= lambda d: d['pixel_sum']) # Sort tiles based on their pixel_sum field\n        sorted_tiles = sorted_tiles[:self.N_tiles] # Get top N tiles\n        return sorted_tiles\n    \n    # Return the tensor of individual tiles\n    def get_individual_tiles(self, slide, transform=None):\n        tiles_info = self._get_tile_locations_from_slide(slide)\n        tiles = torch.empty((self.N_tiles,3, self.tile_size, self.tile_size))\n        for i, tile in enumerate(tiles_info):\n            img = slide.read_region((tile['xloc']*(4**self.level),tile['yloc']*(4**self.level)), self.level, (self.tile_size,self.tile_size))\n            img = torch.clone(transforms.PILToTensor()(img)) # The position is wrt. level 0, so must convert to level 0 coordinates by multiplying by the downsampling factor.\n            img = img[:3,:,:] # Ignore the alpha channel\n            if tile['required_padding']:\n                img[img==0] = 255\n            if transforms:\n                img = img.float()/255.0 # Necessary for the transformations, images are expected to be a tensor with elements between [0,1]\n                img = transform(img)\n            tiles[i,...] = img\n        return tiles","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:13.113473Z","iopub.execute_input":"2023-06-05T10:24:13.114305Z","iopub.status.idle":"2023-06-05T10:24:13.132649Z","shell.execute_reply.started":"2023-06-05T10:24:13.114268Z","shell.execute_reply":"2023-06-05T10:24:13.131823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PANDADataModule(pl.LightningModule):\n    def __init__(self, stage:str='train', fold=None, num_classes:int=6, train_csv=None):\n        super().__init__()\n        self.stage=stage\n        self.num_classes=num_classes-1\n        self.normalize = transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225])\n        self.transform = transforms.Compose([\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.ColorJitter(brightness=0, contrast=0, saturation=0, hue=[-0.03,0.04])\n        ])\n        \n        if stage=='test':\n            self.tiler = Tiler(N_tiles = 36, tile_size=2**8, level=1)\n            self.img_dir = '/kaggle/input/prostate-cancer-grade-assessment/test_images/'\n            self.df = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/test.csv')\n#             self.img_dir = '/kaggle/input/prostate-cancer-grade-assessment/train_images/'\n#             self.df = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\n        else: # If train/val, check whether folds are provided\n            self.img_dir = '/kaggle/input/ismi-group3-panda-36-256-256-res1-tiles/tiled_images/tiled_images'\n            train_csv = train_csv\n            if fold is None:\n                if stage == 'train':\n                    self.df = train_csv[:10000]\n                elif stage == 'val':\n                    self.df = train_csv[10000:]\n            else: \n                if stage == 'train':\n                    train = fold[0]\n                    self.df = train_csv.loc[train,:]\n                elif stage == 'val':\n                    val = fold[1]\n                    self.df = train_csv.loc[val,:]\n\n    def convert_to_ordinal(self, grade: int):\n        ordinal = torch.zeros(self.num_classes)\n        ordinal[0:grade] = 1\n        return ordinal\n            \n    # Returns the individual tiles and ISUP label \n    def __getitem__(self, idx):\n        if self.stage=='train' or self.stage=='val':\n            item_name = self.df.iloc[idx].loc['image_id'] # Get the name of the sample\n            item_dir = os.path.join(self.img_dir, f'{item_name}_tiled.png') \n            img = torchvision.io.read_image(item_dir)\n            \n            ## Augmentation\n            img = img.float()/255.0 # Necessary for the transformations, images are expected to be a tensor with elements between [0,1]\n            # Only augment the data when training\n            if self.stage == 'train': \n                img = self.transform(img)\n            img = self.normalize(img) # Always normalize the input\n            \n            ## Splitting the tiled_image into individual tiles\n            rows = torch.tensor_split(img[None, :, :, :], 6, dim=3) # this way they go from left to right and then next row if you swap dimension order its up down and the column\n            rows=torch.cat(rows, dim=0)\n            \n            tiles = torch.tensor_split(rows, 6, dim=2)\n            tiles=torch.cat(tiles, dim=0)\n\n            return tiles, self.convert_to_ordinal(int(self.df.iloc[idx].loc['isup_grade'])), idx\n    \n        elif self.stage=='test':\n            item_name = self.df.iloc[idx].loc['image_id'] # Get the name of the sample\n            file_path = os.path.join(self.img_dir, f'{item_name}.tiff')\n            slide = openslide.OpenSlide(file_path)\n            tiles = self.tiler.get_individual_tiles(slide, transform=self.normalize)\n            return tiles, self.df.iloc[idx].loc['isup_grade']\n        \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:13.135334Z","iopub.execute_input":"2023-06-05T10:24:13.13625Z","iopub.status.idle":"2023-06-05T10:24:13.159393Z","shell.execute_reply.started":"2023-06-05T10:24:13.136217Z","shell.execute_reply":"2023-06-05T10:24:13.158482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity checking that the tiling dataloader is working\ndata_testing = False\n\nif data_testing:\n    \n    train_data = PANDADataModule(stage=\"train\", train_csv=df)\n    train_loader = torch.utils.data.DataLoader(train_data, batch_size=1, shuffle=False)\n\n    for i, (tiles, label) in enumerate(train_loader): \n        plt.imshow(tiles[0][0].permute(1,2,0))\n        plt.show()\n        plt.imshow(tiles[0][1].permute(1,2,0))\n        plt.show()\n        break\n        \n    for i, (tiles, label) in enumerate(train_loader):\n        if i < 4:\n            fig, axs = plt.subplots(1,36, figsize=(20,3))\n            for i,ax in enumerate(axs):\n                ax.imshow(tiles[0,i,...].permute(1,2,0)) # Get back the width x height x rgb shape\n                ax.axis(\"off\")\n            plt.show()\n        else:\n            break","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:13.757382Z","iopub.execute_input":"2023-06-05T10:24:13.757861Z","iopub.status.idle":"2023-06-05T10:24:13.772042Z","shell.execute_reply.started":"2023-06-05T10:24:13.757821Z","shell.execute_reply":"2023-06-05T10:24:13.771047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Network","metadata":{}},{"cell_type":"code","source":"class ModelModule(pl.LightningModule):\n    \"\"\" \n    densenetblur121d, convnext_pico, nvidia_efficientnet_widese_b0, tf_efficientnet_b1 \n    \"\"\"\n    def __init__(self, name, model, num_classes = 6, n_tiles=36, train_more=True, weight_tensor:torch.Tensor = torch.Tensor([1, 1.2, 1.5, 1.8, 2.2])):\n        super().__init__()\n        if not name:\n            self.name = model\n        else:\n            self.name = name\n        self.n_tiles = n_tiles\n        self.train_more = train_more \n        self.train_losses = []\n        self.val_losses = []\n        self.epoch_train_losses = []\n        self.epoch_val_losses = []\n        self.num_classes=num_classes-1\n        self.criterion = nn.BCEWithLogitsLoss(pos_weight=weight_tensor)\n        self.L2_norm = 0.001\n        self.dropout_proba = 0.1\n        \n        if model.startswith('convnext'):\n            self.model = timm.create_model(model, pretrained=True)\n            # Replace last layer with a 6 class output layer\n            if self.train_more: # add the last block as well \n                self.encoder = nn.Sequential(self.model.stem, self.model.stages[:-1])\n                down_sample_to = 32\n                last_block = ConvN.ConvNeXtStage(in_chs=256, out_chs=down_sample_to, norm_layer=LayerNorm2d, conv_mlp=True)\n                self.model.head.norm = LayerNorm2d(down_sample_to, eps=1e-06)\n                self.fine_tune = nn.Sequential(last_block, self.model.norm_pre, *list(self.model.head.children())[:-1])\n                self.classifier = nn.Sequential(nn.Conv1d(in_channels=self.n_tiles, out_channels=self.num_classes+1, kernel_size=down_sample_to),\n                                                nn.GELU(),\n                                                nn.Flatten(),\n                                                nn.Linear(self.num_classes+1, self.num_classes))\n            else: \n                enc = list(self.model.children())[:-1] + list(self.model.head)[:-1]\n                self.encoder = nn.Sequential(*enc)\n                self.classifier = torch.nn.Linear(in_features=self.model.head[-1].in_features*self.n_tiles, out_features=self.num_classes, bias=True)\n                \n        elif model.startswith('densenet'):\n            self.model = timm.create_model(model, pretrained=True)\n            \n            if self.train_more:\n                if model.endswith('161'): # depending on the size of the model different depths apply thus last layer is a different one \n                    final_layer = self.model.features.denseblock4.denselayer24\n                elif model.endswith('121') or model.endswith('121d'):\n                    final_layer = self.model.features.denseblock4.denselayer16\n                    \n                self.encoder = nn.Sequential(list(self.model.children())[0][:-2])\n                \n                self.fine_tune = nn.Sequential(self.model.features.denseblock4, self.model.features.norm5, \n                                                nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=(1, 1), stride=(1, 1), bias=False),\n                                                nn.ReLU(),\n                                                nn.BatchNorm2d(num_features=512),\n                                                nn.Conv2d(in_channels=512, out_channels=128, kernel_size=(1, 1), stride=(1, 1), bias=False),\n                                                nn.ReLU(),\n                                                nn.BatchNorm2d(num_features=128),\n                                               self.model.global_pool)\n                self.classifier =  self.get_classifier(activation=nn.ReLU, dropout_proba=self.dropout_proba)\n\n            else:\n                self.encoder = nn.Sequential(*list(self.model.children())[:-1])\n                self.classifier = torch.nn.Linear(in_features=self.model.classifier.in_features*self.n_tiles, out_features=self.num_classes, bias=True)\n\n                \n        elif model.startswith('tf_efficientnet'):\n            if model.endswith('0'):\n                weight_path = f'/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth'\n            elif model.endswith('1'):\n                weight_path = '/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b1/1/tf_efficientnet_b1_aa-ea7a6ee0.pth'\n            self.model = timm.create_model(model, \n                                       checkpoint_path=weight_path)\n\n            if self.train_more:\n                self.encoder = nn.Sequential(self.model.conv_stem, self.model.bn1, self.model.blocks[:-1])\n                # Change the conv_head to downsample instead of upsample\n                self.model.conv_head = torch.nn.Conv2d(in_channels=320, out_channels=128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n                norm_layer = nn.BatchNorm2d\n                act_layer = nn.SiLU\n                norm_act_layer = get_norm_act_layer(norm_layer, act_layer)\n                # Change the batchnorm + activation accordingly\n                self.model.bn2 = norm_act_layer(128, eps=0.001, inplace=True)\n                self.fine_tune = nn.Sequential(self.model.blocks[-1], self.model.conv_head, self.model.bn2, self.model.global_pool)\n                self.classifier =  self.get_classifier(activation=nn.SiLU, dropout_proba=self.dropout_proba)\n            else:\n                self.classifier = torch.nn.Linear(in_features=self.model.classifier.in_features*self.n_tiles, out_features=self.num_classes, bias=True)\n                self.encoder = nn.Sequential(*list(self.model.children())[:-1])\n        \n        elif model.startswith('nvidia_efficientnet'):\n            self.model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub',  model, pretrained=True) \n\n            if self.train_more:\n                self.encoder = nn.Sequential(self.model.stem, self.model.layers[:-1])\n                # Make last conv 320 -> 128 instead of 320 -> 1280\n                self.model.features.conv = nn.Conv2d(in_channels=320, out_channels=128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n                self.model.features.bn = nn.BatchNorm2d(128, eps=0.001, momentum=0.01)\n                self.fine_tune = nn.Sequential(self.model.layers[-1], self.model.features, self.model.classifier[:-1])\n                self.classifier =  self.get_classifier(activation=nn.SiLU, dropout_proba=self.dropout_proba)\n            else:\n                enc = list(self.model.children())[:-1] + list(self.model.classifier)[:-1]\n                self.encoder = nn.Sequential(*enc)\n                self.classifier = torch.nn.Linear(in_features=self.model.classifier.fc.in_features*self.n_tiles, out_features=self.num_classes, bias=True)\n\n        else: \n            raise NotImplementedError('Model type not implemented')\n            \n            \n    def get_classifier(self, activation=nn.ReLU, dropout_proba=0.0):\n        return nn.Sequential(torch.nn.Linear(in_features=128*self.n_tiles, out_features=256, bias=True),\n                                            activation(),\n                                            nn.BatchNorm1d(256),\n                                            nn.Dropout(p=dropout_proba),\n                                            torch.nn.Linear(in_features=256, out_features=self.num_classes, bias=True))\n    \n    # For the individual tiles\n    def forward(self, x):\n        batch_size = x.shape[0]\n        # Stack all the tiles in the batch, the network expects a 4D input, we can use (batch_size*N_tiles, RGB, height, width), to be unstacked after conv -> (batch,N_tiles*classifier.in_features).\n        x = x.view(-1,x.shape[2],x.shape[3],x.shape[4])\n        with torch.no_grad():\n            x = self.encoder(x)\n\n        if self.train_more: \n            x= self.fine_tune(x)\n            \n        if self.name.startswith('convnext'):\n            x = x.view(batch_size, self.n_tiles, -1) # #(batch_size * N_tiles, single_tile_features) -> (batch_size, N_tiles, single_tile_features)\n        else:\n            x = x.view(batch_size,-1) #(batch_size * N_tiles, single_tile_features) -> (batch_size, N_tiles*single_tile_features)\n            \n        x = self.classifier(x)\n        return x\n        \n    def training_step(self, batch, batch_idx):\n        x,y,_ = batch\n        # Make target label ordinal\n        # y = ordinal(y)\n        x = x.float()\n        y_hat = self.forward(x)\n        loss = self.criterion(y_hat,y)\n        self.epoch_train_losses.append(loss.detach())\n        self.log(\"train_loss\", loss.detach())\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        x,y,_ = batch\n        # Make target label ordinal\n        # y = ordinal(y)\n        x = x.float()\n        y_hat = self.forward(x)\n        loss = self.criterion(y_hat,y)\n        self.epoch_val_losses.append(loss.detach())\n        self.log(\"val_loss\", loss.detach())\n        return loss\n    \n    def test_step(self, batch, batch_idx):\n        loss = self.criterion(y_hat,y)\n        return loss\n    \n    def on_train_epoch_end(self):\n        avg = torch.mean(torch.Tensor(self.epoch_train_losses))\n        print(f'Train loss: {avg}')\n        print(f'finished epoch {self.current_epoch}')\n        self.train_losses.append(avg)\n        self.epoch_train_losses = []\n        trainer.save_checkpoint(\"latest_model_\"+self.name+\".ckpt\") # built-in checkpointing does not work as expected so I did this \n\n        \n    def on_validation_epoch_end(self):\n        avg = torch.mean(torch.Tensor(self.epoch_val_losses))\n        print(f'Validation loss: {avg}')\n        \n        if self.current_epoch == 0 and len(self.val_losses) != 0:\n            print('cleared after validation sanity check')\n            self.val_losses = []\n        \n        if len(self.val_losses) != 0: # otherwise cannot compute min\n            if avg < torch.min(torch.Tensor(self.val_losses)):\n                trainer.save_checkpoint(\"best_model_\"+self.name+\".ckpt\") # built-in checkpointing does not work as expected so I did this \n        else:\n            trainer.save_checkpoint(\"best_model_\"+self.name+\".ckpt\") # built-in checkpointing does not work as expected so I did this \n        \n        self.val_losses.append(avg)\n        self.epoch_val_losses = []\n\n        \n    def configure_optimizers(self):\n        if self.train_more:\n            params = list(self.fine_tune.parameters()) + list(self.classifier.parameters())\n            optimizer = torch.optim.Adam(params=params, weight_decay=self.L2_norm)\n        else: \n            optimizer = torch.optim.Adam(params=self.classifier.parameters(), weight_decay=self.L2_norm)\n        return optimizer","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:14.742699Z","iopub.execute_input":"2023-06-05T10:24:14.743125Z","iopub.status.idle":"2023-06-05T10:24:14.80527Z","shell.execute_reply.started":"2023-06-05T10:24:14.743093Z","shell.execute_reply":"2023-06-05T10:24:14.804368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"batch_size = 16\nn_epochs = 4000\n\nname = \"tf_efficientnet_b1\"\n# model = ModelModule('densenetblur121d', train_more=True)\n# model = ModelModule('convnext_pico', train_more=True)\n# model = ModelModule('tf_efficientnet_b1', train_more=True)\n\nmodel = ModelModule(name, 'tf_efficientnet_b1', train_more=True)\n\ncheckpointed = None\nif checkpointed is not None:\n    print('Loading from checkpoint')\n    model = ModelModule.load_from_checkpoint(name=name, model=name, checkpoint_path=checkpointed)\n\ntrain_data = PANDADataModule(stage=\"train\", fold=folds[1], train_csv=df)\ntrain_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\nval_data = PANDADataModule(stage=\"val\", fold=folds[1], train_csv=df)\nval_loader = torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:36.832873Z","iopub.execute_input":"2023-06-05T10:24:36.833225Z","iopub.status.idle":"2023-06-05T10:24:37.811715Z","shell.execute_reply.started":"2023-06-05T10:24:36.833197Z","shell.execute_reply":"2023-06-05T10:24:37.810779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"do_train = False\nif do_train:\n    # if you stop training and start again later \"versions += 1\" this way you keep track of the order of log files\n    logger = TensorBoardLogger(save_dir=\"tb_logs\", version=1, name=name)\n    timer = Timer(duration=\"00:10:00:00\")\n    \n#     trainer = pl.Trainer(log_every_n_steps=16, limit_train_batches=0.01, limit_val_batches=0.1, max_epochs=1, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer]) # limits to speed up the whole thing a little  \n    trainer = pl.Trainer(log_every_n_steps=16, max_epochs=n_epochs, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer]) # limits to speed up the whole thing a little  \n    trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:38.797014Z","iopub.execute_input":"2023-06-05T10:24:38.797693Z","iopub.status.idle":"2023-06-05T10:24:38.803659Z","shell.execute_reply.started":"2023-06-05T10:24:38.797658Z","shell.execute_reply":"2023-06-05T10:24:38.802797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hard Negative Mining","metadata":{}},{"cell_type":"code","source":"class HardNegWrapperDataModule(pl.LightningModule):\n    def __init__(self, wrappermodule, hard_neg_idxs):\n        super().__init__()\n        self.DM = wrappermodule\n        self.hard_neg_idxs = hard_neg_idxs\n    \n    def __getitem__(self, idx):\n        return self.DM.__getitem__(self.hard_neg_idxs[idx]) #just to be clear, here we index the list containing indexes :)\n        \n    def __len__(self):\n        return len(self.hard_neg_idxs)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:24:44.139542Z","iopub.execute_input":"2023-06-05T10:24:44.140042Z","iopub.status.idle":"2023-06-05T10:24:44.149346Z","shell.execute_reply.started":"2023-06-05T10:24:44.140004Z","shell.execute_reply":"2023-06-05T10:24:44.147354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 16\nn_epochs = 4000","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:25:14.931826Z","iopub.execute_input":"2023-06-05T10:25:14.932196Z","iopub.status.idle":"2023-06-05T10:25:14.939698Z","shell.execute_reply.started":"2023-06-05T10:25:14.93217Z","shell.execute_reply":"2023-06-05T10:25:14.93865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#classify all samples to find hard negatives (save indexes of hard negative samples) (takes roughly 1.5 hour)\n## LOAD BEST MODEL\nmodel = ModelModule.load_from_checkpoint(name=name, model=name, checkpoint_path=\"/kaggle/input/effnet-training2/AAA training effnet/run 4/best_model_tf_efficientnet_b1.ckpt\").to(device)\nhardneg_idxs = torch.empty(0, dtype=torch.uint8).to(device)\nsig = nn.Sigmoid()\nmodel.eval()\nfor i, (x, y, idxs) in tqdm(enumerate(train_loader), total=len(train_loader), desc=\"hard negative mining\"): \n    x = x.to(device).float()\n    y = y.to(device)\n    idxs = idxs.to(device)\n    with torch.no_grad():\n        y_hat = model(x).to(device)\n    \n    #put a gismoid on linear layer outputs (just like in loss function)\n    y_hat = sig(y_hat)\n\n    #do hard negative mining for zero vs non-zero labels\n    threshold = 0.5\n    y_hat = torch.where(y_hat>threshold, 1, 0)\n\n    y_0 = torch.all(y == torch.zeros_like(y , device=device), axis=1)\n    y_hat_0 = torch.all(y_hat == torch.zeros_like(y_hat , device=device), axis=1)\n\n    hard_neg_mask = ~(y_0==y_hat_0)\n\n    hardneg_idxs = torch.cat((hardneg_idxs, idxs[hard_neg_mask]), 0)\n    \n#make new dataset with hardnegative samples\nhardneg_idxs = hardneg_idxs.tolist()\nhardneg_data = HardNegWrapperDataModule(wrappermodule=train_data, hard_neg_idxs=hardneg_idxs)\nhardneg_loader = torch.utils.data.DataLoader(hardneg_data, batch_size=batch_size, shuffle=True)\nmodel.train()\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T10:26:56.752897Z","iopub.execute_input":"2023-06-05T10:26:56.753334Z","iopub.status.idle":"2023-06-05T11:55:35.281907Z","shell.execute_reply.started":"2023-06-05T10:26:56.753301Z","shell.execute_reply":"2023-06-05T11:55:35.280855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_hardneg_idxs = True\nif save_hardneg_idxs:\n    file_name = f'hardneg_idxs_{name}.pkl'\n    with open(file_name, 'wb') as file:\n        pickle.dump(hardneg_idxs, file)\n        print(f'Object successfully saved to \"{file_name}\"')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:55:35.283811Z","iopub.execute_input":"2023-06-05T11:55:35.284246Z","iopub.status.idle":"2023-06-05T11:55:35.290964Z","shell.execute_reply.started":"2023-06-05T11:55:35.284213Z","shell.execute_reply":"2023-06-05T11:55:35.289772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train one epoch on the hardneg samples","metadata":{}},{"cell_type":"code","source":"model = ModelModule.load_from_checkpoint(name=name, model=name, checkpoint_path=\"/kaggle/input/effnet-training2/AAA training effnet/run 4/best_model_tf_efficientnet_b1.ckpt\").to(device)\nn_epochs = 1\ndo_train = True\nif do_train:\n    # if you stop training and start again later \"versions += 1\" this way you keep track of the order of log files\n    logger = TensorBoardLogger(save_dir=\"tb_logs\", version=1, name=name)\n    timer = Timer(duration=\"00:07:00:00\")\n    \n#     trainer = pl.Trainer(log_every_n_steps=16, limit_train_batches=0.02, limit_val_batches=0.1, max_epochs=1, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer]) # limits to speed up the whole thing a little  \n    trainer = pl.Trainer(log_every_n_steps=16, max_epochs=n_epochs, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer])\n    trainer.fit(model=model, train_dataloaders=hardneg_loader, val_dataloaders=val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:50:31.4872Z","iopub.execute_input":"2023-06-05T12:50:31.48762Z","iopub.status.idle":"2023-06-05T13:00:50.715416Z","shell.execute_reply.started":"2023-06-05T12:50:31.487569Z","shell.execute_reply":"2023-06-05T13:00:50.714475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Retrain after hardneg mining","metadata":{}},{"cell_type":"code","source":"n_epochs = 1\ndo_train = True\nif do_train:\n    # if you stop training and start again later \"versions += 1\" this way you keep track of the order of log files\n    logger = TensorBoardLogger(save_dir=\"tb_logs\", version=1, name=name)\n    timer = Timer(duration=\"00:07:00:00\")\n    \n#     trainer = pl.Trainer(log_every_n_steps=16, limit_train_batches=0.01, limit_val_batches=0.1, max_epochs=1, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer]) # limits to speed up the whole thing a little  \n    trainer = pl.Trainer(log_every_n_steps=16, max_epochs=n_epochs, accelerator='gpu', enable_progress_bar=True, logger=logger, callbacks=[timer]) # limits to speed up the whole thing a little  \n    trainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders=val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T13:02:18.194555Z","iopub.execute_input":"2023-06-05T13:02:18.19496Z","iopub.status.idle":"2023-06-05T14:21:08.994212Z","shell.execute_reply.started":"2023-06-05T13:02:18.194929Z","shell.execute_reply":"2023-06-05T14:21:08.993254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Make zip from output directory\nimport shutil\n# # shutil.rmtree('/kaggle/working/tb_logs')\n# myfile = \"/kaggle/working/latest_model_tf_efficientnet_b1.ckpt\"\n# # If file exists, delete it.\n# if os.path.isfile(myfile):\n#     os.remove(myfile)\n# else:\n#     # If it fails, inform the user.\n#     print(\"Error: %s file not found\" % myfile)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:43:50.976675Z","iopub.execute_input":"2023-06-05T12:43:50.977034Z","iopub.status.idle":"2023-06-05T12:43:50.994024Z","shell.execute_reply.started":"2023-06-05T12:43:50.977006Z","shell.execute_reply":"2023-06-05T12:43:50.99304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Validation","metadata":{}},{"cell_type":"code","source":"def inference(model, dataloader, device):\n    model.eval()\n    model.to(device)\n    preds = []\n    sigm = nn.Sigmoid()\n    threshold = 0.5\n    for i, tiles in tqdm(enumerate(dataloader), total=len(dataloader)):\n        tiles = tiles[0].to(device).float() ## img[0] because we get the tiles and the labels, we only want the tiles\n        with torch.no_grad():\n            output = model(tiles)\n            output = sigm(output)\n            output = torch.where(output>threshold, 1, 0)\n            pred = torch.sum(output, axis=1)\n\n        preds.append(pred.to('cpu').numpy()) # add the predictions of this batch to the overall list\n    preds = np.concatenate(preds) # Make it a single list of predictions over all batches\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-06-02T15:35:39.514563Z","iopub.execute_input":"2023-06-02T15:35:39.514938Z","iopub.status.idle":"2023-06-02T15:35:39.523353Z","shell.execute_reply.started":"2023-06-02T15:35:39.514909Z","shell.execute_reply":"2023-06-02T15:35:39.522432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nprint(f'Using {device} for inference')","metadata":{"execution":{"iopub.status.busy":"2023-06-02T15:35:39.835896Z","iopub.execute_input":"2023-06-02T15:35:39.836956Z","iopub.status.idle":"2023-06-02T15:35:39.842796Z","shell.execute_reply.started":"2023-06-02T15:35:39.836918Z","shell.execute_reply":"2023-06-02T15:35:39.841472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import cohen_kappa_score\n\ndef quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')","metadata":{"execution":{"iopub.status.busy":"2023-06-02T15:37:02.771733Z","iopub.execute_input":"2023-06-02T15:37:02.772129Z","iopub.status.idle":"2023-06-02T15:37:03.539429Z","shell.execute_reply.started":"2023-06-02T15:37:02.7721Z","shell.execute_reply":"2023-06-02T15:37:03.538423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get predictions and compute accuracy\nfrom tqdm.notebook import tqdm\n# model = model.load_from_checkpoint('/kaggle/working/best_model.ckpt')\n# model = torch.jit.load('/kaggle/working/baseline_NVIDIA_Efficientnet_wideSE_b0.pt')\nmodel = ModelModule.load_from_checkpoint(name=name, model=name, checkpoint_path=\"/kaggle/working/best_model_\"+name+\".ckpt\").to(device)\npredictions = inference(model,val_loader,device)\nprint(f\"Accuracy: {np.sum(predictions==val_data.df['isup_grade'])/len(predictions)}\")\nprint(f\"Quadratic weighted kappa: {quadratic_weighted_kappa(predictions, val_data.df['isup_grade'])}\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T15:35:40.2718Z","iopub.execute_input":"2023-06-02T15:35:40.274395Z","iopub.status.idle":"2023-06-02T15:36:09.893099Z","shell.execute_reply.started":"2023-06-02T15:35:40.274359Z","shell.execute_reply":"2023-06-02T15:36:09.890035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SAVE MODEL FOR SUBMISSION","metadata":{}},{"cell_type":"code","source":"# Load best network from checkpoint\nmodel = ModelModule.load_from_checkpoint(name=name, model=name, checkpoint_path=\"/kaggle/working/best_model_\"+name+\".ckpt\").to(device)\n# Save entire model\nmodel_scripted = torch.jit.script(model) # Export to TorchScript\nmodel_scripted.save(name+'_submission.pt') # Save","metadata":{},"execution_count":null,"outputs":[]}]}