{"cells":[{"metadata":{},"cell_type":"markdown","source":"# PANDA Challenge Inference\n\nThere are 4 models trained on 4 different stains from the data. Eg: a reference stain is selected from a dataset, the whole dataset is stain normalized to the reference stain and then a classification network is trained by selecting 36 256x256 tiles from that image and stacking tiles to form a full image of resolution 1536x1536.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Installing necessary libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/spamspip/spams-2.6.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/imagecodecs/imagecodecs-2020.5.30-cp37-cp37m-manylinux2010_x86_64.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEBUG = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using efficientnet pytorch as a model from and staintools library to perform stain normalization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path = [\n    '../input/efficientnet/EfficientNet-PyTorch-master/',\n    '../input/staintoolszip/',\n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import skimage.io\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom efficientnet_pytorch import model as enet\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import staintools\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment'\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\ndf_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))\n\nmodel_dir = '../input/4stains'\nimage_folder = os.path.join(data_dir, 'test_images')\nis_test = os.path.exists(image_folder)  # IF test_images is not exists, we will use some train images.\nimage_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')\ntrain_folder = os.path.join(data_dir, 'train_images')\n\ndf = df_test if is_test else df_train.loc[:1]\n\ntile_size = 256\nimage_size = 256\nn_tiles = 36\nbatch_size = 12\nnum_workers = 12\n\ndevice = torch.device('cuda')\n\nprint(image_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class enetv2(nn.Module):\n    def __init__(self, backbone, out_dim):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x\n    \n    \ndef load_models(model_files):\n    models = []\n    for model_f in model_files:\n        model_f = os.path.join(model_dir, model_f)\n        backbone = 'efficientnet-b0'\n        model = enetv2(backbone, out_dim=5)\n        model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=False)\n        model.eval()\n        model.to(device)\n        models.append(model)\n        print(f'{model_f} loaded!')\n    return models\n\n\nmodel_files = [\n    'b0-stain-bb9_best_fold0.pth','b0-stain-964_best_fold1.pth','b0-stain-917_best_fold2.pth',\n    'b0-stain-c11_best_fold3.pth'\n]\n\nmodels = load_models(model_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sampleMeanStd(img):\n\n\n    img  = img.astype(\"float32\")\n\n    b_ch=np.mean(img[:,:,0])\n    g_ch=np.mean(img[:,:,1])\n    r_ch=np.mean(img[:,:,2])\n\n    #Individual channel-wise mean subtraction\n    img -= np.array((b_ch,g_ch,r_ch))\n\n    b_ch=np.std(img[:,:,0])\n    g_ch=np.std(img[:,:,1])\n    r_ch=np.std(img[:,:,2])\n\n    img /= np.array((b_ch,g_ch,r_ch))\n\n    return img\n\ndef get_tiles(img, mode=0,forReference=False):\n        result = []\n        h, w, c = img.shape\n        pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n        pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n\n        img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)\n        img3 = img2.reshape(\n            img2.shape[0] // tile_size,\n            tile_size,\n            img2.shape[1] // tile_size,\n            tile_size,\n            3\n        )\n\n        img3 = img3.transpose(0,2,1,3,4).reshape(-1, tile_size, tile_size,3)\n        n_tiles_with_info = (img3.reshape(img3.shape[0],-1).sum(1) < tile_size ** 2 * 3 * 245).sum()\n#         print(n_tiles_with_info)\n        if len(img) < n_tiles:\n            img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)\n        if forReference:\n            idxs = np.argsort(img3.reshape(img3.shape[0],-1).sum(-1))[:n_tiles]#[:]\n        else:\n            idxs = np.argsort(img3.reshape(img3.shape[0],-1).sum(-1))[:n_tiles_with_info]#[:n_tiles]\n        img3 = img3[idxs]\n        for i in range(len(img3)):\n            result.append({'img':img3[i], 'idx':i})\n        return result, n_tiles_with_info\n    \ndef getItemCustom(img_id,n_tiles=36):\n    tiff_file = os.path.join(train_folder, f'{img_id}.tiff')\n    image = skimage.io.MultiImage(tiff_file)[1]\n    #print(image.shape)\n    tiles, OK = get_tiles(image,forReference=True)\n\n    \n    idxes = list(range(n_tiles))\n#     idxes = np.asarray(idxes) + n_tiles if self.sub_imgs else idxes\n\n    n_row_tiles = int(np.sqrt(n_tiles))\n    images = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n    for h in range(n_row_tiles):\n        for w in range(n_row_tiles):\n            i = h * n_row_tiles + w\n\n            if len(tiles) > idxes[i]:\n                this_img = tiles[idxes[i]]['img']\n            else:\n                this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n            #this_img = 255 - this_img\n            h1 = h * image_size\n            w1 = w * image_size\n            images[h1:h1+image_size, w1:w1+image_size] = this_img\n\n#         images = 255 - images\n#         images = images.astype(np.float32)\n#         images /= 255\n    #images = sampleMeanStd(images)\n    #print(images.shape)\n    #images = images.transpose(2, 0, 1)\n    #print(images.shape)\n\n    return images #torch.tensor(images)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fetching 4 images as 4 reference stain from the training data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"reference_stain = getItemCustom('001c62abd11fa4b57bf7a6c603a11bb9')\nreference_stain = np.uint8(reference_stain)\nreference_stain = cv2.resize(reference_stain,(1024,1024))\nreference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)\nnormalizer = staintools.StainNormalizer(method='vahadane')\nnormalizer.fit(reference_stain)\n\nreference_stain = getItemCustom('008069b542b0439ed69b194674051964')\nreference_stain = np.uint8(reference_stain)\nreference_stain = cv2.resize(reference_stain,(1024,1024))\nreference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)\nnormalizer2 = staintools.StainNormalizer(method='vahadane')\nnormalizer2.fit(reference_stain)\n\nreference_stain = getItemCustom('0005f7aaab2800f6170c399693a96917')\nreference_stain = np.uint8(reference_stain)\nreference_stain = cv2.resize(reference_stain,(1024,1024))\nreference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)\nnormalizer3 = staintools.StainNormalizer(method='vahadane')\nnormalizer3.fit(reference_stain)\n\nreference_stain = getItemCustom('999a911f00a8647b3603859bf62c8c11')\nreference_stain = np.uint8(reference_stain)\nreference_stain = cv2.resize(reference_stain,(1024,1024))\nreference_stain = staintools.LuminosityStandardizer.standardize(reference_stain)\nnormalizer4 = staintools.StainNormalizer(method='vahadane')\nnormalizer4.fit(reference_stain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDADataset(Dataset):\n    def __init__(self,\n                 df,\n                 image_size,\n                 n_tiles=n_tiles,\n                 tile_mode=0,\n                 rand=False,\n                 sub_imgs=False,\n                 normalizer = None\n                ):\n\n        self.df = df.reset_index(drop=True)\n        self.image_size = image_size\n        self.n_tiles = n_tiles\n        self.tile_mode = tile_mode\n        self.rand = rand\n        self.sub_imgs = sub_imgs\n        self.normalizer= normalizer\n\n    def __len__(self):\n        return self.df.shape[0]\n\n#     def __getitem__(self, index):\n#         row = self.df.iloc[index]\n#         img_id = row.image_id\n        \n#         tiff_file = os.path.join(image_folder, f'{img_id}.tiff')\n#         image = skimage.io.MultiImage(tiff_file)[1]\n#         tiles, OK = get_tiles(image, self.tile_mode)\n\n#         if self.rand:\n#             idxes = np.random.choice(list(range(self.n_tiles)), self.n_tiles, replace=False)\n#         else:\n#             idxes = list(range(self.n_tiles))\n#         idxes = np.asarray(idxes) + self.n_tiles if self.sub_imgs else idxes\n\n#         n_row_tiles = int(np.sqrt(self.n_tiles))\n#         images = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n#         for h in range(n_row_tiles):\n#             for w in range(n_row_tiles):\n#                 i = h * n_row_tiles + w\n    \n#                 if len(tiles) > idxes[i]:\n#                     this_img = tiles[idxes[i]]['img']\n#                 else:\n#                     this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n#                 #this_img = 255 - this_img\n#                 h1 = h * image_size\n#                 w1 = w * image_size\n#                 images[h1:h1+image_size, w1:w1+image_size] = this_img\n\n#         image_to_transform = np.copy(images)\n#         image_to_transform = np.uint8(image_to_transform)\n#         image_to_transform = cv2.resize(image_to_transform,(1024,1024))\n#         try:\n#             image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)\n#             normalized_image = self.normalizer.transform(image_to_transform)\n#             normalized_image = cv2.resize(normalized_image,(1536,1536))\n#             images = normalized_image\n#         except Exception as e:\n#             print(e)\n#         images = 255 - images\n#         images = images.astype(np.float32)\n#         images /= 255\n#         images = images.transpose(2, 0, 1)\n\n#         return torch.tensor([images,images])\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        img_id = row.image_id\n        \n        tiff_file = os.path.join(image_folder, f'{img_id}.tiff')\n        image = skimage.io.MultiImage(tiff_file)[1]\n        tiles, tiles_count = get_tiles(image, self.tile_mode)\n#         print(tiles_count)\n\n#         if self.rand:\n#             idxes = np.random.choice(list(range(self.n_tiles)), self.n_tiles, replace=False)\n#         else:\n        idxes = list(range(tiles_count))\n        if tiles_count < 36:\n            idxes = list(range(self.n_tiles))\n        if tiles_count > 45:\n            idxes = list(range(self.n_tiles*2))\n        #idxes = np.asarray(idxes) + self.n_tiles if self.sub_imgs else idxes\n        \n#         idxes1,tiles1 = idxes[:36],tiles[:36]\n#         idxes2,tiles2 = idxes[36:],tiles[36:]\n\n        n_row_tiles = int(np.sqrt(self.n_tiles))\n        images = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n        for h in range(n_row_tiles):\n            for w in range(n_row_tiles):\n                i = h * n_row_tiles + w\n    \n                if len(tiles) > idxes[i]:\n                    this_img = tiles[idxes[i]]['img']\n                else:\n                    this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n                #this_img = 255 - this_img\n                h1 = h * image_size\n                w1 = w * image_size\n                images[h1:h1+image_size, w1:w1+image_size] = this_img\n        \n        if tiles_count > 45:\n            #create another image\n            images2 = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n            for h in range(n_row_tiles):\n                for w in range(n_row_tiles):\n                    i = h * n_row_tiles + w\n\n                    if len(tiles) > idxes[i+36]:\n                        this_img = tiles[idxes[i+36]]['img']\n                    else:\n                        this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n                    #this_img = 255 - this_img\n                    h1 = h * image_size\n                    w1 = w * image_size\n                    images2[h1:h1+image_size, w1:w1+image_size] = this_img\n        else:\n            images2 = np.copy(images)\n\n        image_to_transform = np.copy(images)\n        image_to_transform = np.uint8(image_to_transform)\n        image_to_transform = cv2.resize(image_to_transform,(1024,1024))\n        try:\n            image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)\n            normalized_image = self.normalizer.transform(image_to_transform)\n            normalized_image = cv2.resize(normalized_image,(1536,1536))\n            images = normalized_image\n        except Exception as e:\n            print(e)\n        images = 255 - images\n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n        \n        if tiles_count > 45:\n            image_to_transform = np.copy(images2)\n            image_to_transform = np.uint8(image_to_transform)\n            image_to_transform = cv2.resize(image_to_transform,(1024,1024))\n            try:\n                image_to_transform = staintools.LuminosityStandardizer.standardize(image_to_transform)\n                normalized_image = self.normalizer.transform(image_to_transform)\n                normalized_image = cv2.resize(normalized_image,(1536,1536))\n                images2 = normalized_image\n            except Exception as e:\n                print(e)\n            images2 = 255 - images2\n            images2 = images2.astype(np.float32)\n            images2 /= 255\n            images2 = images2.transpose(2, 0, 1)\n        else:\n            images2 = 255 - images2\n            images2 = images2.astype(np.float32)\n            images2 /= 255\n            images2 = images2.transpose(2, 0, 1)\n            \n\n        return torch.tensor([images,images2])#,['1','2']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# if not is_test:\n#     dataset_show = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer)\n#     from pylab import rcParams\n#     rcParams['figure.figsize'] = 20,10\n#     for i in range(2):\n#         f, axarr = plt.subplots(1,5)\n#         for p in range(5):\n#             idx = np.random.randint(0, len(dataset_show))\n#             img = dataset_show[idx]\n#             axarr[p].imshow(1. - img.transpose(0, 1).transpose(1,2).squeeze())\n#             axarr[p].set_title(str(idx))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer)  # mode == 0\nloader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)\n\ndataset2 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer2)  \nloader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)\n\ndataset3 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer3)  \nloader3 = DataLoader(dataset3, batch_size=batch_size, num_workers=num_workers, shuffle=False)\n\ndataset4 = PANDADataset(df, image_size, n_tiles, tile_mode=0,normalizer=normalizer4)  \nloader4 = DataLoader(dataset4, batch_size=batch_size, num_workers=num_workers, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with torch.no_grad():\n#     for data in tqdm(loader):\n#         print(data.shape)\n#         data = data.to(device)\n#         logits = models[0](data)\n#         print(logits.shape)\n#         break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# logits[0],logits[1],logits[2],logits[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# torch.mean(logits,dim=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LOGITS = []\n# with torch.no_grad():\n#     for data in tqdm(loader):\n#         all_logits = []\n#         for i in range(data.shape[0]):\n#             img1,img2 = data[i][0],data[i][1]\n#             logits = models[0](data[i].to(device))\n#             logits = torch.mean(logits,dim=0)\n#             all_logits.append(logits)\n\n#         logits = torch.stack(all_logits)\n#         LOGITS.append(logits)\n        \n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time()\nLOGITS = []\nLOGITS2 = []\nLOGITS3 = []\nLOGITS4 = []\nwith torch.no_grad():\n    for data in tqdm(loader):\n        all_logits = []\n        for i in range(data.shape[0]):\n            img1,img2 = data[i][0],data[i][1]\n            logits = models[0](data[i].to(device))\n            logits = torch.mean(logits,dim=0)\n            all_logits.append(logits)\n\n        logits = torch.stack(all_logits)\n        LOGITS.append(logits)\n\nwith torch.no_grad():\n    for data in tqdm(loader2):\n        all_logits = []\n        for i in range(data.shape[0]):\n            img1,img2 = data[i][0],data[i][1]\n            logits = models[1](data[i].to(device))\n            logits = torch.mean(logits,dim=0)\n            all_logits.append(logits)\n\n        logits = torch.stack(all_logits)\n        LOGITS2.append(logits)\n\nwith torch.no_grad():\n    for data in tqdm(loader3):\n        all_logits = []\n        for i in range(data.shape[0]):\n            img1,img2 = data[i][0],data[i][1]\n            logits = models[2](data[i].to(device))\n            logits = torch.mean(logits,dim=0)\n            all_logits.append(logits)\n\n        logits = torch.stack(all_logits)\n        LOGITS3.append(logits)\n\nwith torch.no_grad():\n    for data in tqdm(loader4):\n        all_logits = []\n        for i in range(data.shape[0]):\n            img1,img2 = data[i][0],data[i][1]\n            logits = models[3](data[i].to(device))\n            logits = torch.mean(logits,dim=0)\n            all_logits.append(logits)\n\n        logits = torch.stack(all_logits)\n        LOGITS4.append(logits)\n        \n# with torch.no_grad():\n#     for data in tqdm(loader4):\n#         data = data.to(device)\n#         logits = models[3](data)\n#         LOGITS4.append(logits)\n\nLOGITS_all = (torch.cat(LOGITS).sigmoid().cpu() + torch.cat(LOGITS2).sigmoid().cpu() + torch.cat(LOGITS3).sigmoid().cpu() + torch.cat(LOGITS4).sigmoid().cpu()) / 4\n#LOGITS_all = (torch.cat(LOGITS).sigmoid().cpu() + torch.cat(LOGITS2).sigmoid().cpu() ) / 2\n# LOGITS_all = torch.cat(LOGITS).sigmoid().cpu()\nPREDS = LOGITS_all.sum(1).round().numpy()\n\ndf['isup_grade'] = PREDS.astype(int)\ndf[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)\nprint('Total time taken',time()-start_time)\nprint(df.head())\nprint()\nprint(df.isup_grade.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}