{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:36.836925Z","iopub.execute_input":"2022-09-18T06:04:36.837636Z","iopub.status.idle":"2022-09-18T06:04:36.8492Z","shell.execute_reply.started":"2022-09-18T06:04:36.837549Z","shell.execute_reply":"2022-09-18T06:04:36.846104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !conda install -y --channel conda-forge pyvips\n!conda install ../input/pyvips-install/pyvips/*.tar.bz2","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:36.851629Z","iopub.execute_input":"2022-09-18T06:04:36.852458Z","iopub.status.idle":"2022-09-18T06:04:43.185383Z","shell.execute_reply.started":"2022-09-18T06:04:36.852424Z","shell.execute_reply":"2022-09-18T06:04:43.184174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nfrom openslide import OpenSlide\nimport tifffile as tiff\nimport os\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:43.187217Z","iopub.execute_input":"2022-09-18T06:04:43.187961Z","iopub.status.idle":"2022-09-18T06:04:43.237582Z","shell.execute_reply.started":"2022-09-18T06:04:43.187918Z","shell.execute_reply":"2022-09-18T06:04:43.23668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\n# test_image_paths = glob.glob(\"../input/mayo-clinic-strip-ai/train/*.tif\")\ntest_image_paths = glob.glob(\"../input/mayo-clinic-strip-ai/test/*.tif\")","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:43.240687Z","iopub.execute_input":"2022-09-18T06:04:43.24104Z","iopub.status.idle":"2022-09-18T06:04:43.247527Z","shell.execute_reply.started":"2022-09-18T06:04:43.241013Z","shell.execute_reply":"2022-09-18T06:04:43.246391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scale = 4\n# output_dir = \"/kaggle/working/\"\n# too_big_for_process = []\n# # for path in test_image_paths[180:200]:\n# for path in test_image_paths:\n\n#     print(path)\n#     slide = OpenSlide(path)\n\n#     if slide.dimensions[0]*slide.dimensions[1] < 4131662535:\n#         image_id = os.path.splitext(os.path.basename(path))[0]\n#         image = tiff.imread(path)\n#         print(f\"{image.shape}\")\n#         cv2.imwrite(os.path.join(output_dir, f\"{image_id}.jpg\"), image[::scale,::scale,::-1])\n#         del image\n#         gc.collect()\n#     else:\n#         print(\"Skip process for avoiding OOM possibility.\")\n#         too_big_for_process.append(path)\n\n# test_image_paths = glob.glob(\"/kaggle/working/*.jpg\") ","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:43.249279Z","iopub.execute_input":"2022-09-18T06:04:43.249627Z","iopub.status.idle":"2022-09-18T06:04:43.255376Z","shell.execute_reply.started":"2022-09-18T06:04:43.249592Z","shell.execute_reply":"2022-09-18T06:04:43.254067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nimport torch\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n# from efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nfrom torchvision import models\nimport torch.nn as nn\nimport timm\nimport torchvision\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform\nimport pyvips\nimport scipy.stats\nimport random\nfrom fastai.vision import *\nfrom fastai.layers import AdaptiveConcatPool2d, Flatten, Mish","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:43.257307Z","iopub.execute_input":"2022-09-18T06:04:43.257749Z","iopub.status.idle":"2022-09-18T06:04:45.217273Z","shell.execute_reply.started":"2022-09-18T06:04:43.257715Z","shell.execute_reply":"2022-09-18T06:04:45.216194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, arch='tf_efficientnetv2_s', n=1, pre=False):\n        super().__init__()\n#         m = torch.hub.load('../input/facebookresearchsemisupervisedimagenet1kmodels/semi-supervised-ImageNet1K-models-master', model=arch, source='local')\n        m = timm.create_model(arch, pretrained=False)\n        self.enc = nn.Sequential(*list(m.children())[:-2])       \n        nc = list(m.children())[-1].in_features\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),Flatten(),nn.Linear(2*nc,512),\n                            Mish(),nn.BatchNorm1d(512), nn.Dropout(0.5),nn.Linear(512,n))\n        \n    def forward(self, x):\n        x = [x for x in x]\n        shape = x[0].shape\n        n = 16\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n        #x: bs*N x 3 x 128 x 128\n        x = self.enc(x)\n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n        #concatenate the output for tiles into a single map\n        x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous()\\\n          .view(-1,shape[1],shape[2]*n,shape[3])\n        #x: bs x C x N*4 x 4\n        x = self.head(x)\n        #x: bs x n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:45.219032Z","iopub.execute_input":"2022-09-18T06:04:45.219604Z","iopub.status.idle":"2022-09-18T06:04:45.230309Z","shell.execute_reply.started":"2022-09-18T06:04:45.219567Z","shell.execute_reply":"2022-09-18T06:04:45.229157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model2(nn.Module):\n    def __init__(self, arch='swinv2_tiny_window16_256', n=1, pre=False):\n        super().__init__()\n#         m = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', arch)\n        m = timm.create_model(arch, pretrained=pre, num_classes=0)\n        self.enc = m\n#         nc = list(m.children())[-1].in_features\n        self.nc=768\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),Flatten(),nn.Linear(2*self.nc,512),\n                            Mish(),nn.BatchNorm1d(512), nn.Dropout(0.5),nn.Linear(512,n))\n        \n    def forward(self, x):\n        x = [x for x in x]\n        shape = x[0].shape\n        n = N\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n        #x: bs*N x 3 x 128 x 128\n        x = self.enc(x)\n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n#         print(x.shape)\n        #concatenate the output for tiles into a single map\n        x =  x.view(-1, 16, self.nc, 1).permute(0,2,1,3).contiguous() # 1024 for swinv2_base_window16_256, 768 for swinv2_tiny_window8_256, 192 for vit_tiny_patch16_384, 384 for vit_small, deit3_small_patch16_384_in21ft1k\n        #x: bs x C x N*4 x 4\n        x = self.head(x)\n        #x: bs x n\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:45.232016Z","iopub.execute_input":"2022-09-18T06:04:45.232468Z","iopub.status.idle":"2022-09-18T06:04:45.242953Z","shell.execute_reply.started":"2022-09-18T06:04:45.232424Z","shell.execute_reply":"2022-09-18T06:04:45.242019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\ntile_sz=384\nsz = 384\nsz2 = 256\nN=64\nims_per_batch = 16","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:45.244432Z","iopub.execute_input":"2022-09-18T06:04:45.244778Z","iopub.status.idle":"2022-09-18T06:04:45.291969Z","shell.execute_reply.started":"2022-09-18T06:04:45.244743Z","shell.execute_reply":"2022-09-18T06:04:45.290775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_class = 2\n# backbone = timm.create_model(model_name, pretrained=False, num_classes=0).to(device)\n# model = nn.Sequential(\n#     backbone,\n#     nn.Dropout(0.2),\n#     nn.Linear(backbone.num_features, n_class)\n# ).to(device)\n\nmodel = Model().to(device)\n\n# model_path = \"../input/mayoinfermodelbfjctp/20220821_models/kaggle/working/models/*.pth\"\nmodel_path = \"../input/mayo-concat-tile-pooling/models/*.pth\"\nmodel_paths = glob.glob(model_path)\n\nmodel_paths","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:45.295476Z","iopub.execute_input":"2022-09-18T06:04:45.295779Z","iopub.status.idle":"2022-09-18T06:04:47.515102Z","shell.execute_reply.started":"2022-09-18T06:04:45.295754Z","shell.execute_reply":"2022-09-18T06:04:47.514163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model2 = Model2().to(device)\nmodel2_path = \"../input/mayo-concat-tile-pooling-transformer/models/*.pth\"\nmodel2_paths = glob.glob(model2_path)\n\nmodel2_paths","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:47.516747Z","iopub.execute_input":"2022-09-18T06:04:47.51718Z","iopub.status.idle":"2022-09-18T06:04:48.130723Z","shell.execute_reply.started":"2022-09-18T06:04:47.517144Z","shell.execute_reply":"2022-09-18T06:04:48.129732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile(img, sz=128, N=64):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    scores = []\n    for im in img:\n        scores.append(len(cv2.imencode(\".jpg\", im)[1]))\n#     idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n#     img = img[idxs]\n    scores, img = zip(*sorted(zip(scores, img), reverse=True, key=lambda x: x[0]))\n    high_info_ind = pd.Series(scores).where(pd.Series(scores) > 30000).idxmin()\n#     print(high_info_ind is not np.nan)\n    \n    bg_ind = pd.Series(scores).where(pd.Series(scores) > 10000).idxmin()\n    bg_cand = img[bg_ind:]\n    for bgi, bg in enumerate(bg_cand):\n        bg = bg.reshape(bg.shape[0] * bg.shape[1], bg.shape[2])\n        white, _ = scipy.stats.mode(bg, axis=0)\n        diff_to_white = (255,255,255) - white\n        if sum(diff_to_white[0]) < 128*3:\n            break\n        else:\n            diff_to_white = [[0,0,0]]\n    \n    if high_info_ind < N or high_info_ind is np.nan:\n        return img[:N], bg_cand[bgi], diff_to_white\n    else:\n        high_info_indexes = random.sample(list(range(high_info_ind)), N)\n        img2 = []\n        for i in high_info_indexes:\n            img2.append(img[i])\n        return img2, bg_cand[bgi], diff_to_white\n\ndef vips2numpy(vi):\n    format_to_dtype = {\n       'uchar': np.uint8,\n       'char': np.int8,\n       'ushort': np.uint16,\n       'short': np.int16,\n       'uint': np.uint32,\n       'int': np.int32,\n       'float': np.float32,\n       'double': np.float64,\n       'complex': np.complex64,\n       'dpcomplex': np.complex128,\n    }\n    return np.ndarray(buffer=vi.write_to_memory(),dtype=format_to_dtype[vi.format],shape=[vi.height, vi.width, vi.bands])\n\ndef return_tiled_images(image_path, transform, N=64, ims_per_batch=16 ,max_size=20000, crop_size=384):\n    image = pyvips.Image.thumbnail(image_path, max_size)\n    image = vips2numpy(image)\n    width, height, c = image.shape\n    print(f\"Input width: {width} height: {height}\")\n    images, bg, diff_to_white = tile(image, sz=crop_size, N=N)\n    output_images = []\n    for idx, img in enumerate(images):\n#         img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n#         img = cv2.imencode(\".jpg\", img, [cv2.IMWRITE_JPEG_QUALITY, 100])[1]\n#         img = cv2.imdecode(img, flags=cv2.IMREAD_COLOR)\n        img = img + diff_to_white\n        img = img / img.max()\n        img = np.clip(img * 255, a_min = 0, a_max = 255).astype(np.uint8)\n        img = transform(img)\n        \n        output_images.append(img)\n    \n    img_indexes = random.sample(list(range(0, N)), N)\n    \n    batched_images = []\n    one_batch = []\n    for i, index in enumerate(img_indexes):\n        one_batch.append(output_images[index])\n        if i%ims_per_batch==(ims_per_batch-1):\n            batched_images.append(torch.stack(one_batch, dim=0))\n            one_batch = []\n\n    batched_images = torch.stack(batched_images, dim=0)\n#     del img, image, images, output_images; gc.collect()\n    return batched_images","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:48.132356Z","iopub.execute_input":"2022-09-18T06:04:48.132997Z","iopub.status.idle":"2022-09-18T06:04:48.153275Z","shell.execute_reply.started":"2022-09-18T06:04:48.132957Z","shell.execute_reply":"2022-09-18T06:04:48.152307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((sz, sz)),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n])\n\npreds = []\n\nfor path in test_image_paths:\n    images = return_tiled_images(image_path=path, transform=transform, N=N, max_size=20000, crop_size=tile_sz)\n    images2 = []\n    for batch in images:\n        images2.append(torchvision.transforms.Resize((sz2, sz2))(batch))\n        \n    images2 = torch.stack(images2, dim=0)\n    images = images.to(device)\n    images2 = images2.to(device)\n\n    \n    for i, m_path in enumerate(model_paths):\n        model.load_state_dict(\n        torch.load(\n            m_path, map_location=device\n            )\n        )\n        model.train(False)\n        with torch.cuda.amp.autocast():\n#             pred = model(torch.unsqueeze(images, 0))\n            pred = model(images)\n\n        pred = torch.sigmoid(pred).to('cpu').detach().numpy().copy()\n\n        pred_ce = ((1 - pred)**2).mean()\n        pred_laa = (pred**2).mean()\n\n        preds.append((path, i, \"model1\", pred_ce, pred_laa))\n\n    for i, m_path in enumerate(model2_paths):\n        model2.load_state_dict(\n        torch.load(\n            m_path, map_location=device\n            )\n        )\n        model2.train(False)\n        \n        with torch.cuda.amp.autocast():\n#             pred = model(torch.unsqueeze(images, 0))\n            pred2 = model2(images2)\n    \n        pred2 = torch.sigmoid(pred2).to('cpu').detach().numpy().copy()\n#         print(pred2)\n\n        pred2_ce = ((1 - pred2)**2).mean()\n        pred2_laa = (pred2**2).mean()\n\n        preds.append((path, i, \"model2\", pred2_ce, pred2_laa))\n\n        \n    del images, images2\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:04:48.157739Z","iopub.execute_input":"2022-09-18T06:04:48.158285Z","iopub.status.idle":"2022-09-18T06:06:21.44017Z","shell.execute_reply.started":"2022-09-18T06:04:48.158238Z","shell.execute_reply":"2022-09-18T06:06:21.438966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if len(too_big_for_process)>0:\n#     for i in too_big_for_process:\n#         preds.append((i, 0, 0.82, 0.28))","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.442313Z","iopub.execute_input":"2022-09-18T06:06:21.442633Z","iopub.status.idle":"2022-09-18T06:06:21.447777Z","shell.execute_reply.started":"2022-09-18T06:06:21.442604Z","shell.execute_reply":"2022-09-18T06:06:21.446826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images = return_tiled_images(image_path=path, transform=transform, N=N, max_size=20000, crop_size=tile_sz)\n# images2 = []\n# for batch in images:\n#     images2.append(torchvision.transforms.Resize((sz2, sz2))(batch))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.449583Z","iopub.execute_input":"2022-09-18T06:06:21.45032Z","iopub.status.idle":"2022-09-18T06:06:21.459087Z","shell.execute_reply.started":"2022-09-18T06:06:21.450281Z","shell.execute_reply":"2022-09-18T06:06:21.458074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data = torch.zeros([4, 16, 3, 256, 256], dtype=torch.float16).to(device)\n# with torch.cuda.amp.autocast():\n#     t = model2(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.46068Z","iopub.execute_input":"2022-09-18T06:06:21.461084Z","iopub.status.idle":"2022-09-18T06:06:21.471017Z","shell.execute_reply.started":"2022-09-18T06:06:21.461049Z","shell.execute_reply":"2022-09-18T06:06:21.470042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.472188Z","iopub.execute_input":"2022-09-18T06:06:21.472538Z","iopub.status.idle":"2022-09-18T06:06:21.480075Z","shell.execute_reply.started":"2022-09-18T06:06:21.472503Z","shell.execute_reply":"2022-09-18T06:06:21.478846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def path_to_patient_id(path):\n    return os.path.basename(path).split(\"_\")[0]\n\ndf = pd.DataFrame(preds, columns=(\"path\", \"fold\", \"model\", \"CE\", \"LAA\"))\ndf[\"patient_id\"] = df[\"path\"].map(path_to_patient_id)","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.481648Z","iopub.execute_input":"2022-09-18T06:06:21.482047Z","iopub.status.idle":"2022-09-18T06:06:21.491706Z","shell.execute_reply.started":"2022-09-18T06:06:21.482012Z","shell.execute_reply":"2022-09-18T06:06:21.490544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:07:58.842621Z","iopub.execute_input":"2022-09-18T06:07:58.843052Z","iopub.status.idle":"2022-09-18T06:07:58.86027Z","shell.execute_reply.started":"2022-09-18T06:07:58.843017Z","shell.execute_reply":"2022-09-18T06:07:58.859283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"patient_id\").mean().drop(\"fold\", axis=1).to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.496102Z","iopub.execute_input":"2022-09-18T06:06:21.497273Z","iopub.status.idle":"2022-09-18T06:06:21.512944Z","shell.execute_reply.started":"2022-09-18T06:06:21.49721Z","shell.execute_reply":"2022-09-18T06:06:21.511736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(\"patient_id\").mean().drop(\"fold\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-18T06:06:21.514741Z","iopub.execute_input":"2022-09-18T06:06:21.51637Z","iopub.status.idle":"2022-09-18T06:06:21.534818Z","shell.execute_reply.started":"2022-09-18T06:06:21.516328Z","shell.execute_reply":"2022-09-18T06:06:21.533806Z"},"trusted":true},"execution_count":null,"outputs":[]}]}