{"cells":[{"metadata":{},"cell_type":"markdown","source":"The code below selects 20 256x256 tiles for each image and mask based on the maximum number of infected tissue pixels in masks. The kernel also provides computed image stats. Please check my kernels to see how to use this data. \n![](https://i.ibb.co/RzSWP56/convert.png)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, gc\nimport cv2\nimport skimage.io\nimport openslide\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport pandas as pd\nimport numpy as np\nimport cv2\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport PIL\nfrom IPython.display import Image, display","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN = '../input/prostate-cancer-grade-assessment/train_images/'\nMASKS = '../input/prostate-cancer-grade-assessment/train_label_masks/'\nOUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'\nsz = 256\nN = 16\nTIFF_LEVEL = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"names = [name[:-10] for name in os.listdir(MASKS)]\n# i = 9\n# pos = (int) (len(names) / 11)\n# start = (pos*i)\n# end = (pos*(i+1))\n# names = names[ start : end]\n# len(names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"done_names = ['4cbde67c6d4feb90b93497fa08b413f7']\nnames = [x for x in names if x in done_names]\nlen(names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# names = [x for x in names if x not in done_names]\n# print(len(names))\n# names = names[0:1856]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(ID, path, level=2):\n    image_path = path + ID + '.tiff'\n    img = skimage.io.MultiImage(image_path)\n    img = img[level]\n    return img\n\ndef read_mask_img(ID, path, level=2):\n    image_path = path + ID + '_mask.tiff'\n    img = skimage.io.MultiImage(image_path)\n    img = img[level]\n    return img\n\ndef img_to_gray(img):\n    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    return img_gray\n\ndef mask_img(img_gray, tol=210):\n    mask = (img_gray < tol).astype(np.uint8)\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_mask = read_mask_img(names[0], MASKS)\n# img_mask = img_mask * 255\n\nimg1 = read_img(names[0], TRAIN)\nimg_gray1 = img_to_gray(img1)\nmask1 = mask_img(img_gray1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, ax = plt.subplots(3, 2, figsize=(20, 25))\n\n# for i, img in enumerate(names[:1]):\n#     image = skimage.io.MultiImage(os.path.join(TRAIN,names[i]+'.tiff'))[TIFF_LEVEL]\n#     mask = openslide.OpenSlide(os.path.join(MASKS,names[i]+'_mask.tiff'))\n#     ax[i][0].imshow(image)\n#     ax[i][0].set_title(f'{img} - Original - Label: {names[i]}')\n    \n#     mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n#     cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n        \n#     ax[i][1].imshow(np.asarray(mask_data)[:,:,0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5)\n#     ax[i][1].set_title(f'{img} - Mask - Label: {names[i]}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tile(img, mask = 0, idxs = None, index = 0):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    if mask == 0:\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                    constant_values=255)\n    else:\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                    constant_values=0)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        if mask == 0:\n            img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n        else:\n            img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=0)\n    if index == 0:\n        idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[::-1][:N]\n    \n    img = img[idxs]\n    return img, idxs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\n# names = [name[:-10] for name in os.listdir(MASKS)]\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for name in tqdm(names):\n        masks, idxs = tile(skimage.io.MultiImage(os.path.join(MASKS,name+'_mask.tiff'))[TIFF_LEVEL], mask = 1)\n#         for i in range(N):\n#             mask = cv2.imencode('.png',masks[i][:,:,0])[1]\n#             mask_out.writestr(f'{name}_{i}.png', mask)\n#             mask = None\n#             gc.collect()\n        masks = None\n        gc.collect()\n        \n        imgs, idxs = tile(skimage.io.MultiImage(os.path.join(TRAIN,name+'.tiff'))[TIFF_LEVEL], idxs = idxs, index = 1)\n        for i in range(N):\n            x_tot.append((imgs[i]/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((imgs[i]/255.0)**2).reshape(-1,3).mean(0))\n            #if read with PIL RGB turns into BGR\n            img = cv2.imencode('.png',cv2.cvtColor(imgs[i], cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{name}_{i}.png', img)\n            img = None\n            gc.collect()\n            \n        imgs = None\n        idxs = None\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', np.sqrt(img_std))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}