{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport skimage.io\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport pandas as pd\nimport numpy as np\nimport shutil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /root/.kaggle/\n!cp ../input/my-kaggle-api/kaggle.json /root/.kaggle/\n!chmod 600 /root/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/panda_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /tmp/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = '''{\n  \"title\": \"Panda_Dataset_medium_25_256_256\",\n  \"id\": \"raghaw/panda-dataset-medium-25-256-256\",\n  \"licenses\": [\n    {\n      \"name\": \"CC0-1.0\"\n    }\n  ]\n}\n'''\ntext_file = open(\"/tmp/panda_dataset/dataset-metadata.json\", 'w+')\nn = text_file.write(data)\ntext_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN = '../input/prostate-cancer-grade-assessment/train_images/'\nMASKS = '../input/prostate-cancer-grade-assessment/train_label_masks/'\nOUT_TRAIN = '/tmp/panda_dataset/train_medium_25_256_256.zip'\nOUT_MASKS = '/tmp/panda_dataset/masks_medium_25_256_256.zip'\nsz = 256\nN = 25","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tile(img, mask):\n    result = []\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=0)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz,3)\n    mask = mask.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        mask = np.pad(mask,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=0)\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    img = img[idxs]\n    mask = mask[idxs]\n    for i in range(len(img)):\n        result.append({'img':img[i], 'mask':mask[i], 'idx':i})\n    return result","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\nnames = [name[:-5] for name in os.listdir(TRAIN)]\nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for name in tqdm(names):\n        img = skimage.io.MultiImage(os.path.join(TRAIN,name+'.tiff'))[1]\n        mask_exist = os.path.isfile(os.path.join(MASKS,name+'_mask.tiff'))\n        if mask_exist:\n            mask = skimage.io.MultiImage(os.path.join(MASKS,name+'_mask.tiff'))[1]\n        else:\n            mask = np.zeros_like(img)\n        tiles = tile(img,mask)\n        for t in tiles:\n            img,mask,idx = t['img'],t['mask'],t['idx']\n            x_tot.append((img/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n            #if read with PIL RGB turns into BGR\n            img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{name}_{idx}.png', img)\n            if mask_exist:\n                mask = cv2.imencode('.png',mask[:,:,0])[1]\n                mask_out.writestr(f'{name}_{idx}.png', mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', np.sqrt(img_std))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!sleep 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip -q /tmp/panda_dataset/train_medium_25_256_256.zip -d /tmp/train_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = os.listdir(\"/tmp/train_images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = set([file.split(\"_\")[0] for file in train_files])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/prostate-cancer-grade-assessment/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if len(files) != df.shape[0]:\n    df = df[df.image_id.isin(files)]\n    df.to_csv(\"/tmp/panda_dataset/train.csv\", index = False)\nelse:\n    shutil.copy2(\"../input/prostate-cancer-grade-assessment/train.csv\", \"/tmp/panda_dataset/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!sleep 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l /tmp/panda_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!kaggle datasets create -p /tmp/panda_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /tmp/train_images","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}