{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Basic Python and Machine learning libraries\nimport os, sys, random, time, cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport skimage.io\nfrom PIL import Image\nfrom scipy import stats\nfrom IPython.display import display\nfrom tqdm.notebook import tqdm\nimport zipfile\n\nimport albumentations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_path = '../input/prostate-cancer-grade-assessment/train_images/0ab626bb7e1eb8c7b1eb6ac3fb129349.tiff'\nimage = skimage.io.MultiImage(img_path)[1]\nplt.figure(figsize=(10,10))\n# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nplt.imshow(image)\nprint(type(image), image.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !mkdir  -p /root/.kaggle/\n# !cp ../input/kaggle-api/kaggle.json /root/.kaggle\n# os.environ['KAGGLE_CONFIG_DIR'] = \"/root/.kaggle\"\n# !chmod 600 /root/.kaggle/kaggle.json\n\n\n!mkdir -p ~/.kaggle\n!cp ../input/tokken-api/kaggle.json ~/.kaggle/kaggle.json\n!chmod 600 ~/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #create ds folder:\n# !mkdir /kaggle/working/proc-prostate-cancer-grade-assessment\n# #init ds\n# !kaggle datasets metadata -p /kaggle/working/proc-prostate-cancer-grade-assessment feascr/proc-prostate-cancer-grade-assessment\n\n\n# #create ds\n\n\n# # with open('/kaggle/working/proc-prostate-cancer-grade-assessment/somefile.txt', 'w') as f:\n# #     f.write('Hello\\n')\n# # !kaggle datasets create -p /kaggle/working/proc-prostate-cancer-grade-assessment","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p /tmp/panda_dataset\n!ls /tmp/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_metadata = '''{\n  \"title\": \"PANDA dataset medium 256_64_64\",\n  \"id\": \"blablamc/panda-dataset-medium-256-64-64\",\n  \"licenses\": [\n    {\n      \"name\": \"CC0-1.0\"\n    }\n  ]\n}\n'''\nwith open(\"/tmp/panda_dataset/dataset-metadata.json\", 'w+') as text_file:\n    n = text_file.write(dataset_metadata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    pwd = '/kaggle/working/'\n    data_dir = '../input/prostate-cancer-grade-assessment/'\n    train_images_dir = '../input/prostate-cancer-grade-assessment/train_images/'\n    train_masks_dir = '../input/prostate-cancer-grade-assessment/train_label_masks/'\n    image_size = 64\n    tile_size = 64\n    tile_mode = 0\n    n_tiles = 256\n    seed = 2020\n    out_train = '/tmp/panda_dataset/train_images.zip'\n    out_masks = '/tmp/panda_dataset/train_label_masks.zip'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\nseed_everything(Config.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(Config.data_dir+'train.csv')\ndisplay(train_df.head())\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tiles(img, mask, tile_size, n_tiles, mode=0):\n    result = []\n    h, w, c = img.shape\n    pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n    pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n\n    img = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)\n    img = img.reshape(\n            img.shape[0] // tile_size,\n            tile_size,\n            img.shape[1] // tile_size,\n            tile_size,\n            3\n        )\n    img = img.transpose(0,2,1,3,4).reshape(-1, tile_size, tile_size,3)\n    \n    \n    if mask is not None:\n        mask = np.pad(mask,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=0)\n        mask = mask.reshape(\n                mask.shape[0] // tile_size,\n                tile_size,\n                mask.shape[1] // tile_size,\n                tile_size,\n                3\n            )\n        mask = mask.transpose(0,2,1,3,4).reshape(-1, tile_size, tile_size,3)\n    \n    if len(img) < n_tiles:\n        img = np.pad(img,[[0,n_tiles-len(img)],[0,0],[0,0],[0,0]], constant_values=255)\n        if mask is not None:\n            mask = np.pad(mask,[[0,n_tiles-len(mask)],[0,0],[0,0],[0,0]], constant_values=0)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:n_tiles]\n    img = img[idxs]\n    if mask is not None:\n        mask = mask[idxs]\n    for i in range(len(img)):\n        if mask is not None:\n            result.append({'img':img[i], 'mask':mask[i], 'idx':i})\n        else:\n            result.append({'img':img[i], 'idx':i})\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile(Config.out_train, 'w') as img_out,\\\n        zipfile.ZipFile(Config.out_masks, 'w') as mask_out:\n    for i in tqdm(range(len(train_df))):\n        img_path = os.path.join(Config.train_images_dir, train_df['image_id'].values[i]) + '.tiff'\n        mask_path = os.path.join(Config.train_masks_dir, train_df['image_id'].values[i]) + '_mask.tiff'\n        img = skimage.io.MultiImage(img_path)[1]\n        mask = skimage.io.MultiImage(mask_path)\n        mask_not_exist = not mask\n        if mask_not_exist:\n            tiles = get_tiles(img, None, Config.tile_size, Config.n_tiles, Config.tile_mode)\n        else:\n            mask = mask[1]\n            tiles = get_tiles(img, mask, Config.tile_size, Config.n_tiles, Config.tile_mode)\n        for t in tiles:\n            if mask_not_exist:\n                img, mask, idx = t['img'], None, t['idx']\n            else:\n                img, mask, idx = t['img'],t['mask'],t['idx']\n            #if read with PIL RGB turns into BGR\n            img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f\"{train_df['image_id'].values[i]}_{idx}.png\", img)\n            if not mask_not_exist:\n            # TO DO SAVE IMG TO FOLDER VIA CV OR PIL + ADD KAGGLE DATASET SUPPORT\n                mask = cv2.imencode('.png',mask[:,:,0])[1]\n                mask_out.writestr(f\"{train_df['image_id'].values[i]}_mask_{idx}.png\", mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!unzip -q /tmp/panda_dataset/train_images.zip -d /tmp/train_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = os.listdir(\"/tmp/train_images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = set([file.split(\"_\")[0] for file in train_files])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\n\nif len(files) != train_df.shape[0]:\n    print('YES')\n    train_df = train_df[df.image_id.isin(files)]\n    train_df.to_csv(\"/tmp/panda_dataset/train.csv\", index = False)\nelse:\n    shutil.copy2(\"../input/prostate-cancer-grade-assessment/train.csv\", \"/tmp/panda_dataset/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l /tmp/panda_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!kaggle datasets create -p /tmp/panda_dataset -u","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /tmp/train_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FOR DOWNLOADING FILES FROM DRIVE\n# import torchvision\n# torchvision.datasets.utils.download_file_from_google_drive('https://drive.google.com/file/d/1Ex-kv2F9yF15DREprw1fVLTqrb2xa1v6/view?usp=sharing', '/kaggle/working', filename='kaggle.json', md5=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # !mkdir  -p /root/.kaggle/\n# !mkdir ~/.kaggle\n# !cp /kaggle/working/kaggle.json ~/.kaggle/kaggle.json\n# !chmod 600 ~/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MIGHT HELP\n# !echo '{\"username\":\"feascr\",\"key\":\"07fa08279a5745a468798a76cb4dbfb7\"}' > ~/.kaggle/kaggle.json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}