{"cells":[{"metadata":{},"cell_type":"markdown","source":"Thanks to [PANDA: Resize and Save Train Data](https://www.kaggle.com/xhlulu/panda-resize-and-save-train-data) by xhulu. \n\nPlease upvote this amazing reference kernel."},{"metadata":{},"cell_type":"markdown","source":"### Dependencies"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom PIL import Image\nimport cv2\nfrom tqdm.notebook import tqdm\nimport skimage.io\nfrom skimage.transform import resize, rescale\nimport openslide","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset Preparation"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment/train_images/'\nmask_dir = '../input/prostate-cancer-grade-assessment/train_label_masks/'\nmask_files = os.listdir(mask_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_id = train_labels.image_id[0]\npath = data_dir + img_id + '.tiff'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Performance Check"},{"metadata":{"trusted":true},"cell_type":"code","source":"%time biopsy = openslide.OpenSlide(path)\n%time biopsy2 = skimage.io.MultiImage(path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%timeit img = biopsy.get_thumbnail(size=(331, 331))\n%timeit out = resize(biopsy2[-1], (331, 331))\n%timeit out = cv2.resize(biopsy2[-1], (331, 331))\n%timeit out = Image.fromarray(biopsy2[-1]).resize((331, 331))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out = cv2.resize(biopsy2[-1],(331,331))\n%timeit Image.fromarray(out).save(img_id+'.png')\n%timeit cv2.imwrite(img_id+'.png',out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = skimage.io.MultiImage(mask_dir + mask_files[1])\nimg = skimage.io.MultiImage(data_dir + mask_files[1].replace(\"_mask\", \"\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask[-1].shape, img[-1].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Resize Images and Save Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"save_dir = \"/kaggle/train_images/\"\nos.makedirs(save_dir, exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for img_id in tqdm(train_labels.image_id):\n    load_path = data_dir + img_id + '.tiff'\n    save_path = save_dir + img_id + '.png'\n    \n    biopsy = skimage.io.MultiImage(load_path)\n    img = cv2.resize(biopsy[-1], (331, 331))\n    cv2.imwrite(save_path, img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"save_mask_dir = '/kaggle/train_label_masks/'\nos.makedirs(save_mask_dir, exist_ok=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for mask_file in tqdm(mask_files):\n    load_path = mask_dir + mask_file\n    save_path = save_mask_dir + mask_file.replace('.tiff', '.png')\n    \n    mask = skimage.io.MultiImage(load_path)\n    img = cv2.resize(mask[-1], (331, 331))\n    cv2.imwrite(save_path, img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Convert to TF-Record and Save Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_root = \"/kaggle/train_images/\"\n\ntf_record_dir = os.path.join(data_root, \"kaggle/tfrecord_data/\")\ntf_record_array_dir = os.path.join(data_root, \"kaggle/tfrecord_array_data/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def write_to_tfrecords(decoded_resolution=None):\n    if decoded_resolution:\n        record_dir = os.path.join(tf_record_array_dir, str(decoded_resolution))\n    else:\n        record_dir = tf_record_dir\n\n    if os.path.exists(record_dir):\n        return\n    os.makedirs(record_dir, exist_ok=True)\n\n    print(\"Converting images to TFRecords...\")\n    records_per_shard = 50\n\n    shard_number = 0\n    path_template = os.path.join(record_dir, \"shard_{0:04d}.tfrecords\")\n    writer = tf.io.TFRecordWriter(path_template.format(shard_number))\n    for i, (image_path, label) in enumerate(get_paths_and_labels()):\n        if i and not (i % records_per_shard):\n            shard_number += 1\n            writer.close()\n            writer = tf.io.TFRecordWriter(path_template.format(shard_number))\n\n    with open(image_path, \"rb\") as f:\n        image_bytes = f.read()\n\n    if decoded_resolution:\n        image = tf.io.decode_png(image_bytes)\n        image = tf.cast(image, tf.float32)\n        image = tf.image.resize(image, (decoded_resolution,) * 2)\n        if image.shape[2] == 1:\n            image = tf.tile(image, (1, 1, 3))\n            image_bytes = tf.io.encode_jpeg(tf.cast(image, tf.uint8)).numpy()\n\n    record_bytes = tf.train.Example(features=tf.train.Features(feature={\n        \"image\": tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_bytes])),\n        \"label\": tf.train.Feature(int64_list=tf.train.Int64List(value=[label]))\n    })).SerializeToString()\n\n    writer.write(record_bytes)\n\n    writer.close()\n    print(\"TFRecord conversion complete.\")\n\n\nRECORD_PATTERN = os.path.join(tf_record_dir, \"*.tfrecords\")\nRESIZED_RECORD_PATTERN = os.path.join(tf_record_array_dir, \"{}\", \"*.tfrecords\")\nRECORD_SCHEMA = {\n    \"image\": tf.io.FixedLenFeature([], dtype=tf.string),\n    \"label\": tf.io.FixedLenFeature([1], dtype=tf.int64)\n}\n\n#write_to_tfrecords()\n\nassert RESOLUTION[0] == RESOLUTION[1], \"Resize is hard coded to square images.\"\nwrite_to_tfrecords(RESOLUTION[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!tar -czf train_images.tar.gz ../train_images/*.png\n!tar -czf train_label_masks.tar.gz ../train_label_masks/*.png","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}