{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob\nfrom openslide import OpenSlide\nimport tifffile as tiff\nimport cv2\nimport gc\nimport os\nimport pandas as pd\n\nfiles = glob.glob(\"../input/mayo-clinic-strip-ai/train/*.tif\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-16T11:56:16.026519Z","iopub.execute_input":"2022-08-16T11:56:16.027717Z","iopub.status.idle":"2022-08-16T11:56:16.470756Z","shell.execute_reply.started":"2022-08-16T11:56:16.027625Z","shell.execute_reply":"2022-08-16T11:56:16.469427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We can inspect image pixel size without loading image on RAM by using openslide","metadata":{}},{"cell_type":"code","source":"df = []\nfor file in files:\n    slide = OpenSlide(file)\n    filesize = os.path.getsize(file)\n    df.append((file, slide.dimensions[0], slide.dimensions[1], slide.dimensions[0]*slide.dimensions[1], filesize))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T11:56:16.475357Z","iopub.execute_input":"2022-08-16T11:56:16.475706Z","iopub.status.idle":"2022-08-16T11:56:36.05836Z","shell.execute_reply.started":"2022-08-16T11:56:16.475676Z","shell.execute_reply":"2022-08-16T11:56:36.056887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(df, columns=[\"path\", \"width\", \"height\", \"pixel_size\", \"file_size\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-16T11:56:36.060759Z","iopub.execute_input":"2022-08-16T11:56:36.061221Z","iopub.status.idle":"2022-08-16T11:56:36.076121Z","shell.execute_reply.started":"2022-08-16T11:56:36.061147Z","shell.execute_reply":"2022-08-16T11:56:36.074757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sort_values('pixel_size', ascending=False).reset_index()\ndf.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T11:56:36.081489Z","iopub.execute_input":"2022-08-16T11:56:36.082649Z","iopub.status.idle":"2022-08-16T11:56:36.120701Z","shell.execute_reply.started":"2022-08-16T11:56:36.082499Z","shell.execute_reply":"2022-08-16T11:56:36.119198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"filesize_inspection.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-16T11:56:36.122684Z","iopub.execute_input":"2022-08-16T11:56:36.123128Z","iopub.status.idle":"2022-08-16T11:56:36.136686Z","shell.execute_reply.started":"2022-08-16T11:56:36.123084Z","shell.execute_reply":"2022-08-16T11:56:36.135129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ↓These files can be read by tifffile.","metadata":{}},{"cell_type":"code","source":"import tqdm\nscale = 4\n\nfor path in tqdm.tqdm(df[\"path\"][1:]):\n    # This downsample method is reffered from https://www.kaggle.com/code/tmyok1984/mayo-convert-tif-to-jpg, thanks!\n    image = tiff.imread(path)\n    cv2.imwrite(\"test.jpg\", image[::scale,::scale,::-1])\n    del image\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T11:56:36.139083Z","iopub.execute_input":"2022-08-16T11:56:36.139634Z","iopub.status.idle":"2022-08-16T15:30:46.489627Z","shell.execute_reply.started":"2022-08-16T11:56:36.139591Z","shell.execute_reply":"2022-08-16T15:30:46.488084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ↓This file cannot be read by tifffile!!","metadata":{}},{"cell_type":"code","source":"df[\"path\"][0]","metadata":{"execution":{"iopub.status.busy":"2022-08-16T15:30:46.494816Z","iopub.execute_input":"2022-08-16T15:30:46.495126Z","iopub.status.idle":"2022-08-16T15:30:46.509982Z","shell.execute_reply.started":"2022-08-16T15:30:46.495097Z","shell.execute_reply":"2022-08-16T15:30:46.508479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image = tiff.imread(df[\"path\"][0])\n# cv2.imwrite(\"test.jpg\", image[::scale,::scale,::-1])\n# del image\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T15:30:46.512024Z","iopub.execute_input":"2022-08-16T15:30:46.512714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Therefore, it may be possible to prevent OOM by skipping processing when pixel_size is `4131662535` or larger at the time of submission.\n## ※Strictly speaking, the amount of used memory and pixel_size do not correlate perfectly, and the amount of memory used varies depending on the process being performed, so `4131662535` is just a guideline.","metadata":{}}]}