{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport openslide\nfrom openslide import OpenSlide\nimport cv2 \nimport tqdm\nfrom tqdm import tqdm\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-09-03T03:56:00.678719Z","iopub.execute_input":"2022-09-03T03:56:00.67922Z","iopub.status.idle":"2022-09-03T03:56:06.6823Z","shell.execute_reply.started":"2022-09-03T03:56:00.679162Z","shell.execute_reply":"2022-09-03T03:56:06.681397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/mayo-clinic-strip-ai/train.csv')\ntest = pd.read_csv('../input/mayo-clinic-strip-ai/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-03T03:53:00.240651Z","iopub.execute_input":"2022-09-03T03:53:00.241063Z","iopub.status.idle":"2022-09-03T03:53:00.266839Z","shell.execute_reply.started":"2022-09-03T03:53:00.241031Z","shell.execute_reply":"2022-09-03T03:53:00.265975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"image_path\"] = train[\"image_id\"].apply(lambda x: \"../input/mayo-clinic-strip-ai/train/\" + x + \".tif\")\ntest[\"image_path\"]  = test[\"image_id\"].apply(lambda x: \"../input/mayo-clinic-strip-ai/test/\" + x + \".tif\")","metadata":{"execution":{"iopub.status.busy":"2022-09-03T03:54:18.039202Z","iopub.execute_input":"2022-09-03T03:54:18.039759Z","iopub.status.idle":"2022-09-03T03:54:18.060159Z","shell.execute_reply.started":"2022-09-03T03:54:18.03969Z","shell.execute_reply":"2022-09-03T03:54:18.058842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(image_path,size_of_image):\n    slide=OpenSlide(image_path)\n    region= (1000,1000)    \n    size  = (5000, 5000)\n    image = slide.read_region(region, 0, size)\n    image = tf.image.resize(image, (size_of_image, size_of_image))\n    image = np.array(image)    \n    return image\n","metadata":{"execution":{"iopub.status.busy":"2022-09-03T03:52:52.76959Z","iopub.execute_input":"2022-09-03T03:52:52.770029Z","iopub.status.idle":"2022-09-03T03:52:52.776486Z","shell.execute_reply.started":"2022-09-03T03:52:52.769997Z","shell.execute_reply":"2022-09-03T03:52:52.775655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_size_256x256=[]\n\nfor i in tqdm(train['image_path']):\n    img=preprocess(i,256)\n    train_images_size_256x256.append(img)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T03:56:19.863015Z","iopub.execute_input":"2022-09-03T03:56:19.863408Z","iopub.status.idle":"2022-09-03T04:43:28.868367Z","shell.execute_reply.started":"2022-09-03T03:56:19.863376Z","shell.execute_reply":"2022-09-03T04:43:28.866428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comparing tqdm with pqdm and seeing which is better for use case","metadata":{}},{"cell_type":"code","source":"pip install pqdm","metadata":{"execution":{"iopub.status.busy":"2022-09-03T04:43:28.871903Z","iopub.execute_input":"2022-09-03T04:43:28.873116Z","iopub.status.idle":"2022-09-03T04:43:43.755894Z","shell.execute_reply.started":"2022-09-03T04:43:28.873061Z","shell.execute_reply":"2022-09-03T04:43:43.754724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pqdm.processes import pqdm","metadata":{"execution":{"iopub.status.busy":"2022-09-03T04:43:43.758202Z","iopub.execute_input":"2022-09-03T04:43:43.758606Z","iopub.status.idle":"2022-09-03T04:43:43.792218Z","shell.execute_reply.started":"2022-09-03T04:43:43.758566Z","shell.execute_reply":"2022-09-03T04:43:43.791241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def job_256x256(image_path):\n    train_images_size_256x256=[]\n    img=preprocess(image_path,256)\n    train_images_size_256x256.append(img)\n    return train_images_size_256x256\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:01:14.207188Z","iopub.execute_input":"2022-09-03T05:01:14.207667Z","iopub.status.idle":"2022-09-03T05:01:14.214631Z","shell.execute_reply.started":"2022-09-03T05:01:14.207629Z","shell.execute_reply":"2022-09-03T05:01:14.213242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_256x256 = pqdm(train['image_path'], job_256x256, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:01:14.925272Z","iopub.execute_input":"2022-09-03T05:01:14.9258Z","iopub.status.idle":"2022-09-03T05:13:17.866424Z","shell.execute_reply.started":"2022-09-03T05:01:14.925754Z","shell.execute_reply":"2022-09-03T05:13:17.865143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(train_images_size_256x256[0])","metadata":{"execution":{"iopub.status.busy":"2022-09-03T04:59:58.68869Z","iopub.execute_input":"2022-09-03T04:59:58.689167Z","iopub.status.idle":"2022-09-03T04:59:58.892927Z","shell.execute_reply.started":"2022-09-03T04:59:58.68913Z","shell.execute_reply":"2022-09-03T04:59:58.891728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_256x256 = np.array(images_256x256).reshape(754,256,256,4)\nimages_256x256.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:16:14.18318Z","iopub.execute_input":"2022-09-03T05:16:14.183612Z","iopub.status.idle":"2022-09-03T05:16:14.50927Z","shell.execute_reply.started":"2022-09-03T05:16:14.183578Z","shell.execute_reply":"2022-09-03T05:16:14.508298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(images_256x256[0])","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:16:29.826368Z","iopub.execute_input":"2022-09-03T05:16:29.826842Z","iopub.status.idle":"2022-09-03T05:16:30.083606Z","shell.execute_reply.started":"2022-09-03T05:16:29.826805Z","shell.execute_reply":"2022-09-03T05:16:30.082093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We can clearly see the pqdm is much faster than tqdm","metadata":{}},{"cell_type":"code","source":"test_images_256x256 = pqdm(test['image_path'], job_256x256, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:20:14.245543Z","iopub.execute_input":"2022-09-03T05:20:14.254266Z","iopub.status.idle":"2022-09-03T05:20:27.045693Z","shell.execute_reply.started":"2022-09-03T05:20:14.253833Z","shell.execute_reply":"2022-09-03T05:20:27.044571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset Size 512x512 and 1024x1024","metadata":{}},{"cell_type":"code","source":"def job_512x512(image_path):\n    train_images_size_512x512=[]\n    img=preprocess(image_path,512)\n    train_images_size_512x512.append(img)\n    train_images_size_512x512 = np.array(train_images_size_512x512).reshape(754,512,512,4)\n    return train_images_size_512x512","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:22:04.276137Z","iopub.execute_input":"2022-09-03T05:22:04.276913Z","iopub.status.idle":"2022-09-03T05:22:04.293727Z","shell.execute_reply.started":"2022-09-03T05:22:04.276846Z","shell.execute_reply":"2022-09-03T05:22:04.290869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_512x512 = pqdm(train['image_path'], job_512x512, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:23:03.28011Z","iopub.execute_input":"2022-09-03T05:23:03.280641Z","iopub.status.idle":"2022-09-03T05:40:27.822571Z","shell.execute_reply.started":"2022-09-03T05:23:03.280602Z","shell.execute_reply":"2022-09-03T05:40:27.82159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images_512x512 = pqdm(test['image_path'], job_512x512, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:45:47.722081Z","iopub.execute_input":"2022-09-03T05:45:47.72261Z","iopub.status.idle":"2022-09-03T05:45:56.918613Z","shell.execute_reply.started":"2022-09-03T05:45:47.72257Z","shell.execute_reply":"2022-09-03T05:45:56.917088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def job_1024x1024(image_path):\n    train_images_size_1024x1024=[]\n    img=preprocess(image_path,512)\n    train_images_size_1024x1024.append(img)\n    train_images_size_1024x1024 = np.array(train_images_size_1024x1024).reshape(754,1024,1024,4)\n    return train_images_size_1024x1024","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:45:56.920978Z","iopub.execute_input":"2022-09-03T05:45:56.921374Z","iopub.status.idle":"2022-09-03T05:45:56.928641Z","shell.execute_reply.started":"2022-09-03T05:45:56.921338Z","shell.execute_reply":"2022-09-03T05:45:56.927049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_1024x1024 = pqdm(train['image_path'], job_1024x1024, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:45:56.93036Z","iopub.execute_input":"2022-09-03T05:45:56.930791Z","iopub.status.idle":"2022-09-03T05:57:50.26988Z","shell.execute_reply.started":"2022-09-03T05:45:56.93073Z","shell.execute_reply":"2022-09-03T05:57:50.268026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images_1024x1024 = pqdm(test['image_path'], job_1024x1024, n_jobs=4)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T05:57:50.27346Z","iopub.execute_input":"2022-09-03T05:57:50.273924Z","iopub.status.idle":"2022-09-03T05:57:56.831433Z","shell.execute_reply.started":"2022-09-03T05:57:50.273883Z","shell.execute_reply":"2022-09-03T05:57:56.829757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_256x256 = {'image_256x256': images_256x256}\nnp.save('images_256x256',image_256x256)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:03:52.819697Z","iopub.execute_input":"2022-09-03T06:03:52.821066Z","iopub.status.idle":"2022-09-03T06:03:59.128299Z","shell.execute_reply.started":"2022-09-03T06:03:52.821018Z","shell.execute_reply":"2022-09-03T06:03:59.127178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_512x512 = {'image_512x512': np.asarray(images_512x512)}\nnp.save('images_512x512',image_512x512)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:03:59.130078Z","iopub.execute_input":"2022-09-03T06:03:59.131162Z","iopub.status.idle":"2022-09-03T06:03:59.206418Z","shell.execute_reply.started":"2022-09-03T06:03:59.131122Z","shell.execute_reply":"2022-09-03T06:03:59.205125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_1024x1024 = {'image_1024x1024': images_1024x1024}\nnp.save('images_1024x1024',image_1024x1024)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:04:03.437797Z","iopub.execute_input":"2022-09-03T06:04:03.438274Z","iopub.status.idle":"2022-09-03T06:04:03.501693Z","shell.execute_reply.started":"2022-09-03T06:04:03.438235Z","shell.execute_reply":"2022-09-03T06:04:03.500682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-03T03:49:19.077353Z","iopub.execute_input":"2022-09-03T03:49:19.078082Z"}}},{"cell_type":"code","source":"test_image_256x256 = {'test_image_256x256': test_images_256x256}\nnp.save('test_images_256x256',test_image_256x256)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:04:08.939724Z","iopub.execute_input":"2022-09-03T06:04:08.940224Z","iopub.status.idle":"2022-09-03T06:04:08.973727Z","shell.execute_reply.started":"2022-09-03T06:04:08.940182Z","shell.execute_reply":"2022-09-03T06:04:08.972337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_512x512 = {'test_image_512x512': test_images_512x512}\nnp.save('test_images_512x512',test_image_512x512)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:04:14.16777Z","iopub.execute_input":"2022-09-03T06:04:14.168257Z","iopub.status.idle":"2022-09-03T06:04:14.176257Z","shell.execute_reply.started":"2022-09-03T06:04:14.168215Z","shell.execute_reply":"2022-09-03T06:04:14.174476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image_1024x1024 = {'test_image_1024x1024': test_images_1024x1024}\nnp.save('test_images_1024x1024',test_image_1024x1024)","metadata":{"execution":{"iopub.status.busy":"2022-09-03T06:04:18.835061Z","iopub.execute_input":"2022-09-03T06:04:18.836001Z","iopub.status.idle":"2022-09-03T06:04:18.843319Z","shell.execute_reply.started":"2022-09-03T06:04:18.835953Z","shell.execute_reply":"2022-09-03T06:04:18.842021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}