{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is not my dicovery. @**kaggleqrdl** found it and discussed here: https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369684#2057282\n\nI created quick implementation how to speed up dicom processing significantly.\n\nGPU / 500 images (Parallel - 2 jobs):\n- pydicom -> 396.74 sec\n- dicomsdl -> 243.39 sec \n","metadata":{}},{"cell_type":"code","source":"%%capture\n\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-09T09:46:54.572602Z","iopub.execute_input":"2022-12-09T09:46:54.573162Z","iopub.status.idle":"2022-12-09T09:47:06.061096Z","shell.execute_reply.started":"2022-12-09T09:46:54.573113Z","shell.execute_reply":"2022-12-09T09:47:06.059067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nimport cv2\nimport pydicom\n\nfrom joblib import Parallel, delayed\nimport glob\nimport time\nimport numpy as np\nimport os\nfrom matplotlib import pyplot as plt","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-09T09:47:06.064286Z","iopub.execute_input":"2022-12-09T09:47:06.06474Z","iopub.status.idle":"2022-12-09T09:47:06.071926Z","shell.execute_reply.started":"2022-12-09T09:47:06.064696Z","shell.execute_reply":"2022-12-09T09:47:06.070558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir_pydicom = '/kaggle/working/png_file_py/'\nimage_dir_dicomsdl = '/kaggle/working/png_file_dic/'\n\nos.makedirs(image_dir_pydicom, exist_ok=True)\nos.makedirs(image_dir_dicomsdl, exist_ok=True)\n\nIMAGES_TO_PROCESS = 500","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:47:06.07438Z","iopub.execute_input":"2022-12-09T09:47:06.075829Z","iopub.status.idle":"2022-12-09T09:47:06.088599Z","shell.execute_reply.started":"2022-12-09T09:47:06.075705Z","shell.execute_reply":"2022-12-09T09:47:06.087226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm\")\nlen(train_images)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:47:06.091433Z","iopub.execute_input":"2022-12-09T09:47:06.091832Z","iopub.status.idle":"2022-12-09T09:47:20.22646Z","shell.execute_reply.started":"2022-12-09T09:47:06.091796Z","shell.execute_reply":"2022-12-09T09:47:20.225228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512, save_folder=None, dicom_process = True, extension=\"png\"):\n    \n    patient = f.split('/')[-2]\n    image_name = f.split('/')[-1][:-4]\n    if dicom_process:\n        dicom = pydicom.dcmread(f)\n        img = dicom.pixel_array\n\n        img = (img - img.min()) / (img.max() - img.min())\n\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":  \n            img = 1 - img\n            \n        image = (img * 255).astype(np.uint8)\n    else:\n        \n        dicom = dicoml.open(f)\n        img = dicom.pixelData()\n\n        img = (img - img.min()) / (img.max() - img.min())\n\n        if dicom.getPixelDataInfo()['PhotometricInterpretation'] == \"MONOCHROME1\":\n            img = 1 - img\n\n        image = (img * 255).astype(np.uint8)\n    \n    img = cv2.resize(image, (size, size))\n\n    file_name = f'{save_folder}' + f\"{patient}_{image_name}.{extension}\"\n\n    cv2.imwrite(file_name, img)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:47:20.228269Z","iopub.execute_input":"2022-12-09T09:47:20.229562Z","iopub.status.idle":"2022-12-09T09:47:20.244219Z","shell.execute_reply.started":"2022-12-09T09:47:20.229503Z","shell.execute_reply":"2022-12-09T09:47:20.242924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### pydicom - speed test","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n        \nParallel(n_jobs=4)(\n    delayed(process)(f, size = 512, save_folder = image_dir_pydicom, dicom_process = True)\n    for f in train_images[:IMAGES_TO_PROCESS]\n)\n\nprint(time.time() - start_time)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:47:20.245728Z","iopub.execute_input":"2022-12-09T09:47:20.24661Z","iopub.status.idle":"2022-12-09T09:51:03.916903Z","shell.execute_reply.started":"2022-12-09T09:47:20.246569Z","shell.execute_reply":"2022-12-09T09:51:03.915645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_files = glob.glob(f'{image_dir_pydicom}*.png')\nfor idx, i in enumerate(out_files[:4]):\n    im = cv2.imread(i)\n    print(f'file {i} size in bytes: {os.path.getsize(i)}, shape: {im.shape}')\n    plt.imshow(im)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:51:03.918682Z","iopub.execute_input":"2022-12-09T09:51:03.919116Z","iopub.status.idle":"2022-12-09T09:51:04.893519Z","shell.execute_reply.started":"2022-12-09T09:51:03.919076Z","shell.execute_reply":"2022-12-09T09:51:04.892109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dicomsdl - speed test","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n        \nParallel(n_jobs=4)(\n    delayed(process)(f, size = 512, save_folder = image_dir_dicomsdl, dicom_process = False)\n    for f in train_images[:IMAGES_TO_PROCESS]\n)\n\nprint(time.time() - start_time)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:51:04.895124Z","iopub.execute_input":"2022-12-09T09:51:04.896035Z","iopub.status.idle":"2022-12-09T09:53:21.214411Z","shell.execute_reply.started":"2022-12-09T09:51:04.895993Z","shell.execute_reply":"2022-12-09T09:53:21.213043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_files = glob.glob(f'{image_dir_dicomsdl}*.png')\nfor i in out_files[:4]:\n    im = cv2.imread(i)\n    print(f'file {i} size in bytes: {os.path.getsize(i)}, shape: {im.shape}')\n    plt.imshow(im)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-12-09T09:53:21.215632Z","iopub.execute_input":"2022-12-09T09:53:21.216029Z","iopub.status.idle":"2022-12-09T09:53:22.152937Z","shell.execute_reply.started":"2022-12-09T09:53:21.215994Z","shell.execute_reply":"2022-12-09T09:53:22.151668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see there is improvement in speed but further investigation in image quality is needed.","metadata":{}}]}