{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  DICOM Image Read Benchmark Testing\n\n## DICOM : Digital Imaging and Communication in Medicine\n\nDigital Imaging and Communications in Medicine (DICOM) is an international standard used for medical images.<br>\nSuch as X-rays, MRIs, ultrasounds, and other medical imaging modalities.\n\n> https://www.dicomstandard.org/\n\n### Testing 3 library for reading DICOM Images: `dicomsdl` - `pydicom` - `tensorflow_io`\n\n> **%timeit** vs **%time**\n>> **%timeit**: is recommended for benchmarking because it automatically runs the code multiple times and calculates the average execution time.<br>\nThis helps to get a more accurate measurement of the performance and to avoid the impact of other processes running on the system.<br><br>\n>> **%time**: is another option, but it only runs the code once, which may not be representative of the overall performance.<br>\nHowever, %time can be useful if you want to inspect the detailed timing information for each line of code.\n\nBoth %timeit and %time can give you an idea of how long it takes to run the function, but %timeit is usually the better choice for benchmarking and performance testing.","metadata":{}},{"cell_type":"code","source":"%%writefile requirements.txt\npydicom\ndicomsdl\npython-gdcm\npylibjpeg\npylibjpeg-libjpeg","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:15:39.352032Z","iopub.execute_input":"2023-12-23T00:15:39.352851Z","iopub.status.idle":"2023-12-23T00:15:39.358099Z","shell.execute_reply.started":"2023-12-23T00:15:39.352816Z","shell.execute_reply":"2023-12-23T00:15:39.357166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, sys, platform\n\n!{sys.executable} -m pip install -r requirements.txt -Uq\nprint(\"Platform:\", platform.system())  # platform.platform()\nprint(\"Python  :\", platform.python_version())  # sys.version\nprint(\"Actv Env:\", os.getenv('CONDA_DEFAULT_ENV', 'Not Found Conda Env'))","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:15:41.284912Z","iopub.execute_input":"2023-12-23T00:15:41.286096Z","iopub.status.idle":"2023-12-23T00:15:57.4048Z","shell.execute_reply.started":"2023-12-23T00:15:41.286049Z","shell.execute_reply":"2023-12-23T00:15:57.403764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing Related Libraries\n\n```\n# TF_CPP_MIN_LOG_LEVEL\n0 = all messages are logged (default behavior)\n1 = INFO messages are not printed\n2 = INFO and WARNING messages are not printed\n3 = INFO, WARNING, and ERROR messages are not printed\n``` ","metadata":{}},{"cell_type":"code","source":"import os\n# Disable TensorFlow warnings, before you import tensorflow\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # FATAL or any {'0', '1', '2'}\n\nimport tensorflow as tf\ntf.get_logger().setLevel('FATAL')  # or any {DEBUG, INFO, WARN, ERROR, FATAL}\n# Can also be set using the AUTOGRAPH_VERBOSITY environment variable\ntf.autograph.set_verbosity(1)\n\nimport tensorflow_io as tfio","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:16:07.836292Z","iopub.execute_input":"2023-12-23T00:16:07.836649Z","iopub.status.idle":"2023-12-23T00:16:19.230564Z","shell.execute_reply.started":"2023-12-23T00:16:07.836618Z","shell.execute_reply":"2023-12-23T00:16:19.229756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nimport joblib\n# import gpuparallel\nfrom time import time\nfrom tqdm.auto import tqdm\nfrom multiprocess import Pool, cpu_count; print(cpu_count())\n\nimport pylibjpeg; print(\"pylibjpeg\", pylibjpeg.__version__)\nimport libjpeg; print(\"libjpeg\", libjpeg.__version__)\nimport dicomsdl; print(\"dicomsdl\", dicomsdl.DICOMSDL_VERSION)\nimport pydicom; print(\"pydicom\", pydicom.__version__)\nimport gdcm; print(\"gdcm\", gdcm.GDCM_VERSION)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:16:19.232336Z","iopub.execute_input":"2023-12-23T00:16:19.233484Z","iopub.status.idle":"2023-12-23T00:16:19.498793Z","shell.execute_reply.started":"2023-12-23T00:16:19.233454Z","shell.execute_reply":"2023-12-23T00:16:19.497845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\ntrain_path = \"/kaggle/input/rsna-breast-cancer-detection/train_images\"\ntrain_folder_paths = sorted(glob(train_path + '/*'))\ntotal_train_folder = len(train_folder_paths)\n\ntrain_file_paths   = sorted(glob(train_path + '/*/*.dcm'))\ntotal_train_file   = len(train_file_paths)\n\nprint(train_path)\nprint(f'Train size:\\n'\n      f'\\tFolder : {total_train_folder:<10}'\n      f'\\tImages : {total_train_file:<10}')","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:16:19.499996Z","iopub.execute_input":"2023-12-23T00:16:19.500252Z","iopub.status.idle":"2023-12-23T00:18:06.045135Z","shell.execute_reply.started":"2023-12-23T00:16:19.500223Z","shell.execute_reply":"2023-12-23T00:18:06.04434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dicomsdl = dicomsdl.open(train_file_paths[0]).pixelData(storedvalue=True)\nimage_pydicom  = pydicom.dcmread(train_file_paths[0]).pixel_array \nimage_tfio     = tfio.image.decode_dicom_image(tf.io.read_file(train_file_paths[0]), dtype=tf.uint16).numpy().squeeze()\n\nplt.figure(figsize=(15, 5))\nplt.suptitle('dicom_image_dicomsdl - dicom_image_pydicom - dicom_image_tfio', fontsize=26)\nsamples = [image_dicomsdl, image_pydicom, image_tfio]\n\nfor i, sample in enumerate(samples, 1):\n    plt.subplot(1, 3, i)\n    plt.imshow(np.expand_dims(sample, axis=2), cmap='gray');\n    plt.axis(\"Off\")\n    \nprint('dicomsdl Shape:', image_dicomsdl.shape)\nprint('pydicom  Shape:', image_pydicom.shape)\nprint('tfio     Shape:', image_tfio.shape)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:19:25.119499Z","iopub.execute_input":"2023-12-23T00:19:25.120352Z","iopub.status.idle":"2023-12-23T00:19:36.096766Z","shell.execute_reply.started":"2023-12-23T00:19:25.120316Z","shell.execute_reply":"2023-12-23T00:19:36.095851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dicomsdl","metadata":{}},{"cell_type":"code","source":"image_dicom = dicomsdl.open(train_file_paths[0])\n\nprint(image_dicom.getPixelDataInfo())\nprint('PhotometricInterpretation :', image_dicom.PhotometricInterpretation)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:19:39.283985Z","iopub.execute_input":"2023-12-23T00:19:39.284356Z","iopub.status.idle":"2023-12-23T00:19:39.296398Z","shell.execute_reply.started":"2023-12-23T00:19:39.284325Z","shell.execute_reply":"2023-12-23T00:19:39.295463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pydicom","metadata":{}},{"cell_type":"code","source":"image_dicom = pydicom.dcmread(train_file_paths[0])\n\nprint(image_dicom)\nprint('PhotometricInterpretation :', image_dicom.PhotometricInterpretation)\nprint('TransferSyntaxUID         :', image_dicom.file_meta.TransferSyntaxUID)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:19:40.482493Z","iopub.execute_input":"2023-12-23T00:19:40.483171Z","iopub.status.idle":"2023-12-23T00:19:40.492381Z","shell.execute_reply.started":"2023-12-23T00:19:40.483137Z","shell.execute_reply":"2023-12-23T00:19:40.491398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tensorflow_io\n\n**⚠️ Warning**: tensorflow_io produces some errors for this dataset.\n\n**ℹ️ INFO**: tensorflow_io OLD VERSION for TPU CAN BE REQUIRED GOOGLE CLOUD - GCS_PATH.\n\n**tensor Array**\n\n- https://www.tensorflow.org/io/tutorials/dicom\n- https://colab.research.google.com/github/tensorflow/io/blob/master/docs/tutorials/dicom.ipynb#scrollTo=WodUv8O1VKmr","metadata":{}},{"cell_type":"code","source":"image_dicom = tf.io.read_file(train_file_paths[0]).numpy()\nimage_data  = tfio.image.decode_dicom_image(image_dicom, dtype=tf.uint16).numpy().squeeze()\ntag_value   = tfio.image.decode_dicom_data(image_dicom, tfio.image.dicom_tags.PhotometricInterpretation).numpy().decode()\n\nprint(image_dicom[:250])\nprint('PhotometricInterpretation :', tag_value)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:19:41.976301Z","iopub.execute_input":"2023-12-23T00:19:41.976655Z","iopub.status.idle":"2023-12-23T00:19:44.509311Z","shell.execute_reply.started":"2023-12-23T00:19:41.976626Z","shell.execute_reply":"2023-12-23T00:19:44.508417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"def read_dicom_image(path=train_file_paths[0], isDicomsdl=False, isPydicom=False, isTfio=False):\n    if isDicomsdl:\n        img_dicom = dicomsdl.open(path)\n        img_data  = img_dicom.pixelData(storedvalue=True)\n        tag_value = img_dicom.PhotometricInterpretation\n        \n    elif isPydicom:\n        img_dicom = pydicom.dcmread(path)\n        img_data  = img_dicom.pixel_array\n        tag_value = img_dicom.PhotometricInterpretation\n        \n    elif isTfio:\n        img_dicom = tf.io.read_file(path)\n        img_data  = tfio.image.decode_dicom_image(img_dicom, dtype=tf.uint16).numpy().squeeze()\n        tag_value = tfio.image.decode_dicom_data(img_dicom, tfio.image.dicom_tags.PhotometricInterpretation).numpy().decode()\n        \n    #return img_data, tag_value","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:19:46.462611Z","iopub.execute_input":"2023-12-23T00:19:46.463025Z","iopub.status.idle":"2023-12-23T00:19:46.469892Z","shell.execute_reply.started":"2023-12-23T00:19:46.462993Z","shell.execute_reply":"2023-12-23T00:19:46.469008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**timeit is more accurate, for three reasons:**\n\n- it repeats the tests many times to eliminate the influence of other tasks on your machine, such as disk flushing and OS scheduling.\n- it disables the garbage collector to prevent that process from skewing the results by scheduling a collection run at an inopportune moment.\n- it picks the most accurate timer for your OS, time.time or time.clock in Python 2 and time.perf_counter() on Python 3. See timeit.default_timer.\n\nhttps://stackoverflow.com/questions/17579357/time-time-vs-timeit-timeit","metadata":{}},{"cell_type":"markdown","source":"# GPU Results\n\n- https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster","metadata":{}},{"cell_type":"code","source":"%timeit -r 500 -n 10 -p 3 read_dicom_image(train_file_paths[1], isDicomsdl=1)\n%time read_dicom_image(train_file_paths[1], isDicomsdl=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:20:10.891912Z","iopub.execute_input":"2023-12-23T00:20:10.892275Z","iopub.status.idle":"2023-12-23T00:20:25.960552Z","shell.execute_reply.started":"2023-12-23T00:20:10.892245Z","shell.execute_reply":"2023-12-23T00:20:25.959681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%timeit -r 500 -n 10 -p 3 read_dicom_image(train_file_paths[1], isPydicom=1)\n%time read_dicom_image(train_file_paths[1], isPydicom=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:20:25.962258Z","iopub.execute_input":"2023-12-23T00:20:25.962537Z","iopub.status.idle":"2023-12-23T00:20:40.529462Z","shell.execute_reply.started":"2023-12-23T00:20:25.962512Z","shell.execute_reply":"2023-12-23T00:20:40.528553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%timeit -r 10 -n 2 -p 3 read_dicom_image(train_file_paths[1], isTfio=1)\n%time read_dicom_image(train_file_paths[1], isTfio=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:20:40.530512Z","iopub.execute_input":"2023-12-23T00:20:40.530804Z","iopub.status.idle":"2023-12-23T00:21:17.130885Z","shell.execute_reply.started":"2023-12-23T00:20:40.530778Z","shell.execute_reply":"2023-12-23T00:21:17.129841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  Convert Dicom to j2k","metadata":{}},{"cell_type":"code","source":"def convert_dicom_to_j2k(file, save_folder=\"\"):\n    # Read the DICOM file\n    dcmfile = pydicom.dcmread(file)\n\n    # Check if the DICOM file is compressed using JPEG 2000 Lossless\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        # Use DicomBytesIO for better memory handling\n        with open(file, 'rb') as f_in:\n            raw = pydicom.filebase.DicomBytesIO(f_in.read())\n            ds  = pydicom.dcmread(raw)\n        # Find the offset of the JPEG 2000 header\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")\n        # Extract the JPEG 2000 codestream starting from the offset\n        jpeg2000_codestream = bytearray(ds.PixelData[offset:])\n\n        # Save the JPEG 2000 codestream to a new file\n        # Extract patient and image information from the file path\n        patient   = os.path.basename(os.path.dirname(file))\n        image     = os.path.splitext(os.path.basename(file))[0]\n        save_path = os.path.join(save_folder, f\"{patient}_{image}.jp2\")\n        with open(save_path, \"wb\") as f_out:\n            f_out.write(jpeg2000_codestream)\n\n# Example usage:\nconvert_dicom_to_j2k(train_file_paths[0], \"./\")","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:21:17.133128Z","iopub.execute_input":"2023-12-23T00:21:17.133485Z","iopub.status.idle":"2023-12-23T00:21:17.157133Z","shell.execute_reply.started":"2023-12-23T00:21:17.133452Z","shell.execute_reply":"2023-12-23T00:21:17.156218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob('*.jp2')","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:21:17.158483Z","iopub.execute_input":"2023-12-23T00:21:17.158863Z","iopub.status.idle":"2023-12-23T00:21:17.165841Z","shell.execute_reply.started":"2023-12-23T00:21:17.158828Z","shell.execute_reply":"2023-12-23T00:21:17.16477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\n# Open the JP2 image using Pillow\njp2_image = Image.open('10006_1459541791.jp2')\n\n# Convert to a NumPy array for display\njp2_array = plt.imread('10006_1459541791.jp2')\n\n# Display the image\nplt.imshow(jp2_array)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-23T00:21:58.441861Z","iopub.execute_input":"2023-12-23T00:21:58.442237Z","iopub.status.idle":"2023-12-23T00:22:02.315757Z","shell.execute_reply.started":"2023-12-23T00:21:58.442205Z","shell.execute_reply":"2023-12-23T00:22:02.314817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## End of Project","metadata":{}}]}