{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"👋 In this kernel we will load and prepare 16-bit versions of the competition images. This notebook is heavily based on @hengck23's TensorRT example and Kaggle community efforts with my own additions/refactorings.","metadata":{}},{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-success\">\n    <b>Data source:</b> Radiological Society of North America. (2022, Nov 29). RSNA Screening Mammography Breast Cancer Detection, Version 1. Retrieved 2023 Feb 9 from [https://www.kaggle.com/competitions/rsna-breast-cancer-detection/data].\n</div>","metadata":{}},{"cell_type":"markdown","source":"# Initial imports","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nimport glob\n\nimport gc\n\nfrom pathlib import Path\n\nfrom timeit import default_timer as timer\nfrom joblib import Parallel, delayed\n\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:49:45.889946Z","iopub.execute_input":"2023-02-14T19:49:45.890627Z","iopub.status.idle":"2023-02-14T19:49:45.961466Z","shell.execute_reply.started":"2023-02-14T19:49:45.890505Z","shell.execute_reply":"2023-02-14T19:49:45.960628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:49:45.962838Z","iopub.execute_input":"2023-02-14T19:49:45.963215Z","iopub.status.idle":"2023-02-14T19:49:45.988403Z","shell.execute_reply.started":"2023-02-14T19:49:45.963174Z","shell.execute_reply":"2023-02-14T19:49:45.98749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ['CUDA_MODULE_LOADING']='LAZY'","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:49:45.989874Z","iopub.execute_input":"2023-02-14T19:49:45.990228Z","iopub.status.idle":"2023-02-14T19:49:45.995214Z","shell.execute_reply.started":"2023-02-14T19:49:45.990194Z","shell.execute_reply":"2023-02-14T19:49:45.994308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install dependencies","metadata":{}},{"cell_type":"markdown","source":"## `DICOM`","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/rsna-2022-whl/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install /kaggle/input/rsna-2022-whl/python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:49:45.998727Z","iopub.execute_input":"2023-02-14T19:49:45.999349Z","iopub.status.idle":"2023-02-14T19:51:15.030682Z","shell.execute_reply.started":"2023-02-14T19:49:45.999314Z","shell.execute_reply":"2023-02-14T19:51:15.029289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `png`","metadata":{}},{"cell_type":"markdown","source":"https://gitlab.com/drj11/pypng is licensed with the MIT licence.","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/pypng-0202207150-py3-none-any/wheelhouse/pypng-0.20220715.0-py3-none-any.whl  # private dataset for offline use","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:15.033923Z","iopub.execute_input":"2023-02-14T19:51:15.035878Z","iopub.status.idle":"2023-02-14T19:51:44.765413Z","shell.execute_reply.started":"2023-02-14T19:51:15.035827Z","shell.execute_reply":"2023-02-14T19:51:44.764178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `nvjpg2k` Python binding for decoding","metadata":{}},{"cell_type":"markdown","source":"https://github.com/louis-she/nvjpeg2k-python is published with a MIT license.","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/nvjpeg2k/nvjpeg2k.so ./","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:44.77117Z","iopub.execute_input":"2023-02-14T19:51:44.771603Z","iopub.status.idle":"2023-02-14T19:51:45.773871Z","shell.execute_reply.started":"2023-02-14T19:51:44.77151Z","shell.execute_reply":"2023-02-14T19:51:45.772559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/nvjpeg2k/')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:45.776054Z","iopub.execute_input":"2023-02-14T19:51:45.776484Z","iopub.status.idle":"2023-02-14T19:51:45.781903Z","shell.execute_reply.started":"2023-02-14T19:51:45.776431Z","shell.execute_reply":"2023-02-14T19:51:45.780837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport dicomsdl\nimport nvjpeg2k\nimport png","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:45.783406Z","iopub.execute_input":"2023-02-14T19:51:45.784259Z","iopub.status.idle":"2023-02-14T19:51:46.021262Z","shell.execute_reply.started":"2023-02-14T19:51:45.78422Z","shell.execute_reply":"2023-02-14T19:51:46.020296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load `metadata` and images and save results","metadata":{}},{"cell_type":"markdown","source":"Adapted from https://www.kaggle.com/code/hengck23/3hr-tensorrt-nextvit-example","metadata":{}},{"cell_type":"code","source":"def time_to_str(t, mode='min'):\n    if mode=='min':\n        t  = int(t)/60\n        hr = t//60\n        min = t%60\n        return '%2d hr %02d min'%(hr,min)\n\n    elif mode=='sec':\n        t   = int(t)\n        min = t//60\n        sec = t%60\n        return '%2d min %02d sec'%(min,sec)\n\n    else:\n        raise NotImplementedError","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.022652Z","iopub.execute_input":"2023-02-14T19:51:46.023113Z","iopub.status.idle":"2023-02-14T19:51:46.030142Z","shell.execute_reply.started":"2023-02-14T19:51:46.023071Z","shell.execute_reply":"2023-02-14T19:51:46.028981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm_loc = Path('/kaggle/input/rsna-breast-cancer-detection/train_images/')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.031938Z","iopub.execute_input":"2023-02-14T19:51:46.032413Z","iopub.status.idle":"2023-02-14T19:51:46.040603Z","shell.execute_reply.started":"2023-02-14T19:51:46.032373Z","shell.execute_reply":"2023-02-14T19:51:46.039586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"csv_file = Path('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntrain_meta = pd.read_csv(csv_file)\ntrain_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.042177Z","iopub.execute_input":"2023-02-14T19:51:46.042575Z","iopub.status.idle":"2023-02-14T19:51:46.130686Z","shell.execute_reply.started":"2023-02-14T19:51:46.042529Z","shell.execute_reply":"2023-02-14T19:51:46.129774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add `TransferSyntaxUID` into the `DataFrame`\n\nThere are `ds` with two different `TransferSyntaxUID` in the competition dataset, namely `1.2.840.10008.1.2.4.70` (JPEG Lossless, Nonhierarchical, First- Order Prediction) and `1.2.840.10008.1.2.4.90` (JPEG 2000 Image Compression, Lossless Only).","metadata":{}},{"cell_type":"markdown","source":"Adapted from https://www.kaggle.com/code/hengck23/3hr-tensorrt-nextvit-example","metadata":{}},{"cell_type":"code","source":"def retrieve_transfer_syntax_uid(df, dcm_dir):\n    transfer_uid = {}\n    machine_id = df.machine_id.unique()\n    for i in machine_id:\n        d = df[df.machine_id == i].iloc[0]\n        f = f'{dcm_dir}/{d.patient_id}/{d.image_id}.dcm'\n        ds = pydicom.dcmread(f)\n        transfer_uid[i] = ds.file_meta.TransferSyntaxUID\n    return transfer_uid","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.132093Z","iopub.execute_input":"2023-02-14T19:51:46.132451Z","iopub.status.idle":"2023-02-14T19:51:46.140413Z","shell.execute_reply.started":"2023-02-14T19:51:46.132423Z","shell.execute_reply":"2023-02-14T19:51:46.139185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transfer_id = retrieve_transfer_syntax_uid(train_meta, dcm_loc)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.14697Z","iopub.execute_input":"2023-02-14T19:51:46.148182Z","iopub.status.idle":"2023-02-14T19:51:46.840187Z","shell.execute_reply.started":"2023-02-14T19:51:46.148134Z","shell.execute_reply":"2023-02-14T19:51:46.839218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta.loc[:, 'i'] = np.arange(len(train_meta))\ntrain_meta.loc[:, 'TransferSyntaxUID'] = train_meta.machine_id.map(transfer_id)\ntrain_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.841786Z","iopub.execute_input":"2023-02-14T19:51:46.842167Z","iopub.status.idle":"2023-02-14T19:51:46.866468Z","shell.execute_reply.started":"2023-02-14T19:51:46.842129Z","shell.execute_reply":"2023-02-14T19:51:46.865554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add `StudyInstanceUID` into the `DataFrame`","metadata":{}},{"cell_type":"code","source":"def retrieve_study_uid(df, dcm_dir):  # modified for this notebook\n    study_uid = {}\n    pat_id = df.patient_id.unique()\n    for i in pat_id:\n        d = df[df.patient_id == i].iloc[0]\n        f = f'{dcm_dir}/{d.patient_id}/{d.image_id}.dcm'\n        ds = pydicom.dcmread(f, stop_before_pixels=True)  # https://github.com/pydicom/pydicom/discussions/1763\n        study_uid[i] = ds.StudyInstanceUID\n    return study_uid","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.868101Z","iopub.execute_input":"2023-02-14T19:51:46.869088Z","iopub.status.idle":"2023-02-14T19:51:46.87611Z","shell.execute_reply.started":"2023-02-14T19:51:46.869039Z","shell.execute_reply":"2023-02-14T19:51:46.875099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_timer = timer()  # this will take some time\n\nstudy_ids = retrieve_study_uid(train_meta, dcm_loc)\n\nprint(time_to_str(timer() - start_timer, 'sec'))","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:51:46.878031Z","iopub.execute_input":"2023-02-14T19:51:46.878351Z","iopub.status.idle":"2023-02-14T19:55:42.253282Z","shell.execute_reply.started":"2023-02-14T19:51:46.878321Z","shell.execute_reply":"2023-02-14T19:55:42.252198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta['StudyInstanceUID'] = train_meta['patient_id'].map(study_ids)\ntrain_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.254513Z","iopub.execute_input":"2023-02-14T19:55:42.255377Z","iopub.status.idle":"2023-02-14T19:55:42.288155Z","shell.execute_reply.started":"2023-02-14T19:55:42.255346Z","shell.execute_reply":"2023-02-14T19:55:42.286515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_meta.StudyInstanceUID.unique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.289849Z","iopub.execute_input":"2023-02-14T19:55:42.290375Z","iopub.status.idle":"2023-02-14T19:55:42.304573Z","shell.execute_reply.started":"2023-02-14T19:55:42.290337Z","shell.execute_reply":"2023-02-14T19:55:42.303198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_meta.patient_id.unique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.30644Z","iopub.execute_input":"2023-02-14T19:55:42.306833Z","iopub.status.idle":"2023-02-14T19:55:42.316537Z","shell.execute_reply.started":"2023-02-14T19:55:42.306798Z","shell.execute_reply":"2023-02-14T19:55:42.3153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add column `projection`","metadata":{}},{"cell_type":"code","source":"train_meta['projection'] = train_meta[['laterality', 'view']].astype(str).apply('-'.join, axis=1)\ntrain_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.317881Z","iopub.execute_input":"2023-02-14T19:55:42.318555Z","iopub.status.idle":"2023-02-14T19:55:42.810489Z","shell.execute_reply.started":"2023-02-14T19:55:42.318484Z","shell.execute_reply":"2023-02-14T19:55:42.809533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Projections\n\n#### Standard views:\n\nCC: cranio caudal\n\nMLO: mediolateral oblique\n\n#### Possible supplementary views:\n\nML: mediolateral\n\nLM: lateromedial\n\nAT: axillary tail\n\nLMO: lateromedial oblique\n\nFB: caudocranial\n\nISO: inferosuperior oblique\n\nSIO: superoinferior oblique\n\nXCCL: exaggerated craniocaudal medial\n\nXCCM: exaggerated craniocaudal lateral","metadata":{}},{"cell_type":"code","source":"list(train_meta['projection'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.813082Z","iopub.execute_input":"2023-02-14T19:55:42.813403Z","iopub.status.idle":"2023-02-14T19:55:42.826536Z","shell.execute_reply.started":"2023-02-14T19:55:42.813373Z","shell.execute_reply":"2023-02-14T19:55:42.825207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split `metadata` according to `TransferSyntaxUID`","metadata":{}},{"cell_type":"markdown","source":"Adapted from https://www.kaggle.com/code/hengck23/3hr-tensorrt-nextvit-example","metadata":{}},{"cell_type":"code","source":"j2k_meta = train_meta[train_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90'].reset_index(drop=True)\nj2k_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.830153Z","iopub.execute_input":"2023-02-14T19:55:42.830416Z","iopub.status.idle":"2023-02-14T19:55:42.872504Z","shell.execute_reply.started":"2023-02-14T19:55:42.830391Z","shell.execute_reply":"2023-02-14T19:55:42.871532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_j2k_meta = train_meta[train_meta.TransferSyntaxUID != '1.2.840.10008.1.2.4.90'].reset_index(drop=True)  # train_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.70'\nnon_j2k_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.875616Z","iopub.execute_input":"2023-02-14T19:55:42.875875Z","iopub.status.idle":"2023-02-14T19:55:42.912539Z","shell.execute_reply.started":"2023-02-14T19:55:42.87585Z","shell.execute_reply":"2023-02-14T19:55:42.91167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(j2k_meta['projection'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.914507Z","iopub.execute_input":"2023-02-14T19:55:42.91513Z","iopub.status.idle":"2023-02-14T19:55:42.924731Z","shell.execute_reply.started":"2023-02-14T19:55:42.915092Z","shell.execute_reply":"2023-02-14T19:55:42.923633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(non_j2k_meta['projection'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.926589Z","iopub.execute_input":"2023-02-14T19:55:42.927006Z","iopub.status.idle":"2023-02-14T19:55:42.937654Z","shell.execute_reply.started":"2023-02-14T19:55:42.926972Z","shell.execute_reply":"2023-02-14T19:55:42.93649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Choose only the four standard projections","metadata":{}},{"cell_type":"code","source":"j2k_df = pd.DataFrame()\nj2k_df = j2k_meta.loc[(j2k_meta['projection'] == 'R-MLO') | (j2k_meta['projection'] == 'L-MLO') | (j2k_meta['projection'] == 'R-CC') | (j2k_meta['projection'] == 'L-CC')]\nj2k_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.939438Z","iopub.execute_input":"2023-02-14T19:55:42.939917Z","iopub.status.idle":"2023-02-14T19:55:42.974438Z","shell.execute_reply.started":"2023-02-14T19:55:42.939883Z","shell.execute_reply":"2023-02-14T19:55:42.973587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_j2k_df = pd.DataFrame()\nnon_j2k_df = non_j2k_meta.loc[(non_j2k_meta['projection'] == 'R-MLO') | (non_j2k_meta['projection'] == 'L-MLO') | (non_j2k_meta['projection'] == 'R-CC') | (non_j2k_meta['projection'] == 'L-CC')]\nnon_j2k_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:42.975584Z","iopub.execute_input":"2023-02-14T19:55:42.975903Z","iopub.status.idle":"2023-02-14T19:55:43.014438Z","shell.execute_reply.started":"2023-02-14T19:55:42.975869Z","shell.execute_reply":"2023-02-14T19:55:43.013565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initialize `nvjpeg2k` decoder","metadata":{}},{"cell_type":"code","source":"j2k_decoder = nvjpeg2k.Decoder()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.015825Z","iopub.execute_input":"2023-02-14T19:55:43.016248Z","iopub.status.idle":"2023-02-14T19:55:43.307693Z","shell.execute_reply.started":"2023-02-14T19:55:43.016214Z","shell.execute_reply":"2023-02-14T19:55:43.306598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define `DICOM` processing functions","metadata":{}},{"cell_type":"markdown","source":"Adapted from https://www.kaggle.com/code/hengck23/3hr-tensorrt-nextvit-example","metadata":{}},{"cell_type":"code","source":"def process_j2k(df, dcm_dir, image_dir, is_voi_lut=True):\n    for t, d in tqdm(df.iterrows(), total=len(df)):\n        dcm_file = f'{dcm_dir}/{d.patient_id}/{d.image_id}.dcm'\n        dc = pydicom.dcmread(dcm_file)\n        \n        height = dc.Rows\n        width = dc.Columns\n        \n        offset = dc.PixelData.find(b'\\x00\\x00\\x00\\x0C')\n        jpeg_stream = bytearray(dc.PixelData[offset:])\n        image = j2k_decoder.decode(jpeg_stream)\n\n        if is_voi_lut:\n            image = pydicom.pixel_data_handlers.util.apply_voi_lut(image, dc)\n            \n        image = image.reshape((height, width))  # (height, width)\n        \n        if dc.PhotometricInterpretation == 'MONOCHROME1':  # ranges from bright to dark with ascending pixel values\n            image = image.max() - image\n        elif dc.PhotometricInterpretation == 'MONOCHROME2':  # ranges from dark to bright with ascending pixel values\n            pass\n        \n        laterality = dc.ImageLaterality\n        view = d.view\n        \n        bitdepth = dc.BitsAllocated\n        # bitsstored = dc.BitsStored\n\n        image = image.astype(np.float64)\n        image /= image.max()\n        image *= pow(2, bitdepth) - 1\n        image = image.astype(np.uint16)\n        \n        if laterality == 'R':\n            image = np.fliplr(image)\n        \n        filepath = Path(f'{image_dir}/{d.machine_id}/{d.patient_id}/{d.image_id}.png')\n        os.makedirs(f'{image_dir}/{d.machine_id}/{d.patient_id}', exist_ok=True)\n        save_as_png(image, filepath, height, width, bitdepth)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.309533Z","iopub.execute_input":"2023-02-14T19:55:43.309963Z","iopub.status.idle":"2023-02-14T19:55:43.322413Z","shell.execute_reply.started":"2023-02-14T19:55:43.309923Z","shell.execute_reply":"2023-02-14T19:55:43.321561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicomsdl_to_numpy_image(ds, index=0):\n    # https://stackoverflow.com/questions/44659924/returning-numpy-arrays-via-pybind11\n    info = ds.getPixelDataInfo()\n    if info['SamplesPerPixel'] != 1:\n        raise RuntimeError('SamplesPerPixel != 1')\n\n    shape = [info['Rows'], info['Cols']]\n    dtype = info['dtype']\n    outarr = np.empty(shape, dtype=dtype)\n    ds.copyFrameData(index, outarr)\n    return outarr","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.323661Z","iopub.execute_input":"2023-02-14T19:55:43.323931Z","iopub.status.idle":"2023-02-14T19:55:43.34131Z","shell.execute_reply.started":"2023-02-14T19:55:43.323906Z","shell.execute_reply":"2023-02-14T19:55:43.340348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicomsdl_parallel_process(d, dcm_dir, image_dir, is_voi_lut):\n    dcm_file = f'{dcm_dir}/{d.patient_id}/{d.image_id}.dcm'\n    ds = dicomsdl.open(dcm_file)\n    \n    height = ds.Rows\n    width = ds.Columns\n    \n    image = dicomsdl_to_numpy_image(ds)\n\n    if is_voi_lut:\n        dc = pydicom.dcmread(dcm_file)\n        image = pydicom.pixel_data_handlers.util.apply_voi_lut(image, dc)\n\n    image = image.reshape((height, width))  # (height, width)\n        \n    if ds.PhotometricInterpretation == 'MONOCHROME1':  # ranges from bright to dark with ascending pixel values\n        image = image.max() - image\n    elif ds.PhotometricInterpretation == 'MONOCHROME2':  # ranges from dark to bright with ascending pixel values\n        pass\n        \n    laterality = dc.ImageLaterality\n    view = d.view\n        \n    bitdepth = ds.BitsAllocated\n    # bitsstored = ds.BitsStored\n\n    image = image.astype(np.float64)\n    image /= image.max()\n    image *= pow(2, bitdepth) - 1\n    image = image.astype(np.uint16)\n    \n    if laterality == 'R':\n        image = np.fliplr(image)\n        \n    filepath = Path(f'{image_dir}/{d.machine_id}/{d.patient_id}/{d.image_id}.png')\n    os.makedirs(f'{image_dir}/{d.machine_id}/{d.patient_id}', exist_ok=True)\n    save_as_png(image, filepath, height, width, bitdepth)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.343285Z","iopub.execute_input":"2023-02-14T19:55:43.343572Z","iopub.status.idle":"2023-02-14T19:55:43.355105Z","shell.execute_reply.started":"2023-02-14T19:55:43.343538Z","shell.execute_reply":"2023-02-14T19:55:43.354176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_non_j2k(df, dcm_dir, image_dir, n_jobs, is_voi_lut=True):\n    #https://stackoverflow.com/questions/56659294/does-joblib-parallel-keep-the-original-order-of-data-passed\n    Parallel(n_jobs=n_jobs)(\n        delayed(dicomsdl_parallel_process)(d, dcm_dir, image_dir, is_voi_lut)\n        for t,d in tqdm(df.iterrows(), total=len(df))\n    )","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.356631Z","iopub.execute_input":"2023-02-14T19:55:43.356997Z","iopub.status.idle":"2023-02-14T19:55:43.371467Z","shell.execute_reply.started":"2023-02-14T19:55:43.356961Z","shell.execute_reply":"2023-02-14T19:55:43.370468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save as `png`","metadata":{}},{"cell_type":"code","source":"def save_as_png(image, png_fname, height, width, bitdepth=12):\n    with open(png_fname, 'wb') as f:\n        writer = png.Writer(height=image.shape[0], width=image.shape[1], bitdepth=bitdepth, greyscale=True)\n        writer.write(f, image.tolist())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.372646Z","iopub.execute_input":"2023-02-14T19:55:43.373406Z","iopub.status.idle":"2023-02-14T19:55:43.38293Z","shell.execute_reply.started":"2023-02-14T19:55:43.373379Z","shell.execute_reply":"2023-02-14T19:55:43.381951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"png_save_dir = Path('/kaggle/tmp/~png')  # cache all png images","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.386319Z","iopub.execute_input":"2023-02-14T19:55:43.386655Z","iopub.status.idle":"2023-02-14T19:55:43.393925Z","shell.execute_reply.started":"2023-02-14T19:55:43.38663Z","shell.execute_reply":"2023-02-14T19:55:43.393072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = True  # only one examination for `j2k` and `non_j2k` will be processed","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.395045Z","iopub.execute_input":"2023-02-14T19:55:43.396745Z","iopub.status.idle":"2023-02-14T19:55:43.406252Z","shell.execute_reply.started":"2023-02-14T19:55:43.396709Z","shell.execute_reply":"2023-02-14T19:55:43.405276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Processing j2k: {len(j2k_df)}')\nstart_timer = timer()\nif test:\n    process_j2k(j2k_df.loc[j2k_df['patient_id'] == 10011], dcm_loc, png_save_dir)\nelse:\n    process_j2k(j2k_df, dcm_loc, png_save_dir)\nprint(time_to_str(timer() - start_timer, 'sec'))\n    \nprint(f'Processing non-j2k: {len(non_j2k_df)}')\nstart_timer = timer()\nif test:\n    process_non_j2k(non_j2k_df.loc[non_j2k_df['patient_id'] == 10038], dcm_loc, png_save_dir, n_jobs=2)\nelse:\n    process_non_j2k(non_j2k_df, dcm_loc, png_save_dir, n_jobs=2)  \nprint(time_to_str(timer() - start_timer, 'sec'))","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:43.408171Z","iopub.execute_input":"2023-02-14T19:55:43.408937Z","iopub.status.idle":"2023-02-14T19:55:50.664498Z","shell.execute_reply.started":"2023-02-14T19:55:43.408895Z","shell.execute_reply":"2023-02-14T19:55:50.66333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show `tmp` folder contents","metadata":{}},{"cell_type":"code","source":"!ls -la /kaggle/tmp/~png/","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:50.66636Z","iopub.execute_input":"2023-02-14T19:55:50.666752Z","iopub.status.idle":"2023-02-14T19:55:51.667945Z","shell.execute_reply.started":"2023-02-14T19:55:50.666711Z","shell.execute_reply":"2023-02-14T19:55:51.666795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(Path('/kaggle/tmp/~png/21/10011').glob('*'))","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:51.670364Z","iopub.execute_input":"2023-02-14T19:55:51.670948Z","iopub.status.idle":"2023-02-14T19:55:51.680376Z","shell.execute_reply.started":"2023-02-14T19:55:51.670905Z","shell.execute_reply":"2023-02-14T19:55:51.679185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(Path('/kaggle/tmp/~png/216/10038').glob('*'))","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:51.681921Z","iopub.execute_input":"2023-02-14T19:55:51.68313Z","iopub.status.idle":"2023-02-14T19:55:51.692418Z","shell.execute_reply.started":"2023-02-14T19:55:51.683092Z","shell.execute_reply":"2023-02-14T19:55:51.691224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize results","metadata":{}},{"cell_type":"code","source":"file_lst = list(Path('/kaggle/tmp/~png/216/10038').glob('*'))","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:51.699547Z","iopub.execute_input":"2023-02-14T19:55:51.699874Z","iopub.status.idle":"2023-02-14T19:55:51.705933Z","shell.execute_reply.started":"2023-02-14T19:55:51.699845Z","shell.execute_reply":"2023-02-14T19:55:51.704944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in file_lst:\n    print(f'File: {str(file)}')\n    \n    img = cv2.imread(str(file), -1)\n    fig = plt.figure(figsize=(7, 7))\n    ax = fig.add_subplot(1,1,1)\n    ax.imshow(img, cmap='gray')\n    plt.show()\n    \n    unique_colors = len(np.unique(img))\n    mi = np.min(img)\n    ma = np.max(img)\n    print(f'Unique colors: {unique_colors}; Min value: {mi}; Max value: {ma}')\n    \n    del img","metadata":{"execution":{"iopub.status.busy":"2023-02-14T19:55:51.707495Z","iopub.execute_input":"2023-02-14T19:55:51.707888Z","iopub.status.idle":"2023-02-14T19:55:54.039692Z","shell.execute_reply.started":"2023-02-14T19:55:51.707852Z","shell.execute_reply":"2023-02-14T19:55:54.038536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusions\n\nWith the steps above we have achieved 16-bit dataset comprising of the **four standard view representatives**, namely R-MLO, L-MLO, R-CC and L-CC, for training purposes with images flipped to to have the same orientation. 👍","metadata":{}}]}