{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\nimport io\nimport shutil\nimport subprocess\nfrom glob import glob\n\nimport gc\n\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sb\nfrom PIL import Image\n\nimport tensorflow as tf\n\nfrom tqdm.contrib.concurrent import process_map","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-14T17:39:21.000238Z","iopub.execute_input":"2022-10-14T17:39:21.000717Z","iopub.status.idle":"2022-10-14T17:39:27.040081Z","shell.execute_reply.started":"2022-10-14T17:39:21.000606Z","shell.execute_reply":"2022-10-14T17:39:27.038878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Quick fix for pydicom's inability to read compressed data\n# https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341187\n\ncommand = \"pip install -qU ../input/for-pydicom/python_gdcm-3.0.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ../input/for-pydicom/pylibjpeg-1.4.0-py3-none-any.whl --find-links frozen_packages --no-index\"\nsubprocess.run(command.split())","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:39:27.042273Z","iopub.execute_input":"2022-10-14T17:39:27.042977Z","iopub.status.idle":"2022-10-14T17:39:39.471419Z","shell.execute_reply.started":"2022-10-14T17:39:27.042928Z","shell.execute_reply":"2022-10-14T17:39:39.469931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nfrom pydicom.tag import Tag\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:39:39.473466Z","iopub.execute_input":"2022-10-14T17:39:39.474148Z","iopub.status.idle":"2022-10-14T17:39:39.918946Z","shell.execute_reply.started":"2022-10-14T17:39:39.474098Z","shell.execute_reply":"2022-10-14T17:39:39.917641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = \"../input/rsna-2022-cervical-spine-fracture-detection\"","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:39:39.921954Z","iopub.execute_input":"2022-10-14T17:39:39.922442Z","iopub.status.idle":"2022-10-14T17:39:39.928445Z","shell.execute_reply.started":"2022-10-14T17:39:39.922395Z","shell.execute_reply":"2022-10-14T17:39:39.92725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/raddar/convert-dicom-to-np-array-the-correct-way/notebook\n\ndef dicom2jpg(input_path, output_path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(input_path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # Depending on this value, X-ray may look inverted, so fix that\n    if fix_monochrome and dicom.PhotometricInterpretation==\"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    # Normalize image\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    height, width = data.shape\n    # Convert numpy array to PIL\n    data = Image.fromarray(data)\n\n    if not height == width == 512:\n        # Resize respecting aspect ratio\n        data.thumbnail((512, 512))        \n        # If dimensions are unequal then apply padding\n        if height != width:\n            # Create background\n            background = Image.new(data.mode, size=(512, 512), color=\"white\")\n            # Paste image on top-left of background\n            background.paste(data, (0,0))\n            # Copy image\n            data = background.copy()\n            \n            del background\n            gc.collect()\n    \n    # Create output directory if it does not exist\n    os.makedirs(os.path.dirname(output_path), exist_ok=True)\n    \n#     # Save image in output directory\n#     data.save(output_path)\n\n    # Convert PIL to bytes\n    img_byte_arr = io.BytesIO()\n    data.save(img_byte_arr, format=\"JPEG\")\n    img_byte_arr = img_byte_arr.getvalue()\n\n    with tf.io.TFRecordWriter(output_path) as writer:\n        feature = {\n            \"image\": tf.train.Feature(\n                bytes_list = tf.train.BytesList(value=[img_byte_arr])\n            )\n        }\n        example = tf.train.Example(features=tf.train.Features(feature=feature))\n        writer.write(example.SerializeToString())\n\n    \n    del data, img_byte_arr\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:39:39.930428Z","iopub.execute_input":"2022-10-14T17:39:39.931001Z","iopub.status.idle":"2022-10-14T17:39:39.945787Z","shell.execute_reply.started":"2022-10-14T17:39:39.930907Z","shell.execute_reply":"2022-10-14T17:39:39.944266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ninput_paths = glob(f\"{base_path}/train_images/*/*.dcm\", recursive=True)\ninput_paths = sorted(input_paths)\n\n# name = \"d0000to010k\"\n# output_paths = [f\"{name}/\" + path.replace(f\"{base_path}/\", \"\").replace(\".dcm\", \"\").replace(\".\", \"_\") + \".jpg\" for path in input_paths]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:39:39.9476Z","iopub.execute_input":"2022-10-14T17:39:39.948174Z","iopub.status.idle":"2022-10-14T17:40:27.363167Z","shell.execute_reply.started":"2022-10-14T17:39:39.948123Z","shell.execute_reply":"2022-10-14T17:40:27.361917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output_paths = [\n#     \"train_images/\" + path.replace(f\"{base_path}/train_images/\", \"\").replace(\".dcm\", \"\").replace(\".\", \"_\").replace(\"/\", \"__\") + \".jpg\" \n#     for path in input_paths\n# ]\n\noutput_paths = [\n    \"train_images/\" + path.replace(f\"{base_path}/train_images/\", \"\").replace(\".dcm\", \"\").replace(\".\", \"_\").replace(\"/\", \"__\") + \".tfr\" \n    for path in input_paths\n]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:40:27.364627Z","iopub.execute_input":"2022-10-14T17:40:27.365115Z","iopub.status.idle":"2022-10-14T17:40:28.437466Z","shell.execute_reply.started":"2022-10-14T17:40:27.365064Z","shell.execute_reply":"2022-10-14T17:40:28.435969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Save multiple images at a time\n_ = process_map(dicom2jpg, input_paths[:10_000], output_paths[:10_000], max_workers=10, chunksize=2) ","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:40:28.438858Z","iopub.execute_input":"2022-10-14T17:40:28.439227Z","iopub.status.idle":"2022-10-14T17:53:52.813132Z","shell.execute_reply.started":"2022-10-14T17:40:28.439193Z","shell.execute_reply":"2022-10-14T17:53:52.811625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example = tf.data.TFRecordDataset(\"train_images/1_2_826_0_1_3680043_10608__216.tfr\")","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:53:52.823811Z","iopub.execute_input":"2022-10-14T17:53:52.824215Z","iopub.status.idle":"2022-10-14T17:53:52.836228Z","shell.execute_reply.started":"2022-10-14T17:53:52.82418Z","shell.execute_reply":"2022-10-14T17:53:52.835145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for raw_record in example: \n#     example = tf.train.Example()\n#     example.ParseFromString(raw_record.numpy())\n#     example = example.features.feature[\"image\"].bytes_list.value[0]\n#     example = Image.open(io.BytesIO(example))\n#     example = np.array(example)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:53:52.837391Z","iopub.execute_input":"2022-10-14T17:53:52.838463Z","iopub.status.idle":"2022-10-14T17:53:52.845935Z","shell.execute_reply.started":"2022-10-14T17:53:52.838424Z","shell.execute_reply":"2022-10-14T17:53:52.845086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:53:52.847517Z","iopub.execute_input":"2022-10-14T17:53:52.848331Z","iopub.status.idle":"2022-10-14T17:53:52.85616Z","shell.execute_reply.started":"2022-10-14T17:53:52.848285Z","shell.execute_reply":"2022-10-14T17:53:52.855103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" ","metadata":{},"execution_count":null,"outputs":[]}]}