{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Pulmonary Embolism Detection. Data Analysis and visualization.

\n\n
","metadata":{}},{"cell_type":"markdown","source":"### Hello everyone! In this kernel I am going to present basic EDA for this dataset, prepare some interesting image animations and show how to work with metadata of medical images. Let's start!","metadata":{}},{"cell_type":"markdown","source":"
\n

Quick navigation

\n\n* [1. Basic Data Overview](#1)\n* [2. Image Overview](#2)\n* [3. Image Animation](#3)\n* [4. Images Analysis](#4)\n* [5. Metadata Analysis](#5)","metadata":{}},{"cell_type":"markdown","source":"\n

1. Basic Data Overview

","metadata":{}},{"cell_type":"code","source":"!conda install -c conda-forge gdcm -y\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport pydicom as dcm\nimport matplotlib\nimport plotly.express as px\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport glob\nimport gdcm\nfrom matplotlib import animation, rc\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\n\nTRAIN_DIR = \"../input/rsna-str-pulmonary-embolism-detection/train/\"\nfiles = glob.glob('../input/rsna-str-pulmonary-embolism-detection/train/*/*/*.dcm')\n\nrc('animation', html='jshtml')\n\nnp.random.seed(666)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T09:16:02.353575Z","iopub.execute_input":"2022-08-29T09:16:02.353856Z","iopub.status.idle":"2022-08-29T09:35:07.915868Z","shell.execute_reply.started":"2022-08-29T09:16:02.353828Z","shell.execute_reply":"2022-08-29T09:35:07.915002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\ntest = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-08-29T09:35:07.918782Z","iopub.execute_input":"2022-08-29T09:35:07.919334Z","iopub.status.idle":"2022-08-29T09:35:11.427904Z","shell.execute_reply.started":"2022-08-29T09:35:07.919291Z","shell.execute_reply":"2022-08-29T09:35:11.427141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:11.431278Z","iopub.execute_input":"2022-08-29T09:35:11.431559Z","iopub.status.idle":"2022-08-29T09:35:11.460951Z","shell.execute_reply.started":"2022-08-29T09:35:11.431531Z","shell.execute_reply":"2022-08-29T09:35:11.460311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = [\n 'pe_present_on_image', 'negative_exam_for_pe', 'qa_motion', \n 'qa_contrast', 'flow_artifact', 'rv_lv_ratio_gte_1', \n 'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe', \n 'true_filling_defect_not_pe', 'rightsided_pe', \n 'acute_and_chronic_pe', 'central_pe', 'indeterminate'\n]\n\nfig = make_subplots(rows=5, cols=3)\n\ntraces = [\n go.Bar(\n x=[0, 1], \n y=[\n len(train[train[col]==0]),\n len(train[train[col]==1])\n ], \n name=col,\n text = [\n str(round(100 * len(train[train[col]==0]) / len(train), 2)) + '%',\n str(round(100 * len(train[train[col]==1]) / len(train), 2)) + '%'\n ],\n textposition='auto'\n ) for col in cols\n]\n\nfor i in range(len(traces)):\n fig.append_trace(traces[i], (i // 3) + 1, (i % 3) +1)\n\nfig.update_layout(\n title_text='Train columns',\n height=1200,\n width=1000\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:11.462907Z","iopub.execute_input":"2022-08-29T09:35:11.463503Z","iopub.status.idle":"2022-08-29T09:35:16.789305Z","shell.execute_reply.started":"2022-08-29T09:35:11.463464Z","shell.execute_reply":"2022-08-29T09:35:16.788459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see how many records for every column in training set have non zero values.","metadata":{}},{"cell_type":"code","source":"x = train.drop(['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID'], axis=1).sum(axis=0).sort_values().reset_index()\nx.columns = ['column', 'nonzero_records']\n\nfig = px.bar(\n x, \n x='nonzero_records', \n y='column', \n orientation='h', \n title='Columns and non zero samples', \n height=800, \n width=800\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:16.79238Z","iopub.execute_input":"2022-08-29T09:35:16.792942Z","iopub.status.idle":"2022-08-29T09:35:17.496725Z","shell.execute_reply.started":"2022-08-29T09:35:16.792894Z","shell.execute_reply":"2022-08-29T09:35:17.49604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train.drop(['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID'], axis=1).astype(bool).sum(axis=1).reset_index()\ndata.columns = ['row', 'count']\ndata = data.groupby(['count'])['row'].count().reset_index()\n\nfig = px.bar(\n data, \n y=data['row'], \n x=\"count\", \n title='Number of activations in for every sample in training set', \n width=800, \n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:17.499918Z","iopub.execute_input":"2022-08-29T09:35:17.500317Z","iopub.status.idle":"2022-08-29T09:35:17.688881Z","shell.execute_reply.started":"2022-08-29T09:35:17.500286Z","shell.execute_reply":"2022-08-29T09:35:17.687976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train.drop(['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID'], axis=1).astype(bool).sum(axis=1).reset_index()\ndata.columns = ['row', 'count']\ndata = data.groupby(['count'])['row'].count().reset_index()\n\nfig = px.pie(\n data, \n values=round((100 * data['row'] / len(train)), 2), \n names=\"count\", \n title='Number of activations for every sample (Percent)', \n width=800, \n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:17.690633Z","iopub.execute_input":"2022-08-29T09:35:17.691027Z","iopub.status.idle":"2022-08-29T09:35:17.893339Z","shell.execute_reply.started":"2022-08-29T09:35:17.690981Z","shell.execute_reply":"2022-08-29T09:35:17.892426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that major number of samples in training set have only 1 activation (~65%).","metadata":{}},{"cell_type":"code","source":"data = train[[\n 'pe_present_on_image', 'negative_exam_for_pe', 'qa_motion', \n 'qa_contrast', 'flow_artifact', 'rv_lv_ratio_gte_1', \n 'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe', \n 'true_filling_defect_not_pe', 'rightsided_pe', \n 'acute_and_chronic_pe', 'central_pe', 'indeterminate'\n]]\n\nf = plt.figure(figsize=(16, 16))\nplt.matshow(data.corr(), fignum=f.number)\nplt.xticks(range(data.shape[1]), data.columns, fontsize=13, rotation=70)\nplt.yticks(range(data.shape[1]), data.columns, fontsize=13)\ncb = plt.colorbar()\ncb.ax.tick_params(labelsize=13)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:17.894885Z","iopub.execute_input":"2022-08-29T09:35:17.895383Z","iopub.status.idle":"2022-08-29T09:35:19.160751Z","shell.execute_reply.started":"2022-08-29T09:35:17.895346Z","shell.execute_reply":"2022-08-29T09:35:19.159839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Total number (dirictories) in training set {}'.format(len(os.listdir(TRAIN_DIR))))","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:19.161791Z","iopub.execute_input":"2022-08-29T09:35:19.162034Z","iopub.status.idle":"2022-08-29T09:35:19.170088Z","shell.execute_reply.started":"2022-08-29T09:35:19.162011Z","shell.execute_reply":"2022-08-29T09:35:19.169197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:19.171852Z","iopub.execute_input":"2022-08-29T09:35:19.172211Z","iopub.status.idle":"2022-08-29T09:35:19.186087Z","shell.execute_reply.started":"2022-08-29T09:35:19.172175Z","shell.execute_reply":"2022-08-29T09:35:19.185032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

2. Image Overview

","metadata":{}},{"cell_type":"markdown","source":"Image Sample.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\nax.imshow(\n dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/4833c9b6a5d0/57e3e3c5f910/f4fdc88f2ace.dcm\").pixel_array\n)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:19.187765Z","iopub.execute_input":"2022-08-29T09:35:19.188158Z","iopub.status.idle":"2022-08-29T09:35:19.4195Z","shell.execute_reply.started":"2022-08-29T09:35:19.188102Z","shell.execute_reply":"2022-08-29T09:35:19.418659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image = dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/4833c9b6a5d0/57e3e3c5f910/f4fdc88f2ace.dcm\").pixel_array\nprint('Image shape: ', test_image.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:19.420597Z","iopub.execute_input":"2022-08-29T09:35:19.421035Z","iopub.status.idle":"2022-08-29T09:35:19.428778Z","shell.execute_reply.started":"2022-08-29T09:35:19.420999Z","shell.execute_reply":"2022-08-29T09:35:19.427924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Meta information.","metadata":{}},{"cell_type":"code","source":"dcm.dcmread(\"../input/rsna-str-pulmonary-embolism-detection/train/4833c9b6a5d0/57e3e3c5f910/f4fdc88f2ace.dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:19.429983Z","iopub.execute_input":"2022-08-29T09:35:19.430793Z","iopub.status.idle":"2022-08-29T09:35:19.443349Z","shell.execute_reply.started":"2022-08-29T09:35:19.430738Z","shell.execute_reply":"2022-08-29T09:35:19.442698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check some of our randomly selected images.","metadata":{}},{"cell_type":"code","source":"f, plots = plt.subplots(6, 6, sharex='col', sharey='row', figsize=(17, 17))\n\nfor i in range(36):\n plots[i // 6, i % 6].axis('off')\n plots[i // 6, i % 6].imshow(dcm.dcmread(np.random.choice(files[:10000])).pixel_array)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:19.444724Z","iopub.execute_input":"2022-08-29T09:35:19.445246Z","iopub.status.idle":"2022-08-29T09:35:24.364499Z","shell.execute_reply.started":"2022-08-29T09:35:19.445211Z","shell.execute_reply":"2022-08-29T09:35:24.361952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

3. Image Animation

","metadata":{}},{"cell_type":"code","source":"def load_slice(path):\n slices = [dcm.read_file(path + '/' + s) for s in os.listdir(path)]\n slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n \n try:\n slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n except:\n slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n \n for s in slices:\n s.SliceThickness = slice_thickness\n \n return slices\n\n### Source: https://www.kaggle.com/allunia/pulmonary-fibrosis-dicom-preprocessing\ndef transform_to_hu(slices):\n images = np.stack([file.pixel_array for file in slices])\n images = images.astype(np.int16)\n images[images <= -1000] = 0\n\n for n in range(len(slices)):\n intercept = slices[n].RescaleIntercept\n slope = slices[n].RescaleSlope\n \n if slope != 1:\n images[n] = slope * images[n].astype(np.float64)\n images[n] = images[n].astype(np.int16)\n \n images[n] += np.int16(intercept)\n \n return np.array(images, dtype=np.int16)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:24.36619Z","iopub.execute_input":"2022-08-29T09:35:24.366773Z","iopub.status.idle":"2022-08-29T09:35:24.381888Z","shell.execute_reply.started":"2022-08-29T09:35:24.366728Z","shell.execute_reply":"2022-08-29T09:35:24.380713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sequence of images for some slice.","metadata":{}},{"cell_type":"code","source":"first_patient = load_slice('../input/rsna-str-pulmonary-embolism-detection/train/eac9014cea52/90cc14605905')\nfirst_patient_pixels = transform_to_hu(first_patient)\n\nfig, plots = plt.subplots(16, 10, sharex='col', sharey='row', figsize=(20, 25))\n\nfor i in range(160):\n plots[i // 10, i % 10].axis('off')\n plots[i // 10, i % 10].imshow(first_patient_pixels[i], cmap=plt.cm.viridis) ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:24.383828Z","iopub.execute_input":"2022-08-29T09:35:24.384339Z","iopub.status.idle":"2022-08-29T09:35:48.469998Z","shell.execute_reply.started":"2022-08-29T09:35:24.384292Z","shell.execute_reply":"2022-08-29T09:35:48.468931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's do animation (Inspired and slightly modified from https://www.kaggle.com/avloss/eda-with-animation).","metadata":{}},{"cell_type":"code","source":"scans = glob.glob('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/*/*/')\nprint('Total number of scans: ', len(scans))","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:35:48.471653Z","iopub.execute_input":"2022-08-29T09:35:48.472202Z","iopub.status.idle":"2022-08-29T09:35:56.7496Z","shell.execute_reply.started":"2022-08-29T09:35:48.472164Z","shell.execute_reply":"2022-08-29T09:35:56.74881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_scan(path):\n fragments = glob.glob(path + '/*')\n \n slices = []\n for f in fragments:\n img = dcm.dcmread(f)\n img_data = img.pixel_array\n length = int(img.InstanceNumber)\n slices.append((length, img_data))\n slices.sort()\n return [s[1] for s in slices]\n\n\ndef animate(ims):\n fig = plt.figure(figsize=(11, 11))\n plt.axis('off')\n im = plt.imshow(ims[0])\n\n def animate_func(i):\n im.set_array(ims[i])\n return [im]\n\n return animation.FuncAnimation(fig, animate_func, frames = len(ims), interval = 1000//24)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:56.750853Z","iopub.execute_input":"2022-08-29T09:35:56.751224Z","iopub.status.idle":"2022-08-29T09:35:56.760346Z","shell.execute_reply.started":"2022-08-29T09:35:56.751193Z","shell.execute_reply":"2022-08-29T09:35:56.759232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"movie = animate(read_scan(scans[666]))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-29T09:35:56.761604Z","iopub.execute_input":"2022-08-29T09:35:56.762125Z","iopub.status.idle":"2022-08-29T09:36:07.529138Z","shell.execute_reply.started":"2022-08-29T09:35:56.762063Z","shell.execute_reply":"2022-08-29T09:36:07.528296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"movie","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:36:07.530374Z","iopub.execute_input":"2022-08-29T09:36:07.530807Z","iopub.status.idle":"2022-08-29T09:36:39.49553Z","shell.execute_reply.started":"2022-08-29T09:36:07.530774Z","shell.execute_reply":"2022-08-29T09:36:39.494234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

4. Images Analysis

","metadata":{}},{"cell_type":"markdown","source":"Plotly is really heavy for this task so I didn't use it here :)","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\n\nfor n in range(100):\n loaded = dcm.dcmread(np.random.choice(files[:]))\n image = loaded.pixel_array.flatten()\n rescaled_image = image * loaded.RescaleSlope + loaded.RescaleIntercept\n sns.distplot(image.flatten())\n\nplt.title(\"HU unit distributions for 100 examples\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:36:39.497654Z","iopub.execute_input":"2022-08-29T09:36:39.49838Z","iopub.status.idle":"2022-08-29T09:38:07.892135Z","shell.execute_reply.started":"2022-08-29T09:36:39.498324Z","shell.execute_reply":"2022-08-29T09:38:07.89118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scans = [dcm.dcmread(files[i]) for i in range(500)]","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:38:07.893473Z","iopub.execute_input":"2022-08-29T09:38:07.89384Z","iopub.status.idle":"2022-08-29T09:38:16.152164Z","shell.execute_reply.started":"2022-08-29T09:38:07.893808Z","shell.execute_reply":"2022-08-29T09:38:16.151397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hu_scans = transform_to_hu(scans)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:38:16.15402Z","iopub.execute_input":"2022-08-29T09:38:16.154576Z","iopub.status.idle":"2022-08-29T09:38:16.982534Z","shell.execute_reply.started":"2022-08-29T09:38:16.154535Z","shell.execute_reply":"2022-08-29T09:38:16.981593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = hu_scans[13]\na = img.reshape((512, 512, 1))\na = np.concatenate([a, a, a], axis=2)\n\nfig = make_subplots(1, 2)\nimg = hu_scans[0]\n\nfig.add_trace(go.Image(z=a), 1, 1)\nfig.add_trace(go.Histogram(x=img.ravel(), opacity=1), 1, 2)\n\nfig.update_layout(\n height=600, \n width=800,\n title='Image in HU and HU values distribution'\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:38:16.983893Z","iopub.execute_input":"2022-08-29T09:38:16.984264Z","iopub.status.idle":"2022-08-29T09:38:18.676705Z","shell.execute_reply.started":"2022-08-29T09:38:16.984227Z","shell.execute_reply":"2022-08-29T09:38:18.674265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = hu_scans[55]\na = img.reshape((512, 512, 1))\na = np.concatenate([a, a, a], axis=2)\n\nfig = make_subplots(1, 2)\nimg = hu_scans[0]\n\nfig.add_trace(go.Image(z=a), 1, 1)\nfig.add_trace(go.Histogram(x=img.ravel(), opacity=1), 1, 2)\nfig.update_layout(\n height=600, \n width=800,\n title='Image in HU and HU values distribution'\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:38:18.678569Z","iopub.execute_input":"2022-08-29T09:38:18.679268Z","iopub.status.idle":"2022-08-29T09:38:20.318933Z","shell.execute_reply.started":"2022-08-29T09:38:18.679216Z","shell.execute_reply":"2022-08-29T09:38:20.318081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = hu_scans[90]\na = img.reshape((512, 512, 1))\na = np.concatenate([a, a, a], axis=2)\n\nfig = make_subplots(1, 2)\nimg = hu_scans[0]\n\nfig.add_trace(go.Image(z=a), 1, 1)\nfig.add_trace(go.Histogram(x=img.ravel(), opacity=1), 1, 2)\n\nfig.update_layout(\n height=600, \n width=800,\n title='Image in HU and HU values distribution'\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:38:20.320289Z","iopub.execute_input":"2022-08-29T09:38:20.320802Z","iopub.status.idle":"2022-08-29T09:38:22.116106Z","shell.execute_reply.started":"2022-08-29T09:38:20.320765Z","shell.execute_reply":"2022-08-29T09:38:22.114402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

5. Metadata Analysis

","metadata":{}},{"cell_type":"markdown","source":"You can find the full list of available metadata fields here.","metadata":{}},{"cell_type":"markdown","source":"* Specific Character Set **(0008,0005)** \n * Identifies the Character Set that expands or replaces the Basic Graphic Set (ISO 646) for values of Data Elements that have Value Representation of SH, LO, ST, PN, LT, UC or UT.\n\n\n* Image Type **(0008,0008)** \n * Contains the highest level summary of what is in the SOP Instance.\n\n\n* SOP Class UID **(0008, 0016)**\n * Uniquely identifies the SOP Class.\n\n\n* SOP Instance UID **(0008, 0018)**\n * Uniquely identifies the SOP Instance\n \n\n* Modality **(0008, 0060)**\n * Type of equipment that originally acquired the data used to create the images in this Series.\n * CT: Computed Tomography\n \n\n* Slice Thickness **(0018, 0050)**\n * Nominal slice thickness, in mm.\n \n \n* KVP **(0018, 0060)** \n * Peak kilo voltage output of the X-Ray generator used.\n \n\n* Gantry/Detector Tilt **(0018, 1120)** \n * The angle in degrees of the detector face relative to the patient's major (Head to Feet) axis (or the table supporting the patient). Positive tilt is towards the patient's feet.\n \n \n* Table Height **(0018, 1130)**\n * The distance in mm of the top of the patient table to the center of rotation; below the center is positive.\n \n \n* Rotation Direction **(0018, 1140)**\n * Direction of rotation of the source when relevant, about nearest principal axis of equipment.\n * CW: clockwise\n * CC: counter clockwise\n \n \n* X-Ray Tube Current **(0018, 1151)**\n * X-Ray Tube Current in mA.\n \n \n* Exposure **(0018, 1152)**\n * The exposure expressed in mAs, for example calculated from Exposure Time and X-Ray Tube Current.\n \n\n* Convolution Kernel **(0018, 1210)** \n * A label describing the convolution kernel or algorithm used to reconstruct the data.\n \n \n* Patient Position **(0018, 5100)**\n * Patient position descriptor relative to the equipment.\n \n \n* Study Instance UID **(0020, 000d)** \n * Unique identifier for the Study.\n \n \n* Series Instance UID **(0020, 000e)** \n * Unique identifier of the Series.\n \n \n* Series Number **(0020, 0011)** \n * A number that identifies this Series.\n \n\n* Instance Number **(0020, 0013)**\n * A number that identifies this image.\n \n \n* Image Position (Patient) **(0020, 0032)**\n * The x, y, and z coordinates of the upper left hand corner (center of the first voxel transmitted) of the image, in mm.\n \n \n* Image Orientation (Patient) **(0020, 0037)** \n * Specifies the direction cosines of the first row and the first column with respect to the patient. These Attributes shall be provide as a pair. Row value for the x, y, and z axes respectively followed by the Column value for the x, y, and z axes respectively.\n \n \n* Frame of Reference UID **(0020, 0052) **\n * Uniquely identifies the Frame of Reference for a Series. \n \n \n* Samples per Pixel **(0028, 0002)** \n * Number of samples (planes) in this image.\n \n \n* Photometric Interpretation **(0028, 0004)** \n * Specifies the intended interpretation of the pixel data.\n \n \n* Rows **(0028, 0010)**\n * Number of rows in the image.\n \n \n* Columns **(0028, 0011) **\n * Number of columns in the image.\n \n\n* Pixel Spacing **(0028, 0030) **\n * Physical distance in the patient between the center of each pixel, specified by a numeric pair - adjacent row spacing (delimiter) adjacent column spacing in mm.\n \n \n* Bits Allocated **(0028, 0100)** \n * Number of bits allocated for each pixel sample. Each sample shall have the same number of bits allocated. Bits Allocated (0028,0100) shall be either 1, or a multiple of 8.\n \n \n* Bits Stored **(0028, 0101)** \n * Number of bits stored for each pixel sample. Each sample shall have the same number of bits stored.\n \n \n* High Bit **(0028, 0102)** \n * Most significant bit for pixel sample data. Each sample shall have the same high bit. High Bit (0028,0102) shall be one less than Bits Stored (0028,0101).\n \n \n* Pixel Representation **(0028, 0103)**\n * Data representation of the pixel samples. Each sample shall have the same pixel representation.\n \n \n* Window Center **(0028, 1050)**\n * Defines a Window Center for display.\n \n\n* Window Width **(0028, 1051)**\n * Window Width for display.\n \n \n* Rescale Intercept **(0028, 1052)** \n * The value b in relationship between stored values (SV) and the output units.\n * Output units = m*SV+b\n * If Image Type (0008,0008) Value 1 is ORIGINAL and Value 3 is not LOCALIZER, and Multi-energy CT Acquisition (0018,9361) is either absent or NO, output units shall be Hounsfield Units (HU).\n \n \n* Rescale Slope **(0028, 1053)**\n * m in the equation specified in Rescale Intercept (0028,1052).\n \n \n* Pixel Data **(7fe0, 0010)** \n * A data stream of the pixel samples that comprise the Image.","metadata":{}},{"cell_type":"code","source":"im_path = list()\n\nfor i in os.listdir(TRAIN_DIR): \n for j in os.listdir(TRAIN_DIR + i):\n x = i+'/'+j\n im_path.append(x)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:38:22.118006Z","iopub.execute_input":"2022-08-29T09:38:22.118469Z","iopub.status.idle":"2022-08-29T09:38:25.791192Z","shell.execute_reply.started":"2022-08-29T09:38:22.118436Z","shell.execute_reply":"2022-08-29T09:38:25.790366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixelspacing_r = []\npixelspacing_c = []\nrows = []\ncolumns = []\nids = []\nslice_thicknesses = []\nkvp = []\nmodality = []\ntable_height = []\nx_ray = []\nexposure = []\npatient_position = []\ndetector_tilt = []\nbits_allocated = []\nrescale_intercept = []\nrescale_slope = []\nphotometric_interpretation = []\nconvolution_kernel = [] \n\nfor i in im_path:\n ids.append(i.split('/')[0]+'_'+i.split('/')[1])\n example_dcm = os.listdir(TRAIN_DIR + i + \"/\")[0]\n dataset = dcm.dcmread(TRAIN_DIR + i + \"/\" + example_dcm)\n\n spacing = dataset.PixelSpacing\n pixelspacing_r.append(spacing[0])\n pixelspacing_c.append(spacing[1])\n rows.append(dataset.Rows)\n columns.append(dataset.Columns)\n slice_thicknesses.append(dataset.SliceThickness)\n kvp.append(dataset.KVP)\n modality.append(dataset.Modality)\n table_height.append(dataset.TableHeight)\n x_ray.append(dataset.XRayTubeCurrent)\n exposure.append(dataset.Exposure)\n patient_position.append(dataset.PatientPosition)\n detector_tilt.append(dataset.GantryDetectorTilt)\n bits_allocated.append(dataset.BitsAllocated)\n rescale_intercept.append(dataset.RescaleIntercept)\n rescale_slope.append(dataset.RescaleSlope)\n photometric_interpretation.append(dataset.PhotometricInterpretation)\n convolution_kernel.append(dataset.ConvolutionKernel)\n \nscan_properties = pd.DataFrame(data=ids, columns=[\"ID\"])\nscan_properties.loc[:, \"pixelspacing_r\"] = pixelspacing_r\nscan_properties.loc[:, \"pixelspacing_c\"] = pixelspacing_c\nscan_properties.loc[:, \"rows\"] = rows\nscan_properties.loc[:, \"columns\"] = columns\nscan_properties.loc[:, \"slice_thicknesses\"] = slice_thicknesses\nscan_properties.loc[:, \"kvp\"] = kvp\nscan_properties.loc[:, \"modality\"] = modality\nscan_properties.loc[:, \"table_height\"] = table_height\nscan_properties.loc[:, \"x_ray_tube_current\"] = x_ray\nscan_properties.loc[:, \"exposure\"] = exposure\nscan_properties.loc[:, \"patient_position\"] = patient_position\nscan_properties.loc[:, \"gantry/detector_tilt\"] = detector_tilt\nscan_properties.loc[:, \"bits_allocated\"] = bits_allocated\nscan_properties.loc[:, \"rescale_intercept\"] = rescale_intercept\nscan_properties.loc[:, \"rescale_slope\"] = rescale_slope\nscan_properties.loc[:, \"photometric_interpretation\"] = photometric_interpretation\nscan_properties.loc[:, \"convolution_kernel\"] = convolution_kernel\n\nscan_properties","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:38:25.793338Z","iopub.execute_input":"2022-08-29T09:38:25.793953Z","iopub.status.idle":"2022-08-29T09:40:31.424758Z","shell.execute_reply.started":"2022-08-29T09:38:25.793899Z","shell.execute_reply":"2022-08-29T09:40:31.423841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Unique rows number: ', scan_properties['rows'].unique().tolist())\nprint('Unique columns number: ', scan_properties['columns'].unique().tolist())","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:40:31.426099Z","iopub.execute_input":"2022-08-29T09:40:31.426476Z","iopub.status.idle":"2022-08-29T09:40:31.432899Z","shell.execute_reply.started":"2022-08-29T09:40:31.426439Z","shell.execute_reply":"2022-08-29T09:40:31.432026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we can see that all images have 512x512 size that is good for modeling.","metadata":{}},{"cell_type":"code","source":"print('Number of inconsistencies in pixel spacing: ', len(scan_properties[scan_properties['pixelspacing_r'] != scan_properties['pixelspacing_c']]))","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:40:31.434491Z","iopub.execute_input":"2022-08-29T09:40:31.435007Z","iopub.status.idle":"2022-08-29T09:40:31.447792Z","shell.execute_reply.started":"2022-08-29T09:40:31.434967Z","shell.execute_reply":"2022-08-29T09:40:31.446883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = scan_properties['slice_thicknesses'].value_counts().reset_index()\ndata.columns = ['slice_thicknesses', 'count']\ndata['slice_thicknesses'] = 'st: ' + data['slice_thicknesses'].astype(str)\n\nfig = px.bar(\n data, \n x=\"slice_thicknesses\", \n y=\"count\", \n title='slice_thicknesses distribution', \n width=700,\n height=500,\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.536672Z","iopub.execute_input":"2022-08-29T09:40:31.537086Z","iopub.status.idle":"2022-08-29T09:40:31.585585Z","shell.execute_reply.started":"2022-08-29T09:40:31.537042Z","shell.execute_reply":"2022-08-29T09:40:31.584719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = scan_properties['kvp'].value_counts().reset_index()\ndata.columns = ['kvp', 'count']\ndata['kvp'] = 'kvp: ' + data['kvp'].astype(str)\nfig = px.bar(\n data, \n x=\"kvp\", \n y=\"count\", \n title='Peak kilovoltage distribution', \n width=700,\n height=500\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.587089Z","iopub.execute_input":"2022-08-29T09:40:31.587623Z","iopub.status.idle":"2022-08-29T09:40:31.634968Z","shell.execute_reply.started":"2022-08-29T09:40:31.587582Z","shell.execute_reply":"2022-08-29T09:40:31.634072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(\n scan_properties, \n \"pixelspacing_r\", \n nbins=100, \n title='Pixel spacing distribution', \n width=700,\n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.449318Z","iopub.execute_input":"2022-08-29T09:40:31.449751Z","iopub.status.idle":"2022-08-29T09:40:31.532022Z","shell.execute_reply.started":"2022-08-29T09:40:31.449676Z","shell.execute_reply":"2022-08-29T09:40:31.531173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(\n scan_properties, \n \"table_height\", \n nbins=100, \n title='Table_height distribution', \n width=700,\n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.636246Z","iopub.execute_input":"2022-08-29T09:40:31.636572Z","iopub.status.idle":"2022-08-29T09:40:31.701083Z","shell.execute_reply.started":"2022-08-29T09:40:31.636537Z","shell.execute_reply":"2022-08-29T09:40:31.700305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(\n scan_properties, \n \"x_ray_tube_current\", \n nbins=100, \n title='x_ray_tube_current distribution', \n width=700,\n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.702231Z","iopub.execute_input":"2022-08-29T09:40:31.702703Z","iopub.status.idle":"2022-08-29T09:40:31.763245Z","shell.execute_reply.started":"2022-08-29T09:40:31.702665Z","shell.execute_reply":"2022-08-29T09:40:31.762396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(\n scan_properties, \n \"exposure\", \n nbins=100, \n title='exposure distribution', \n width=700,\n height=500\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T09:40:31.764817Z","iopub.execute_input":"2022-08-29T09:40:31.765193Z","iopub.status.idle":"2022-08-29T09:40:31.826788Z","shell.execute_reply.started":"2022-08-29T09:40:31.765157Z","shell.execute_reply":"2022-08-29T09:40:31.825947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = scan_properties['patient_position'].value_counts().reset_index()\ndata.columns = ['patient_position', 'count']\n\nfig = px.bar(\n data, \n x=\"patient_position\", \n y=\"count\", \n title='patient_position distribution', \n width=700,\n height=500\n)\n\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.828469Z","iopub.execute_input":"2022-08-29T09:40:31.828797Z","iopub.status.idle":"2022-08-29T09:40:31.877081Z","shell.execute_reply.started":"2022-08-29T09:40:31.828754Z","shell.execute_reply":"2022-08-29T09:40:31.876103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = scan_properties[\"rescale_intercept\"].value_counts().reset_index()\ndata.columns = [\"rescale_intercept\", 'count']\nfig = px.bar(\n data, \n x=\"rescale_intercept\", \n y=\"count\", \n title='\"rescale_intercept\" distribution', \n width=700,\n height=500\n)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T09:40:31.878674Z","iopub.execute_input":"2022-08-29T09:40:31.879027Z","iopub.status.idle":"2022-08-29T09:40:31.926875Z","shell.execute_reply.started":"2022-08-29T09:40:31.878992Z","shell.execute_reply":"2022-08-29T09:40:31.926015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}