{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport pydicom\nimport scipy.ndimage\nimport gdcm\n\nfrom os import listdir, mkdir\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"listdir(\"../input/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basepath = \"../input/rsna-str-pulmonary-embolism-detection/\"\nlistdir(basepath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(basepath + \"train.csv\")\ntest_df = pd.read_csv(basepath + \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a list of unique Study Ids\nlist_of_studies = train_df.StudyInstanceUID.unique()\nlist_of_studies.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a list of file directories for each study \ntrain_df[\"dcm_path\"] = basepath + \"train/\" + train_df.StudyInstanceUID + \"/\" + train_df.SeriesInstanceUID + \"/\" + train_df.SOPInstanceUID + \".dcm\"\nlist_of_directories = train_df.dcm_path.unique()\nlist_of_directories.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make list of positive PE studies (negative_exam_for_pe = 0)\npositive_studies = train_df.loc[train_df[\"negative_exam_for_pe\"] == 0]\npositive_studies = positive_studies.StudyInstanceUID.unique()\npositive_studies.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make list of negative PE studies (negative_exam_for_pe = 1)\nnegative_studies_imgs = train_df.loc[train_df[\"negative_exam_for_pe\"] == 1]\nnegative_studies_list = negative_studies_imgs.StudyInstanceUID.unique()\nnegative_studies_list.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# count how many positive images there are\npositive_images = train_df.loc[train_df[\"pe_present_on_image\"] == 1]\npositive_images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"negative_studies_imgs.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"negative_studies_imgs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"positive_images.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's build a dataset that has a shuffled mix of pe positive and negative images"},{"metadata":{"trusted":true},"cell_type":"code","source":"negative_subset = negative_studies_imgs.iloc[0:positive_images.shape[0]]\ncombined_set = positive_images.append(negative_subset)\nrandom_indexes = np.arange(0,combined_set.shape[0] )\nfor i in range(3):\n    np.random.shuffle(random_indexes)\nmixed_set = combined_set.sample(frac=1).reset_index(drop=True)\n#mixed_set = combined_set.iloc[random_indexes[0]]\n#for i in range(1,20000):\n#    mixed_set = mixed_set.append(combined_set.iloc[random_indexes[i]])\n#    if( i % 1000 == 0):\n#        print(\"mixed {} records\".format(i))\n                              \nmixed_set.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_indexes.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dicom_array_and_sort(dcm_path):\n    dicoms = [pydicom.dcmread(file) for file in dcm_path]\n    M = float(dicoms[0].RescaleSlope)\n    B = float(dicoms[0].RescaleIntercept)\n    # Assume all images are axial\n    z_pos = [float(d.ImagePositionPatient[-1]) for d in dicoms]\n    dicoms = np.asarray([d.pixel_array for d in dicoms])\n    dicoms = dicoms[np.argsort(z_pos)]\n    dicoms = dicoms * M\n    dicoms = dicoms + B\n    return dicoms, np.asarray(dcm_path)[np.argsort(z_pos)]\n\n\ndef load_dicom_array(dcm_path):\n    dicoms = [pydicom.dcmread(file) for file in dcm_path]\n    M = float(dicoms[0].RescaleSlope)\n    B = float(dicoms[0].RescaleIntercept)\n    # Assume all images are axial\n    #z_pos = [float(d.ImagePositionPatient[-1]) for d in dicoms]\n    dicoms = np.asarray([d.pixel_array for d in dicoms])\n    #dicoms = dicoms[np.argsort(z_pos)]\n    dicoms = dicoms * M\n    dicoms = dicoms + B\n    return dicoms, np.asarray(dcm_path)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def CT_window(img, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    #X = (X*255.0).astype('uint8')\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the TFExample Data type for training models\n# Our TFRecord format will include the CT Image and metadata of the image, including the prediction label (is PE present)\n\nimport tensorflow as tf\n\n\nPE_WINDOW_LEVEL = 100\nPE_WINDOW_WIDTH = 700\n\n# Utilities serialize data into a TFRecord\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef image_example(image, study_id, image_name, pred_label):\n    image_shape = image.shape\n    image_bytes = image.tostring()\n    feature = {\n        'height': _int64_feature(image_shape[0]),\n        'width': _int64_feature(image_shape[1]),\n        'image_raw': _bytes_feature(image_bytes),\n        'study_id': _bytes_feature(study_id.encode()),\n        'img_name': _bytes_feature(image_name.encode()),\n        'pred_label':  _int64_feature(pred_label)\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))\n\ndef create_tfrecord( images_array, image_file_names, output_path):\n    num_records = images_array.__len__()\n    total_records = 0\n    #opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)\n    opts = tf.io.TFRecordOptions(compression_type=\"GZIP\")\n    with tf.io.TFRecordWriter(output_path, opts) as writer:\n        for index in range(num_records):\n            img_file_name = image_file_names[index]\n            img_file_name = img_file_name.split(\"/\")[-1]\n            img_name = img_file_name.split(\".\")[0]\n            img_data = train_df.loc[train_df[\"SOPInstanceUID\"] == img_name]\n            pred_label = img_data[\"pe_present_on_image\"].values[0]\n            study_id = img_data[\"StudyInstanceUID\"].values[0]\n            # the line below write the original CT image\n            #tf_example = image_example(images_array[index], study_id, img_name, pred_label)\n            # the 2 lines below apply a PE Window function prior to writing the image\n            windowed_image = CT_window(images_array[index], PE_WINDOW_LEVEL, PE_WINDOW_WIDTH)\n            tf_example = image_example(windowed_image, study_id, img_name, pred_label)\n            writer.write(tf_example.SerializeToString())\n            total_records = total_records + 1\n            print(\"*\",end='')\n            #print(\"wrote {}\".format(img_name))\n        writer.close()\n        \n    print(\"wrote {} records\".format(total_records))\n    return total_records\n\n# Create a dictionary describing the features.\nimage_feature_description = {\n    'height': tf.io.FixedLenFeature([], tf.int64),\n    'width': tf.io.FixedLenFeature([], tf.int64),\n    'image_raw': tf.io.FixedLenFeature([], tf.string),\n    'study_id': tf.io.FixedLenFeature([], tf.string),\n    'img_name': tf.io.FixedLenFeature([], tf.string),\n    'pred_label': tf.io.FixedLenFeature([], tf.int64)\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, image_feature_description)\n\n\ndef read_tf_dataset(storage_file_path):\n    encoded_image_dataset = tf.data.TFRecordDataset(storage_file_path, compression_type=\"GZIP\")\n    parsed_image_dataset = encoded_image_dataset.map(_parse_image_function)\n    return parsed_image_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\n\n#mixed_set, random_indexes\n\ndef write_tfrecord_parts( image_data, output_path, file_prefix, number_dirs, records_per_dir, parts_per_record ):\n    for dir_number in range(number_dirs):\n        print('working on directory number {}'.format(dir_number))\n        dir_path = output_path+'dir{}/'.format(dir_number)\n        # create directory\n        if os.path.exists(dir_path):\n            shutil.rmtree(dir_path)\n        os.mkdir(dir_path)\n        for part_number in range(records_per_dir):\n            print(\"working on part {}\".format(part_number))\n            dataset_file_path = dir_path+file_prefix+'dir{}_part{}.tfrecords'.format(dir_number,part_number)\n            lower_range = part_number * parts_per_record\n            upper_range = lower_range + parts_per_record\n            image_set = mixed_set[lower_range:upper_range]\n            dicom_images, dicom_image_file_paths = load_dicom_array(image_set.dcm_path)\n            num_records = create_tfrecord( dicom_images, dicom_image_file_paths, dataset_file_path)\n    \noutput_path = '/kaggle/working/'\nfile_prefix = '/pe_window_shuffled_'\n\nwrite_tfrecord_parts( mixed_set, output_path, file_prefix, 5, 40, 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l '/kaggle/working/dir0'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}