{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pydicom\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_rows = 500\npd.options.display.max_columns = 100\npd.options.display.max_colwidth = 200","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TRAIN_IMG_PATH = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/\"\nTEST_IMG_PATH = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/\"\nTRAIN_DATA_PATH = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv\"\nSUBMISSION_PATH = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loaddataset(input):\n    \"\"\"\n    Read csv file and return dataframe\n    \"\"\"\n    return pd.read_csv(input)\n\ndef removeduplicates(df, _keep='first', _inplace=True):\n    df_copy = df.copy()\n    \"\"\"\n    Remove duplicates keeping first row.\n    \"\"\"\n    df_copy = df_copy.drop_duplicates(keep=_keep, inplace=_inplace)\n    return df_copy\n\ndef splitcolumn(df):\n    \"\"\"\n    Method read column value and split in multiple columns. This is very much specific to data.\n    \"\"\"\n    df['Hemorrhage'] = df['ID'].apply(lambda x : str(x).rsplit('_',1)[1])\n    df['ID'] = df['ID'].apply(lambda x : str(x).rsplit('_',1)[0])\n    return df\n\ndef pivot_dataframe(args, df, column_name, value, indexcolumn, IMG_PATH):\n    \"\"\"\n    This method is used to convert row wise data to column wise.\n    column name for which pivoting is required.\n    value to keep for pivoted columns\n    indexcolumn on which dataframe has to be indexed.\n    \"\"\"\n    df = pd.pivot_table(df, columns=column_name, values=value, index=indexcolumn).reset_index()\n    df['filepath'] = f'{IMG_PATH}' + df['ID'] + f'.dcm'\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = loaddataset(TRAIN_DATA_PATH)\ntest = loaddataset(SUBMISSION_PATH)\nprint(f'Shape of train dataset before removing duplicates {train.shape}')\nprint(f'Shape of test dataset before removing duplicates {test.shape}')\nremoveduplicates(train, 'first', True)\nremoveduplicates(test, 'first', True)\nprint(f'Shape of train dataset after removing duplicates {train.shape}')\nprint(f'Shape of test dataset after removing duplicates {test.shape}')\ntrain = splitcolumn(train)\ntest = splitcolumn(test)\ntrain = pivot_dataframe(train, train,['Hemorrhage'],'Label',['ID'], TRAIN_IMG_PATH)\ntest = pivot_dataframe(test, test,['Hemorrhage'],'Label',['ID'], TEST_IMG_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dicom_value(x, cast=int):\n    if type(x) in [pydicom.multival.MultiValue, tuple]:\n        return cast(x[0])\n    else:\n        return cast(x)\n\n\ndef cast(value):\n    if type(value) is pydicom.valuerep.MultiValue:\n        return tuple(value)\n    return value\n\n\ndef get_dicom_raw(dicom):\n    return {attr:cast(getattr(dicom,attr)) for attr in dir(dicom) if attr[0].isupper() and attr not in ['PixelData']}\n\n\ndef rescale_image(image, slope, intercept, bits, pixel):\n    # In some cases intercept value is wrong and can be fixed\n    # Ref. https://www.kaggle.com/jhoward/cleaning-the-data-for-rapid-prototyping-fastai\n    if bits == 12 and pixel == 0 and intercept > -100:\n        image = image.copy() + 1000\n        px_mode = 4096\n        image[image>=px_mode] = image[image>=px_mode] - px_mode\n        intercept = -1000\n    return image.astype(np.float32) * slope + intercept\n\n\ndef apply_window(image, center, width):\n    image = image.copy()\n    min_value = center - width // 2\n    max_value = center + width // 2\n    image[image < min_value] = min_value\n    image[image > max_value] = max_value\n    return image\n\n\ndef get_windowed_ratio(image, center, width):\n    # get ratio of pixels within the window\n    windowed = apply_window(image, center, width)\n    return len(np.where((windowed > 0) & (windowed < 80))[0]) / windowed.size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_record(id, img_path):\n\n    #id, labels = item\n\n    #path = '%s/%s.dcm' % (dirname, id)\n    dicom = pydicom.dcmread(img_path)\n    \n    record = {\n        'ID': id,\n    }\n    record.update(get_dicom_raw(dicom))\n    raw = dicom.pixel_array\n    slope = float(record['RescaleSlope'])\n    intercept = float(record['RescaleIntercept'])\n    center = get_dicom_value(record['WindowCenter'])\n    width = get_dicom_value(record['WindowWidth'])\n    bits= record['BitsStored']\n    pixel = record['PixelRepresentation']\n\n    image = rescale_image(raw, slope, intercept, bits, pixel)\n    doctor = apply_window(image, center, width)\n    brain = apply_window(image, 40, 80)\n\n    record.update({\n        'raw_max': raw.max(),\n        'raw_min': raw.min(),\n        'raw_mean': raw.mean(),\n        'raw_std' : raw.std(),\n        'raw_diff': raw.max() - raw.min(),\n        'doctor_max': doctor.max(),\n        'doctor_min': doctor.min(),\n        'doctor_mean': doctor.mean(),\n        'doctor_std' : doctor.std(),\n        'doctor_diff': doctor.max() - doctor.min(),\n        'brain_max': brain.max(),\n        'brain_min': brain.min(),\n        'brain_mean': brain.mean(),\n        'brain_std' : brain.std(),\n        'brain_diff': brain.max() - brain.min(),\n        'brain_ratio': get_windowed_ratio(image, 40, 80),\n    })\n    return record","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dicommetadata(df):\n    dicom_metadata=[]\n    ids = []\n    for index, row in df.iterrows():\n        try:\n            record = create_record(row['ID'], row['filepath'])\n            dicom_metadata.append(record)\n        except:\n            ids.append(row['ID'])\n            continue\n    return dicom_metadata, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dicom_metadata, corrupted_ids = dicommetadata(train)\ndicom_metadata_test, corrupted_ids_test = dicommetadata(test)\ndicom_df = pd.DataFrame(dicom_metadata)\ndicom_df_test = pd.DataFrame(dicom_metadata_test)\ndicom_df.to_pickle('dicom_df.pkl')\ndicom_df_test.to_pickle('dicom_df_test.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train corrupted ids \", len(corrupted_ids))\nprint(\"test corrupted ids \", len(corrupted_ids_test))\nprint(dicom_df.shape)\nprint(dicom_df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_corrupted_images(ids, df):\n    df_temp = df.copy()\n    for id in ids:\n        df_temp = df_temp.drop(df_temp[df_temp['ID'] == id].index, axis=0)\n    return df_temp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_uncorrupted = remove_corrupted_images(corrupted_ids, train)\ntest_uncorrupted = remove_corrupted_images(corrupted_ids_test, test)\ntrain_uncorrupted.to_pickle('train_uncorrupted.pkl')\ntest_uncorrupted.to_pickle('test_uncorrupted.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train uncorrupted shape \", train_uncorrupted.shape)\nprint(\"test uncorrupted shape \", test_uncorrupted.shape)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}