{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport pydicom # For accessing DICOM files\nimport numpy as np\nimport pandas as pd \nimport random as rn\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\n# Importing Libraries for random forest classifier\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n%matplotlib inline\n\ninput_path = '../input/rsna-str-pulmonary-embolism-detection/'\nimg_trainpath = '../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/'\nimg_testpath = '../input/rsna-str-pulmonary-embolism-detection/test/00268ff88746/75d23269adbd/'\ntrain_csv = input_path + 'train.csv'\ntest_csv = input_path + 'test.csv'\n\nseed = 1234\nnp.random.seed(seed)\nrn.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_df = pd.read_csv(train_csv)\ntest_csv_df = pd.read_csv(test_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images = os.listdir(img_trainpath)\ntest_images = os.listdir(img_testpath)\nfirst_dicom_file = pydicom.dcmread(img_trainpath + train_images[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meta_cols=['ImageType','SOPClassUID','SOPInstanceUID',\n               'Modality','SliceThickness','KVP',\n               'TableHeight','RotationDirection','Exposure',\n               'ConvolutionKernel','PatientPosition',\n               'StudyInstanceUID','SeriesInstanceUID','SeriesNumber','InstanceNumber',\n               'ImagePositionPatient','ImageOrientationPatient','PhotometricInterpretation',\n               'Rows','Columns','PixelSpacing','BitsAllocated','BitsStored',\n               'HighBit','PixelRepresentation','WindowCenter','WindowWidth',\n               'RescaleIntercept','RescaleSlope','PixelData','SamplesPerPixel']\ncol_dict_train = {col: [] for col in meta_cols}\ncol_dict_test = {col: [] for col in meta_cols}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for img in tqdm(train_images): \n    dicom_object = pydicom.dcmread(img_trainpath + img)\n    for col in meta_cols: \n        col_dict_train[col].append(str(getattr(dicom_object, col)))\nmeta_df_train = pd.DataFrame(col_dict_train)\ndel col_dict_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for img in tqdm(test_images): \n    dicom_object = pydicom.dcmread(img_testpath + img)\n    for col in meta_cols: \n        col_dict_test[col].append(str(getattr(dicom_object, col)))\nmeta_df_test = pd.DataFrame(col_dict_test)\ndel col_dict_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for df in [meta_df_train, meta_df_test]:\n    # ImagePositionPatient\n    ipp1 = []\n    ipp2 = []\n    ipp3 = []\n    for value in df['ImagePositionPatient'].fillna('[-9999,-9999,-9999]').values:\n        value_list = eval(value)\n        ipp1.append(float(value_list[0]))\n        ipp2.append(float(value_list[1]))\n        ipp3.append(float(value_list[2]))\n    df['ImagePositionPatient_1'] = ipp1\n    df['ImagePositionPatient_2'] = ipp2\n    df['ImagePositionPatient_3'] = ipp3\n    \n    # ImageOrientationPatient\n    iop1 = []\n    iop2 = []\n    iop3 = []\n    iop4 = []\n    iop5 = []\n    iop6 = []\n    # Fill missing values and collect all Image Orientation information\n    for value in df['ImageOrientationPatient'].fillna('[-9999,-9999,-9999,-9999,-9999,-9999]').values:\n        value_list = eval(value)\n        iop1.append(float(value_list[0]))\n        iop2.append(float(value_list[1]))\n        iop3.append(float(value_list[2]))\n        iop4.append(float(value_list[3]))\n        iop5.append(float(value_list[4]))\n        iop6.append(float(value_list[5]))\n    df['ImageOrientationPatient_1'] = iop1\n    df['ImageOrientationPatient_2'] = iop2\n    df['ImageOrientationPatient_3'] = iop3\n    df['ImageOrientationPatient_4'] = iop4\n    df['ImageOrientationPatient_5'] = iop5\n    df['ImageOrientationPatient_6'] = iop6\n    \n    # Pixel Spacing\n    ps1 = []\n    ps2 = []\n    # Fill missing values and collect all pixal spacing features\n    for value in df['PixelSpacing'].fillna('[-9999,-9999]').values:\n        value_list = eval(value)\n        ps1.append(float(value_list[0]))\n        ps2.append(float(value_list[1]))\n    df['PixelSpacing_1'] = ps1\n    df['PixelSpacing_2'] = ps2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save to CSV\nmeta_df_train.to_csv('train_with_metadata.csv', index=False)\nmeta_df_test.to_csv('test_with_metadata.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_traincsv_data = './train_with_metadata.csv'\ndf = pd.read_csv(input_traincsv_data, header=None)\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"column_names=['ImageType','SOPClassUID','SOPInstanceUID',\n               'Modality','SliceThickness','KVP',\n               'TableHeight','RotationDirection','Exposure',\n               'ConvolutionKernel','PatientPosition',\n               'StudyInstanceUID','SeriesInstanceUID','SeriesNumber','InstanceNumber',\n               'ImagePositionPatient','ImageOrientationPatient','PhotometricInterpretation',\n               'Rows','Columns','PixelSpacing','BitsAllocated','BitsStored',\n               'HighBit','PixelRepresentation','WindowCenter','WindowWidth',\n               'RescaleIntercept','RescaleSlope','PixelData','SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2',\n              'ImagePositionPatient_3','ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n              'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2']\ndf.columns = column_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df.drop(['ImageType','SOPClassUID','Modality','SliceThickness','KVP',\n             'TableHeight','RotationDirection','Exposure','ConvolutionKernel','PatientPosition',\n             'SeriesInstanceUID','SeriesNumber','InstanceNumber','ImagePositionPatient','ImageOrientationPatient',\n             'PhotometricInterpretation','Rows','Columns','PixelSpacing','BitsAllocated','BitsStored','HighBit',\n             'PixelRepresentation','WindowCenter','WindowWidth','RescaleIntercept','RescaleSlope','PixelData',\n             'SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2','ImagePositionPatient_3',\n             'ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n             'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df.drop(['SOPInstanceUID','StudyInstanceUID','ImageType'],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.33, random_state = 42)\nX_train.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape, y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\nx_encoder = ce.OrdinalEncoder(cols=['SOPClassUID','Modality','SliceThickness','KVP','TableHeight','RotationDirection','Exposure','ConvolutionKernel','PatientPosition','SeriesInstanceUID','SeriesNumber','InstanceNumber','ImagePositionPatient','ImageOrientationPatient','PhotometricInterpretation',\n               'Rows','Columns','PixelSpacing','BitsAllocated','BitsStored','HighBit','PixelRepresentation','WindowCenter','WindowWidth',\n               'RescaleIntercept','RescaleSlope','PixelData','SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2',\n                                  'ImagePositionPatient_3','ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n                                  'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2'])\nX_train = x_encoder.fit_transform(X_train)\nX_test = x_encoder.fit_transform(X_test)\n\n\ny_encoder = ce.OrdinalEncoder(cols=['SOPInstanceUID','StudyInstanceUID'])\ny_train = y_encoder.fit_transform(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\ny_encoder = ce.OrdinalEncoder(cols=['SOPInstanceUID','StudyInstanceUID'])\ny_test = y_encoder.fit_transform(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yts = y_test.SOPInstanceUID.values\nprint(yts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xts = X_test.values\nprint(xts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtr = X_train.values\nprint(xtr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ytr=y_train.SOPInstanceUID.values\nprint(ytr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nrandom_forest_classifier = RandomForestClassifier(random_state=0)\nrandom_forest_classifier.fit(xtr, ytr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_forest_classifier.score(xtr,ytr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_pred = random_forest_classifier.predict(xts)\nrandom_forest_classifier.score(xts,yts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_scores = pd.Series(random_forest_classifier.feature_importances_, index=X_train.columns).sort_values(ascending=False)\nfeature_scores","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Linear Regression Classifier**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nimport seaborn as sns\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrc = LinearRegression()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"column_names=['ImageType','SOPClassUID','SOPInstanceUID',\n               'Modality','SliceThickness','KVP',\n               'TableHeight','RotationDirection','Exposure',\n               'ConvolutionKernel','PatientPosition',\n               'StudyInstanceUID','SeriesInstanceUID','SeriesNumber','InstanceNumber',\n               'ImagePositionPatient','ImageOrientationPatient','PhotometricInterpretation',\n               'Rows','Columns','PixelSpacing','BitsAllocated','BitsStored',\n               'HighBit','PixelRepresentation','WindowCenter','WindowWidth',\n               'RescaleIntercept','RescaleSlope','PixelData','SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2',\n              'ImagePositionPatient_3','ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n              'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2']\ndf.columns = column_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df.drop(['ImageType','SOPClassUID','Modality','SliceThickness','KVP',\n             'TableHeight','RotationDirection','Exposure','ConvolutionKernel','PatientPosition',\n             'SeriesInstanceUID','SeriesNumber','InstanceNumber','ImagePositionPatient','ImageOrientationPatient',\n             'PhotometricInterpretation','Rows','Columns','PixelSpacing','BitsAllocated','BitsStored','HighBit',\n             'PixelRepresentation','WindowCenter','WindowWidth','RescaleIntercept','RescaleSlope','PixelData',\n             'SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2','ImagePositionPatient_3',\n             'ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n             'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df.drop(['SOPInstanceUID','StudyInstanceUID','ImageType'],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.33, random_state = 42)\nX_train.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\nx_encoder = ce.OrdinalEncoder(cols=['SOPClassUID','Modality','SliceThickness','KVP','TableHeight','RotationDirection','Exposure','ConvolutionKernel','PatientPosition','SeriesInstanceUID','SeriesNumber','InstanceNumber','ImagePositionPatient','ImageOrientationPatient','PhotometricInterpretation',\n               'Rows','Columns','PixelSpacing','BitsAllocated','BitsStored','HighBit','PixelRepresentation','WindowCenter','WindowWidth',\n               'RescaleIntercept','RescaleSlope','PixelData','SamplesPerPixel','ImagePositionPatient_1','ImagePositionPatient_2',\n                                  'ImagePositionPatient_3','ImageOrientationPatient_1','ImageOrientationPatient_2','ImageOrientationPatient_3','ImageOrientationPatient_4',\n                                  'ImageOrientationPatient_5','ImageOrientationPatient_6','PixelSpacing_1','PixelSpacing_2'])\nX_train = x_encoder.fit_transform(X_train)\nX_test = x_encoder.fit_transform(X_test)\n\n\ny_encoder = ce.OrdinalEncoder(cols=['SOPInstanceUID','StudyInstanceUID'])\ny_train = y_encoder.fit_transform(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\ny_encoder = ce.OrdinalEncoder(cols=['SOPInstanceUID','StudyInstanceUID'])\ny_test = y_encoder.fit_transform(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtr = X_train.values\nprint(Xtr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xts = X_test.values\nprint(Xts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Ytr = y_train.SOPInstanceUID.values\nprint(ytr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Yts= y_test.SOPInstanceUID.values\nprint(Yts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrc.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = lrc.predict(Xts)\nprint(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(metrics.mean_squared_error(Yts,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(metrics.mean_absolute_error(Yts,y_pred))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}