{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y\n!cp ../input/gdcm-data/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np \nimport pandas as pd \nimport os\nfrom pydicom import dcmread # for dcm files\nimport pydicom\nimport cv2\nimport csv\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nfrom plotly.offline import iplot, init_notebook_mode\nfrom keras.losses import categorical_crossentropy\nfrom keras.optimizers import Adadelta\nimport plotly.graph_objs as go\nfrom matplotlib.pyplot import cm\nfrom keras.models import Model\nimport numpy as np\nimport keras\nimport h5py\nimport gdcm\nimport PIL\nimport tensorflow as tf\nfrom skimage import measure, morphology\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LIST OF PATHS"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\npath = '../input/rsna-str-pulmonary-embolism-detection/test'\nimage_paths_test = []\nSOPs_test = []\nseries_test = []\nstudies_test = []\nstudies_paths_test = []\nseries_paths_test = []\nimport os\nfor dirname, _, filenames in os.walk(path):\n    for filename in filenames:\n        temp = os.path.join(dirname, filename)\n        \n        temp_series_path = temp[0:temp.rfind('/')]\n        series_paths_test.append(temp_series_path)\n        \n        temp_series = temp_series_path[temp_series_path.rfind('/')+1:len(temp_series_path)]\n        series_test.append(temp_series)\n        \n        temp_study_path = temp_series_path[0:temp_series_path.rfind('/')]\n        studies_paths_test.append(temp_study_path)\n        \n        temp_study = temp_study_path[temp_study_path.rfind('/')+1:len(temp_study_path)]\n        studies_test.append(temp_study)\n        \n        image_paths_test.append(os.path.join(dirname, filename))\n        SOPs_test.append(temp[len(temp)-16:len(temp)-4])\n        \n        \n\nimage_paths_test = np.asarray(image_paths_test)\nSOPs_test = np.asarray(SOPs_test)\nseries_test = np.asarray(series_test)\nstudies_test = np.asarray(studies_test)\nstudies_paths_test = np.asarray(studies_paths_test)\nseries_paths_test = np.asarray(series_paths_test)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nimage_paths_test = []\nSOPs_test = []\nseries_test = []\nstudies_test = []\nstudies_paths_test = []\nseries_paths_test = []\nimport os\nfor dirname, _, filenames in os.walk('../input/rsna-str-pulmonary-embolism-detection/test'):\n    for filename in filenames:\n        temp = os.path.join(dirname, filename)\n        \n        temp_series_path = temp[0:temp.rfind('/')]\n        series_paths_test.append(temp_series_path)\n        \n        temp_series = temp_series_path[temp_series_path.rfind('/')+1:len(temp_series_path)]\n        series_test.append(temp_series)\n        \n        temp_study_path = temp_series_path[0:temp_series_path.rfind('/')]\n        studies_paths_test.append(temp_study_path)\n        \n        temp_study = temp_study_path[temp_study_path.rfind('/')+1:len(temp_study_path)]\n        studies_test.append(temp_study)\n        \n        image_paths_test.append(os.path.join(dirname, filename))\n        SOPs_test.append(temp[len(temp)-16:len(temp)-4])\n        \n        \n\nimage_paths_test = np.asarray(image_paths_test)\nSOPs_test = np.asarray(SOPs_test)\nseries_test = np.asarray(series_test)\nstudies_test = np.asarray(studies_test)\nstudies_paths_test = np.asarray(studies_paths_test)\nseries_paths_test = np.asarray(series_paths_test)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nprint(temp)\ntemp_series_path = temp[0:temp.rfind('/')]\ntemp_series = temp_series_path[temp_series_path.rfind('/')+1:len(temp_series_path)]\ntemp_study_path = temp_series_path[0:temp_series_path.rfind('/')]\ntemp_study = temp_study_path[temp_study_path.rfind('/')+1:len(temp_study_path)]\n\nprint(temp_study_path)\nprint(\"study\",temp_study)\n\nprint(temp_series_path)\nprint(\"series\",temp_series)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open ('../input/rsna-str-pulmonary-embolism-detection/test.csv') as csvfile:  \n    readCSVFeatures = csv.reader(csvfile, delimiter=',')\n    test_data = list(csv.reader(csvfile))\ntest_data = test_data[1:len(test_data )]    \ntest_data = np.asarray(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_test_csv = test_data[:,0]\ntest_path = np.array(['../input/rsna-str-pulmonary-embolism-detection/test/' for _ in range(len(test_data[:,0]))])\nslash_array = np.array(['/' for _ in range(len(test_data[:,0]))])\ndcm_array = np.array(['.dcm' for _ in range(len(test_data[:,0]))])\nstudy_path = np.char.add(test_path,test_data[:,0]) \nstudies_test = test_data[:,0]\ntemp_array = np.char.add(study_path,slash_array) \nseries_path = np.char.add(temp_array,test_data[:,1]) \nseries_path = np.char.add(series_path,slash_array) \ntemp_array = np.char.add(series_path,slash_array) \nsops_path = np.char.add(temp_array,test_data[:,2]) \nSOPs_test = test_data[:,2]\nsops_path = np.char.add(sops_path,dcm_array) \nstudy_path_temp,study_path_unique_indexes =  np.unique(studies_test,return_index = True)\nsorted_path  =np.sort(study_path_unique_indexes)\nstudy_path_unique = study_path[sorted_path]\nstudies_test_unique = studies_test[sorted_path]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(studies_test[-1])\nprint(sops_path[-1])\nprint(test_data[:,0][-1])\nprint(len(sops_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nprint(len(test_data))\nprint(len(image_paths_test))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nprint(image_paths_test[0])\nprint(studies_test[0])\nstudies_test_index= np.argsort(studies_test)\nimage_paths_test= image_paths_test[studies_test_index]\nstudies_test= studies_test[studies_test_index]\nSOPs_test = SOPs_test[studies_test_index]\nseries_test = series_test[studies_test_index]\nstudies_paths_test = studies_paths_test[studies_test_index] \nseries_paths_test = series_paths_test[studies_test_index]\n\nprint(image_paths_test[0])\nprint(studies_test[0])\n\nstudy_test_csv = test_data[:,0]\nstudy_test_csv_index= np.argsort(study_test_csv)\ntest_data = test_data[study_test_csv_index]\nstudy_test_csv = study_test_csv[study_test_csv_index]\n\nprint(image_paths_test[0])\nprint(test_data[0])\nprint(studies_test[0])\nprint(studies_paths_test[0])\nstudy_test_csv_unique,study_test_csv_uniq_indexes =  np.unique(study_test_csv,return_index = True)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef preprocess_image(path):\n    dataset = dcmread(path)\n    image_dene = np.asarray(dataset.pixel_array)\n    preprocessed_image= cv2.resize(image_dene , (150, 150),  interpolation = cv2.INTER_AREA)\n    preprocessed_image = cv2.merge((preprocessed_image,preprocessed_image,preprocessed_image))\n    preprocessed_image = tf.keras.applications.xception.preprocess_input(preprocessed_image) \n\n    return preprocessed_image\n    \n    \n      \n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.models.load_model('../input/model-last-second-epoch/model_last_second_epoch.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncount_no = 80\n\n#image_paths_test = image_paths_test[0:study_test_csv_uniq_indexes[1700]]\nlen_dataset = len(sops_path)\n\npred_all = np.asarray([])\nfor count in range(0,count_no):\n    print(count)\n    start = int((len_dataset/count_no)*count)\n    end= int((len_dataset/count_no)*(count+1))\n    x_test = np.ones((end-start, 150, 150, 3),  dtype=np.int16)\n    print(\"start\",start)\n    print(\"end\",end)\n    for i in range(start,end):\n        if i % 1000 == 0 :\n            print(i) \n        x_test[i-start]= preprocess_image(sops_path[i])\n    #pred = np.ones((end-start),  dtype=np.int16)\n    pred = model.predict(x_test)\n    pred_binary = np.ones(len(x_test))\n    \n    print(len(np.where(pred>0.5)[0]))\n    print(len(np.where(pred<=0.5)[0]))\n    \n    pred_binary[np.where(pred>0.5)[0]] = 1\n    pred_binary[np.where(pred<=0.5)[0]] = 0    \n    \n    #pred_all = np.concatenate((pred_all,pred_binary),axis = None)\n    pred_all = np.concatenate((pred_all,pred),axis = None)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ncount_no = 100\n\nimage_paths_test = image_paths_test[0:study_test_csv_uniq_indexes[1700]]\nlen_dataset = len(image_paths_test)\nprint(len_dataset)\npred_all = []\nfor i in range(0,len_dataset):\n    if i %100== 0:\n        print(i)\n    x_test= preprocess_image(image_paths_test[i])\n    #print(x_test.shape)\n    x_test = np.reshape(x_test,(1,150,150,3))\n    pred = model.predict(x_test)\n    if pred>0.5:\n        pred_binary = 1\n    if pred<=0.5:\n        pred_binary = 0    \n \n    \n    pred_all.append(pred_binary)\n\npred_all = np.asarray(pred_all)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(len(pred_all))\n#print(len(SOPs_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\"\"\"\n\nx_test = np.ones((int(len(image_paths_test)/10), 150, 150, 3),  dtype=np.int16)\nfor i in range(0,int(len(image_paths_test)/10)):\n    if i % 1000 == 0 :\n        print(i) \n    x_test[i]= preprocess_image(image_paths_test[i])\n\npred = model.predict(x_test)\npred_binary = np.ones(len(x_test))\nprint(len(np.where(pred>0.5)[0]))\nprint(len(np.where(pred<=0.5)[0]))\n\npred_binary[np.where(pred>0.5)[0]] = 1\npred_binary[np.where(pred<=0.5)[0]] = 0  \n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nnegative_exam = np.zeros(len(study_test_csv_unique))\nrv_lv_ratio_lt_1 = np.zeros(len(study_test_csv_unique))\nrv_lv_ratio_gte_1 = np.zeros(len(study_test_csv_unique))\nleftsided_pe = np.zeros(len(study_test_csv_unique))\nchronic_pe = np.zeros(len(study_test_csv_unique))\nrightsided_pe = np.zeros(len(study_test_csv_unique))\nacute_and_chronic_pe = np.zeros(len(study_test_csv_unique))\ncentral_pe = np.zeros(len(study_test_csv_unique))\nindeterminate = np.zeros(len(study_test_csv_unique))\n\nfor x in range(len(study_test_csv_unique)):       \n    del_index = study_test_csv_uniq_indexes[x]\n    if x < len(study_test_csv_unique)-1:\n        del_index_2 = study_test_csv_uniq_indexes[x+1]   \n    else:  \n        del_index_2 = len(SOPs_test)\n    negative = True\n    for i in range(del_index,del_index_2):\n          \n        if pred_all[i] == 1:\n            negative = False\n    \n    if negative = True:\n        negative_exam[x] = 1\n    \n    if negative = False:\n        negative_exam[x] = 0       \n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SOPs_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#del_index = studies_test_unique[0]\n#del_index_2 = studies_test_unique[1]\n\nimport csv\n\nwith open('submission.csv', 'w', newline='') as file:\n    writer = csv.writer(file) \n    writer.writerow([\"id\", \"label\"])\n    for x in range(len(studies_test_unique)):       \n        writer.writerow([str(studies_test_unique[x])+\"_negative_exam_for_pe\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_rv_lv_ratio_gte_1\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_rv_lv_ratio_lt_1\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_leftsided_pe\", str(0.5)])    \n        writer.writerow([str(studies_test_unique[x])+\"_chronic_pe\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_rightsided_pe\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_acute_and_chronic_pe\", str(0.5)])\n        writer.writerow([str(studies_test_unique[x])+\"_central_pe\", str(0.5)])   \n        writer.writerow([str(studies_test_unique[x])+\"_indeterminate\", str(0.5)]) \n    \n\n    for i in range(0,len(SOPs_test)):\n        writer.writerow([str(SOPs_test[i]), str(pred_all[i])]) \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open ('./submission.csv') as csvfile:  \n    readCSVFeatures = csv.reader(csvfile, delimiter=',')\n    submission_data = list(csv.reader(csvfile))\n\nsubmission_data = submission_data[1:len(submission_data)]    \nsubmission_data = np.asarray(submission_data)\nprint(len(submission_data))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(9*len(studies_test_unique)+len(sops_path))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}