{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":24800,"databundleVersionId":1831594,"sourceType":"competition"}],"dockerImageVersionId":30260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom as dicom\nimport matplotlib.image as mpimg\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\n\ndef yaxpb(pxvalue,bt,wt):\n    if pxvalue < bt:\n        y=0\n    elif pxvalue > wt:\n        y=255\n    else:\n        y=pxvalue*255/(wt-bt)-255*bt/(wt-bt)\n    return y\n\n\ndef DicomtoRGB(dicomfile,bt,wt):\n    \"\"\"Create new image(numpy array) filled with certain color in RGB\"\"\"\n    # Create black blank image\n    image = np.zeros((dicomfile.shape[0], dicomfile.shape[1], 3), np.uint8)\n    #loops on image height and width\n    i=0\n    j=0\n    while i<dicomfile.shape[0]:\n        j=0\n        while j<dicomfile.shape[1]:\n            color = yaxpb(dicomfile[i][j],bt,wt) #linear transformation to be adapted\n            image[i][j] = (color,color,color)## same R,G, B value to obtain greyscale\n            j=j+1\n        i=i+1\n    return image\n\n\ndicom_file='../input/vinbigdata-chest-xray-abnormalities-detection/test/02d04b6b6883fd92c12a3dde5d2ff6c0.dicom'\nds = dicom.read_file(dicom_file)\ndcm_sample=ds.pixel_array*128\nprint(type(dcm_sample))\n\nimage=DicomtoRGB(dcm_sample,bt=10000,wt=4000)\nprint(type(image))\n\ngray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n\nplt.imshow(gray,cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T06:07:02.207651Z","iopub.execute_input":"2022-10-15T06:07:02.208112Z","iopub.status.idle":"2022-10-15T06:07:42.764846Z","shell.execute_reply.started":"2022-10-15T06:07:02.208075Z","shell.execute_reply":"2022-10-15T06:07:42.763564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dicom viewing\nimport pydicom as dicom\nimport matplotlib.image as mpimg\nimport numpy as np\nimport matplotlib.pyplot as plt\n# specify your image path\nimport tensorflow as tf\nimport tensorflow_io as tfio\n\nimage_bytes = tf.io.read_file('../input/vinbigdata-chest-xray-abnormalities-detection/train/04bb8bd7ee6f88a16623fe5c6dd4da91.dicom')\n\nimage = tfio.image.decode_dicom_image(image_bytes, dtype=tf.uint16)\n\n#skipped = tfio.image.decode_dicom_image(image_bytes, on_error='skip', dtype=tf.uint8)\n\n#lossy_image = tfio.image.decode_dicom_image(image_bytes, scale='auto', on_error='lossy', dtype=tf.uint8)\nplt.imshow(np.squeeze(image.numpy()),cmap='gray')\nplt.show()\nplt.imsave('./hell.jpg',np.squeeze(image.numpy()),cmap='gray')\n\n\"\"\"image_path = '../input/vinbigdata-chest-xray-abnormalities-detection/test/01f8f17dba6c5e67ba5bcf30ce70ea3f.dicom'\nds = dicom.dcmread(image_path)\n\npixel_array_numpy = ds.pixel_array.astype(float)\nimage_2d=(np.maximum(pixel_array_numpy,0) / pixel_array_numpy.max()) * 255.0\nimage_2d=np.uint8(image_2d)\n\n#img=np.expand_dims(pixel_array_numpy,2)\n#print('img   ',img.shape)\n#R, G, B = img[:,:,0], img[:,:,1], img[:,:,2]\n#imgGray = 0.2989 * R + 0.5870 * G + 0.1140 * B\n#plt.imshow(pixel_array_numpy)\nplt.imshow(image_2d)\nplt.show()\n#mpimg.imsave('./hell.jpg',pixel_array_numpy)\"\"\"\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:53:25.02557Z","iopub.execute_input":"2022-10-15T19:53:25.026079Z","iopub.status.idle":"2022-10-15T19:53:26.982481Z","shell.execute_reply.started":"2022-10-15T19:53:25.026044Z","shell.execute_reply":"2022-10-15T19:53:26.981139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil,os\nimport numpy as np\nimport pandas as pf","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:43:59.099871Z","iopub.execute_input":"2022-10-15T19:43:59.100311Z","iopub.status.idle":"2022-10-15T19:43:59.1058Z","shell.execute_reply.started":"2022-10-15T19:43:59.100274Z","shell.execute_reply":"2022-10-15T19:43:59.104545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import Image, display\nimport numpy as np\nimport os\nfrom os.path import join\nfrom PIL import ImageFile\nimport pandas as pd\nfrom matplotlib import cm\nimport seaborn as sns\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, GlobalAveragePooling2D\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import ResNet50\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error, roc_auc_score, classification_report, confusion_matrix\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn import svm\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.isotonic import IsotonicRegression\nimport re\nimport pydicom as dicom\nimport matplotlib.image as mpimg\nImageFile.LOAD_TRUNCATED_IMAGES = True\nplt.style.use('fivethirtyeight')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:44:08.396774Z","iopub.execute_input":"2022-10-15T19:44:08.397162Z","iopub.status.idle":"2022-10-15T19:44:09.273391Z","shell.execute_reply.started":"2022-10-15T19:44:08.397131Z","shell.execute_reply":"2022-10-15T19:44:09.272031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dicom viewing\nimport pydicom as dicom\nimport matplotlib.image as mpimg\nimport numpy as np\nimport matplotlib.pyplot as plt\n# specify your image path\nimport tensorflow as tf\nimport tensorflow_io as tfio","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:44:17.450298Z","iopub.execute_input":"2022-10-15T19:44:17.450706Z","iopub.status.idle":"2022-10-15T19:44:17.456807Z","shell.execute_reply.started":"2022-10-15T19:44:17.450674Z","shell.execute_reply":"2022-10-15T19:44:17.455888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making disease directory\nos.mkdir('./cardiomegaly')","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:57:10.143019Z","iopub.execute_input":"2022-10-15T19:57:10.143446Z","iopub.status.idle":"2022-10-15T19:57:10.1492Z","shell.execute_reply.started":"2022-10-15T19:57:10.143414Z","shell.execute_reply":"2022-10-15T19:57:10.147685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('./hell')","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:54:35.728392Z","iopub.execute_input":"2022-10-15T19:54:35.728844Z","iopub.status.idle":"2022-10-15T19:54:35.738191Z","shell.execute_reply.started":"2022-10-15T19:54:35.728807Z","shell.execute_reply":"2022-10-15T19:54:35.736729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making directory for not disease\nos.mkdir('./NotDisease')","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:57:36.139288Z","iopub.execute_input":"2022-10-15T19:57:36.139676Z","iopub.status.idle":"2022-10-15T19:57:36.145407Z","shell.execute_reply.started":"2022-10-15T19:57:36.139645Z","shell.execute_reply":"2022-10-15T19:57:36.144174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file=pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\ndata=file.values","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:57:46.402483Z","iopub.execute_input":"2022-10-15T19:57:46.402905Z","iopub.status.idle":"2022-10-15T19:57:46.510688Z","shell.execute_reply.started":"2022-10-15T19:57:46.402871Z","shell.execute_reply":"2022-10-15T19:57:46.509057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#deletiong NoTDiseas\nshutil.rmtree('./NotDisease')","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:54:49.592022Z","iopub.execute_input":"2022-10-15T19:54:49.592443Z","iopub.status.idle":"2022-10-15T19:54:49.600129Z","shell.execute_reply.started":"2022-10-15T19:54:49.592408Z","shell.execute_reply":"2022-10-15T19:54:49.5989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copying not diseases folder\nlabels=list(file['class_name'].unique())\nlabels.remove('Nodule/Mass')\n\nlabels","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:57:52.690179Z","iopub.execute_input":"2022-10-15T19:57:52.690589Z","iopub.status.idle":"2022-10-15T19:57:52.705814Z","shell.execute_reply.started":"2022-10-15T19:57:52.690557Z","shell.execute_reply":"2022-10-15T19:57:52.704506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label in labels:\n    for row in data:\n        if str(row[1])==label:\n                image_path = '../input/vinbigdata-chest-xray-abnormalities-detection/train/'+str(row[0])+'.dicom'\n                image_bytes = tf.io.read_file(image_path)\n\n                image = tfio.image.decode_dicom_image(image_bytes, dtype=tf.uint16)\n\n                #skipped = tfio.image.decode_dicom_image(image_bytes, on_error='skip', dtype=tf.uint8)\n\n                #lossy_image = tfio.image.decode_dicom_image(image_bytes, scale='auto', on_error='lossy', dtype=tf.uint8)\n                plt.imsave('./NotDisease/'+str(row[0])+'.jpg',np.squeeze(image.numpy()))\n                #mpimg.imsave('./hell/'+str(row[0])+'.jpg',np.squeeze(image.numpy()))\n                #shutil.copy('../input/vinbigdata-chest-xray-abnormalities-detection/train/'+str(row[0])+'.dicom','./NotDisease/'+label+'/')\n                break\n            ","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:58:01.871867Z","iopub.execute_input":"2022-10-15T19:58:01.872412Z","iopub.status.idle":"2022-10-15T19:58:18.467846Z","shell.execute_reply.started":"2022-10-15T19:58:01.872363Z","shell.execute_reply":"2022-10-15T19:58:18.466657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('./NotDisease')))","metadata":{"execution":{"iopub.status.busy":"2022-10-15T19:58:34.204876Z","iopub.execute_input":"2022-10-15T19:58:34.205314Z","iopub.status.idle":"2022-10-15T19:58:34.211837Z","shell.execute_reply.started":"2022-10-15T19:58:34.20528Z","shell.execute_reply":"2022-10-15T19:58:34.210422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copying disease data\nfor row in data:\n    if str(row[1])=='Cardiomegaly':\n        image_path = '../input/vinbigdata-chest-xray-abnormalities-detection/train/'+str(row[0])+'.dicom'\n        image_bytes = tf.io.read_file(image_path)\n\n        image = tfio.image.decode_dicom_image(image_bytes, dtype=tf.uint16)\n\n        #skipped = tfio.image.decode_dicom_image(image_bytes, on_error='skip', dtype=tf.uint8)\n\n        #lossy_image = tfio.image.decode_dicom_image(image_bytes, scale='auto', on_error='lossy', dtype=tf.uint8)\n        plt.imsave('./cardiomegaly/'+str(row[0])+'.jpg',np.squeeze(image.numpy()))\n        #mpimg.imsave('./cardiomegaly/'+str(row[0])+'.jpg',np.squeeze(image.numpy()))\n        #shutil.copy('../input/vinbigdata-chest-xray-abnormalities-detection/train/'+str(row[0])+'.dicom','./cardiomegaly/')","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:00:14.207271Z","iopub.execute_input":"2022-10-15T20:00:14.207738Z","iopub.status.idle":"2022-10-15T20:31:18.212861Z","shell.execute_reply.started":"2022-10-15T20:00:14.2077Z","shell.execute_reply":"2022-10-15T20:31:18.210723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#took 853 images for cardiomegaly\nprint(len(os.listdir('./cardiomegaly')))","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:11.553141Z","iopub.execute_input":"2022-10-15T20:34:11.553582Z","iopub.status.idle":"2022-10-15T20:34:11.562011Z","shell.execute_reply.started":"2022-10-15T20:34:11.553549Z","shell.execute_reply":"2022-10-15T20:34:11.560677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import desired disease images from natural images\ntrain_img_dir_n = \"./cardiomegaly\"  #train directory\ntrain_img_paths_n = [join(train_img_dir_n,filename) for filename in os.listdir(train_img_dir_n)]","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:15.184184Z","iopub.execute_input":"2022-10-15T20:34:15.184725Z","iopub.status.idle":"2022-10-15T20:34:15.193787Z","shell.execute_reply.started":"2022-10-15T20:34:15.184682Z","shell.execute_reply":"2022-10-15T20:34:15.192525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split in train test and validation\ntrain_img_paths, test_img_paths_disease = train_test_split(train_img_paths_n, test_size=0.20, random_state=42)\ntrain_img_paths, val_img_paths_disease = train_test_split(train_img_paths, test_size=0.20, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:17.650255Z","iopub.execute_input":"2022-10-15T20:34:17.650989Z","iopub.status.idle":"2022-10-15T20:34:17.658949Z","shell.execute_reply.started":"2022-10-15T20:34:17.650951Z","shell.execute_reply":"2022-10-15T20:34:17.657611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#second class for other than specific disease\nnot_disease_images_path = \"./NotDisease/\"\ntest_img_paths_no_disease = []\n\"\"\"for d in [d for d in os.listdir(not_disease_images_path)]:\n    test_img_dir_na = not_disease_images_path+d\n    test_img_paths_no_disease.append([join(test_img_dir_na,filename) for filename in os.listdir(test_img_dir_na)])\"\"\"\nfor im in os.listdir('./NotDisease'):\n    test_img_paths_no_disease.append(not_disease_images_path+im)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:20.348823Z","iopub.execute_input":"2022-10-15T20:34:20.349255Z","iopub.status.idle":"2022-10-15T20:34:20.355832Z","shell.execute_reply.started":"2022-10-15T20:34:20.349219Z","shell.execute_reply":"2022-10-15T20:34:20.354674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img_paths_no_disease","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:24.201959Z","iopub.execute_input":"2022-10-15T20:34:24.202388Z","iopub.status.idle":"2022-10-15T20:34:24.210727Z","shell.execute_reply.started":"2022-10-15T20:34:24.202353Z","shell.execute_reply":"2022-10-15T20:34:24.209362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img_paths_no_disease_flat = [item for sublist in test_img_paths_no_disease for item in sublist]\ntest_img_paths_no_disease, val_img_paths_no_disease = train_test_split(test_img_paths_no_disease_flat, test_size = 0.25, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:27.191748Z","iopub.execute_input":"2022-10-15T20:34:27.192171Z","iopub.status.idle":"2022-10-15T20:34:27.199191Z","shell.execute_reply.started":"2022-10-15T20:34:27.192138Z","shell.execute_reply":"2022-10-15T20:34:27.198121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create test dataframe\nall_test_paths = test_img_paths_disease+test_img_paths_no_disease\ntest_path_df = pd.DataFrame({\n    'path': all_test_paths,\n    'is_disease': [1 if path in test_img_paths_disease else 0 for path in all_test_paths]\n})\ntest_path_df = shuffle(test_path_df,random_state = 0).reset_index(drop = True)\nall_test_paths = test_path_df['path'].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:30.85493Z","iopub.execute_input":"2022-10-15T20:34:30.855431Z","iopub.status.idle":"2022-10-15T20:34:30.867632Z","shell.execute_reply.started":"2022-10-15T20:34:30.855373Z","shell.execute_reply":"2022-10-15T20:34:30.86667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create validation dataframe\nall_val_paths = val_img_paths_disease+val_img_paths_no_disease\nval_path_df = pd.DataFrame({\n    'path': all_val_paths,\n    'is_disease': [1 if path in val_img_paths_disease else 0 for path in all_val_paths]\n})\nval_path_df = shuffle(val_path_df,random_state = 0).reset_index(drop = True)\nall_val_paths = val_path_df['path'].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:33.936117Z","iopub.execute_input":"2022-10-15T20:34:33.936553Z","iopub.status.idle":"2022-10-15T20:34:33.947269Z","shell.execute_reply.started":"2022-10-15T20:34:33.936518Z","shell.execute_reply":"2022-10-15T20:34:33.945929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_paths","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:31:58.590342Z","iopub.execute_input":"2022-10-15T20:31:58.591443Z","iopub.status.idle":"2022-10-15T20:31:58.612143Z","shell.execute_reply.started":"2022-10-15T20:31:58.591396Z","shell.execute_reply":"2022-10-15T20:31:58.610629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_ok_paths=[f for f in all_test_paths if (f.endswith(\".jpg\"))]","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:47.989303Z","iopub.execute_input":"2022-10-15T20:34:47.989758Z","iopub.status.idle":"2022-10-15T20:34:47.995643Z","shell.execute_reply.started":"2022-10-15T20:34:47.989721Z","shell.execute_reply":"2022-10-15T20:34:47.99451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_test_ok_paths)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:34:58.341782Z","iopub.execute_input":"2022-10-15T20:34:58.342323Z","iopub.status.idle":"2022-10-15T20:34:58.349797Z","shell.execute_reply.started":"2022-10-15T20:34:58.342284Z","shell.execute_reply":"2022-10-15T20:34:58.348462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_val_ok_paths=[f for f in all_val_paths if (f.endswith(\".jpg\"))]","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:35:01.243641Z","iopub.execute_input":"2022-10-15T20:35:01.24426Z","iopub.status.idle":"2022-10-15T20:35:01.251642Z","shell.execute_reply.started":"2022-10-15T20:35:01.244164Z","shell.execute_reply":"2022-10-15T20:35:01.250268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_val_ok_paths)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:35:03.41318Z","iopub.execute_input":"2022-10-15T20:35:03.413646Z","iopub.status.idle":"2022-10-15T20:35:03.42105Z","shell.execute_reply.started":"2022-10-15T20:35:03.413607Z","shell.execute_reply":"2022-10-15T20:35:03.419673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare images for resnet50\n#dicom viewing\nimport pydicom as dicom\nimport matplotlib.image as mpimg\nimage_size = 224\n\ndef read_and_prep_images(img_paths, img_height=image_size, img_width=image_size):\n\n    #imgs = [load_img(img_path, target_size=(img_height, img_width)) for img_path in img_paths\n    imgs=[]\n    for img_path in img_paths:\n        try:\n            imgs.append(load_img(img_path,target_size=(img_height,img_width)))\n        except:\n            continue\n        \n    img_array = np.array([img_to_array(img) for img in imgs])\n    \"\"\"img_array=[]\n    for img in imgs:\n        try:\n            img_array.append(np.array([img_to_array(img)]))\n        except:\n            continue\"\"\"\n    \n    output = img_array\n    #output = preprocess_input(img_array)\n    return(output)\n\nX_train = read_and_prep_images(train_img_paths)\nX_test = read_and_prep_images(all_test_ok_paths)\nX_val = read_and_prep_images(all_val_paths)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:35:07.965741Z","iopub.execute_input":"2022-10-15T20:35:07.966127Z","iopub.status.idle":"2022-10-15T20:36:55.15424Z","shell.execute_reply.started":"2022-10-15T20:35:07.966096Z","shell.execute_reply":"2022-10-15T20:36:55.152984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# X : images numpy array\nresnet_model = ResNet50(input_shape=(image_size, image_size, 3),include_top=False, pooling='avg')  # Since top layer is the fc layer used for predictions\n\nX_train = resnet_model.predict(X_train)\nX_test = resnet_model.predict(X_test)\nX_val = resnet_model.predict(X_val)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:37:08.934868Z","iopub.execute_input":"2022-10-15T20:37:08.935439Z","iopub.status.idle":"2022-10-15T20:38:33.087671Z","shell.execute_reply.started":"2022-10-15T20:37:08.935395Z","shell.execute_reply":"2022-10-15T20:38:33.086671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply standard scaler to output from resnet50\nss = StandardScaler()\nss.fit(X_train)\nX_train = ss.transform(X_train)\nX_test = ss.transform(X_test)\nX_val = ss.transform(X_val)\n\n# Take PCA to reduce feature space dimensionality\npca = PCA(n_components=512, whiten=True)\npca = pca.fit(X_train)\nprint('Explained variance percentage = %0.2f' % sum(pca.explained_variance_ratio_))\nX_train = pca.transform(X_train)\nX_test = pca.transform(X_test)\nX_val = pca.transform(X_val)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T07:50:44.666844Z","iopub.execute_input":"2022-10-14T07:50:44.667273Z","iopub.status.idle":"2022-10-14T07:50:45.05404Z","shell.execute_reply.started":"2022-10-14T07:50:44.66724Z","shell.execute_reply":"2022-10-14T07:50:45.052489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-10-14T07:45:15.554128Z","iopub.execute_input":"2022-10-14T07:45:15.554534Z","iopub.status.idle":"2022-10-14T07:45:15.560503Z","shell.execute_reply.started":"2022-10-14T07:45:15.554501Z","shell.execute_reply":"2022-10-14T07:45:15.559362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oc_svm_clf = svm.OneClassSVM(gamma=0.001, kernel='rbf', nu=0.08)  # Obtained using grid search\nif_clf = IsolationForest(contamination=0.08, max_features=1.0, max_samples=1.0, n_estimators=40)  # Obtained using grid search\n\noc_svm_clf.fit(X_train)\nif_clf.fit(X_train)\n\noc_svm_preds = oc_svm_clf.predict(X_test)\nif_preds = if_clf.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:39:01.084749Z","iopub.execute_input":"2022-10-15T20:39:01.085475Z","iopub.status.idle":"2022-10-15T20:39:01.872622Z","shell.execute_reply.started":"2022-10-15T20:39:01.085428Z","shell.execute_reply":"2022-10-15T20:39:01.871308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(all_test_ok_paths))\nprint(len(oc_svm_preds))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:14:47.185397Z","iopub.execute_input":"2022-10-14T17:14:47.185866Z","iopub.status.idle":"2022-10-14T17:14:47.193684Z","shell.execute_reply.started":"2022-10-14T17:14:47.185829Z","shell.execute_reply":"2022-10-14T17:14:47.191977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(oc_svm_preds))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T17:16:01.732945Z","iopub.execute_input":"2022-10-14T17:16:01.733395Z","iopub.status.idle":"2022-10-14T17:16:01.740568Z","shell.execute_reply.started":"2022-10-14T17:16:01.733359Z","shell.execute_reply":"2022-10-14T17:16:01.739036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_oc_svm_preds=list(oc_svm_preds)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:39:25.146316Z","iopub.execute_input":"2022-10-15T20:39:25.146788Z","iopub.status.idle":"2022-10-15T20:39:25.153302Z","shell.execute_reply.started":"2022-10-15T20:39:25.146751Z","shell.execute_reply":"2022-10-15T20:39:25.151685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_test_ok_paths)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:40:36.439077Z","iopub.execute_input":"2022-10-15T20:40:36.439554Z","iopub.status.idle":"2022-10-15T20:40:36.44753Z","shell.execute_reply.started":"2022-10-15T20:40:36.439517Z","shell.execute_reply":"2022-10-15T20:40:36.44629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(222):\n    print(all_test_ok_paths[i],'   ',l_oc_svm_preds[i])","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:39:41.133292Z","iopub.execute_input":"2022-10-15T20:39:41.133904Z","iopub.status.idle":"2022-10-15T20:39:41.145992Z","shell.execute_reply.started":"2022-10-15T20:39:41.133846Z","shell.execute_reply":"2022-10-15T20:39:41.144857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oc_svm_clf.predict()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trying this oc_svm on notdisease\nbase='./NotDisease'\nfor file in os.listdir(base):\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_if_preds=list(if_preds)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:40:52.531855Z","iopub.execute_input":"2022-10-15T20:40:52.532285Z","iopub.status.idle":"2022-10-15T20:40:52.537778Z","shell.execute_reply.started":"2022-10-15T20:40:52.532248Z","shell.execute_reply":"2022-10-15T20:40:52.536623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(222):\n    print(all_test_ok_paths[i],'   ',l_if_preds[i])","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:41:00.334943Z","iopub.execute_input":"2022-10-15T20:41:00.335364Z","iopub.status.idle":"2022-10-15T20:41:00.346541Z","shell.execute_reply.started":"2022-10-15T20:41:00.335329Z","shell.execute_reply":"2022-10-15T20:41:00.344863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_val_ok_paths)","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:41:46.145687Z","iopub.execute_input":"2022-10-15T20:41:46.146125Z","iopub.status.idle":"2022-10-15T20:41:46.154841Z","shell.execute_reply.started":"2022-10-15T20:41:46.146087Z","shell.execute_reply":"2022-10-15T20:41:46.153188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svm_if_results=pd.DataFrame({\n  'path': all_test_paths,\n  'oc_svm_preds': [0 if x == -1 else 1 for x in oc_svm_preds],\n  'if_preds': [0 if x == -1 else 1 for x in if_preds]\n  \n})\n\n\nsvm_if_results=svm_if_results.merge(test_path_df)\nsvm_if_results.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T07:52:50.525368Z","iopub.execute_input":"2022-10-14T07:52:50.525844Z","iopub.status.idle":"2022-10-14T07:52:50.563849Z","shell.execute_reply.started":"2022-10-14T07:52:50.525801Z","shell.execute_reply":"2022-10-14T07:52:50.562194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\npickle.dump(oc_svm_clf,open('./OneClass_SVM_Cardiomegaly.sav','wb'))\npickle.dump(if_clf,open('./OneClass_IF_Cardiomegaly.sav','wb'))","metadata":{"execution":{"iopub.status.busy":"2022-10-15T20:42:16.945248Z","iopub.execute_input":"2022-10-15T20:42:16.945695Z","iopub.status.idle":"2022-10-15T20:42:16.959515Z","shell.execute_reply.started":"2022-10-15T20:42:16.945659Z","shell.execute_reply":"2022-10-15T20:42:16.957923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(if_clf,open('./OneClass_IsolationForest_Cardiomegaly.sav','wb'))","metadata":{"execution":{"iopub.status.busy":"2022-10-13T06:36:43.744037Z","iopub.execute_input":"2022-10-13T06:36:43.744943Z","iopub.status.idle":"2022-10-13T06:36:43.756561Z","shell.execute_reply.started":"2022-10-13T06:36:43.744894Z","shell.execute_reply":"2022-10-13T06:36:43.754889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(svm_if_results.values)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T07:02:54.88865Z","iopub.execute_input":"2022-10-13T07:02:54.889243Z","iopub.status.idle":"2022-10-13T07:02:54.901808Z","shell.execute_reply.started":"2022-10-13T07:02:54.889194Z","shell.execute_reply":"2022-10-13T07:02:54.900872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#finding last cardiomegaly\nl=df.loc[df['']]","metadata":{},"execution_count":null,"outputs":[]}]}