{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import random\nimport cv2\nimport pydicom\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom utilities_x_ray import read_xray,showXray\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def seedAll(seed=355):\n# os.environ[\"PYTHONHASHSEED\"] = str(seed)\n# np.random.seed(seed)\n# tf.random.set_seed(seed)\n# random.seed(seed)\n# seedAll()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

First Look at the data

     "},{"metadata":{},"cell_type":"markdown","source":"## 1. DataFrames"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/train.csv')\nss = pd.read_csv('../input/vinbigdata-chest-xray-abnormalities-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.image_id.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train.image_id == '03e6ecfa6f6fb33dfeac6ca4f9b459c9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.class_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### augment this dataset to reduce class imbalance"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.rad_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### R9, R8, R10 seem most hardworking"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_none = train[train.class_name == 'No finding']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_none.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_none.rad_id.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reliable_annotators = ['R8', 'R9', 'R10']\nreliable_annotators = ['R9']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_none_reliable = train_none[train_none.rad_id.isin(reliable_annotators)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_none_reliable.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_reliable = train[train.rad_id.isin(reliable_annotators)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_reliable.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_reliable.class_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.class_name.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### now we will only focus on train reliable"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_reliable","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# docs = train[train.image_id == '03e6ecfa6f6fb33dfeac6ca4f9b459c9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"ss.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The submission file must contain the image id and the prediction string in the format \"a b (c,d,e,f)\"
where\n"},{"metadata":{},"cell_type":"markdown","source":"## 2. Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8,10))\nplt.imshow(read_xray('../input/vinbigdata-chest-xray-abnormalities-detection/train/03e6ecfa6f6fb33dfeac6ca4f9b459c9.dicom'),cmap=plt.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"showXray('../input/vinbigdata-chest-xray-abnormalities-detection/train/03e6ecfa6f6fb33dfeac6ca4f9b459c9.dicom',train,with_boxes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs = train[train.image_id == '03e6ecfa6f6fb33dfeac6ca4f9b459c9'] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs['x'] = docs['x_max'] - docs['x_min']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs['y'] = docs['y_max'] - docs['y_min']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs['area'] = docs['y'] * docs['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs2 = docs.sort_values(by=['area'], ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs2 = docs2.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train2 = docs.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# docs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def complete_overlap(row1, row2):\n \"\"\"\n is box1 from row1 completely inside box2 from row2\n \"\"\"\n x_min_row1 = row1['x_min']\n x_min_row2 = row2['x_min']\n \n x_max_row1 = row1['x_max']\n x_max_row2 = row2['x_max']\n \n \n if (x_min_row1 > x_min_row2) and (x_max_row1 < x_max_row2):\n return True\n\n\n return False\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"complete_overlap(docs.iloc[1], docs.iloc[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"complete_overlap(docs.iloc[3], docs.iloc[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"showXray('../input/vinbigdata-chest-xray-abnormalities-detection/train/03e6ecfa6f6fb33dfeac6ca4f9b459c9.dicom',docs2,with_boxes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

EDA

     "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Number of rows in train dataframe: {}\".format(train.shape[0]))\nprint(\"Number of Unique images in train set: {}\".format(train.image_id.nunique()))\nprint(\"Number of Classes: {}\\n\".format(train.class_name.nunique()))\nprint(\"Class Names: {}\".format(list(train.class_name.unique())))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Null Values:\")\ntrain.isna().sum().to_frame().rename(columns={0:'Null Value count'}).style.background_gradient('viridis')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The number of null values are same as the number of samples that do not have any abnormality"},{"metadata":{},"cell_type":"markdown","source":"### The Distribution of Classes\nWe can see there is a huge class imbalance. The number of negative examples are very high and a few abnormalities have very few examples "},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(9,6))\nsns.countplot(train[\"class_id\"]);\nplt.title(\"Class Distributions\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Distribution of Radiologists"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(9,6))\nsns.countplot(train[\"rad_id\"]);\nplt.title(\"rad_id Distributions\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

An Intuition of the Data

     

\n
Before proceeding further let us try and get an intuition of the data and what exactly we need to do.
\n
In this competition we have been given 15000 images for training. Parallelly we have a dataframe containing the ground truths for various abnormalities. Every sample in the datframe contains:
\n \nImportant points to be noted here are:\n\n

Information about dicom can be found: Here

\n

Procedure to extract DICOM metadata can be found in: this notebook

"},{"metadata":{},"cell_type":"markdown","source":"

Data Preparation

     "},{"metadata":{"trusted":true},"cell_type":"code","source":"class_names = sorted(train.class_name.unique())\ndel class_names[class_names.index('No finding')]\nclass_names = class_names+['No finding']\nclasses = dict(zip(list(range(15)),class_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepareDataFrame(train_df= train):\n train_df = train_df.fillna(0)\n# train_df = train_df.head(10)\n \n cols = ['image_id','label']+list(range(4*len(class_names[:-1])))\n return_df = pd.DataFrame(columns=cols)\n \n for image in tqdm(train_df.image_id.unique()):\n# print('image=', image)\n df = train_df.query(\"image_id==@image\")\n# print('df=', df)\n\n label = np.zeros(15)\n for cls in df.class_id.unique():\n# print('cls=', cls)\n label[int(cls)]=1\n# print('label=', label)\n \n bboxes_df = df.groupby('class_id')[['x_min','y_min','x_max','y_max']].mean().round()\n# print('bboxes_df=', bboxes_df)\n \n bboxes_list = [0 for i in range(60)]\n for ind in list(bboxes_df.index):\n bboxes_list[4*ind:4*ind+4] = list(bboxes_df.loc[ind,:].values)\n return_df.loc[len(return_df),:] = [image]+[label]+bboxes_list[:-4]\n \n# print('===========\\n')\n \n return return_df\ntrain_df = prepareDataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### filter just on class-name for now"},{"metadata":{"trusted":true},"cell_type":"code","source":"docs = train_df[train_df.image_id == '03e6ecfa6f6fb33dfeac6ca4f9b459c9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs.iloc[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs.iloc[0].label ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_cols = ['image_id', 'label']\ntrain_df = train_df[my_cols]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs = train_df[train_df.image_id == '03e6ecfa6f6fb33dfeac6ca4f9b459c9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"docs.iloc[0].label ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### now split for model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# help(KFold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generateFolds(n_splits = None):\n kf = KFold(n_splits= n_splits)\n for id,(tr_,val_) in enumerate(kf.split(train_df[\"image_id\"],train_df[\"label\"])):\n train_df.loc[val_,'kfold'] = int(id)\n train_df[\"kfold\"].astype(int)\n\ngenerateFolds(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.kfold.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df.head(2000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DataLoader:\n def __init__(self,path = None,train_df=train_df,val_df=None):\n self.path = path\n self.df = train_df\n self.val_df = val_df\n self.train_list = [f'{img}.npy' for img in train_df[\"image_id\"].unique()]\n np.random.shuffle(self.train_list)\n self.test_list = [f'{img}.npy' for img in val_df[\"image_id\"].unique()]\n np.random.shuffle(self.test_list)\n \n def read_image(self):\n for img in self.train_list:\n im_name = img.split('.npy')[0]\n image = np.load(self.path+img)\n temp = self.df[self.df.image_id==im_name]\n c_label,bb = temp.iloc[0,1],temp.iloc[0,2:].values.astype('float')\n yield image,c_label,bb\n \n \n def batch_generator(self,items,batch_size):\n a=[]\n i=0\n for item in items:\n a.append(item)\n i+=1\n\n if i%batch_size==0:\n yield a\n a=[]\n if len(a) is not 0:\n yield a\n \n def flow(self,batch_size):\n \"\"\"\n flow from given directory in batches\n ==========================================\n batch_size: size of the batch\n \"\"\"\n while True:\n for bat in self.batch_generator(self.read_image(),batch_size):\n batch_images = []\n batch_c_labels = []\n batch_bb = []\n for im,im_c_label,im_bb in bat:\n batch_images.append(im)\n batch_c_labels.append(im_c_label)\n batch_bb.append(im_bb)\n batch_images = np.stack(batch_images,axis=0)\n\n# batch_labels = (np.stack(batch_c_labels,axis=0),np.stack(batch_bb,axis=0))\n batch_labels = np.stack(batch_c_labels,axis=0)\n yield batch_images,batch_labels\n \n def getVal(self):\n images = []\n c_labels = []\n bb_labels = []\n for img in self.test_list:\n im_name = img.split('.npy')[0]\n image = np.load(self.path+img)\n temp = self.val_df[self.val_df.image_id==im_name]\n c_label,bb = temp.iloc[0,1],temp.iloc[0,2:].values.astype('float')\n images.append(image)\n c_labels.append(c_label)\n bb_labels.append(bb)\n\n# return np.stack(images,axis=0),(np.stack(c_labels,axis=0),np.stack(bb_labels,axis=0))\n return np.stack(images,axis=0),np.stack(c_labels,axis=0)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# help(np.stack)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

Model Building and Training

     "},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n# import tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import regularizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# help(regularizers.l2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.metrics import Recall, Precision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_v1():\n in1 = tf.keras.layers.Input(shape=(256,256,1))\n \n# out1 = tf.keras.layers.Conv2D(4,(3,3),activation=\"relu\")(in1)\n out1 = tf.keras.layers.Conv2D(32,(3,3),\n activation=\"relu\",\n padding='same')(in1)\n out1 = tf.keras.layers.MaxPooling2D((2,2))(out1)\n \n out1 = tf.keras.layers.Conv2D(32,(3,3),\n activation=\"relu\",\n padding='same')(out1)\n out1 = tf.keras.layers.MaxPooling2D((2,2))(out1)\n\n out1 = tf.keras.layers.Flatten()(out1)\n\n out2 = tf.keras.layers.Dense(30,activation=\"relu\")(out1)\n out2 = tf.keras.layers.Dense(30,activation=\"relu\")(out2)\n out2 = tf.keras.layers.Dense(15,\n activation=\"sigmoid\",\n name='class_out', \n kernel_regularizer=regularizers.l2(0.01))(out2)\n\n model = tf.keras.Model(inputs=in1,outputs=out2)\n model.compile(loss={'class_out':'categorical_crossentropy'},\n optimizer=\"adam\",\n metrics=[Recall(), Precision(), 'accuracy'])\n\n\n return model ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"model = build_v1()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"

Training Loop

"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def getTest(path=None):\n# images = []\n# for img in tqdm(os.listdir(path)):\n# im_name = img.split('.npy')[0]\n# image = np.load(path+img)\n# images.append(image)\n# return np.stack(images,axis=0)\n\n# # X_test = getTest('../input/xraynumpy/images/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_label = np.zeros((len(X_test),15))\n# bb_label = np.zeros((len(X_test),56))\n\nfor fold in range(3):\n print(f'\\nFold: {fold}\\n')\n \n# X_train = train_df[train_df.kfold!=fold].drop('kfold',axis=1)\n# X_val = train_df[train_df.kfold==fold].drop('kfold',axis=1)\n X_train = train_df[train_df.kfold!=fold]\n X_val = train_df[train_df.kfold==fold]\n print('X_train.shape=', X_train.shape)\n print('X_train.head()=', X_train.head())\n \n print('-----------\\n')\n \n print('X_val.shape=', X_val.shape)\n print('X_val.head()=', X_val.head())\n\n \n print('-----------\\n')\n dl = DataLoader('../input/xraynumpy/images/train/',X_train,X_val)\n train_set = dl.flow(batch_size=32)\n\n X_eval,Y_eval = dl.getVal()\n# print('X_eval[0]=', X_eval[0])\n print('X_eval.shape=', X_eval.shape)\n\n# print('Y_eval[0]=', Y_eval[0])\n print('Y_eval.shape=', Y_eval.shape)\n \n \n chckpt = tf.keras.callbacks.ModelCheckpoint(f'./model_f{fold}.hdf5',monitor='val_loss',mode='min',save_best_only=True)\n \n K.clear_session()\n model = build_v1()\n \n print('-----------\\n')\n model.fit(train_set,\n epochs=10,\n steps_per_epoch=int(15000/32),\n validation_data = (X_eval,Y_eval),\n callbacks = [chckpt]\n )\n \n break\n \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chckpt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls {'./model_f0.hdf5'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.load_weights('./model_f0.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y_eval.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y_eval[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = model.predict(X_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### now predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_test.shape\nX_eval.shape\n# X_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# c = model.predict(X_test)\nc = model.predict(X_eval)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_eval[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val.iloc[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"showXray('../input/vinbigdata-chest-xray-abnormalities-detection/train/9a5094b2563a1ef3ff50dc5c7ff71345.dicom',train,with_boxes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_label+=c\n# bb_label+=b\n# class_label = class_label/5\n# bb_label = bb_label/5\n# np.save('./class_label.npy',class_label)\n# np.save('./bb_label.npy',bb_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### get perf metrics per class"},{"metadata":{"trusted":true},"cell_type":"code","source":"y = Y_eval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(X_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_true_pos(y, pred, th=0.5):\n pred_t = (pred > th)\n return np.sum((pred_t == True) & (y == 1))\n\n\ndef get_true_neg(y, pred, th=0.5):\n pred_t = (pred > th)\n return np.sum((pred_t == False) & (y == 0))\n\n\ndef get_false_neg(y, pred, th=0.5):\n pred_t = (pred > th)\n return np.sum((pred_t == False) & (y == 1))\n\n\ndef get_false_pos(y, pred, th=0.5):\n pred_t = (pred > th)\n return np.sum((pred_t == True) & (y == 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def true_positives(y, pred, th=0.5):\n \"\"\"\n Count true positives.\n\n Args:\n y (np.array): ground truth, size (n_examples)\n pred (np.array): model output, size (n_examples)\n th (float): cutoff value for positive prediction from model\n Returns:\n TP (int): true positives\n \"\"\"\n TP = 0\n \n # get thresholded predictions\n thresholded_preds = pred >= th\n\n # compute TP\n TP = np.sum((y == 1) & (thresholded_preds == 1))\n \n return TP\n\ndef true_negatives(y, pred, th=0.5):\n \"\"\"\n Count true negatives.\n\n Args:\n y (np.array): ground truth, size (n_examples)\n pred (np.array): model output, size (n_examples)\n th (float): cutoff value for positive prediction from model\n Returns:\n TN (int): true negatives\n \"\"\"\n TN = 0\n \n # get thresholded predictions\n thresholded_preds = pred >= th\n\n ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n \n # compute TN\n TN = np.sum((y == 0) & (thresholded_preds == 0))\n\n ### END CODE HERE ###\n \n return TN\n\ndef false_positives(y, pred, th=0.5):\n \"\"\"\n Count false positives.\n\n Args:\n y (np.array): ground truth, size (n_examples)\n pred (np.array): model output, size (n_examples)\n th (float): cutoff value for positive prediction from model\n Returns:\n FP (int): false positives\n \"\"\"\n FP = 0\n \n # get thresholded predictions\n thresholded_preds = pred >= th\n \n ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n\n # compute FP\n FP = np.sum((y == 0) & (thresholded_preds == 1))\n\n ### END CODE HERE ###\n \n return FP\n\ndef false_negatives(y, pred, th=0.5):\n \"\"\"\n Count false positives.\n\n Args:\n y (np.array): ground truth, size (n_examples)\n pred (np.array): model output, size (n_examples)\n th (float): cutoff value for positive prediction from model\n Returns:\n FN (int): false negatives\n \"\"\"\n FN = 0\n \n # get thresholded predictions\n thresholded_preds = pred >= th\n\n ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n \n # compute FN\n FN = np.sum((y == 1) & (thresholded_preds == 0))\n\n ### END CODE HERE ###\n \n return FN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_accuracy(y, pred, th=0.5):\n \"\"\"\n Compute accuracy of predictions at threshold.\n\n Args:\n y (np.array): ground truth, size (n_examples)\n pred (np.array): model output, size (n_examples)\n th (float): cutoff value for positive prediction from model\n Returns:\n accuracy (float): accuracy of predictions at threshold\n \"\"\"\n accuracy = 0.0\n \n ### START CODE HERE (REPLACE INSTANCES OF 'None' with your code) ###\n \n # get TP, FP, TN, FN using our previously defined functions\n TP = true_positives(y, pred, th)\n FP = false_positives(y, pred, th)\n TN = true_negatives(y, pred, th)\n FN = false_negatives(y, pred, th)\n\n # Compute accuracy using TP, FP, TN, FN\n accuracy = (TP + TN) / (TP + TN + FP + FN) \n \n ### END CODE HERE ###\n \n return accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_performance_metrics(y, \n pred, \n class_labels, \n tp=get_true_pos,\n tn=get_true_neg, \n fp=get_false_pos,\n fn=get_false_neg,\n acc=None, \n prevalence=None, \n spec=None,\n sens=None, \n ppv=None, \n npv=None, \n auc=None, \n f1=None,\n thresholds=[]):\n if len(thresholds) != len(class_labels):\n thresholds = [.5] * len(class_labels)\n\n columns = [\"\", \"TP\", \"TN\", \"FP\", \"FN\", \"Accuracy\", \"Prevalence\",\n \"Sensitivity\",\n \"Specificity\", \"PPV\", \"NPV\", \"AUC\", \"F1\", \"Threshold\"]\n df = pd.DataFrame(columns=columns)\n for i in range(len(class_labels)):\n df.loc[i] = [\"\"] + [0] * (len(columns) - 1)\n df.loc[i][0] = class_labels[i]\n df.loc[i][1] = round(tp(y[:, i], pred[:, i]),\n 3) if tp != None else \"Not Defined\"\n df.loc[i][2] = round(tn(y[:, i], pred[:, i]),\n 3) if tn != None else \"Not Defined\"\n df.loc[i][3] = round(fp(y[:, i], pred[:, i]),\n 3) if fp != None else \"Not Defined\"\n df.loc[i][4] = round(fn(y[:, i], pred[:, i]),\n 3) if fn != None else \"Not Defined\"\n df.loc[i][5] = round(acc(y[:, i], pred[:, i], thresholds[i]),\n 3) if acc != None else \"Not Defined\"\n df.loc[i][6] = round(prevalence(y[:, i]),\n 3) if prevalence != None else \"Not Defined\"\n df.loc[i][7] = round(sens(y[:, i], pred[:, i], thresholds[i]),\n 3) if sens != None else \"Not Defined\"\n df.loc[i][8] = round(spec(y[:, i], pred[:, i], thresholds[i]),\n 3) if spec != None else \"Not Defined\"\n df.loc[i][9] = round(ppv(y[:, i], pred[:, i], thresholds[i]),\n 3) if ppv != None else \"Not Defined\"\n df.loc[i][10] = round(npv(y[:, i], pred[:, i], thresholds[i]),\n 3) if npv != None else \"Not Defined\"\n df.loc[i][11] = round(auc(y[:, i], pred[:, i]),\n 3) if auc != None else \"Not Defined\"\n df.loc[i][12] = round(f1(y[:, i], pred[:, i] > thresholds[i]),\n 3) if f1 != None else \"Not Defined\"\n df.loc[i][13] = round(thresholds[i], 3)\n\n df = df.set_index(\"\")\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_labels = list(classes.values())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_performance_metrics(y, \n y_pred, \n class_labels, \n acc=get_accuracy, \n auc=roc_auc_score,\n f1=f1_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### now try different and simpler model"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_v2():\n in1 = tf.keras.layers.Input(shape=(256,256,1))\n \n out1 = tf.keras.layers.Conv2D(64,(3,3),\n activation=\"relu\")(in1)\n \n \n out1 = tf.keras.layers.MaxPooling2D((2,2))(out1)\n\n out1 = tf.keras.layers.Conv2D(64,(3,3),\n activation=\"relu\")(out1)\n \n out1 = tf.keras.layers.MaxPooling2D((2,2))(out1)\n\n out1 = tf.keras.layers.Flatten()(out1)\n\n out2 = tf.keras.layers.Dense(128,activation=\"relu\")(out1)\n\n out2 = tf.keras.layers.Dense(15,\n activation=\"sigmoid\")(out2)\n\n model2 = tf.keras.Model(inputs=in1,outputs=out2)\n\n return model2 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = build_v2()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model2.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\nmodel2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_label = np.zeros((len(X_test),15))\n# bb_label = np.zeros((len(X_test),56))\n\nfor fold in range(3):\n print(f'\\nFold: {fold}\\n')\n \n# X_train = train_df[train_df.kfold!=fold].drop('kfold',axis=1)\n# X_val = train_df[train_df.kfold==fold].drop('kfold',axis=1)\n X_train = train_df[train_df.kfold!=fold]\n X_val = train_df[train_df.kfold==fold]\n print('X_train.shape=', X_train.shape)\n print('X_train.head()=', X_train.head())\n \n print('-----------\\n')\n \n print('X_val.shape=', X_val.shape)\n print('X_val.head()=', X_val.head())\n\n \n print('-----------\\n')\n dl = DataLoader('../input/xraynumpy/images/train/',X_train,X_val)\n train_set = dl.flow(batch_size=32)\n\n X_eval,Y_eval = dl.getVal()\n# print('X_eval[0]=', X_eval[0])\n print('X_eval.shape=', X_eval.shape)\n\n# print('Y_eval[0]=', Y_eval[0])\n print('Y_eval.shape=', Y_eval.shape)\n \n \n chckpt = tf.keras.callbacks.ModelCheckpoint(f'./model2_f{fold}.hdf5',monitor='val_loss',mode='min',save_best_only=True)\n \n K.clear_session()\n# model = build_v1()\n \n print('-----------\\n')\n model2.fit(train_set,\n epochs=10,\n steps_per_epoch=int(15000/32),\n validation_data = (X_eval,Y_eval),\n callbacks = [chckpt]\n )\n \n break\n \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model2.predict(X_eval)\nmodel2.evaluate(X_eval, Y_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = Y_eval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred2 = model2.predict(X_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_performance_metrics(y, \n y_pred2, \n class_labels, \n acc=get_accuracy, \n auc=roc_auc_score,\n f1=f1_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### retrain model after changing compile method\n\nhttps://datascience.stackexchange.com/questions/25752/how-does-keras-calculate-accuracy-for-multi-label-classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = build_v2()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_label = np.zeros((len(X_test),15))\n# bb_label = np.zeros((len(X_test),56))\n\nfor fold in range(3):\n print(f'\\nFold: {fold}\\n')\n \n# X_train = train_df[train_df.kfold!=fold].drop('kfold',axis=1)\n# X_val = train_df[train_df.kfold==fold].drop('kfold',axis=1)\n X_train = train_df[train_df.kfold!=fold]\n X_val = train_df[train_df.kfold==fold]\n print('X_train.shape=', X_train.shape)\n print('X_train.head()=', X_train.head())\n \n print('-----------\\n')\n \n print('X_val.shape=', X_val.shape)\n print('X_val.head()=', X_val.head())\n\n \n print('-----------\\n')\n dl = DataLoader('../input/xraynumpy/images/train/',X_train,X_val)\n train_set = dl.flow(batch_size=32)\n\n X_eval,Y_eval = dl.getVal()\n# print('X_eval[0]=', X_eval[0])\n print('X_eval.shape=', X_eval.shape)\n\n# print('Y_eval[0]=', Y_eval[0])\n print('Y_eval.shape=', Y_eval.shape)\n \n \n chckpt = tf.keras.callbacks.ModelCheckpoint(f'./model2_f{fold}.hdf5',monitor='val_loss',mode='min',save_best_only=True)\n \n K.clear_session()\n# model = build_v1()\n \n print('-----------\\n')\n model2.fit(train_set,\n epochs=10,\n steps_per_epoch=int(15000/32),\n validation_data = (X_eval,Y_eval),\n callbacks = [chckpt]\n )\n \n break\n \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred2 = model2.predict(X_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_performance_metrics(y, \n y_pred2, \n class_labels, \n acc=get_accuracy, \n auc=roc_auc_score,\n f1=f1_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2b = build_v2()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2b.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_label = np.zeros((len(X_test),15))\n# bb_label = np.zeros((len(X_test),56))\n\nfor fold in range(3):\n print(f'\\nFold: {fold}\\n')\n \n# X_train = train_df[train_df.kfold!=fold].drop('kfold',axis=1)\n# X_val = train_df[train_df.kfold==fold].drop('kfold',axis=1)\n X_train = train_df[train_df.kfold!=fold]\n X_val = train_df[train_df.kfold==fold]\n print('X_train.shape=', X_train.shape)\n print('X_train.head()=', X_train.head())\n \n print('-----------\\n')\n \n print('X_val.shape=', X_val.shape)\n print('X_val.head()=', X_val.head())\n\n \n print('-----------\\n')\n dl = DataLoader('../input/xraynumpy/images/train/',X_train,X_val)\n train_set = dl.flow(batch_size=32)\n\n X_eval,Y_eval = dl.getVal()\n# print('X_eval[0]=', X_eval[0])\n print('X_eval.shape=', X_eval.shape)\n\n# print('Y_eval[0]=', Y_eval[0])\n print('Y_eval.shape=', Y_eval.shape)\n \n \n chckpt = tf.keras.callbacks.ModelCheckpoint(f'./model2_f{fold}.hdf5',monitor='val_loss',mode='min',save_best_only=True)\n \n K.clear_session()\n# model = build_v1()\n \n print('-----------\\n')\n model2b.fit(train_set,\n epochs=10,\n steps_per_epoch=int(15000/32),\n validation_data = (X_eval,Y_eval),\n callbacks = [chckpt]\n )\n \n break\n \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred2b = model2b.predict(X_eval)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = get_performance_metrics(y, \n y_pred2b, \n class_labels, \n acc=get_accuracy, \n auc=roc_auc_score,\n f1=f1_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Work in Progress....\n

To Do:

\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}