{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport matplotlib.pyplot as plt\nimport collections\nfrom tqdm import tqdm_notebook as tqdm\nfrom datetime import datetime\n\nfrom math import ceil, floor, log\nimport cv2\n\nimport tensorflow as tf\nimport keras\n\nimport sys\n\n# from keras_applications.resnet import ResNet50\nfrom keras.applications import *\nfrom keras.layers import *\nfrom sklearn.model_selection import ShuffleSplit\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport PIL\nimport tensorflow as tf\nimport pathlib\nfrom tensorflow import keras\nfrom PIL import Image\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom keras.applications import *\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.applications.xception import Xception\nfrom keras.callbacks import EarlyStopping\nfrom keras.utils import plot_model\nfrom keras.callbacks import TensorBoard\n#from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\n#from sklearn.model_selection import train_test_split\n#from tqdm import tqdm\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\ninput_path = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/\"\ntest_images_dir = input_path + 'stage_2_test/'\ntrain_images_dir = input_path + 'stage_2_train/'","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:03.491141Z","iopub.execute_input":"2021-05-22T19:07:03.491518Z","iopub.status.idle":"2021-05-22T19:07:03.502706Z","shell.execute_reply.started":"2021-05-22T19:07:03.491463Z","shell.execute_reply":"2021-05-22T19:07:03.501354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/ID_000012eaf.dcm'\nimg = pydicom.dcmread(path)\nimg","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:03.504634Z","iopub.execute_input":"2021-05-22T19:07:03.505056Z","iopub.status.idle":"2021-05-22T19:07:03.541694Z","shell.execute_reply.started":"2021-05-22T19:07:03.505018Z","shell.execute_reply":"2021-05-22T19:07:03.540824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_dcm(dcm):\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n\ndef window_image(dcm, window_center, window_width):\n    \n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        correct_dcm(dcm)\n    \n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n\n    return img\n\ndef bsb_window(dcm):\n    brain_img = window_image(dcm, 40, 80)\n    subdural_img = window_image(dcm, 80, 200)\n    soft_img = window_image(dcm, 40, 380)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n\n    return bsb_img\n\n# Sanity Check\n# Example dicoms: ID_2669954a7, ID_5c8b5d701, ID_52c9913b1\n\ndicom = pydicom.dcmread(train_images_dir + 'ID_5c8b5d701' + '.dcm')\n#                                     ID  Label\n# 4045566          ID_5c8b5d701_epidural      0\n# 4045567  ID_5c8b5d701_intraparenchymal      1\n# 4045568  ID_5c8b5d701_intraventricular      0\n# 4045569      ID_5c8b5d701_subarachnoid      1\n# 4045570          ID_5c8b5d701_subdural      1\n# 4045571               ID_5c8b5d701_any      1\nplt.imshow(bsb_window(dicom), cmap=plt.cm.bone);\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:03.54395Z","iopub.execute_input":"2021-05-22T19:07:03.544392Z","iopub.status.idle":"2021-05-22T19:07:03.743043Z","shell.execute_reply.started":"2021-05-22T19:07:03.544352Z","shell.execute_reply":"2021-05-22T19:07:03.741938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def window_with_correction(dcm, window_center, window_width):\n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        correct_dcm(dcm)\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef window_without_correction(dcm, window_center, window_width):\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef window_testing(img, window):\n    brain_img = window(img, 40, 80)\n    subdural_img = window(img, 80, 200)\n    soft_img = window(img, 40, 380)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n\n    return bsb_img\n\n# example of a \"bad data point\" (i.e. (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100) == True)\ndicom = pydicom.dcmread(train_images_dir + \"ID_036db39b7\" + \".dcm\")\n\nfig, ax = plt.subplots(1, 2)\n\nax[0].imshow(window_testing(dicom, window_without_correction), cmap=plt.cm.bone);\nax[0].set_title(\"original\")\nax[1].imshow(window_testing(dicom, window_with_correction), cmap=plt.cm.bone);\nax[1].set_title(\"corrected\");","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:03.745105Z","iopub.execute_input":"2021-05-22T19:07:03.745457Z","iopub.status.idle":"2021-05-22T19:07:04.019435Z","shell.execute_reply.started":"2021-05-22T19:07:03.745419Z","shell.execute_reply":"2021-05-22T19:07:04.018357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/rsna-csv-files/RSNA_DATA/good_slices.csv',index_col = 'Unnamed: 0')\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:04.020968Z","iopub.execute_input":"2021-05-22T19:07:04.021317Z","iopub.status.idle":"2021-05-22T19:07:05.073816Z","shell.execute_reply.started":"2021-05-22T19:07:04.021277Z","shell.execute_reply":"2021-05-22T19:07:05.072996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:05.075933Z","iopub.execute_input":"2021-05-22T19:07:05.076195Z","iopub.status.idle":"2021-05-22T19:07:05.093829Z","shell.execute_reply.started":"2021-05-22T19:07:05.076161Z","shell.execute_reply":"2021-05-22T19:07:05.092935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"code","source":"from tqdm import trange\ndef get_partition_labels(df,frac = 0.25):\n    partition = dict()\n    labels = dict()\n    df = df.sample(frac = frac, random_state= 1)\n    for i in trange(len(df)):\n        id_ = df.Image.iloc[i]\n        #print('id = ',id_,type(id_))\n        label = df.iloc[i,1:7].to_numpy(dtype = 'int32')\n        #print('label =',label)\n        labels[id_] = label\n    \n    training = df.sample(frac = 0.6,random_state= 1)\n    validation = df.drop(training.index, axis = 0)\n    test = validation.sample(frac = 0.5,random_state= 1)\n    validation = validation.drop(test.index, axis = 0) \n    partition['train'] = list(training.Image)\n    partition['validation'] = list(validation.Image)\n    partition['test'] = list(test.Image)\n    return partition,labels","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:05.095026Z","iopub.execute_input":"2021-05-22T19:07:05.095528Z","iopub.status.idle":"2021-05-22T19:07:05.105484Z","shell.execute_reply.started":"2021-05-22T19:07:05.095488Z","shell.execute_reply":"2021-05-22T19:07:05.104813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"partition,labels = get_partition_labels(df,frac = 0.25)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:05.107257Z","iopub.execute_input":"2021-05-22T19:07:05.107714Z","iopub.status.idle":"2021-05-22T19:07:22.245764Z","shell.execute_reply.started":"2021-05-22T19:07:05.107673Z","shell.execute_reply":"2021-05-22T19:07:22.244784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _read(path, desired_size):\n    \"\"\"Will be used in DataGenerator\"\"\"\n    \n    dcm = pydicom.dcmread(path)\n    \n    try:\n        img = bsb_window(dcm)\n    except:\n        img = np.zeros(desired_size)\n    \n    \n    img = cv2.resize(img, desired_size[:2], interpolation=cv2.INTER_LINEAR)\n    return img\n\n# Another sanity check \nplt.imshow(\n    _read(train_images_dir+'ID_5c8b5d701'+'.dcm', (256, 256,3)), cmap=plt.cm.bone\n);","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:22.250073Z","iopub.execute_input":"2021-05-22T19:07:22.25209Z","iopub.status.idle":"2021-05-22T19:07:22.462257Z","shell.execute_reply.started":"2021-05-22T19:07:22.252048Z","shell.execute_reply":"2021-05-22T19:07:22.461417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loader","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport keras\n#Data loader specialized \nclass DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, labels, batch_size=32, dim=(32,32,32), n_channels=1,\n                 n_classes=10, shuffle=True):\n        'Initialization'\n        self.dim = dim\n        self.batch_size = batch_size\n        self.labels = labels\n        self.list_IDs = list_IDs\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.shuffle = shuffle\n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n\n        # Generate data\n        if self.labels is not None:\n            X, y = self.__data_generation(list_IDs_temp)\n            return X, y\n        else:\n            X = self.__data_generation(list_IDs_temp)\n            return X\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n    def __data_generation(self, list_IDs_temp):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = np.empty((self.batch_size, *self.dim),dtype = 'float32') #,self.n_channels))\n        if self.labels is not None: # training phase\n            #Use self.n_classes instead of 6 \n            y = np.empty((self.batch_size,6), dtype='float32')\n            #print(y.shape)\n            #print(X.shape)\n            # Generate data\n            for i, ID in enumerate(list_IDs_temp):\n                # Store sample\n                image_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'\n                X[i,] = _read(image_dir+ID+'.dcm',self.dim)\n                #print(X)\n                # Store class\n                y[i] = self.labels[ID]\n                #print(y[i])\n            return X,y\n        \n        else: # test phase\n            for i, ID in enumerate(list_IDs_temp):\n                path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/'\n                X[i,] = _read(path+ID+'.dcm',self.dim)\n            \n            return X","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:22.463599Z","iopub.execute_input":"2021-05-22T19:07:22.464087Z","iopub.status.idle":"2021-05-22T19:07:22.477038Z","shell.execute_reply.started":"2021-05-22T19:07:22.464048Z","shell.execute_reply":"2021-05-22T19:07:22.476285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss functions ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef weighted_log_loss(y_true, y_pred):\n    \"\"\"\n    Can be used as the loss function in model.compile()\n    ---------------------------------------------------\n    \"\"\"\n    \n    class_weights =  tf.constant([2., 1., 1., 1., 1., 1.])\n    \n    eps = tf.keras.backend.epsilon()\n    \n    \n    y_pred = tf.clip_by_value(y_pred, eps, 1.0-eps)\n\n    out = -(         y_true  * tf.math.log(      y_pred) * class_weights\n            + (1.0 - y_true) * tf.math.log(1.0 - y_pred) * class_weights)\n    \n    return tf.reduce_mean(out, axis=-1)\n\n\n####\ndef weighted_log_loss_V2(y_true, y_pred):\n    \"\"\"\n    Can be used as the loss function in model.compile()\n    ---------------------------------------------------\n    \"\"\"\n    \n    class_weights =  tf.constant([2., 1., 1., 1., 1., 1.])\n    \n    eps = tf.keras.backend.epsilon()\n    \n    \n    y_pred = tf.clip_by_value(y_pred, eps, 1.0-eps)\n\n    out = -(         y_true  * tf.math.log(      y_pred) * class_weights\n            + (1.0 - y_true) * tf.math.log(1.0 - y_pred) * class_weights)\n    \n    return tf.reduce_mean(out, axis=-1)\n####\n\ndef _normalized_weighted_average(arr, weights=None):\n    \"\"\"\n    A simple Keras implementation that mimics that of \n    numpy.average(), specifically for this competition\n    \"\"\"\n    \n    if weights is not None:\n        scl = K.sum(weights)\n        weights = K.expand_dims(weights, axis=1)\n        return K.sum(K.dot(arr, weights), axis=1) / scl\n    return K.mean(arr, axis=1)\n\n\ndef weighted_loss(y_true, y_pred):\n    \"\"\"\n    Will be used as the metric in model.compile()\n    ---------------------------------------------\n    \n    Similar to the custom loss function 'weighted_log_loss()' above\n    but with normalized weights, which should be very similar \n    to the official competition metric:\n        https://www.kaggle.com/kambarakun/lb-probe-weights-n-of-positives-scoring\n    and hence:\n        sklearn.metrics.log_loss with sample weights\n    \"\"\"\n    \n    class_weights = tf.constant([2., 1., 1., 1., 1., 1.])\n    \n    eps = tf.keras.backend.epsilon()\n    \n    y_pred = tf.clip_by_value(y_pred, eps, 1.0-eps)\n\n    loss = -(        y_true  * tf.math.log(      y_pred)\n            + (1.0 - y_true) * tf.math.log(1.0 - y_pred))\n    \n    loss_samples = _normalized_weighted_average(loss, class_weights)\n    \n    return tf.reduce_mean(loss_samples)\n\n\ndef weighted_log_loss_metric(trues, preds):\n    \"\"\"\n    Will be used to calculate the log loss \n    of the validation set in PredictionCheckpoint()\n    ------------------------------------------\n    \"\"\"\n    class_weights = [2., 1., 1., 1., 1., 1.]\n    \n    epsilon = 1e-7\n    \n    preds = np.clip(preds, epsilon, 1-epsilon)\n    loss = trues * np.log(preds) + (1 - trues) * np.log(1 - preds)\n    loss_samples = np.average(loss, axis=1, weights=class_weights)\n\n    return - loss_samples.mean()\n'''\ndef log_loss(y_pred,y_true):\n    #y_pred = np.clip(y_pred, 1e-7, 1.0-1e-7)\n    ones = np.where(y_true)\n    zeros = np.where(y_true == 0)\n    likelihood = np.prod(y_pred[ones]) *  np.prod(y_pred[zeros])\n    logloss = -1 * np.log(likelihood)\n    return logloss\n'''\n\ndef log_loss(y_pred,y_true):\n    \"\"\"\n    Will be used to calculate the log loss \n    of the validation set in metrics\n    ------------------------------------------\n    \"\"\"\n    eps = 1e-15\n    \n    y_pred = tf.clip_by_value(y_pred, eps, 1.0 - eps)\n    \n    ones = tf.where(y_true)\n    zeros = tf.where(y_true == 0)\n    \n    likelihood = tf.math.reduce_prod(tf.gather(y_pred,ones)) *  tf.math.reduce_prod(1 - tf.gather(y_pred,zeros))\n    logloss = -1 * tf.math.log(likelihood)\n    return logloss\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:22.481574Z","iopub.execute_input":"2021-05-22T19:07:22.481823Z","iopub.status.idle":"2021-05-22T19:07:22.499194Z","shell.execute_reply.started":"2021-05-22T19:07:22.481798Z","shell.execute_reply":"2021-05-22T19:07:22.49808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"code","source":"params = {'dim':(224,224,3),\n         'batch_size':32,\n         'n_classes':6,\n         'n_channels':0,\n         'shuffle':True}\nepochs = 4\n# Generators\ntraining_generator = DataGenerator(partition['train'], labels, **params)\nvalidation_generator = DataGenerator(partition['validation'], labels, **params)\n\n\n# Design model\nmodel = Sequential([\n    tf.keras.applications.InceptionResNetV2(\n    include_top=False,\n    weights=\"imagenet\",\n    input_shape=params['dim']),\n    Flatten(),\n    Dense(30,activation = 'relu'),\n    Dense(6,activation = 'sigmoid')\n])\nmodel = Sequential([\n    tf.keras.applications.ResNet50(\n    include_top=False,\n    weights=\"imagenet\",\n    input_shape=params['dim'],\n    pooling='max'),\n    Flatten(),\n    Dense(30,activation = 'relu'),\n    Dense(6,activation = 'sigmoid')\n])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:22.500878Z","iopub.execute_input":"2021-05-22T19:07:22.501364Z","iopub.status.idle":"2021-05-22T19:07:32.192511Z","shell.execute_reply.started":"2021-05-22T19:07:22.501325Z","shell.execute_reply":"2021-05-22T19:07:32.191564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Callbacks","metadata":{}},{"cell_type":"code","source":"#Callbacks\n#Callbacks\n#%load_ext tensorboard\n#%tensorboard --logdir logs\n\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nif 'checkpoint' not in os.listdir('./'):\n    os.mkdir('./checkpoint')\nmy_callbacks = [\n    ModelCheckpoint(filepath='./checkpoint', monitor = 'val_weighted_loss' ,save_best_only=True,verbose = 3),\n    ReduceLROnPlateau(monitor= 'val_weighted_loss', factor=0.1, patience= 2, verbose=1,mode='auto', min_delta=0.0001)]\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:32.193866Z","iopub.execute_input":"2021-05-22T19:07:32.194185Z","iopub.status.idle":"2021-05-22T19:07:32.203793Z","shell.execute_reply.started":"2021-05-22T19:07:32.19415Z","shell.execute_reply":"2021-05-22T19:07:32.202946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[weighted_loss,log_loss,keras.metrics.BinaryAccuracy(),tf.keras.metrics.AUC(num_thresholds=200, curve='ROC',summation_method='interpolation', multi_label=True, label_weights=[2, 1, 1, 1, 1, 1])])","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:07:32.204981Z","iopub.execute_input":"2021-05-22T19:07:32.205298Z","iopub.status.idle":"2021-05-22T19:07:32.244385Z","shell.execute_reply.started":"2021-05-22T19:07:32.205264Z","shell.execute_reply":"2021-05-22T19:07:32.243711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"epochs = 10\n# Train model on dataset\nmodel.fit(training_generator,\n         validation_data=validation_generator,\n         epochs=epochs,\n         workers = 4, \n         use_multiprocessing= True,\n         callbacks = my_callbacks)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:10:37.606702Z","iopub.execute_input":"2021-05-22T19:10:37.607068Z","iopub.status.idle":"2021-05-22T21:18:33.84567Z","shell.execute_reply.started":"2021-05-22T19:10:37.607035Z","shell.execute_reply":"2021-05-22T21:18:33.844517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('./model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:52:11.105152Z","iopub.execute_input":"2021-05-22T21:52:11.105687Z","iopub.status.idle":"2021-05-22T21:52:11.446803Z","shell.execute_reply.started":"2021-05-22T21:52:11.105636Z","shell.execute_reply":"2021-05-22T21:52:11.445999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"params = {'dim':(224,224,3),\n         'batch_size':32,\n         'n_classes':6,\n         'n_channels':0,\n         'shuffle':False}\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:52:13.045462Z","iopub.execute_input":"2021-05-22T21:52:13.045955Z","iopub.status.idle":"2021-05-22T21:52:13.053127Z","shell.execute_reply.started":"2021-05-22T21:52:13.045909Z","shell.execute_reply":"2021-05-22T21:52:13.052095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator = DataGenerator(partition['test'], labels, **params)\nmodel.evaluate(test_generator,verbose = 1, use_multiprocessing = True, workers = 4)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:52:13.267184Z","iopub.execute_input":"2021-05-22T21:52:13.267667Z","iopub.status.idle":"2021-05-22T21:55:17.216Z","shell.execute_reply.started":"2021-05-22T21:52:13.267625Z","shell.execute_reply":"2021-05-22T21:55:17.214661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get True Labels","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\ny_true = []\nfor labels in tqdm(test_generator):\n    batch = labels[1]\n    for label in batch:\n        y_true.append(label)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:55:17.218848Z","iopub.execute_input":"2021-05-22T21:55:17.219245Z","iopub.status.idle":"2021-05-22T21:57:48.056875Z","shell.execute_reply.started":"2021-05-22T21:55:17.219215Z","shell.execute_reply":"2021-05-22T21:57:48.055404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = np.array(y_true)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:57:48.058419Z","iopub.execute_input":"2021-05-22T21:57:48.058826Z","iopub.status.idle":"2021-05-22T21:57:48.078627Z","shell.execute_reply.started":"2021-05-22T21:57:48.058788Z","shell.execute_reply":"2021-05-22T21:57:48.077929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = model.predict(test_generator,workers=4, use_multiprocessing=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T21:57:48.080197Z","iopub.execute_input":"2021-05-22T21:57:48.080765Z","iopub.status.idle":"2021-05-22T22:00:19.334747Z","shell.execute_reply.started":"2021-05-22T21:57:48.080724Z","shell.execute_reply":"2021-05-22T22:00:19.333331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\ny_pred_copy = copy.deepcopy(test_preds)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:19.339334Z","iopub.execute_input":"2021-05-22T22:00:19.339741Z","iopub.status.idle":"2021-05-22T22:00:19.345831Z","shell.execute_reply.started":"2021-05-22T22:00:19.3397Z","shell.execute_reply":"2021-05-22T22:00:19.344687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=y_pred_copy\ny_pred[y_pred >= 0.5] = 1\ny_pred[y_pred < 0.5] = 0","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:19.347579Z","iopub.execute_input":"2021-05-22T22:00:19.348145Z","iopub.status.idle":"2021-05-22T22:00:19.357682Z","shell.execute_reply.started":"2021-05-22T22:00:19.348108Z","shell.execute_reply":"2021-05-22T22:00:19.356944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix\nfrom sklearn.datasets import make_classification\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.svm import SVC\ncm_multi = multilabel_confusion_matrix(y_true, y_pred, sample_weight=None, labels=None)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:19.35924Z","iopub.execute_input":"2021-05-22T22:00:19.359692Z","iopub.status.idle":"2021-05-22T22:00:20.056953Z","shell.execute_reply.started":"2021-05-22T22:00:19.359594Z","shell.execute_reply":"2021-05-22T22:00:20.056152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,cm in enumerate(cm_multi):\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=None)\n    disp.plot() \n  ","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:20.060164Z","iopub.execute_input":"2021-05-22T22:00:20.06042Z","iopub.status.idle":"2021-05-22T22:00:21.289861Z","shell.execute_reply.started":"2021-05-22T22:00:20.060394Z","shell.execute_reply":"2021-05-22T22:00:21.289018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission Part","metadata":{}},{"cell_type":"code","source":"def get_test_dcms():\n    path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv'\n    df1 = pd.read_csv(path)\n    df1[\"Image\"] = df1[\"ID\"].str.slice(stop=12)\n    df1[\"Diagnosis\"] = df1[\"ID\"].str.slice(start=13)\n    df1 = df1.loc[:, [ \"Diagnosis\", \"Image\",\"Label\"]]\n    df1 = df1.pivot(index = 'Image', columns = 'Diagnosis', values = 'Label')\n    return list(df1.index)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:21.293873Z","iopub.execute_input":"2021-05-22T22:00:21.295824Z","iopub.status.idle":"2021-05-22T22:00:21.304542Z","shell.execute_reply.started":"2021-05-22T22:00:21.295783Z","shell.execute_reply":"2021-05-22T22:00:21.303509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pred(path = '../input/model.h5'):\n    params = {'dim':(224,224,3),\n         'batch_size':16,\n         'n_classes':6,\n         'n_channels':0,\n         'shuffle':True}\n    dcms = get_test_dcms() #121232 element\n    test_generator = DataGenerator(dcms,labels = None,**params)\n    model = tf.keras.models.load_model( path, compile = False)\n    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[weighted_loss,log_loss,keras.metrics.BinaryAccuracy(),tf.keras.metrics.AUC(num_thresholds=200, curve='ROC',summation_method='interpolation', multi_label=True, label_weights=[2, 1, 1, 1, 1, 1])])\n    preds = model.predict(test_generator,verbose = 1,use_multiprocessing = True, workers = 4)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:00:21.309047Z","iopub.execute_input":"2021-05-22T22:00:21.311604Z","iopub.status.idle":"2021-05-22T22:00:21.322227Z","shell.execute_reply.started":"2021-05-22T22:00:21.311566Z","shell.execute_reply":"2021-05-22T22:00:21.321252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = get_pred(path = './model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:03:53.303652Z","iopub.execute_input":"2021-05-22T22:03:53.304017Z","iopub.status.idle":"2021-05-22T22:26:43.276594Z","shell.execute_reply.started":"2021-05-22T22:03:53.303986Z","shell.execute_reply":"2021-05-22T22:26:43.275254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = get_pred(path = './checkpoint/saved_model.pb')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_submission(preds):\n    dcms = get_test_dcms()\n    deneme = pd.DataFrame(preds, columns=[ 'epidural', 'intraparenchymal', 'intraventricular','subarachnoid', 'subdural','any'], index=dcms)\n    values = deneme.stack().values\n    path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv'\n    submission = pd.read_csv(path)\n    submission.drop_duplicates()\n    submission.Label = values\n    #print(submission.head())\n    submission.to_csv('./submission.csv',index = None)\n    return submission\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:26:43.282866Z","iopub.execute_input":"2021-05-22T22:26:43.283214Z","iopub.status.idle":"2021-05-22T22:26:43.290498Z","shell.execute_reply.started":"2021-05-22T22:26:43.283186Z","shell.execute_reply":"2021-05-22T22:26:43.289284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_submission(preds)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T22:26:43.292Z","iopub.execute_input":"2021-05-22T22:26:43.292691Z","iopub.status.idle":"2021-05-22T22:26:48.540923Z","shell.execute_reply.started":"2021-05-22T22:26:43.292652Z","shell.execute_reply":"2021-05-22T22:26:48.540172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}