{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport pandas as pd\nimport cv2\nimport pathlib \nimport os\nimport string\nfrom PIL import Image\nfrom keras import backend as K\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-28T13:15:56.930582Z","iopub.execute_input":"2023-08-28T13:15:56.931283Z","iopub.status.idle":"2023-08-28T13:15:56.937416Z","shell.execute_reply.started":"2023-08-28T13:15:56.93125Z","shell.execute_reply":"2023-08-28T13:15:56.936423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Introduction","metadata":{}},{"cell_type":"markdown","source":"- What does the convolutional neural network depend on in making its decision? The next project shows the way in which areas can be identified from the medical images through which a decision can be made whether there is a cancerous tumor in the medical images or not and generate a heat map that helps determine the places of most interest for the convolutional neural network. Which helped her make her decision.\n- Two methodologies were used to track the location of the cancer tumor based on a convolutional neural network whose task is to classify the presence of a cancerous disease, and thus the study explains and interprets what the convolutional neural network sees and what it relies on in making the final classification decision.\n- GradCAM / GradCAM++ is used to interpret what a convolutional neural network sees.","metadata":{}},{"cell_type":"markdown","source":"# Preparation and processing of dataset","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nge = ImageDataGenerator(rescale = 1/255,\n                        rotation_range=0.2, \n                        width_shift_range=0.05, \n                        height_shift_range=0.05,\n                        fill_mode = 'constant', \n                        validation_split = 0.2, \n                        horizontal_flip = True, \n                        vertical_flip = True,\n                        zoom_range = 0.2\n                        )","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:54:51.361184Z","iopub.execute_input":"2023-08-28T14:54:51.361577Z","iopub.status.idle":"2023-08-28T14:54:51.368843Z","shell.execute_reply.started":"2023-08-28T14:54:51.361546Z","shell.execute_reply":"2023-08-28T14:54:51.366888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datasetfolder = '/kaggle/input/brian-tumor-dataset/Brain Tumor Data Set/Brain Tumor Data Set'\ndataflowtraining = ge.flow_from_directory(directory = datasetfolder,\n                                 target_size = (224, 224),\n                                 color_mode = 'rgb',\n                                 batch_size = 32, \n                                 shuffle = True,\n                                 subset = 'training')\ndataflowvalidation = ge.flow_from_directory(directory = datasetfolder, \n                                           target_size = (224, 224), \n                                           color_mode = 'rgb', \n                                           batch_size = 32, \n                                           shuffle = True,\n                                           subset = 'validation')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:54:53.199568Z","iopub.execute_input":"2023-08-28T14:54:53.200698Z","iopub.status.idle":"2023-08-28T14:54:55.174879Z","shell.execute_reply.started":"2023-08-28T14:54:53.200654Z","shell.execute_reply":"2023-08-28T14:54:55.173904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = dataflowvalidation.next()\nnp.min(images), np.max(images)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T13:18:53.51585Z","iopub.execute_input":"2023-08-28T13:18:53.516225Z","iopub.status.idle":"2023-08-28T13:18:53.967699Z","shell.execute_reply.started":"2023-08-28T13:18:53.516195Z","shell.execute_reply":"2023-08-28T13:18:53.966703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimages, labels = dataflowvalidation.next()\nplt.figure(figsize = (12, 12))\nfor i in range(32):\n    plt.subplot(6, 6, (i + 1))\n    plt.imshow(images[i])\n    plt.title(labels[i])\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:54:56.153158Z","iopub.execute_input":"2023-08-28T14:54:56.153535Z","iopub.status.idle":"2023-08-28T14:55:01.547238Z","shell.execute_reply.started":"2023-08-28T14:54:56.153503Z","shell.execute_reply":"2023-08-28T14:55:01.545621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convolutional neural network architecture proposal\n\nWe note here, that I did not freeze the first layers of the pre-trained neural network, but rather used weights to give a good or acceptable range and initialization to the weights included in the neural network, instead of starting from a random initialization of the values of the weights.","metadata":{}},{"cell_type":"code","source":"from keras.applications import DenseNet121\nbasemodel = DenseNet121(weights = 'imagenet', include_top = False, \n                        input_shape = (224, 224, 3), pooling = None)\nx = tf.keras.layers.Flatten()(basemodel.output)\nx = tf.keras.layers.Dropout(0.7)(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.Dense(16, activation = 'relu',)(x)\nx = tf.keras.layers.Dropout(0.5)(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.Dense(2, activation = 'softmax')(x)\nm = tf.keras.models.Model(inputs = basemodel.input, outputs = x)\nm.compile(loss = 'binary_crossentropy', \n          optimizer = tf.keras.optimizers.Adam(learning_rate = 0.0001), \n         metrics = ['accuracy', tf.keras.metrics.Precision(name = 'precision'), \n                   tf.keras.metrics.Recall(name = 'recall')])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T13:19:28.713618Z","iopub.execute_input":"2023-08-28T13:19:28.714272Z","iopub.status.idle":"2023-08-28T13:19:32.290153Z","shell.execute_reply.started":"2023-08-28T13:19:28.714229Z","shell.execute_reply":"2023-08-28T13:19:32.289174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = m.fit(dataflowtraining, epochs = 60, batch_size = 32,\n             validation_data = dataflowvalidation,\n             callbacks = [\n                tf.keras.callbacks.EarlyStopping(patience = 8, monitor = 'val_loss', mode = 'min', \n                                                restore_best_weights = True), \n                tf.keras.callbacks.ReduceLROnPlateau(patience = 6, monitor = 'val_loss', \n                                                          mode = 'min', factor = 0.1)\n            ])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T13:19:34.987226Z","iopub.execute_input":"2023-08-28T13:19:34.987993Z","iopub.status.idle":"2023-08-28T14:29:09.076812Z","shell.execute_reply.started":"2023-08-28T13:19:34.987958Z","shell.execute_reply":"2023-08-28T14:29:09.07579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"m.evaluate(dataflowtraining)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:30:24.764433Z","iopub.execute_input":"2023-08-28T14:30:24.764846Z","iopub.status.idle":"2023-08-28T14:31:47.094887Z","shell.execute_reply.started":"2023-08-28T14:30:24.764813Z","shell.execute_reply":"2023-08-28T14:31:47.093714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m.evaluate(dataflowvalidation)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:31:47.09709Z","iopub.execute_input":"2023-08-28T14:31:47.097453Z","iopub.status.idle":"2023-08-28T14:31:58.074132Z","shell.execute_reply.started":"2023-08-28T14:31:47.09742Z","shell.execute_reply":"2023-08-28T14:31:58.073092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize = (12, 6))\nmetrics = ['loss', 'precision', 'recall', 'accuracy']\nfor i in range(4):\n    plt.subplot(2, 2, (i + 1))\n    plt.plot(hist.history[metrics[i]], label = metrics[i])\n    plt.plot(hist.history['val_{}'.format(metrics[i])], label = 'val_{}'.format(metrics[i]))\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:32:03.848399Z","iopub.execute_input":"2023-08-28T14:32:03.848798Z","iopub.status.idle":"2023-08-28T14:32:04.613575Z","shell.execute_reply.started":"2023-08-28T14:32:03.848761Z","shell.execute_reply":"2023-08-28T14:32:04.612673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m.save('/kaggle/working/final_tumor_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T14:33:18.153406Z","iopub.execute_input":"2023-08-28T14:33:18.153786Z","iopub.status.idle":"2023-08-28T14:33:19.605029Z","shell.execute_reply.started":"2023-08-28T14:33:18.153747Z","shell.execute_reply":"2023-08-28T14:33:19.60405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Heat Map:\nIn this section, the focus was on the medical images in which there was a cancerous tumor, and the neural network was asked to predict that it contained a cancerous tumor or not, and at the same time to identify the areas that were relied upon in making a decision that the cancerous tumor was present within it.","metadata":{}},{"cell_type":"code","source":"def readtumorImages(imagespathes):\n    images = []\n    for img in imagespathes:\n        img = cv2.imread(str(img))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = img/255\n        img = cv2.resize(img, (224, 224))\n        images.append(img)\n    return np.array(images)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:15.413747Z","iopub.execute_input":"2023-08-28T15:07:15.414878Z","iopub.status.idle":"2023-08-28T15:07:15.42179Z","shell.execute_reply.started":"2023-08-28T15:07:15.414828Z","shell.execute_reply":"2023-08-28T15:07:15.420573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = readtumorImages(list(pathlib.Path('/kaggle/input/brian-tumor-dataset/Brain Tumor Data Set/Brain Tumor Data Set/Brain Tumor').glob('*.*'))[:1000])\nlabels = tf.keras.utils.to_categorical(tf.zeros(shape = (images.shape[0])), num_classes = 2)\nimages.shape, labels.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:17.813464Z","iopub.execute_input":"2023-08-28T15:07:17.813897Z","iopub.status.idle":"2023-08-28T15:07:24.26947Z","shell.execute_reply.started":"2023-08-28T15:07:17.813863Z","shell.execute_reply":"2023-08-28T15:07:24.268511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indexs = np.random.choice(range(images.shape[0]), size = (images.shape[0], ))\nimages = images[indexs]\nlabels = labels[indexs]\nimages.shape, labels.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:24.271298Z","iopub.execute_input":"2023-08-28T15:07:24.271779Z","iopub.status.idle":"2023-08-28T15:07:24.779133Z","shell.execute_reply.started":"2023-08-28T15:07:24.271726Z","shell.execute_reply":"2023-08-28T15:07:24.778007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = tf.keras.models.load_model('/kaggle/working/final_tumor_model.h5')","metadata":{"execution":{"iopub.status.busy":"2023-08-28T12:06:32.759502Z","iopub.execute_input":"2023-08-28T12:06:32.75989Z","iopub.status.idle":"2023-08-28T12:06:37.667389Z","shell.execute_reply.started":"2023-08-28T12:06:32.75986Z","shell.execute_reply":"2023-08-28T12:06:37.666326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Grad-Cam","metadata":{}},{"cell_type":"code","source":"def gradCam(image, true_label, layer_conv_name):\n    model_grad = tf.keras.models.Model(inputs = m.input, \n                                  outputs = [m.get_layer(layer_conv_name).output, \n                                             m.output])\n    with tf.GradientTape() as tape:\n        conv_output, predictions = model_grad(image)\n        tape.watch(conv_output)\n        loss = tf.losses.binary_crossentropy(true_label, predictions)\n    grad = tape.gradient(loss, conv_output)\n    grad = K.mean(tf.abs(grad), axis = (0, 1, 2))\n    conv_output = np.squeeze(conv_output.numpy())\n    for i in range(conv_output.shape[-1]):\n        conv_output[:,:, i] = conv_output[:,:, i]*grad[i]\n    heatmap = tf.reduce_mean(conv_output, axis = -1)\n    heatmap = np.maximum(heatmap, 0)\n    heatmap = heatmap/tf.reduce_max(heatmap)\n    heatmap = cv2.resize(heatmap.numpy(), (224, 224))\n    return np.squeeze(heatmap), np.squeeze(image)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:48.650657Z","iopub.execute_input":"2023-08-28T15:07:48.65161Z","iopub.status.idle":"2023-08-28T15:07:48.663102Z","shell.execute_reply.started":"2023-08-28T15:07:48.651563Z","shell.execute_reply":"2023-08-28T15:07:48.661923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getHeatMap(images, labels):\n    heatmaps = []\n    for index in range(128):\n        heatmap, image = gradCam(images[index: index + 1], \n                                               labels[index: index + 1], \n                                           'relu')\n        heatmaps.append(heatmap)\n    return np.array(heatmaps)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:50.466427Z","iopub.execute_input":"2023-08-28T15:07:50.466828Z","iopub.status.idle":"2023-08-28T15:07:50.473299Z","shell.execute_reply.started":"2023-08-28T15:07:50.466793Z","shell.execute_reply":"2023-08-28T15:07:50.471852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"heatmaps = getHeatMap(images, labels)\nheatmaps.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:07:53.573862Z","iopub.execute_input":"2023-08-28T15:07:53.575032Z","iopub.status.idle":"2023-08-28T15:10:37.073385Z","shell.execute_reply.started":"2023-08-28T15:07:53.574975Z","shell.execute_reply":"2023-08-28T15:10:37.072392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Grad-Cam++","metadata":{}},{"cell_type":"code","source":"def grad_cam_plus_plus(image, true_label, conv_layer):\n    gradModel = tf.keras.models.Model(inputs = m.input, outputs = [\n        m.get_layer(conv_layer).output, \n        m.output\n    ])\n    with tf.GradientTape() as tape:\n        conv_output, predict = gradModel(image)\n        tape.watch(conv_output)\n        score = predict[:, np.argmax(predict,)]\n    grads = tape.gradient(score, conv_output)\n    grads = K.mean(tf.abs(grads), axis = [0, 1, 2])\n    first = K.exp(score)*grads\n    second = K.exp(score)*grads*grads\n    third = K.exp(score)*grads*grads*grads\n    conv_output = np.squeeze(conv_output)\n    x = conv_output\n    x = np.sum(np.sum(x, axis = 0), axis = 0)\n    grads = (second)/(2*second + third*x)\n    conv_output = np.array(conv_output)\n    for i in range(conv_output.shape[-1]):\n        conv_output[:,:,i] = conv_output[:,:,i]*grads[i]\n    conv_output = tf.reduce_mean(conv_output, axis = -1)\n    heatmap = np.maximum(conv_output, 0)\n    heatmap = heatmap/np.max(heatmap)\n    heatmap = cv2.resize(heatmap, (224, 224))\n    return heatmap","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:10:37.075575Z","iopub.execute_input":"2023-08-28T15:10:37.075958Z","iopub.status.idle":"2023-08-28T15:10:37.08941Z","shell.execute_reply.started":"2023-08-28T15:10:37.075924Z","shell.execute_reply":"2023-08-28T15:10:37.088398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getHeatMap_plus_plus(images, labels):\n    heatmaps = []\n    for index in range(128):\n        heatmap = grad_cam_plus_plus(images[index: index + 1], \n                                               labels[index: index + 1], \n                                           'relu')\n        heatmaps.append(heatmap)\n    return np.array(heatmaps)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:10:37.090798Z","iopub.execute_input":"2023-08-28T15:10:37.091224Z","iopub.status.idle":"2023-08-28T15:10:37.102387Z","shell.execute_reply.started":"2023-08-28T15:10:37.09119Z","shell.execute_reply":"2023-08-28T15:10:37.101284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GradCamplusheatmaps = getHeatMap_plus_plus(images, labels)\nGradCamplusheatmaps.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:10:37.106148Z","iopub.execute_input":"2023-08-28T15:10:37.10713Z","iopub.status.idle":"2023-08-28T15:13:23.238968Z","shell.execute_reply.started":"2023-08-28T15:10:37.107069Z","shell.execute_reply":"2023-08-28T15:13:23.237715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_compare(images, gradcam_heatmaps, \n                 gradcamplus_heatmaps, labels):\n  plt.figure(figsize = (12, 50))\n  index = 0\n  n = 0\n  for i in range(120):\n    plt.subplot(20, 6, (i + 1))\n    if index == 0:\n      plt.imshow(images[n])\n      plt.title('Image-class: {}'.format(labels[n]))\n      index = 1\n    elif index == 1:\n      plt.imshow(images[n])\n      plt.imshow(gradcam_heatmaps[n], alpha = 0.6, cmap = 'jet')\n      plt.title('Grad-Cam')\n      index = 2\n    elif index == 2:\n      plt.imshow(images[n])\n      plt.imshow(gradcamplus_heatmaps[n], alpha = 0.6, cmap = 'jet')\n      plt.title('Grad-Cam++')\n      index = 0\n      n = n + 1\n  plt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:13:23.240823Z","iopub.execute_input":"2023-08-28T15:13:23.242813Z","iopub.status.idle":"2023-08-28T15:13:23.252033Z","shell.execute_reply.started":"2023-08-28T15:13:23.242776Z","shell.execute_reply":"2023-08-28T15:13:23.250747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = np.argmax(labels, axis = 1)\nl.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:13:23.253804Z","iopub.execute_input":"2023-08-28T15:13:23.255004Z","iopub.status.idle":"2023-08-28T15:13:23.272136Z","shell.execute_reply.started":"2023-08-28T15:13:23.254966Z","shell.execute_reply":"2023-08-28T15:13:23.270677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndraw_compare(images[:40], heatmaps[:40], \n                 GradCamplusheatmaps[:40], l[:40])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:13:23.27389Z","iopub.execute_input":"2023-08-28T15:13:23.275094Z","iopub.status.idle":"2023-08-28T15:13:44.386558Z","shell.execute_reply.started":"2023-08-28T15:13:23.275036Z","shell.execute_reply":"2023-08-28T15:13:44.383847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_compare(images[40:80], heatmaps[40:80], \n                 GradCamplusheatmaps[40:80], l[40:80])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:13:44.388516Z","iopub.execute_input":"2023-08-28T15:13:44.389179Z","iopub.status.idle":"2023-08-28T15:14:06.577796Z","shell.execute_reply.started":"2023-08-28T15:13:44.389143Z","shell.execute_reply":"2023-08-28T15:14:06.57293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_compare(images[80:120], heatmaps[80:120], \n                 GradCamplusheatmaps[80:120], l[80:120])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T15:14:06.579536Z","iopub.execute_input":"2023-08-28T15:14:06.580455Z","iopub.status.idle":"2023-08-28T15:14:27.488065Z","shell.execute_reply.started":"2023-08-28T15:14:06.580415Z","shell.execute_reply":"2023-08-28T15:14:27.48678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References:\n* https://arxiv.org/pdf/1610.02391.pdf\n* https://arxiv.org/pdf/1710.11063.pdf\n* https://arxiv.org/pdf/1910.01279.pdf\n* https://arxiv.org/ftp/arxiv/papers/2008/2008.00299.pdf\n* https://arxiv.org/pdf/2008.02312.pdf","metadata":{}}]}