{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Intracranial Hemorrhage Detection : Trainer Notebook for Deep RNN\n\nThis is only meant for training the Deep RNN model over small split."},{"metadata":{},"cell_type":"markdown","source":"### Download all the files inside /kaggle/working/ your own server\n\n* train_small.json    : http://shashankdeeplearning.000webhostapp.com/train_small.json\n* model_trained_last.h5 : http://shashankdeeplearning.000webhostapp.com/model_trained_last.h5\n* last_session_data.json : http://shashankdeeplearning.000webhostapp.com/last_session_data.json\n* sample_weights : http://shashankdeeplearning.000webhostapp.com/sample_weights.json\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# when training the smaller model\n\n!wget \"http://shashankdeeplearning.000webhostapp.com/train_small.json\" -O \"train_small.json\"\n!wget \"http://shashankdeeplearning.000webhostapp.com/model_trained_last.h5\" -O \"model_trained_last.h5\"\n!wget \"http://shashankdeeplearning.000webhostapp.com/last_session_data.json\" -O \"last_session_data.json\"\n!wget \"http://shashankdeeplearning.000webhostapp.com/sample_weights.json\" -O \"sample_weights.json\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make sure all the required files are present before trying to train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"working_dir = '/kaggle/working/'\ndataset_dir = '/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'\n\nassert os.path.exists(working_dir + 'train_small.json'), \"Upload train_small.json to /kaggle/working/\"\nassert os.path.exists(working_dir + 'model_trained_last.h5'), \"Upload model_trained_last.h5 file to /kaggle/working/\"\nassert os.path.exists(working_dir + 'last_session_data.json'), \"Upload last_session_data.json file to /kaggle/working/\"\nassert os.path.exists(working_dir + 'sample_weights.json'), \"Upload sample_weights.json file to /kaggle/working/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make neccessary imports for keras and the custom data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\nimport numpy as np\nimport pydicom\nimport json\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = load_model('model_trained_last.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load the data from last session (last_session.json)"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('last_session_data.json') as file:\n    last_session_data = json.load(file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define the DataGenerator Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DicomCuboidBatchGenerator:\n\n    def __init__(self, partitionLabel, batchSize=32, shuffle=True, resuming=False, last_session_data=None):\n        \"\"\" Initialize DicomCubeBatchGenerator \"\"\"\n\n        self.partitionLabel = partitionLabel\n        self.batchSize = batchSize\n        self.shuffle = shuffle\n        with open(f'sample_weights.json') as file:\n            self.sampleWeightData = json.load(file)\n        with open(f'{partitionLabel}.json') as file:\n            self.labelData = json.load(file)\n        self.datasetSize = len(self.labelData)\n        if resuming:\n            self.randomIndexList = np.array(last_session_data['randomIndexList'])\n            self.preventChangeAtResumedEpoch = True\n        else:\n            self.randomIndexList = np.arange(0, self.datasetSize)\n            self.preventChangeAtResumedEpoch = False\n        \n\n    def on_epoch_begin(self):\n        \"\"\" After each epoch shuffle the indexes \"\"\"\n        \n        if self.preventChangeAtResumedEpoch:\n            self.preventChangeAtResumedEpoch = False\n        elif self.shuffle:\n            np.random.shuffle(self.randomIndexList)\n\n    def __len__(self):\n        \"\"\" Number of batches per epoch \"\"\"\n\n        return np.int(np.ceil(self.datasetSize / self.batchSize))\n\n    def getBatch(self, index):\n        \"\"\" Return batch[index] \"\"\"\n\n        batchStartIndex = index * self.batchSize\n        batchEndIndex = (index + 1) * self.batchSize # actually end index + 1\n\n        if batchEndIndex > self.datasetSize:\n            batchEndIndex = self.datasetSize\n\n        return self.generateBatch(batchStartIndex, batchEndIndex)\n\n    def generateBatch(self, batchStartIndex, batchEndIndex):\n        \"\"\" Generate the batch for the given range \"\"\"\n\n        curBatchSize = batchEndIndex - batchStartIndex\n\n        X = []\n        Y = []\n        SW = []\n\n        for i in range(batchStartIndex, batchEndIndex):\n\n            actIdx = self.randomIndexList[i]\n\n            dicomFilename = 'ID_' + self.labelData[actIdx][0] + '.dcm'\n            \n            imageCube = self.getImageCubeFromDicom(dicomFilename)\n            X.append(imageCube)\n            labelRowVector = np.array(self.labelData[actIdx][1])\n            Y.append(labelRowVector)\n            SW.append(self.sampleWeightData[str(self.labelData[actIdx][1])])\n\n        X = np.array(X)\n        X = X.reshape(*X.shape, 1)  # add a single channel dimension\n        assert X.shape == (curBatchSize, 10, 768, 768, 1), f\"Incorrect input batch shape {X.shape}\"\n\n        Y = np.array(Y)\n        assert Y.shape == (curBatchSize, 6), f\"Incorrect output batch shape {Y.shape}\"\n        \n        SW = np.array(SW)\n        assert SW.shape == (curBatchSize,), f\"Incorrect sample weight list shape {SW.shape}\"\n        \n        return X, Y, SW\n\n    def getImageCubeFromDicom(self, filename):\n        \"\"\" Read DICOM data and convert it to cube with multiple windows \"\"\"\n\n        filedir = '/kaggle/input/rsna-intracranial-hemorrhage-detection/'\n        filedir += 'rsna-intracranial-hemorrhage-detection/'\n        filedir += 'stage_2_train/'\n        \n        filepath = filedir + filename\n\n        dicomData = pydicom.dcmread(filepath)\n\n        windows = [['default', 0, 0],\n                    ['brain', 40, 80],\n                    ['subdural-min', 50, 130],\n                    ['subdural-mid', 75, 215],\n                    ['subdural-max', 100, 300],\n                    ['tissue-min', 20, 350],\n                    ['tissue-mid', 40, 375],\n                    ['tissue-max', 60, 400],\n                    ['bone', 600, 2800],\n                    ['grey-white', 32, 8]]\n\n        if type(dicomData.WindowCenter) == pydicom.multival.MultiValue:\n            windows[0][1] = dicomData.WindowCenter[0]\n        else:\n            windows[0][1] = dicomData.WindowCenter\n\n        if type(dicomData.WindowWidth) == pydicom.multival.MultiValue:\n            windows[0][2] = dicomData.WindowWidth[0]\n        else:\n            windows[0][2] = dicomData.WindowWidth\n\n        imageCube = []\n        for window in windows:\n            image = self.getImageForWindow(dicomData, window[1], window[2])\n            imageCube.append(image)\n\n        imageCube = np.array(imageCube)\n\n        assert imageCube.shape == (10, 768, 768), \"Cube shape incorrect\"\n\n        return imageCube\n\n    def getImageForWindow(self, ds, windowCenter, windowWidth):\n        \"\"\" Get the image for given window settings \"\"\"\n        \n        # perform linear transformation on the original pixel_array\n        img = ds.pixel_array * ds.RescaleSlope + ds.RescaleIntercept\n        \n        # pad the image to make sure it's 768 x 768\n        l = (768 - ds.Columns) // 2\n        r = 768 - ds.Columns - l\n        t = (768 - ds.Rows) // 2\n        b = 768 - ds.Rows - t\n        img = np.pad(img, ((t, b), (l, r)), mode='constant', constant_values=img[0,0])\n        \n        # perform windowing\n        img_min = windowCenter - windowWidth // 2\n        img_max = windowCenter + windowWidth // 2\n        img[img < img_min] = img_min\n        img[img > img_max] = img_max\n        \n        # rescale the image\n        img = (img - img_min) / (img_max - img_min)\n\n        assert img.shape == (768, 768), \"Image shape incorrect\"\n\n        return img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Instantiate custom data generators for training and validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"batchSize = last_session_data['batchSize']\n\nif last_session_data['last_completed'] == True:\n    training_gen = DicomCuboidBatchGenerator('train_small', batchSize)\nelse:\n    training_gen = DicomCuboidBatchGenerator('train_small', batchSize, True, True, last_session_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define the number of epochs and steps per epoch.\n* Steps per epoch is equal to number of batches.\n* Number of batches = ceil (length of training data / batch size)\n* DicomCuboidGenerator code is written in such a way that it's len attribute returns exactly this.\n* So we can simply use len(training_gen) to get the number of batches which in turn will be equal to the number of steps."},{"metadata":{"trusted":false},"cell_type":"code","source":"steps_training = len(training_gen)\n\nnumber_of_epochs = last_session_data['number_of_epochs']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Custom training code (for 5.5 hours) with Auto Save feature"},{"metadata":{"_kg_hide-output":false,"trusted":false},"cell_type":"code","source":"# get last session variables\nlast_executing_epoch_index = last_session_data['last_executing_epoch_index']\nlast_executed_batch_index = last_session_data['last_executed_batch_index']\n\n#start the training, set it to train for 5 hours (5*60*60 seconds)\nstart = time.time()\ntimeToStop = False\n\nprint(f'Resuming from : EPOCH {last_executing_epoch_index+1}/{number_of_epochs} and BATCH {last_executed_batch_index+2}/{steps_training}')\n\nfor epoch in range(last_executing_epoch_index, number_of_epochs):\n    \n    training_gen.on_epoch_begin()\n    \n    for batch_index in range(last_executed_batch_index+1, steps_training):\n        \n        print(f'Running EPOCH {epoch+1}/{number_of_epochs} and BATCH {batch_index+1}/{steps_training}', end='\\r')\n        X, Y, SW = training_gen.getBatch(batch_index)\n        \n        RM = True if batch_index == 0 else False # reset metrics only at the beginning of each epoch\n\n        model.train_on_batch(X, Y, sample_weight=SW, reset_metrics=RM) # train the current batch\n        \n        last_executed_batch_index = batch_index # update the last executed batch index\n                \n        # stop if time is more than 5 hours\n        if (time.time()-start) > (5.5*60*60):\n            timeToStop = True\n            break\n    \n    # breaks the outer loop if inner loop was terminated via break\n    if timeToStop:\n        break\n\nprint(f'Last Run : EPOCH {epoch+1}/{number_of_epochs} and BATCH {batch_index+1}/{steps_training}')\nif not timeToStop:\n    print('All epochs completed')\nelse:\n    print('Resume the process next time')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make the last save (doesn't matter if the whole process is complete or not)"},{"metadata":{"trusted":false},"cell_type":"code","source":"model.save('model_trained_last.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare the last_session_data file"},{"metadata":{"trusted":false},"cell_type":"code","source":"last_session_data = dict() # reset the previous last_session_data dictionary\n\nlast_session_data['batchSize'] = batchSize\nlast_session_data['number_of_epochs'] = number_of_epochs\n\n# save the batchSize, last_executing_epoch_index, last_executed_batch_index and number_of_epochs\nlast_session_data['last_executing_epoch_index'] = 0 if not timeToStop else epoch\nlast_session_data['last_executed_batch_index'] = -1 if not timeToStop else last_executed_batch_index\nlast_session_data['last_completed'] = True if not timeToStop else False\n\n# save the randomIndexList\nlast_session_data['randomIndexList'] = list(map(int, list(training_gen.randomIndexList)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write the last_session_data to a json file"},{"metadata":{"trusted":false},"cell_type":"code","source":"with open('last_session_data.json', 'w') as file:\n    json.dump(last_session_data, file, indent=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Don't forget to download the following files\n* model_trained_last.h5\n* last_session_data.json"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":4}