{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Set Up**","metadata":{}},{"cell_type":"markdown","source":"Handle imports and initialize a few directory paths","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os,cv2,keras\nimport json\nimport math\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.applications import ResNet50#, preprocess_input\nimport numpy as np\nimport tensorflow as tf\nimport keras.layers as KL\nimport keras.backend as K\nfrom keras.utils import Sequence\nimport time\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n#Set random seeds\nseed = 232\nnp.random.seed(seed)\n#tf.set_random_seed(seed)\n\n#set image inputs\ndataset_path = '/kaggle/input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled/'\ntrainingImageDirPath = '/kaggle/input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled/train_images/'\ntrainingAnnotationsPath = '/kaggle/input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled/train_annotations.json'\ntestingImageDirPath = '/kaggle/input/vinbigdata-coco-dataset-with-wbf-3x-downscaled/vinbigdata-coco-dataset-with-wbf-3x-downscaled/val_images/'\n#trainingAnnotationsPath = '/kaggle/input/vinbigdata-chest-xray-abnormalities-detection/train.csv'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Demo**\n\nCheck out some xrays","metadata":{}},{"cell_type":"code","source":"#Plot some examples\nfig, ax = plt.subplots(2, 1, figsize=(15, 7))\nax = ax.ravel()\nplt.tight_layout()\n\nfor i, _set in enumerate(['train_images/', 'val_images/']):\n    set_path = dataset_path+_set\n    ax[i].imshow(plt.imread(set_path+os.listdir(set_path)[0]), cmap='gray')\n    ax[i].set_title('Example Image'.format(_set))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**HELPER FUNCTIONS**","metadata":{}},{"cell_type":"markdown","source":"**Build Constants**\n\nInitialize some constants","metadata":{}},{"cell_type":"code","source":"#RCNN Constants\n#We will only use square images, what is the n in nxn?\nN_REGION_IMG_SIZE = 120\nN_RPN_TRAIN_IMAGE_SIZE = 227\nN_INPUT_CHANNELS = 3\n\nRPN_INPUT_SHAPE = (N_RPN_TRAIN_IMAGE_SIZE, N_RPN_TRAIN_IMAGE_SIZE, N_INPUT_CHANNELS)\nANCHOR_STRIDE = 2\nANCHOR_SCALES = (32, 64, 128, 256, 512)\nANCHORS_PER_LOCATION = len(ANCHOR_SCALES)^2\n#TRAIN_ANCHORS_PER_IMAGE = 36\nRPN_BBOX_STD_DEV = np.array([0.1, 0.1, 0.2, 0.2])\nN_FEATURE_MAP_SIZE = 13\nANCHOR_IOU_FG_THRESHOLD = 0.5\nANCHOR_IOU_BG_THRESHOLD = 0.1\n#How many anchors we should validate and find loss with?\nTRAINING_SET_SIZE = 10\n\n\n#This constant must be hand written to match the AlexNet\nFEATURE_MAP_SHAPE = (1, 13, 13, 256)\n\n\nLEARNING_RATE = 0.001\nLEARNING_MOMENTUM = 0.9\nBATCH_SIZE = 8\nEPOCHS = 10","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Compute IOU Data**\n\nCompute the IOU data of two rectangles. The IOU of two rectangles is the ratio of how much of the rectangles overlap to the amount that the rectangles don't overlap, measured in 2-D area.","metadata":{}},{"cell_type":"code","source":"def computeIouList(rect1, rect2List):\n    rect1area= (rect1[2] - rect1[0]) * (rect1[3] - rect1[1])\n    rect2area = (rect2List[:, 2] - rect2List[:, 0]) * (rect2List[:, 3] - rect2List[:, 1]) # area = width * height\n    \n    #Get X, y coordinates of intersecting rectangle\n    xMin = np.maximum(rect1[0], rect2List[:, 0])\n    yMin = np.maximum(rect1[1], rect2List[:, 1])\n    xMax = np.minimum(rect1[2], rect2List[:, 2])\n    yMax = np.minimum(rect1[3], rect2List[:, 3])\n    \n    #get area of intersecting rectangle\n    intersection = np.maximum(0, xMax - xMin + 1) * np.maximum(0, yMax - yMin + 1)\n    union = rect1area + rect2area[:] - intersection[:]\n    iou = intersection / union\n    \n    return iou","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Get Images From Rect Data**\n\nIt is important that all of our regions are squares, so that they are compatible with the neural network that we build. For this reason we must resize all the images we pull in by a constant value.","metadata":{}},{"cell_type":"code","source":"#Perform Preprocessing on Rect Proposals\ndef GetRectImageData(rects, image):\n    props = []\n    \n    (x, y, x2, y2) = rects\n    (x, y, x2, y2) = (int(x), int(y), int(x2), int(y2))\n        \n    #extract rect from image\n    rect = image[y:x2, x:y2]\n    #convert color\n    RGBrect = cv2.cvtColor(rect, cv2.COLOR_BGR2RGB)\n    #Resize rects to NxN\n    squareRect = cv2.resize(RGBrect, (N_REGION_RPN_TRAIN_IMAGE_SIZE, N_REGION_RPN_TRAIN_IMAGE_SIZE))\n        \n    return squareRect\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Generate Anchors**\n\nGiven a feature map, we want to generate some anchors. We will use the constants for image size and anchor stride for this calculation. The challenge is that we are trying to generate evenly spaced anchor points in a full size image using a feature map. Most of this function consists of generating every possible combination of box shape and size, then spacing them out evenly throughout the possible range of values in the image.","metadata":{}},{"cell_type":"code","source":"def GetAnchors(featureMapNSide):\n    featureStrides = int(N_RPN_TRAIN_IMAGE_SIZE/featureMapNSide) # 512/13 ~= 39\n\n    # All combinations of anchor points\n    #ex: Range from 0 to 13 with a stride of 1\n    # (0,2,3,6,...10,12) * 39 = (0, 39, 78, 117 ... 507)\n    start = int(ANCHOR_STRIDE/2)\n    end = int(N_RPN_TRAIN_IMAGE_SIZE - start* featureStrides)\n    print(end)\n    \n    x = np.arange(start*featureStrides, end, ANCHOR_STRIDE * featureStrides)\n    y = np.arange(start*featureStrides, end, ANCHOR_STRIDE * featureStrides)\n    x, y = np.meshgrid(x, y)  #shapes: 39x39\n    \n    #print(x)\n    #print(x[0][5])\n\n    # All combinations of indices, and shapes\n    anchorBBoxes = []\n    scale = len(x)\n    for idy in range(scale):\n        for idx in range(scale):\n            for width in ANCHOR_SCALES:\n                for height in ANCHOR_SCALES:\n                    BBox = (x[idy][idx] - width/2, y[idy][idx] - height/2, x[idy][idx] + width/2, y[idy][idx] + height/2)\n                    anchorBBoxes.append(BBox)\n\n    # Anchors are created for each feature map\n    #print('Num of generated anchors:\\t',len(anchorBBoxes))\n    anchors = np.reshape(anchorBBoxes, (scale, scale, len(ANCHOR_SCALES), len(ANCHOR_SCALES), 4))\n    return anchors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculate Anchor Ground Truth Data**\n\nWe need to get the indicies, the deltas of the boxes, and the labels for the ground truth data. Many things happen in this function. The first thing that happens is that we load the deltas for the boxes from the ground truth, then we find the IOUs of the boxes from the ground truth. If the IOU is above a certain amount, we label that anchor as foreground, and save it to an array which will get sent to the neural network.","metadata":{}},{"cell_type":"code","source":"\n#Make sure BBoxes is a np.array before adding\n#This won't be implemented with batching\ndef CalculateDeltasAndLabels(BBoxes, anchors):  \n    #Get anchors in 1-d Array\n    linearAnchors = np.ndarray.flatten(anchors)\n    anchorCount = (anchors.shape[0] * anchors.shape[1] * anchors.shape[2] * anchors.shape[3])\n    boxCount = len(BBoxes)\n    \n    fgAnchors = []\n    bgAnchors = []\n    \n    deltasOut = np.zeros((anchorCount, 4))\n    bboxIous = np.zeros((boxCount,anchorCount)) #Intersection over union score for each bbox-anchor pair\n    bboxDeltas = np.zeros((boxCount,anchorCount, 4)) #desired delta x,y,h,w for each bbox-anchor pair --> RPN shoult predict these\n    \n    #For each box, calculate the boxes, distance from each anchor\n    for bboxnum, bbox in enumerate(BBoxes):\n        #Calculate Bounding Box Deltas\n        AnchorsArray = np.reshape(anchors, (anchorCount,4))\n        aw = AnchorsArray[:,2] - AnchorsArray[:,0]\n        ah = AnchorsArray[:,3] - AnchorsArray[:,1]\n        acx = AnchorsArray[:,0] + aw[:]/2\n        acy = AnchorsArray[:,1] + ah[:]/2\n        \n        (bbw, bbh) = (bbox[2] - bbox[0], bbox[3] - bbox[1])\n        (bcx, bcy) = (bbox[0] + bbw/2, bbox[1] - bbh/2)\n        \n        deltasGroup = (bbw - aw[:], bbh - ah[:], bcx - acx[:], bcy - acy[:])\n        \n        bboxDeltas[bboxnum] = list(zip(*deltasGroup))\n        #Calculate IOU\n        bboxIous[bboxnum] = computeIouList(bbox, AnchorsArray)\n\n    #For each anchor, find the nearest BBox and the IOU\n    for anchor in range(anchorCount):\n        bestBboxix = np.argmax(bboxIous[:,anchor])\n        deltasOut[anchor] = bboxDeltas[bestBboxix, anchor]  \n        iou = bboxIous[bestBboxix, anchor]\n        \n        if iou > ANCHOR_IOU_FG_THRESHOLD:\n            fgAnchors.append([0,anchor])\n            \n        elif iou < ANCHOR_IOU_BG_THRESHOLD:\n            bgAnchors.append([0,anchor])\n            \n    if fgAnchors == []:\n        fgAnchors = np.array([[-1,-1]])\n    if bgAnchors == []:\n        bgAnchors = np.array([[-1,-1]])\n    \n    return  np.array(fgAnchors), np.array(bgAnchors), deltasOut","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **TRAIN THE NETWORK**","metadata":{}},{"cell_type":"markdown","source":"***Data Pre-processing***\n\nWe want to load the imageURLS and bounding boxe data for every photo in the directory. We can do this because it won't take too much memory but we should not load any images except in a fit generator for the model.","metadata":{}},{"cell_type":"code","source":"#load images into Local memory\ntrainingImagePaths = os.listdir(trainingImageDirPath)\n\nindex = 0\nimgDirSize = len(trainingImagePaths)\n\nannotationsFile = open(trainingAnnotationsPath)\ntrainingMetadata = json.load(annotationsFile)\n\ndataCollection = []\nfor imgPath in trainingImagePaths:\n    matchedImageMetadata = None\n    \n    #GET IMAGE METADATA\n    for imageMetadata in trainingMetadata['images']:\n        if imageMetadata['file_name'] == str(\"train_images/\" + imgPath):\n            matchedImageMetadata = imageMetadata\n            break\n    \n    #GET ANNOTATIONS FOR IMAGE\n    BBoxesDims = []\n    for annotationsData in trainingMetadata['annotations']:\n        if (annotationsData['image_id'] != matchedImageMetadata['id']):\n            continue\n            \n        bboxDims = annotationsData[\"bbox\"]\n        bboxDims[2] = bboxDims[0] +  bboxDims[2]\n        bboxDims[3] = bboxDims[1] +  bboxDims[3]\n        BBoxesDims.append(bboxDims)\n        \n    dataCollection.append({\"imgPath\":imgPath,\"bboxesDims\":BBoxesDims})#\"img\": img,\"bboxesDims\": BBoxesDims})\n    \n    if (math.remainder(index,250) == 0):\n        print(\"Progress: \" + str(index) + \" out of \" + str(imgDirSize) + \" images processed\")\n    index += 1\n\nprint(\"Done Loading\")\n#loop over images","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Feature Map Extractor***\n\nWe use the first 5 layers of an Alex Net to extract a feature map from the input data. ","metadata":{}},{"cell_type":"code","source":"def RunPartialAlexNet(inputTensor):\n    x = KL.ZeroPadding2D((3, 3))(inputTensor)\n    y = KL.Conv2D(filters=96, kernel_size=(11,11), strides=(4,4), activation='relu', input_shape=RPN_INPUT_SHAPE)(x)\n    y = KL.BatchNormalization()(y)\n    y = KL.MaxPool2D(pool_size=(3,3), strides=(2,2))(y)\n    L1 = y#KL.UpSampling2D(size=(2,2))(y)\n    y = KL.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\")(L1)\n    y = KL.BatchNormalization()(y)\n    y = KL.MaxPool2D(pool_size=(3,3), strides=(2,2))(y)\n    L2 = y#KL.UpSampling2D(size=(2,2))(y)\n    y = KL.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\")(L2)\n    y = KL.BatchNormalization()(y)\n    L3 = y#KL.UpSampling2D(size=(2,2))(y)\n    y = KL.Conv2D(filters=384, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\")(L3)\n    y = KL.BatchNormalization()(y)\n    L4 = y#KL.UpSampling2D(size=(2,2))(y)\n    y = KL.Conv2D(filters=256, kernel_size=(3,3), strides=(1,1), activation='relu', padding=\"same\", name=\"LastLayer\")(L4)\n    L5 = y#KL.UpSampling2D(size=(3,3))(y)\n    \n    return [L1, L2, L3, L4, L5]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Build Data Generator***","metadata":{}},{"cell_type":"markdown","source":"Do to the massive size of the data, we had to use a tensorflow input generator. This allows us to avoid loading all of the images into memory at the same time. The __getitem__ function is called once for each image, with a total of __len__ times. At that time we calculate the ground truth data, get the feature map, and build this data as inputs to send to the network.","metadata":{}},{"cell_type":"code","source":"#FeatureMapSize is actually programmatic but it's found after the point where we need\nclass RPNInputGenerator(Sequence):\n    def __init__(self, DataCollection, mode = 'train'):\n        self.DataCollection = DataCollection\n        self.anchors = None\n        self.mode = mode\n        \n    def __len__(self):\n        return int(len(self.DataCollection))\n        \n    def __getitem__(self, idx):\n        idxData = self.DataCollection[idx]\n            \n        img = cv2.imread(trainingImageDirPath + '/' + idxData[\"imgPath\"])\n        (H,W,_) = img.shape\n\n        featureMap = self.GetFeatureMap(img)\n        \n        if (self.mode == 'train'):\n            BBoxes = idxData[\"bboxesDims\"]\n            BBoxesConverted = self.ConvertBoundingBoxes(BBoxes, H, W)\n        \n            if (self.anchors is None):\n                FeatureMapShape = featureMap.shape[1]\n                self.anchors = GetAnchors(FeatureMapShape)\n            \n            fgAnchors, bgAnchors, gtDeltas = CalculateDeltasAndLabels(BBoxesConverted, self.anchors)\n        \n            inputs = [featureMap, fgAnchors, bgAnchors, gtDeltas]\n        else:\n            inputs = [featureMap]\n        \n        return inputs\n\n    def GetFeatureMap(self, img):\n        RGBimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        #Resize rects to 227\n        resizedImg = cv2.resize(RGBimg, (N_RPN_TRAIN_IMAGE_SIZE, N_RPN_TRAIN_IMAGE_SIZE))\n                            \n        input = resizedImg.reshape((1,) + resizedImg.shape)\n        \n        tfInput = tf.constant(input, dtype=tf.float32)\n        \n        #Get feature map\n        [_,_,_,_,featureMap] = RunPartialAlexNet(tfInput)\n        \n        return featureMap\n    \n    def ConvertBoundingBoxes(self, BoundingBoxes, H, W):\n        HRatio = (N_RPN_TRAIN_IMAGE_SIZE/H)\n        WRatio = (N_RPN_TRAIN_IMAGE_SIZE/W)\n        \n        newBoxes = []\n        for Bbox in BoundingBoxes:\n            xn = int(np.multiply(Bbox[0], WRatio))\n            yn = int(np.multiply(Bbox[1], HRatio))\n            xm = int(np.multiply(Bbox[2], WRatio))\n            ym = int(np.multiply(Bbox[3], HRatio))\n            \n            newBox = (xn, yn, xm, ym)\n            \n            newBoxes.append(newBox)\n            \n        return np.array(newBoxes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Build RPN***","metadata":{}},{"cell_type":"markdown","source":"In this section we build the RPN. An RPN (Region proposal network) is a network which is trained to identify regions in an image where it is likely that objects will exist. The core of the RPN is a CNN which is given a feature map and gives out 2 sets of outputs: foreground or background labels, and bounding box deltas.\n\nTo understand what bounding box deltas represent, refer to the previous section on anchor points. To reiterate, anchor points are spread throughout the image and multiple bounding boxes are drawn with the anchor point in the center. The bounding box deltas then represent the predicted difference between anchor point box and the actual regions.\n\nThe labels are a binary classification of whether or not the network perceives the items to be abnormalities in the image.","metadata":{}},{"cell_type":"markdown","source":"*RPN Loss Function*\n\nThe loss function is the main focus of our RPN. This function is actually a combination of two loss functions at once, one which calculates the categorical crossentropy of the labels, and one which calculates the smooth l1 loss of the box deltas. Essentially, we calculate these values by comparing the values of each anchor.\n\nThere are some manipulations we have to do in order to get the data to work though. For example, since we do not care about calculating the delta loss for incorrect labels so we only use anchors indicated as foreground. There is also one special case. If there are no anchors identified as foreground in the image, either because there wasn't an accurate enough anchor box proposal, or because there does not exist any abnormalities in the image, we instead use the background anchors for label loss and assume the delta loss is 0.\n","metadata":{}},{"cell_type":"code","source":"def smoothL1(yK,yP):\n    x = tf.abs(yK-yP)\n    mask = tf.cast(tf.less(x,1.0), \"float32\")\n\n    # Loss calculation for smooth l1\n    loss = (mask * (0.5 * x ** 2)) + (1 - mask) * (x - 0.5)\n    return loss\n\n@tf.function\ndef rpnLoss(rpnLogits, rpnDeltas, gtDeltas, fgAnchors, bgAnchors):\n    #Get Predicted Labels of anchors meant to be foreground\n    #If there are no fg anchors, we can use bg anchors to get loss\n    anchorsToUse = fgAnchors\n    if (tf.reduce_all(tf.equal(fgAnchors\n                               ,tf.constant([[-1,-1]])))):\n        anchorsToUse = bgAnchors\n    \n    selPredictedLabels = tf.gather_nd(rpnLogits, anchorsToUse, name=\"GetPredictedLabels\")\n    onepy = np.ones(1000)\n    gtLabels = tf.gather(onepy, anchorsToUse[:,1], name=\"GetGTLabels\")\n    \n    #Compare labels\n    lf = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n    classLoss = lf(gtLabels, selPredictedLabels)\n    classLoss = tf.reduce_mean(classLoss)\n    \n    #If there are no fg anchors, deltas loss is 0, we cannot use bg anchors for delta loss\n    if (not tf.reduce_all(tf.equal(fgAnchors\n                               ,tf.constant([[-1,-1]])))):\n        selRpnDeltas = tf.cast(tf.gather_nd(rpnDeltas, fgAnchors, name=\"selRpnDeltas\"),tf.float32)\n        selGtDeltas = tf.cast(tf.gather(gtDeltas, fgAnchors[:,1], name=\"selGtDeltas\"), tf.float32)\n        \n        #Compare the deltas\n        deltaLoss = smoothL1(selGtDeltas, selRpnDeltas)\n        deltaLoss = tf.reduce_mean(deltaLoss)\n    else:\n        #deltaLoss = smoothL1(selRpnDeltas, selRpnDeltas)\n        deltaLoss = tf.constant(0.0)\n    \n    return classLoss, deltaLoss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Function to Build RPN Model*\n\nThe core CNN which makes up the RPN is a simple CNN with 3 significant layers. The input is in the form a feature map, derived from passing the resized, original image into the first 5 layers of an AlexNet. The first layer is a convolution on the input which morphs the channels to 512. Then there are two other layers, a classification layer with a depth of two multiplied by the amount of anchor boxes at each anchor point. The two represents the binary classification. The final layer has a depth of four multiplied by the amount of anchor boxes at each point. Here, the four represents the movement of the top left corner of the bounding box, expressed in (x,y) coordinates, and movement of the center of the bounding box, expressed in (x,y) coordinates.","metadata":{}},{"cell_type":"code","source":"def BaseRPN(featureMap):\n    initializer = tf.keras.initializers.GlorotNormal(seed = None)\n    input_ = tf.keras.layers.Input(shape=featureMap.shape, name=\"rpn_INPUT\")\n\n    #Shared base convolution for all methods\n    shared = tf.keras.layers.Conv2D(512, (3,3), padding='same', activation='relu'\n                    , strides=1, name='rpnSharedConvolution',kernel_initializer=initializer)(input_)\n    \n    #CLS LAYER\n    #Generate Anchor classification [batch, height, width, ANCHORS_PER_LOCATION * w]\n    x = tf.keras.layers.Conv2D(2*ANCHORS_PER_LOCATION, (1, 1), padding='valid'\n                    , activation='linear',name='rpnClassification',kernel_initializer=initializer)(shared) \n    \n    #Reshape to [batch, anchors, 2]\n    rpnClassLogits = tf.keras.layers.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 2]))(x)\n    #Softmax on last dimension\n    rpnProbs = tf.keras.layers.Activation(\"softmax\", name=\"rpn_class_xxx\")(rpnClassLogits) # --> BG/FG\n    \n    #REG LAYER\n    # Bounding box refinement. [batch, H, W, anchors per location * depth]\n    #Depth is [x,y, log(w) log(h)]\n    #Also generate 4 delta coordinates\n    x = tf.keras.layers.Conv2D(ANCHORS_PER_LOCATION*4, (1, 1), padding=\"valid\", activation='linear', name='rpnPredictBoundingBoxes',kernel_initializer=initializer)(shared) \n    # Reshape to [batch, anchors, 4]\n    rpnBBox = tf.keras.layers.Lambda(lambda t: tf.reshape(t, [tf.shape(t)[0], -1, 4]))(x)\n    \n    outputs = [rpnClassLogits, rpnProbs, rpnBBox]\n    rpnModel = tf.keras.models.Model([input_], outputs, name=\"RPNModel\")\n\n    return rpnModel\n\n#Build RPN model, handling a single feature map per img\ndef buildRPNModel(mode='train'):\n        rpnInput = KL.Input(shape=FEATURE_MAP_SHAPE, name=\"FeatureMap\")\n        \n        rpNetwork = BaseRPN(rpnInput)\n        \n        rpnOutputs = rpNetwork(rpnInput)\n        \n        rpnLogits, rpnProbs, rpnDeltas = rpnOutputs\n\n        fgAnchors = KL.Input(shape=[None,2], name=\"fgAnchors\", dtype=tf.int32)\n        bgAnchors = KL.Input(shape=[None,2], name=\"bgAnchors\", dtype=tf.int32)\n        gtDeltas = KL.Input(shape=[None, 4], name=\"gtDeltas\", dtype=tf.float32)\n        \n        classLoss, deltaLoss = KL.Lambda(lambda x: rpnLoss(*x), name=\"rpnLossFn\")(\n                                   [rpnLogits, rpnDeltas, gtDeltas, fgAnchors, bgAnchors])\n                                   \n        # Inputs and outputs of the model\n        if mode == 'train':\n            inputs = [rpnInput, fgAnchors, bgAnchors, gtDeltas]\n            outputs = [rpnLogits, rpnProbs, rpnDeltas, classLoss, deltaLoss]\n        elif mode == 'inference':\n            inputs = [rpnInput]\n            outputs = [rpnProbs, rpnDeltas]\n            \n        return tf.keras.models.Model(inputs, outputs, name=\"RPN\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Compile the RPN*\n\nWe use a learning rate of .0001, momentum of 0.9, and clipnorm of 0.5. These values are empirically derived from a Mask RCNN developed by Matterport, Inc in Sunnyvale, CA. Alot of the internal infrastructure of our implementation is actually based off of their work, with major modifications of course to simplify and adapt the implementation to our data set.\n\nWhen compiling, we manually add our loss function as the metrics for the network as well.","metadata":{}},{"cell_type":"code","source":"Optimizer = tf.keras.optimizers.SGD(learning_rate=0.0001\n                                   , momentum= 0.9,clipnorm=0.5)\n\nRpnNetwork = buildRPNModel('train')\n\nRpnNetwork.summary()\n\nlossLayer = RpnNetwork.get_layer('rpnLossFn')\nlossFn = tf.reduce_mean(lossLayer.output, keepdims=True)\nRpnNetwork.add_loss(lossFn)\n\nRpnNetwork.compile(optimizer=Optimizer, loss=[None] * len(RpnNetwork.outputs))\n\nlossLayer = RpnNetwork.get_layer('rpnLossFn')\nRpnNetwork.metrics.append(lossLayer)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Train The Model*\n\nInitialize the RPN input generator. Send it to the network.","metadata":{}},{"cell_type":"code","source":"Inputs = RPNInputGenerator(dataCollection, 'train')\n\nRpnNetwork.fit_generator(Inputs, len(Inputs), epochs=EPOCHS)\n#dataCollection\n#rn50.fit_generator(Inputs, len(Inputs), epochs=EPOCHS)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.models.save_model(RpnNetwork, '/kaggle/working/saved_models/TrainedRPN-05-14-1.h5')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***PUT CODE HERE TO TRAIN THE CNN***","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Test the Trained Network**","metadata":{}},{"cell_type":"markdown","source":"**Load Regions**\n\n*Load Trained RPN*","metadata":{}},{"cell_type":"code","source":"# Load the Model back from file\n#with open(ModelFileName, 'rb') as file:  \n#    RpnNetwork = pickle.load(file)\n    \nRPNTrained = tf.keras.models.load_model('/kaggle/working/saved_models/TrainedRPN-05-14-1.h5')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similar to training, load only image paths, but do not load images.","metadata":{}},{"cell_type":"code","source":"#load images into Local memory\ntestingImagePaths = os.listdir(testingImageDirPath)\n\nindex = 0\nimgDirSize = len(testingImagePaths)\n\ndataCollection = []\nfor imgPath in testingImagePaths:\n    matchedImageMetadata = None\n    \n    dataCollection.append({\"imgPath\":imgPath})\n    \n    if (math.remainder(index,250) == 0):\n        print(\"Progress: \" + str(index) + \" out of \" + str(imgDirSize) + \" images processed\")\n    index += 1\n    \nprint(\"Done Loading\")\n#loop over images","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"InputGenerator = RPNInputGenerator(DataCollection, mode = 'inference')\nregions = []\nfor ix in range(InputGenerator.__len__()):\n    input = InputGenerator.__getitem__(ix)\n    Probs, BBoxes = RPNTrained.Predict(input)\n    \n    \n    \nConvertBBoxes->","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}