{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, sys, random\nimport xml.etree.ElementTree as ET\nimport mxnet as mx\nimport cv2\nfrom matplotlib import pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nfrom skimage import io\nfrom shutil import copyfile\nimport sys\nimport time\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# > Data Format for RCNN is: ImageID, Xmin, Xmax, Ymin, Ymax, Class"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Writing train.csv to annotation.txt\n\ntrain_df = pd.read_csv('train.csv')\n\n# for training\nwith open(\"annotation.txt\", \"w+\") as f:\n  for idx, row in train_df.iterrows():\n      img = cv2.imread('train/' + row['ImageID'])\n      #print(img)  #none \n      height, width = img.shape[:2]\n      x1 = int(row['XMin'] * width)\n      x2 = int(row['XMax'] * width)\n      y1 = int(row['YMin'] * height)\n      y2 = int(row['YMax'] * height)\n      \n      train_file_path = 'D:/5-deep_learning_with_tensorflow/module 7 cnn/7-faster_r-cnn/SAR_faster-rcnn/train'\n      fileName = os.path.join(train_file_path, row['ImageID'])\n      className = row['ClassName']\n      f.write(fileName + ',' + str(x1) + ',' + str(y1) + ',' + str(x2) + ',' + str(y2) + ',' + className + '\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Traininf=g"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport pprint\nimport sys\nimport time\nimport numpy as np\nfrom optparse import OptionParser\nimport pickle\nimport math\nimport cv2\nimport copy\nfrom matplotlib import pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport os\n\nfrom sklearn.metrics import average_precision_score\n\nfrom keras import backend as K\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.layers import Flatten, Dense, Input, Conv2D, MaxPooling2D, Dropout\nfrom keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D, TimeDistributed\nfrom keras.engine.topology import get_source_inputs\nfrom keras.utils import layer_utils\nfrom keras.utils.data_utils import get_file\nfrom keras.objectives import categorical_crossentropy\n\nfrom keras.models import Model\nfrom keras.utils import generic_utils\nfrom keras.engine import Layer, InputSpec\nfrom keras import initializers, regularizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n\n\tdef __init__(self):\n\n        # Print the process or not\n\t\tself.verbose = True\n\n\t\t# Name of base network\n\t\tself.network = 'vgg'\n\n\t\t# Setting for data augmentation\n\t\tself.use_horizontal_flips = False\n\t\tself.use_vertical_flips = False\n\t\tself.rot_90 = False\n\n\t\t# Anchor box scales\n    # Note that if im_size is smaller, anchor_box_scales should be scaled\n    # Original anchor_box_scales in the paper is [128, 256, 512]\n\t\tself.anchor_box_scales = [64, 128, 256] \n\n\t\t# Anchor box ratios\n\t\tself.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]\n\n\t\t# Size to resize the smallest side of the image\n\t\t# Original setting in paper is 600. Set to 300 in here to save training time\n\t\tself.im_size = 300\n\n\t\t# image channel-wise mean to subtract\n\t\tself.img_channel_mean = [103.939, 116.779, 123.68]\n\t\tself.img_scaling_factor = 1.0\n\n\t\t# number of ROIs at once\n\t\tself.num_rois = 4\n\n\t\t# stride at the RPN (this depends on the network configuration)\n\t\tself.rpn_stride = 16\n\n\t\tself.balanced_classes = False\n\n\t\t# scaling the stdev\n\t\tself.std_scaling = 4.0\n\t\tself.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]\n\n\t\t# overlaps for RPN\n\t\tself.rpn_min_overlap = 0.3\n\t\tself.rpn_max_overlap = 0.7\n\n\t\t# overlaps for classifier ROIs\n\t\tself.classifier_min_overlap = 0.1\n\t\tself.classifier_max_overlap = 0.5\n\n\t\t# placeholder for the class mapping, automatically generated by the parser\n\t\tself.class_mapping = None\n\n\t\tself.model_path = None\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parser the data from annotation file\ndef get_data(input_path):\n\t\"\"\"Parse the data from annotation file\n\t\n\tArgs:\n\t\tinput_path: annotation file path\n\n\tReturns:\n\t\tall_data: list(filepath, width, height, list(bboxes))\n\t\tclasses_count: dict{key:class_name, value:count_num} \n\t\t\te.g. {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745}\n\t\tclass_mapping: dict{key:class_name, value: idx}\n\t\t\te.g. {'Car': 0, 'Mobile phone': 1, 'Person': 2}\n\t\"\"\"\n\tfound_bg = False\n\tall_imgs = {}\n\n\tclasses_count = {}\n\n\tclass_mapping = {}\n\n\tvisualise = True\n\n\ti = 1\n\t\n\twith open(input_path,'r') as f:\n\n\t\tprint('Parsing annotation files')\n\n\t\tfor line in f:\n\n\t\t\t# Print process\n\t\t\tsys.stdout.write('\\r'+'idx=' + str(i))\n\t\t\ti += 1\n\n\t\t\tline_split = line.strip().split(',')\n\n\t\t\t(filename,x1,y1,x2,y2,class_name) = line_split\n\n\t\t\tif class_name not in classes_count:\n\t\t\t\tclasses_count[class_name] = 1\n\t\t\telse:\n\t\t\t\tclasses_count[class_name] += 1\n\n\t\t\tif class_name not in class_mapping:\n\t\t\t\tif class_name == 'bg' and found_bg == False:\n\t\t\t\t\tprint('Found class name with special name bg. Will be treated as a background region (this is usually for hard negative mining).')\n\t\t\t\t\tfound_bg = True\n\t\t\t\tclass_mapping[class_name] = len(class_mapping)\n\n\t\t\tif filename not in all_imgs:\n\t\t\t\tall_imgs[filename] = {}\n\t\t\t\t\n\t\t\t\timg = cv2.imread(filename)\n\t\t\t\t(rows,cols) = img.shape[:2]\n\t\t\t\tall_imgs[filename]['filepath'] = filename\n\t\t\t\tall_imgs[filename]['width'] = cols\n\t\t\t\tall_imgs[filename]['height'] = rows\n\t\t\t\tall_imgs[filename]['bboxes'] = []\n\t\t\t\t\n\t\t\tall_imgs[filename]['bboxes'].append({'class': class_name, 'x1': int(x1), 'x2': int(x2), 'y1': int(y1), 'y2': int(y2)})\n\n\n\t\tall_data = []\n\t\tfor key in all_imgs:\n\t\t\tall_data.append(all_imgs[key])\n\t\t\n\t\t# make sure the bg class is last in the list\n\t\tif found_bg:\n\t\t\tif class_mapping['bg'] != len(class_mapping) - 1:\n\t\t\t\tkey_to_switch = [key for key in class_mapping.keys() if class_mapping[key] == len(class_mapping)-1][0]\n\t\t\t\tval_to_switch = class_mapping['bg']\n\t\t\t\tclass_mapping['bg'] = len(class_mapping) - 1\n\t\t\t\tclass_mapping[key_to_switch] = val_to_switch\n\t\t\n\t\treturn all_data, classes_count, class_mapping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define ROI Pooling Convolutional Layer\n\nclass RoiPoolingConv(Layer):\n    '''ROI pooling layer for 2D inputs.\n    See Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition,\n    K. He, X. Zhang, S. Ren, J. Sun\n    # Arguments\n        pool_size: int\n            Size of pooling region to use. pool_size = 7 will result in a 7x7 region.\n        num_rois: number of regions of interest to be used\n    # Input shape\n        list of two 4D tensors [X_img,X_roi] with shape:\n        X_img:\n        `(1, rows, cols, channels)`\n        X_roi:\n        `(1,num_rois,4)` list of rois, with ordering (x,y,w,h)\n    # Output shape\n        3D tensor with shape:\n        `(1, num_rois, channels, pool_size, pool_size)`\n    '''\n    def __init__(self, pool_size, num_rois, **kwargs):\n\n        self.dim_ordering = K.image_dim_ordering()\n        self.pool_size = pool_size\n        self.num_rois = num_rois\n\n        #inheritance super class with any number of arg\n        super(RoiPoolingConv, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.nb_channels = input_shape[0][3]   \n\n    def compute_output_shape(self, input_shape):\n        return None, self.num_rois, self.pool_size, self.pool_size, self.nb_channels\n\n    def call(self, x, mask=None):\n\n        assert(len(x) == 2)\n\n        # x[0] is image with shape (rows, cols, channels)\n        img = x[0]\n\n        # x[1] is roi with shape (num_rois,4) with ordering (x,y,w,h)\n        rois = x[1]\n\n        input_shape = K.shape(img)\n\n        outputs = []\n\n        for roi_idx in range(self.num_rois):\n\n            x = rois[0, roi_idx, 0]\n            y = rois[0, roi_idx, 1]\n            w = rois[0, roi_idx, 2]\n            h = rois[0, roi_idx, 3]\n\n            x = K.cast(x, 'int32')\n            y = K.cast(y, 'int32')\n            w = K.cast(w, 'int32')\n            h = K.cast(h, 'int32')\n\n            # Resized roi of the image to pooling size (7x7)\n            rs = tf.image.resize_images(img[:, y:y+h, x:x+w, :], (self.pool_size, self.pool_size))\n            outputs.append(rs)\n                \n\n        final_output = K.concatenate(outputs, axis=0)\n\n        # Reshape to (1, num_rois, pool_size, pool_size, nb_channels)\n        # Might be (1, 4, 7, 7, 3)\n        final_output = K.reshape(final_output, (1, self.num_rois, self.pool_size, self.pool_size, self.nb_channels))\n\n        # permute_dimensions is similar to transpose\n        final_output = K.permute_dimensions(final_output, (0, 1, 2, 3, 4))\n\n        return final_output\n    \n    \n    def get_config(self):\n        config = {'pool_size': self.pool_size,\n                  'num_rois': self.num_rois}\n        base_config = super(RoiPoolingConv, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# VGG-16 (BAse Network)\n\ndef get_img_output_length(width, height):\n    def get_output_length(input_length):\n        return input_length//16\n\n    return get_output_length(width), get_output_length(height)    \n\ndef nn_base(input_tensor=None, trainable=False):\n\n\n    input_shape = (None, None, 3)\n\n    if input_tensor is None:\n        img_input = Input(shape=input_shape)\n    else:\n        if not K.is_keras_tensor(input_tensor):\n            img_input = Input(tensor=input_tensor, shape=input_shape)\n        else:\n            img_input = input_tensor\n\n    bn_axis = 3\n\n    # Block 1\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)\n    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)\n    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n    # Block 3\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)\n    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n    # Block 4\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n    # Block 5\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)\n    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)\n    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#RPN layer\n\ndef rpn_layer(base_layers, num_anchors):\n    \"\"\"Create a rpn layer\n        Step1: Pass through the feature map from base layer to a 3x3 512 channels convolutional layer\n                Keep the padding 'same' to preserve the feature map's size\n        Step2: Pass the step1 to two (1,1) convolutional layer to replace the fully connected layer\n                classification layer: num_anchors (9 in here) channels for 0, 1 sigmoid activation output\n                regression layer: num_anchors*4 (36 in here) channels for computing the regression of bboxes with linear activation\n    Args:\n        base_layers: vgg in here\n        num_anchors: 9 in here\n\n    Returns:\n        [x_class, x_regr, base_layers]\n        x_class: classification for whether it's an object\n        x_regr: bboxes regression\n        base_layers: vgg in here\n    \"\"\"\n    x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)\n\n    x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)\n    x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)\n\n    return [x_class, x_regr, base_layers]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Classifier layer\n\ndef classifier_layer(base_layers, input_rois, num_rois, nb_classes = 4):\n    \n\n    input_shape = (num_rois,7,7,512)\n\n    pooling_regions = 7\n\n    # out_roi_pool.shape = (1, num_rois, channels, pool_size, pool_size)\n    # num_rois (4) 7x7 roi pooling\n    out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois])\n\n    # Flatten the convlutional layer and connected to 2 FC and 2 dropout\n    out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n    out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out)\n    out = TimeDistributed(Dropout(0.5))(out)\n\n    # There are two output layer\n    # out_class: softmax acivation function for classify the class name of the object\n    # out_regr: linear activation function for bboxes coordinates regression\n    out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out)\n    # note: no regression target for bg class\n    out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out)\n\n    return [out_class, out_regr]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calculate IoU (Intersection of Union)\n\ndef union(au, bu, area_intersection):\n\tarea_a = (au[2] - au[0]) * (au[3] - au[1])\n\tarea_b = (bu[2] - bu[0]) * (bu[3] - bu[1])\n\tarea_union = area_a + area_b - area_intersection\n\treturn area_union\n\n\ndef intersection(ai, bi):\n\tx = max(ai[0], bi[0])\n\ty = max(ai[1], bi[1])\n\tw = min(ai[2], bi[2]) - x\n\th = min(ai[3], bi[3]) - y\n\tif w < 0 or h < 0:\n\t\treturn 0\n\treturn w*h\n\n\ndef iou(a, b):\n\t# a and b should be (x1,y1,x2,y2)\n\n\tif a[0] >= a[2] or a[1] >= a[3] or b[0] >= b[2] or b[1] >= b[3]:\n\t\treturn 0.0\n\n\tarea_i = intersection(a, b)\n\tarea_u = union(a, b, area_i)\n\n\treturn float(area_i) / float(area_u + 1e-6)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calculate the rpn for all anchors of all images\n\ndef calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):\n\t\n\tdownscale = float(C.rpn_stride) \n\tanchor_sizes = C.anchor_box_scales   # 128, 256, 512\n\tanchor_ratios = C.anchor_box_ratios  # 1:1, 1:2*sqrt(2), 2*sqrt(2):1\n\tnum_anchors = len(anchor_sizes) * len(anchor_ratios) # 3x3=9\n\n\t# calculate the output map size based on the network architecture\n\t(output_width, output_height) = img_length_calc_function(resized_width, resized_height)\n\n\tn_anchratios = len(anchor_ratios)    # 3\n\t\n\t# initialise empty output objectives\n\ty_rpn_overlap = np.zeros((output_height, output_width, num_anchors))\n\ty_is_box_valid = np.zeros((output_height, output_width, num_anchors))\n\ty_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))\n\n\tnum_bboxes = len(img_data['bboxes'])\n\n\tnum_anchors_for_bbox = np.zeros(num_bboxes).astype(int)\n\tbest_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)\n\tbest_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)\n\tbest_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)\n\tbest_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)\n\n\t# get the GT box coordinates, and resize to account for image resizing\n\tgta = np.zeros((num_bboxes, 4))\n\tfor bbox_num, bbox in enumerate(img_data['bboxes']):\n\t\t# get the GT box coordinates, and resize to account for image resizing\n\t\tgta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))\n\t\tgta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))\n\t\tgta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))\n\t\tgta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))\n\t\n\t# rpn ground truth\n\n\tfor anchor_size_idx in range(len(anchor_sizes)):\n\t\tfor anchor_ratio_idx in range(n_anchratios):\n\t\t\tanchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]\n\t\t\tanchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]\t\n\t\t\t\n\t\t\tfor ix in range(output_width):\t\t\t\t\t\n\t\t\t\t# x-coordinates of the current anchor box\t\n\t\t\t\tx1_anc = downscale * (ix + 0.5) - anchor_x / 2\n\t\t\t\tx2_anc = downscale * (ix + 0.5) + anchor_x / 2\t\n\t\t\t\t\n\t\t\t\t# ignore boxes that go across image boundaries\t\t\t\t\t\n\t\t\t\tif x1_anc < 0 or x2_anc > resized_width:\n\t\t\t\t\tcontinue\n\t\t\t\t\t\n\t\t\t\tfor jy in range(output_height):\n\n\t\t\t\t\t# y-coordinates of the current anchor box\n\t\t\t\t\ty1_anc = downscale * (jy + 0.5) - anchor_y / 2\n\t\t\t\t\ty2_anc = downscale * (jy + 0.5) + anchor_y / 2\n\n\t\t\t\t\t# ignore boxes that go across image boundaries\n\t\t\t\t\tif y1_anc < 0 or y2_anc > resized_height:\n\t\t\t\t\t\tcontinue\n\n\t\t\t\t\t# bbox_type indicates whether an anchor should be a target\n\t\t\t\t\t# Initialize with 'negative'\n\t\t\t\t\tbbox_type = 'neg'\n\n\t\t\t\t\t# this is the best IOU for the (x,y) coord and the current anchor\n\t\t\t\t\t# note that this is different from the best IOU for a GT bbox\n\t\t\t\t\tbest_iou_for_loc = 0.0\n\n\t\t\t\t\tfor bbox_num in range(num_bboxes):\n\t\t\t\t\t\t\n\t\t\t\t\t\t# get IOU of the current GT box and the current anchor box\n\t\t\t\t\t\tcurr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])\n\t\t\t\t\t\t# calculate the regression targets if they will be needed\n\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\tcx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0\n\t\t\t\t\t\t\tcy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0\n\t\t\t\t\t\t\tcxa = (x1_anc + x2_anc)/2.0\n\t\t\t\t\t\t\tcya = (y1_anc + y2_anc)/2.0\n\n\t\t\t\t\t\t\ttx = (cx - cxa) / (x2_anc - x1_anc)\n\t\t\t\t\t\t\tty = (cy - cya) / (y2_anc - y1_anc)\n\t\t\t\t\t\t\ttw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))\n\t\t\t\t\t\t\tth = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))\n\t\t\t\t\t\t\n\t\t\t\t\t\tif img_data['bboxes'][bbox_num]['class'] != 'bg':\n\n\t\t\t\t\t\t\t# all GT boxes should be mapped to an anchor box, so we keep track of which anchor box was best\n\t\t\t\t\t\t\tif curr_iou > best_iou_for_bbox[bbox_num]:\n\t\t\t\t\t\t\t\tbest_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]\n\t\t\t\t\t\t\t\tbest_iou_for_bbox[bbox_num] = curr_iou\n\t\t\t\t\t\t\t\tbest_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]\n\t\t\t\t\t\t\t\tbest_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]\n\n\t\t\t\t\t\t\t# we set the anchor to positive if the IOU is >0.7 (it does not matter if there was another better box, it just indicates overlap)\n\t\t\t\t\t\t\tif curr_iou > C.rpn_max_overlap:\n\t\t\t\t\t\t\t\tbbox_type = 'pos'\n\t\t\t\t\t\t\t\tnum_anchors_for_bbox[bbox_num] += 1\n\t\t\t\t\t\t\t\t# we update the regression layer target if this IOU is the best for the current (x,y) and anchor position\n\t\t\t\t\t\t\t\tif curr_iou > best_iou_for_loc:\n\t\t\t\t\t\t\t\t\tbest_iou_for_loc = curr_iou\n\t\t\t\t\t\t\t\t\tbest_regr = (tx, ty, tw, th)\n\n\t\t\t\t\t\t\t# if the IOU is >0.3 and <0.7, it is ambiguous and no included in the objective\n\t\t\t\t\t\t\tif C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:\n\t\t\t\t\t\t\t\t# gray zone between neg and pos\n\t\t\t\t\t\t\t\tif bbox_type != 'pos':\n\t\t\t\t\t\t\t\t\tbbox_type = 'neutral'\n\n\t\t\t\t\t# turn on or off outputs depending on IOUs\n\t\t\t\t\tif bbox_type == 'neg':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'neutral':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0\n\t\t\t\t\telif bbox_type == 'pos':\n\t\t\t\t\t\ty_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\ty_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1\n\t\t\t\t\t\tstart = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)\n\t\t\t\t\t\ty_rpn_regr[jy, ix, start:start+4] = best_regr\n\n\t# we ensure that every bbox has at least one positive RPN region\n\n\tfor idx in range(num_anchors_for_bbox.shape[0]):\n\t\tif num_anchors_for_bbox[idx] == 0:\n\t\t\t# no box with an IOU greater than zero ...\n\t\t\tif best_anchor_for_bbox[idx, 0] == -1:\n\t\t\t\tcontinue\n\t\t\ty_is_box_valid[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\ty_rpn_overlap[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *\n\t\t\t\tbest_anchor_for_bbox[idx,3]] = 1\n\t\t\tstart = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])\n\t\t\ty_rpn_regr[\n\t\t\t\tbest_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]\n\n\ty_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))\n\ty_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)\n\n\ty_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))\n\ty_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)\n\n\ty_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))\n\ty_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)\n\n\tpos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))\n\tneg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))\n\n\tnum_pos = len(pos_locs[0])\n\n\t# one issue is that the RPN has many more negative than positive regions, so we turn off some of the negative\n\t# regions. We also limit it to 256 regions.\n\tnum_regions = 256\n\n\tif len(pos_locs[0]) > num_regions/2:\n\t\tval_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)\n\t\ty_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0\n\t\tnum_pos = num_regions/2\n\n\tif len(neg_locs[0]) + num_pos > num_regions:\n\t\tval_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)\n\t\ty_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0\n\n\ty_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)\n\ty_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)\n\n\treturn np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get new image size and augment the image\n\ndef get_new_img_size(width, height, img_min_side=300):\n\tif width <= height:\n\t\tf = float(img_min_side) / width\n\t\tresized_height = int(f * height)\n\t\tresized_width = img_min_side\n\telse:\n\t\tf = float(img_min_side) / height\n\t\tresized_width = int(f * width)\n\t\tresized_height = img_min_side\n\n\treturn resized_width, resized_height\n\ndef augment(img_data, config, augment=True):\n\tassert 'filepath' in img_data\n\tassert 'bboxes' in img_data\n\tassert 'width' in img_data\n\tassert 'height' in img_data\n\n\timg_data_aug = copy.deepcopy(img_data)\n\n\timg = cv2.imread(img_data_aug['filepath'])\n\n\tif augment:\n\t\trows, cols = img.shape[:2]\n\n\t\tif config.use_horizontal_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 1)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\tbbox['x1'] = cols - x2\n\n\t\tif config.use_vertical_flips and np.random.randint(0, 2) == 0:\n\t\t\timg = cv2.flip(img, 0)\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\tbbox['y1'] = rows - y2\n\n\t\tif config.rot_90:\n\t\t\tangle = np.random.choice([0,90,180,270],1)[0]\n\t\t\tif angle == 270:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 0)\n\t\t\telif angle == 180:\n\t\t\t\timg = cv2.flip(img, -1)\n\t\t\telif angle == 90:\n\t\t\t\timg = np.transpose(img, (1,0,2))\n\t\t\t\timg = cv2.flip(img, 1)\n\t\t\telif angle == 0:\n\t\t\t\tpass\n\n\t\t\tfor bbox in img_data_aug['bboxes']:\n\t\t\t\tx1 = bbox['x1']\n\t\t\t\tx2 = bbox['x2']\n\t\t\t\ty1 = bbox['y1']\n\t\t\t\ty2 = bbox['y2']\n\t\t\t\tif angle == 270:\n\t\t\t\t\tbbox['x1'] = y1\n\t\t\t\t\tbbox['x2'] = y2\n\t\t\t\t\tbbox['y1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = cols - x1\n\t\t\t\telif angle == 180:\n\t\t\t\t\tbbox['x2'] = cols - x1\n\t\t\t\t\tbbox['x1'] = cols - x2\n\t\t\t\t\tbbox['y2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = rows - y2\n\t\t\t\telif angle == 90:\n\t\t\t\t\tbbox['x1'] = rows - y2\n\t\t\t\t\tbbox['x2'] = rows - y1\n\t\t\t\t\tbbox['y1'] = x1\n\t\t\t\t\tbbox['y2'] = x2        \n\t\t\t\telif angle == 0:\n\t\t\t\t\tpass\n\n\timg_data_aug['width'] = img.shape[1]\n\timg_data_aug['height'] = img.shape[0]\n\treturn img_data_aug, img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Generate the ground_truth anchors\n\ndef get_anchor_gt(all_img_data, C, img_length_calc_function, mode='train'):\n\t\"\"\" Yield the ground-truth anchors as Y (labels)\n\t\t\n\tArgs:\n\t\tall_img_data: list(filepath, width, height, list(bboxes))\n\t\tC: config\n\t\timg_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size\n\t\tmode: 'train' or 'test'; 'train' mode need augmentation\n\n\tReturns:\n\t\tx_img: image data after resized and scaling (smallest size = 300px)\n\t\tY: [y_rpn_cls, y_rpn_regr]\n\t\timg_data_aug: augmented image data (original image with augmentation)\n\t\tdebug_img: show image for debug\n\t\tnum_pos: show number of positive anchors for debug\n\t\"\"\"\n\twhile True:\n\n\t\tfor img_data in all_img_data:\n\t\t\ttry:\n\n\t\t\t\t# read in image, and optionally add augmentation\n\n\t\t\t\tif mode == 'train':\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=True)\n\t\t\t\telse:\n\t\t\t\t\timg_data_aug, x_img = augment(img_data, C, augment=False)\n\n\t\t\t\t(width, height) = (img_data_aug['width'], img_data_aug['height'])\n\t\t\t\t(rows, cols, _) = x_img.shape\n\n\t\t\t\tassert cols == width\n\t\t\t\tassert rows == height\n\n\t\t\t\t# get image dimensions for resizing\n\t\t\t\t(resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n\t\t\t\t# resize the image so that smalles side is length = 300px\n\t\t\t\tx_img = cv2.resize(x_img, (resized_width, resized_height), interpolation=cv2.INTER_CUBIC)\n\t\t\t\tdebug_img = x_img.copy()\n\n\t\t\t\ttry:\n\t\t\t\t\ty_rpn_cls, y_rpn_regr, num_pos = calc_rpn(C, img_data_aug, width, height, resized_width, resized_height, img_length_calc_function)\n\t\t\t\texcept:\n\t\t\t\t\tcontinue\n\n\t\t\t\t# Zero-center by mean pixel, and preprocess image\n\n\t\t\t\tx_img = x_img[:,:, (2, 1, 0)]  # BGR -> RGB\n\t\t\t\tx_img = x_img.astype(np.float32)\n\t\t\t\tx_img[:, :, 0] -= C.img_channel_mean[0]\n\t\t\t\tx_img[:, :, 1] -= C.img_channel_mean[1]\n\t\t\t\tx_img[:, :, 2] -= C.img_channel_mean[2]\n\t\t\t\tx_img /= C.img_scaling_factor\n\n\t\t\t\tx_img = np.transpose(x_img, (2, 0, 1))\n\t\t\t\tx_img = np.expand_dims(x_img, axis=0)\n\n\t\t\t\ty_rpn_regr[:, y_rpn_regr.shape[1]//2:, :, :] *= C.std_scaling\n\n\t\t\t\tx_img = np.transpose(x_img, (0, 2, 3, 1))\n\t\t\t\ty_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))\n\t\t\t\ty_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))\n\n\t\t\t\tyield np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], img_data_aug, debug_img, num_pos\n\n\t\t\texcept Exception as e:\n\t\t\t\tprint(e)\n\t\t\t\tcontinue","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define loss functions for all four outputs\n\nlambda_rpn_regr = 1.0\nlambda_rpn_class = 1.0\n\nlambda_cls_regr = 1.0\nlambda_cls_class = 1.0\n\nepsilon = 1e-4\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_loss_regr(num_anchors):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def rpn_loss_regr_fixed_num(y_true, y_pred):\n\n        # x is the difference between true value and predicted vaue\n        x = y_true[:, :, :, 4 * num_anchors:] - y_pred\n\n        # absolute value of x\n        x_abs = K.abs(x)\n\n        # If x_abs <= 1.0, x_bool = 1\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)\n\n        return lambda_rpn_regr * K.sum(\n            y_true[:, :, :, :4 * num_anchors] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :, :4 * num_anchors])\n\n    return rpn_loss_regr_fixed_num\n\n\ndef rpn_loss_cls(num_anchors):\n    \"\"\"Loss function for rpn classification\n    Args:\n        num_anchors: number of anchors (9 in here)\n        y_true[:, :, :, :9]: [0,1,0,0,0,0,0,1,0] means only the second and the eighth box is valid which contains pos or neg anchor => isValid\n        y_true[:, :, :, 9:]: [0,1,0,0,0,0,0,0,0] means the second box is pos and eighth box is negative\n    Returns:\n        lambda * sum((binary_crossentropy(isValid*y_pred,y_true))) / N\n    \"\"\"\n    def rpn_loss_cls_fixed_num(y_true, y_pred):\n\n            return lambda_rpn_class * K.sum(y_true[:, :, :, :num_anchors] * K.binary_crossentropy(y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(epsilon + y_true[:, :, :, :num_anchors])\n\n    return rpn_loss_cls_fixed_num\n\n\ndef class_loss_regr(num_classes):\n    \"\"\"Loss function for rpn regression\n    Args:\n        num_anchors: number of anchors (9 in here)\n    Returns:\n        Smooth L1 loss function \n                           0.5*x*x (if x_abs < 1)\n                           x_abx - 0.5 (otherwise)\n    \"\"\"\n    def class_loss_regr_fixed_num(y_true, y_pred):\n        x = y_true[:, :, 4*num_classes:] - y_pred\n        x_abs = K.abs(x)\n        x_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')\n        return lambda_cls_regr * K.sum(y_true[:, :, :4*num_classes] * (x_bool * (0.5 * x * x) + (1 - x_bool) * (x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4*num_classes])\n    return class_loss_regr_fixed_num\n\n\ndef class_loss_cls(y_true, y_pred):\n    return lambda_cls_class * K.mean(categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):\n    # code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/\n    # if there are no boxes, return an empty list\n\n    # Process explanation:\n    #   Step 1: Sort the probs list\n    #   Step 2: Find the larget prob 'Last' in the list and save it to the pick list\n    #   Step 3: Calculate the IoU with 'Last' box and other boxes in the list. If the IoU is larger than overlap_threshold, delete the box from list\n    #   Step 4: Repeat step 2 and step 3 until there is no item in the probs list \n    if len(boxes) == 0:\n        return []\n\n    # grab the coordinates of the bounding boxes\n    x1 = boxes[:, 0]\n    y1 = boxes[:, 1]\n    x2 = boxes[:, 2]\n    y2 = boxes[:, 3]\n\n    np.testing.assert_array_less(x1, x2)\n    np.testing.assert_array_less(y1, y2)\n\n    # if the bounding boxes integers, convert them to floats --\n    # this is important since we'll be doing a bunch of divisions\n    if boxes.dtype.kind == \"i\":\n        boxes = boxes.astype(\"float\")\n\n    # initialize the list of picked indexes\t\n    pick = []\n\n    # calculate the areas\n    area = (x2 - x1) * (y2 - y1)\n\n    # sort the bounding boxes \n    idxs = np.argsort(probs)\n\n    # keep looping while some indexes still remain in the indexes\n    # list\n    while len(idxs) > 0:\n        # grab the last index in the indexes list and add the\n        # index value to the list of picked indexes\n        last = len(idxs) - 1\n        i = idxs[last]\n        pick.append(i)\n\n        # find the intersection\n\n        xx1_int = np.maximum(x1[i], x1[idxs[:last]])\n        yy1_int = np.maximum(y1[i], y1[idxs[:last]])\n        xx2_int = np.minimum(x2[i], x2[idxs[:last]])\n        yy2_int = np.minimum(y2[i], y2[idxs[:last]])\n\n        ww_int = np.maximum(0, xx2_int - xx1_int)\n        hh_int = np.maximum(0, yy2_int - yy1_int)\n\n        area_int = ww_int * hh_int\n\n        # find the union\n        area_union = area[i] + area[idxs[:last]] - area_int\n\n        # compute the ratio of overlap\n        overlap = area_int/(area_union + 1e-6)\n\n        # delete all indexes from the index list that have\n        idxs = np.delete(idxs, np.concatenate(([last],\n            np.where(overlap > overlap_thresh)[0])))\n\n        if len(pick) >= max_boxes:\n            break\n\n    # return only the bounding boxes that were picked using the integer data type\n    boxes = boxes[pick].astype(\"int\")\n    probs = probs[pick]\n    return boxes, probs\n\ndef apply_regr_np(X, T):\n    \"\"\"Apply regression layer to all anchors in one feature map\n\n    Args:\n        X: shape=(4, 18, 25) the current anchor type for all points in the feature map\n        T: regression layer shape=(4, 18, 25)\n\n    Returns:\n        X: regressed position and size for current anchor\n    \"\"\"\n    try:\n        x = X[0, :, :]\n        y = X[1, :, :]\n        w = X[2, :, :]\n        h = X[3, :, :]\n\n        tx = T[0, :, :]\n        ty = T[1, :, :]\n        tw = T[2, :, :]\n        th = T[3, :, :]\n\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n\n        w1 = np.exp(tw.astype(np.float64)) * w\n        h1 = np.exp(th.astype(np.float64)) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n\n        x1 = np.round(x1)\n        y1 = np.round(y1)\n        w1 = np.round(w1)\n        h1 = np.round(h1)\n        return np.stack([x1, y1, w1, h1])\n    except Exception as e:\n        print(e)\n        return X\n    \ndef apply_regr(x, y, w, h, tx, ty, tw, th):\n    # Apply regression to x, y, w and h\n    try:\n        cx = x + w/2.\n        cy = y + h/2.\n        cx1 = tx * w + cx\n        cy1 = ty * h + cy\n        w1 = math.exp(tw) * w\n        h1 = math.exp(th) * h\n        x1 = cx1 - w1/2.\n        y1 = cy1 - h1/2.\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        w1 = int(round(w1))\n        h1 = int(round(h1))\n\n        return x1, y1, w1, h1\n\n    except ValueError:\n        return x, y, w, h\n    except OverflowError:\n        return x, y, w, h\n    except Exception as e:\n        print(e)\n        return x, y, w, h\n\ndef calc_iou(R, img_data, C, class_mapping):\n    \"\"\"Converts from (x1,y1,x2,y2) to (x,y,w,h) format\n\n    Args:\n        R: bboxes, probs\n    \"\"\"\n    bboxes = img_data['bboxes']\n    (width, height) = (img_data['width'], img_data['height'])\n    # get image dimensions for resizing\n    (resized_width, resized_height) = get_new_img_size(width, height, C.im_size)\n\n    gta = np.zeros((len(bboxes), 4))\n\n    for bbox_num, bbox in enumerate(bboxes):\n        # get the GT box coordinates, and resize to account for image resizing\n        # gta[bbox_num, 0] = (40 * (600 / 800)) / 16 = int(round(1.875)) = 2 (x in feature map)\n        gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))\n        gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))\n        gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))\n\n    x_roi = []\n    y_class_num = []\n    y_class_regr_coords = []\n    y_class_regr_label = []\n    IoUs = [] # for debugging only\n\n    # R.shape[0]: number of bboxes (=300 from non_max_suppression)\n    for ix in range(R.shape[0]):\n        (x1, y1, x2, y2) = R[ix, :]\n        x1 = int(round(x1))\n        y1 = int(round(y1))\n        x2 = int(round(x2))\n        y2 = int(round(y2))\n\n        best_iou = 0.0\n        best_bbox = -1\n        # Iterate through all the ground-truth bboxes to calculate the iou\n        for bbox_num in range(len(bboxes)):\n            curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])\n\n            # Find out the corresponding ground-truth bbox_num with larget iou\n            if curr_iou > best_iou:\n                best_iou = curr_iou\n                best_bbox = bbox_num\n\n        if best_iou < C.classifier_min_overlap:\n                continue\n        else:\n            w = x2 - x1\n            h = y2 - y1\n            x_roi.append([x1, y1, w, h])\n            IoUs.append(best_iou)\n\n            if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:\n                # hard negative example\n                cls_name = 'bg'\n            elif C.classifier_max_overlap <= best_iou:\n                cls_name = bboxes[best_bbox]['class']\n                cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0\n                cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0\n\n                cx = x1 + w / 2.0\n                cy = y1 + h / 2.0\n\n                tx = (cxg - cx) / float(w)\n                ty = (cyg - cy) / float(h)\n                tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))\n                th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))\n            else:\n                print('roi = {}'.format(best_iou))\n                raise RuntimeError\n\n        class_num = class_mapping[cls_name]\n        class_label = len(class_mapping) * [0]\n        class_label[class_num] = 1\n        y_class_num.append(copy.deepcopy(class_label))\n        coords = [0] * 4 * (len(class_mapping) - 1)\n        labels = [0] * 4 * (len(class_mapping) - 1)\n        if cls_name != 'bg':\n            label_pos = 4 * class_num\n            sx, sy, sw, sh = C.classifier_regr_std\n            coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]\n            labels[label_pos:4+label_pos] = [1, 1, 1, 1]\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n        else:\n            y_class_regr_coords.append(copy.deepcopy(coords))\n            y_class_regr_label.append(copy.deepcopy(labels))\n\n    if len(x_roi) == 0:\n        return None, None, None, None\n\n    # bboxes that iou > C.classifier_min_overlap for all gt bboxes in 300 non_max_suppression bboxes\n    X = np.array(x_roi)\n    # one hot code for bboxes from above => x_roi (X)\n    Y1 = np.array(y_class_num)\n    # corresponding labels and corresponding gt bboxes\n    Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)\n\n    return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):\n\t\"\"\"Convert rpn layer to roi bboxes\n\n\tArgs: (num_anchors = 9)\n\t\trpn_layer: output layer for rpn classification \n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 18) if resized image is 400 width and 300\n\t\tregr_layer: output layer for rpn regression\n\t\t\tshape (1, feature_map.height, feature_map.width, num_anchors)\n\t\t\tMight be (1, 18, 25, 72) if resized image is 400 width and 300\n\t\tC: config\n\t\tuse_regr: Wether to use bboxes regression in rpn\n\t\tmax_boxes: max bboxes number for non-max-suppression (NMS)\n\t\toverlap_thresh: If iou in NMS is larger than this threshold, drop the box\n\n\tReturns:\n\t\tresult: boxes from non-max-suppression (shape=(300, 4))\n\t\t\tboxes: coordinates for bboxes (on the feature map)\n\t\"\"\"\n\tregr_layer = regr_layer / C.std_scaling\n\n\tanchor_sizes = C.anchor_box_scales   # (3 in here)\n\tanchor_ratios = C.anchor_box_ratios  # (3 in here)\n\n\tassert rpn_layer.shape[0] == 1\n\n\t(rows, cols) = rpn_layer.shape[1:3]\n\n\tcurr_layer = 0\n\n\t# A.shape = (4, feature_map.height, feature_map.width, num_anchors) \n\t# Might be (4, 18, 25, 18) if resized image is 400 width and 300\n\t# A is the coordinates for 9 anchors for every point in the feature map \n\t# => all 18x25x9=4050 anchors cooridnates\n\tA = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))\n\n\tfor anchor_size in anchor_sizes:\n\t\tfor anchor_ratio in anchor_ratios:\n\t\t\t# anchor_x = (128 * 1) / 16 = 8  => width of current anchor\n\t\t\t# anchor_y = (128 * 2) / 16 = 16 => height of current anchor\n\t\t\tanchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride\n\t\t\tanchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride\n\t\t\t\n\t\t\t# curr_layer: 0~8 (9 anchors)\n\t\t\t# the Kth anchor of all position in the feature map (9th in total)\n\t\t\tregr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4] # shape => (18, 25, 4)\n\t\t\tregr = np.transpose(regr, (2, 0, 1)) # shape => (4, 18, 25)\n\n\t\t\t# Create 18x25 mesh grid\n\t\t\t# For every point in x, there are all the y points and vice versa\n\t\t\t# X.shape = (18, 25)\n\t\t\t# Y.shape = (18, 25)\n\t\t\tX, Y = np.meshgrid(np.arange(cols),np. arange(rows))\n\n\t\t\t# Calculate anchor position and size for each feature map point\n\t\t\tA[0, :, :, curr_layer] = X - anchor_x/2 # Top left x coordinate\n\t\t\tA[1, :, :, curr_layer] = Y - anchor_y/2 # Top left y coordinate\n\t\t\tA[2, :, :, curr_layer] = anchor_x       # width of current anchor\n\t\t\tA[3, :, :, curr_layer] = anchor_y       # height of current anchor\n\n\t\t\t# Apply regression to x, y, w and h if there is rpn regression layer\n\t\t\tif use_regr:\n\t\t\t\tA[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)\n\n\t\t\t# Avoid width and height exceeding 1\n\t\t\tA[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])\n\n\t\t\t# Convert (x, y , w, h) to (x1, y1, x2, y2)\n\t\t\t# x1, y1 is top left coordinate\n\t\t\t# x2, y2 is bottom right coordinate\n\t\t\tA[2, :, :, curr_layer] += A[0, :, :, curr_layer]\n\t\t\tA[3, :, :, curr_layer] += A[1, :, :, curr_layer]\n\n\t\t\t# Avoid bboxes drawn outside the feature map\n\t\t\tA[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])\n\t\t\tA[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])\n\t\t\tA[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])\n\t\t\tA[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])\n\n\t\t\tcurr_layer += 1\n\n\tall_boxes = np.reshape(A.transpose((0, 3, 1, 2)), (4, -1)).transpose((1, 0))  # shape=(4050, 4)\n\tall_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))                   # shape=(4050,)\n\n\tx1 = all_boxes[:, 0]\n\ty1 = all_boxes[:, 1]\n\tx2 = all_boxes[:, 2]\n\ty2 = all_boxes[:, 3]\n\n\t# Find out the bboxes which is illegal and delete them from bboxes list\n\tidxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))\n\n\tall_boxes = np.delete(all_boxes, idxs, 0)\n\tall_probs = np.delete(all_probs, idxs, 0)\n\n\t# Apply non_max_suppression\n\t# Only extract the bboxes. Don't need rpn probs in the later process\n\tresult = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]\n\n\treturn result\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Start Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = 'D:/5-deep_learning_with_tensorflow/module 7 cnn/7-faster_r-cnn/SAR_faster-rcnn'\n\ntrain_path =  'D:/5-deep_learning_with_tensorflow/module 7 cnn/7-faster_r-cnn/SAR_faster-rcnn/annotation.txt' # Training data (annotation file)\n\nnum_rois = 4 # Number of RoIs to process at once.\n\n# Augmentation flag\nhorizontal_flips = True # Augment with horizontal flips in training. \nvertical_flips = True   # Augment with vertical flips in training. \nrot_90 = True           # Augment with 90 degree rotations in training. \n\noutput_weight_path = os.path.join(base_path, 'model/model_frcnn_vgg.hdf5')\n\nrecord_path = os.path.join(base_path, 'model/record.csv') # Record data (used to save the losses, classification accuracy and mean average precision)\n\nbase_weight_path = os.path.join(base_path, 'model/vgg16_weights_tf_dim_ordering_tf_kernels.h5')\n\nconfig_output_filename = os.path.join(base_path, 'model_vgg_config.pickle')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the config\nC = Config()\n\nC.use_horizontal_flips = horizontal_flips\nC.use_vertical_flips = vertical_flips\nC.rot_90 = rot_90\n\nC.record_path = record_path\nC.model_path = output_weight_path\nC.num_rois = num_rois\n\nC.base_net_weights = base_weight_path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This step will spend some time to load the data        #\n#--------------------------------------------------------#\nst = time.time()\ntrain_imgs, classes_count, class_mapping = get_data(train_path)\nprint()\nprint('Spend %0.2f mins to load the data' % ((time.time()-st)/60) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 'bg' not in classes_count:\n\tclasses_count['bg'] = 0\n\tclass_mapping['bg'] = len(class_mapping)\n# e.g.\n#    classes_count: {'Car': 2383, 'Mobile phone': 1108, 'Person': 3745, 'bg': 0}\n#    class_mapping: {'Person': 0, 'Car': 1, 'Mobile phone': 2, 'bg': 3}\nC.class_mapping = class_mapping\n\nprint('Training images per class:')\npprint.pprint(classes_count)\nprint('Num classes (including bg) = {}'.format(len(classes_count)))\nprint(class_mapping)\n\n# Save the configuration\nwith open(config_output_filename, 'wb') as config_f:\n\tpickle.dump(C,config_f)\n\tprint('Config has been written to {}, and can be loaded when testing to ensure correct results'.format(config_output_filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shuffle the images with seed\nrandom.seed(1)\nrandom.shuffle(train_imgs)\n\nprint('Num train samples (images) {}'.format(len(train_imgs)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get train data generator which generate X, Y, image_data\ndata_gen_train = get_anchor_gt(train_imgs, C, get_img_output_length, mode='train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Original image: height=%d width=%d'%(image_data['height'], image_data['width']))\nprint('Resized image:  height=%d width=%d C.im_size=%d'%(X.shape[1], X.shape[2], C.im_size))\nprint('Feature map size: height=%d width=%d C.rpn_stride=%d'%(Y[0].shape[1], Y[0].shape[2], C.rpn_stride))\nprint(X.shape)\nprint(str(len(Y))+\" includes 'y_rpn_cls' and 'y_rpn_regr'\")\nprint('Shape of y_rpn_cls {}'.format(Y[0].shape))\nprint('Shape of y_rpn_regr {}'.format(Y[1].shape))\nprint(image_data)\n\nprint('Number of positive anchors for this image: %d' % (debug_num_pos))\nif debug_num_pos==0:\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['height']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['height'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['width']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['width'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n\n    plt.grid()\n    plt.imshow(img)\n    plt.show()\nelse:\n    cls = Y[0][0]\n    pos_cls = np.where(cls==1)\n    print(pos_cls)\n    regr = Y[1][0]\n    pos_regr = np.where(regr==1)\n    print(pos_regr)\n    print('y_rpn_cls for possible pos anchor: {}'.format(cls[pos_cls[0][0],pos_cls[1][0],:]))\n    print('y_rpn_regr for positive anchor: {}'.format(regr[pos_regr[0][0],pos_regr[1][0],:]))\n\n    gt_x1, gt_x2 = image_data['bboxes'][0]['x1']*(X.shape[2]/image_data['width']), image_data['bboxes'][0]['x2']*(X.shape[2]/image_data['width'])\n    gt_y1, gt_y2 = image_data['bboxes'][0]['y1']*(X.shape[1]/image_data['height']), image_data['bboxes'][0]['y2']*(X.shape[1]/image_data['height'])\n    gt_x1, gt_y1, gt_x2, gt_y2 = int(gt_x1), int(gt_y1), int(gt_x2), int(gt_y2)\n\n    img = debug_img.copy()\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    color = (0, 255, 0)\n    #   cv2.putText(img, 'gt bbox', (gt_x1, gt_y1-5), cv2.FONT_HERSHEY_DUPLEX, 0.7, color, 1)\n    cv2.rectangle(img, (gt_x1, gt_y1), (gt_x2, gt_y2), color, 2)\n    cv2.circle(img, (int((gt_x1+gt_x2)/2), int((gt_y1+gt_y2)/2)), 3, color, -1)\n\n    # Add text\n    textLabel = 'gt bbox'\n    (retval,baseLine) = cv2.getTextSize(textLabel,cv2.FONT_HERSHEY_COMPLEX,0.5,1)\n    textOrg = (gt_x1, gt_y1+5)\n    cv2.rectangle(img, (textOrg[0] - 5, textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (0, 0, 0), 2)\n    cv2.rectangle(img, (textOrg[0] - 5,textOrg[1]+baseLine - 5), (textOrg[0]+retval[0] + 5, textOrg[1]-retval[1] - 5), (255, 255, 255), -1)\n    cv2.putText(img, textLabel, textOrg, cv2.FONT_HERSHEY_DUPLEX, 0.5, (0, 0, 0), 1)\n\n    # Draw positive anchors according to the y_rpn_regr\n    for i in range(debug_num_pos):\n\n        color = (100+i*(155/4), 0, 100+i*(155/4))\n\n        idx = pos_regr[2][i*4]/4\n        anchor_size = C.anchor_box_scales[int(idx/3)]\n        anchor_ratio = C.anchor_box_ratios[2-int((idx+1)%3)]\n\n        center = (pos_regr[1][i*4]*C.rpn_stride, pos_regr[0][i*4]*C.rpn_stride)\n        print('Center position of positive anchor: ', center)\n        cv2.circle(img, center, 3, color, -1)\n        anc_w, anc_h = anchor_size*anchor_ratio[0], anchor_size*anchor_ratio[1]\n        cv2.rectangle(img, (center[0]-int(anc_w/2), center[1]-int(anc_h/2)), (center[0]+int(anc_w/2), center[1]+int(anc_h/2)), color, 2)\n#         cv2.putText(img, 'pos anchor bbox '+str(i+1), (center[0]-int(anc_w/2), center[1]-int(anc_h/2)-5), cv2.FONT_HERSHEY_DUPLEX, 0.5, color, 1)\n\nprint('Green bboxes is ground-truth bbox. Others are positive anchors')\nplt.figure(figsize=(8,8))\nplt.grid()\nplt.imshow(img)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}