{"cells":[{"metadata":{"_uuid":"3f701b35-203f-4d73-aa6a-0d2a642948f9","_cell_guid":"459430f3-a81c-43e0-911a-00825bc62472","trusted":true},"cell_type":"code","source":"import argparse\nimport os\nimport numpy as np\nfrom keras.layers import Conv2D, Input, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D\nfrom keras.layers.merge import add, concatenate\nfrom keras.models import Model\nimport struct\nimport cv2\n\nnp.set_printoptions(threshold=np.nan)\nos.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\nos.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n\nargparser = argparse.ArgumentParser(\n description='test yolov3 network with coco weights')\n\nargparser.add_argument(\n '-w',\n '--weights',\n help='path to weights file')\n\nargparser.add_argument(\n '-i',\n '--image',\n help='path to image file')\n\nclass WeightReader:\n def __init__(self, weight_file):\n with open(weight_file, 'rb') as w_f:\n major, = struct.unpack('i', w_f.read(4))\n minor, = struct.unpack('i', w_f.read(4))\n revision, = struct.unpack('i', w_f.read(4))\n\n if (major*10 + minor) >= 2 and major < 1000 and minor < 1000:\n w_f.read(8)\n else:\n w_f.read(4)\n\n transpose = (major > 1000) or (minor > 1000)\n \n binary = w_f.read()\n\n self.offset = 0\n self.all_weights = np.frombuffer(binary, dtype='float32')\n \n def read_bytes(self, size):\n self.offset = self.offset + size\n return self.all_weights[self.offset-size:self.offset]\n\n def load_weights(self, model):\n for i in range(106):\n try:\n conv_layer = model.get_layer('conv_' + str(i))\n print(\"loading weights of convolution #\" + str(i))\n\n if i not in [81, 93, 105]:\n norm_layer = model.get_layer('bnorm_' + str(i))\n\n size = np.prod(norm_layer.get_weights()[0].shape)\n\n beta = self.read_bytes(size) # bias\n gamma = self.read_bytes(size) # scale\n mean = self.read_bytes(size) # mean\n var = self.read_bytes(size) # variance \n\n weights = norm_layer.set_weights([gamma, beta, mean, var]) \n\n if len(conv_layer.get_weights()) > 1:\n bias = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape))\n kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n \n kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n kernel = kernel.transpose([2,3,1,0])\n conv_layer.set_weights([kernel, bias])\n else:\n kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape))\n kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape)))\n kernel = kernel.transpose([2,3,1,0])\n conv_layer.set_weights([kernel])\n except ValueError:\n print(\"no convolution #\" + str(i)) \n \n def reset(self):\n self.offset = 0\n\nclass BoundBox:\n def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None):\n self.xmin = xmin\n self.ymin = ymin\n self.xmax = xmax\n self.ymax = ymax\n \n self.objness = objness\n self.classes = classes\n\n self.label = -1\n self.score = -1\n\n def get_label(self):\n if self.label == -1:\n self.label = np.argmax(self.classes)\n \n return self.label\n \n def get_score(self):\n if self.score == -1:\n self.score = self.classes[self.get_label()]\n \n return self.score\n\ndef _conv_block(inp, convs, skip=True):\n x = inp\n count = 0\n \n for conv in convs:\n if count == (len(convs) - 2) and skip:\n skip_connection = x\n count += 1\n \n if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top\n x = Conv2D(conv['filter'], \n conv['kernel'], \n strides=conv['stride'], \n padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top\n name='conv_' + str(conv['layer_idx']), \n use_bias=False if conv['bnorm'] else True)(x)\n if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x)\n if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x)\n\n return add([skip_connection, x]) if skip else x\n\ndef _interval_overlap(interval_a, interval_b):\n x1, x2 = interval_a\n x3, x4 = interval_b\n\n if x3 < x1:\n if x4 < x1:\n return 0\n else:\n return min(x2,x4) - x1\n else:\n if x2 < x3:\n return 0\n else:\n return min(x2,x4) - x3 \n\ndef _sigmoid(x):\n return 1. / (1. + np.exp(-x))\n\ndef bbox_iou(box1, box2):\n intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])\n intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])\n \n intersect = intersect_w * intersect_h\n\n w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin\n w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin\n \n union = w1*h1 + w2*h2 - intersect\n \n return float(intersect) / union\n\ndef make_yolov3_model():\n input_image = Input(shape=(None, None, 3))\n\n # Layer 0 => 4\n x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0},\n {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1},\n {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2},\n {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}])\n\n # Layer 5 => 8\n x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5},\n {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6},\n {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}])\n\n # Layer 9 => 11\n x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9},\n {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}])\n\n # Layer 12 => 15\n x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12},\n {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13},\n {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}])\n\n # Layer 16 => 36\n for i in range(7):\n x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3},\n {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}])\n \n skip_36 = x\n \n # Layer 37 => 40\n x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37},\n {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38},\n {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}])\n\n # Layer 41 => 61\n for i in range(7):\n x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3},\n {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}])\n \n skip_61 = x\n \n # Layer 62 => 65\n x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62},\n {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63},\n {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}])\n\n # Layer 66 => 74\n for i in range(3):\n x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3},\n {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}])\n \n # Layer 75 => 79\n x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75},\n {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76},\n {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77},\n {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78},\n {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False)\n\n # Layer 80 => 82\n yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80},\n {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False)\n\n # Layer 83 => 86\n x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False)\n x = UpSampling2D(2)(x)\n x = concatenate([x, skip_61])\n\n # Layer 87 => 91\n x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87},\n {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88},\n {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89},\n {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90},\n {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False)\n\n # Layer 92 => 94\n yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92},\n {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False)\n\n # Layer 95 => 98\n x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}], skip=False)\n x = UpSampling2D(2)(x)\n x = concatenate([x, skip_36])\n\n # Layer 99 => 106\n yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99},\n {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100},\n {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101},\n {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102},\n {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103},\n {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104},\n {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False)\n\n model = Model(input_image, [yolo_82, yolo_94, yolo_106]) \n return model\n\ndef preprocess_input(image, net_h, net_w):\n new_h, new_w, _ = image.shape\n\n # determine the new size of the image\n if (float(net_w)/new_w) < (float(net_h)/new_h):\n new_h = (new_h * net_w)/new_w\n new_w = net_w\n else:\n new_w = (new_w * net_h)/new_h\n new_h = net_h\n\n # resize the image to the new size\n resized = cv2.resize(image[:,:,::-1]/255., (int(new_w), int(new_h)))\n\n # embed the image into the standard letter box\n new_image = np.ones((net_h, net_w, 3)) * 0.5\n new_image[int((net_h-new_h)//2):int((net_h+new_h)//2), int((net_w-new_w)//2):int((net_w+new_w)//2), :] = resized\n new_image = np.expand_dims(new_image, 0)\n\n return new_image\n\ndef decode_netout(netout, anchors, obj_thresh, nms_thresh, net_h, net_w):\n grid_h, grid_w = netout.shape[:2]\n nb_box = 3\n netout = netout.reshape((grid_h, grid_w, nb_box, -1))\n nb_class = netout.shape[-1] - 5\n\n boxes = []\n\n netout[..., :2] = _sigmoid(netout[..., :2])\n netout[..., 4:] = _sigmoid(netout[..., 4:])\n netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:]\n netout[..., 5:] *= netout[..., 5:] > obj_thresh\n\n for i in range(grid_h*grid_w):\n row = i / grid_w\n col = i % grid_w\n \n for b in range(nb_box):\n # 4th element is objectness score\n objectness = netout[int(row)][int(col)][b][4]\n #objectness = netout[..., :4]\n \n if(objectness.all() <= obj_thresh): continue\n \n # first 4 elements are x, y, w, and h\n x, y, w, h = netout[int(row)][int(col)][b][:4]\n\n x = (col + x) / grid_w # center position, unit: image width\n y = (row + y) / grid_h # center position, unit: image height\n w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width\n h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height \n \n # last elements are class probabilities\n classes = netout[int(row)][col][b][5:]\n \n box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)\n #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes)\n\n boxes.append(box)\n\n return boxes\n\ndef correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):\n if (float(net_w)/image_w) < (float(net_h)/image_h):\n new_w = net_w\n new_h = (image_h*net_w)/image_w\n else:\n new_h = net_w\n new_w = (image_w*net_h)/image_h\n \n for i in range(len(boxes)):\n x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w\n y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h\n \n boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)\n boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)\n boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)\n boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)\n \ndef do_nms(boxes, nms_thresh):\n if len(boxes) > 0:\n nb_class = len(boxes[0].classes)\n else:\n return\n \n for c in range(nb_class):\n sorted_indices = np.argsort([-box.classes[c] for box in boxes])\n\n for i in range(len(sorted_indices)):\n index_i = sorted_indices[i]\n\n if boxes[index_i].classes[c] == 0: continue\n\n for j in range(i+1, len(sorted_indices)):\n index_j = sorted_indices[j]\n\n if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:\n boxes[index_j].classes[c] = 0\n \ndef draw_boxes(image, boxes, labels, obj_thresh):\n for box in boxes:\n label_str = ''\n label = -1\n \n for i in range(len(labels)):\n if box.classes[i] > obj_thresh:\n label_str += labels[i]\n label = i\n print(labels[i] + ': ' + str(box.classes[i]*100) + '%')\n \n if label >= 0:\n cv2.rectangle(image, (box.xmin,box.ymin), (box.xmax,box.ymax), (0,255,0), 3)\n cv2.putText(image, \n label_str + ' ' + str(box.get_score()), \n (box.xmin, box.ymin - 13), \n cv2.FONT_HERSHEY_SIMPLEX, \n 1e-3 * image.shape[0], \n (0,255,0), 2)\n \n return image \n\ndef _main_(args):\n weights_path = args.weights\n image_path = args.image\n\n # set some parameters\n net_h, net_w = 416, 416\n obj_thresh, nms_thresh = 0.5, 0.45\n anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]]\n labels = [\"person\", \"bicycle\", \"car\", \"motorbike\", \"aeroplane\", \"bus\", \"train\", \"truck\", \\\n \"boat\", \"traffic light\", \"fire hydrant\", \"stop sign\", \"parking meter\", \"bench\", \\\n \"bird\", \"cat\", \"dog\", \"horse\", \"sheep\", \"cow\", \"elephant\", \"bear\", \"zebra\", \"giraffe\", \\\n \"backpack\", \"umbrella\", \"handbag\", \"tie\", \"suitcase\", \"frisbee\", \"skis\", \"snowboard\", \\\n \"sports ball\", \"kite\", \"baseball bat\", \"baseball glove\", \"skateboard\", \"surfboard\", \\\n \"tennis racket\", \"bottle\", \"wine glass\", \"cup\", \"fork\", \"knife\", \"spoon\", \"bowl\", \"banana\", \\\n \"apple\", \"sandwich\", \"orange\", \"broccoli\", \"carrot\", \"hot dog\", \"pizza\", \"donut\", \"cake\", \\\n \"chair\", \"sofa\", \"pottedplant\", \"bed\", \"diningtable\", \"toilet\", \"tvmonitor\", \"laptop\", \"mouse\", \\\n \"remote\", \"keyboard\", \"cell phone\", \"microwave\", \"oven\", \"toaster\", \"sink\", \"refrigerator\", \\\n \"book\", \"clock\", \"vase\", \"scissors\", \"teddy bear\", \"hair drier\", \"toothbrush\"]\n\n # make the yolov3 model to predict 80 classes on COCO\n yolov3 = make_yolov3_model()\n\n # load the weights trained on COCO into the model\n weight_reader = WeightReader(weights_path)\n weight_reader.load_weights(yolov3)\n\n # preprocess the image\n image = cv2.imread(image_path)\n image_h, image_w, _ = image.shape\n new_image = preprocess_input(image, net_h, net_w)\n\n # run the prediction\n yolos = yolov3.predict(new_image)\n boxes = []\n\n for i in range(len(yolos)):\n # decode the output of the network\n boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, nms_thresh, net_h, net_w)\n\n # correct the sizes of the bounding boxes\n correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w)\n\n # suppress non-maximal boxes\n do_nms(boxes, nms_thresh) \n\n # draw bounding boxes on the image using labels\n draw_boxes(image, boxes, labels, obj_thresh) \n \n # write the image with bounding boxes to file\n cv2.imwrite(image_path[:-4] + '_detected' + image_path[-4:], (image).astype('uint8')) \n\nif __name__ == '__main__':\n args = argparser.parse_args()\n _main_(args)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}