Support Keras 2.0

.gitignore

@@ -92,3 +92,6 @@ ENV/
 
 # Rope project settings
 .ropeproject
+
+# Pycharm project settings
+.idea/

README.md

@@ -5,4 +5,5 @@ For forward pass for 300x300 model, please, follow `SSD.ipynb` for examples. For
 
 Weights are ported from the original models and are available [here](https://mega.nz/#F!7RowVLCL!q3cEVRK9jyOSB9el3SssIA). You need `weights_SSD300.hdf5`, `weights_300x300_old.hdf5` is for the old version of architecture with 3x3 convolution for `pool6`.
 
-This code was tested with `Keras` v1.2.2, `Tensorflow` v1.0.0, `OpenCV` v3.1.0-dev
+This code was tested with `Keras` v1.2.2, `Tensorflow` v1.0.0, `OpenCV` v3.1.0-dev\
+Also support newest `Keras` v2.0.1 (using ssd_v2.py)

feature_flow.py

@@ -0,0 +1,208 @@
+# coding: utf-8
+
+from keras.applications.imagenet_utils import preprocess_input
+from keras.preprocessing import image
+from scipy.misc import imread
+import tensorflow as tf
+from keras import backend as K
+import time
+
+from plot_util import *
+from flow_util import *
+from ssd_v2 import SSD300v2
+from ssd_conv4_3 import SSD300_conv4_3
+from ssd_utils import BBoxUtility
+
+voc_classes = ['Aeroplane', 'Bicycle', 'Bird', 'Boat', 'Bottle',
+               'Bus', 'Car', 'Cat', 'Chair', 'Cow', 'Diningtable',
+               'Dog', 'Horse', 'Motorbike', 'Person', 'Pottedplant',
+               'Sheep', 'Sofa', 'Train', 'Tvmonitor']
+
+NUM_CLASSES = len(voc_classes) + 1
+network_size = 1024
+batch_size = 2
+input_shape = (network_size, network_size, 3)
+colors = plt.cm.hsv(np.linspace(0, 1, 21)).tolist()
+
+use_feature_flow = True
+use_dump_file = False
+plot_activation_enable = False
+#image_files = ['/home/cory/cedl/vid/videos/vid04/0270.jpg', '/home/cory/cedl/vid/videos/vid04/0275.jpg']
+#image_files = ['/home/cory/KITTI_Dataset/data_tracking_image_2/training/image_02/0000/000015.png',
+#               '/home/cory/KITTI_Dataset/data_tracking_image_2/training/image_02/0000/000018.png']
+
+
+# magic case: vid04 270 - 299
+
+# image_files = ['/home/cory/ssd_keras/GTAV/GD1015.png', '/home/cory/ssd_keras/GTAV/GD1020.png']
+image_files = ['/home/cory/ssd_keras/GTAV/GD1293.png', '/home/cory/ssd_keras/GTAV/GD1295.png']
+# '/home/cory/ssd_keras/GTAV/GD21.png'
+# '/home/cory/cedl/vid/videos/vid04/1000.jpg'
+
+
+def get_detections(result):
+    detections = map(lambda r: {'label': r[0],
+                                'conf': r[1],
+                                'xmin': r[2],
+                                'ymin': r[3],
+                                'xmax': r[4],
+                                'ymax': r[5]},
+                     result)
+    return detections
+
+
+def get_layer_output(model, inputs, output_layer_name):
+    immediate_layer = K.function([model.input, K.learning_phase()],
+                                 [model.get_layer(name=output_layer_name).output])
+    output = immediate_layer([inputs, 1])[0]
+    return output
+
+
+def get_layer_predict(model, input_layer_name, input_layer_feature):
+    immediate_layer = K.function([model.get_layer(name=input_layer_name), K.learning_phase()],
+                                 [model.output])
+    model_predict = immediate_layer([input_layer_feature, 1])[0]
+    return model_predict
+
+
+def load_inputs(file_list):
+    inputs = []
+    images = []
+    for file in file_list:
+        img = image.load_img(file, target_size=(network_size, network_size))
+        inputs.append(image.img_to_array(img))
+        images.append(imread(file))
+    return inputs, images
+
+
+def run_network(model, inputs):
+    time_begin = time.time()
+    predictions = model.predict(inputs, batch_size=batch_size, verbose=1)
+    time_elapsed = time.time() - time_begin
+    print('elapsed time {:0.4f} sec  {:.4f} fps'.format(time_elapsed, batch_size / time_elapsed))
+    return predictions
+
+
+def compare_model_layer(model1, input1, layer1, model2, input2, layer2, plot_activation_enable=False):
+    layer_output1 = get_layer_output(model=model1, inputs=input1, output_layer_name=layer1)
+    layer_output2 = get_layer_output(model=model2, inputs=input2, output_layer_name=layer2)
+    diff = (layer_output1 - layer_output2)
+
+    print('layer_output1 sum =', sum(layer_output1[0].ravel()))
+    print('layer_output2 sum =', sum(layer_output2[0].ravel()))
+    print('diff min={:f} max={:f} sum={:f}'.format(
+        min(np.absolute(diff).ravel()),
+        max(np.absolute(diff).ravel()),
+        sum(np.absolute(diff).ravel())))
+    eq = np.array_equal(layer_output1, layer_output2)
+    if eq:
+        print('equal')
+    else:
+        print('not equal')
+
+    if plot_activation_enable:
+        plot_feature_map(layer_output1[0], 'feature_map_1')
+        plot_feature_map(layer_output2[0], 'feature_map_2')
+
+
+def plot_detections(image_list, detection_result):
+    # for each image
+    for i, img in enumerate(image_list):
+        detections = get_detections(detection_result[i])
+        detections = list(filter(lambda x: x['conf'] > 0.8, detections))
+        fig = imshow_fig(img, title='frame_{:d}'.format(i+1))
+
+        current_axis = fig.gca()
+        for det in detections:
+            xmin = int(round(det['xmin'] * img.shape[1]))
+            ymin = int(round(det['ymin'] * img.shape[0]))
+            xmax = int(round(det['xmax'] * img.shape[1]))
+            ymax = int(round(det['ymax'] * img.shape[0]))
+            conf = det['conf']
+            label = int(det['label'])
+            label_name = voc_classes[label - 1]
+            display_txt = '{:0.2f}, {}'.format(conf, label_name)
+            # print(display_txt)
+            coords = (xmin, ymin), xmax - xmin + 1, ymax - ymin + 1
+            color = colors[label]
+            current_axis.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=2))
+            current_axis.text(xmin, ymin, display_txt, bbox={'facecolor': color, 'alpha': 0.5})
+        fig.show()
+
+
+def feature_flow():
+    bbox_util = BBoxUtility(NUM_CLASSES)
+    raw_inputs, images = load_inputs(image_files)
+    inputs = preprocess_input(np.array(raw_inputs))
+
+    dump_activation_layer = 'conv4_2'
+    compare_layer_name = 'conv6_2'
+    print('dump_activation_layer', dump_activation_layer)
+    print('target_layer_name', compare_layer_name)
+
+    # normal SSD network
+    model1 = SSD300v2(input_shape, num_classes=NUM_CLASSES)
+    model1.load_weights('weights_SSD300.hdf5', by_name=True)
+    predictions = run_network(model1, inputs)
+    results = bbox_util.detection_out(predictions)
+    plot_detections(images, results)
+
+    # get dump layer's output (as input for flow network)
+    input_img2 = inputs[1:2, :, :, :]
+    layer_dump = get_layer_output(model=model1, inputs=input_img2, output_layer_name=dump_activation_layer)
+    print('layer_dump.shape = ', layer_dump.shape)
+
+    # flow (raw rgb)
+    flow_rgb = compute_flow(image_files[1], image_files[0])
+
+    print('flow.shape', flow_rgb.shape)
+    imshow_fig(cv2.cvtColor(draw_hsv(flow_rgb), cv2.COLOR_BGR2RGB), title='flow_rgb')
+
+    # flow (re-sized for feature map)
+    flow_feature = get_flow_for_filter(flow_rgb)
+    # imshow_fig(flow_feature[:, :, 0], title='flow_feature_y', cmap='gray')
+    # imshow_fig(flow_feature[:, :, 1], title='flow_feature_x', cmap='gray')
+
+    # warp image by flow_rgb
+    iimg1 = cv2.imread(image_files[0])
+    img_warp = warp_flow(iimg1, flow_rgb)
+    imshow_fig(cv2.cvtColor(img_warp, cv2.COLOR_BGR2RGB), title='frame_2_warp')
+
+    # shift feature
+    shifted_feature = shift_filter(layer_dump, flow_feature)
+
+    # flow net
+    model2 = SSD300_conv4_3((128, 128, 512), num_classes=NUM_CLASSES)
+    model2.load_weights('weights_SSD300.hdf5', by_name=True)
+    predictions = run_network(model2, shifted_feature)
+    results = bbox_util.detection_out(predictions)
+    plot_detections(images[1:2], results)
+
+    # get specific layer's output and compare them (for debugging)
+    compare_model_layer(model1, input_img2, compare_layer_name,
+                        model2, shifted_feature, compare_layer_name,
+                        True)
+
+    sess.close()
+    plt.show()
+
+
+def get_flow_for_filter(flow):
+    filter_map_width = 128
+    flow_ratio_y = flow.shape[0] / filter_map_width
+    flow_ratio_x = flow.shape[1] / filter_map_width
+    flow_small = np.asarray([cv2.resize(flow[:, :, 0] / flow_ratio_y, (filter_map_width, filter_map_width)),
+                             cv2.resize(flow[:, :, 1] / flow_ratio_x, (filter_map_width, filter_map_width))])
+    flow_small = flow_small.transpose([1, 2, 0])
+    print('flow_small.shape', flow_small.shape)
+    return flow_small
+
+
+if __name__ == '__main__':
+    config = tf.ConfigProto(
+        device_count={'GPU': 1}
+    )
+    config.gpu_options.per_process_gpu_memory_fraction = 0.5
+    sess = tf.Session(config=config)
+    K.set_session(sess)
+    feature_flow()

flow_util.py

@@ -0,0 +1,50 @@
+import matplotlib.pyplot as plt
+import cv2
+import numpy as np
+from subprocess import check_output
+
+
+def shift_filter(feature, flow):
+    # feature shape = (None, 128, 128, 512)
+    shifted_feature = list()
+    for feat in feature:
+        print(feat.shape)
+        for i in range(feat.shape[-1]):
+            act2d = feat[..., i]
+            act2d = act2d[:, :, np.newaxis]
+            res = warp_flow(act2d, flow)
+            shifted_feature.append(res)
+
+            if False:
+                print('act2d', act2d.shape, sum(act2d.ravel()))
+                print('flow', flow.shape, sum(flow.ravel()))
+                plt.figure(11)
+                plt.imshow(act2d[:, :, 0], cmap='gray')
+                plt.figure(12)
+                plt.imshow(flow[..., 0], cmap='gray')
+                plt.figure(13)
+                plt.imshow(flow[..., 1], cmap='gray')
+                plt.figure(14)
+                plt.imshow(res, cmap='gray')
+                plt.show()
+                pass
+
+    return np.asarray([shifted_feature]).swapaxes(1, 2).swapaxes(2, 3)
+
+
+def compute_flow(image_path1, image_path2):
+    flow_cmd = './run_flow.sh ' + image_path1 + ' ' + image_path2
+    check_output([flow_cmd], shell=True)
+    flow = np.load('./flow.npy')
+    flow = flow.transpose(1, 2, 0)
+    # flow.shape should be (height, width, 2)
+    return flow
+
+
+def warp_flow(img, flow):
+    h, w = flow.shape[:2]
+    flow_map = flow.copy()
+    flow_map[:, :, 0] += np.arange(w)
+    flow_map[:, :, 1] += np.arange(h)[:, np.newaxis]
+    res = cv2.remap(img, flow_map, None, cv2.INTER_LINEAR)
+    return res

plot_util.py

@@ -0,0 +1,56 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2
+import math
+
+
+def imshow_fig(img, title='', **kwargs):
+    h = img.shape[0]
+    w = img.shape[1]
+    dpi = 96
+    fig = plt.figure(figsize=(w/dpi, h/dpi))
+    fig.add_axes([0., 0., 1., 1.])
+    fig.canvas.set_window_title(title)
+    plt.imshow(img, **kwargs)
+    plt.axis('off')
+    return fig
+
+
+def plot_feature_map(activations, title=''):
+
+    num_channel = activations.shape[2]
+    act_border = activations.shape[0]
+    map_border_num = int(math.ceil(math.sqrt(num_channel)))
+    map_border = act_border * map_border_num
+    print('create act map {:d} x {:d}'.format(map_border, map_border))
+    act_map = np.zeros((map_border, map_border))
+
+    print(activations.shape)
+    all_sum = 0
+    for i_x in range(map_border_num):
+        for i_y in range(map_border_num):
+            idx = i_x * map_border_num + i_y
+            if idx >= num_channel:
+                break
+            act = activations[:, :, idx]
+            act_map[i_x*act_border:(i_x+1)*act_border, i_y*act_border:(i_y+1)*act_border] = act
+            act_sum = sum(sum(act))
+            all_sum += act_sum
+            # print('filter-{:d}  act_sum={:f}'.format(idx, act_sum))
+
+    print('all_sum = {:f}'.format(all_sum))
+    fig = imshow_fig(act_map, title, cmap='gray')
+    fig.show()
+
+
+def draw_hsv(flow):
+    h, w = flow.shape[:2]
+    fx, fy = flow[:,:,0], flow[:,:,1]
+    ang = np.arctan2(fy, fx) + np.pi
+    v = np.sqrt(fx*fx+fy*fy)
+    hsv = np.zeros((h, w, 3), np.uint8)
+    hsv[...,0] = ang*(180/np.pi/2)
+    hsv[...,1] = 255
+    hsv[...,2] = np.minimum(v*4, 255)
+    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
+    return bgr