diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a0114d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +build/* +*.pyc diff --git a/CMakeLists.txt b/CMakeLists.txt index 904cda2..fd3f277 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,14 @@ cmake_minimum_required(VERSION 2.8) project( gms_match_demo) +set( CMAKE_CXX_STANDARD 11 ) -# OpenCV +# OpenCV find_package( OpenCV REQUIRED ) -# -include_directories( +# +include_directories( include -${OpenCV_INCLUDE_DIRS} +${OpenCV_INCLUDE_DIRS} ) diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..45c628a --- /dev/null +++ b/data/README.md @@ -0,0 +1,12 @@ +# README on Photos + +This describes the photos kf_\*.png +They photos are drawn from a video sequence. The numbers denote the keyframe number. For example kf_972.png and kf_973.png are consecutive frames. + +There are 3 separate physical locations + +- kf_8, kf_12, kf_558, kf_589 are captured in same physical location. + +- kf_972, kf_973, kf_2397 are captured in same physical location. + +- kf_1140, kf_2582, kf_2583 are captured in same physical location. diff --git a/data/kf_1140.png b/data/kf_1140.png new file mode 100644 index 0000000..5199542 Binary files /dev/null and b/data/kf_1140.png differ diff --git a/data/kf_12.png b/data/kf_12.png new file mode 100644 index 0000000..e3b245c Binary files /dev/null and b/data/kf_12.png differ diff --git a/data/kf_2397.png b/data/kf_2397.png new file mode 100644 index 0000000..097d7fa Binary files /dev/null and b/data/kf_2397.png differ diff --git a/data/kf_2581.png b/data/kf_2581.png new file mode 100644 index 0000000..fdba883 Binary files /dev/null and b/data/kf_2581.png differ diff --git a/data/kf_2582.png b/data/kf_2582.png new file mode 100644 index 0000000..23a4ca5 Binary files /dev/null and b/data/kf_2582.png differ diff --git a/data/kf_2583.png b/data/kf_2583.png new file mode 100644 index 0000000..23a4ca5 Binary files /dev/null and b/data/kf_2583.png differ diff --git a/data/kf_558.png b/data/kf_558.png new file mode 100644 index 0000000..0c870ba Binary files /dev/null and b/data/kf_558.png differ diff --git a/data/kf_589.png b/data/kf_589.png new file mode 100644 index 0000000..7eb0430 Binary files /dev/null and b/data/kf_589.png differ diff --git a/data/kf_8.png b/data/kf_8.png new file mode 100644 index 0000000..6d88cfe Binary files /dev/null and b/data/kf_8.png differ diff --git a/data/kf_972.png b/data/kf_972.png new file mode 100644 index 0000000..bcda126 Binary files /dev/null and b/data/kf_972.png differ diff --git a/data/kf_973.png b/data/kf_973.png new file mode 100644 index 0000000..e12d159 Binary files /dev/null and b/data/kf_973.png differ diff --git a/data/nn_right.jpg b/data/nn_right.jpg index 991a83c..3cb3a93 100644 Binary files a/data/nn_right.jpg and b/data/nn_right.jpg differ diff --git a/include/Header.h b/include/Header.h index d0e0760..4a6822d 100644 --- a/include/Header.h +++ b/include/Header.h @@ -9,6 +9,7 @@ #include #include #include +#include using namespace std; using namespace cv; @@ -16,4 +17,3 @@ using namespace cv; #include using cuda::GpuMat; #endif - diff --git a/include/gms_matcher.h b/include/gms_matcher.h index 793d4b0..255ee13 100644 --- a/include/gms_matcher.h +++ b/include/gms_matcher.h @@ -3,7 +3,7 @@ #define THRESH_FACTOR 6 -// 8 possible rotation and each one is 3 X 3 +// 8 possible rotation and each one is 3 X 3 const int mRotationPatterns[8][9] = { 1,2,3, 4,5,6, @@ -45,8 +45,8 @@ const double mScaleRatios[5] = { 1.0, 1.0 / 2, 1.0 / sqrt(2.0), sqrt(2.0), 2.0 } class gms_matcher { public: - // OpenCV Keypoints & Correspond Image Size & Nearest Neighbor Matches - gms_matcher(const vector &vkp1, const Size size1, const vector &vkp2, const Size size2, const vector &vDMatches) + // OpenCV Keypoints & Correspond Image Size & Nearest Neighbor Matches + gms_matcher(const vector &vkp1, const Size size1, const vector &vkp2, const Size size2, const vector &vDMatches) { // Input initialize NormalizePoints(vkp1, size1, mvP1); @@ -58,7 +58,7 @@ class gms_matcher mGridSizeLeft = Size(20, 20); mGridNumberLeft = mGridSizeLeft.width * mGridSizeLeft.height; - // Initialize the neihbor of left grid + // Initialize the neihbor of left grid mGridNeighborLeft = Mat::zeros(mGridNumberLeft, 9, CV_32SC1); InitalizeNiehbors(mGridNeighborLeft, mGridSizeLeft); }; @@ -87,14 +87,14 @@ class gms_matcher // value : how many matches from idx_left to idx_right Mat mMotionStatistics; - // + // vector mNumberPointsInPerCellLeft; // Inldex : grid_idx_left // Value : grid_idx_right vector mCellPairs; - // Every Matches has a cell-pair + // Every Matches has a cell-pair // first : grid_idx_left // second : grid_idx_right vector > mvMatchPairs; @@ -110,7 +110,7 @@ class gms_matcher public: // Get Inlier Mask - // Return number of inliers + // Return number of inliers int GetInlierMask(vector &vbInliers, bool WithScale = false, bool WithRotation = false); private: @@ -177,7 +177,7 @@ class gms_matcher return x + y * mGridSizeRight.width; } - // Assign Matches to Cell Pairs + // Assign Matches to Cell Pairs void AssignMatchPairs(int GridType); // Verify Cell Pairs @@ -193,7 +193,7 @@ class gms_matcher for (int yi = -1; yi <= 1; yi++) { for (int xi = -1; xi <= 1; xi++) - { + { int idx_xx = idx_x + xi; int idx_yy = idx_y + yi; @@ -222,13 +222,13 @@ class gms_matcher mGridSizeRight.height = mGridSizeLeft.height * mScaleRatios[Scale]; mGridNumberRight = mGridSizeRight.width * mGridSizeRight.height; - // Initialize the neihbor of right grid + // Initialize the neihbor of right grid mGridNeighborRight = Mat::zeros(mGridNumberRight, 9, CV_32SC1); InitalizeNiehbors(mGridNeighborRight, mGridSizeRight); } - // Run + // Run int run(int RotationType); }; @@ -292,7 +292,7 @@ int gms_matcher::GetInlierMask(vector &vbInliers, bool WithScale, bool Wit vbInliers = mvbInlierMask; max_inlier = num_inlier; } - + } return max_inlier; } @@ -356,7 +356,7 @@ void gms_matcher::VerifyCellPairs(int RotationType) { int idx_grid_rt = mCellPairs[i]; const int *NB9_lt = mGridNeighborLeft.ptr(i); - const int *NB9_rt = mGridNeighborRight.ptr(idx_grid_rt); + const int *NB9_rt = mGridNeighborRight.ptr(idx_grid_rt); int score = 0; double thresh = 0; @@ -389,13 +389,13 @@ int gms_matcher::run(int RotationType) { mMotionStatistics = Mat::zeros(mGridNumberLeft, mGridNumberRight, CV_32SC1); mvMatchPairs.assign(mNumberMatches, pair(0, 0)); - for (int GridType = 1; GridType <= 4; GridType++) + for (int GridType = 1; GridType <= 4; GridType++) { // initialize mMotionStatistics.setTo(0); mCellPairs.assign(mGridNumberLeft, -1); mNumberPointsInPerCellLeft.assign(mGridNumberLeft, 0); - + AssignMatchPairs(GridType); VerifyCellPairs(RotationType); @@ -421,7 +421,7 @@ inline Mat DrawInlier(Mat &src1, Mat &src2, vector &kpt1, vector &kpt1, vector &kpt1, vector(src.cols * 1.0 / ratio); resize(src, src, Size(width, height)); } - - - - diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..a5c89e5 --- /dev/null +++ b/python/README.md @@ -0,0 +1,42 @@ +# Usage of gms_matcher.py + +Cntains 2 classes, viz. GmsMatcher and GmsRobe. + +## GmsMatcher +This class is the original work as presented in the paper. +``` +@inproceedings{bian2017gms, + title={GMS: Grid-based Motion Statistics for Fast, Ultra-robust Feature Correspondence}, + author={JiaWang Bian and Wen-Yan Lin and Yasuyuki Matsushita and Sai-Kit Yeung and Tan Dat Nguyen and Ming-Ming Cheng}, + booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, + year={2017} +} +``` + +For demo usage see function, `demo_gms_original_implementation()` in `demo.py` + + +## GmsRobe +This class extents the functionality of GmsMatcher. +The GmsMatcher returns an array of DMatch. This might be sometimes convinent to use. +As you need to make reference to the keypoints to get to the image co-ordinates in question. + +GmsRobe class defines the following functions + - match2( imC, imP )
+ Given current image(imC) and prev image(imP) returns GMS matches co-ordinates (x,y)_i + Essentially this is a thin wrapper around the original GmsMatcher class + + + - match3( imC, imP, imCm )
+ To find 3way correspondences, ie. a set of point co-ordinates in imC which also occur in imP and imCm. a<-->b<-->c. Note: The order in which the images is given is critical. + + + - match2_guided( imC, pt1, imP )
+ Given current image (imC), with pts_C 2xN as input points on imC, the + objective is to find these points in previous image (imP) + + The way we do this is to first compute all the matches between imC and imP. + Then filter these matches to include only those pysically close to input points. + + +Following functions return 2xN matrix with (x,y) of matches. diff --git a/python/demo.py b/python/demo.py new file mode 100644 index 0000000..7ee1dc9 --- /dev/null +++ b/python/demo.py @@ -0,0 +1,125 @@ +## This script has demo use cases for the original GmsMatcher class as well as +## for the GmsRobe class. + +import math +from enum import Enum + +import cv2 +import numpy as np +cv2.ocl.setUseOpenCL(False) + +import time +import code + + +from gms_matcher import GmsMatcher +from gms_matcher import GmsRobe +from gms_matcher import DrawingType + +def imresize(src, height): + ratio = src.shape[0] * 1.0/height + width = int(src.shape[1] * 1.0/ratio) + return cv2.resize(src, (width, height)) + +def demo_match3(): + + imgP = cv2.imread("../data/kf_1140.png") #curr + imgC = cv2.imread("../data/kf_2583.png") #prev + imgCm = cv2.imread("../data/kf_2581.png") #currm + + gms = GmsRobe() + + print 'Test gmsrobe.match3()' + startT = time.time() + pts_C, pts_P, pts_Cm = gms.match3( imgC, imgP, imgCm ) + print 'gmsrobe.match3 took (ms): %4.2f' %(1000.*(time.time() - startT ) ) + + gridd = gms.plot_3way_match( imgC, pts_C, imgP, pts_P, imgCm, pts_Cm, show_random_points=40 ) + gridd2 = gms.plot_3way_match( imgC, pts_C, imgP, pts_P, imgCm, pts_Cm, show_random_points=-1, enable_lines=False ) + cv2.imshow( 'showing 3way match. Randomly selected 40 points only', gridd ) + cv2.imshow( 'showing 3way match.', gridd2 ) + cv2.waitKey(0) + + + +def demo_match2_guided(): + + print 'Test gmsrobe.match2_guided()' + imgP = cv2.imread("../data/kf_1140.png") #curr + imgC = cv2.imread("../data/kf_2583.png") #prev + + orb = cv2.ORB_create(50) + kp1 = orb.detect(imgC, None) + print 'nORB Pts:', len(kp1) + pts_C = np.transpose( np.array([ np.array(k.pt) for k in kp1 ]) ) #2xN + + + gms = GmsRobe() + + startT = time.time() + ptC, ptP = gms.match2_guided( imgC, pts_C, imgP ) + print 'Elapsed total (ms): %4.2f' %(1000.0 * (time.time() - startT ) ) + print 'nGuided Pts: ', ptC.shape[1] + + cv2.imshow( 'orb points on curr', gms.plot_points_on_image( imgC, pts_C ) ) + cv2.imshow( 'xcanvas', gms.plot_point_sets( imgC, ptC, imgP, ptP ) ) + cv2.waitKey(0) + + + + +def demo_match2(): + print 'Test simple gmsrobe.match2()' + imgP = cv2.imread("../data/kf_972.png") #curr + imgC = cv2.imread("../data/kf_973.png") #prev + + gms = GmsRobe() + + startT = time.time() + ptC, ptP = gms.match2( imgC, imgP ) + print 'Elapsed total (ms): %4.2f' %(1000.0 * (time.time() - startT ) ) + + + xcanvas = gms.plot_point_sets( imgC, ptC, imgP, ptP, enable_lines=False ) + xcanvas2 = gms.plot_point_sets( imgC, ptC, imgP, ptP, show_random_points=60 ) + + # r = np.random.randint( 0, ptC.shape[1], 50 ) + # xcanvas = gms.plot_point_sets( imgC, ptC[:,r], imgCm, ptCm[:,r] ) + cv2.imshow( 'All matches', xcanvas ) + cv2.imshow( 'Only show 60 random matches', xcanvas2 ) + cv2.waitKey(0) + + + +def demo_gms_original_implementation(): + print 'Test for Original GMS Implementation, ie. class GmsMatcher' + # img1 = cv2.imread("../data/nn_left.jpg") + # img2 = cv2.imread("../data/nn_right.jpg") + + # img1 = imresize(img1, 240) + # img2 = imresize(img2, 240) + + img1 = cv2.imread("../data/kf_2581.png") + img2 = cv2.imread("../data/kf_2583.png") + + + gms = GmsMatcher(verbosity=1) + startT = time.time() + matches = gms.compute_matches(img1, img2) + print 'gms.compute_matches took (ms): %4.2f' %(1000.*(time.time() - startT ) ) + gms.draw_matches(img1, img2, DrawingType.POINTS_AND_TEXT) + +if __name__ == '__main__': + # Try any of the 4 demos. + which_demo = raw_input('a:demo_match3\nb:demo_match2_guided\nc:demo_match2\nd:demo_gms_original_implementation\nWhich demo do you want to try? ') + print( '%s' %(which_demo)) + + if which_demo == 'a': + demo_match3() + if which_demo == 'b': + demo_match2_guided() + if which_demo == 'c': + demo_match2() + + if which_demo == 'd': + demo_gms_original_implementation() diff --git a/python/gms_matcher.py b/python/gms_matcher.py index c6cc9f0..5d09e63 100644 --- a/python/gms_matcher.py +++ b/python/gms_matcher.py @@ -1,8 +1,29 @@ +## Adopted from : https://github.com/JiawangBian/GMS-Feature-Matcher +## Acknowledment to original Authors +# @inproceedings{bian2017gms, +# title={GMS: Grid-based Motion Statistics for Fast, Ultra-robust Feature Correspondence}, +# author={JiaWang Bian and Wen-Yan Lin and Yasuyuki Matsushita and Sai-Kit Yeung and Tan Dat Nguyen and Ming-Ming Cheng}, +# booktitle={IEEE Conference on Computer Vision and Pattern Recognition}, +# year={2017} +# } +# +# +# Added class GmsRobe by mpkuse +# This class can be used to expand macthes, and to find 3way matches. the +# core matcher is based on GMS-Feature-Matcher by Jiawang Bian. +# Date: 1st Nov, 2017 + + + import math from enum import Enum import cv2 import numpy as np +cv2.ocl.setUseOpenCL(False) + +import time +import code THRESHOLD_FACTOR = 6 @@ -43,10 +64,37 @@ class DrawingType(Enum): ONLY_LINES = 1 LINES_AND_POINTS = 2 + POINTS_AND_TEXT = 3 class GmsMatcher: - def __init__(self, descriptor, matcher): + def __init__(self, descriptor=None, matcher=None, n=10000, verbosity=0): + """ + descriptor : The Feature point descriptor. If it is None, we initialize a ORB + extractor with 10000 features. This can be altered with the argument n. + + matcher : The matcher. If None provided we initialize a brute force matcher. + + verbose : if 0 will not print out message. 5 will print all messages. + """ + self.verbosity = verbosity + + if descriptor is None: + if n<10: + descriptor = cv2.ORB_create(10000) + self._print_filt( 'Create ORB detector with n=10000', 1) + else: + descriptor = cv2.ORB_create(n) + self._print_filt( 'Create ORB detector with n=%d' %(n), 1 ) + descriptor.setFastThreshold(0) + + if matcher is None: + if cv2.__version__.startswith('3'): + matcher = cv2.BFMatcher(cv2.NORM_HAMMING) + else: + matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING) + + self.scale_ratios = [1.0, 1.0 / 2, 1.0 / math.sqrt(2.0), math.sqrt(2.0), 2.0] # Normalized vectors of 2D points self.normalized_points1 = [] @@ -89,6 +137,10 @@ def __init__(self, descriptor, matcher): self.keypoints_image1 = [] self.keypoints_image2 = [] + def _print_filt( self, msg, ch ): + if self.verbosity > ch: + print( msg ) + def empty_matches(self): self.normalized_points1 = [] self.normalized_points2 = [] @@ -96,23 +148,34 @@ def empty_matches(self): self.gms_matches = [] def compute_matches(self, img1, img2): - self.keypoints_image1, descriptors_image1 = self.descriptor.detectAndCompute(img1, np.array([])) - self.keypoints_image2, descriptors_image2 = self.descriptor.detectAndCompute(img2, np.array([])) + startKeypts = time.time() + self.keypoints_image1, descriptors_image1 = self.descriptor.detectAndCompute(img1, None )#np.array([])) + self.keypoints_image2, descriptors_image2 = self.descriptor.detectAndCompute(img2, None )#np.array([])) + self._print_filt( 'compute_matches(): detectAndCompute took (ms): %4.2f' %(1000.*(time.time() - startKeypts ) ), 1 ) + size1 = Size(img1.shape[1], img1.shape[0]) size2 = Size(img2.shape[1], img2.shape[0]) if self.gms_matches: self.empty_matches() - + + startMatcher = time.time() all_matches = self.matcher.match(descriptors_image1, descriptors_image2) + # code.interact( local=locals() ) + self._print_filt( 'compute_matches(): self.matcher.match took (ms): %4.2f' %(1000.*(time.time() - startMatcher ) ), 1 ) + self.normalize_points(self.keypoints_image1, size1, self.normalized_points1) self.normalize_points(self.keypoints_image2, size2, self.normalized_points2) + self.matches_number = len(all_matches) self.convert_matches(all_matches, self.matches) self.initialize_neighbours(self.grid_neighbor_left, self.grid_size_left) - mask, num_inliers = self.get_inlier_mask(False, False) - print('Found', num_inliers, 'matches') + startVote = time.time() + mask, num_inliers = self.get_inlier_mask(False, False) #This is the most expensive function call, which inturn calls run() + self._print_filt( '%s, %s, %s' %('Found', num_inliers, 'matches'), 1 ) + self._print_filt( 'compute_matches(): GMS Voting took (ms): %4.2f' %(1000.*(time.time() - startVote ) ), 1 ) + for i in range(len(mask)): if mask[i]: @@ -327,8 +390,18 @@ def draw_matches(self, src1, src2, drawing_type): cv2.circle(output, tuple(map(int, left)), 1, (0, 255, 255), 2) cv2.circle(output, tuple(map(int, right)), 1, (0, 255, 0), 2) + elif drawing_type == DrawingType.POINTS_AND_TEXT: + for i in range(len(self.gms_matches)): + left = self.keypoints_image1[self.gms_matches[i].queryIdx].pt + right = tuple(sum(x) for x in zip(self.keypoints_image2[self.gms_matches[i].trainIdx].pt, (src1.shape[1], 0))) + cv2.circle(output, tuple(map(int, left)), 1, (0, 255, 255)) + cv2.circle(output, tuple(map(int, right)), 1, (0, 255, 0) ) + cv2.putText(output, str(i), tuple(map(int, left)), cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + cv2.putText(output, str(i), tuple(map(int, right)), cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + cv2.imshow('show', output) cv2.waitKey() + return output class Size: @@ -337,27 +410,292 @@ def __init__(self, width, height): self.height = height -def imresize(src, height): - ratio = src.shape[0] * 1.0/height - width = int(src.shape[1] * 1.0/ratio) - return cv2.resize(src, (width, height)) + +class GmsRobe: + ## This class is suppose to extent the functonality of GmsMatcher. All of the + ## Following functions return 2xN matrix with (x,y) of matches. No image resizing + ## a) match2( imC, imP ) + ## b) match3( imC, imP, imCm ) + ## c) match2_guided( imC, pt1, imP ) + def __init__(self, descriptor=None, matcher=None, n=10000, verbosity=0): + """ + descriptor : The Feature point descriptor. If it is None, we initialize a ORB + extractor with 10000 features. This can be altered with the argument n. + + matcher : The matcher. If None provided we initialize a brute force matcher. + + verbose : if 0 will not print out message. 5 will print all messages. + """ + self.gms = GmsMatcher(descriptor, matcher, n, verbosity) + + + def match2( self, imC, imP ): + """ Given current image(imC) and prev image(imP) returns GMS matches co-ordinates (x,y)_i + Essentially this is a thin wrapper around the original GmsMatcher class. + + Typical usage with 240x320 images takes ~ 350 ms (Brute Force matcher) + """ + matches = self.gms.compute_matches(imC, imP) + return self._matches_to_cords( matches, self.gms.keypoints_image1, self.gms.keypoints_image2 ) + + def match2_guided( self, imC, pts_C, imP ): + """ Given current image (imC), with pts_C 2xN as input points on imC, the + objective is to find these points in previous image (imP) + + The way we do this is to first compute all the matches between imC and imP. + Then filter these matches to include only those pysically close to input points. + + Typical usage with 240x320 images and pts_C ~ 50 takes 354 ms overall. + Part-A: Takes 352ms (Brute Force matcher) + Part-B: Takes 2ms + """ + + timeA = time.time() + gmsC, gmsP = self.match2( imC, imP ) + # cv2.imshow( 'org GMS C--P', self.plot_point_sets( imC, gmsC, imP, gmsP ) ) + + print 'ElapsedA (ms): %4.2f' %(1000.0 * (time.time() - timeA )) + + + # Now go thru each of pts_C, find nn of each of pts_C in gmsC. Only retain + # the pair gmsC_i <--> gmsP_i if gmsC_i is within 1px of pts_C_i. + L = [] + timeB = time.time() + for i in range( pts_C.shape[1] ): + + # find nearest neighbour of pts_C_i in gmsC. TODO Consider using FLANN for this. + diff = gmsC - np.expand_dims( pts_C[:,i],1 ) #2xN + diff_norm = np.linalg.norm( diff, axis=0 ) #1xN + minval = diff_norm.min() + minarg = diff_norm.argmin() #This is essentially like 1-NN + + + if minval < 1.0 : + # print minval, minarg + L.append( minarg ) + + print 'ElapsedB (ms): %4.2f' %(1000.0 * (time.time() - timeB )) + + # L is list of index on gmsC <--> gmsP which are found in pts_C. + # code.interact(local=locals() ) + return gmsC[:,L], gmsP[:,L] + + + + def match3( self, imC, imP, imCm ): + """ To find 3way correspondences. Note: The order in which the images is given is critical + imC : current image + imP : previous image + imCm : current-1 image + + + Takes about 775 ms. + Part-A: 727 ms (Brute Force matcher) + Part-B: 47 ms + """ + + # Get C<-->P and C<-->Cm + startA = time.time() + _gmsC, _gmsP = self.match2( imC, imP ) #2xN, 2xN + __gmsC, __gmsCm = self.match2( imC, imCm ) # This will be a larger set than _gmsC <--> _gmsP. 2xN, 2xN + print 'gmsrobe.match3.A took (ms): %4.2f' %(1000.*(time.time() - startA ) ) + + + # print 'C<-- %d -->P' %(_gmsC.shape[1]) + # print 'C<-- %d -->Cm' %(__gmsC.shape[1]) + + # r = np.random.randint( 0, __gmsC.shape[1], 50 ) + # cv2.imshow( 'C--P', gms.plot_point_sets( imC, _gmsC, imP, _gmsP ) ) + # cv2.imshow( 'C--Cm', self.plot_point_sets( imC, __gmsC, imCm, __gmsCm ) ) + # # cv2.imshow( 'C', self.plot_points_on_image(imC, __gmsC[:,r]) ) + # # cv2.imshow( 'Cm', self.plot_points_on_image(imCm, __gmsCm[:,r]) ) + # cv2.waitKey(0) + + + + # loop thru _gmsC <--> _gmsCm. Find nearest cords of _gmsC_i in __gmsC. + startB = time.time() + L1 = [] + L2 = [] + for i in range( _gmsC.shape[1] ): + diff = __gmsC - np.expand_dims( _gmsC[:,i] ,1 ) #2xN + diff_norm = np.linalg.norm( diff, axis=0 ) #1xN + minval = diff_norm.min() + minarg = diff_norm.argmin() #This is essentially like 1-NN + + if minval < 1.0: + # print i, minarg + L1.append(i) + L2.append(minarg) + + print 'gmsrobe.match3.B took (ms): %4.2f' %(1000.*(time.time() - startB ) ) + return _gmsC[:,L1], _gmsP[:,L1], __gmsCm[:,L2] -if __name__ == '__main__': - img1 = cv2.imread("../data/nn_left.jpg") - img2 = cv2.imread("../data/nn_right.jpg") - img1 = imresize(img1, 480) - img2 = imresize(img2, 480) - orb = cv2.ORB_create(10000) - orb.setFastThreshold(0) - if cv2.__version__.startswith('3'): - matcher = cv2.BFMatcher(cv2.NORM_HAMMING) - else: - matcher = cv2.BFMatcher_create(cv2.NORM_HAMMING) - gms = GmsMatcher(orb, matcher) - matches = gms.compute_matches(img1, img2) - gms.draw_matches(img1, img2, DrawingType.ONLY_LINES) + ################################################## + ################# Utilities ##################### + ################################################## + def _matches_to_cords( self, matches, kp1, kp2 ): + """ Given the DMatches array and keypoints, returns list of cords""" + N = len(matches) + cords_1 = np.zeros( (2,N) ) + cords_2 = np.zeros( (2,N) ) + for i in range(N): + left = kp1[matches[i].queryIdx].pt + right = kp2[matches[i].trainIdx].pt + + cords_1[0,i] = left[0] + cords_1[1,i] = left[1] + + cords_2[0,i] = right[0] + cords_2[1,i] = right[1] + + # print '---',i + # print left + # print right + + return cords_1, cords_2 + + + + ################################################## + ########## Plotting / Visualization ############# + ################################################## + + + def plot_3way_match( self, curr_im, pts_curr, prev_im, pts_prev, curr_m_im, pts_curr_m, enable_text=True, enable_lines=False, show_random_points=-1 ): + """ pts_curr, pts_prev, pts_curr_m : 2xN numpy matrix + + :flags: + enable_text: Enable/disable printing point index in image + enable_lines: Enable/disable lines drawing + show_random_points: Only show this many number of random matches. Mainly to avoid clutter. + + + returns : # grid : [ [curr, prev], [curr-1 X ] ] + """ + + print 'pts_curr.shape ', pts_curr.shape + print 'pts_prev.shape ', pts_prev.shape + print 'pts_curr_m.shape ', pts_curr_m.shape + assert( (pts_curr.shape[1] == pts_prev.shape[1]) and (pts_curr_m.shape[1] == pts_prev.shape[1]) ) + assert( (pts_curr.shape[0] == pts_prev.shape[0]) and (pts_curr_m.shape[0] == pts_prev.shape[0]) ) + assert( pts_curr.shape[0] == 2 ) + # all 3 should have same number of points + + + zero_image = np.zeros( curr_im.shape, dtype='uint8' ) + cv2.putText( zero_image, str(pts_curr.shape[1]), (10,200), cv2.FONT_HERSHEY_SIMPLEX, 2, 255 ) + + + r1 = np.concatenate( ( curr_im, prev_im ), axis=1 ) + r2 = np.concatenate( ( curr_m_im, zero_image ), axis=1 ) + gridd = np.concatenate( (r1,r2), axis=0 ) + + N = pts_curr.shape[1] + if show_random_points < 0 or show_random_points > N: + spann = range(N) + else: + spann = np.random.randint( 0, N, show_random_points ) + + for xi in spann: + point_c = tuple( np.int0(pts_curr[:,xi]) ) + point_p = tuple( np.int0(pts_prev[:,xi]) ) + point_cm = tuple( np.int0(pts_curr_m[:,xi]) ) + + + point_p__ = tuple( np.int0(pts_prev[:,xi]) + [curr_im.shape[1],0] ) + point_cm__ = tuple( np.int0(pts_curr_m[:,xi]) + [0,curr_im.shape[0]] ) + + + ######## C --- P + cv2.circle( gridd, point_c, 4, (0,255,0) ) + cv2.circle( gridd, point_p__, 4, (0,255,0) ) + if enable_lines: + cv2.line( gridd, point_c, point_p__, (255,0,0) ) + if enable_text: + cv2.putText(gridd, str(xi), point_c, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + cv2.putText(gridd, str(xi), point_p__, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + + + ####### C --- Cm + cv2.circle( gridd, point_cm__, 4, (0,255,0) ) + if enable_lines: + cv2.line( gridd, point_c, point_cm__, (255,30,255) ) + if enable_text: + cv2.putText(gridd, str(xi), point_cm__, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + + + + + return gridd + + + def plot_point_sets( self, im1, pt1, im2, pt2, mask=None, enable_text=True, enable_lines=False, show_random_points=-1 ): + """ pt1, pt2 : 2xN array + + :flags: + enable_text: Enable/disable printing point index in image + enable_lines: Enable/disable lines drawing + show_random_points: Only show this many number of random matches. Mainly to avoid clutter. + + """ + assert( pt1.shape[0] == 2 ) + assert( pt2.shape[0] == 2 ) + assert( pt1.shape[1] == pt2.shape[1] ) + + #TODO: if im1 and im2 are 2 channel, this might cause issues + + xcanvas = np.concatenate( (im1, im2), axis=1 ) + N = pt1.shape[1] + if show_random_points < 0 or show_random_points > N: + spann = range(N) + else: + spann = np.random.randint( 0, N, show_random_points ) + + for xi in spann: + if (mask is not None) and (mask[xi,0] == 0): + continue + + point_left = tuple( np.int0(pt1[:,xi]) ) + point_right = tuple(np.int0(pt2[:,xi]) ) + point_right__ = tuple(np.int0(pt2[:,xi]) + [im1.shape[1],0]) + cv2.circle( xcanvas, point_left, 4, (255,0,255) ) + cv2.circle( xcanvas, point_right__, 4, (255,0,255) ) + + if enable_lines: + cv2.line( xcanvas, point_left, point_right__, (255,0,0) ) + + if enable_text: + cv2.putText(xcanvas, str(xi), point_left, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + cv2.putText(xcanvas, str(xi), point_right__, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + + + return xcanvas + + def plot_points_on_image( self, img, pts, enable_text=True ): + """ img : Image; pts: 2xN array """ + + xcanvas = img.copy() + for xi in range( pts.shape[1] ): + point_ = tuple( np.int0(pts[:,xi]) ) + cv2.circle( xcanvas, point_, 4, (255,0,255) ) + + if enable_text: + cv2.putText(xcanvas, str(xi), point_, cv2.FONT_HERSHEY_COMPLEX_SMALL, 0.4, (255,0,0) ) + + return xcanvas + + + + + + +def imresize(src, height): + ratio = src.shape[0] * 1.0/height + width = int(src.shape[1] * 1.0/ratio) + return cv2.resize(src, (width, height)) diff --git a/src/demo.cpp b/src/demo.cpp index 313fb2a..83abb2b 100644 --- a/src/demo.cpp +++ b/src/demo.cpp @@ -1,6 +1,6 @@ // GridMatch.cpp : Defines the entry point for the console application. -//#define USE_GPU +//#define USE_GPU #include "Header.h" #include "gms_matcher.h" @@ -8,8 +8,11 @@ void GmsMatch(Mat &img1, Mat &img2); void runImagePair(){ - Mat img1 = imread("../data/nn_left.jpg"); - Mat img2 = imread("../data/nn_right.jpg"); + // Mat img1 = imread("../data/nn_left.jpg"); + // Mat img2 = imread("../data/nn_right.jpg"); + + Mat img1 = imread("../data/kf_1140.png"); + Mat img2 = imread("../data/kf_2583.png"); imresize(img1, 480); imresize(img2, 480); @@ -32,12 +35,15 @@ int main() void GmsMatch(Mat &img1, Mat &img2){ + vector kp1, kp2; Mat d1, d2; vector matches_all, matches_gms; Ptr orb = ORB::create(10000); + orb->setFastThreshold(0); + orb->detectAndCompute(img1, Mat(), kp1, d1); orb->detectAndCompute(img2, Mat(), kp2, d2); @@ -49,6 +55,7 @@ void GmsMatch(Mat &img1, Mat &img2){ BFMatcher matcher(NORM_HAMMING); matcher.match(d1, d2, matches_all); #endif + cout << "BFMatcher : " << t1.elapsed() << endl; // GMS filter int num_inliers = 0; @@ -67,9 +74,12 @@ void GmsMatch(Mat &img1, Mat &img2){ } } - Mat show = DrawInlier(img1, img2, kp1, kp2, matches_gms, 1); + cout << "Elapsed time c++: (sec)" << t.elapsed() << endl; + + + Mat show = DrawInlier(img1, img2, kp1, kp2, matches_gms, 3); imshow("show", show); waitKey(); + cout << "Write file : output.png\n"; + imwrite( "output.png", show ); } - -