Details | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
3 | PointedEar | 1 | /*------------------------------------------------------------------------------------------*\ |
2 | This file contains material supporting chapter 9 of the cookbook: |
||
3 | Computer Vision Programming using the OpenCV Library. |
||
4 | by Robert Laganiere, Packt Publishing, 2011. |
||
5 | |||
6 | This program is free software; permission is hereby granted to use, copy, modify, |
||
7 | and distribute this source code, or portions thereof, for any purpose, without fee, |
||
8 | subject to the restriction that the copyright notice may not be removed |
||
9 | or altered from any source or altered source distribution. |
||
10 | The software is released on an as-is basis and without any warranties of any kind. |
||
11 | In particular, the software is not guaranteed to be fault-tolerant or free from failure. |
||
12 | The author disclaims all warranties with regard to this software, any use, |
||
13 | and any consequent failure, is purely the responsibility of the user. |
||
14 | |||
15 | Copyright (C) 2010-2011 Robert Laganiere, www.laganiere.name |
||
16 | \*------------------------------------------------------------------------------------------*/ |
||
17 | |||
18 | #if !defined MATCHER |
||
19 | #define MATCHER |
||
20 | |||
21 | #include <vector> |
||
22 | #include <opencv2/core/core.hpp> |
||
23 | #include <opencv2/imgproc/imgproc.hpp> |
||
24 | #include <opencv2/highgui/highgui.hpp> |
||
25 | #include <opencv2/features2d/features2d.hpp> |
||
26 | |||
27 | class RobustMatcher { |
||
28 | |||
29 | private: |
||
30 | |||
31 | // pointer to the feature point detector object |
||
32 | cv::Ptr<cv::FeatureDetector> detector; |
||
33 | // pointer to the feature descriptor extractor object |
||
34 | cv::Ptr<cv::DescriptorExtractor> extractor; |
||
35 | float ratio; // max ratio between 1st and 2nd NN |
||
36 | bool refineF; // if true will refine the F matrix |
||
37 | double distance; // min distance to epipolar |
||
38 | double confidence; // confidence level (probability) |
||
39 | |||
40 | public: |
||
41 | |||
42 | RobustMatcher() : ratio(0.65f), refineF(true), confidence(0.99), distance(3.0) { |
||
43 | |||
44 | // SURF is the default feature |
||
45 | detector= new cv::SurfFeatureDetector(); |
||
46 | extractor= new cv::SurfDescriptorExtractor(); |
||
47 | } |
||
48 | |||
49 | // Set the feature detector |
||
50 | void setFeatureDetector(cv::Ptr<cv::FeatureDetector>& detect) { |
||
51 | |||
52 | detector= detect; |
||
53 | } |
||
54 | |||
55 | // Set descriptor extractor |
||
56 | void setDescriptorExtractor(cv::Ptr<cv::DescriptorExtractor>& desc) { |
||
57 | |||
58 | extractor= desc; |
||
59 | } |
||
60 | |||
61 | // Set the minimum distance to epipolar in RANSAC |
||
62 | void setMinDistanceToEpipolar(double d) { |
||
63 | |||
64 | distance= d; |
||
65 | } |
||
66 | |||
67 | // Set confidence level in RANSAC |
||
68 | void setConfidenceLevel(double c) { |
||
69 | |||
70 | confidence= c; |
||
71 | } |
||
72 | |||
73 | // Set the NN ratio |
||
74 | void setRatio(float r) { |
||
75 | |||
76 | ratio= r; |
||
77 | } |
||
78 | |||
79 | // if you want the F matrix to be recalculated |
||
80 | void refineFundamental(bool flag) { |
||
81 | |||
82 | refineF= flag; |
||
83 | } |
||
84 | |||
85 | // Clear matches for which NN ratio is > than threshold |
||
86 | // return the number of removed points |
||
87 | // (corresponding entries being cleared, i.e. size will be 0) |
||
88 | int ratioTest(std::vector<std::vector<cv::DMatch>>& matches) { |
||
89 | |||
90 | int removed=0; |
||
91 | |||
92 | // for all matches |
||
93 | for (std::vector<std::vector<cv::DMatch>>::iterator matchIterator= matches.begin(); |
||
94 | matchIterator!= matches.end(); ++matchIterator) { |
||
95 | |||
96 | // if 2 NN has been identified |
||
97 | if (matchIterator->size() > 1) { |
||
98 | |||
99 | // check distance ratio |
||
100 | if ((*matchIterator)[0].distance/(*matchIterator)[1].distance > ratio) { |
||
101 | |||
102 | matchIterator->clear(); // remove match |
||
103 | removed++; |
||
104 | } |
||
105 | |||
106 | } else { // does not have 2 neighbours |
||
107 | |||
108 | matchIterator->clear(); // remove match |
||
109 | removed++; |
||
110 | } |
||
111 | } |
||
112 | |||
113 | return removed; |
||
114 | } |
||
115 | |||
116 | // Insert symmetrical matches in symMatches vector |
||
117 | void symmetryTest(const std::vector<std::vector<cv::DMatch>>& matches1, |
||
118 | const std::vector<std::vector<cv::DMatch>>& matches2, |
||
119 | std::vector<cv::DMatch>& symMatches) { |
||
120 | |||
121 | // for all matches image 1 -> image 2 |
||
122 | for (std::vector<std::vector<cv::DMatch>>::const_iterator matchIterator1= matches1.begin(); |
||
123 | matchIterator1!= matches1.end(); ++matchIterator1) { |
||
124 | |||
125 | if (matchIterator1->size() < 2) // ignore deleted matches |
||
126 | continue; |
||
127 | |||
128 | // for all matches image 2 -> image 1 |
||
129 | for (std::vector<std::vector<cv::DMatch>>::const_iterator matchIterator2= matches2.begin(); |
||
130 | matchIterator2!= matches2.end(); ++matchIterator2) { |
||
131 | |||
132 | if (matchIterator2->size() < 2) // ignore deleted matches |
||
133 | continue; |
||
134 | |||
135 | // Match symmetry test |
||
136 | if ((*matchIterator1)[0].queryIdx == (*matchIterator2)[0].trainIdx && |
||
137 | (*matchIterator2)[0].queryIdx == (*matchIterator1)[0].trainIdx) { |
||
138 | |||
139 | // add symmetrical match |
||
140 | symMatches.push_back(cv::DMatch((*matchIterator1)[0].queryIdx, |
||
141 | (*matchIterator1)[0].trainIdx, |
||
142 | (*matchIterator1)[0].distance)); |
||
143 | break; // next match in image 1 -> image 2 |
||
144 | } |
||
145 | } |
||
146 | } |
||
147 | } |
||
148 | |||
149 | // Identify good matches using RANSAC |
||
150 | // Return fundemental matrix |
||
151 | cv::Mat ransacTest(const std::vector<cv::DMatch>& matches, |
||
152 | const std::vector<cv::KeyPoint>& keypoints1, |
||
153 | const std::vector<cv::KeyPoint>& keypoints2, |
||
154 | std::vector<cv::DMatch>& outMatches) { |
||
155 | |||
156 | // Convert keypoints into Point2f |
||
157 | std::vector<cv::Point2f> points1, points2; |
||
158 | for (std::vector<cv::DMatch>::const_iterator it= matches.begin(); |
||
159 | it!= matches.end(); ++it) { |
||
160 | |||
161 | // Get the position of left keypoints |
||
162 | float x= keypoints1[it->queryIdx].pt.x; |
||
163 | float y= keypoints1[it->queryIdx].pt.y; |
||
164 | points1.push_back(cv::Point2f(x,y)); |
||
165 | // Get the position of right keypoints |
||
166 | x= keypoints2[it->trainIdx].pt.x; |
||
167 | y= keypoints2[it->trainIdx].pt.y; |
||
168 | points2.push_back(cv::Point2f(x,y)); |
||
169 | } |
||
170 | |||
171 | // Compute F matrix using RANSAC |
||
172 | std::vector<uchar> inliers(points1.size(),0); |
||
173 | cv::Mat fundemental= cv::findFundamentalMat( |
||
174 | cv::Mat(points1),cv::Mat(points2), // matching points |
||
175 | inliers, // match status (inlier ou outlier) |
||
176 | CV_FM_RANSAC, // RANSAC method |
||
177 | distance, // distance to epipolar line |
||
178 | confidence); // confidence probability |
||
179 | |||
180 | // extract the surviving (inliers) matches |
||
181 | std::vector<uchar>::const_iterator itIn= inliers.begin(); |
||
182 | std::vector<cv::DMatch>::const_iterator itM= matches.begin(); |
||
183 | // for all matches |
||
184 | for ( ;itIn!= inliers.end(); ++itIn, ++itM) { |
||
185 | |||
186 | if (*itIn) { // it is a valid match |
||
187 | |||
188 | outMatches.push_back(*itM); |
||
189 | } |
||
190 | } |
||
191 | |||
192 | std::cout << "Number of matched points (after cleaning): " << outMatches.size() << std::endl; |
||
193 | |||
194 | if (refineF) { |
||
195 | // The F matrix will be recomputed with all accepted matches |
||
196 | |||
197 | // Convert keypoints into Point2f for final F computation |
||
198 | points1.clear(); |
||
199 | points2.clear(); |
||
200 | |||
201 | for (std::vector<cv::DMatch>::const_iterator it= outMatches.begin(); |
||
202 | it!= outMatches.end(); ++it) { |
||
203 | |||
204 | // Get the position of left keypoints |
||
205 | float x= keypoints1[it->queryIdx].pt.x; |
||
206 | float y= keypoints1[it->queryIdx].pt.y; |
||
207 | points1.push_back(cv::Point2f(x,y)); |
||
208 | // Get the position of right keypoints |
||
209 | x= keypoints2[it->trainIdx].pt.x; |
||
210 | y= keypoints2[it->trainIdx].pt.y; |
||
211 | points2.push_back(cv::Point2f(x,y)); |
||
212 | } |
||
213 | |||
214 | // Compute 8-point F from all accepted matches |
||
215 | fundemental= cv::findFundamentalMat( |
||
216 | cv::Mat(points1),cv::Mat(points2), // matching points |
||
217 | CV_FM_8POINT); // 8-point method |
||
218 | } |
||
219 | |||
220 | return fundemental; |
||
221 | } |
||
222 | |||
223 | // Match feature points using symmetry test and RANSAC |
||
224 | // returns fundemental matrix |
||
225 | cv::Mat match(cv::Mat& image1, cv::Mat& image2, // input images |
||
226 | std::vector<cv::DMatch>& matches, // output matches and keypoints |
||
227 | std::vector<cv::KeyPoint>& keypoints1, std::vector<cv::KeyPoint>& keypoints2) { |
||
228 | |||
229 | // 1a. Detection of the SURF features |
||
230 | detector->detect(image1,keypoints1); |
||
231 | detector->detect(image2,keypoints2); |
||
232 | |||
233 | std::cout << "Number of SURF points (1): " << keypoints1.size() << std::endl; |
||
234 | std::cout << "Number of SURF points (2): " << keypoints2.size() << std::endl; |
||
235 | |||
236 | // 1b. Extraction of the SURF descriptors |
||
237 | cv::Mat descriptors1, descriptors2; |
||
238 | extractor->compute(image1,keypoints1,descriptors1); |
||
239 | extractor->compute(image2,keypoints2,descriptors2); |
||
240 | |||
241 | std::cout << "descriptor matrix size: " << descriptors1.rows << " by " << descriptors1.cols << std::endl; |
||
242 | |||
243 | // 2. Match the two image descriptors |
||
244 | |||
245 | // Construction of the matcher |
||
246 | cv::BruteForceMatcher<cv::L2<float>> matcher; |
||
247 | |||
248 | // from image 1 to image 2 |
||
249 | // based on k nearest neighbours (with k=2) |
||
250 | std::vector<std::vector<cv::DMatch>> matches1; |
||
251 | matcher.knnMatch(descriptors1,descriptors2, |
||
252 | matches1, // vector of matches (up to 2 per entry) |
||
253 | 2); // return 2 nearest neighbours |
||
254 | |||
255 | // from image 2 to image 1 |
||
256 | // based on k nearest neighbours (with k=2) |
||
257 | std::vector<std::vector<cv::DMatch>> matches2; |
||
258 | matcher.knnMatch(descriptors2,descriptors1, |
||
259 | matches2, // vector of matches (up to 2 per entry) |
||
260 | 2); // return 2 nearest neighbours |
||
261 | |||
262 | std::cout << "Number of matched points 1->2: " << matches1.size() << std::endl; |
||
263 | std::cout << "Number of matched points 2->1: " << matches2.size() << std::endl; |
||
264 | |||
265 | // 3. Remove matches for which NN ratio is > than threshold |
||
266 | |||
267 | // clean image 1 -> image 2 matches |
||
268 | int removed= ratioTest(matches1); |
||
269 | std::cout << "Number of matched points 1->2 (ratio test) : " << matches1.size()-removed << std::endl; |
||
270 | // clean image 2 -> image 1 matches |
||
271 | removed= ratioTest(matches2); |
||
272 | std::cout << "Number of matched points 1->2 (ratio test) : " << matches2.size()-removed << std::endl; |
||
273 | |||
274 | // 4. Remove non-symmetrical matches |
||
275 | std::vector<cv::DMatch> symMatches; |
||
276 | symmetryTest(matches1,matches2,symMatches); |
||
277 | |||
278 | std::cout << "Number of matched points (symmetry test): " << symMatches.size() << std::endl; |
||
279 | |||
280 | // 5. Validate matches using RANSAC |
||
281 | cv::Mat fundemental= ransacTest(symMatches, keypoints1, keypoints2, matches); |
||
282 | |||
283 | // return the found fundemental matrix |
||
284 | return fundemental; |
||
285 | } |
||
286 | }; |
||
287 | |||
288 | #endif |