View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.estimators;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.numerical.robust.RANSACRobustEstimator;
23  import com.irurueta.numerical.robust.RANSACRobustEstimatorListener;
24  import com.irurueta.numerical.robust.RobustEstimator;
25  import com.irurueta.numerical.robust.RobustEstimatorException;
26  import com.irurueta.numerical.robust.RobustEstimatorMethod;
27  
28  import java.util.ArrayList;
29  import java.util.List;
30  
31  /**
32   * Finds the best pinhole camera for provided collections of matched 2D/3D
33   * points using RANSAC + UPnP algorithms.
34   */
35  public class RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator extends
36          UPnPPointCorrespondencePinholeCameraRobustEstimator {
37  
38      /**
39       * Constant defining default threshold to determine whether points are
40       * inliers or not.
41       * By default, 1.0 is considered a good value for cases where measures are
42       * done on pixels, since typically the minimum resolution is 1 pixel.
43       */
44      public static final double DEFAULT_THRESHOLD = 1.0;
45  
46      /**
47       * Minimum value that can be set as threshold.
48       * Threshold must be strictly greater than 0.0.
49       */
50      public static final double MIN_THRESHOLD = 0.0;
51  
52      /**
53       * Indicates that by default inliers will only be computed but not kept.
54       */
55      public static final boolean DEFAULT_COMPUTE_AND_KEEP_INLIERS = false;
56  
57      /**
58       * Indicates that by default residuals will only be computed but not kept.
59       */
60      public static final boolean DEFAULT_COMPUTE_AND_KEEP_RESIDUALS = false;
61  
62      /**
63       * Threshold to determine whether points are inliers or not when testing
64       * possible estimation solutions.
65       * The threshold refers to the amount of error (i.e. distance) a possible
66       * solution has on a matched pair of points.
67       */
68      private double threshold;
69  
70      /**
71       * Indicates whether inliers must be computed and kept.
72       */
73      private boolean computeAndKeepInliers;
74  
75      /**
76       * Indicates whether residuals must be computed and kept.
77       */
78      private boolean computeAndKeepResiduals;
79  
80      /**
81       * Constructor.
82       */
83      public RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator() {
84          super();
85          threshold = DEFAULT_THRESHOLD;
86          computeAndKeepInliers = DEFAULT_COMPUTE_AND_KEEP_INLIERS;
87          computeAndKeepResiduals = DEFAULT_COMPUTE_AND_KEEP_RESIDUALS;
88      }
89  
90      /**
91       * Constructor with listener.
92       *
93       * @param listener listener to be notified of events such as when estimation
94       *                 starts, ends or its progress significantly changes.
95       */
96      public RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
97              final PinholeCameraRobustEstimatorListener listener) {
98          super(listener);
99          threshold = DEFAULT_THRESHOLD;
100         computeAndKeepInliers = DEFAULT_COMPUTE_AND_KEEP_INLIERS;
101         computeAndKeepResiduals = DEFAULT_COMPUTE_AND_KEEP_RESIDUALS;
102     }
103 
104     /**
105      * Constructor with lists of points to be used to estimate a pinhole camera.
106      * Points in the lists located at the same position are considered to be
107      * matched. Hence, both lists must have the same size, and their size must
108      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES (6 points).
109      *
110      * @param points3D list of 3D points used to estimate a pinhole camera.
111      * @param points2D list of corresponding projected 2D points used to
112      *                 estimate a pinhole camera.
113      * @throws IllegalArgumentException if provided lists of points don't have
114      *                                  the same size or their size is smaller than required minimum size (6
115      *                                  correspondences).
116      */
117     public RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
118             final List<Point3D> points3D, final List<Point2D> points2D) {
119         super(points3D, points2D);
120         threshold = DEFAULT_THRESHOLD;
121         computeAndKeepInliers = DEFAULT_COMPUTE_AND_KEEP_INLIERS;
122         computeAndKeepResiduals = DEFAULT_COMPUTE_AND_KEEP_RESIDUALS;
123     }
124 
125     /**
126      * Constructor with listener and lists of points to be used to estimate a
127      * pinhole camera.
128      * Points in the lists located at the same position are considered to be
129      * matched. Hence, both lists must have the same size, and their size must
130      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES (6 points).
131      *
132      * @param listener listener to be notified of events such as when estimation
133      *                 starts, ends or its progress significantly changes.
134      * @param points3D list of 3D points used to estimate a pinhole camera.
135      * @param points2D list of corresponding projected 2D points used to
136      *                 estimate a pinhole camera.
137      * @throws IllegalArgumentException if provided lists of points don't have
138      *                                  the same size or their size is smaller than required minimum size (6
139      *                                  correspondences).
140      */
141     public RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
142             final PinholeCameraRobustEstimatorListener listener,
143             final List<Point3D> points3D, final List<Point2D> points2D) {
144         super(listener, points3D, points2D);
145         threshold = DEFAULT_THRESHOLD;
146         computeAndKeepInliers = DEFAULT_COMPUTE_AND_KEEP_INLIERS;
147         computeAndKeepResiduals = DEFAULT_COMPUTE_AND_KEEP_RESIDUALS;
148     }
149 
150     /**
151      * Returns threshold to determine whether points are inliers or not when
152      * testing possible estimation solutions.
153      * The threshold refers to the amount of error (i.e. Euclidean distance) a
154      * possible solution has on projected 2D points.
155      *
156      * @return threshold to determine whether points are inliers or not when
157      * testing possible estimation solutions.
158      */
159     public double getThreshold() {
160         return threshold;
161     }
162 
163     /**
164      * Sets threshold to determine whether points are inliers or not when
165      * testing possible estimation solutions.
166      * The threshold refers to the amount of error (i.e. Euclidean distance) a
167      * possible solution has on projected 2D points.
168      *
169      * @param threshold threshold to be set.
170      * @throws IllegalArgumentException if provided value is equal or less than
171      *                                  zero.
172      * @throws LockedException          if robust estimator is locked because an
173      *                                  estimation is already in progress.
174      */
175     public void setThreshold(final double threshold) throws LockedException {
176         if (isLocked()) {
177             throw new LockedException();
178         }
179         if (threshold <= MIN_THRESHOLD) {
180             throw new IllegalArgumentException();
181         }
182         this.threshold = threshold;
183     }
184 
185     /**
186      * Indicates whether inliers must be computed and kept.
187      *
188      * @return true if inliers must be computed and kept, false if inliers
189      * only need to be computed but not kept.
190      */
191     public boolean isComputeAndKeepInliersEnabled() {
192         return computeAndKeepInliers;
193     }
194 
195     /**
196      * Specifies whether inliers must be computed and kept.
197      *
198      * @param computeAndKeepInliers true if inliers must be computed and kept,
199      *                              false if inliers only need to be computed but not kept.
200      * @throws LockedException if estimator is locked.
201      */
202     public void setComputeAndKeepInliersEnabled(final boolean computeAndKeepInliers) throws LockedException {
203         if (isLocked()) {
204             throw new LockedException();
205         }
206         this.computeAndKeepInliers = computeAndKeepInliers;
207     }
208 
209     /**
210      * Indicates whether residuals must be computed and kept.
211      *
212      * @return true if residuals must be computed and kept, false if residuals
213      * only need to be computed but not kept.
214      */
215     public boolean isComputeAndKeepResidualsEnabled() {
216         return computeAndKeepResiduals;
217     }
218 
219     /**
220      * Specifies whether residuals must be computed and kept.
221      *
222      * @param computeAndKeepResiduals true if residuals must be computed and
223      *                                kept, false if residuals only need to be computed but not kept.
224      * @throws LockedException if estimator is locked.
225      */
226     public void setComputeAndKeepResidualsEnabled(final boolean computeAndKeepResiduals) throws LockedException {
227         if (isLocked()) {
228             throw new LockedException();
229         }
230         this.computeAndKeepResiduals = computeAndKeepResiduals;
231     }
232 
233     /**
234      * Estimates a pinhole camera using a robust estimator and
235      * the best set of matched 2D/3D point correspondences or 2D line/3D plane
236      * correspondences found using the robust estimator.
237      *
238      * @return a pinhole camera.
239      * @throws LockedException          if robust estimator is locked because an
240      *                                  estimation is already in progress.
241      * @throws NotReadyException        if provided input data is not enough to start
242      *                                  the estimation.
243      * @throws RobustEstimatorException if estimation fails for any reason
244      *                                  (i.e. numerical instability, no solution available, etc).
245      */
246     @SuppressWarnings("DuplicatedCode")
247     @Override
248     public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
249         if (isLocked()) {
250             throw new LockedException();
251         }
252         if (!isReady()) {
253             throw new NotReadyException();
254         }
255 
256         // pinhole camera estimator using UPnP (Uncalibrated Perspective-n-Point)
257         // algorithm
258         final var nonRobustEstimator = new UPnPPointCorrespondencePinholeCameraEstimator();
259 
260         nonRobustEstimator.setPlanarConfigurationAllowed(planarConfigurationAllowed);
261         nonRobustEstimator.setNullspaceDimension2Allowed(nullspaceDimension2Allowed);
262         nonRobustEstimator.setPlanarThreshold(planarThreshold);
263         nonRobustEstimator.setSkewness(skewness);
264         nonRobustEstimator.setHorizontalPrincipalPoint(horizontalPrincipalPoint);
265         nonRobustEstimator.setVerticalPrincipalPoint(verticalPrincipalPoint);
266 
267         // suggestions
268         nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
269         nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
270         nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
271         nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
272         nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
273         nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
274         nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
275         nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
276         nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
277         nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
278         nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
279         nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
280         nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
281         nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
282 
283         final var innerEstimator = new RANSACRobustEstimator<>(new RANSACRobustEstimatorListener<PinholeCamera>() {
284 
285             // point to be reused when computing residuals
286             private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
287 
288             // 3D points for a subset of samples
289             private final List<Point3D> subset3D = new ArrayList<>();
290 
291             // 2D points for a subset of samples
292             private final List<Point2D> subset2D = new ArrayList<>();
293 
294             @Override
295             public double getThreshold() {
296                 return threshold;
297             }
298 
299             @Override
300             public int getTotalSamples() {
301                 return points3D.size();
302             }
303 
304             @Override
305             public int getSubsetSize() {
306                 return PointCorrespondencePinholeCameraEstimator.MIN_NUMBER_OF_POINT_CORRESPONDENCES;
307             }
308 
309             @Override
310             public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
311                 subset3D.clear();
312                 subset3D.add(points3D.get(samplesIndices[0]));
313                 subset3D.add(points3D.get(samplesIndices[1]));
314                 subset3D.add(points3D.get(samplesIndices[2]));
315                 subset3D.add(points3D.get(samplesIndices[3]));
316                 subset3D.add(points3D.get(samplesIndices[4]));
317                 subset3D.add(points3D.get(samplesIndices[5]));
318 
319                 subset2D.clear();
320                 subset2D.add(points2D.get(samplesIndices[0]));
321                 subset2D.add(points2D.get(samplesIndices[1]));
322                 subset2D.add(points2D.get(samplesIndices[2]));
323                 subset2D.add(points2D.get(samplesIndices[3]));
324                 subset2D.add(points2D.get(samplesIndices[4]));
325                 subset2D.add(points2D.get(samplesIndices[5]));
326 
327                 try {
328                     nonRobustEstimator.setLists(subset3D, subset2D);
329 
330                     final var cam = nonRobustEstimator.estimate();
331                     solutions.add(cam);
332                 } catch (final Exception e) {
333                     // if points configuration is degenerate, no solution is
334                     // added
335                 }
336             }
337 
338             @Override
339             public double computeResidual(final PinholeCamera currentEstimation, final int i) {
340                 // pick i-th points
341                 final var point3D = points3D.get(i);
342                 final var point2D = points2D.get(i);
343 
344                 // project point3D into test point
345                 currentEstimation.project(point3D, testPoint);
346 
347                 // compare test point and 2D point
348                 return testPoint.distanceTo(point2D);
349             }
350 
351             @Override
352             public boolean isReady() {
353                 return RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this.isReady();
354             }
355 
356             @Override
357             public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
358                 if (listener != null) {
359                     listener.onEstimateStart(RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this);
360                 }
361             }
362 
363             @Override
364             public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
365                 if (listener != null) {
366                     listener.onEstimateEnd(RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this);
367                 }
368             }
369 
370             @Override
371             public void onEstimateNextIteration(final RobustEstimator<PinholeCamera> estimator, final int iteration) {
372                 if (listener != null) {
373                     listener.onEstimateNextIteration(
374                             RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this, iteration);
375                 }
376             }
377 
378             @Override
379             public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, final float progress) {
380                 if (listener != null) {
381                     listener.onEstimateProgressChange(
382                             RANSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this, progress);
383                 }
384             }
385         });
386 
387         try {
388             locked = true;
389             inliersData = null;
390             innerEstimator.setComputeAndKeepInliersEnabled(computeAndKeepInliers || refineResult);
391             innerEstimator.setComputeAndKeepResidualsEnabled(computeAndKeepResiduals || refineResult);
392             innerEstimator.setConfidence(confidence);
393             innerEstimator.setMaxIterations(maxIterations);
394             innerEstimator.setProgressDelta(progressDelta);
395             final var result = innerEstimator.estimate();
396             inliersData = innerEstimator.getInliersData();
397             return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
398         } catch (final com.irurueta.numerical.LockedException e) {
399             throw new LockedException(e);
400         } catch (final com.irurueta.numerical.NotReadyException e) {
401             throw new NotReadyException(e);
402         } finally {
403             locked = false;
404         }
405     }
406 
407     /**
408      * Returns method being used for robust estimation.
409      *
410      * @return method being used for robust estimation.
411      */
412     @Override
413     public RobustEstimatorMethod getMethod() {
414         return RobustEstimatorMethod.RANSAC;
415     }
416 
417     /**
418      * Gets standard deviation used for Levenberg-Marquardt fitting during
419      * refinement.
420      * Returned value gives an indication of how much variance each residual
421      * has.
422      * Typically, this value is related to the threshold used on each robust
423      * estimation, since residuals of found inliers are within the range of
424      * such threshold.
425      *
426      * @return standard deviation used for refinement.
427      */
428     @Override
429     protected double getRefinementStandardDeviation() {
430         return threshold;
431     }
432 }