View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.estimators;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.numerical.robust.MSACRobustEstimator;
23  import com.irurueta.numerical.robust.MSACRobustEstimatorListener;
24  import com.irurueta.numerical.robust.RobustEstimator;
25  import com.irurueta.numerical.robust.RobustEstimatorException;
26  import com.irurueta.numerical.robust.RobustEstimatorMethod;
27  
28  import java.util.ArrayList;
29  import java.util.List;
30  
31  /**
32   * Finds the best pinhole camera for provided collections of matched 2D/3D
33   * points using MSAC + UPnP algorithms.
34   */
35  @SuppressWarnings("DuplicatedCode")
36  public class MSACUPnPPointCorrespondencePinholeCameraRobustEstimator extends
37          UPnPPointCorrespondencePinholeCameraRobustEstimator {
38  
39      /**
40       * Constant defining default threshold to determine whether points are
41       * inliers or not.
42       * By default, 1.0 is considered a good value for cases where measures are
43       * done on pixels, since typically the minimum resolution is 1 pixel.
44       */
45      public static final double DEFAULT_THRESHOLD = 1.0;
46  
47      /**
48       * Minimum value that can be set as threshold.
49       * Threshold must be strictly greater than 0.0.
50       */
51      public static final double MIN_THRESHOLD = 0.0;
52  
53      /**
54       * Threshold to determine whether points are inliers or not when testing
55       * possible estimation solutions.
56       * The threshold refers to the amount of error (i.e. distance) a possible
57       * solution has on a matched pair of points.
58       */
59      private double threshold;
60  
61      /**
62       * Constructor.
63       */
64      public MSACUPnPPointCorrespondencePinholeCameraRobustEstimator() {
65          super();
66          threshold = DEFAULT_THRESHOLD;
67      }
68  
69      /**
70       * Constructor with listener.
71       *
72       * @param listener listener to be notified of events such as when estimation
73       *                 starts, ends or its progress significantly changes.
74       */
75      public MSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
76              final PinholeCameraRobustEstimatorListener listener) {
77          super(listener);
78          threshold = DEFAULT_THRESHOLD;
79      }
80  
81      /**
82       * Constructor with lists of points to be used to estimate a pinhole camera.
83       * Points in the lists located at the same position are considered to be
84       * matched. Hence, both lists must have the same size, and their size must
85       * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES (6 points).
86       *
87       * @param points3D list of 3D points used to estimate a pinhole camera.
88       * @param points2D list of corresponding projected 2D points used to
89       *                 estimate a pinhole camera.
90       * @throws IllegalArgumentException if provided lists of points don't have
91       *                                  the same size or their size is smaller than required minimum size (6
92       *                                  correspondences).
93       */
94      public MSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
95              final List<Point3D> points3D, final List<Point2D> points2D) {
96          super(points3D, points2D);
97          threshold = DEFAULT_THRESHOLD;
98      }
99  
100     /**
101      * Constructor with listener and lists of points to be used to estimate a
102      * pinhole camera.
103      * Points in the lists located at the same position are considered to be
104      * matched. Hence, both lists must have the same size, and their size must
105      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES (6 points).
106      *
107      * @param listener listener to be notified of events such as when estimation
108      *                 starts, ends or its progress significantly changes.
109      * @param points3D list of 3D points used to estimate a pinhole camera.
110      * @param points2D list of corresponding projected 2D points used to
111      *                 estimate a pinhole camera.
112      * @throws IllegalArgumentException if provided lists of points don't have
113      *                                  the same size or their size is smaller than required minimum size (6
114      *                                  correspondences).
115      */
116     public MSACUPnPPointCorrespondencePinholeCameraRobustEstimator(
117             final PinholeCameraRobustEstimatorListener listener,
118             final List<Point3D> points3D, final List<Point2D> points2D) {
119         super(listener, points3D, points2D);
120         threshold = DEFAULT_THRESHOLD;
121     }
122 
123     /**
124      * Returns threshold to determine whether points are inliers or not when
125      * testing possible estimation solutions.
126      * The threshold refers to the amount of error (i.e. Euclidean distance) a
127      * possible solution has on projected 2D points.
128      *
129      * @return threshold to determine whether points are inliers or not when
130      * testing possible estimation solutions.
131      */
132     public double getThreshold() {
133         return threshold;
134     }
135 
136     /**
137      * Sets threshold to determine whether points are inliers or not when
138      * testing possible estimation solutions.
139      * The threshold refers to the amount of error (i.e. Euclidean distance) a
140      * possible solution has on projected 2D points.
141      *
142      * @param threshold threshold to be set.
143      * @throws IllegalArgumentException if provided value is equal or less than
144      *                                  zero.
145      * @throws LockedException          if robust estimator is locked because an
146      *                                  estimation is already in progress.
147      */
148     public void setThreshold(final double threshold) throws LockedException {
149         if (isLocked()) {
150             throw new LockedException();
151         }
152         if (threshold <= MIN_THRESHOLD) {
153             throw new IllegalArgumentException();
154         }
155         this.threshold = threshold;
156     }
157 
158     /**
159      * Estimates a pinhole camera using a robust estimator and
160      * the best set of matched 2D/3D point correspondences or 2D line/3D plane
161      * correspondences found using the robust estimator.
162      *
163      * @return a pinhole camera.
164      * @throws LockedException          if robust estimator is locked because an
165      *                                  estimation is already in progress.
166      * @throws NotReadyException        if provided input data is not enough to start
167      *                                  the estimation.
168      * @throws RobustEstimatorException if estimation fails for any reason
169      *                                  (i.e. numerical instability, no solution available, etc).
170      */
171     @Override
172     public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
173         if (isLocked()) {
174             throw new LockedException();
175         }
176         if (!isReady()) {
177             throw new NotReadyException();
178         }
179 
180         // pinhole camera estimator using UPnP (Uncalibrated Perspective-n-Point) algorithm
181         final var nonRobustEstimator = new UPnPPointCorrespondencePinholeCameraEstimator();
182 
183         nonRobustEstimator.setPlanarConfigurationAllowed(planarConfigurationAllowed);
184         nonRobustEstimator.setNullspaceDimension2Allowed(nullspaceDimension2Allowed);
185         nonRobustEstimator.setPlanarThreshold(planarThreshold);
186         nonRobustEstimator.setSkewness(skewness);
187         nonRobustEstimator.setHorizontalPrincipalPoint(horizontalPrincipalPoint);
188         nonRobustEstimator.setVerticalPrincipalPoint(verticalPrincipalPoint);
189 
190         // suggestions
191         nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
192         nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
193         nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
194         nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
195         nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
196         nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
197         nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
198         nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
199         nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
200         nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
201         nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
202         nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
203         nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
204         nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
205 
206         final var innerEstimator = new MSACRobustEstimator<>(new MSACRobustEstimatorListener<PinholeCamera>() {
207 
208             // point to be reused when computing residuals
209             private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
210 
211             // 3D points for a subset of samples
212             private final List<Point3D> subset3D = new ArrayList<>();
213 
214             // 2D points for a subset of samples
215             private final List<Point2D> subset2D = new ArrayList<>();
216 
217             @Override
218             public double getThreshold() {
219                 return threshold;
220             }
221 
222             @Override
223             public int getTotalSamples() {
224                 return points3D.size();
225             }
226 
227             @Override
228             public int getSubsetSize() {
229                 return PointCorrespondencePinholeCameraEstimator.MIN_NUMBER_OF_POINT_CORRESPONDENCES;
230             }
231 
232             @Override
233             public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
234                 subset3D.clear();
235                 subset3D.add(points3D.get(samplesIndices[0]));
236                 subset3D.add(points3D.get(samplesIndices[1]));
237                 subset3D.add(points3D.get(samplesIndices[2]));
238                 subset3D.add(points3D.get(samplesIndices[3]));
239                 subset3D.add(points3D.get(samplesIndices[4]));
240                 subset3D.add(points3D.get(samplesIndices[5]));
241 
242                 subset2D.clear();
243                 subset2D.add(points2D.get(samplesIndices[0]));
244                 subset2D.add(points2D.get(samplesIndices[1]));
245                 subset2D.add(points2D.get(samplesIndices[2]));
246                 subset2D.add(points2D.get(samplesIndices[3]));
247                 subset2D.add(points2D.get(samplesIndices[4]));
248                 subset2D.add(points2D.get(samplesIndices[5]));
249 
250                 try {
251                     nonRobustEstimator.setLists(subset3D, subset2D);
252 
253                     final var cam = nonRobustEstimator.estimate();
254                     solutions.add(cam);
255                 } catch (final Exception e) {
256                     // if points configuration is degenerate, no solution is
257                     // added
258                 }
259             }
260 
261             @Override
262             public double computeResidual(final PinholeCamera currentEstimation, final int i) {
263                 // pick i-th points
264                 final var point3D = points3D.get(i);
265                 final var point2D = points2D.get(i);
266 
267                 // project point3D into test point
268                 currentEstimation.project(point3D, testPoint);
269 
270                 // compare test point and 2D point
271                 return testPoint.distanceTo(point2D);
272             }
273 
274             @Override
275             public boolean isReady() {
276                 return MSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this.isReady();
277             }
278 
279             @Override
280             public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
281                 if (listener != null) {
282                     listener.onEstimateStart(MSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this);
283                 }
284             }
285 
286             @Override
287             public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
288                 if (listener != null) {
289                     listener.onEstimateEnd(MSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this);
290                 }
291             }
292 
293             @Override
294             public void onEstimateNextIteration(final RobustEstimator<PinholeCamera> estimator, int iteration) {
295                 if (listener != null) {
296                     listener.onEstimateNextIteration(
297                             MSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this, iteration);
298                 }
299             }
300 
301             @Override
302             public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, float progress) {
303                 if (listener != null) {
304                     listener.onEstimateProgressChange(
305                             MSACUPnPPointCorrespondencePinholeCameraRobustEstimator.this, progress);
306                 }
307             }
308         });
309 
310         try {
311             locked = true;
312             inliersData = null;
313             innerEstimator.setConfidence(confidence);
314             innerEstimator.setMaxIterations(maxIterations);
315             innerEstimator.setProgressDelta(progressDelta);
316             final var result = innerEstimator.estimate();
317             inliersData = innerEstimator.getInliersData();
318             return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
319         } catch (final com.irurueta.numerical.LockedException e) {
320             throw new LockedException(e);
321         } catch (final com.irurueta.numerical.NotReadyException e) {
322             throw new NotReadyException(e);
323         } finally {
324             locked = false;
325         }
326     }
327 
328     /**
329      * Returns method being used for robust estimation.
330      *
331      * @return method being used for robust estimation.
332      */
333     @Override
334     public RobustEstimatorMethod getMethod() {
335         return RobustEstimatorMethod.MSAC;
336     }
337 
338     /**
339      * Gets standard deviation used for Levenberg-Marquardt fitting during
340      * refinement.
341      * Returned value gives an indication of how much variance each residual
342      * has.
343      * Typically, this value is related to the threshold used on each robust
344      * estimation, since residuals of found inliers are within the range of
345      * such threshold.
346      *
347      * @return standard deviation used for refinement.
348      */
349     @Override
350     protected double getRefinementStandardDeviation() {
351         return threshold;
352     }
353 }