View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.estimators;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.numerical.robust.PROMedSRobustEstimator;
23  import com.irurueta.numerical.robust.PROMedSRobustEstimatorListener;
24  import com.irurueta.numerical.robust.RobustEstimator;
25  import com.irurueta.numerical.robust.RobustEstimatorException;
26  import com.irurueta.numerical.robust.RobustEstimatorMethod;
27  
28  import java.util.ArrayList;
29  import java.util.List;
30  
31  /**
32   * Finds the best pinhole camera for provided collections of matched 2D/3D
33   * points using PROMedS algorithm.
34   */
35  @SuppressWarnings("DuplicatedCode")
36  public class PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator extends
37          DLTPointCorrespondencePinholeCameraRobustEstimator {
38  
39      /**
40       * Default value to be used for stop threshold. Stop threshold can be used
41       * to keep the algorithm iterating in case that best estimated threshold
42       * using median of residuals is not small enough. Once a solution is found
43       * that generates a threshold below this value, the algorithm will stop.
44       * The stop threshold can be used to prevent the LMedS algorithm iterating
45       * too many times in cases where samples have a very similar accuracy.
46       * For instance, in cases where proportion of outliers is very small (close
47       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
48       * iterate for a long time trying to find the best solution when indeed
49       * there is no need to do that if a reasonable threshold has already been
50       * reached.
51       * Because of this behaviour the stop threshold can be set to a value much
52       * lower than the one typically used in RANSAC, and yet the algorithm could
53       * still produce even smaller thresholds in estimated results.
54       */
55      public static final double DEFAULT_STOP_THRESHOLD = 1.0;
56  
57      /**
58       * Minimum allowed stop threshold value.
59       */
60      public static final double MIN_STOP_THRESHOLD = 0.0;
61  
62      /**
63       * Threshold to be used to keep the algorithm iterating in case that best
64       * estimated threshold using median of residuals is not small enough. Once
65       * a solution is found that generates a threshold below this value, the
66       * algorithm will stop.
67       * The stop threshold can be used to prevent the LMedS algorithm iterating
68       * too many times in cases where samples have a very similar accuracy.
69       * For instance, in cases where proportion of outliers is very small (close
70       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
71       * iterate for a long time trying to find the best solution when indeed
72       * there is no need to do that if a reasonable threshold has already been
73       * reached.
74       * Because of this behaviour the stop threshold can be set to a value much
75       * lower than the one typically used in RANSAC, and yet the algorithm could
76       * still produce even smaller thresholds in estimated results.
77       */
78      private double stopThreshold;
79  
80      /**
81       * Quality scores corresponding to each pair of matched points.
82       * The larger the score value the better the quality of the matching.
83       */
84      private double[] qualityScores;
85  
86      /**
87       * Constructor.
88       */
89      public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator() {
90          super();
91          stopThreshold = DEFAULT_STOP_THRESHOLD;
92      }
93  
94      /**
95       * Constructor with lists of points to be used to estimate a pinhole camera.
96       * Points in the list located at the same position are considered to be
97       * matched. Hence, both lists must have the same size, and their size must
98       * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
99       *
100      * @param points3D list of 3D points used to estimate a pinhole camera.
101      * @param points2D list of corresponding projected 2D points used to
102      *                 estimate a pinhole camera.
103      * @throws IllegalArgumentException if provided lists of points don't have
104      *                                  the same size or their size is smaller than required minimum size
105      *                                  (6 correspondences).
106      */
107     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
108             final List<Point3D> points3D, final List<Point2D> points2D) {
109         super(points3D, points2D);
110         stopThreshold = DEFAULT_STOP_THRESHOLD;
111     }
112 
113     /**
114      * Constructor.
115      *
116      * @param listener listener to be notified of events such as when estimation
117      *                 starts, ends or its progress significantly changes.
118      */
119     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
120             final PinholeCameraRobustEstimatorListener listener) {
121         super(listener);
122         stopThreshold = DEFAULT_STOP_THRESHOLD;
123     }
124 
125     /**
126      * Constructor with listener and lists of points to be used ot estimate a
127      * pinhole camera.
128      * Points in the list located at the same position are considered to be
129      * matched. Hence, both lists must have the same size, and their size must
130      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
131      *
132      * @param listener listener to be notified of events such as when estimation
133      *                 starts, ends or its progress significantly changes.
134      * @param points3D list of 3D points used to estimate a pinhole camera.
135      * @param points2D list of corresponding projected 2D points used to
136      *                 estimate a pinhole camera.
137      * @throws IllegalArgumentException if provided lists of points don't have
138      *                                  the same size or their size is smaller than required minimum size
139      *                                  (6 correspondences).
140      */
141     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
142             final PinholeCameraRobustEstimatorListener listener,
143             final List<Point3D> points3D, final List<Point2D> points2D) {
144         super(listener, points3D, points2D);
145         stopThreshold = DEFAULT_STOP_THRESHOLD;
146     }
147 
148     /**
149      * Constructor.
150      *
151      * @param qualityScores quality scores corresponding to each pair of matched
152      *                      points.
153      * @throws IllegalArgumentException if provided quality scores length is
154      *                                  smaller than MINIMUM_SIZE (i.e. 3 samples).
155      */
156     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(final double[] qualityScores) {
157         super();
158         stopThreshold = DEFAULT_STOP_THRESHOLD;
159         internalSetQualityScores(qualityScores);
160     }
161 
162     /**
163      * Constructor with lists of points to be used to estimate a pinhole camera.
164      * Points in the list located at the same position are considered to be
165      * matched. Hence, both lists must have the same size, and their size must
166      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
167      *
168      * @param points3D      list of 3D points used to estimate a pinhole camera.
169      * @param points2D      list of corresponding projected 2D points used to
170      *                      estimate a pinhole camera.
171      * @param qualityScores quality scores corresponding to each pair of matched
172      *                      points.
173      * @throws IllegalArgumentException if provided lists of points and array
174      *                                  of quality scores don't have the same size or their size is smaller than
175      *                                  6 correspondences.
176      */
177     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
178             final List<Point3D> points3D, final List<Point2D> points2D, final double[] qualityScores) {
179         super(points3D, points2D);
180 
181         if (qualityScores.length != points3D.size()) {
182             throw new IllegalArgumentException();
183         }
184 
185         stopThreshold = DEFAULT_STOP_THRESHOLD;
186         internalSetQualityScores(qualityScores);
187     }
188 
189     /**
190      * Constructor.
191      *
192      * @param listener      listener to be notified of events such as when estimation
193      *                      starts, ends or its progress significantly changes.
194      * @param qualityScores quality scores corresponding to each pair of matched
195      *                      points.
196      * @throws IllegalArgumentException if provided quality scores length is
197      *                                  smaller than MINIMUM_SIZE (i.e. 3 samples).
198      */
199     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
200             final PinholeCameraRobustEstimatorListener listener, final double[] qualityScores) {
201         super(listener);
202         stopThreshold = DEFAULT_STOP_THRESHOLD;
203         internalSetQualityScores(qualityScores);
204     }
205 
206     /**
207      * Constructor with listener and lists of points to be used ot estimate a
208      * pinhole camera.
209      * Points in the list located at the same position are considered to be
210      * matched. Hence, both lists must have the same size, and their size must
211      * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
212      *
213      * @param listener      listener to be notified of events such as when estimation
214      *                      starts, ends or its progress significantly changes.
215      * @param points3D      list of 3D points used to estimate a pinhole camera.
216      * @param points2D      list of corresponding projected 2D points used to
217      *                      estimate a pinhole camera.
218      * @param qualityScores quality scores corresponding to each pair of matched
219      *                      points.
220      * @throws IllegalArgumentException if provided lists of points don't have
221      *                                  the same size or their size is smaller than
222      *                                  MIN_NUMBER_OF_POINT_CORRESPONDENCES.
223      */
224     public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
225             final PinholeCameraRobustEstimatorListener listener,
226             final List<Point3D> points3D, final List<Point2D> points2D, final double[] qualityScores) {
227         super(listener, points3D, points2D);
228 
229         if (qualityScores.length != points3D.size()) {
230             throw new IllegalArgumentException();
231         }
232 
233         stopThreshold = DEFAULT_STOP_THRESHOLD;
234         internalSetQualityScores(qualityScores);
235     }
236 
237     /**
238      * Returns threshold to be used to keep the algorithm iterating in case that
239      * best estimated threshold using median of residuals is not small enough.
240      * Once a solution is found that generates a threshold below this value, the
241      * algorithm will stop.
242      * As in LMedS, the stop threshold can be used to prevent the PROMedS
243      * algorithm iterating too many times in cases where samples have a very
244      * similar accuracy.
245      * For instance, in cases where proportion of outliers is very small (close
246      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
247      * iterate for a long time trying to find the best solution when indeed
248      * there is no need to do that if a reasonable threshold has already been
249      * reached.
250      * Because of this behaviour the stop threshold can be set to a value much
251      * lower than the one typically used in RANSAC, and yet the algorithm could
252      * still produce even smaller thresholds in estimated results.
253      *
254      * @return stop threshold to stop the algorithm prematurely when a certain
255      * accuracy has been reached.
256      */
257     public double getStopThreshold() {
258         return stopThreshold;
259     }
260 
261     /**
262      * Sets threshold to be used to keep the algorithm iterating in case that
263      * best estimated threshold using median of residuals is not small enough.
264      * Once a solution is found that generates a threshold below this value, the
265      * algorithm will stop.
266      * As in LMedS, the stop threshold can be used to prevent the PROMedS
267      * algorithm iterating too many times in cases where samples have a very
268      * similar accuracy.
269      * For instance, in cases where proportion of outliers is very small (close
270      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
271      * iterate for a long time trying to find the best solution when indeed
272      * there is no need to do that if a reasonable threshold has already been
273      * reached.
274      * Because of this behaviour the stop threshold can be set to a value much
275      * lower than the one typically used in RANSAC, and yet the algorithm could
276      * still produce even smaller thresholds in estimated results.
277      *
278      * @param stopThreshold stop threshold to stop the algorithm prematurely
279      *                      when a certain accuracy has been reached.
280      * @throws IllegalArgumentException if provided value is zero or negative.
281      * @throws LockedException          if robust estimator is locked because an
282      *                                  estimation is already in progress.
283      */
284     public void setStopThreshold(final double stopThreshold) throws LockedException {
285         if (isLocked()) {
286             throw new LockedException();
287         }
288         if (stopThreshold <= MIN_STOP_THRESHOLD) {
289             throw new IllegalArgumentException();
290         }
291 
292         this.stopThreshold = stopThreshold;
293     }
294 
295     /**
296      * Returns quality scores corresponding to each pair of matched points.
297      * The larger the score value the better the quality of the matching.
298      *
299      * @return quality scores corresponding to each pair of matched points.
300      */
301     @Override
302     public double[] getQualityScores() {
303         return qualityScores;
304     }
305 
306     /**
307      * Sets quality scores corresponding to each pair of matched points.
308      * The larger the score value the better the quality of the matching.
309      *
310      * @param qualityScores quality scores corresponding to each pair of matched
311      *                      points.
312      * @throws LockedException          if robust estimator is locked because an
313      *                                  estimation is already in progress.
314      * @throws IllegalArgumentException if provided quality scores length is
315      *                                  smaller than MINIMUM_SIZE (i.e. 3 samples).
316      */
317     @Override
318     public void setQualityScores(final double[] qualityScores) throws LockedException {
319         if (isLocked()) {
320             throw new LockedException();
321         }
322         internalSetQualityScores(qualityScores);
323     }
324 
325     /**
326      * Indicates if estimator is ready to start the affine 2D transformation
327      * estimation.
328      * This is true when input data (i.e. lists of matched points and quality
329      * scores) are provided and a minimum of MINIMUM_SIZE points are available.
330      *
331      * @return true if estimator is ready, false otherwise.
332      */
333     @Override
334     public boolean isReady() {
335         return super.isReady() && qualityScores != null && qualityScores.length == points3D.size();
336     }
337 
338     /**
339      * Estimates an affine 2D transformation using a robust estimator and
340      * the best set of matched 2D point correspondences found using the robust
341      * estimator.
342      *
343      * @return an affine 2D transformation.
344      * @throws LockedException          if robust estimator is locked because an
345      *                                  estimation is already in progress.
346      * @throws NotReadyException        if provided input data is not enough to start
347      *                                  the estimation.
348      * @throws RobustEstimatorException if estimation fails for any reason
349      *                                  (i.e. numerical instability, no solution available, etc).
350      */
351     @Override
352     public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
353         if (isLocked()) {
354             throw new LockedException();
355         }
356         if (!isReady()) {
357             throw new NotReadyException();
358         }
359 
360         // pinhole camera estimator using DLT (Direct Linear Transform) algorithm
361         final var nonRobustEstimator = new DLTPointCorrespondencePinholeCameraEstimator();
362 
363         nonRobustEstimator.setLMSESolutionAllowed(false);
364         nonRobustEstimator.setPointCorrespondencesNormalized(normalizeSubsetPointCorrespondences);
365 
366         // suggestions
367         nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
368         nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
369         nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
370         nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
371         nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
372         nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
373         nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
374         nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
375         nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
376         nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
377         nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
378         nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
379         nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
380         nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
381 
382         final var innerEstimator = new PROMedSRobustEstimator<>(new PROMedSRobustEstimatorListener<PinholeCamera>() {
383 
384             // point to be reused when computing residuals
385             private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
386 
387             // 3D points for a subset of samples
388             private final List<Point3D> subset3D = new ArrayList<>();
389 
390             // 2D points for a subset of samples
391             private final List<Point2D> subset2D = new ArrayList<>();
392 
393             @Override
394             public double getThreshold() {
395                 return stopThreshold;
396             }
397 
398             @Override
399             public int getTotalSamples() {
400                 return points3D.size();
401             }
402 
403             @Override
404             public int getSubsetSize() {
405                 return PointCorrespondencePinholeCameraRobustEstimator.MIN_NUMBER_OF_POINT_CORRESPONDENCES;
406             }
407 
408             @Override
409             public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
410                 subset3D.clear();
411                 subset3D.add(points3D.get(samplesIndices[0]));
412                 subset3D.add(points3D.get(samplesIndices[1]));
413                 subset3D.add(points3D.get(samplesIndices[2]));
414                 subset3D.add(points3D.get(samplesIndices[3]));
415                 subset3D.add(points3D.get(samplesIndices[4]));
416                 subset3D.add(points3D.get(samplesIndices[5]));
417 
418                 subset2D.clear();
419                 subset2D.add(points2D.get(samplesIndices[0]));
420                 subset2D.add(points2D.get(samplesIndices[1]));
421                 subset2D.add(points2D.get(samplesIndices[2]));
422                 subset2D.add(points2D.get(samplesIndices[3]));
423                 subset2D.add(points2D.get(samplesIndices[4]));
424                 subset2D.add(points2D.get(samplesIndices[5]));
425 
426                 try {
427                     nonRobustEstimator.setLists(subset3D, subset2D);
428 
429                     final var cam = nonRobustEstimator.estimate();
430                     solutions.add(cam);
431                 } catch (final Exception e) {
432                     // if points configuration is degenerate, no solution is added
433                 }
434             }
435 
436             @Override
437             public double computeResidual(final PinholeCamera currentEstimation, final int i) {
438                 // pick i-th points
439                 final var point3D = points3D.get(i);
440                 final var point2D = points2D.get(i);
441 
442                 // project point3D into test point
443                 currentEstimation.project(point3D, testPoint);
444 
445                 // compare test point and 2D point
446                 return testPoint.distanceTo(point2D);
447             }
448 
449             @Override
450             public boolean isReady() {
451                 return PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this.isReady();
452             }
453 
454             @Override
455             public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
456                 if (listener != null) {
457                     listener.onEstimateStart(PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this);
458                 }
459             }
460 
461             @Override
462             public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
463                 if (listener != null) {
464                     listener.onEstimateEnd(PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this);
465                 }
466             }
467 
468             @Override
469             public void onEstimateNextIteration(
470                     final RobustEstimator<PinholeCamera> estimator, final int iteration) {
471                 if (listener != null) {
472                     listener.onEstimateNextIteration(
473                             PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this, iteration);
474                 }
475             }
476 
477             @Override
478             public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, final float progress) {
479                 if (listener != null) {
480                     listener.onEstimateProgressChange(
481                             PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this, progress);
482                 }
483             }
484 
485             @Override
486             public double[] getQualityScores() {
487                 return qualityScores;
488             }
489         });
490 
491         try {
492             locked = true;
493             inliersData = null;
494             innerEstimator.setConfidence(confidence);
495             innerEstimator.setMaxIterations(maxIterations);
496             innerEstimator.setProgressDelta(progressDelta);
497             final var result = innerEstimator.estimate();
498             inliersData = innerEstimator.getInliersData();
499             return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
500         } catch (final com.irurueta.numerical.LockedException e) {
501             throw new LockedException(e);
502         } catch (final com.irurueta.numerical.NotReadyException e) {
503             throw new NotReadyException(e);
504         } finally {
505             locked = false;
506         }
507     }
508 
509     /**
510      * Returns method being used for robust estimation.
511      *
512      * @return method being used for robust estimation.
513      */
514     @Override
515     public RobustEstimatorMethod getMethod() {
516         return RobustEstimatorMethod.PROMEDS;
517     }
518 
519     /**
520      * Gets standard deviation used for Levenberg-Marquardt fitting during
521      * refinement.
522      * Returned value gives an indication of how much variance each residual
523      * has.
524      * Typically, this value is related to the threshold used on each robust
525      * estimation, since residuals of found inliers are within the range of
526      * such threshold.
527      *
528      * @return standard deviation used for refinement.
529      */
530     @Override
531     protected double getRefinementStandardDeviation() {
532         final var inliersData = (PROMedSRobustEstimator.PROMedSInliersData) getInliersData();
533 
534         // avoid setting a threshold too strict
535         final var threshold = inliersData.getEstimatedThreshold();
536         return Math.max(threshold, stopThreshold);
537     }
538 
539     /**
540      * Sets quality scores corresponding to each pair of matched points.
541      * This method is used internally and does not check whether instance is
542      * locked or not.
543      *
544      * @param qualityScores quality scores to be set.
545      * @throws IllegalArgumentException if provided quality scores length is
546      *                                  smaller than MINIMUM_SIZE.
547      */
548     private void internalSetQualityScores(double[] qualityScores) {
549         if (qualityScores.length < MIN_NUMBER_OF_POINT_CORRESPONDENCES) {
550             throw new IllegalArgumentException();
551         }
552 
553         this.qualityScores = qualityScores;
554     }
555 }