View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.estimators;
17  
18  import com.irurueta.geometry.Line2D;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Plane;
21  import com.irurueta.numerical.robust.*;
22  
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  /**
27   * Finds the best pinhole camera for provided collections of matched lines and
28   * planes using PROMedS algorithm.
29   */
30  @SuppressWarnings("DuplicatedCode")
31  public class PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator
32          extends DLTLinePlaneCorrespondencePinholeCameraRobustEstimator {
33  
34      /**
35       * Default value to be used for stop threshold. Stop threshold can be used
36       * to keep the algorithm iterating in case that best estimated threshold
37       * using median of residuals is not small enough. Once a solution is found
38       * that generates a threshold below this value, the algorithm will stop.
39       * The stop threshold can be used to prevent the LMedS algorithm iterating
40       * too many times in cases where samples have a very similar accuracy.
41       * For instance, in cases where proportion of outliers is very small (close
42       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
43       * iterate for a long time trying to find the best solution when indeed
44       * there is no need to do that if a reasonable threshold has already been
45       * reached.
46       * Because of this behaviour the stop threshold can be set to a value much
47       * lower than the one typically used in RANSAC, and yet the algorithm could
48       * still produce even smaller thresholds in estimated results.
49       */
50      public static final double DEFAULT_STOP_THRESHOLD = 1.0;
51  
52      /**
53       * Minimum allowed stop threshold value.
54       */
55      public static final double MIN_STOP_THRESHOLD = 0.0;
56  
57      /**
58       * Threshold to be used to keep the algorithm iterating in case that best
59       * estimated threshold using median of residuals is not small enough. Once
60       * a solution is found that generates a threshold below this value, the
61       * algorithm will stop.
62       * The stop threshold can be used to prevent the LMedS algorithm iterating
63       * too many times in cases where samples have a very similar accuracy.
64       * For instance, in cases where proportion of outliers is very small (close
65       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
66       * iterate for a long time trying to find the best solution when indeed
67       * there is no need to do that if a reasonable threshold has already been
68       * reached.
69       * Because of this behaviour the stop threshold can be set to a value much
70       * lower than the one typically used in RANSAC, and yet the algorithm could
71       * still produce even smaller thresholds in estimated results.
72       */
73      private double stopThreshold;
74  
75      /**
76       * Quality scores corresponding to each pair of matched points.
77       * The larger the score value the better the quality of the matching.
78       */
79      private double[] qualityScores;
80  
81      /**
82       * Constructor.
83       */
84      public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator() {
85          super();
86          stopThreshold = DEFAULT_STOP_THRESHOLD;
87      }
88  
89      /**
90       * Constructor with lists of matched planes and 2D lines to estimate a
91       * pinhole camera.
92       * Points and lines in the lists located at the same position are considered
93       * to be matched. Hence, both lists must have the same size, and their size
94       * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
95       * (4 matches).
96       *
97       * @param planes list of planes used to estimate a pinhole camera.
98       * @param lines  list of corresponding projected 2D lines used to estimate
99       *               a pinhole camera.
100      * @throws IllegalArgumentException if provided lists don't have the same
101      *                                  size or their size is smaller than required minimum size (4 matches).
102      */
103     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
104             final List<Plane> planes, final List<Line2D> lines) {
105         super(planes, lines);
106         stopThreshold = DEFAULT_STOP_THRESHOLD;
107     }
108 
109     /**
110      * Constructor with listener.
111      *
112      * @param listener listener to be notified of events such as when estimation
113      *                 starts, ends or its progress significantly changes.
114      */
115     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
116             final PinholeCameraRobustEstimatorListener listener) {
117         super(listener);
118         stopThreshold = DEFAULT_STOP_THRESHOLD;
119     }
120 
121     /**
122      * Constructor with listener and lists of matched planes and 2D lines to
123      * estimate a pinhole camera.
124      * Points and lines in the lists located at the same position are considered
125      * to be matched. Hence, both lists must have the same size, and their size
126      * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
127      * (4 matches).
128      *
129      * @param listener listener to be notified of events such as when estimation
130      *                 starts, ends or its progress significantly changes.
131      * @param planes   list of planes used to estimate a pinhole camera.
132      * @param lines    list of corresponding projected 2D lines used to estimate
133      *                 a pinhole camera.
134      * @throws IllegalArgumentException if provided lists don't have the same
135      *                                  size or their size is smaller than required minimum size (4 matches).
136      */
137     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
138             final PinholeCameraRobustEstimatorListener listener, final List<Plane> planes, final List<Line2D> lines) {
139         super(listener, planes, lines);
140         stopThreshold = DEFAULT_STOP_THRESHOLD;
141     }
142 
143     /**
144      * Constructor.
145      *
146      * @param qualityScores quality scores corresponding to each pair of matched
147      *                      points.
148      * @throws IllegalArgumentException if provided quality scores length is
149      *                                  smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
150      */
151     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(final double[] qualityScores) {
152         super();
153         stopThreshold = DEFAULT_STOP_THRESHOLD;
154         internalSetQualityScores(qualityScores);
155     }
156 
157     /**
158      * Constructor with lists of matched planes and 2D lines to estimate a
159      * pinhole camera.
160      * Points and lines in the lists located at the same position are considered
161      * to be matched. Hence, both lists must have the same size, and their size
162      * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
163      * (4 matches).
164      *
165      * @param planes        list of planes used to estimate a pinhole camera.
166      * @param lines         list of corresponding projected 2D lines used to estimate
167      *                      a pinhole camera.
168      * @param qualityScores quality scores corresponding to each pair of matched
169      *                      points.
170      * @throws IllegalArgumentException if provided lists or quality scores
171      *                                  don't have the same size or their size is smaller than required minimum
172      *                                  size (4 matches).
173      */
174     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
175             final List<Plane> planes, final List<Line2D> lines, final double[] qualityScores) {
176         super(planes, lines);
177 
178         if (qualityScores.length != planes.size()) {
179             throw new IllegalArgumentException();
180         }
181 
182         stopThreshold = DEFAULT_STOP_THRESHOLD;
183         internalSetQualityScores(qualityScores);
184     }
185 
186     /**
187      * Constructor with listener.
188      *
189      * @param listener      listener to be notified of events such as when estimation
190      *                      starts, ends or its progress significantly changes.
191      * @param qualityScores quality scores corresponding to each pair of matched
192      *                      points.
193      * @throws IllegalArgumentException if provided quality scores length is
194      *                                  smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
195      */
196     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
197             final PinholeCameraRobustEstimatorListener listener, final double[] qualityScores) {
198         super(listener);
199         stopThreshold = DEFAULT_STOP_THRESHOLD;
200         internalSetQualityScores(qualityScores);
201     }
202 
203     /**
204      * Constructor with listener and lists of matched planes and 2D lines to
205      * estimate a pinhole camera.
206      * Points and lines in the lists located at the same position are considered
207      * to be matched. Hence, both lists must have the same size, and their size
208      * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
209      * (4 matches).
210      *
211      * @param listener      listener to be notified of events such as when estimation
212      *                      starts, ends or its progress significantly changes.
213      * @param planes        list of planes used to estimate a pinhole camera.
214      * @param lines         list of corresponding projected 2D lines used to estimate
215      *                      a pinhole camera.
216      * @param qualityScores quality scores corresponding to each pair of matched
217      *                      points.
218      * @throws IllegalArgumentException if provided lists don't have the same
219      *                                  size or their size is smaller than required minimum size (4 matches).
220      */
221     public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
222             final PinholeCameraRobustEstimatorListener listener, final List<Plane> planes, final List<Line2D> lines,
223             final double[] qualityScores) {
224         super(listener, planes, lines);
225 
226         if (qualityScores.length != planes.size()) {
227             throw new IllegalArgumentException();
228         }
229 
230         stopThreshold = DEFAULT_STOP_THRESHOLD;
231         internalSetQualityScores(qualityScores);
232     }
233 
234     /**
235      * Returns threshold to be used to keep the algorithm iterating in case that
236      * best estimated threshold using median of residuals is not small enough.
237      * Once a solution is found that generates a threshold below this value, the
238      * algorithm will stop.
239      * As in LMedS, the stop threshold can be used to prevent the PROMedS
240      * algorithm iterating too many times in cases where samples have a very
241      * similar accuracy.
242      * For instance, in cases where proportion of outliers is very small (close
243      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
244      * iterate for a long time trying to find the best solution when indeed
245      * there is no need to do that if a reasonable threshold has already been
246      * reached.
247      * Because of this behaviour the stop threshold can be set to a value much
248      * lower than the one typically used in RANSAC, and yet the algorithm could
249      * still produce even smaller thresholds in estimated results.
250      *
251      * @return stop threshold to stop the algorithm prematurely when a certain
252      * accuracy has been reached.
253      */
254     public double getStopThreshold() {
255         return stopThreshold;
256     }
257 
258     /**
259      * Sets threshold to be used to keep the algorithm iterating in case that
260      * best estimated threshold using median of residuals is not small enough.
261      * Once a solution is found that generates a threshold below this value, the
262      * algorithm will stop.
263      * As in LMedS, the stop threshold can be used to prevent the PROMedS
264      * algorithm iterating too many times in cases where samples have a very
265      * similar accuracy.
266      * For instance, in cases where proportion of outliers is very small (close
267      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
268      * iterate for a long time trying to find the best solution when indeed
269      * there is no need to do that if a reasonable threshold has already been
270      * reached.
271      * Because of this behaviour the stop threshold can be set to a value much
272      * lower than the one typically used in RANSAC, and yet the algorithm could
273      * still produce even smaller thresholds in estimated results.
274      *
275      * @param stopThreshold stop threshold to stop the algorithm prematurely
276      *                      when a certain accuracy has been reached.
277      * @throws IllegalArgumentException if provided value is zero or negative.
278      * @throws LockedException          if robust estimator is locked because an
279      *                                  estimation is already in progress.
280      */
281     public void setStopThreshold(final double stopThreshold) throws LockedException {
282         if (isLocked()) {
283             throw new LockedException();
284         }
285         if (stopThreshold <= MIN_STOP_THRESHOLD) {
286             throw new IllegalArgumentException();
287         }
288 
289         this.stopThreshold = stopThreshold;
290     }
291 
292     /**
293      * Returns quality scores corresponding to each pair of matched points.
294      * The larger the score value the better the quality of the matching.
295      *
296      * @return quality scores corresponding to each pair of matched points.
297      */
298     @Override
299     public double[] getQualityScores() {
300         return qualityScores;
301     }
302 
303     /**
304      * Sets quality scores corresponding to each pair of matched points.
305      * The larger the score value the better the quality of the matching.
306      *
307      * @param qualityScores quality scores corresponding to each pair of matched
308      *                      points.
309      * @throws LockedException          if robust estimator is locked because an
310      *                                  estimation is already in progress.
311      * @throws IllegalArgumentException if provided quality scores length is
312      *                                  smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
313      */
314     @Override
315     public void setQualityScores(final double[] qualityScores) throws LockedException {
316         if (isLocked()) {
317             throw new LockedException();
318         }
319         internalSetQualityScores(qualityScores);
320     }
321 
322     /**
323      * Indicates if estimator is ready to start the affine 2D transformation
324      * estimation.
325      * This is true when input data (i.e. lists of matched points and quality
326      * scores) are provided and a minimum of MINIMUM_SIZE points are available.
327      *
328      * @return true if estimator is ready, false otherwise.
329      */
330     @Override
331     public boolean isReady() {
332         return super.isReady() && qualityScores != null && qualityScores.length == planes.size();
333     }
334 
335     /**
336      * Estimates a pinhole camera using a robust estimator and
337      * the best set of matched 2D line/3D plane correspondences found using the
338      * robust estimator.
339      *
340      * @return a pinhole camera.
341      * @throws LockedException          if robust estimator is locked because an
342      *                                  estimation is already in progress.
343      * @throws NotReadyException        if provided input data is not enough to start
344      *                                  the estimation.
345      * @throws RobustEstimatorException if estimation fails for any reason
346      *                                  (i.e. numerical instability, no solution available, etc).
347      */
348     @Override
349     public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
350         if (isLocked()) {
351             throw new LockedException();
352         }
353         if (!isReady()) {
354             throw new NotReadyException();
355         }
356 
357         // pinhole camera estimator using DLT (Direct Linear Transform) algorithm
358         final var nonRobustEstimator = new DLTLinePlaneCorrespondencePinholeCameraEstimator();
359 
360         nonRobustEstimator.setLMSESolutionAllowed(false);
361 
362         // suggestions
363         nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
364         nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
365         nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
366         nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
367         nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
368         nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
369         nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
370         nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
371         nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
372         nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
373         nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
374         nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
375         nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
376         nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
377 
378         final var innerEstimator = new PROMedSRobustEstimator<>(new PROMedSRobustEstimatorListener<PinholeCamera>() {
379 
380             // 3D planes for a subset of samples
381             private final List<Plane> subsetPlanes = new ArrayList<>();
382 
383             // 2D lines for a subset of samples
384             private final List<Line2D> subsetLines = new ArrayList<>();
385 
386             @Override
387             public double getThreshold() {
388                 return stopThreshold;
389             }
390 
391             @Override
392             public int getTotalSamples() {
393                 return planes.size();
394             }
395 
396             @Override
397             public int getSubsetSize() {
398                 return LinePlaneCorrespondencePinholeCameraEstimator.MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES;
399             }
400 
401             @Override
402             public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
403                 subsetPlanes.clear();
404                 subsetPlanes.add(planes.get(samplesIndices[0]));
405                 subsetPlanes.add(planes.get(samplesIndices[1]));
406                 subsetPlanes.add(planes.get(samplesIndices[2]));
407                 subsetPlanes.add(planes.get(samplesIndices[3]));
408 
409                 subsetLines.clear();
410                 subsetLines.add(lines.get(samplesIndices[0]));
411                 subsetLines.add(lines.get(samplesIndices[1]));
412                 subsetLines.add(lines.get(samplesIndices[2]));
413                 subsetLines.add(lines.get(samplesIndices[3]));
414 
415                 try {
416                     nonRobustEstimator.setLists(subsetPlanes, subsetLines);
417 
418                     final var cam = nonRobustEstimator.estimate();
419                     solutions.add(cam);
420                 } catch (final Exception e) {
421                     // if lines/planes configuration is degenerate, no solution is added
422                 }
423             }
424 
425             @Override
426             public double computeResidual(PinholeCamera currentEstimation, int i) {
427                 final var inputLine = lines.get(i);
428                 final var inputPlane = planes.get(i);
429 
430                 return singleBackprojectionResidual(currentEstimation, inputLine, inputPlane);
431             }
432 
433             @Override
434             public boolean isReady() {
435                 return PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this.isReady();
436             }
437 
438             @Override
439             public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
440                 if (listener != null) {
441                     listener.onEstimateStart(
442                             PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this);
443                 }
444             }
445 
446             @Override
447             public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
448                 if (listener != null) {
449                     listener.onEstimateEnd(PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this);
450                 }
451             }
452 
453             @Override
454             public void onEstimateNextIteration(final RobustEstimator<PinholeCamera> estimator, int iteration) {
455                 if (listener != null) {
456                     listener.onEstimateNextIteration(
457                             PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this, iteration);
458                 }
459             }
460 
461             @Override
462             public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, float progress) {
463                 if (listener != null) {
464                     listener.onEstimateProgressChange(
465                             PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this, progress);
466                 }
467             }
468 
469             @Override
470             public double[] getQualityScores() {
471                 return qualityScores;
472             }
473         });
474 
475         try {
476             locked = true;
477             inliersData = null;
478             innerEstimator.setConfidence(confidence);
479             innerEstimator.setMaxIterations(maxIterations);
480             innerEstimator.setProgressDelta(progressDelta);
481             final var result = innerEstimator.estimate();
482             inliersData = innerEstimator.getInliersData();
483             return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
484 
485         } catch (final com.irurueta.numerical.LockedException e) {
486             throw new LockedException(e);
487         } catch (final com.irurueta.numerical.NotReadyException e) {
488             throw new NotReadyException(e);
489         } finally {
490             locked = false;
491         }
492     }
493 
494     /**
495      * Returns method being used for robust estimation.
496      *
497      * @return method being used for robust estimation.
498      */
499     @Override
500     public RobustEstimatorMethod getMethod() {
501         return RobustEstimatorMethod.PROMEDS;
502     }
503 
504     /**
505      * Gets standard deviation used for Levenberg-Marquardt fitting during
506      * refinement.
507      * Returned value gives an indication of how much variance each residual
508      * has.
509      * Typically, this value is related to the threshold used on each robust
510      * estimation, since residuals of found inliers are within the range of
511      * such threshold.
512      *
513      * @return standard deviation used for refinement.
514      */
515     @Override
516     protected double getRefinementStandardDeviation() {
517         final var inliersData = (PROMedSRobustEstimator.PROMedSInliersData) getInliersData();
518 
519         // avoid setting a threshold too strict
520         final var threshold = inliersData.getEstimatedThreshold();
521         return Math.max(threshold, stopThreshold);
522     }
523 
524     /**
525      * Sets quality scores corresponding to each pair of matched points.
526      * This method is used internally and does not check whether instance is
527      * locked or not.
528      *
529      * @param qualityScores quality scores to be set.
530      * @throws IllegalArgumentException if provided quality scores length is
531      *                                  smaller than MINIMUM_SIZE.
532      */
533     private void internalSetQualityScores(final double[] qualityScores) {
534         if (qualityScores.length < MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES) {
535             throw new IllegalArgumentException();
536         }
537 
538         this.qualityScores = qualityScores;
539     }
540 }