View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.sfm;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.geometry.estimators.LockedException;
23  import com.irurueta.geometry.estimators.NotReadyException;
24  import com.irurueta.numerical.robust.PROMedSRobustEstimator;
25  import com.irurueta.numerical.robust.PROMedSRobustEstimatorListener;
26  import com.irurueta.numerical.robust.RobustEstimator;
27  import com.irurueta.numerical.robust.RobustEstimatorException;
28  import com.irurueta.numerical.robust.RobustEstimatorMethod;
29  
30  import java.util.ArrayList;
31  import java.util.List;
32  
33  /**
34   * Robustly triangulates 3D points from matched 2D points and their
35   * corresponding cameras on several views using PROMedS algorithm.
36   */
37  public class PROMedSRobustSinglePoint3DTriangulator extends RobustSinglePoint3DTriangulator {
38  
39      /**
40       * Default value to be used for stop threshold. Stop threshold can be used
41       * to keep the algorithm iterating in case that best estimated threshold
42       * using median of residuals is not small enough. Once a solution is found
43       * that generates a threshold below this value, the algorithm will stop.
44       * The stop threshold can be used to prevent the LMedS algorithm iterating
45       * too many times in cases where samples have a very similar accuracy.
46       * For instance, in cases where proportion of outliers is very small (close
47       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
48       * iterate for a long time trying to find the best solution when indeed
49       * there is no need to do that if a reasonable threshold has already been
50       * reached.
51       * Because of this behaviour the stop threshold can be set to a value much
52       * lower than the one typically used in RANSAC, and yet the algorithm could
53       * still produce even smaller thresholds in estimated results.
54       */
55      public static final double DEFAULT_STOP_THRESHOLD = 1e-3;
56  
57      /**
58       * Minimum allowed stop threshold value.
59       */
60      public static final double MIN_STOP_THRESHOLD = 0.0;
61  
62      /**
63       * Threshold to be used to keep the algorithm iterating in case that best
64       * estimated threshold using median of residuals is not small enough. Once
65       * a solution is found that generates a threshold below this value, the
66       * algorithm will stop.
67       * The stop threshold can be used to prevent the LMedS algorithm iterating
68       * too many times in cases where samples have a very similar accuracy.
69       * For instance, in cases where proportion of outliers is very small (close
70       * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
71       * iterate for a long time trying to find the best solution when indeed
72       * there is no need to do that if a reasonable threshold has already been
73       * reached.
74       * Because of this behaviour the stop threshold can be set to a value much
75       * lower than the one typically used in RANSAC, and yet the algorithm could
76       * still produce even smaller thresholds in estimated results.
77       */
78      private double stopThreshold;
79  
80      /**
81       * Quality scores corresponding to each provided point.
82       * The larger the score value the better the quality of the sample.
83       */
84      private double[] qualityScores;
85  
86      /**
87       * Constructor.
88       */
89      public PROMedSRobustSinglePoint3DTriangulator() {
90          super();
91          stopThreshold = DEFAULT_STOP_THRESHOLD;
92      }
93  
94      /**
95       * Constructor.
96       *
97       * @param listener listener to be notified of events such as when estimation
98       *                 starts, ends or its progress significantly changes.
99       */
100     public PROMedSRobustSinglePoint3DTriangulator(final RobustSinglePoint3DTriangulatorListener listener) {
101         super(listener);
102         stopThreshold = DEFAULT_STOP_THRESHOLD;
103     }
104 
105     /**
106      * Constructor.
107      *
108      * @param points  Matched 2D points. Each point in the list is assumed to be
109      *                projected by the corresponding camera in the list.
110      * @param cameras List of cameras associated to the matched 2D point on the
111      *                same position as the camera on the list.
112      * @throws IllegalArgumentException if provided lists don't have the same
113      *                                  length or their length is less than 2 views, which is the minimum
114      *                                  required to compute triangulation.
115      */
116     public PROMedSRobustSinglePoint3DTriangulator(final List<Point2D> points, final List<PinholeCamera> cameras) {
117         super(points, cameras);
118         stopThreshold = DEFAULT_STOP_THRESHOLD;
119     }
120 
121     /**
122      * Constructor.
123      *
124      * @param points   Matched 2D points. Each point in the list is assumed to be
125      *                 projected by the corresponding camera in the list.
126      * @param cameras  List of cameras associated to the matched 2D point on the
127      *                 same position as the camera on the list.
128      * @param listener listener to be notified of events such as when estimation
129      *                 starts, ends or its progress significantly changes.
130      * @throws IllegalArgumentException if provided lists don't have the same
131      *                                  length or their length is less than 2 views, which is the minimum
132      *                                  required to compute triangulation.
133      */
134     public PROMedSRobustSinglePoint3DTriangulator(
135             final List<Point2D> points, final List<PinholeCamera> cameras,
136             final RobustSinglePoint3DTriangulatorListener listener) {
137         super(points, cameras, listener);
138         stopThreshold = DEFAULT_STOP_THRESHOLD;
139     }
140 
141     /**
142      * Constructor.
143      *
144      * @param qualityScores quality scores corresponding to each provided view.
145      * @throws IllegalArgumentException if provided quality scores length is
146      *                                  smaller than required size (i.e. 2 views).
147      */
148     public PROMedSRobustSinglePoint3DTriangulator(final double[] qualityScores) {
149         this();
150         internalSetQualityScores(qualityScores);
151     }
152 
153     /**
154      * Constructor.
155      *
156      * @param qualityScores quality scores corresponding to each provided view.
157      * @param listener      listener to be notified of events such as when estimation
158      *                      starts, ends or its progress significantly changes.
159      * @throws IllegalArgumentException if provided quality scores length is
160      *                                  smaller than required size (i.e. 2 views).
161      */
162     public PROMedSRobustSinglePoint3DTriangulator(final double[] qualityScores,
163                                                   final RobustSinglePoint3DTriangulatorListener listener) {
164         this(listener);
165         internalSetQualityScores(qualityScores);
166     }
167 
168     /**
169      * Constructor.
170      *
171      * @param points        Matched 2D points. Each point in the list is assumed to be
172      *                      projected by the corresponding camera in the list.
173      * @param cameras       List of cameras associated to the matched 2D point on the
174      *                      same position as the camera on the list.
175      * @param qualityScores quality scores corresponding to each provided view.
176      * @throws IllegalArgumentException if provided lists or quality scores
177      *                                  don't have the same length or their length is less than 2 views,
178      *                                  which is the minimum required to compute triangulation.
179      */
180     public PROMedSRobustSinglePoint3DTriangulator(
181             final List<Point2D> points, final List<PinholeCamera> cameras, final double[] qualityScores) {
182         this(points, cameras);
183         internalSetQualityScores(qualityScores);
184     }
185 
186     /**
187      * Constructor.
188      *
189      * @param points        Matched 2D points. Each point in the list is assumed to be
190      *                      projected by the corresponding camera in the list.
191      * @param cameras       List of cameras associated to the matched 2D point on the
192      *                      same position as the camera on the list.
193      * @param qualityScores quality scores corresponding to each provided view.
194      * @param listener      listener to be notified of events such as when estimation
195      *                      starts, ends or its progress significantly changes.
196      * @throws IllegalArgumentException if provided lists or quality scores
197      *                                  don't have the same length or their length is less than 2 views,
198      *                                  which is the minimum required to compute triangulation.
199      */
200     public PROMedSRobustSinglePoint3DTriangulator(final List<Point2D> points,
201                                                   final List<PinholeCamera> cameras,
202                                                   final double[] qualityScores,
203                                                   final RobustSinglePoint3DTriangulatorListener listener) {
204         this(points, cameras, listener);
205         internalSetQualityScores(qualityScores);
206     }
207 
208     /**
209      * Returns threshold to be used to keep the algorithm iterating in case that
210      * best estimated threshold using median of residuals is not small enough.
211      * Once a solution is found that generates a threshold below this value, the
212      * algorithm will stop.
213      * The stop threshold can be used to prevent the LMedS algorithm iterating
214      * too many times in cases where samples have a very similar accuracy.
215      * For instance, in cases where proportion of outliers is very small (close
216      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
217      * iterate for a long time trying to find the best solution when indeed
218      * there is no need to do that if a reasonable threshold has already been
219      * reached.
220      * Because of this behaviour the stop threshold can be set to a value much
221      * lower than the one typically used in RANSAC, and yet the algorithm could
222      * still produce even smaller thresholds in estimated results.
223      *
224      * @return stop threshold to stop the algorithm prematurely when a certain
225      * accuracy has been reached.
226      */
227     public double getStopThreshold() {
228         return stopThreshold;
229     }
230 
231     /**
232      * Sets threshold to be used to keep the algorithm iterating in case that
233      * best estimated threshold using median of residuals is not small enough.
234      * Once a solution is found that generates a threshold below this value, the
235      * algorithm will stop.
236      * The stop threshold can be used to prevent the LMedS algorithm iterating
237      * too many times in cases where samples have a very similar accuracy.
238      * For instance, in cases where proportion of outliers is very small (close
239      * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
240      * iterate for a long time trying to find the best solution when indeed
241      * there is no need to do that if a reasonable threshold has already been
242      * reached.
243      * Because of this behaviour the stop threshold can be set to a value much
244      * lower than the one typically used in RANSAC, and yet the algorithm could
245      * still produce even smaller thresholds in estimated results.
246      *
247      * @param stopThreshold stop threshold to stop the algorithm prematurely
248      *                      when a certain accuracy has been reached.
249      * @throws IllegalArgumentException if provided value is zero or negative.
250      * @throws LockedException          if robust estimator is locked because an
251      *                                  estimation is already in progress.
252      */
253     public void setStopThreshold(final double stopThreshold) throws LockedException {
254         if (isLocked()) {
255             throw new LockedException();
256         }
257         if (stopThreshold <= MIN_STOP_THRESHOLD) {
258             throw new IllegalArgumentException();
259         }
260 
261         this.stopThreshold = stopThreshold;
262     }
263 
264     /**
265      * Returns quality scores corresponding to each provided view.
266      * The larger the score value the better the quality of the sampled view.
267      *
268      * @return quality scores corresponding to each view.
269      */
270     @Override
271     public double[] getQualityScores() {
272         return qualityScores;
273     }
274 
275     /**
276      * Sets quality scores corresponding to each provided view.
277      * The larger the score value the better the quality of the sampled view.
278      *
279      * @param qualityScores quality scores corresponding to each view.
280      * @throws LockedException          if robust estimator is locked because an
281      *                                  estimation is already in progress.
282      * @throws IllegalArgumentException if provided quality scores length is
283      *                                  smaller than MIN_REQUIRED_VIEWS (i.e. 2 views).
284      */
285     @Override
286     public void setQualityScores(final double[] qualityScores) throws LockedException {
287         if (isLocked()) {
288             throw new LockedException();
289         }
290         internalSetQualityScores(qualityScores);
291     }
292 
293     /**
294      * Indicates if triangulator is ready to start the 3D point triangulation.
295      * This is true when input data (i.e. 2D points, cameras and quality scores)
296      * are provided and a minimum of 2 views are available.
297      *
298      * @return true if estimator is ready, false otherwise.
299      */
300     @Override
301     public boolean isReady() {
302         return super.isReady() && qualityScores != null && qualityScores.length == points2D.size();
303     }
304 
305 
306     /**
307      * Triangulates provided matched 2D points being projected by each
308      * corresponding camera into a single 3D point.
309      * At least 2 matched 2D points and their corresponding 2 cameras are
310      * required to compute triangulation. If more views are provided, an
311      * averaged solution can be found.
312      *
313      * @return computed triangulated 3D point.
314      * @throws LockedException          if this instance is locked.
315      * @throws NotReadyException        if lists of points and cameras don't have the
316      *                                  same length or less than 2 views are provided.
317      * @throws RobustEstimatorException if estimation fails for any reason
318      *                                  (i.e. numerical instability, no solution available, etc).
319      */
320     @SuppressWarnings("DuplicatedCode")
321     @Override
322     public Point3D triangulate() throws LockedException, NotReadyException, RobustEstimatorException {
323         if (isLocked()) {
324             throw new LockedException();
325         }
326         if (!isReady()) {
327             throw new NotReadyException();
328         }
329 
330         final var innerEstimator = new PROMedSRobustEstimator<Point3D>(new PROMedSRobustEstimatorListener<>() {
331 
332                             // point to be reused when computing residuals
333                             private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
334 
335                             // non-robust 3D point triangulator
336                             private final SinglePoint3DTriangulator triangulator =
337                                     SinglePoint3DTriangulator.create(useHomogeneousSolution
338                                             ? Point3DTriangulatorType.LMSE_HOMOGENEOUS_TRIANGULATOR
339                                             : Point3DTriangulatorType.LMSE_INHOMOGENEOUS_TRIANGULATOR);
340 
341                             // subset of 2D points
342                             private final List<Point2D> subsetPoints = new ArrayList<>();
343 
344                             // subst of cameras
345                             private final List<PinholeCamera> subsetCameras = new ArrayList<>();
346 
347                             @Override
348                             public double getThreshold() {
349                                 return stopThreshold;
350                             }
351 
352                             @Override
353                             public int getTotalSamples() {
354                                 return points2D.size();
355                             }
356 
357                             @Override
358                             public int getSubsetSize() {
359                                 return MIN_REQUIRED_VIEWS;
360                             }
361 
362                             @Override
363                             public void estimatePreliminarSolutions(
364                                     final int[] samplesIndices, final List<Point3D> solutions) {
365                                 subsetPoints.clear();
366                                 subsetPoints.add(points2D.get(samplesIndices[0]));
367                                 subsetPoints.add(points2D.get(samplesIndices[1]));
368 
369                                 subsetCameras.clear();
370                                 subsetCameras.add(cameras.get(samplesIndices[0]));
371                                 subsetCameras.add(cameras.get(samplesIndices[1]));
372 
373                                 try {
374                                     triangulator.setPointsAndCameras(subsetPoints, subsetCameras);
375                                     final var triangulated = triangulator.triangulate();
376                                     solutions.add(triangulated);
377                                 } catch (final Exception e) {
378                                     // if anything fails, no solution is added
379                                 }
380                             }
381 
382                             @Override
383                             public double computeResidual(final Point3D currentEstimation, final int i) {
384                                 final var point2D = points2D.get(i);
385                                 final var camera = cameras.get(i);
386 
387                                 // project estimated point with camera
388                                 camera.project(currentEstimation, testPoint);
389 
390                                 // return distance of projected point respect to the original one
391                                 // as a residual
392                                 return testPoint.distanceTo(point2D);
393                             }
394 
395                             @Override
396                             public boolean isReady() {
397                                 return PROMedSRobustSinglePoint3DTriangulator.this.isReady();
398                             }
399 
400                             @Override
401                             public void onEstimateStart(final RobustEstimator<Point3D> estimator) {
402                                 if (listener != null) {
403                                     listener.onTriangulateStart(PROMedSRobustSinglePoint3DTriangulator.this);
404                                 }
405                             }
406 
407                             @Override
408                             public void onEstimateEnd(final RobustEstimator<Point3D> estimator) {
409                                 if (listener != null) {
410                                     listener.onTriangulateEnd(PROMedSRobustSinglePoint3DTriangulator.this);
411                                 }
412                             }
413 
414                             @Override
415                             public void onEstimateNextIteration(
416                                     final RobustEstimator<Point3D> estimator, final int iteration) {
417                                 if (listener != null) {
418                                     listener.onTriangulateNextIteration(
419                                             PROMedSRobustSinglePoint3DTriangulator.this, iteration);
420                                 }
421                             }
422 
423                             @Override
424                             public void onEstimateProgressChange(
425                                     final RobustEstimator<Point3D> estimator, final float progress) {
426                                 if (listener != null) {
427                                     listener.onTriangulateProgressChange(
428                                             PROMedSRobustSinglePoint3DTriangulator.this, progress);
429                                 }
430                             }
431 
432                             @Override
433                             public double[] getQualityScores() {
434                                 return qualityScores;
435                             }
436                         });
437 
438         try {
439             locked = true;
440             innerEstimator.setConfidence(confidence);
441             innerEstimator.setMaxIterations(maxIterations);
442             innerEstimator.setProgressDelta(progressDelta);
443             return innerEstimator.estimate();
444         } catch (final com.irurueta.numerical.LockedException e) {
445             throw new LockedException(e);
446         } catch (final com.irurueta.numerical.NotReadyException e) {
447             throw new NotReadyException(e);
448         } finally {
449             locked = false;
450         }
451     }
452 
453     /**
454      * Returns method being used for robust estimation.
455      *
456      * @return method being used for robust estimation.
457      */
458     @Override
459     public RobustEstimatorMethod getMethod() {
460         return RobustEstimatorMethod.PROMEDS;
461     }
462 
463     /**
464      * Sets quality scores corresponding to each provided view.
465      * This method is used internally and does not check whether instance is
466      * locked or not.
467      *
468      * @param qualityScores quality scores to be set.
469      * @throws IllegalArgumentException if provided quality scores length is
470      *                                  smaller than MINIMUM_SIZE.
471      */
472     private void internalSetQualityScores(final double[] qualityScores) {
473         if (qualityScores.length < MIN_REQUIRED_VIEWS) {
474             throw new IllegalArgumentException();
475         }
476 
477         this.qualityScores = qualityScores;
478     }
479 }