View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.sfm;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.geometry.estimators.LockedException;
23  import com.irurueta.geometry.estimators.NotReadyException;
24  import com.irurueta.numerical.robust.PROSACRobustEstimator;
25  import com.irurueta.numerical.robust.PROSACRobustEstimatorListener;
26  import com.irurueta.numerical.robust.RobustEstimator;
27  import com.irurueta.numerical.robust.RobustEstimatorException;
28  import com.irurueta.numerical.robust.RobustEstimatorMethod;
29  
30  import java.util.ArrayList;
31  import java.util.List;
32  
33  /**
34   * Robustly triangulates 3D points from matched 2D points and their
35   * corresponding cameras on several views using PROSAC algorithm.
36   */
37  public class PROSACRobustSinglePoint3DTriangulator extends RobustSinglePoint3DTriangulator {
38  
39      /**
40       * Constant defining default threshold to determine whether samples are
41       * inliers or not.
42       * By default, 1.0 is considered a good value for cases where 2D point
43       * measures are done on pixels, since typically the minimum resolution is 1
44       * pixel.
45       */
46      public static final double DEFAULT_THRESHOLD = 1.0;
47  
48      /**
49       * Minimum value that can be set as threshold.
50       * Threshold must be strictly greater than 0.0.
51       */
52      public static final double MIN_THRESHOLD = 0.0;
53  
54      /**
55       * Threshold to determine whether samples are inliers or not when testing
56       * possible estimation solutions.
57       * The threshold refers to the amount of projection error (i.e. distance of
58       * projected solution using each camera).
59       */
60      private double threshold;
61  
62      /**
63       * Quality scores corresponding to each provided view.
64       * The larger the score value the better the quality of the sample.
65       */
66      private double[] qualityScores;
67  
68      /**
69       * Constructor.
70       */
71      public PROSACRobustSinglePoint3DTriangulator() {
72          super();
73          threshold = DEFAULT_THRESHOLD;
74      }
75  
76      /**
77       * Constructor.
78       *
79       * @param listener listener to be notified of events such as when estimation
80       *                 starts, ends or its progress significantly changes.
81       */
82      public PROSACRobustSinglePoint3DTriangulator(final RobustSinglePoint3DTriangulatorListener listener) {
83          super(listener);
84          threshold = DEFAULT_THRESHOLD;
85      }
86  
87      /**
88       * Constructor.
89       *
90       * @param points  Matched 2D points. Each point in the list is assumed to be
91       *                projected by the corresponding camera in the list.
92       * @param cameras List of cameras associated to the matched 2D point on the
93       *                same position as the camera on the list.
94       * @throws IllegalArgumentException if provided lists don't have the same
95       *                                  length or their length is less than 2 views, which is the minimum
96       *                                  required to compute triangulation.
97       */
98      public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points, final List<PinholeCamera> cameras) {
99          super(points, cameras);
100         threshold = DEFAULT_THRESHOLD;
101     }
102 
103     /**
104      * Constructor.
105      *
106      * @param points   Matched 2D points. Each point in the list is assumed to be
107      *                 projected by the corresponding camera in the list.
108      * @param cameras  List of cameras associated to the matched 2D point on the
109      *                 same position as the camera on the list.
110      * @param listener listener to be notified of events such as when estimation
111      *                 starts, ends or its progress significantly changes.
112      * @throws IllegalArgumentException if provided lists don't have the same
113      *                                  length or their length is less than 2 views, which is the minimum
114      *                                  required to compute triangulation.
115      */
116     public PROSACRobustSinglePoint3DTriangulator(
117             final List<Point2D> points, final List<PinholeCamera> cameras,
118             final RobustSinglePoint3DTriangulatorListener listener) {
119         super(points, cameras, listener);
120         threshold = DEFAULT_THRESHOLD;
121     }
122 
123     /**
124      * Constructor.
125      *
126      * @param qualityScores quality scores corresponding to each provided view.
127      * @throws IllegalArgumentException if provided quality scores length is
128      *                                  smaller than required size (i.e. 2 views).
129      */
130     public PROSACRobustSinglePoint3DTriangulator(final double[] qualityScores) {
131         this();
132         internalSetQualityScores(qualityScores);
133     }
134 
135     /**
136      * Constructor.
137      *
138      * @param qualityScores quality scores corresponding to each provided view.
139      * @param listener      listener to be notified of events such as when estimation
140      *                      starts, ends or its progress significantly changes.
141      * @throws IllegalArgumentException if provided quality scores length is
142      *                                  smaller than required size (i.e. 2 views).
143      */
144     public PROSACRobustSinglePoint3DTriangulator(final double[] qualityScores,
145                                                  final RobustSinglePoint3DTriangulatorListener listener) {
146         this(listener);
147         internalSetQualityScores(qualityScores);
148     }
149 
150     /**
151      * Constructor.
152      *
153      * @param points        Matched 2D points. Each point in the list is assumed to be
154      *                      projected by the corresponding camera in the list.
155      * @param cameras       List of cameras associated to the matched 2D point on the
156      *                      same position as the camera on the list.
157      * @param qualityScores quality scores corresponding to each provided view.
158      * @throws IllegalArgumentException if provided lists or quality scores
159      *                                  don't have the same length or their length is less than 2 views,
160      *                                  which is the minimum required to compute triangulation.
161      */
162     public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points,
163                                                  final List<PinholeCamera> cameras,
164                                                  final double[] qualityScores) {
165         this(points, cameras);
166         internalSetQualityScores(qualityScores);
167     }
168 
169     /**
170      * Constructor.
171      *
172      * @param points        Matched 2D points. Each point in the list is assumed to be
173      *                      projected by the corresponding camera in the list.
174      * @param cameras       List of cameras associated to the matched 2D point on the
175      *                      same position as the camera on the list.
176      * @param qualityScores quality scores corresponding to each provided view.
177      * @param listener      listener to be notified of events such as when estimation
178      *                      starts, ends or its progress significantly changes.
179      * @throws IllegalArgumentException if provided lists or quality scores
180      *                                  don't have the same length or their length is less than 2 views,
181      *                                  which is the minimum required to compute triangulation.
182      */
183     public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points,
184                                                  final List<PinholeCamera> cameras,
185                                                  final double[] qualityScores,
186                                                  final RobustSinglePoint3DTriangulatorListener listener) {
187         this(points, cameras, listener);
188         internalSetQualityScores(qualityScores);
189     }
190 
191     /**
192      * Returns threshold to determine whether points are inliers or not when
193      * testing possible estimation solutions.
194      * The threshold refers to the amount of error (i.e. Euclidean distance) a
195      * possible solution has on projected 2D points.
196      *
197      * @return threshold to determine whether points are inliers or not when
198      * testing possible estimation solutions.
199      */
200     public double getThreshold() {
201         return threshold;
202     }
203 
204     /**
205      * Sets threshold to determine whether points are inliers or not when
206      * testing possible estimation solutions.
207      * The threshold refers to the amount of error (i.e. Euclidean distance) a
208      * possible solution has on projected 2D points.
209      *
210      * @param threshold threshold to be set.
211      * @throws IllegalArgumentException if provided value is equal or less than
212      *                                  zero.
213      * @throws LockedException          if robust estimator is locked because an
214      *                                  estimation is already in progress.
215      */
216     public void setThreshold(final double threshold) throws LockedException {
217         if (isLocked()) {
218             throw new LockedException();
219         }
220         if (threshold <= MIN_THRESHOLD) {
221             throw new IllegalArgumentException();
222         }
223         this.threshold = threshold;
224     }
225 
226     /**
227      * Returns quality scores corresponding to each provided view.
228      * The larger the score value the better the quality of the sampled view.
229      *
230      * @return quality scores corresponding to each view.
231      */
232     @Override
233     public double[] getQualityScores() {
234         return qualityScores;
235     }
236 
237     /**
238      * Sets quality scores corresponding to each provided view.
239      * The larger the score value the better the quality of the sampled view.
240      *
241      * @param qualityScores quality scores corresponding to each view.
242      * @throws LockedException          if robust estimator is locked because an
243      *                                  estimation is already in progress.
244      * @throws IllegalArgumentException if provided quality scores length is
245      *                                  smaller than MIN_REQUIRED_VIEWS (i.e. 2 views).
246      */
247     @Override
248     public void setQualityScores(final double[] qualityScores) throws LockedException {
249         if (isLocked()) {
250             throw new LockedException();
251         }
252         internalSetQualityScores(qualityScores);
253     }
254 
255     /**
256      * Indicates if triangulator is ready to start the 3D point triangulation.
257      * This is true when input data (i.e. 2D points, cameras and quality scores)
258      * are provided and a minimum of 2 views are available.
259      *
260      * @return true if estimator is ready, false otherwise.
261      */
262     @Override
263     public boolean isReady() {
264         return super.isReady() && qualityScores != null && qualityScores.length == points2D.size();
265     }
266 
267 
268     /**
269      * Triangulates provided matched 2D points being projected by each
270      * corresponding camera into a single 3D point.
271      * At least 2 matched 2D points and their corresponding 2 cameras are
272      * required to compute triangulation. If more views are provided, an
273      * averaged solution can be found.
274      *
275      * @return computed triangulated 3D point.
276      * @throws LockedException          if this instance is locked.
277      * @throws NotReadyException        if lists of points and cameras don't have the
278      *                                  same length or less than 2 views are provided.
279      * @throws RobustEstimatorException if estimation fails for any reason
280      *                                  (i.e. numerical instability, no solution available, etc).
281      */
282     @SuppressWarnings("DuplicatedCode")
283     @Override
284     public Point3D triangulate() throws LockedException, NotReadyException, RobustEstimatorException {
285         if (isLocked()) {
286             throw new LockedException();
287         }
288         if (!isReady()) {
289             throw new NotReadyException();
290         }
291 
292         final var innerEstimator = new PROSACRobustEstimator<Point3D>(new PROSACRobustEstimatorListener<>() {
293 
294             // point to be reused when computing residuals
295             private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
296 
297             // non-robust 3D point triangulator
298             private final SinglePoint3DTriangulator triangulator =
299                     SinglePoint3DTriangulator.create(useHomogeneousSolution
300                             ? Point3DTriangulatorType.LMSE_HOMOGENEOUS_TRIANGULATOR
301                             : Point3DTriangulatorType.LMSE_INHOMOGENEOUS_TRIANGULATOR);
302 
303             // subset of 2D points
304             private final List<Point2D> subsetPoints = new ArrayList<>();
305 
306             // subset of cameras
307             private final List<PinholeCamera> subsetCameras = new ArrayList<>();
308 
309             @Override
310             public double getThreshold() {
311                 return threshold;
312             }
313 
314             @Override
315             public int getTotalSamples() {
316                 return points2D.size();
317             }
318 
319             @Override
320             public int getSubsetSize() {
321                 return MIN_REQUIRED_VIEWS;
322             }
323 
324             @Override
325             public void estimatePreliminarSolutions(final int[] samplesIndices, final List<Point3D> solutions) {
326                 subsetPoints.clear();
327                 subsetPoints.add(points2D.get(samplesIndices[0]));
328                 subsetPoints.add(points2D.get(samplesIndices[1]));
329 
330                 subsetCameras.clear();
331                 subsetCameras.add(cameras.get(samplesIndices[0]));
332                 subsetCameras.add(cameras.get(samplesIndices[1]));
333 
334                 try {
335                     triangulator.setPointsAndCameras(subsetPoints, subsetCameras);
336                     final var triangulated = triangulator.triangulate();
337                     solutions.add(triangulated);
338                 } catch (final Exception e) {
339                     // if anything fails, no solution is added
340                 }
341             }
342 
343             @Override
344             public double computeResidual(final Point3D currentEstimation, final int i) {
345                 final var point2D = points2D.get(i);
346                 final var camera = cameras.get(i);
347 
348                 // project estimated point with camera
349                 camera.project(currentEstimation, testPoint);
350 
351                 // return distance of projected point respect to the original one
352                 // as a residual
353                 return testPoint.distanceTo(point2D);
354             }
355 
356             @Override
357             public boolean isReady() {
358                 return PROSACRobustSinglePoint3DTriangulator.this.isReady();
359             }
360 
361             @Override
362             public void onEstimateStart(final RobustEstimator<Point3D> estimator) {
363                 if (listener != null) {
364                     listener.onTriangulateStart(PROSACRobustSinglePoint3DTriangulator.this);
365                 }
366             }
367 
368             @Override
369             public void onEstimateEnd(final RobustEstimator<Point3D> estimator) {
370                 if (listener != null) {
371                     listener.onTriangulateEnd(PROSACRobustSinglePoint3DTriangulator.this);
372                 }
373             }
374 
375             @Override
376             public void onEstimateNextIteration(final RobustEstimator<Point3D> estimator, final int iteration) {
377                 if (listener != null) {
378                     listener.onTriangulateNextIteration(
379                             PROSACRobustSinglePoint3DTriangulator.this, iteration);
380                 }
381             }
382 
383             @Override
384             public void onEstimateProgressChange(final RobustEstimator<Point3D> estimator, final float progress) {
385                 if (listener != null) {
386                     listener.onTriangulateProgressChange(
387                             PROSACRobustSinglePoint3DTriangulator.this, progress);
388                 }
389             }
390 
391             @Override
392             public double[] getQualityScores() {
393                 return qualityScores;
394             }
395         });
396 
397         try {
398             locked = true;
399             innerEstimator.setConfidence(confidence);
400             innerEstimator.setMaxIterations(maxIterations);
401             innerEstimator.setProgressDelta(progressDelta);
402             return innerEstimator.estimate();
403         } catch (final com.irurueta.numerical.LockedException e) {
404             throw new LockedException(e);
405         } catch (final com.irurueta.numerical.NotReadyException e) {
406             throw new NotReadyException(e);
407         } finally {
408             locked = false;
409         }
410     }
411 
412     /**
413      * Returns method being used for robust estimation.
414      *
415      * @return method being used for robust estimation.
416      */
417     @Override
418     public RobustEstimatorMethod getMethod() {
419         return RobustEstimatorMethod.PROSAC;
420     }
421 
422     /**
423      * Sets quality scores corresponding to each provided view.
424      * This method is used internally and does not check whether instance is
425      * locked or not.
426      *
427      * @param qualityScores quality scores to be set.
428      * @throws IllegalArgumentException if provided quality scores length is
429      *                                  smaller than MINIMUM_SIZE.
430      */
431     private void internalSetQualityScores(final double[] qualityScores) {
432         if (qualityScores.length < MIN_REQUIRED_VIEWS) {
433             throw new IllegalArgumentException();
434         }
435 
436         this.qualityScores = qualityScores;
437     }
438 }