View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.sfm;
17  
18  import com.irurueta.geometry.CoordinatesType;
19  import com.irurueta.geometry.PinholeCamera;
20  import com.irurueta.geometry.Point2D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.geometry.estimators.LockedException;
23  import com.irurueta.geometry.estimators.NotReadyException;
24  
25  import java.util.List;
26  
27  /**
28   * Base class to triangulate matched 2D points into a single 3D one by using
29   * 2D points correspondences on different views along with the corresponding
30   * cameras on each of those views.
31   * Subclasses will implement different types of triangulators that can provide
32   * either LMSE or weighted solutions using either homogeneous or inhomogeneous
33   * systems of equations.
34   * Inhomogeneous methods are suitable only for cases where finite points and
35   * cameras are being used. If points or cameras are located very far or at
36   * infinity, triangulation will fail when using inhomogeneous methods.
37   * Homogeneous methods are suitable for any case, however, if points and
38   * cameras are close and well-defined, inhomogeneous methods might yield better
39   * accuracy (although the difference is minimal).
40   */
41  public abstract class SinglePoint3DTriangulator {
42  
43      /**
44       * Default triangulator type.
45       */
46      public static final Point3DTriangulatorType DEFAULT_TYPE = Point3DTriangulatorType.LMSE_HOMOGENEOUS_TRIANGULATOR;
47  
48      /**
49       * Minimum required number of views to triangulate 3D points.
50       */
51      public static final int MIN_REQUIRED_VIEWS = 2;
52  
53      /**
54       * Matched 2D points. Each point in the list is assumed to be projected by
55       * the corresponding camera in the list.
56       */
57      protected List<Point2D> points2D;
58  
59      /**
60       * List of cameras associated to the matched 2D point on the same position
61       * as the camera on the list.
62       */
63      protected List<PinholeCamera> cameras;
64  
65      /**
66       * Listener to handle events generated by instances of this class.
67       */
68      protected SinglePoint3DTriangulatorListener listener;
69  
70      /**
71       * Indicates whether this instance is locked doing computations.
72       */
73      protected boolean locked;
74  
75      /**
76       * Constructor.
77       */
78      protected SinglePoint3DTriangulator() {
79      }
80  
81      /**
82       * Constructor.
83       *
84       * @param points2D list of matched 2D points on each view. Each point in the
85       *                 list is assumed to be projected by the corresponding camera in the list.
86       * @param cameras  camera for each view where 2D points are represented.
87       * @throws IllegalArgumentException if provided lists don't have the same
88       *                                  length or their length is less than 2 views, which is the minimum
89       *                                  required to compute triangulation.
90       */
91      protected SinglePoint3DTriangulator(final List<Point2D> points2D, final List<PinholeCamera> cameras) {
92          internalSetPointsAndCameras(points2D, cameras);
93      }
94  
95      /**
96       * Constructor.
97       *
98       * @param listener listener to notify events generated by instances of this
99       *                 class.
100      */
101     protected SinglePoint3DTriangulator(final SinglePoint3DTriangulatorListener listener) {
102         this.listener = listener;
103     }
104 
105     /**
106      * Constructor.
107      *
108      * @param points2D list of matched 2D points on each view. Each point in the
109      *                 list is assumed to be projected by the corresponding camera in the list.
110      * @param cameras  cameras for each view where 2D points are represented.
111      * @param listener listener to notify events generated by instances of this
112      *                 class.
113      * @throws IllegalArgumentException if provided lists don't have the same
114      *                                  length or their length is less than 2 views, which is the minimum
115      *                                  required to compute triangulation.
116      */
117     protected SinglePoint3DTriangulator(final List<Point2D> points2D, final List<PinholeCamera> cameras,
118                                         final SinglePoint3DTriangulatorListener listener) {
119         this(points2D, cameras);
120         this.listener = listener;
121     }
122 
123     /**
124      * Returns list of matched 2D points on each view. Each point in the list is
125      * assumed to be projected by the corresponding camera.
126      *
127      * @return list of matched 2D points on each view.
128      */
129     public List<Point2D> getPoints2D() {
130         return points2D;
131     }
132 
133     /**
134      * Returns cameras for each view where 2D points are represented.
135      *
136      * @return cameras for each view where 2D points are represented.
137      */
138     public List<PinholeCamera> getCameras() {
139         return cameras;
140     }
141 
142     /**
143      * Sets list of matched 2D points for each view and their corresponding
144      * cameras used to project them.
145      *
146      * @param points2D list of matched 2D points on each view. Each point in the
147      *                 list is assumed to be projected by the corresponding camera in the list.
148      * @param cameras  cameras for each view where 2D points are represented.
149      * @throws LockedException          if this instance is locked.
150      * @throws IllegalArgumentException if provided lists don't have the same
151      *                                  length or their length is less than 2 views, which is the minimum
152      *                                  required to compute triangulation.
153      */
154     public void setPointsAndCameras(final List<Point2D> points2D, final List<PinholeCamera> cameras)
155             throws LockedException {
156         if (isLocked()) {
157             throw new LockedException();
158         }
159         internalSetPointsAndCameras(points2D, cameras);
160     }
161 
162     /**
163      * Indicates whether this instance is locked because computations are being
164      * done.
165      *
166      * @return true if instance is locked, false otherwise.
167      */
168     public boolean isLocked() {
169         return locked;
170     }
171 
172     /**
173      * Indicates whether this instance is ready to start the triangulation.
174      * An instance is ready when both lists of 2D points and cameras are
175      * provided, both lists have the same length and at least data for 2 views
176      * is provided.
177      *
178      * @return true if this instance is ready, false otherwise.
179      */
180     public boolean isReady() {
181         return areValidPointsAndCameras(points2D, cameras);
182     }
183 
184     /**
185      * Returns listener to be notified of events generated by instances of this
186      * class.
187      *
188      * @return listener to be notified of events generated by instances of this
189      * class.
190      */
191     public SinglePoint3DTriangulatorListener getListener() {
192         return listener;
193     }
194 
195     /**
196      * Sets listener to be notified of events generated by instances of this
197      * class.
198      *
199      * @param listener listener to be notified of events generated by instances
200      *                 of this class.
201      * @throws LockedException if this instance is locked.
202      */
203     public void setListener(final SinglePoint3DTriangulatorListener listener) throws LockedException {
204         if (isLocked()) {
205             throw new LockedException();
206         }
207 
208         this.listener = listener;
209     }
210 
211     /**
212      * Triangulates provided matched 2D points being projected by each
213      * corresponding camera into a single 3D point.
214      * At least 2 matched 2D points and their corresponding 2 cameras are
215      * required to compute triangulation. If more views are provided, an
216      * averaged solution can be found.
217      *
218      * @return computed triangulated 3D point.
219      * @throws LockedException               if this instance is locked.
220      * @throws NotReadyException             if lists of points and cameras don't have the
221      *                                       same length or less than 2 views are provided.
222      * @throws Point3DTriangulationException if triangulation fails for some
223      *                                       other reason (i.e. degenerate geometry, numerical
224      *                                       instabilities, etc.).
225      */
226     public Point3D triangulate() throws LockedException, NotReadyException, Point3DTriangulationException {
227         final var result = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
228         triangulate(result);
229         return result;
230     }
231 
232     /**
233      * Triangulates provided matched 2D points being projected by each
234      * corresponding camera into a single 3D point.
235      * At least 2 matched 2D points and their corresponding 2 cameras are
236      * required to compute triangulation. If more views are provided, an
237      * averaged solution can be found.
238      *
239      * @param result instance where data for triangulated 3D point is stored.
240      * @throws LockedException               if this instance is locked.
241      * @throws NotReadyException             if lists of points and cameras don't have the
242      *                                       same length or less than 2 views are provided.
243      * @throws Point3DTriangulationException if triangulation fails for some
244      *                                       other reason (i.e. degenerate geometry, numerical
245      *                                       instabilities, etc.).
246      */
247     public void triangulate(final Point3D result) throws LockedException, NotReadyException,
248             Point3DTriangulationException {
249         if (isLocked()) {
250             throw new LockedException();
251         }
252         if (!isReady()) {
253             throw new NotReadyException();
254         }
255 
256         triangulate(points2D, cameras, result);
257     }
258 
259     /**
260      * Indicates whether provided points and cameras are valid to start the
261      * triangulation.
262      * In order to triangulate points, at least two cameras and their
263      * corresponding 2 matched 2D points are required.
264      * If more views are provided, an averaged solution can be found.
265      *
266      * @param points2D list of matched points on each view.
267      * @param cameras  cameras for each view where 2D points are represented.
268      * @return true if data is enough to start triangulation, false otherwise.
269      */
270     public static boolean areValidPointsAndCameras(final List<Point2D> points2D, final List<PinholeCamera> cameras) {
271         return points2D != null && cameras != null && points2D.size() == cameras.size()
272                 && points2D.size() >= MIN_REQUIRED_VIEWS;
273     }
274 
275     /**
276      * Creates a new 3D point triangulator instance using provided type.
277      *
278      * @param type a triangulator type.
279      * @return a 3D point triangulator instance.
280      */
281     public static SinglePoint3DTriangulator create(final Point3DTriangulatorType type) {
282         return switch (type) {
283             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator();
284             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator();
285             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator();
286             default -> new LMSEHomogeneousSinglePoint3DTriangulator();
287         };
288     }
289 
290     /**
291      * Creates a new 3D point triangulator instance using provided lists of
292      * points and corresponding cameras along with provided type.
293      *
294      * @param points2D list of matched 2D points on each view. Each point in the
295      *                 list is assumed to be projected by the corresponding camera in the list.
296      * @param cameras  camera for each view where 2D points are represented.
297      * @param type     a triangulator type.
298      * @return a 3D point triangulator instance.
299      * @throws IllegalArgumentException if provided lists don't have the same
300      *                                  length or their length is less than 2 views, which is the minimum
301      *                                  required to compute triangulation.
302      */
303     public static SinglePoint3DTriangulator create(
304             final List<Point2D> points2D, final List<PinholeCamera> cameras, final Point3DTriangulatorType type) {
305         return switch (type) {
306             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator(points2D,
307                     cameras);
308             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator(points2D,
309                     cameras);
310             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator(points2D,
311                     cameras);
312             default -> new LMSEHomogeneousSinglePoint3DTriangulator(points2D, cameras);
313         };
314     }
315 
316     /**
317      * Creates a new 3D point triangulator instance using provided lists of
318      * points, weights and corresponding cameras along with provided type.
319      *
320      * @param points2D list of matched 2D points on each view. Each point in the
321      *                 list is assumed to be projected by the corresponding camera in the list.
322      * @param cameras  camera for each view where 2D points are represented.
323      * @param weights  weights assigned to each view.
324      * @param type     a triangulator type.
325      * @return a 3D point triangulator instance.
326      * @throws IllegalArgumentException if provided lists or weights don't have
327      *                                  the same length or their length is less than 2 views, which is the
328      *                                  minimum required to compute triangulation.
329      */
330     public static SinglePoint3DTriangulator create(
331             final List<Point2D> points2D, final List<PinholeCamera> cameras, final double[] weights,
332             final Point3DTriangulatorType type) {
333         return switch (type) {
334             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator(points2D,
335                     cameras, weights);
336             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator(points2D,
337                     cameras, weights);
338             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator(points2D, cameras);
339             default -> new LMSEHomogeneousSinglePoint3DTriangulator(points2D, cameras);
340         };
341     }
342 
343     /**
344      * Creates a new 3D point triangulator instance using provided listener and
345      * type.
346      *
347      * @param listener listener to notify events generated by instances of this
348      *                 class.
349      * @param type     a triangulator type.
350      * @return a 3D point triangulator instance.
351      */
352     public static SinglePoint3DTriangulator create(
353             final SinglePoint3DTriangulatorListener listener, final Point3DTriangulatorType type) {
354         return switch (type) {
355             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator(listener);
356             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator(listener);
357             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator(listener);
358             default -> new LMSEHomogeneousSinglePoint3DTriangulator(listener);
359         };
360     }
361 
362     /**
363      * Creates a new 3D point triangulator instance using provided lists of
364      * points and corresponding cameras, listener and provided type.
365      *
366      * @param points2D list of matched 2D points on each view. Each point in the
367      *                 list is assumed to be projected by the corresponding camera in the list.
368      * @param cameras  camera for each view where 2D points are represented.
369      * @param listener listener to notify events generated by instances of this
370      *                 class.
371      * @param type     a triangulator type.
372      * @return a 3D point triangulator instance.
373      * @throws IllegalArgumentException if provided lists don't have the same
374      *                                  length or their length is less than 2 views, which is the minimum
375      *                                  required to compute triangulation.
376      */
377     public static SinglePoint3DTriangulator create(
378             final List<Point2D> points2D, final List<PinholeCamera> cameras,
379             final SinglePoint3DTriangulatorListener listener, final Point3DTriangulatorType type) {
380         return switch (type) {
381             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator(points2D,
382                     cameras, listener);
383             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator(points2D,
384                     cameras, listener);
385             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator(points2D, cameras,
386                     listener);
387             default -> new LMSEHomogeneousSinglePoint3DTriangulator(points2D, cameras, listener);
388         };
389     }
390 
391     /**
392      * Creates a new 3D point triangulator instance using provided lists of
393      * points, weights, corresponding cameras, listener and provided type.
394      *
395      * @param points2D list of matched 2D points on each view. Each point in the
396      *                 list is assumed to be projected by the corresponding camera in the list.
397      * @param cameras  camera for each view where 2D points are represented.
398      * @param weights  weights assigned to each view.
399      * @param listener listener to notify events generated by instances of this
400      *                 class.
401      * @param type     a triangulator type.
402      * @return a 3D point triangulator instance.
403      * @throws IllegalArgumentException if provided lists or weights don't have
404      *                                  the same length or their length is less than 2 views, which is the
405      *                                  minimum required to compute triangulation.
406      */
407     public static SinglePoint3DTriangulator create(
408             final List<Point2D> points2D, final List<PinholeCamera> cameras, final double[] weights,
409             final SinglePoint3DTriangulatorListener listener, final Point3DTriangulatorType type) {
410         return switch (type) {
411             case WEIGHTED_INHOMOGENEOUS_TRIANGULATOR -> new WeightedInhomogeneousSinglePoint3DTriangulator(points2D,
412                     cameras, weights, listener);
413             case WEIGHTED_HOMOGENEOUS_TRIANGULATOR -> new WeightedHomogeneousSinglePoint3DTriangulator(points2D,
414                     cameras, weights, listener);
415             case LMSE_INHOMOGENEOUS_TRIANGULATOR -> new LMSEInhomogeneousSinglePoint3DTriangulator(points2D, cameras,
416                     listener);
417             default -> new LMSEHomogeneousSinglePoint3DTriangulator(points2D, cameras, listener);
418         };
419     }
420 
421     /**
422      * Creates a new 3D point triangulator instance using default type.
423      *
424      * @return a 3D point triangulator instance.
425      */
426     public static SinglePoint3DTriangulator create() {
427         return create(DEFAULT_TYPE);
428     }
429 
430     /**
431      * Creates a new 3D point triangulator instance using provided lists of
432      * points and corresponding cameras along with default type.
433      *
434      * @param points2D list of matched 2D points on each view. Each point in the
435      *                 list is assumed to be projected by the corresponding camera in the list.
436      * @param cameras  camera for each view where 2D points are represented.
437      * @return a 3D point triangulator instance.
438      * @throws IllegalArgumentException if provided lists don't have the same
439      *                                  length or their length is less than 2 views, which is the minimum
440      *                                  required to compute triangulation.
441      */
442     public static SinglePoint3DTriangulator create(final List<Point2D> points2D, final List<PinholeCamera> cameras) {
443         return create(points2D, cameras, DEFAULT_TYPE);
444     }
445 
446     /**
447      * Creates a new 3D point triangulator instance using provided lists of
448      * points, weights and corresponding cameras along with default type.
449      *
450      * @param points2D list of matched 2D points on each view. Each point in the
451      *                 list is assumed to be projected by the corresponding camera in the list.
452      * @param cameras  camera for each view where 2D points are represented.
453      * @param weights  weights assigned to each view.
454      * @return a 3D point triangulator instance.
455      * @throws IllegalArgumentException if provided lists or weights don't have
456      *                                  the same length or their length is less than 2 views, which is the
457      *                                  minimum required to compute triangulation.
458      */
459     public static SinglePoint3DTriangulator create(
460             final List<Point2D> points2D, final List<PinholeCamera> cameras, double[] weights) {
461         return create(points2D, cameras, weights, DEFAULT_TYPE);
462     }
463 
464     /**
465      * Creates a new 3D point triangulator instance using provided listener and
466      * default type.
467      *
468      * @param listener listener to notify events generated by instances of this
469      *                 class.
470      * @return a 3D point triangulator instance.
471      */
472     public static SinglePoint3DTriangulator create(final SinglePoint3DTriangulatorListener listener) {
473         return create(listener, DEFAULT_TYPE);
474     }
475 
476     /**
477      * Creates a new 3D point triangulator instance using provided lists of
478      * points and corresponding cameras, listener and default type.
479      *
480      * @param points2D list of matched 2D points on each view. Each point in the
481      *                 list is assumed to be projected by the corresponding camera in the list.
482      * @param cameras  camera for each view where 2D points are represented.
483      * @param listener listener to notify events generated by instances of this
484      *                 class.
485      * @return a 3D point triangulator instance.
486      * @throws IllegalArgumentException if provided lists don't have the same
487      *                                  length or their length is less than 2 views, which is the minimum
488      *                                  required to compute triangulation.
489      */
490     public static SinglePoint3DTriangulator create(
491             final List<Point2D> points2D, final List<PinholeCamera> cameras,
492             final SinglePoint3DTriangulatorListener listener) {
493         return create(points2D, cameras, listener, DEFAULT_TYPE);
494     }
495 
496     /**
497      * Creates a new 3D point triangulator instance using provided lists of
498      * points, weights, corresponding cameras, listener and default type.
499      *
500      * @param points2D list of matched 2D points on each view. Each point in the
501      *                 list is assumed to be projected by the corresponding camera in the list.
502      * @param cameras  camera for each view where 2D points are represented.
503      * @param weights  weights assigned to each view.
504      * @param listener listener to notify events generated by instances of this
505      *                 class.
506      * @return a 3D point triangulator instance.
507      * @throws IllegalArgumentException if provided lists or weights don't have
508      *                                  the same length or their length is less than 2 views, which is the
509      *                                  minimum required to compute triangulation.
510      */
511     public static SinglePoint3DTriangulator create(final List<Point2D> points2D,
512                                                    final List<PinholeCamera> cameras,
513                                                    final double[] weights,
514                                                    final SinglePoint3DTriangulatorListener listener) {
515         return create(points2D, cameras, weights, listener, DEFAULT_TYPE);
516     }
517 
518     /**
519      * Returns type of triangulator (a combination of homogeneous or
520      * inhomogeneous type along with an LMSE or weighted strategy).
521      *
522      * @return type of triangulator.
523      */
524     public abstract Point3DTriangulatorType getType();
525 
526     /**
527      * Internal method to triangulate provided matched 2D points being projected
528      * by each corresponding camera into a single 3D point.
529      * At least 2 matched 2D points and their corresponding 2 cameras are
530      * required to compute triangulation. If more views are provided, an
531      * averaged solution is found.
532      * This method does not check whether instance is locked or ready.
533      *
534      * @param points2D matched 2D points. Each point in the list is assumed to
535      *                 be projected by the corresponding camera in the list.
536      * @param cameras  list of cameras associated to the matched 2D point on the
537      *                 same position as the camera on the list.
538      * @param result   instance where triangulated 3D point is stored.
539      * @throws Point3DTriangulationException if triangulation fails for some
540      *                                       other reason (i.e. degenerate geometry, numerical
541      *                                       instabilities, etc.).
542      */
543     protected abstract void triangulate(
544             final List<Point2D> points2D, final List<PinholeCamera> cameras, final Point3D result)
545             throws Point3DTriangulationException;
546 
547     /**
548      * Internal method to sets list of matched 2D points for each view and their
549      * corresponding cameras used to project them.
550      * This method does not check whether instance is locked.
551      *
552      * @param points2D list of matched 2D points on each view. Each point in the
553      *                 list is assumed to be projected by the corresponding camera in the list.
554      * @param cameras  cameras for each view where 2D points are represented.
555      * @throws IllegalArgumentException if provided lists don't have the same
556      *                                  length or their length is less than 2 views, which is the minimum
557      *                                  required to compute triangulation.
558      */
559     private void internalSetPointsAndCameras(final List<Point2D> points2D, final List<PinholeCamera> cameras) {
560         if (!areValidPointsAndCameras(points2D, cameras)) {
561             throw new IllegalArgumentException();
562         }
563 
564         this.points2D = points2D;
565         this.cameras = cameras;
566     }
567 }