View Javadoc
1   /*
2    * Copyright (C) 2013 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.estimators;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.algebra.Utils;
20  import com.irurueta.algebra.WrongSizeException;
21  import com.irurueta.geometry.HomogeneousPoint2D;
22  import com.irurueta.geometry.NotAvailableException;
23  import com.irurueta.geometry.PinholeCamera;
24  import com.irurueta.geometry.Point2D;
25  import com.irurueta.geometry.Point3D;
26  import com.irurueta.geometry.ProjectiveTransformation2D;
27  import com.irurueta.geometry.ProjectiveTransformation3D;
28  import com.irurueta.geometry.refiners.DecomposedPointCorrespondencePinholeCameraRefiner;
29  
30  import java.util.ArrayList;
31  import java.util.BitSet;
32  import java.util.Collections;
33  import java.util.List;
34  
35  /**
36   * This file contains abstract implementation for pinhole camera estimators
37   * based on point correspondences.
38   */
39  @SuppressWarnings("DuplicatedCode")
40  public abstract class PointCorrespondencePinholeCameraEstimator extends PinholeCameraEstimator {
41  
42      /**
43       * Minimum number of required point correspondences to estimate a pinhole
44       * camera.
45       */
46      public static final int MIN_NUMBER_OF_POINT_CORRESPONDENCES = 6;
47  
48      /**
49       * Indicates if by default provided point correspondences are normalized to
50       * increase the accuracy of the estimation.
51       */
52      public static final boolean DEFAULT_NORMALIZE_POINT_CORRESPONDENCES = true;
53  
54      /**
55       * Defines tiny value considered as machine precision.
56       */
57      public static final double EPS = 1e-8;
58  
59      /**
60       * List of corresponding 3D points.
61       */
62      protected List<Point3D> points3D;
63  
64      /**
65       * List of corresponding 2D points.
66       */
67      protected List<Point2D> points2D;
68  
69      /**
70       * Indicates if provided point correspondences are normalized to increase
71       * the accuracy of the estimation.
72       */
73      private boolean normalizePointCorrespondences;
74  
75      /**
76       * Constructor.
77       */
78      protected PointCorrespondencePinholeCameraEstimator() {
79          super();
80          normalizePointCorrespondences = DEFAULT_NORMALIZE_POINT_CORRESPONDENCES;
81      }
82  
83      /**
84       * Constructor.
85       *
86       * @param listener listener to be notified of events such as when estimation
87       *                 starts, ends or estimation progress changes.
88       */
89      protected PointCorrespondencePinholeCameraEstimator(final PinholeCameraEstimatorListener listener) {
90          super(listener);
91          normalizePointCorrespondences = DEFAULT_NORMALIZE_POINT_CORRESPONDENCES;
92      }
93  
94      /**
95       * Constructor.
96       *
97       * @param points3D list of corresponding 3D points.
98       * @param points2D list of corresponding 2D points.
99       * @throws IllegalArgumentException if any of the lists are null.
100      * @throws WrongListSizesException  if provided lists of points don't have
101      *                                  the same size and enough points.
102      */
103     protected PointCorrespondencePinholeCameraEstimator(final List<Point3D> points3D, final List<Point2D> points2D)
104             throws WrongListSizesException {
105         super();
106         internalSetLists(points3D, points2D);
107         normalizePointCorrespondences = DEFAULT_NORMALIZE_POINT_CORRESPONDENCES;
108     }
109 
110     /**
111      * Constructor.
112      *
113      * @param points3D list of corresponding 3D points.
114      * @param points2D list of corresponding 2D points.
115      * @param listener listener to be notified of events such as when estimation
116      *                 starts, ends or estimation progress changes.
117      * @throws IllegalArgumentException if any of the lists are null.
118      * @throws WrongListSizesException  if provided lists of points don't have
119      *                                  the same size and enough points.
120      */
121     protected PointCorrespondencePinholeCameraEstimator(
122             final List<Point3D> points3D, final List<Point2D> points2D, final PinholeCameraEstimatorListener listener)
123             throws WrongListSizesException {
124         super(listener);
125         internalSetLists(points3D, points2D);
126         normalizePointCorrespondences = DEFAULT_NORMALIZE_POINT_CORRESPONDENCES;
127     }
128 
129     /**
130      * Internal method to set list of corresponding points (it does not check
131      * if estimator is locked).
132      *
133      * @param points3D list of corresponding 3D points.
134      * @param points2D list of corresponding 2D points.
135      * @throws IllegalArgumentException if any of the lists are null.
136      * @throws WrongListSizesException  if provided lists of points don't have
137      *                                  the same size and enough points.
138      */
139     private void internalSetLists(final List<Point3D> points3D, final List<Point2D> points2D)
140             throws WrongListSizesException {
141 
142         if (points3D == null || points2D == null) {
143             throw new IllegalArgumentException();
144         }
145 
146         if (!areValidLists(points3D, points2D)) {
147             throw new WrongListSizesException();
148         }
149 
150         this.points3D = points3D;
151         this.points2D = points2D;
152     }
153 
154     /**
155      * Sets list of corresponding points.
156      *
157      * @param points3D list of corresponding 3D points.
158      * @param points2D list of corresponding 2D points.
159      * @throws LockedException          if estimator is locked.
160      * @throws IllegalArgumentException if any of the lists are null.
161      * @throws WrongListSizesException  if provided lists of points don't have
162      *                                  the same size and enough points.
163      */
164     public void setLists(final List<Point3D> points3D, final List<Point2D> points2D) throws LockedException,
165             WrongListSizesException {
166         if (isLocked()) {
167             throw new LockedException();
168         }
169 
170         internalSetLists(points3D, points2D);
171     }
172 
173     /**
174      * Returns list of corresponding 3D points.
175      * Notice that this method returns an unmodifiable list of points to avoid
176      * undesired modifications.
177      *
178      * @return list of corresponding 3D points.
179      * @throws NotAvailableException if list of points is not yet available.
180      */
181     public List<Point3D> getPoints3D() throws NotAvailableException {
182         if (points3D == null) {
183             throw new NotAvailableException();
184         }
185 
186         // to avoid undesired modifications
187         return Collections.unmodifiableList(points3D);
188     }
189 
190     /**
191      * Returns list of corresponding 2D points.
192      * Notice that this method returns an unmodifiable list of points to avoid
193      * undesired modifications.
194      *
195      * @return list of corresponding 2D points.
196      * @throws NotAvailableException if list of points is not yet available.
197      */
198     public List<Point2D> getPoints2D() throws NotAvailableException {
199         if (points2D == null) {
200             throw new NotAvailableException();
201         }
202 
203         // to avoid undesired modifications
204         return Collections.unmodifiableList(points2D);
205     }
206 
207     /**
208      * Indicates if lists of corresponding 2D/3D points are valid.
209      * Lists are considered valid if they have the same number of points and
210      * both have more than the required minimum of correspondences (which is 6).
211      *
212      * @param points3D list of corresponding 3D points.
213      * @param points2D list of corresponding 2D points.
214      * @return true if corresponding 2D/3D points are valid, false otherwise.
215      */
216     public static boolean areValidLists(final List<Point3D> points3D, final List<Point2D> points2D) {
217         if (points3D == null || points2D == null) {
218             return false;
219         }
220         return points3D.size() == points2D.size() && points3D.size() >= MIN_NUMBER_OF_POINT_CORRESPONDENCES;
221     }
222 
223     /**
224      * Indicates if lists have already been provided and are available for
225      * retrieval.
226      *
227      * @return true if available, false otherwise.
228      */
229     public boolean areListsAvailable() {
230         return points3D != null && points2D != null;
231     }
232 
233     /**
234      * Indicates if provided point correspondences are normalized to increase
235      * the accuracy of the estimation.
236      *
237      * @return true if input point correspondences will be normalized, false
238      * otherwise.
239      */
240     public boolean arePointCorrespondencesNormalized() {
241         return normalizePointCorrespondences;
242     }
243 
244     /**
245      * Specifies whether provided point correspondences are normalized to
246      * increase the accuracy of the estimation.
247      *
248      * @param normalize true if input point correspondences will be normalized,
249      *                  false otherwise.
250      * @throws LockedException if estimator is locked.
251      */
252     public void setPointCorrespondencesNormalized(final boolean normalize) throws LockedException {
253 
254         if (isLocked()) {
255             throw new LockedException();
256         }
257         normalizePointCorrespondences = normalize;
258     }
259 
260     /**
261      * Transforms 2D points so that they have zero mean and unitary standard
262      * deviation. Provided transformation will be updated containing the
263      * transformation used for input points.
264      *
265      * @param list                  list of input 2D points.
266      * @param inverseTransformation inverseTransformation used on input points.
267      *                              This is an output variable.
268      * @return transformed 2D points.
269      * @throws PinholeCameraEstimatorException if transformation cannot be
270      *                                         computed because point configuration might be degenerate.
271      */
272     private List<Point2D> transformPoints2D(
273             final List<Point2D> list, final ProjectiveTransformation2D inverseTransformation)
274             throws PinholeCameraEstimatorException {
275 
276         // compute image point coordinates limits
277         var minX = Double.MAX_VALUE;
278         var maxX = -Double.MAX_VALUE;
279         var minY = Double.MAX_VALUE;
280         var maxY = -Double.MAX_VALUE;
281         for (final var point : list) {
282             point.normalize();
283             final var x = point.getInhomX();
284             final var y = point.getInhomY();
285 
286             if (x < minX) {
287                 minX = x;
288             }
289             if (x > maxX) {
290                 maxX = x;
291             }
292             if (y < minY) {
293                 minY = y;
294             }
295             if (y > maxY) {
296                 maxY = y;
297             }
298         }
299 
300         // compute size of image plane (based on 2D coordinates limits), scale
301         // and centroid
302         final var width = maxX - minX;
303         final var height = maxY - minY;
304 
305         final var norm = Math.sqrt(width * width + height * height);
306         // points are too close to each other (norm is too small).
307         // This is a degenerate configuration and results will be meaningless
308         if (norm < EPS) {
309             throw new PinholeCameraEstimatorException();
310         }
311 
312         final var scale = 1.0 / norm;
313         final var centroidX = (minX + maxX) / 2.0;
314         final var centroidY = (minY + maxY) / 2.0;
315 
316         // set inverse transformation matrix
317         try {
318             final var inverseTransformationMatrix = new Matrix(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
319                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
320 
321             inverseTransformationMatrix.setElementAt(0, 0, norm);
322             // 1.0 / scale
323             inverseTransformationMatrix.setElementAt(1, 1, norm);
324             inverseTransformationMatrix.setElementAt(2, 2, 1.0);
325             inverseTransformationMatrix.setElementAt(0, 2, centroidX);
326             inverseTransformationMatrix.setElementAt(1, 2, centroidY);
327 
328             inverseTransformation.setT(inverseTransformationMatrix);
329         } catch (final WrongSizeException ignore) {
330             // never happens
331         }
332 
333         inverseTransformation.normalize();
334 
335 
336         // transform list of image points
337         final var transformedPoints = new ArrayList<Point2D>(list.size());
338         for (final var point : list) {
339             point.normalize();
340             final var homX = point.getHomX();
341             final var homY = point.getHomY();
342             final var homW = point.getHomW();
343 
344             final var homPoint = new HomogeneousPoint2D(
345                     scale * (homX - centroidX * homW),
346                     scale * (homY - centroidY * homW), homW);
347             // normalize point to increase accuracy
348             homPoint.normalize();
349             transformedPoints.add(homPoint);
350         }
351 
352         return transformedPoints;
353     }
354 
355     /**
356      * Transforms 3D points so that they have zero mean and unitary standard
357      * deviation. Provided transformation will be updated containing the
358      * transformation used for input points.
359      *
360      * @param list           list of input 3D points.
361      * @param transformation transformation used on input points. This is an
362      *                       output variable.
363      * @return transformed 3D points.
364      * @throws PinholeCameraEstimatorException if transformation cannot be
365      *                                         computed because point configuration might be degenerate.
366      */
367     private List<Point3D> transformPoints3D(final List<Point3D> list, final ProjectiveTransformation3D transformation)
368             throws PinholeCameraEstimatorException {
369 
370         // compute image point coordinates limits
371         var minX = Double.MAX_VALUE;
372         var maxX = -Double.MAX_VALUE;
373         var minY = Double.MAX_VALUE;
374         var maxY = -Double.MAX_VALUE;
375         var minZ = Double.MAX_VALUE;
376         var maxZ = -Double.MAX_VALUE;
377         for (final var point : list) {
378             point.normalize();
379             final var x = point.getInhomX();
380             final var y = point.getInhomY();
381             final var z = point.getInhomZ();
382 
383             if (x < minX) {
384                 minX = x;
385             }
386             if (x > maxX) {
387                 maxX = x;
388             }
389             if (y < minY) {
390                 minY = y;
391             }
392             if (y > maxY) {
393                 maxY = y;
394             }
395             if (z < minZ) {
396                 minZ = z;
397             }
398             if (z > maxZ) {
399                 maxZ = z;
400             }
401         }
402 
403         // compute size of image plane (based on 2D coordinates limits), scale
404         // and centroid
405         final var width = maxX - minX;
406         final var height = maxY - minY;
407         final var depth = maxZ - minZ;
408 
409         final var norm = Math.sqrt(width * width + height * height + depth * depth);
410         // points are too close to each other (norm is too small).
411         // This is a degenerate configuration and results will be meaningless
412         if (norm < EPS) {
413             throw new PinholeCameraEstimatorException();
414         }
415 
416         final var scale = 1.0 / norm;
417         final var centroidX = (minX + maxX) / 2.0;
418         final var centroidY = (minY + maxY) / 2.0;
419         final var centroidZ = (minZ + maxZ) / 2.0;
420 
421         // set transformation matrix
422         try {
423             final var transformMatrix = new Matrix(Point3D.POINT3D_HOMOGENEOUS_COORDINATES_LENGTH,
424                     Point3D.POINT3D_HOMOGENEOUS_COORDINATES_LENGTH);
425             transformMatrix.setElementAt(0, 0, scale);
426             transformMatrix.setElementAt(1, 1, scale);
427             transformMatrix.setElementAt(2, 2, scale);
428             transformMatrix.setElementAt(3, 3, 1.0);
429             transformMatrix.setElementAt(0, 3, -scale * centroidX);
430             transformMatrix.setElementAt(1, 3, -scale * centroidY);
431             transformMatrix.setElementAt(2, 3, -scale * centroidZ);
432 
433             // normalize transformation to increase accuracy
434             transformation.setT(transformMatrix);
435         } catch (final WrongSizeException ignore) {
436             // never happens
437         }
438         transformation.normalize();
439 
440         // transform list of world points
441         final var transformedPoints = new ArrayList<Point3D>(list.size());
442         for (final var point : list) {
443             transformedPoints.add(transformation.transformAndReturnNew(point));
444         }
445 
446         return transformedPoints;
447     }
448 
449     /**
450      * Estimates a pinhole camera.
451      *
452      * @return estimated pinhole camera.
453      * @throws LockedException                 if estimator is locked.
454      * @throws NotReadyException               if input has not yet been provided.
455      * @throws PinholeCameraEstimatorException if an error occurs during
456      *                                         estimation, usually because input data is not valid.
457      */
458     @Override
459     public PinholeCamera estimate() throws LockedException, NotReadyException, PinholeCameraEstimatorException {
460 
461         if (isLocked()) {
462             throw new LockedException();
463         }
464         if (!isReady()) {
465             throw new NotReadyException();
466         }
467 
468         try {
469             locked = true;
470             if (listener != null) {
471                 listener.onEstimateStart(this);
472             }
473 
474             final var inverseTransformation2D = new ProjectiveTransformation2D();
475             final var transformation3D = new ProjectiveTransformation3D();
476             final List<Point2D> inputPoints2D;
477             final List<Point3D> inputPoints3D;
478 
479             if (normalizePointCorrespondences) {
480                 // normalize 2D and 3D points and update transformations
481                 inputPoints2D = transformPoints2D(points2D, inverseTransformation2D);
482                 inputPoints3D = transformPoints3D(points3D, transformation3D);
483             } else {
484                 inputPoints2D = points2D;
485                 inputPoints3D = points3D;
486             }
487 
488             var pinholeCameraMatrix = internalEstimate(inputPoints3D, inputPoints2D);
489 
490             if (normalizePointCorrespondences) {
491                 inverseTransformation2D.normalize();
492                 transformation3D.normalize();
493 
494                 // denormalize pinhole camera
495                 final var invTrans2DMatrix = inverseTransformation2D.asMatrix();
496                 final var trans3DMatrix = transformation3D.asMatrix();
497 
498                 invTrans2DMatrix.multiply(pinholeCameraMatrix);
499                 invTrans2DMatrix.multiply(trans3DMatrix);
500 
501                 pinholeCameraMatrix = invTrans2DMatrix;
502 
503                 // normalize by Frobenius norm to increase accuracy after point
504                 // de-normalization
505                 final var norm = Utils.normF(pinholeCameraMatrix);
506                 pinholeCameraMatrix.multiplyByScalar(1.0 / norm);
507             }
508 
509             final var camera = new PinholeCamera(pinholeCameraMatrix);
510 
511             if (listener != null) {
512                 listener.onEstimateEnd(this);
513             }
514 
515             return attemptRefine(camera);
516 
517         } catch (final PinholeCameraEstimatorException e) {
518             throw e;
519         } catch (final Exception e) {
520             throw new PinholeCameraEstimatorException(e);
521         } finally {
522             locked = false;
523         }
524     }
525 
526     /**
527      * Internal method that actually computes the normalized pinhole camera
528      * internal matrix.
529      * Returned matrix must have norm equal to one and might be estimated using
530      * any convenient algorithm (i.e. DLT or weighted DLT).
531      *
532      * @param points3D list of 3D points. Points might or might not be
533      *                 normalized.
534      * @param points2D list of 2D points. Points might or might not be
535      *                 normalized.
536      * @return matrix of estimated pinhole camera.
537      * @throws PinholeCameraEstimatorException if estimation fails for some
538      *                                         reason (i.e. numerical instability or geometric degeneracy).
539      */
540     protected abstract Matrix internalEstimate(final List<Point3D> points3D, final List<Point2D> points2D)
541             throws PinholeCameraEstimatorException;
542 
543     /**
544      * Attempts to refine provided camera using requested suggestions.
545      * If no suggestions are requested or if refinement fails, provided
546      * camera is returned instead.
547      *
548      * @param pinholeCamera camera to be refined.
549      * @return refined camera.
550      */
551     @Override
552     protected PinholeCamera attemptRefine(final PinholeCamera pinholeCamera) {
553         if (hasSuggestions()) {
554             final var numPoints = points3D.size();
555             final var inliers = new BitSet(numPoints);
556             inliers.set(0, numPoints, true);
557             final var residuals = new double[numPoints];
558 
559             final var refiner = new DecomposedPointCorrespondencePinholeCameraRefiner(pinholeCamera,
560                     false, inliers, residuals, numPoints, points3D, points2D, 0.0);
561             try {
562                 refiner.setMinSuggestionWeight(minSuggestionWeight);
563                 refiner.setMaxSuggestionWeight(maxSuggestionWeight);
564                 refiner.setSuggestionWeightStep(suggestionWeightStep);
565 
566                 refiner.setSuggestSkewnessValueEnabled(suggestSkewnessValueEnabled);
567                 refiner.setSuggestedSkewnessValue(suggestedSkewnessValue);
568                 refiner.setSuggestHorizontalFocalLengthEnabled(suggestHorizontalFocalLengthEnabled);
569                 refiner.setSuggestedHorizontalFocalLengthValue(suggestedHorizontalFocalLengthValue);
570                 refiner.setSuggestVerticalFocalLengthEnabled(suggestVerticalFocalLengthEnabled);
571                 refiner.setSuggestedVerticalFocalLengthValue(suggestedVerticalFocalLengthValue);
572                 refiner.setSuggestAspectRatioEnabled(suggestAspectRatioEnabled);
573                 refiner.setSuggestedAspectRatioValue(suggestedAspectRatioValue);
574                 refiner.setSuggestPrincipalPointEnabled(suggestPrincipalPointEnabled);
575                 refiner.setSuggestedPrincipalPointValue(suggestedPrincipalPointValue);
576                 refiner.setSuggestRotationEnabled(suggestRotationEnabled);
577                 refiner.setSuggestedRotationValue(suggestedRotationValue);
578                 refiner.setSuggestCenterEnabled(suggestCenterEnabled);
579                 refiner.setSuggestedCenterValue(suggestedCenterValue);
580 
581                 final var result = new PinholeCamera();
582                 final var improved = refiner.refine(result);
583 
584                 return improved ? result : pinholeCamera;
585 
586             } catch (final Exception e) {
587                 return pinholeCamera;
588             }
589         } else {
590             return pinholeCamera;
591         }
592     }
593 }