View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.geometry.CoordinatesType;
20  import com.irurueta.geometry.PinholeCamera;
21  import com.irurueta.geometry.Point2D;
22  import com.irurueta.geometry.Point3D;
23  import com.irurueta.geometry.estimators.LockedException;
24  import com.irurueta.geometry.estimators.NotReadyException;
25  import com.irurueta.numerical.EvaluationException;
26  import com.irurueta.numerical.GradientEstimator;
27  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
28  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
29  import com.irurueta.numerical.robust.InliersData;
30  
31  import java.util.BitSet;
32  import java.util.List;
33  
34  /**
35   * A pinhole camera refiner using point correspondences and the
36   * Levenberg-Marquardt algorithm to try to decrease overall error in LMSE terms
37   * among inlier samples by taking the pinhole camera matrix as a whole without
38   * decomposition.
39   * Typically, this refiner is used by a robust estimator, however it can also be
40   * useful in some other situations.
41   */
42  @SuppressWarnings("DuplicatedCode")
43  public class NonDecomposedPointCorrespondencePinholeCameraRefiner extends PointCorrespondencePinholeCameraRefiner {
44  
45      /**
46       * Default value for the weight applied to errors related to suggested
47       * camera parameters during computation of projection residuals.
48       */
49      public static final double DEFAULT_SUGGESTION_ERROR_WEIGHT = 2.0;
50  
51      /**
52       * Dimensions for refinement.
53       */
54      private static final int REFINE_DIMS = 12;
55  
56      /**
57       * Suggestion error weight. This weight is applied to errors related to
58       * suggested camera parameters during computation of projection residuals.
59       */
60      private double suggestionErrorWeight = DEFAULT_SUGGESTION_ERROR_WEIGHT;
61  
62      /**
63       * Constructor.
64       */
65      public NonDecomposedPointCorrespondencePinholeCameraRefiner() {
66      }
67  
68      /**
69       * Constructor.
70       *
71       * @param initialEstimation           initial estimation to be set.
72       * @param keepCovariance              true if covariance of estimation must be kept after
73       *                                    refinement, false otherwise.
74       * @param inliers                     set indicating which of the provided matches are inliers.
75       * @param residuals                   residuals for matched samples.
76       * @param numInliers                  number of inliers on initial estimation.
77       * @param samples1                    1st set of paired samples.
78       * @param samples2                    2nd set of paired samples.
79       * @param refinementStandardDeviation standard deviation used for
80       *                                    Levenberg-Marquardt fitting.
81       */
82      public NonDecomposedPointCorrespondencePinholeCameraRefiner(
83              final PinholeCamera initialEstimation, final boolean keepCovariance, final BitSet inliers,
84              final double[] residuals, final int numInliers, final List<Point3D> samples1, final List<Point2D> samples2,
85              final double refinementStandardDeviation) {
86          super(initialEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2,
87                  refinementStandardDeviation);
88      }
89  
90      /**
91       * Constructor.
92       *
93       * @param initialEstimation           initial estimation to be set.
94       * @param keepCovariance              true if covariance of estimation must be kept after
95       *                                    refinement, false otherwise.
96       * @param inliersData                 inlier data, typically obtained from a robust
97       *                                    estimator.
98       * @param samples1                    1st set of paired samples.
99       * @param samples2                    2nd set of paired samples.
100      * @param refinementStandardDeviation standard deviation used for
101      *                                    Levenberg-Marquardt fitting.
102      */
103     public NonDecomposedPointCorrespondencePinholeCameraRefiner(
104             final PinholeCamera initialEstimation, final boolean keepCovariance, final InliersData inliersData,
105             final List<Point3D> samples1, final List<Point2D> samples2, final double refinementStandardDeviation) {
106         super(initialEstimation, keepCovariance, inliersData, samples1, samples2, refinementStandardDeviation);
107     }
108 
109     /**
110      * Gets suggestion error weight. This weight is applied to errors related to
111      * suggested camera parameters during computation of projection residuals.
112      *
113      * @return suggestion error weight.
114      */
115     public double getSuggestionErrorWeight() {
116         return suggestionErrorWeight;
117     }
118 
119     /**
120      * Sets suggestion error weight. This weight is applied to errors related to
121      * suggested camera parameters during computation of projection residuals.
122      *
123      * @param suggestionErrorWeight suggestion error weight.
124      * @throws LockedException if estimator is locked.
125      */
126     public void setSuggestionErrorWeight(final double suggestionErrorWeight) throws LockedException {
127         if (isLocked()) {
128             throw new LockedException();
129         }
130         this.suggestionErrorWeight = suggestionErrorWeight;
131     }
132 
133     /**
134      * Refines provided initial estimation.
135      * This method always sets a value into provided result instance regardless
136      * of the fact that error has actually improved in LMSE terms or not.
137      *
138      * @param result instance where refined estimation will be stored.
139      * @return true if result improves (decreases) in LMSE terms respect to
140      * initial estimation, false if no improvement has been achieved.
141      * @throws NotReadyException if not enough input data has been provided.
142      * @throws LockedException   if estimator is locked because refinement is
143      *                           already in progress.
144      * @throws RefinerException  if refinement fails for some reason (e.g. unable
145      *                           to converge to a result).
146      */
147     @Override
148     public boolean refine(final PinholeCamera result) throws NotReadyException, LockedException, RefinerException {
149         if (isLocked()) {
150             throw new LockedException();
151         }
152         if (!isReady()) {
153             throw new NotReadyException();
154         }
155 
156         locked = true;
157 
158         if (listener != null) {
159             listener.onRefineStart(this, initialEstimation);
160         }
161 
162         try {
163             initialEstimation.normalize();
164 
165             // output values to be fitted/optimized will contain residuals
166             final var y = new double[numInliers];
167             // input values will contain 3D point and 2D point to compute
168             // residuals
169             final var nDims = Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH
170                     + Point3D.POINT3D_HOMOGENEOUS_COORDINATES_LENGTH;
171             final var x = new Matrix(numInliers, nDims);
172             final var nSamples = inliers.length();
173             var pos = 0;
174             final var initParams = new double[REFINE_DIMS];
175             cameraToParameters(initialEstimation, initParams);
176 
177             final var initResidual = residualPowell(initialEstimation, initParams, suggestionErrorWeight);
178 
179             final var suggestionResidual = hasSuggestions() ? suggestionResidual(initParams, suggestionErrorWeight)
180                     : 0.0;
181             for (var i = 0; i < nSamples; i++) {
182                 if (inliers.get(i)) {
183                     // sample is inlier
184                     final var point2D = samples2.get(i);
185                     final var point3D = samples1.get(i);
186                     point2D.normalize();
187                     point3D.normalize();
188                     x.setElementAt(pos, 0, point2D.getHomX());
189                     x.setElementAt(pos, 1, point2D.getHomY());
190                     x.setElementAt(pos, 2, point2D.getHomW());
191                     x.setElementAt(pos, 3, point3D.getHomX());
192                     x.setElementAt(pos, 4, point3D.getHomY());
193                     x.setElementAt(pos, 5, point3D.getHomZ());
194                     x.setElementAt(pos, 6, point3D.getHomW());
195 
196                     y[pos] = Math.pow(residuals[i], 2.0) + suggestionResidual;
197                     pos++;
198                 }
199             }
200 
201             final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
202 
203                 private final Point2D point2D = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
204 
205                 private final Point3D point3D = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
206 
207                 private final PinholeCamera pinholeCamera = new PinholeCamera();
208 
209                 private final GradientEstimator gradientEstimator = new GradientEstimator(params -> {
210                     parametersToCamera(params, pinholeCamera);
211                     return residualLevenbergMarquardt(pinholeCamera, point3D, point2D, params, suggestionErrorWeight);
212                 });
213 
214                 @Override
215                 public int getNumberOfDimensions() {
216                     return nDims;
217                 }
218 
219                 @Override
220                 public double[] createInitialParametersArray() {
221                     return initParams;
222                 }
223 
224                 @Override
225                 public double evaluate(
226                         final int i, final double[] point, final double[] params, final double[] derivatives)
227                         throws EvaluationException {
228                     point2D.setHomogeneousCoordinates(point[0], point[1], point[2]);
229                     point3D.setHomogeneousCoordinates(point[3], point[4], point[5], point[6]);
230 
231                     point2D.normalize();
232                     point3D.normalize();
233 
234                     parametersToCamera(params, pinholeCamera);
235                     final var y = residualLevenbergMarquardt(pinholeCamera, point3D, point2D, params,
236                             suggestionErrorWeight);
237                     gradientEstimator.gradient(params, derivatives);
238 
239                     return y;
240                 }
241             };
242 
243             final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y, refinementStandardDeviation);
244 
245             fitter.fit();
246 
247             final var finalParams = fitter.getA();
248 
249             parametersToCamera(finalParams, result);
250 
251             if (keepCovariance) {
252                 covariance = fitter.getCovar();
253             }
254 
255             final var finalResidual = residualPowell(result, finalParams, suggestionErrorWeight);
256             final var errorDecreased = finalResidual < initResidual;
257 
258             if (listener != null) {
259                 listener.onRefineEnd(this, initialEstimation, result,
260                         errorDecreased);
261             }
262 
263             return errorDecreased;
264 
265         } catch (final Exception e) {
266             throw new RefinerException(e);
267         } finally {
268             locked = false;
269         }
270     }
271 }