View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.geometry.Line2D;
20  import com.irurueta.geometry.PinholeCamera;
21  import com.irurueta.geometry.Plane;
22  import com.irurueta.geometry.estimators.LockedException;
23  import com.irurueta.geometry.estimators.NotReadyException;
24  import com.irurueta.numerical.EvaluationException;
25  import com.irurueta.numerical.GradientEstimator;
26  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
27  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
28  import com.irurueta.numerical.robust.InliersData;
29  
30  import java.util.BitSet;
31  import java.util.List;
32  
33  /**
34   * A pinhole camera refiner using line/plane correspondences and the
35   * Levenberg-Marquardt algorithm to try to decrease overall error in LMSE terms
36   * among inlier samples by taking the pinhole camera matrix as a whole without
37   * decomposition.
38   * Typically, this refiner is used by a robust estimator, however it can also be
39   * useful in some other situations.
40   */
41  @SuppressWarnings("DuplicatedCode")
42  public class NonDecomposedLinePlaneCorrespondencePinholeCameraRefiner extends
43          LinePlaneCorrespondencePinholeCameraRefiner {
44  
45      /**
46       * Default value for the weight applied to errors related to suggested
47       * camera parameters during computation of projection residuals.
48       */
49      public static final double DEFAULT_SUGGESTION_ERROR_WEIGHT = 2.0;
50  
51      /**
52       * Dimensions for refinement.
53       */
54      private static final int REFINE_DIMS = 12;
55  
56      /**
57       * Suggestion error weight. This weight is applied to errors related to
58       * suggested camera parameters during computation of projection residuals.
59       */
60      private double suggestionErrorWeight = DEFAULT_SUGGESTION_ERROR_WEIGHT;
61  
62      /**
63       * Constructor.
64       */
65      public NonDecomposedLinePlaneCorrespondencePinholeCameraRefiner() {
66      }
67  
68      /**
69       * Constructor.
70       *
71       * @param initialEstimation           initial estimation to be set.
72       * @param keepCovariance              true if covariance of estimation must be kept after
73       *                                    refinement, false otherwise.
74       * @param inliers                     set indicating which of the provided matches are inliers.
75       * @param residuals                   residuals for matched samples.
76       * @param numInliers                  number of inliers on initial estimation.
77       * @param samples1                    1st set of paired samples.
78       * @param samples2                    2nd set of paired samples.
79       * @param refinementStandardDeviation standard deviation used for
80       *                                    Levenberg-Marquardt fitting.
81       */
82      public NonDecomposedLinePlaneCorrespondencePinholeCameraRefiner(
83              final PinholeCamera initialEstimation, final boolean keepCovariance,
84              final BitSet inliers, final double[] residuals, final int numInliers,
85              final List<Plane> samples1, final List<Line2D> samples2, final double refinementStandardDeviation) {
86          super(initialEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2,
87                  refinementStandardDeviation);
88      }
89  
90      /**
91       * Constructor.
92       *
93       * @param initialEstimation           initial estimation to be set.
94       * @param keepCovariance              true if covariance of estimation must be kept after
95       *                                    refinement, false otherwise.
96       * @param inliersData                 inlier data, typically obtained from a robust
97       *                                    estimator.
98       * @param samples1                    1st set of paired samples.
99       * @param samples2                    2nd set of paired samples.
100      * @param refinementStandardDeviation standard deviation used for
101      *                                    Levenberg-Marquardt fitting.
102      */
103     public NonDecomposedLinePlaneCorrespondencePinholeCameraRefiner(
104             final PinholeCamera initialEstimation, final boolean keepCovariance,
105             final InliersData inliersData, final List<Plane> samples1, final List<Line2D> samples2,
106             final double refinementStandardDeviation) {
107         super(initialEstimation, keepCovariance, inliersData, samples1, samples2, refinementStandardDeviation);
108     }
109 
110     /**
111      * Gets suggestion error weight. This weight is applied to errors related to
112      * suggested camera parameters during computation of projection residuals.
113      *
114      * @return suggestion error weight.
115      */
116     public double getSuggestionErrorWeight() {
117         return suggestionErrorWeight;
118     }
119 
120     /**
121      * Sets suggestion error weight. This weight is applied to errors related to
122      * suggested camera parameters during computation of projection residuals.
123      *
124      * @param suggestionErrorWeight suggestion error weight.
125      * @throws LockedException if estimator is locked.
126      */
127     public void setSuggestionErrorWeight(final double suggestionErrorWeight) throws LockedException {
128         if (isLocked()) {
129             throw new LockedException();
130         }
131         this.suggestionErrorWeight = suggestionErrorWeight;
132     }
133 
134     /**
135      * Refines provided initial estimation.
136      * This method always sets a value into provided result instance regardless
137      * of the fact that error has actually improved in LMSE terms or not.
138      *
139      * @param result instance where refined estimation will be stored.
140      * @return true if result improves (decreases) in LMSE terms respect to
141      * initial estimation, false if no improvement has been achieved.
142      * @throws NotReadyException if not enough input data has been provided.
143      * @throws LockedException   if estimator is locked because refinement is
144      *                           already in progress.
145      * @throws RefinerException  if refinement fails for some reason (e.g. unable
146      *                           to converge to a result).
147      */
148     @Override
149     public boolean refine(final PinholeCamera result) throws NotReadyException, LockedException, RefinerException {
150         if (isLocked()) {
151             throw new LockedException();
152         }
153         if (!isReady()) {
154             throw new NotReadyException();
155         }
156 
157         locked = true;
158 
159         if (listener != null) {
160             listener.onRefineStart(this, initialEstimation);
161         }
162 
163         try {
164             initialEstimation.normalize();
165 
166             // output values to be fitted/optimized will contain residuals
167             final var y = new double[numInliers];
168             // input values will contain line and plane to compute residuals
169             final var nDims = Line2D.LINE_NUMBER_PARAMS + Plane.PLANE_NUMBER_PARAMS;
170             final var x = new Matrix(numInliers, nDims);
171             final var nSamples = inliers.length();
172             var pos = 0;
173             final var initParams = new double[REFINE_DIMS];
174             cameraToParameters(initialEstimation, initParams);
175 
176             final var initResidual = residualPowell(initialEstimation, initParams, suggestionErrorWeight);
177 
178             final var suggestionResidual = hasSuggestions() ? suggestionResidual(initParams, suggestionErrorWeight) :
179                     0.0;
180             for (var i = 0; i < nSamples; i++) {
181                 if (inliers.get(i)) {
182                     // sample is inlier
183                     final var line = samples2.get(i);
184                     final var plane = samples1.get(i);
185                     line.normalize();
186                     plane.normalize();
187                     x.setElementAt(pos, 0, line.getA());
188                     x.setElementAt(pos, 1, line.getB());
189                     x.setElementAt(pos, 2, line.getC());
190                     x.setElementAt(pos, 3, plane.getA());
191                     x.setElementAt(pos, 4, plane.getB());
192                     x.setElementAt(pos, 5, plane.getC());
193                     x.setElementAt(pos, 6, plane.getD());
194 
195                     y[pos] = Math.pow(residuals[i], 2.0) + suggestionResidual;
196                     pos++;
197                 }
198             }
199 
200             final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
201 
202                 private final Line2D line = new Line2D();
203 
204                 private final Plane plane = new Plane();
205 
206                 private final PinholeCamera pinholeCamera = new PinholeCamera();
207 
208                 private final GradientEstimator gradientEstimator = new GradientEstimator(params -> {
209                     parametersToCamera(params, pinholeCamera);
210                     return residualLevenbergMarquardt(pinholeCamera, line, plane, params, suggestionErrorWeight);
211                 });
212 
213                 @Override
214                 public int getNumberOfDimensions() {
215                     return nDims;
216                 }
217 
218                 @Override
219                 public double[] createInitialParametersArray() {
220                     return initParams;
221                 }
222 
223                 @Override
224                 public double evaluate(final int i, final double[] point, final double[] params,
225                                        final double[] derivatives) throws EvaluationException {
226                     line.setParameters(point[0], point[1], point[2]);
227                     plane.setParameters(point[3], point[4], point[5], point[6]);
228 
229                     line.normalize();
230                     plane.normalize();
231 
232                     parametersToCamera(params, pinholeCamera);
233                     final var y = residualLevenbergMarquardt(pinholeCamera, line, plane, params, suggestionErrorWeight);
234                     gradientEstimator.gradient(params, derivatives);
235 
236                     return y;
237                 }
238             };
239 
240             final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y, refinementStandardDeviation);
241 
242             fitter.fit();
243 
244             final var finalParams = fitter.getA();
245 
246             parametersToCamera(finalParams, result);
247 
248             if (keepCovariance) {
249                 covariance = fitter.getCovar();
250             }
251 
252             final var finalResidual = residualPowell(result, finalParams, suggestionErrorWeight);
253             final var errorDecreased = finalResidual < initResidual;
254 
255             if (listener != null) {
256                 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
257             }
258 
259             return errorDecreased;
260 
261         } catch (final Exception e) {
262             throw new RefinerException(e);
263         } finally {
264             locked = false;
265         }
266     }
267 }