View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.geometry.*;
20  import com.irurueta.geometry.estimators.LockedException;
21  import com.irurueta.geometry.estimators.NotReadyException;
22  import com.irurueta.numerical.EvaluationException;
23  import com.irurueta.numerical.GradientEstimator;
24  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
25  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
26  import com.irurueta.numerical.robust.InliersData;
27  
28  import java.util.BitSet;
29  import java.util.List;
30  
31  /**
32   * Refines a 3D metric transformation by taking into account an initial
33   * estimation, inlier point matches and their residuals.
34   * This class can be used to find a solution that minimizes error of inliers in
35   * LMSE terms.
36   * Typically, a refiner is used by a robust estimator, however it can also be
37   * useful in some other situations.
38   */
39  @SuppressWarnings("DuplicatedCode")
40  public class MetricTransformation3DRefiner extends
41          PairMatchesAndInliersDataRefiner<MetricTransformation3D, Point3D, Point3D> {
42  
43      /**
44       * Point to be reused when computing residuals.
45       */
46      private final Point3D residualTestPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
47  
48      /**
49       * Quaternion to be reused for refinement computations.
50       */
51      private Quaternion quaternion = new Quaternion();
52  
53      /**
54       * Standard deviation used for Levenberg-Marquardt fitting during
55       * refinement.
56       * Returned value gives an indication of how much variance each residual
57       * has.
58       * Typically, this value is related to the threshold used on each robust
59       * estimation, since residuals of found inliers are within the range of
60       * such threshold.
61       */
62      private double refinementStandardDeviation;
63  
64      /**
65       * Constructor.
66       */
67      public MetricTransformation3DRefiner() {
68      }
69  
70      /**
71       * Constructor.
72       *
73       * @param initialEstimation           initial estimation to be set.
74       * @param keepCovariance              true if covariance of estimation must be kept after
75       *                                    refinement, false otherwise.
76       * @param inliers                     set indicating which of the provided matches are inliers.
77       * @param residuals                   residuals for matched samples.
78       * @param numInliers                  number of inliers on initial estimation.
79       * @param samples1                    1st set of paired samples.
80       * @param samples2                    2nd set of paired samples.
81       * @param refinementStandardDeviation standard deviation used for
82       *                                    Levenberg-Marquardt fitting.
83       */
84      public MetricTransformation3DRefiner(
85              final MetricTransformation3D initialEstimation, final boolean keepCovariance,
86              final BitSet inliers, final double[] residuals, final int numInliers,
87              final List<Point3D> samples1, final List<Point3D> samples2, final double refinementStandardDeviation) {
88          super(initialEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2);
89          this.refinementStandardDeviation = refinementStandardDeviation;
90      }
91  
92      /**
93       * Constructor.
94       *
95       * @param initialEstimation           initial estimation to be set.
96       * @param keepCovariance              true if covariance of estimation must be kept after
97       *                                    refinement, false otherwise.
98       * @param inliersData                 inlier data, typically obtained from a robust
99       *                                    estimator.
100      * @param samples1                    1st set of paired samples.
101      * @param samples2                    2nd set of paired samples.
102      * @param refinementStandardDeviation standard deviation used for
103      *                                    Levenberg-Marquardt fitting.
104      */
105     public MetricTransformation3DRefiner(
106             final MetricTransformation3D initialEstimation, final boolean keepCovariance,
107             final InliersData inliersData, final List<Point3D> samples1, final List<Point3D> samples2,
108             final double refinementStandardDeviation) {
109         super(initialEstimation, keepCovariance, inliersData, samples1, samples2);
110         this.refinementStandardDeviation = refinementStandardDeviation;
111     }
112 
113     /**
114      * Gets standard deviation used for Levenberg-Marquardt fitting during
115      * refinement.
116      * Returned value gives an indication of how much variance each residual
117      * has.
118      * Typically, this value is related to the threshold used on each robust
119      * estimation, since residuals of found inliers are within the range of
120      * such threshold.
121      *
122      * @return standard deviation used for refinement.
123      */
124     public double getRefinementStandardDeviation() {
125         return refinementStandardDeviation;
126     }
127 
128     /**
129      * Sets standard deviation used for Levenberg-Marquardt fitting during
130      * refinement.
131      * Returned value gives an indication of how much variance each residual
132      * has.
133      * Typically, this value is related to the threshold used on each robust
134      * estimation, since residuals of found inliers are within the range of such
135      * threshold.
136      *
137      * @param refinementStandardDeviation standard deviation used for
138      *                                    refinement.
139      * @throws LockedException if estimator is locked.
140      */
141     public void setRefinementStandardDeviation(final double refinementStandardDeviation) throws LockedException {
142         if (isLocked()) {
143             throw new LockedException();
144         }
145         this.refinementStandardDeviation = refinementStandardDeviation;
146     }
147 
148     /**
149      * Refines provided initial estimation.
150      *
151      * @return refines estimation.
152      * @throws NotReadyException if not enough input data has been provided.
153      * @throws LockedException   if estimator is locked because refinement is
154      *                           already in progress.
155      * @throws RefinerException  if refinement fails for some reason (e.g. unable
156      *                           to converge to a result).
157      */
158     @Override
159     public MetricTransformation3D refine() throws NotReadyException, LockedException, RefinerException {
160         final var result = new MetricTransformation3D();
161         refine(result);
162         return result;
163     }
164 
165     /**
166      * Refines provided initial estimation.
167      * This method always sets a value into provided result instance regardless
168      * of the fact that error has actually improved in LMSE terms or not.
169      *
170      * @param result instance where refined estimation will be stored.
171      * @return true if result improves (error decreases) in LMSE terms respect
172      * to initial estimation, false if no improvement has been achieved.
173      * @throws NotReadyException if not enough input data has been provided.
174      * @throws LockedException   if estimator is locked because refinement is
175      *                           already in progress.
176      * @throws RefinerException  if refinement fails for some reason (e.g. unable
177      *                           to converge to a result).
178      */
179     @Override
180     public boolean refine(final MetricTransformation3D result) throws NotReadyException, LockedException,
181             RefinerException {
182         if (isLocked()) {
183             throw new LockedException();
184         }
185         if (!isReady()) {
186             throw new NotReadyException();
187         }
188 
189         locked = true;
190 
191         if (listener != null) {
192             listener.onRefineStart(this, initialEstimation);
193         }
194 
195         final var initialTotalResidual = totalResidual(initialEstimation);
196 
197         try {
198             // parameters: rotation angle + scale + translation
199             final var initParams = new double[1 + Quaternion.N_PARAMS
200                     + EuclideanTransformation3D.NUM_TRANSLATION_COORDS];
201             // copy rotation values
202             if (initialEstimation.getRotation().getType() == Rotation3DType.QUATERNION) {
203                 quaternion = (Quaternion) initialEstimation.getRotation();
204             } else {
205                 quaternion = initialEstimation.getRotation().toQuaternion();
206             }
207             quaternion.normalize();
208 
209             // copy values
210             initParams[0] = initialEstimation.getScale();
211             initParams[1] = quaternion.getA();
212             initParams[2] = quaternion.getB();
213             initParams[3] = quaternion.getC();
214             initParams[4] = quaternion.getD();
215 
216             System.arraycopy(initialEstimation.getTranslation(), 0, initParams, Quaternion.N_PARAMS + 1,
217                     EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
218 
219             // output values to be fitted/optimized will contain residuals
220             final var y = new double[numInliers];
221             // input values will contain 2 sets of 2D points to compute residuals
222             final var nDims = 2 * Point3D.POINT3D_HOMOGENEOUS_COORDINATES_LENGTH;
223             final var x = new Matrix(numInliers, nDims);
224             final var nSamples = inliers.length();
225             var pos = 0;
226             for (var i = 0; i < nSamples; i++) {
227                 if (inliers.get(i)) {
228                     // sample is inlier
229                     final var inputPoint = samples1.get(i);
230                     final var outputPoint = samples2.get(i);
231                     inputPoint.normalize();
232                     outputPoint.normalize();
233                     x.setElementAt(pos, 0, inputPoint.getHomX());
234                     x.setElementAt(pos, 1, inputPoint.getHomY());
235                     x.setElementAt(pos, 2, inputPoint.getHomZ());
236                     x.setElementAt(pos, 3, inputPoint.getHomW());
237                     x.setElementAt(pos, 4, outputPoint.getHomX());
238                     x.setElementAt(pos, 5, outputPoint.getHomY());
239                     x.setElementAt(pos, 6, outputPoint.getHomZ());
240                     x.setElementAt(pos, 7, outputPoint.getHomW());
241 
242                     y[pos] = residuals[i];
243                     pos++;
244                 }
245             }
246 
247             final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
248 
249                 private final Point3D inputPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
250 
251                 private final Point3D outputPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
252 
253                 private final MetricTransformation3D transformation = new MetricTransformation3D();
254 
255                 private final GradientEstimator gradientEstimator = new GradientEstimator(params -> {
256                     // copy values
257                     transformation.setScale(params[0]);
258                     quaternion.setA(params[1]);
259                     quaternion.setB(params[2]);
260                     quaternion.setC(params[3]);
261                     quaternion.setD(params[4]);
262                     transformation.setRotation(quaternion);
263 
264                     System.arraycopy(params, 1 + Quaternion.N_PARAMS, transformation.getTranslation(), 0,
265                             EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
266 
267                     return residual(transformation, inputPoint, outputPoint);
268                 });
269 
270                 @Override
271                 public int getNumberOfDimensions() {
272                     return nDims;
273                 }
274 
275                 @Override
276                 public double[] createInitialParametersArray() {
277                     return initParams;
278                 }
279 
280                 @Override
281                 public double evaluate(final int i, final double[] point, final double[] params,
282                                        final double[] derivatives) throws EvaluationException {
283                     inputPoint.setHomogeneousCoordinates(point[0], point[1], point[2], point[3]);
284                     outputPoint.setHomogeneousCoordinates(point[4], point[5], point[6], point[7]);
285 
286                     // copy values
287                     transformation.setScale(params[0]);
288                     quaternion.setA(params[1]);
289                     quaternion.setB(params[2]);
290                     quaternion.setC(params[3]);
291                     quaternion.setD(params[4]);
292                     transformation.setRotation(quaternion);
293 
294                     System.arraycopy(params, 1 + Quaternion.N_PARAMS, transformation.getTranslation(), 0,
295                             EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
296 
297                     final var y = residual(transformation, inputPoint, outputPoint);
298                     gradientEstimator.gradient(params, derivatives);
299 
300                     return y;
301                 }
302             };
303 
304             final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y,
305                     getRefinementStandardDeviation());
306 
307             fitter.fit();
308 
309             // obtain estimated params
310             final var params = fitter.getA();
311 
312             // update transformation
313             result.setScale(params[0]);
314             quaternion.setA(params[1]);
315             quaternion.setB(params[2]);
316             quaternion.setC(params[3]);
317             quaternion.setD(params[4]);
318             result.setRotation(quaternion);
319             System.arraycopy(params, 1 + Quaternion.N_PARAMS, result.getTranslation(), 0,
320                     EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
321 
322             if (keepCovariance) {
323                 // keep covariance
324                 covariance = fitter.getCovar();
325             }
326 
327             final var finalTotalResidual = totalResidual(result);
328             final var errorDecreased = finalTotalResidual < initialTotalResidual;
329 
330             if (listener != null) {
331                 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
332             }
333 
334             return errorDecreased;
335 
336         } catch (final Exception e) {
337             throw new RefinerException(e);
338         } finally {
339             locked = false;
340         }
341     }
342 
343     /**
344      * Computes the residual between the Euclidean transformation and a pair or
345      * matched points.
346      *
347      * @param transformation a transformation.
348      * @param inputPoint     input 3D point.
349      * @param outputPoint    output 3D point.
350      * @return residual.
351      */
352     private double residual(final MetricTransformation3D transformation, final Point3D inputPoint,
353                             final Point3D outputPoint) {
354         inputPoint.normalize();
355         outputPoint.normalize();
356 
357         transformation.transform(inputPoint, residualTestPoint);
358         return residualTestPoint.distanceTo(outputPoint);
359     }
360 
361     /**
362      * Computes total residual among all provided inlier samples.
363      *
364      * @param transformation a transformation.
365      * @return total residual.
366      */
367     private double totalResidual(final MetricTransformation3D transformation) {
368         var result = 0.0;
369 
370         final var nSamples = inliers.length();
371         for (var i = 0; i < nSamples; i++) {
372             if (inliers.get(i)) {
373                 // sample is inlier
374                 final var inputPoint = samples1.get(i);
375                 final var outputPoint = samples2.get(i);
376                 inputPoint.normalize();
377                 outputPoint.normalize();
378                 result += residual(transformation, inputPoint, outputPoint);
379             }
380         }
381 
382         return result;
383     }
384 }