View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.geometry.CoordinatesType;
20  import com.irurueta.geometry.EuclideanTransformation3D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.geometry.Quaternion;
23  import com.irurueta.geometry.Rotation3DType;
24  import com.irurueta.geometry.estimators.LockedException;
25  import com.irurueta.geometry.estimators.NotReadyException;
26  import com.irurueta.numerical.EvaluationException;
27  import com.irurueta.numerical.GradientEstimator;
28  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
29  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
30  import com.irurueta.numerical.robust.InliersData;
31  
32  import java.util.BitSet;
33  import java.util.List;
34  
35  /**
36   * Refine a 3D Euclidean transformation by taking into account an initial
37   * estimation, inlier point matches and their residuals.
38   * This class can be used to find a solution that minimizes error of inliers in
39   * LMSE terms.
40   * Typically, a refiner is used by a robust estimator, however it can also be
41   * useful in some other situations.
42   */
43  @SuppressWarnings("DuplicatedCode")
44  public class EuclideanTransformation3DRefiner extends
45          PairMatchesAndInliersDataRefiner<EuclideanTransformation3D, Point3D, Point3D> {
46  
47      /**
48       * Point to be reused when computing residuals.
49       */
50      private final Point3D residualTestPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
51  
52      /**
53       * Quaternion to be reused for refinement computations.
54       */
55      private Quaternion quaternion = new Quaternion();
56  
57      /**
58       * Standard deviation used for Levenberg-Marquardt fitting during
59       * refinement.
60       * Returned value gives an indication of how much variance each residual
61       * has.
62       * Typically, this value is related to the threshold used on each robust
63       * estimation, since residuals of found inliers are within the range of
64       * such threshold.
65       */
66      private double refinementStandardDeviation;
67  
68      /**
69       * Constructor.
70       */
71      public EuclideanTransformation3DRefiner() {
72      }
73  
74      /**
75       * Constructor.
76       *
77       * @param initialEstimation           initial estimation to be set.
78       * @param keepCovariance              true if covariance of estimation must be kept after
79       *                                    refinement, false otherwise.
80       * @param inliers                     set indicating which of the provided matches are inliers.
81       * @param residuals                   residuals for matched samples.
82       * @param numInliers                  number of inliers on initial estimation.
83       * @param samples1                    1st set of paired samples.
84       * @param samples2                    2nd set of paired samples.
85       * @param refinementStandardDeviation standard deviation used for
86       *                                    Levenberg-Marquardt fitting.
87       */
88      public EuclideanTransformation3DRefiner(
89              final EuclideanTransformation3D initialEstimation, final boolean keepCovariance, final BitSet inliers,
90              final double[] residuals, final int numInliers, final List<Point3D> samples1, final List<Point3D> samples2,
91              final double refinementStandardDeviation) {
92          super(initialEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2);
93          this.refinementStandardDeviation = refinementStandardDeviation;
94      }
95  
96      /**
97       * Constructor.
98       *
99       * @param initialEstimation           initial estimation to be set.
100      * @param keepCovariance              true if covariance of estimation must be kept after
101      *                                    refinement, false otherwise.
102      * @param inliersData                 inlier data, typically obtained from a robust
103      *                                    estimator.
104      * @param samples1                    1st set of paired samples.
105      * @param samples2                    2nd set of paired samples.
106      * @param refinementStandardDeviation standard deviation used for
107      *                                    Levenberg-Marquardt fitting.
108      */
109     public EuclideanTransformation3DRefiner(
110             final EuclideanTransformation3D initialEstimation, final boolean keepCovariance,
111             final InliersData inliersData, final List<Point3D> samples1,
112             final List<Point3D> samples2, final double refinementStandardDeviation) {
113         super(initialEstimation, keepCovariance, inliersData, samples1, samples2);
114         this.refinementStandardDeviation = refinementStandardDeviation;
115     }
116 
117     /**
118      * Gets standard deviation used for Levenberg-Marquardt fitting during
119      * refinement.
120      * Returned value gives an indication of how much variance each residual
121      * has.
122      * Typically, this value is related to the threshold used on each robust
123      * estimation, since residuals of found inliers are within the range of
124      * such threshold.
125      *
126      * @return standard deviation used for refinement.
127      */
128     public double getRefinementStandardDeviation() {
129         return refinementStandardDeviation;
130     }
131 
132     /**
133      * Sets standard deviation used for Levenberg-Marquardt fitting during
134      * refinement.
135      * Returned value gives an indication of how much variance each residual
136      * has.
137      * Typically, this value is related to the threshold used on each robust
138      * estimation, since residuals of found inliers are within the range of such
139      * threshold.
140      *
141      * @param refinementStandardDeviation standard deviation used for
142      *                                    refinement.
143      * @throws LockedException if estimator is locked.
144      */
145     public void setRefinementStandardDeviation(final double refinementStandardDeviation) throws LockedException {
146         if (isLocked()) {
147             throw new LockedException();
148         }
149         this.refinementStandardDeviation = refinementStandardDeviation;
150     }
151 
152     /**
153      * Refines provided initial estimation.
154      *
155      * @return refines estimation.
156      * @throws NotReadyException if not enough input data has been provided.
157      * @throws LockedException   if estimator is locked because refinement is
158      *                           already in progress.
159      * @throws RefinerException  if refinement fails for some reason (e.g. unable
160      *                           to converge to a result).
161      */
162     @Override
163     public EuclideanTransformation3D refine() throws NotReadyException, LockedException, RefinerException {
164         final var result = new EuclideanTransformation3D();
165         refine(result);
166         return result;
167     }
168 
169     /**
170      * Refines provided initial estimation.
171      * This method always sets a value into provided result instance regardless
172      * of the fact that error has actually improved in LMSE terms or not.
173      *
174      * @param result instance where refined estimation will be stored.
175      * @return true if result improves (error decreases) in LMSE terms respect
176      * to initial estimation, false if no improvement has been achieved.
177      * @throws NotReadyException if not enough input data has been provided.
178      * @throws LockedException   if estimator is locked because refinement is
179      *                           already in progress.
180      * @throws RefinerException  if refinement fails for some reason (e.g. unable
181      *                           to converge to a result).
182      */
183     @Override
184     public boolean refine(final EuclideanTransformation3D result) throws NotReadyException, LockedException,
185             RefinerException {
186         if (isLocked()) {
187             throw new LockedException();
188         }
189         if (!isReady()) {
190             throw new NotReadyException();
191         }
192 
193         locked = true;
194 
195         if (listener != null) {
196             listener.onRefineStart(this, initialEstimation);
197         }
198 
199         final var initialTotalResidual = totalResidual(initialEstimation);
200 
201         try {
202             // parameters: rotation angle + scale + translation
203             final var initParams = new double[Quaternion.N_PARAMS + EuclideanTransformation3D.NUM_TRANSLATION_COORDS];
204             // copy rotation values
205             if (initialEstimation.getRotation().getType() == Rotation3DType.QUATERNION) {
206                 quaternion = (Quaternion) initialEstimation.getRotation();
207             } else {
208                 quaternion = initialEstimation.getRotation().toQuaternion();
209             }
210             quaternion.normalize();
211 
212             // copy values
213             initParams[0] = quaternion.getA();
214             initParams[1] = quaternion.getB();
215             initParams[2] = quaternion.getC();
216             initParams[3] = quaternion.getD();
217 
218             System.arraycopy(initialEstimation.getTranslation(), 0, initParams, Quaternion.N_PARAMS,
219                     EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
220 
221             // output values to be fitted/optimized will contain residuals
222             final var y = new double[numInliers];
223             // input values will contain 2 sets of 2D points to compute residuals
224             final var nDims = 2 * Point3D.POINT3D_HOMOGENEOUS_COORDINATES_LENGTH;
225             final var x = new Matrix(numInliers, nDims);
226             final var nSamples = inliers.length();
227             var pos = 0;
228             for (var i = 0; i < nSamples; i++) {
229                 if (inliers.get(i)) {
230                     // sample is inlier
231                     final var inputPoint = samples1.get(i);
232                     final var outputPoint = samples2.get(i);
233                     inputPoint.normalize();
234                     outputPoint.normalize();
235                     x.setElementAt(pos, 0, inputPoint.getHomX());
236                     x.setElementAt(pos, 1, inputPoint.getHomY());
237                     x.setElementAt(pos, 2, inputPoint.getHomZ());
238                     x.setElementAt(pos, 3, inputPoint.getHomW());
239                     x.setElementAt(pos, 4, outputPoint.getHomX());
240                     x.setElementAt(pos, 5, outputPoint.getHomY());
241                     x.setElementAt(pos, 6, outputPoint.getHomZ());
242                     x.setElementAt(pos, 7, outputPoint.getHomW());
243 
244                     y[pos] = residuals[i];
245                     pos++;
246                 }
247             }
248 
249             final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
250 
251                 private final Point3D inputPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
252 
253                 private final Point3D outputPoint = Point3D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
254 
255                 private final EuclideanTransformation3D transformation = new EuclideanTransformation3D();
256 
257                 private final GradientEstimator gradientEstimator = new GradientEstimator(params -> {
258                     // copy values
259                     quaternion.setA(params[0]);
260                     quaternion.setB(params[1]);
261                     quaternion.setC(params[2]);
262                     quaternion.setD(params[3]);
263                     transformation.setRotation(quaternion);
264 
265                     System.arraycopy(params, Quaternion.N_PARAMS, transformation.getTranslation(), 0,
266                             EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
267 
268                     return residual(transformation, inputPoint, outputPoint);
269                 });
270 
271                 @Override
272                 public int getNumberOfDimensions() {
273                     return nDims;
274                 }
275 
276                 @Override
277                 public double[] createInitialParametersArray() {
278                     return initParams;
279                 }
280 
281                 @Override
282                 public double evaluate(final int i, final double[] point, final double[] params,
283                                        final double[] derivatives) throws EvaluationException {
284                     inputPoint.setHomogeneousCoordinates(point[0], point[1], point[2], point[3]);
285                     outputPoint.setHomogeneousCoordinates(point[4], point[5], point[6], point[7]);
286 
287                     // copy values
288                     quaternion.setA(params[0]);
289                     quaternion.setB(params[1]);
290                     quaternion.setC(params[2]);
291                     quaternion.setD(params[3]);
292                     transformation.setRotation(quaternion);
293 
294                     System.arraycopy(params, Quaternion.N_PARAMS, transformation.getTranslation(), 0,
295                             EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
296 
297                     final var y = residual(transformation, inputPoint, outputPoint);
298                     gradientEstimator.gradient(params, derivatives);
299 
300                     return y;
301                 }
302             };
303 
304             final LevenbergMarquardtMultiDimensionFitter fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator,
305                     x, y, getRefinementStandardDeviation());
306 
307             fitter.fit();
308 
309             // obtain estimated params
310             final var params = fitter.getA();
311 
312             // update transformation
313             quaternion.setA(params[0]);
314             quaternion.setB(params[1]);
315             quaternion.setC(params[2]);
316             quaternion.setD(params[3]);
317             result.setRotation(quaternion);
318             System.arraycopy(params, Quaternion.N_PARAMS, result.getTranslation(), 0,
319                     EuclideanTransformation3D.NUM_TRANSLATION_COORDS);
320 
321             if (keepCovariance) {
322                 // keep covariance
323                 covariance = fitter.getCovar();
324             }
325 
326             final var finalTotalResidual = totalResidual(result);
327             final var errorDecreased = finalTotalResidual < initialTotalResidual;
328 
329             if (listener != null) {
330                 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
331             }
332 
333             return errorDecreased;
334 
335         } catch (final Exception e) {
336             throw new RefinerException(e);
337         } finally {
338             locked = false;
339         }
340     }
341 
342     /**
343      * Computes the residual between the Euclidean transformation and a pair or
344      * matched points.
345      *
346      * @param transformation a transformation.
347      * @param inputPoint     input 2D point.
348      * @param outputPoint    output 2D point.
349      * @return residual.
350      */
351     private double residual(final EuclideanTransformation3D transformation, final Point3D inputPoint,
352                             final Point3D outputPoint) {
353         inputPoint.normalize();
354         outputPoint.normalize();
355 
356         transformation.transform(inputPoint, residualTestPoint);
357         return residualTestPoint.distanceTo(outputPoint);
358     }
359 
360     /**
361      * Computes total residual among all provided inlier samples.
362      *
363      * @param transformation a transformation.
364      * @return total residual.
365      */
366     private double totalResidual(final EuclideanTransformation3D transformation) {
367         var result = 0.0;
368 
369         final var nSamples = inliers.length();
370         for (var i = 0; i < nSamples; i++) {
371             if (inliers.get(i)) {
372                 // sample is inlier
373                 final var inputPoint = samples1.get(i);
374                 final var outputPoint = samples2.get(i);
375                 inputPoint.normalize();
376                 outputPoint.normalize();
377                 result += residual(transformation, inputPoint, outputPoint);
378             }
379         }
380 
381         return result;
382     }
383 }