View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.geometry.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.geometry.AffineTransformation2D;
20  import com.irurueta.geometry.CoordinatesType;
21  import com.irurueta.geometry.Point2D;
22  import com.irurueta.geometry.estimators.LockedException;
23  import com.irurueta.geometry.estimators.NotReadyException;
24  import com.irurueta.numerical.EvaluationException;
25  import com.irurueta.numerical.GradientEstimator;
26  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
27  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
28  import com.irurueta.numerical.robust.InliersData;
29  
30  import java.util.BitSet;
31  import java.util.List;
32  
33  /**
34   * A 2D affine transformation refiner using point correspondences.
35   * This class takes into account an initial estimation, inlier point matches
36   * and their residuals to find a solution that minimizes error of inliers in
37   * LMSE terms.
38   * Typically, a refiner is used by a robust estimator, however it can also be
39   * useful in some other situations.
40   */
41  @SuppressWarnings("DuplicatedCode")
42  public class PointCorrespondenceAffineTransformation2DRefiner extends AffineTransformation2DRefiner<Point2D, Point2D> {
43  
44      /**
45       * Point to be reused when computing residuals.
46       */
47      private final Point2D residualTestPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
48  
49      /**
50       * Constructor.
51       */
52      public PointCorrespondenceAffineTransformation2DRefiner() {
53      }
54  
55      /**
56       * Constructor.
57       *
58       * @param initialEstimation           initial estimation to be set.
59       * @param keepCovariance              true if covariance of estimation must be kept after
60       *                                    refinement, false otherwise.
61       * @param inliers                     set indicating which of the provided matches are inliers.
62       * @param residuals                   residuals for matched samples.
63       * @param numInliers                  number of inliers on initial estimation.
64       * @param samples1                    1st set of paired samples.
65       * @param samples2                    2nd set of paired samples.
66       * @param refinementStandardDeviation standard deviation used for
67       *                                    Levenberg-Marquardt fitting.
68       */
69      public PointCorrespondenceAffineTransformation2DRefiner(
70              final AffineTransformation2D initialEstimation, final boolean keepCovariance, final BitSet inliers,
71              final double[] residuals, final int numInliers, final List<Point2D> samples1, final List<Point2D> samples2,
72              final double refinementStandardDeviation) {
73          super(initialEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2,
74                  refinementStandardDeviation);
75      }
76  
77      /**
78       * Constructor.
79       *
80       * @param initialEstimation           initial estimation to be set.
81       * @param keepCovariance              true if covariance of estimation must be kept after
82       *                                    refinement, false otherwise.
83       * @param inliersData                 inlier data, typically obtained from a robust
84       *                                    estimator.
85       * @param samples1                    1st set of paired samples.
86       * @param samples2                    2nd set of paired samples.
87       * @param refinementStandardDeviation standard deviation used for
88       *                                    Levenberg-Marquardt fitting.
89       */
90      public PointCorrespondenceAffineTransformation2DRefiner(
91              final AffineTransformation2D initialEstimation, final boolean keepCovariance, final InliersData inliersData,
92              final List<Point2D> samples1, final List<Point2D> samples2, final double refinementStandardDeviation) {
93          super(initialEstimation, keepCovariance, inliersData, samples1, samples2, refinementStandardDeviation);
94      }
95  
96      /**
97       * Refines provided initial estimation.
98       * This method always sets a value into provided result instance regardless
99       * of the fact that error has actually improved in LMSE terms or not.
100      *
101      * @param result instance where refined estimation will be stored.
102      * @return true if result improves (error decreases) in LMSE terms respect
103      * to initial estimation, false if no improvement has been achieved.
104      * @throws NotReadyException if not enough input data has been provided.
105      * @throws LockedException   if estimator is locked because refinement is
106      *                           already in progress.
107      * @throws RefinerException  if refinement fails for some reason (e.g. unable
108      *                           to converge to a result).
109      */
110     @Override
111     public boolean refine(final AffineTransformation2D result) throws NotReadyException, LockedException,
112             RefinerException {
113         if (isLocked()) {
114             throw new LockedException();
115         }
116         if (!isReady()) {
117             throw new NotReadyException();
118         }
119 
120         locked = true;
121 
122         if (listener != null) {
123             listener.onRefineStart(this, initialEstimation);
124         }
125 
126         final var initialTotalResidual = totalResidual(initialEstimation);
127 
128         try {
129             final var initParams = new double[AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS
130                     + AffineTransformation2D.NUM_TRANSLATION_COORDS];
131             // copy values for A matrix
132             System.arraycopy(initialEstimation.getA().getBuffer(), 0, initParams, 0,
133                     AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS);
134             // copy values for translation
135             System.arraycopy(initialEstimation.getTranslation(), 0, initParams,
136                     AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS,
137                     AffineTransformation2D.NUM_TRANSLATION_COORDS);
138 
139             // output values to be fitted/optimized will contain residuals
140             final var y = new double[numInliers];
141             // input values will contain 2 sets of 2D points to compute residuals
142             final var nDims = 2 * Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH;
143             final var x = new Matrix(numInliers, nDims);
144             final var nSamples = inliers.length();
145             var pos = 0;
146             for (var i = 0; i < nSamples; i++) {
147                 if (inliers.get(i)) {
148                     // sample is inlier
149                     final var inputPoint = samples1.get(i);
150                     final var outputPoint = samples2.get(i);
151                     inputPoint.normalize();
152                     outputPoint.normalize();
153                     x.setElementAt(pos, 0, inputPoint.getHomX());
154                     x.setElementAt(pos, 1, inputPoint.getHomY());
155                     x.setElementAt(pos, 2, inputPoint.getHomW());
156                     x.setElementAt(pos, 3, outputPoint.getHomX());
157                     x.setElementAt(pos, 4, outputPoint.getHomY());
158                     x.setElementAt(pos, 5, outputPoint.getHomW());
159 
160                     y[pos] = residuals[i];
161                     pos++;
162                 }
163             }
164 
165             final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
166 
167                 private final Point2D inputPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
168 
169                 private final Point2D outputPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
170 
171                 private final AffineTransformation2D transformation = new AffineTransformation2D();
172 
173                 private final GradientEstimator gradientEstimator = new GradientEstimator(params -> {
174                     // copy values for A matrix
175                     System.arraycopy(params, 0, transformation.getA().getBuffer(), 0,
176                             AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS);
177                     // copy values for translation
178                     System.arraycopy(params,
179                             AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS,
180                             transformation.getTranslation(), 0, AffineTransformation2D.NUM_TRANSLATION_COORDS);
181 
182                     return residual(transformation, inputPoint, outputPoint);
183                 });
184 
185                 @Override
186                 public int getNumberOfDimensions() {
187                     return nDims;
188                 }
189 
190                 @Override
191                 public double[] createInitialParametersArray() {
192                     return initParams;
193                 }
194 
195                 @Override
196                 public double evaluate(final int i, final double[] point, final double[] params,
197                                        final double[] derivatives) throws EvaluationException {
198                     inputPoint.setHomogeneousCoordinates(point[0], point[1], point[2]);
199                     outputPoint.setHomogeneousCoordinates(point[3], point[4], point[5]);
200 
201                     // copy values for A matrix
202                     System.arraycopy(params, 0, transformation.getA().getBuffer(), 0,
203                             AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS);
204                     // copy values for translation
205                     System.arraycopy(params,
206                             AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS,
207                             transformation.getTranslation(), 0, AffineTransformation2D.NUM_TRANSLATION_COORDS);
208 
209                     final var y = residual(transformation, inputPoint, outputPoint);
210                     gradientEstimator.gradient(params, derivatives);
211 
212                     return y;
213                 }
214             };
215 
216             final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y,
217                     getRefinementStandardDeviation());
218 
219             fitter.fit();
220 
221             // obtain estimated params
222             final var params = fitter.getA();
223 
224             // update transformation
225 
226             // copy values for A matrix
227             System.arraycopy(params, 0, result.getA().getBuffer(), 0,
228                     AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS);
229             // copy values for translation
230             System.arraycopy(params, AffineTransformation2D.INHOM_COORDS * AffineTransformation2D.INHOM_COORDS,
231                     result.getTranslation(), 0, AffineTransformation2D.NUM_TRANSLATION_COORDS);
232 
233             if (keepCovariance) {
234                 // keep covariance
235                 covariance = fitter.getCovar();
236             }
237 
238             final var finalTotalResidual = totalResidual(result);
239             final var errorDecreased = finalTotalResidual < initialTotalResidual;
240 
241             if (listener != null) {
242                 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
243             }
244 
245             return errorDecreased;
246 
247         } catch (final Exception e) {
248             throw new RefinerException(e);
249         } finally {
250             locked = false;
251         }
252     }
253 
254     /**
255      * Computes the residual between the affine transformation and a pair of
256      * matched points.
257      *
258      * @param transformation a transformation.
259      * @param inputPoint     input 2D point.
260      * @param outputPoint    output 2D point.
261      * @return residual.
262      */
263     private double residual(final AffineTransformation2D transformation, final Point2D inputPoint,
264                             final Point2D outputPoint) {
265         inputPoint.normalize();
266         outputPoint.normalize();
267 
268         transformation.transform(inputPoint, residualTestPoint);
269         return residualTestPoint.distanceTo(outputPoint);
270     }
271 
272     /**
273      * Computes total residual among all provided inlier samples.
274      *
275      * @param transformation a transformation.
276      * @return total residual.
277      */
278     private double totalResidual(final AffineTransformation2D transformation) {
279         var result = 0.0;
280 
281         final var nSamples = inliers.length();
282         for (var i = 0; i < nSamples; i++) {
283             if (inliers.get(i)) {
284                 // sample is inlier
285                 final var inputPoint = samples1.get(i);
286                 final var outputPoint = samples2.get(i);
287                 inputPoint.normalize();
288                 outputPoint.normalize();
289                 result += residual(transformation, inputPoint, outputPoint);
290             }
291         }
292 
293         return result;
294     }
295 }