View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.epipolar.refiners;
17  
18  import com.irurueta.algebra.Matrix;
19  import com.irurueta.ar.epipolar.FundamentalMatrix;
20  import com.irurueta.ar.epipolar.InvalidFundamentalMatrixException;
21  import com.irurueta.geometry.CoordinatesType;
22  import com.irurueta.geometry.InhomogeneousPoint2D;
23  import com.irurueta.geometry.Point2D;
24  import com.irurueta.geometry.Transformation2D;
25  import com.irurueta.geometry.estimators.LockedException;
26  import com.irurueta.geometry.estimators.NotReadyException;
27  import com.irurueta.geometry.refiners.RefinerException;
28  import com.irurueta.numerical.EvaluationException;
29  import com.irurueta.numerical.GradientEstimator;
30  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
31  import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
32  import com.irurueta.numerical.robust.InliersData;
33  
34  import java.util.BitSet;
35  import java.util.List;
36  
37  /**
38   * Refines the epipole of a fundamental matrix formed by an initial epipole
39   * estimation and an estimated homography.
40   * Any fundamental matrix can be expressed as F = [e']x*H, where
41   * e' is the epipole on the right view, []x is the skew matrix, and H is a
42   * non-degenerate homography.
43   * This class refines an initial epipole so that residuals from provided point
44   * correspondences generating fundamental matrix F are reduced.
45   * This class is especially useful in cases where geometry of the scene is
46   * degenerate (e.g. planar scene) and provided point correspondences would
47   * generate an inaccurate fundamental matrix.
48   * This implementation uses Levenberg-Marquardt algorithm for a fast cost
49   * optimization and uses inhomogeneous points for epipole refinement,
50   * which makes it unsuitable for cases when epipoles are at or near infinity
51   * (i.e. pure lateral translations).
52   */
53  public class InhomogeneousRightEpipoleRefiner extends RightEpipoleRefiner {
54  
55      /**
56       * Maximum allowed number of refinement iterations.
57       */
58      public static final int MAX_ITERS = 10;
59  
60      /**
61       * Constructor.
62       */
63      public InhomogeneousRightEpipoleRefiner() {
64      }
65  
66      /**
67       * Constructor.
68       *
69       * @param initialEpipoleEstimation    initial right epipole estimation to be
70       *                                    set and refined.
71       * @param keepCovariance              true if covariance of estimation must be kept after
72       *                                    refinement, false otherwise.
73       * @param inliers                     set indicating which of the provided matches are inliers.
74       * @param residuals                   residuals for matched samples.
75       * @param numInliers                  number of inliers on initial estimation.
76       * @param samples1                    1st set of paired samples.
77       * @param samples2                    2nd set of paired samples.
78       * @param refinementStandardDeviation standard deviation used for
79       *                                    Levenberg-Marquardt fitting.
80       * @param homography                  homography relating samples in two views, which is
81       *                                    used to generate a fundamental matrix and its corresponding
82       *                                    epipolar geometry.
83       */
84      public InhomogeneousRightEpipoleRefiner(
85              final Point2D initialEpipoleEstimation, final boolean keepCovariance, final BitSet inliers,
86              final double[] residuals, final int numInliers, final List<Point2D> samples1, final List<Point2D> samples2,
87              final double refinementStandardDeviation, final Transformation2D homography) {
88          super(initialEpipoleEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2,
89                  refinementStandardDeviation, homography);
90      }
91  
92      /**
93       * Constructor.
94       *
95       * @param initialEpipoleEstimation    initial right epipole estimation to be
96       *                                    set and refined.
97       * @param keepCovariance              true if covariance of estimation must be kept after
98       *                                    refinement, false otherwise.
99       * @param inliersData                 inlier data, typically obtained from a robust
100      *                                    estimator.
101      * @param samples1                    1st set of paired samples.
102      * @param samples2                    2nd set of paired samples.
103      * @param refinementStandardDeviation standard deviation used for
104      *                                    Levenberg-Marquardt fitting.
105      * @param homography                  homography relating samples in two views, which is used
106      *                                    to generate a fundamental matrix and its corresponding
107      *                                    epipolar geometry.
108      */
109     public InhomogeneousRightEpipoleRefiner(
110             final Point2D initialEpipoleEstimation, final boolean keepCovariance, final InliersData inliersData,
111             final List<Point2D> samples1, final List<Point2D> samples2, final double refinementStandardDeviation,
112             final Transformation2D homography) {
113         super(initialEpipoleEstimation, keepCovariance, inliersData, samples1, samples2, refinementStandardDeviation,
114                 homography);
115     }
116 
117     /**
118      * Refines provided initial right epipole estimation.
119      * This method always sets a value into provided result instance regardless
120      * of the fact that error has actually improved in LMSE terms or not.
121      *
122      * @param result instance where refined estimation will be stored.
123      * @return true if result improves (decreases) in LMSE terms respect to
124      * initial estimation, false if no improvement has been achieved.
125      * @throws NotReadyException if not enough input data has been provided.
126      * @throws LockedException   if estimator is locked because refinement is
127      *                           already in progress.
128      * @throws RefinerException  if refinement fails for some reason (e.g. unable
129      *                           to converge to a result).
130      */
131 
132     @Override
133     public boolean refine(final Point2D result) throws NotReadyException, LockedException, RefinerException {
134         if (isLocked()) {
135             throw new LockedException();
136         }
137         if (!isReady()) {
138             throw new NotReadyException();
139         }
140 
141         locked = true;
142 
143         if (listener != null) {
144             listener.onRefineStart(this, initialEstimation);
145         }
146 
147         try {
148             final var epipole = new InhomogeneousPoint2D(initialEstimation);
149             var errorDecreased = false;
150             boolean iterErrorDecreased;
151             var numIter = 0;
152             do {
153                 final var fundamentalMatrix = new FundamentalMatrix();
154                 computeFundamentalMatrix(homography, epipole, fundamentalMatrix);
155                 final var initialTotalResidual = totalResidual(fundamentalMatrix);
156                 final var initParams = epipole.asArray();
157 
158                 // output values to be fitted/optimized will contain residuals
159                 final var y = new double[numInliers];
160                 // input values will contain 2 points to compute residuals
161                 final var nDims = 2 * Point2D.POINT2D_INHOMOGENEOUS_COORDINATES_LENGTH;
162                 final var x = new Matrix(numInliers, nDims);
163                 final var nSamples = inliers.length();
164                 var pos = 0;
165                 Point2D leftPoint;
166                 Point2D rightPoint;
167                 for (var i = 0; i < nSamples; i++) {
168                     if (inliers.get(i)) {
169                         // sample is inlier
170                         leftPoint = samples1.get(i);
171                         rightPoint = samples2.get(i);
172                         leftPoint.normalize();
173                         rightPoint.normalize();
174                         x.setElementAt(pos, 0, leftPoint.getInhomX());
175                         x.setElementAt(pos, 1, leftPoint.getInhomY());
176                         x.setElementAt(pos, 2, rightPoint.getInhomX());
177                         x.setElementAt(pos, 3, rightPoint.getInhomY());
178 
179                         y[pos] = residuals[i];
180                         pos++;
181                     }
182                 }
183 
184                 final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
185 
186                     private final Point2D leftPoint = Point2D.create(CoordinatesType.INHOMOGENEOUS_COORDINATES);
187 
188                     private final Point2D rightPoint = Point2D.create(CoordinatesType.INHOMOGENEOUS_COORDINATES);
189 
190                     private final InhomogeneousPoint2D epipole = new InhomogeneousPoint2D();
191 
192                     private final FundamentalMatrix fundMatrix = new FundamentalMatrix();
193 
194                     @SuppressWarnings("DuplicatedCode")
195                     private final GradientEstimator gradientEstimator = new GradientEstimator(point -> {
196 
197                         try {
198                             epipole.setCoordinates(point);
199                             computeFundamentalMatrix(homography, epipole, fundMatrix);
200                             return residual(fundMatrix, leftPoint, rightPoint);
201                         } catch (final InvalidFundamentalMatrixException e) {
202                             throw new EvaluationException(e);
203                         }
204                     });
205 
206                     @Override
207                     public int getNumberOfDimensions() {
208                         return nDims;
209                     }
210 
211                     @Override
212                     public double[] createInitialParametersArray() {
213                         return initParams;
214                     }
215 
216                     @Override
217                     public double evaluate(
218                             final int i, final double[] point, final double[] params, final double[] derivatives)
219                             throws EvaluationException {
220                         try {
221                             leftPoint.setInhomogeneousCoordinates(point[0], point[1]);
222                             rightPoint.setInhomogeneousCoordinates(point[2], point[3]);
223 
224                             epipole.setCoordinates(params);
225                             computeFundamentalMatrix(homography, epipole, fundMatrix);
226 
227                             final var y = residual(fundMatrix, leftPoint, rightPoint);
228                             gradientEstimator.gradient(params, derivatives);
229 
230                             return y;
231                         } catch (final InvalidFundamentalMatrixException e) {
232                             throw new EvaluationException(e);
233                         }
234                     }
235                 };
236 
237                 final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y,
238                         getRefinementStandardDeviation());
239 
240                 fitter.fit();
241 
242                 // obtain estimated params
243                 final var params = fitter.getA();
244 
245                 // update initial epipole for next iteration
246                 epipole.setInhomogeneousCoordinates(params[0], params[1]);
247 
248                 computeFundamentalMatrix(homography, epipole, fundamentalMatrix);
249 
250                 final var finalTotalResidual = totalResidual(fundamentalMatrix);
251                 iterErrorDecreased = finalTotalResidual < initialTotalResidual;
252                 if (iterErrorDecreased || !errorDecreased) {
253                     errorDecreased = true;
254                     // update final result
255                     result.setInhomogeneousCoordinates(params[0], params[1]);
256 
257                     if (keepCovariance) {
258                         // keep covariance
259                         covariance = fitter.getCovar();
260                     }
261                 }
262                 numIter++;
263             } while (iterErrorDecreased && numIter < MAX_ITERS);
264 
265             if (listener != null) {
266                 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
267             }
268 
269             return errorDecreased;
270 
271         } catch (final Exception e) {
272             throw new RefinerException(e);
273         } finally {
274             locked = false;
275         }
276     }
277 }