1 /*
2 * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.ar.epipolar.refiners;
17
18 import com.irurueta.algebra.Matrix;
19 import com.irurueta.ar.epipolar.FundamentalMatrix;
20 import com.irurueta.ar.epipolar.InvalidFundamentalMatrixException;
21 import com.irurueta.geometry.CoordinatesType;
22 import com.irurueta.geometry.InhomogeneousPoint2D;
23 import com.irurueta.geometry.Point2D;
24 import com.irurueta.geometry.Transformation2D;
25 import com.irurueta.geometry.estimators.LockedException;
26 import com.irurueta.geometry.estimators.NotReadyException;
27 import com.irurueta.geometry.refiners.RefinerException;
28 import com.irurueta.numerical.EvaluationException;
29 import com.irurueta.numerical.GradientEstimator;
30 import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFitter;
31 import com.irurueta.numerical.fitting.LevenbergMarquardtMultiDimensionFunctionEvaluator;
32 import com.irurueta.numerical.robust.InliersData;
33
34 import java.util.BitSet;
35 import java.util.List;
36
37 /**
38 * Refines the epipole of a fundamental matrix formed by an initial epipole
39 * estimation and an estimated homography.
40 * Any fundamental matrix can be expressed as F = [e']x*H, where
41 * e' is the epipole on the right view, []x is the skew matrix, and H is a
42 * non-degenerate homography.
43 * This class refines an initial epipole so that residuals from provided point
44 * correspondences generating fundamental matrix F are reduced.
45 * This class is especially useful in cases where geometry of the scene is
46 * degenerate (e.g. planar scene) and provided point correspondences would
47 * generate an inaccurate fundamental matrix.
48 * This implementation uses Levenberg-Marquardt algorithm for a fast cost
49 * optimization and uses inhomogeneous points for epipole refinement,
50 * which makes it unsuitable for cases when epipoles are at or near infinity
51 * (i.e. pure lateral translations).
52 */
53 public class InhomogeneousRightEpipoleRefiner extends RightEpipoleRefiner {
54
55 /**
56 * Maximum allowed number of refinement iterations.
57 */
58 public static final int MAX_ITERS = 10;
59
60 /**
61 * Constructor.
62 */
63 public InhomogeneousRightEpipoleRefiner() {
64 }
65
66 /**
67 * Constructor.
68 *
69 * @param initialEpipoleEstimation initial right epipole estimation to be
70 * set and refined.
71 * @param keepCovariance true if covariance of estimation must be kept after
72 * refinement, false otherwise.
73 * @param inliers set indicating which of the provided matches are inliers.
74 * @param residuals residuals for matched samples.
75 * @param numInliers number of inliers on initial estimation.
76 * @param samples1 1st set of paired samples.
77 * @param samples2 2nd set of paired samples.
78 * @param refinementStandardDeviation standard deviation used for
79 * Levenberg-Marquardt fitting.
80 * @param homography homography relating samples in two views, which is
81 * used to generate a fundamental matrix and its corresponding
82 * epipolar geometry.
83 */
84 public InhomogeneousRightEpipoleRefiner(
85 final Point2D initialEpipoleEstimation, final boolean keepCovariance, final BitSet inliers,
86 final double[] residuals, final int numInliers, final List<Point2D> samples1, final List<Point2D> samples2,
87 final double refinementStandardDeviation, final Transformation2D homography) {
88 super(initialEpipoleEstimation, keepCovariance, inliers, residuals, numInliers, samples1, samples2,
89 refinementStandardDeviation, homography);
90 }
91
92 /**
93 * Constructor.
94 *
95 * @param initialEpipoleEstimation initial right epipole estimation to be
96 * set and refined.
97 * @param keepCovariance true if covariance of estimation must be kept after
98 * refinement, false otherwise.
99 * @param inliersData inlier data, typically obtained from a robust
100 * estimator.
101 * @param samples1 1st set of paired samples.
102 * @param samples2 2nd set of paired samples.
103 * @param refinementStandardDeviation standard deviation used for
104 * Levenberg-Marquardt fitting.
105 * @param homography homography relating samples in two views, which is used
106 * to generate a fundamental matrix and its corresponding
107 * epipolar geometry.
108 */
109 public InhomogeneousRightEpipoleRefiner(
110 final Point2D initialEpipoleEstimation, final boolean keepCovariance, final InliersData inliersData,
111 final List<Point2D> samples1, final List<Point2D> samples2, final double refinementStandardDeviation,
112 final Transformation2D homography) {
113 super(initialEpipoleEstimation, keepCovariance, inliersData, samples1, samples2, refinementStandardDeviation,
114 homography);
115 }
116
117 /**
118 * Refines provided initial right epipole estimation.
119 * This method always sets a value into provided result instance regardless
120 * of the fact that error has actually improved in LMSE terms or not.
121 *
122 * @param result instance where refined estimation will be stored.
123 * @return true if result improves (decreases) in LMSE terms respect to
124 * initial estimation, false if no improvement has been achieved.
125 * @throws NotReadyException if not enough input data has been provided.
126 * @throws LockedException if estimator is locked because refinement is
127 * already in progress.
128 * @throws RefinerException if refinement fails for some reason (e.g. unable
129 * to converge to a result).
130 */
131
132 @Override
133 public boolean refine(final Point2D result) throws NotReadyException, LockedException, RefinerException {
134 if (isLocked()) {
135 throw new LockedException();
136 }
137 if (!isReady()) {
138 throw new NotReadyException();
139 }
140
141 locked = true;
142
143 if (listener != null) {
144 listener.onRefineStart(this, initialEstimation);
145 }
146
147 try {
148 final var epipole = new InhomogeneousPoint2D(initialEstimation);
149 var errorDecreased = false;
150 boolean iterErrorDecreased;
151 var numIter = 0;
152 do {
153 final var fundamentalMatrix = new FundamentalMatrix();
154 computeFundamentalMatrix(homography, epipole, fundamentalMatrix);
155 final var initialTotalResidual = totalResidual(fundamentalMatrix);
156 final var initParams = epipole.asArray();
157
158 // output values to be fitted/optimized will contain residuals
159 final var y = new double[numInliers];
160 // input values will contain 2 points to compute residuals
161 final var nDims = 2 * Point2D.POINT2D_INHOMOGENEOUS_COORDINATES_LENGTH;
162 final var x = new Matrix(numInliers, nDims);
163 final var nSamples = inliers.length();
164 var pos = 0;
165 Point2D leftPoint;
166 Point2D rightPoint;
167 for (var i = 0; i < nSamples; i++) {
168 if (inliers.get(i)) {
169 // sample is inlier
170 leftPoint = samples1.get(i);
171 rightPoint = samples2.get(i);
172 leftPoint.normalize();
173 rightPoint.normalize();
174 x.setElementAt(pos, 0, leftPoint.getInhomX());
175 x.setElementAt(pos, 1, leftPoint.getInhomY());
176 x.setElementAt(pos, 2, rightPoint.getInhomX());
177 x.setElementAt(pos, 3, rightPoint.getInhomY());
178
179 y[pos] = residuals[i];
180 pos++;
181 }
182 }
183
184 final var evaluator = new LevenbergMarquardtMultiDimensionFunctionEvaluator() {
185
186 private final Point2D leftPoint = Point2D.create(CoordinatesType.INHOMOGENEOUS_COORDINATES);
187
188 private final Point2D rightPoint = Point2D.create(CoordinatesType.INHOMOGENEOUS_COORDINATES);
189
190 private final InhomogeneousPoint2D epipole = new InhomogeneousPoint2D();
191
192 private final FundamentalMatrix fundMatrix = new FundamentalMatrix();
193
194 @SuppressWarnings("DuplicatedCode")
195 private final GradientEstimator gradientEstimator = new GradientEstimator(point -> {
196
197 try {
198 epipole.setCoordinates(point);
199 computeFundamentalMatrix(homography, epipole, fundMatrix);
200 return residual(fundMatrix, leftPoint, rightPoint);
201 } catch (final InvalidFundamentalMatrixException e) {
202 throw new EvaluationException(e);
203 }
204 });
205
206 @Override
207 public int getNumberOfDimensions() {
208 return nDims;
209 }
210
211 @Override
212 public double[] createInitialParametersArray() {
213 return initParams;
214 }
215
216 @Override
217 public double evaluate(
218 final int i, final double[] point, final double[] params, final double[] derivatives)
219 throws EvaluationException {
220 try {
221 leftPoint.setInhomogeneousCoordinates(point[0], point[1]);
222 rightPoint.setInhomogeneousCoordinates(point[2], point[3]);
223
224 epipole.setCoordinates(params);
225 computeFundamentalMatrix(homography, epipole, fundMatrix);
226
227 final var y = residual(fundMatrix, leftPoint, rightPoint);
228 gradientEstimator.gradient(params, derivatives);
229
230 return y;
231 } catch (final InvalidFundamentalMatrixException e) {
232 throw new EvaluationException(e);
233 }
234 }
235 };
236
237 final var fitter = new LevenbergMarquardtMultiDimensionFitter(evaluator, x, y,
238 getRefinementStandardDeviation());
239
240 fitter.fit();
241
242 // obtain estimated params
243 final var params = fitter.getA();
244
245 // update initial epipole for next iteration
246 epipole.setInhomogeneousCoordinates(params[0], params[1]);
247
248 computeFundamentalMatrix(homography, epipole, fundamentalMatrix);
249
250 final var finalTotalResidual = totalResidual(fundamentalMatrix);
251 iterErrorDecreased = finalTotalResidual < initialTotalResidual;
252 if (iterErrorDecreased || !errorDecreased) {
253 errorDecreased = true;
254 // update final result
255 result.setInhomogeneousCoordinates(params[0], params[1]);
256
257 if (keepCovariance) {
258 // keep covariance
259 covariance = fitter.getCovar();
260 }
261 }
262 numIter++;
263 } while (iterErrorDecreased && numIter < MAX_ITERS);
264
265 if (listener != null) {
266 listener.onRefineEnd(this, initialEstimation, result, errorDecreased);
267 }
268
269 return errorDecreased;
270
271 } catch (final Exception e) {
272 throw new RefinerException(e);
273 } finally {
274 locked = false;
275 }
276 }
277 }