View Javadoc
1   /*
2    * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.epipolar;
17  
18  import com.irurueta.algebra.AlgebraException;
19  import com.irurueta.algebra.Complex;
20  import com.irurueta.algebra.Matrix;
21  import com.irurueta.geometry.CoordinatesType;
22  import com.irurueta.geometry.GeometryException;
23  import com.irurueta.geometry.HomogeneousPoint2D;
24  import com.irurueta.geometry.Point2D;
25  import com.irurueta.geometry.ProjectiveTransformation2D;
26  import com.irurueta.geometry.estimators.NotReadyException;
27  import com.irurueta.numerical.NumericalException;
28  import com.irurueta.numerical.roots.LaguerrePolynomialRootsEstimator;
29  
30  /**
31   * Fixes a single matched pair of points so that they perfectly follow a given
32   * epipolar geometry using the Gold Standard method, which is capable to
33   * completely remove errors assuming their gaussianity.
34   * When matching points typically the matching precision is about 1 pixel,
35   * however this makes that matched points under a given epipolar geometry (i.e.
36   * fundamental or essential matrix), do not lie perfectly on the corresponding
37   * epipolar plane or epipolar lines.
38   * The consequence is that triangularization of these matches will fail or
39   * produce inaccurate results.
40   * By fixing matched points using a corrector following a given epipolar
41   * geometry, this effect is alleviated.
42   * This corrector uses the Gold Standard method, which is more expensive to
43   * compute than the Sampson approximation, but is capable to remove larger
44   * errors assuming their gaussianity. Contrary to the Sampson corrector, the
45   * Gold Standard method might fail in some situations, while in those cases
46   * probably the Sampson corrector produces wrong results without failing.
47   */
48  public class GoldStandardSingleCorrector extends SingleCorrector {
49  
50      /**
51       * A tiny value.
52       */
53      private static final double EPS = 1e-6;
54  
55      /**
56       * Constructor.
57       */
58      public GoldStandardSingleCorrector() {
59          super();
60      }
61  
62      /**
63       * Constructor.
64       *
65       * @param fundamentalMatrix fundamental matrix defining the epipolar
66       *                          geometry.
67       */
68      public GoldStandardSingleCorrector(final FundamentalMatrix fundamentalMatrix) {
69          super(fundamentalMatrix);
70      }
71  
72      /**
73       * Constructor.
74       *
75       * @param leftPoint  matched point on left view to be corrected.
76       * @param rightPoint matched point on right view to be corrected.
77       */
78      public GoldStandardSingleCorrector(final Point2D leftPoint, final Point2D rightPoint) {
79          super(leftPoint, rightPoint);
80      }
81  
82      /**
83       * Constructor.
84       *
85       * @param leftPoint         matched point on left view to be corrected.
86       * @param rightPoint        matched point on right view to be corrected.
87       * @param fundamentalMatrix fundamental matrix defining an epipolar geometry.
88       */
89      public GoldStandardSingleCorrector(final Point2D leftPoint, final Point2D rightPoint,
90                                         final FundamentalMatrix fundamentalMatrix) {
91          super(leftPoint, rightPoint, fundamentalMatrix);
92      }
93  
94      /**
95       * Corrects the pair of provided matched points to be corrected.
96       *
97       * @throws NotReadyException   if this instance is not ready (either points or
98       *                             fundamental matrix has not been provided yet).
99       * @throws CorrectionException if correction fails.
100      */
101     @Override
102     public void correct() throws NotReadyException, CorrectionException {
103         if (!isReady()) {
104             throw new NotReadyException();
105         }
106 
107         leftCorrectedPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
108         rightCorrectedPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
109         correct(leftPoint, rightPoint, fundamentalMatrix, leftCorrectedPoint, rightCorrectedPoint);
110     }
111 
112     /**
113      * Gets type of correction being used.
114      *
115      * @return type of correction.
116      */
117     @Override
118     public CorrectorType getType() {
119         return CorrectorType.GOLD_STANDARD;
120     }
121 
122     /**
123      * Corrects the pair of provided matched points to be corrected using
124      * provided fundamental matrix and stores the corrected points into provided
125      * instances.
126      *
127      * @param leftPoint           point on left view to be corrected.
128      * @param rightPoint          point on right view to be corrected.
129      * @param fundamentalMatrix   fundamental matrix defining the epipolar
130      *                            geometry.
131      * @param correctedLeftPoint  point on left view after correction.
132      * @param correctedRightPoint point on right view after correction.
133      * @throws NotReadyException   if provided fundamental matrix is not ready.
134      * @throws CorrectionException if correction fails.
135      */
136     @SuppressWarnings("DuplicatedCode")
137     public static void correct(
138             final Point2D leftPoint, final Point2D rightPoint, final FundamentalMatrix fundamentalMatrix,
139             final Point2D correctedLeftPoint, final Point2D correctedRightPoint) throws NotReadyException,
140             CorrectionException {
141 
142         // normalize to increase accuracy
143         leftPoint.normalize();
144         rightPoint.normalize();
145         fundamentalMatrix.normalize();
146 
147         try {
148             // create transformations to move left and right points to the origin
149             // (0,0,1)
150             final var leftTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
151                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
152             // add terms to move left point to origin
153             leftTranslationMatrix.setElementAt(0, 2, -leftPoint.getInhomX());
154             leftTranslationMatrix.setElementAt(1, 2, -leftPoint.getInhomY());
155 
156             // create inverse transformation
157             var invLeftTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
158                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
159             // add terms to obtain inverse transformation
160             invLeftTranslationMatrix.setElementAt(0, 2, leftPoint.getInhomX());
161             invLeftTranslationMatrix.setElementAt(1, 2, leftPoint.getInhomY());
162 
163             final var rightTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
164                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
165             // add terms to move right point to origin
166             rightTranslationMatrix.setElementAt(0, 2, -rightPoint.getInhomX());
167             rightTranslationMatrix.setElementAt(1, 2, -rightPoint.getInhomY());
168 
169             // create inverse and transposed transformation
170             var invTransRightTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
171                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
172             // add terms to obtain inverse transformation
173             invTransRightTranslationMatrix.setElementAt(2, 0, rightPoint.getInhomX());
174             invTransRightTranslationMatrix.setElementAt(2, 1, rightPoint.getInhomY());
175 
176             // because points have been transformed using left and right
177             // translations (T1 and T2 respectively), then fundamental matrix
178             // becomes (T2^-1)'*F*T1^-1
179             var fundInternalMatrix = fundamentalMatrix.getInternalMatrix();
180             fundInternalMatrix.multiply(invLeftTranslationMatrix);
181             invTransRightTranslationMatrix.multiply(fundInternalMatrix);
182 
183             // compute transformations to rotate epipoles of transformed
184             // fundamental matrix so that they are located at e1 = (1,0,f1) and
185             // e2 = (1,0,f2)
186             final var transformedFundamentalMatrix = new FundamentalMatrix(invTransRightTranslationMatrix);
187             transformedFundamentalMatrix.computeEpipoles();
188             final var leftEpipole = transformedFundamentalMatrix.getLeftEpipole();
189             final var rightEpipole = transformedFundamentalMatrix.getRightEpipole();
190 
191             // normalize so that leftEpipole.x^2 + leftEpipole.y^2 = 1 and
192             // rightEpipole.x^2 + rightEpipole.y^2 = 1, that way transformations
193             // become rotations
194             final var normLeftEpipole = Math.pow(leftEpipole.getHomX(), 2.0) + Math.pow(leftEpipole.getHomY(), 2.0);
195             final var normRightEpipole = Math.pow(rightEpipole.getHomX(), 2.0) + Math.pow(rightEpipole.getHomY(), 2.0);
196 
197             leftEpipole.setHomogeneousCoordinates(
198                     leftEpipole.getHomX() / normLeftEpipole,
199                     leftEpipole.getHomY() / normLeftEpipole,
200                     leftEpipole.getHomW() / normLeftEpipole);
201             rightEpipole.setHomogeneousCoordinates(
202                     rightEpipole.getHomX() / normRightEpipole,
203                     rightEpipole.getHomY() / normRightEpipole,
204                     rightEpipole.getHomW() / normRightEpipole);
205 
206             final var leftRotationMatrix = new Matrix(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
207                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
208             leftRotationMatrix.setElementAt(0, 0, leftEpipole.getHomX());
209             leftRotationMatrix.setElementAt(1, 0, -leftEpipole.getHomY());
210             leftRotationMatrix.setElementAt(0, 1, leftEpipole.getHomY());
211             leftRotationMatrix.setElementAt(1, 1, leftEpipole.getHomX());
212             leftRotationMatrix.setElementAt(2, 2, 1.0);
213 
214             // the inverse of a rotation is its transpose
215             final var invLeftRotationMatrix = leftRotationMatrix.transposeAndReturnNew();
216 
217             final var rightRotationMatrix = new Matrix(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
218                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
219             rightRotationMatrix.setElementAt(0, 0, rightEpipole.getHomX());
220             rightRotationMatrix.setElementAt(1, 0, -rightEpipole.getHomY());
221             rightRotationMatrix.setElementAt(0, 1, rightEpipole.getHomY());
222             rightRotationMatrix.setElementAt(1, 1, rightEpipole.getHomX());
223             rightRotationMatrix.setElementAt(2, 2, 1.0);
224 
225             // again the inverse of rotation is its transpose
226             final var invRightRotationMatrix = rightRotationMatrix.transposeAndReturnNew();
227 
228             // and so the fundamental matrix now becomes: R2*(T2^-1)'*F*T1^-1*R1',
229             // where the middle matrices correspond to previous transformation
230             fundInternalMatrix = invTransRightTranslationMatrix;
231             fundInternalMatrix.multiply(invLeftRotationMatrix);
232             rightRotationMatrix.multiply(fundInternalMatrix);
233             invTransRightTranslationMatrix = rightRotationMatrix;
234 
235             // where F is a transformed fundamental matrix that has the following
236             // form:
237             // [f1*f2*d  -f2*c   -f2*d   ]
238             // [-f1*b    a       b       ]
239             // [-f1*d    c       d       ]
240 
241             // hence:
242             final var a = invTransRightTranslationMatrix.getElementAt(1, 1);
243             final var b = invTransRightTranslationMatrix.getElementAt(1, 2);
244             final var c = invTransRightTranslationMatrix.getElementAt(2, 1);
245             final var d = invTransRightTranslationMatrix.getElementAt(2, 2);
246             final double f1;
247             final double f2;
248             if (Math.abs(b) > Math.abs(d)) {
249                 f1 = -invTransRightTranslationMatrix.getElementAt(1, 0) / b;
250             } else {
251                 f1 = -invTransRightTranslationMatrix.getElementAt(2, 0) / d;
252             }
253             if (Math.abs(c) > Math.abs(d)) {
254                 f2 = -invTransRightTranslationMatrix.getElementAt(0, 1) / c;
255             } else {
256                 f2 = -invTransRightTranslationMatrix.getElementAt(0, 2) / d;
257             }
258 
259             // Hence the polynomial of degree 6 to solve corresponding to the derivative of s(t) is:
260             // g(t) = A*t^6 + B*t^5 + C*t^4 + D*t^3 + E*t^2 + F*t + G
261             // where:
262             final var tmp1 = Math.pow(a, 2.0) * d - a * b * c;
263             final var tmp2 = Math.pow(a, 2.0) + Math.pow(c, 2.0) * Math.pow(f2, 2.0);
264             final var tmp3 = a * b * d - Math.pow(b, 2.0) * c;
265             final var tmp4 = tmp3 * c;
266             final var tmp5 = a * b + c * d * Math.pow(f2, 2.0);
267             final var tmp6 = Math.pow(b, 2.0) + Math.pow(d, 2.0) * Math.pow(f2, 2.0);
268 
269             final var realA = -tmp1 * c * Math.pow(f1, 4.0);
270             final var realB = Math.pow(tmp2, 2.0) - ((Math.pow(a, 2) * d - a * b * c) * d + tmp4) * Math.pow(f1, 4.0);
271             final var realC = (4.0 * tmp2 * tmp5 - (2.0 * tmp1 * c * Math.pow(f1, 2.0) + tmp3 * d * Math.pow(f1, 4.0)));
272             final var realD = 2.0 * (tmp2 * tmp6 + 2.0 * Math.pow(tmp5, 2.0) - (tmp1 * d + tmp4) * Math.pow(f1, 2.0));
273             final var realE = (4.0 * tmp5 * tmp6 - (tmp1 * c + 2.0 * tmp3 * d * Math.pow(f1, 2.0)));
274             final var realF = (Math.pow(tmp6, 2.0) - (tmp1 * d + tmp4));
275             final var realG = -tmp3 * d;
276 
277             final var complexA = new Complex(realA);
278             final var complexB = new Complex(realB);
279             final var complexC = new Complex(realC);
280             final var complexD = new Complex(realD);
281             final var complexE = new Complex(realE);
282             final var complexF = new Complex(realF);
283             final var complexG = new Complex(realG);
284             final var polyParams = new Complex[]{complexG, complexF, complexE, complexD, complexC, complexB, complexA};
285             final var rootEstimator = new LaguerrePolynomialRootsEstimator(polyParams);
286             rootEstimator.estimate();
287             final var roots = rootEstimator.getRoots();
288 
289             // evaluate polynomial s(t) on each root of its derivative to obtain
290             // the global minima (we discard non-real roots)
291             var minimum = Double.MAX_VALUE;
292             Complex bestRoot = null;
293             for (final var root : roots) {
294                 // skip solutions having an imaginary part
295                 if (Math.abs(root.getImaginary()) > EPS) {
296                     continue;
297                 }
298 
299                 final var value = s(root.getReal(), a, b, c, d, f1, f2);
300                 if (value < minimum) {
301                     minimum = value;
302                     bestRoot = root;
303                 }
304             }
305 
306             if (bestRoot == null) {
307                 throw new CorrectionException();
308             }
309 
310             final var tmin = bestRoot.getReal();
311 
312             // and so, the transformed left point is (-tmin^2*f1, tmin, 1 + (tmin*f1)^2)
313             final var transformedLeftPoint = new HomogeneousPoint2D(
314                     -Math.pow(tmin, 2.0) * f1, tmin, 1.0 + Math.pow(tmin * f1, 2.0));
315             // and the right point is (f2*(c*tmin+d)^2, -(a*tmin+b)*(c*tmin+d), (a*t+b)^2 + f2^2*(c*t+d)^2)
316             final var tmp7 = Math.pow(c * tmin + d, 2.0);
317             final var transformedRightPoint = new HomogeneousPoint2D(
318                     f2 * tmp7,
319                     -(a * tmin + b) * (c * tmin + d),
320                     Math.pow(a * tmin + b, 2.0) + Math.pow(f2, 2.0) * tmp7);
321 
322             // undo translation and rotation transformations
323             // correctedLeftPoint = T1^-1*R1'*transformedLeftPoint
324             // correctedRightPoint = T2^-1*R2'*transformedRightPoint
325 
326             // inverse left transformation
327             invLeftTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
328                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
329             // add terms to obtain inverse transformation
330             invLeftTranslationMatrix.setElementAt(0, 2, leftPoint.getInhomX());
331             invLeftTranslationMatrix.setElementAt(1, 2, leftPoint.getInhomY());
332 
333             // inverse right translation transformation
334             final var invRightTranslationMatrix = Matrix.identity(Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH,
335                     Point2D.POINT2D_HOMOGENEOUS_COORDINATES_LENGTH);
336             // add terms to obtain inverse transformation
337             invRightTranslationMatrix.setElementAt(0, 2, rightPoint.getInhomX());
338             invRightTranslationMatrix.setElementAt(1, 2, rightPoint.getInhomY());
339 
340             invLeftTranslationMatrix.multiply(invLeftRotationMatrix);
341             final var invLeftTransformationMatrix = invLeftTranslationMatrix;
342             invRightTranslationMatrix.multiply(invRightRotationMatrix);
343 
344             final var invLeftTransformation = new ProjectiveTransformation2D(invLeftTransformationMatrix);
345             final var invRightTransformation = new ProjectiveTransformation2D(invRightTranslationMatrix);
346 
347             invLeftTransformation.transform(transformedLeftPoint, correctedLeftPoint);
348             invRightTransformation.transform(transformedRightPoint, correctedRightPoint);
349 
350         } catch (final AlgebraException | GeometryException | NumericalException e) {
351             throw new CorrectionException(e);
352         }
353     }
354 
355     /**
356      * Evaluates polynomial to be minimized in order to find best corrected
357      * points.
358      *
359      * @param t  polynomial variable.
360      * @param a  a value for transformed fundamental matrix.
361      * @param b  b value for transformed fundamental matrix.
362      * @param c  c value for transformed fundamental matrix.
363      * @param d  d value for transformed fundamental matrix.
364      * @param f1 f1 value for transformed fundamental matrix.
365      * @param f2 f2 value for transformed fundamental matrix.
366      * @return evaluated polynomial value.
367      */
368     private static double s(final double t, final double a, final double b, final double c,
369                             final double d, final double f1, final double f2) {
370         final var tmp = Math.pow(c * t + d, 2.0);
371         return Math.pow(t, 2.0) / (1 + Math.pow(t * f1, 2.0)) + tmp / (Math.pow(a * t + b, 2.0)
372                 + Math.pow(f2, 2.0) * tmp);
373     }
374 }