View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.sfm;
17  
18  import com.irurueta.ar.slam.AbsoluteOrientationBaseSlamEstimator;
19  import com.irurueta.ar.slam.BaseCalibrationData;
20  import com.irurueta.geometry.MetricTransformation3D;
21  import com.irurueta.geometry.Point3D;
22  import com.irurueta.geometry.Rotation3D;
23  
24  import java.util.ArrayList;
25  
26  /**
27   * Base class in charge of estimating cameras and 3D reconstructed points from
28   * sparse image point correspondences in two views and also in charge of
29   * estimating overall scene scale and absolute orientation by means of SLAM
30   * (Simultaneous Location And Mapping) using data obtained from sensors like
31   * accelerometers or gyroscopes.
32   * NOTE: absolute orientation slam estimators are not very accurate during
33   * estimation of the orientation state, for that reason we take into account
34   * the initial orientation.
35   *
36   * @param <D> type of calibration data.
37   * @param <C> type of configuration.
38   * @param <R> type of re-constructor.
39   * @param <L> type of listener.
40   * @param <S> type of SLAM estimator.
41   */
42  public abstract class BaseAbsoluteOrientationSlamTwoViewsSparseReconstructor<
43          D extends BaseCalibrationData,
44          C extends BaseSlamTwoViewsSparseReconstructorConfiguration<D, C>,
45          R extends BaseSlamTwoViewsSparseReconstructor<D, C, R, L, S>,
46          L extends BaseSlamTwoViewsSparseReconstructorListener<R>,
47          S extends AbsoluteOrientationBaseSlamEstimator<D>> extends
48          BaseSlamTwoViewsSparseReconstructor<D, C, R, L, S> {
49  
50      /**
51       * First sample of orientation received.
52       */
53      protected Rotation3D firstOrientation;
54  
55      /**
56       * Constructor.
57       *
58       * @param configuration configuration for this re-constructor.
59       * @param listener      listener in charge of handling events.
60       * @throws NullPointerException if listener or configuration is not
61       *                              provided.
62       */
63      protected BaseAbsoluteOrientationSlamTwoViewsSparseReconstructor(
64              final C configuration, final L listener) {
65          super(configuration, listener);
66      }
67  
68      /**
69       * Provides a new orientation sample to update SLAM estimator.
70       * If re-constructor is not running, calling this method has no effect.
71       *
72       * @param timestamp   timestamp of accelerometer sample since epoch time and
73       *                    expressed in nanoseconds.
74       * @param orientation new orientation.
75       */
76      public void updateOrientationSample(final long timestamp,
77                                          final Rotation3D orientation) {
78          if (slamEstimator != null) {
79              slamEstimator.updateOrientationSample(timestamp, orientation);
80          }
81          if (firstOrientation == null) {
82              //make a copy of orientation
83              firstOrientation = orientation.toQuaternion();
84          }
85      }
86  
87      /**
88       * Updates scene scale and orientation using SLAM data.
89       *
90       * @return true if scale was successfully updated, false otherwise.
91       */
92      @SuppressWarnings("DuplicatedCode")
93      protected boolean updateScaleAndOrientation() {
94  
95          // obtain baseline (camera separation from slam estimator data
96          final var posX = slamEstimator.getStatePositionX();
97          final var posY = slamEstimator.getStatePositionY();
98          final var posZ = slamEstimator.getStatePositionZ();
99  
100         // to estimate baseline, we assume that first camera is placed at
101         // world origin
102         final var baseline = Math.sqrt(posX * posX + posY * posY + posZ * posZ);
103 
104         try {
105             final var camera1 = estimatedCamera1.getCamera();
106             final var camera2 = estimatedCamera2.getCamera();
107 
108             camera1.decompose();
109             camera2.decompose();
110 
111             final var center1 = camera1.getCameraCenter();
112             final var center2 = camera2.getCameraCenter();
113 
114             // R1' = R1*Rdiff
115             // Rdiff = R1^T*R1'
116 
117             // where R1' is the desired orientation (obtained by sampling a
118             // sensor)
119             // and R1 is always the identity for the 1st camera.
120             // Hence R1' = Rdiff
121 
122             // t1' is the desired translation which is zero for the 1st
123             // camera.
124 
125             // We want: P1' = K*[R1' t1'] = K*[R1' 0]
126             // And we have P1 = K[I 0]
127 
128             // We need a transformation T so that:
129             // P1' = P1*T^-1 = K[I 0][R1' 0]
130             //                      [0   1]
131 
132             // Hence: T^-1 = [R1' 0]
133             //              [0   1]
134 
135             // or T = [R1'^T 0]
136             //       [0     1]
137 
138             // because we are also applying a transformation of scale s,
139             // the combination of both transformations is
140             // T = [s*R1'^T 0]
141             //     [0       1]
142 
143             final var r = firstOrientation.inverseRotationAndReturnNew();
144 
145             final var estimatedBaseline = center1.distanceTo(center2);
146 
147             final var scale = baseline / estimatedBaseline;
148 
149             final var scaleAndOrientationTransformation = new MetricTransformation3D(scale);
150             scaleAndOrientationTransformation.setRotation(r);
151 
152             // update scale of cameras
153             scaleAndOrientationTransformation.transform(camera1);
154             scaleAndOrientationTransformation.transform(camera2);
155 
156             estimatedCamera1.setCamera(camera1);
157             estimatedCamera2.setCamera(camera2);
158 
159             // update scale of reconstructed points
160             final var numPoints = reconstructedPoints.size();
161             final var reconstructedPoints3D = new ArrayList<Point3D>();
162             for (final var reconstructedPoint : reconstructedPoints) {
163                 reconstructedPoints3D.add(reconstructedPoint.getPoint());
164             }
165 
166             scaleAndOrientationTransformation.transformAndOverwritePoints(reconstructedPoints3D);
167 
168             // set scaled points into result
169             for (var i = 0; i < numPoints; i++) {
170                 reconstructedPoints.get(i).setPoint(reconstructedPoints3D.get(i));
171             }
172 
173             return true;
174         } catch (final Exception e) {
175             failed = true;
176             //noinspection unchecked
177             listener.onFail((R) this);
178 
179             return false;
180         }
181     }
182 }