View Javadoc
1   /*
2    * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *         http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  package com.irurueta.ar.sfm;
17  
18  import com.irurueta.ar.slam.BaseCalibrationData;
19  import com.irurueta.ar.slam.BaseSlamEstimator;
20  import com.irurueta.geometry.InhomogeneousPoint3D;
21  import com.irurueta.geometry.MetricTransformation3D;
22  import com.irurueta.geometry.PinholeCamera;
23  import com.irurueta.geometry.PinholeCameraIntrinsicParameters;
24  import com.irurueta.geometry.Point3D;
25  import com.irurueta.geometry.Quaternion;
26  
27  import java.util.ArrayList;
28  
29  /**
30   * Base class in charge of estimating cameras and 3D reconstructed points from
31   * sparse image point correspondences in two views and also in charge of
32   * estimating overall scene scale by means of SLAM (Simultaneous Location And
33   * Mapping) using data obtained from sensors like accelerometers or gyroscopes.
34   *
35   * @param <D> type of calibration data.
36   * @param <C> type of configuration.
37   * @param <R> type of re-constructor.
38   * @param <L> type of listener.
39   * @param <S> type of SLAM estimator.
40   */
41  @SuppressWarnings("DuplicatedCode")
42  public abstract class BaseSlamTwoViewsSparseReconstructor<
43          D extends BaseCalibrationData,
44          C extends BaseSlamTwoViewsSparseReconstructorConfiguration<D, C>,
45          R extends BaseSlamTwoViewsSparseReconstructor<D, C, R, L, S>,
46          L extends BaseSlamTwoViewsSparseReconstructorListener<R>,
47          S extends BaseSlamEstimator<D>> extends BaseTwoViewsSparseReconstructor<C, R, L> {
48  
49      /**
50       * Slam estimator to estimate position, speed, orientation using
51       * accelerometer and gyroscope data.
52       */
53      protected S slamEstimator;
54  
55      /**
56       * Position estimated by means of SLAM. It is reused each time it is notified.
57       */
58      private final InhomogeneousPoint3D slamPosition = new InhomogeneousPoint3D();
59  
60      /**
61       * Camera estimated by means of SLAM. It is reused each time it is notified.
62       */
63      private final PinholeCamera slamCamera = new PinholeCamera();
64  
65      /**
66       * Camera rotation estimated by means of SLAM. It is reused each time it is notified.
67       */
68      private final Quaternion slamRotation = new Quaternion();
69  
70      /**
71       * Constructor.
72       *
73       * @param configuration configuration for this re-constructor.
74       * @param listener      listener in charge of handling events.
75       * @throws NullPointerException if listener or configuration is not
76       *                              provided.
77       */
78      protected BaseSlamTwoViewsSparseReconstructor(final C configuration, final L listener) {
79          super(configuration, listener);
80      }
81  
82      /**
83       * Provides a new accelerometer sample to update SLAM estimation.
84       * This method must be called whenever the accelerometer sensor receives new
85       * data.
86       * If re-constructor is not running, calling this method has no effect.
87       *
88       * @param timestamp     timestamp of accelerometer sample since epoch time and
89       *                      expressed in nanoseconds.
90       * @param accelerationX linear acceleration along x-axis expressed in meters
91       *                      per squared second (m/s^2).
92       * @param accelerationY linear acceleration along y-axis expressed in meters
93       *                      per squared second (m/s^2).
94       * @param accelerationZ linear acceleration along z-axis expressed in meters
95       *                      per squared second (m/s^2).
96       */
97      public void updateAccelerometerSample(
98              final long timestamp, final float accelerationX, final float accelerationY, final float accelerationZ) {
99          if (slamEstimator != null) {
100             slamEstimator.updateAccelerometerSample(timestamp, accelerationX, accelerationY, accelerationZ);
101         }
102     }
103 
104     /**
105      * Provides a new accelerometer sample to update SLAM estimation.
106      * This method must be called whenever the accelerometer sensor receives new
107      * data.
108      * If re-constructor is not running, calling this method has no effect.
109      *
110      * @param timestamp timestamp of accelerometer sample since epoch time and
111      *                  expressed in nanoseconds.
112      * @param data      array containing x,y,z components of linear acceleration
113      *                  expressed in meters per squared second (m/s^2).
114      * @throws IllegalArgumentException if provided array does not have length
115      *                                  3.
116      */
117     public void updateAccelerometerSample(final long timestamp, final float[] data) {
118         if (slamEstimator != null) {
119             slamEstimator.updateAccelerometerSample(timestamp, data);
120         }
121     }
122 
123     /**
124      * Provides a new gyroscope sample to update SLAM estimation.
125      * If re-constructor is not running, calling this method has no effect.
126      *
127      * @param timestamp     timestamp of gyroscope sample since epoch time and
128      *                      expressed in nanoseconds.
129      * @param angularSpeedX angular speed of rotation along x-axis expressed in
130      *                      radians per second (rad/s).
131      * @param angularSpeedY angular speed of rotation along y-axis expressed in
132      *                      radians per second (rad/s).
133      * @param angularSpeedZ angular speed of rotation along z-axis expressed in
134      *                      radians per second (rad/s).
135      */
136     public void updateGyroscopeSample(
137             final long timestamp, final float angularSpeedX, final float angularSpeedY, final float angularSpeedZ) {
138         if (slamEstimator != null) {
139             slamEstimator.updateGyroscopeSample(timestamp, angularSpeedX, angularSpeedY, angularSpeedZ);
140         }
141     }
142 
143     /**
144      * Provides a new gyroscope sample to update SLAM estimation.
145      * If re-constructor is not running, calling this method has no effect.
146      *
147      * @param timestamp timestamp of gyroscope sample since epoch time and
148      *                  expressed in nanoseconds.
149      * @param data      angular speed of rotation along x,y,z axes expressed in
150      *                  radians per second (rad/s).
151      * @throws IllegalArgumentException if provided array does not have length
152      *                                  3.
153      */
154     public void updateGyroscopeSample(final long timestamp, final float[] data) {
155         if (slamEstimator != null) {
156             slamEstimator.updateGyroscopeSample(timestamp, data);
157         }
158     }
159 
160     /**
161      * Set ups calibration data on SLAM estimator if available.
162      */
163     protected void setUpCalibrationData() {
164         D calibrationData = configuration.getCalibrationData();
165         if (calibrationData != null) {
166             slamEstimator.setCalibrationData(calibrationData);
167         }
168     }
169 
170     /**
171      * Configures listener of SLAM estimator
172      */
173     protected void setUpSlamEstimatorListener() {
174         slamEstimator.setListener(new BaseSlamEstimator.BaseSlamEstimatorListener<>() {
175             @Override
176             public void onFullSampleReceived(final BaseSlamEstimator<D> estimator) {
177                 // not used
178             }
179 
180             @Override
181             public void onFullSampleProcessed(final BaseSlamEstimator<D> estimator) {
182                 notifySlamStateAndCamera();
183             }
184 
185             @Override
186             public void onCorrectWithPositionMeasure(final BaseSlamEstimator<D> estimator) {
187                 // not used
188             }
189 
190             @Override
191             public void onCorrectedWithPositionMeasure(final BaseSlamEstimator<D> estimator) {
192                 notifySlamStateAndCamera();
193             }
194 
195             private void notifySlamStateAndCamera() {
196                 notifySlamStateIfNeeded();
197                 notifySlamCameraIfNeeded();
198             }
199         });
200     }
201 
202     /**
203      * Update scene scale using SLAM data.
204      *
205      * @return true if scale was successfully updated, false otherwise.
206      */
207     protected boolean updateScale() {
208         // obtain baseline (camera separation from slam estimator data
209         final var posX = slamEstimator.getStatePositionX();
210         final var posY = slamEstimator.getStatePositionY();
211         final var posZ = slamEstimator.getStatePositionZ();
212 
213         // to estimate baseline, we assume that first camera is placed at
214         // world origin
215         final var baseline = Math.sqrt(posX * posX + posY * posY + posZ * posZ);
216 
217         try {
218             final var camera1 = estimatedCamera1.getCamera();
219             final var camera2 = estimatedCamera2.getCamera();
220 
221             camera1.decompose();
222             camera2.decompose();
223 
224             final var center1 = camera1.getCameraCenter();
225             final var center2 = camera2.getCameraCenter();
226 
227             final var estimatedBaseline = center1.distanceTo(center2);
228 
229             final var scale = baseline / estimatedBaseline;
230 
231             final var scaleTransformation = new MetricTransformation3D(scale);
232 
233             // update scale of cameras
234             scaleTransformation.transform(camera1);
235             scaleTransformation.transform(camera2);
236 
237             estimatedCamera1.setCamera(camera1);
238             estimatedCamera2.setCamera(camera2);
239 
240             // update scale of reconstructed points
241             final var numPoints = reconstructedPoints.size();
242             final var reconstructedPoints3D = new ArrayList<Point3D>();
243             for (final var reconstructedPoint : reconstructedPoints) {
244                 reconstructedPoints3D.add(reconstructedPoint.getPoint());
245             }
246 
247             scaleTransformation.transformAndOverwritePoints(reconstructedPoints3D);
248 
249             // set scaled points into result
250             for (var i = 0; i < numPoints; i++) {
251                 reconstructedPoints.get(i).setPoint(reconstructedPoints3D.get(i));
252             }
253 
254             return true;
255         } catch (final Exception e) {
256             failed = true;
257             //noinspection unchecked
258             listener.onFail((R) this);
259 
260             return false;
261         }
262     }
263 
264     /**
265      * Notifies SLAM state if notification is enabled at configuration time.
266      */
267     private void notifySlamStateIfNeeded() {
268         if (!configuration.isNotifyAvailableSlamDataEnabled()) {
269             return;
270         }
271 
272         final var positionX = slamEstimator.getStatePositionX();
273         final var positionY = slamEstimator.getStatePositionY();
274         final var positionZ = slamEstimator.getStatePositionZ();
275 
276         final var velocityX = slamEstimator.getStateVelocityX();
277         final var velocityY = slamEstimator.getStateVelocityY();
278         final var velocityZ = slamEstimator.getStateVelocityZ();
279 
280         final var accelerationX = slamEstimator.getStateAccelerationX();
281         final var accelerationY = slamEstimator.getStateAccelerationY();
282         final var accelerationZ = slamEstimator.getStateAccelerationZ();
283 
284         final var quaternionA = slamEstimator.getStateQuaternionA();
285         final var quaternionB = slamEstimator.getStateQuaternionB();
286         final var quaternionC = slamEstimator.getStateQuaternionC();
287         final var quaternionD = slamEstimator.getStateQuaternionD();
288 
289         final var angularSpeedX = slamEstimator.getStateAngularSpeedX();
290         final var angularSpeedY = slamEstimator.getStateAngularSpeedY();
291         final var angularSpeedZ = slamEstimator.getStateAngularSpeedZ();
292 
293         //noinspection unchecked
294         listener.onSlamDataAvailable((R) this, positionX, positionY, positionZ,
295                 velocityX, velocityY, velocityZ, accelerationX, accelerationY, accelerationZ,
296                 quaternionA, quaternionB, quaternionC, quaternionD, angularSpeedX, angularSpeedY, angularSpeedZ,
297                 slamEstimator.getStateCovariance());
298     }
299 
300     /**
301      * Notifies estimated camera by means of SLAM if notification is enabled at
302      * configuration time and intrinsics are already available.
303      */
304     private void notifySlamCameraIfNeeded() {
305         if (!configuration.isNotifyEstimatedSlamCameraEnabled()) {
306             return;
307         }
308 
309         PinholeCameraIntrinsicParameters intrinsicParameters = null;
310         if (configuration.getInitialIntrinsic1() != null) {
311             intrinsicParameters = configuration.getInitialIntrinsic1();
312         } else if (configuration.getInitialIntrinsic2() != null) {
313             intrinsicParameters = configuration.getInitialIntrinsic2();
314         }
315 
316         if (intrinsicParameters == null) {
317             return;
318         }
319 
320         final var positionX = slamEstimator.getStatePositionX();
321         final var positionY = slamEstimator.getStatePositionY();
322         final var positionZ = slamEstimator.getStatePositionZ();
323         slamPosition.setInhomogeneousCoordinates(positionX, positionY, positionZ);
324 
325         final var quaternionA = slamEstimator.getStateQuaternionA();
326         final var quaternionB = slamEstimator.getStateQuaternionB();
327         final var quaternionC = slamEstimator.getStateQuaternionC();
328         final var quaternionD = slamEstimator.getStateQuaternionD();
329         slamRotation.setA(quaternionA);
330         slamRotation.setB(quaternionB);
331         slamRotation.setC(quaternionC);
332         slamRotation.setD(quaternionD);
333 
334         slamCamera.setIntrinsicAndExtrinsicParameters(intrinsicParameters, slamRotation, slamPosition);
335 
336         //noinspection unchecked
337         listener.onSlamCameraEstimated((R) this, slamCamera);
338     }
339 }