1 /*
2 * Copyright (C) 2017 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.ar.sfm;
17
18 import com.irurueta.ar.slam.AbsoluteOrientationBaseSlamEstimator;
19 import com.irurueta.ar.slam.BaseCalibrationData;
20 import com.irurueta.geometry.MetricTransformation3D;
21 import com.irurueta.geometry.Point3D;
22 import com.irurueta.geometry.Rotation3D;
23
24 import java.util.ArrayList;
25
26 /**
27 * Base class in charge of estimating cameras and 3D reconstructed points from
28 * sparse image point correspondences in two views and also in charge of
29 * estimating overall scene scale and absolute orientation by means of SLAM
30 * (Simultaneous Location And Mapping) using data obtained from sensors like
31 * accelerometers or gyroscopes.
32 * NOTE: absolute orientation slam estimators are not very accurate during
33 * estimation of the orientation state, for that reason we take into account
34 * the initial orientation.
35 *
36 * @param <D> type of calibration data.
37 * @param <C> type of configuration.
38 * @param <R> type of re-constructor.
39 * @param <L> type of listener.
40 * @param <S> type of SLAM estimator.
41 */
42 public abstract class BaseAbsoluteOrientationSlamTwoViewsSparseReconstructor<
43 D extends BaseCalibrationData,
44 C extends BaseSlamTwoViewsSparseReconstructorConfiguration<D, C>,
45 R extends BaseSlamTwoViewsSparseReconstructor<D, C, R, L, S>,
46 L extends BaseSlamTwoViewsSparseReconstructorListener<R>,
47 S extends AbsoluteOrientationBaseSlamEstimator<D>> extends
48 BaseSlamTwoViewsSparseReconstructor<D, C, R, L, S> {
49
50 /**
51 * First sample of orientation received.
52 */
53 protected Rotation3D firstOrientation;
54
55 /**
56 * Constructor.
57 *
58 * @param configuration configuration for this re-constructor.
59 * @param listener listener in charge of handling events.
60 * @throws NullPointerException if listener or configuration is not
61 * provided.
62 */
63 protected BaseAbsoluteOrientationSlamTwoViewsSparseReconstructor(
64 final C configuration, final L listener) {
65 super(configuration, listener);
66 }
67
68 /**
69 * Provides a new orientation sample to update SLAM estimator.
70 * If re-constructor is not running, calling this method has no effect.
71 *
72 * @param timestamp timestamp of accelerometer sample since epoch time and
73 * expressed in nanoseconds.
74 * @param orientation new orientation.
75 */
76 public void updateOrientationSample(final long timestamp,
77 final Rotation3D orientation) {
78 if (slamEstimator != null) {
79 slamEstimator.updateOrientationSample(timestamp, orientation);
80 }
81 if (firstOrientation == null) {
82 //make a copy of orientation
83 firstOrientation = orientation.toQuaternion();
84 }
85 }
86
87 /**
88 * Updates scene scale and orientation using SLAM data.
89 *
90 * @return true if scale was successfully updated, false otherwise.
91 */
92 @SuppressWarnings("DuplicatedCode")
93 protected boolean updateScaleAndOrientation() {
94
95 // obtain baseline (camera separation from slam estimator data
96 final var posX = slamEstimator.getStatePositionX();
97 final var posY = slamEstimator.getStatePositionY();
98 final var posZ = slamEstimator.getStatePositionZ();
99
100 // to estimate baseline, we assume that first camera is placed at
101 // world origin
102 final var baseline = Math.sqrt(posX * posX + posY * posY + posZ * posZ);
103
104 try {
105 final var camera1 = estimatedCamera1.getCamera();
106 final var camera2 = estimatedCamera2.getCamera();
107
108 camera1.decompose();
109 camera2.decompose();
110
111 final var center1 = camera1.getCameraCenter();
112 final var center2 = camera2.getCameraCenter();
113
114 // R1' = R1*Rdiff
115 // Rdiff = R1^T*R1'
116
117 // where R1' is the desired orientation (obtained by sampling a
118 // sensor)
119 // and R1 is always the identity for the 1st camera.
120 // Hence R1' = Rdiff
121
122 // t1' is the desired translation which is zero for the 1st
123 // camera.
124
125 // We want: P1' = K*[R1' t1'] = K*[R1' 0]
126 // And we have P1 = K[I 0]
127
128 // We need a transformation T so that:
129 // P1' = P1*T^-1 = K[I 0][R1' 0]
130 // [0 1]
131
132 // Hence: T^-1 = [R1' 0]
133 // [0 1]
134
135 // or T = [R1'^T 0]
136 // [0 1]
137
138 // because we are also applying a transformation of scale s,
139 // the combination of both transformations is
140 // T = [s*R1'^T 0]
141 // [0 1]
142
143 final var r = firstOrientation.inverseRotationAndReturnNew();
144
145 final var estimatedBaseline = center1.distanceTo(center2);
146
147 final var scale = baseline / estimatedBaseline;
148
149 final var scaleAndOrientationTransformation = new MetricTransformation3D(scale);
150 scaleAndOrientationTransformation.setRotation(r);
151
152 // update scale of cameras
153 scaleAndOrientationTransformation.transform(camera1);
154 scaleAndOrientationTransformation.transform(camera2);
155
156 estimatedCamera1.setCamera(camera1);
157 estimatedCamera2.setCamera(camera2);
158
159 // update scale of reconstructed points
160 final var numPoints = reconstructedPoints.size();
161 final var reconstructedPoints3D = new ArrayList<Point3D>();
162 for (final var reconstructedPoint : reconstructedPoints) {
163 reconstructedPoints3D.add(reconstructedPoint.getPoint());
164 }
165
166 scaleAndOrientationTransformation.transformAndOverwritePoints(reconstructedPoints3D);
167
168 // set scaled points into result
169 for (var i = 0; i < numPoints; i++) {
170 reconstructedPoints.get(i).setPoint(reconstructedPoints3D.get(i));
171 }
172
173 return true;
174 } catch (final Exception e) {
175 failed = true;
176 //noinspection unchecked
177 listener.onFail((R) this);
178
179 return false;
180 }
181 }
182 }