1 /*
2 * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.ar.sfm;
17
18 import com.irurueta.geometry.CoordinatesType;
19 import com.irurueta.geometry.PinholeCamera;
20 import com.irurueta.geometry.Point2D;
21 import com.irurueta.geometry.Point3D;
22 import com.irurueta.geometry.estimators.LockedException;
23 import com.irurueta.geometry.estimators.NotReadyException;
24 import com.irurueta.numerical.robust.PROSACRobustEstimator;
25 import com.irurueta.numerical.robust.PROSACRobustEstimatorListener;
26 import com.irurueta.numerical.robust.RobustEstimator;
27 import com.irurueta.numerical.robust.RobustEstimatorException;
28 import com.irurueta.numerical.robust.RobustEstimatorMethod;
29
30 import java.util.ArrayList;
31 import java.util.List;
32
33 /**
34 * Robustly triangulates 3D points from matched 2D points and their
35 * corresponding cameras on several views using PROSAC algorithm.
36 */
37 public class PROSACRobustSinglePoint3DTriangulator extends RobustSinglePoint3DTriangulator {
38
39 /**
40 * Constant defining default threshold to determine whether samples are
41 * inliers or not.
42 * By default, 1.0 is considered a good value for cases where 2D point
43 * measures are done on pixels, since typically the minimum resolution is 1
44 * pixel.
45 */
46 public static final double DEFAULT_THRESHOLD = 1.0;
47
48 /**
49 * Minimum value that can be set as threshold.
50 * Threshold must be strictly greater than 0.0.
51 */
52 public static final double MIN_THRESHOLD = 0.0;
53
54 /**
55 * Threshold to determine whether samples are inliers or not when testing
56 * possible estimation solutions.
57 * The threshold refers to the amount of projection error (i.e. distance of
58 * projected solution using each camera).
59 */
60 private double threshold;
61
62 /**
63 * Quality scores corresponding to each provided view.
64 * The larger the score value the better the quality of the sample.
65 */
66 private double[] qualityScores;
67
68 /**
69 * Constructor.
70 */
71 public PROSACRobustSinglePoint3DTriangulator() {
72 super();
73 threshold = DEFAULT_THRESHOLD;
74 }
75
76 /**
77 * Constructor.
78 *
79 * @param listener listener to be notified of events such as when estimation
80 * starts, ends or its progress significantly changes.
81 */
82 public PROSACRobustSinglePoint3DTriangulator(final RobustSinglePoint3DTriangulatorListener listener) {
83 super(listener);
84 threshold = DEFAULT_THRESHOLD;
85 }
86
87 /**
88 * Constructor.
89 *
90 * @param points Matched 2D points. Each point in the list is assumed to be
91 * projected by the corresponding camera in the list.
92 * @param cameras List of cameras associated to the matched 2D point on the
93 * same position as the camera on the list.
94 * @throws IllegalArgumentException if provided lists don't have the same
95 * length or their length is less than 2 views, which is the minimum
96 * required to compute triangulation.
97 */
98 public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points, final List<PinholeCamera> cameras) {
99 super(points, cameras);
100 threshold = DEFAULT_THRESHOLD;
101 }
102
103 /**
104 * Constructor.
105 *
106 * @param points Matched 2D points. Each point in the list is assumed to be
107 * projected by the corresponding camera in the list.
108 * @param cameras List of cameras associated to the matched 2D point on the
109 * same position as the camera on the list.
110 * @param listener listener to be notified of events such as when estimation
111 * starts, ends or its progress significantly changes.
112 * @throws IllegalArgumentException if provided lists don't have the same
113 * length or their length is less than 2 views, which is the minimum
114 * required to compute triangulation.
115 */
116 public PROSACRobustSinglePoint3DTriangulator(
117 final List<Point2D> points, final List<PinholeCamera> cameras,
118 final RobustSinglePoint3DTriangulatorListener listener) {
119 super(points, cameras, listener);
120 threshold = DEFAULT_THRESHOLD;
121 }
122
123 /**
124 * Constructor.
125 *
126 * @param qualityScores quality scores corresponding to each provided view.
127 * @throws IllegalArgumentException if provided quality scores length is
128 * smaller than required size (i.e. 2 views).
129 */
130 public PROSACRobustSinglePoint3DTriangulator(final double[] qualityScores) {
131 this();
132 internalSetQualityScores(qualityScores);
133 }
134
135 /**
136 * Constructor.
137 *
138 * @param qualityScores quality scores corresponding to each provided view.
139 * @param listener listener to be notified of events such as when estimation
140 * starts, ends or its progress significantly changes.
141 * @throws IllegalArgumentException if provided quality scores length is
142 * smaller than required size (i.e. 2 views).
143 */
144 public PROSACRobustSinglePoint3DTriangulator(final double[] qualityScores,
145 final RobustSinglePoint3DTriangulatorListener listener) {
146 this(listener);
147 internalSetQualityScores(qualityScores);
148 }
149
150 /**
151 * Constructor.
152 *
153 * @param points Matched 2D points. Each point in the list is assumed to be
154 * projected by the corresponding camera in the list.
155 * @param cameras List of cameras associated to the matched 2D point on the
156 * same position as the camera on the list.
157 * @param qualityScores quality scores corresponding to each provided view.
158 * @throws IllegalArgumentException if provided lists or quality scores
159 * don't have the same length or their length is less than 2 views,
160 * which is the minimum required to compute triangulation.
161 */
162 public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points,
163 final List<PinholeCamera> cameras,
164 final double[] qualityScores) {
165 this(points, cameras);
166 internalSetQualityScores(qualityScores);
167 }
168
169 /**
170 * Constructor.
171 *
172 * @param points Matched 2D points. Each point in the list is assumed to be
173 * projected by the corresponding camera in the list.
174 * @param cameras List of cameras associated to the matched 2D point on the
175 * same position as the camera on the list.
176 * @param qualityScores quality scores corresponding to each provided view.
177 * @param listener listener to be notified of events such as when estimation
178 * starts, ends or its progress significantly changes.
179 * @throws IllegalArgumentException if provided lists or quality scores
180 * don't have the same length or their length is less than 2 views,
181 * which is the minimum required to compute triangulation.
182 */
183 public PROSACRobustSinglePoint3DTriangulator(final List<Point2D> points,
184 final List<PinholeCamera> cameras,
185 final double[] qualityScores,
186 final RobustSinglePoint3DTriangulatorListener listener) {
187 this(points, cameras, listener);
188 internalSetQualityScores(qualityScores);
189 }
190
191 /**
192 * Returns threshold to determine whether points are inliers or not when
193 * testing possible estimation solutions.
194 * The threshold refers to the amount of error (i.e. Euclidean distance) a
195 * possible solution has on projected 2D points.
196 *
197 * @return threshold to determine whether points are inliers or not when
198 * testing possible estimation solutions.
199 */
200 public double getThreshold() {
201 return threshold;
202 }
203
204 /**
205 * Sets threshold to determine whether points are inliers or not when
206 * testing possible estimation solutions.
207 * The threshold refers to the amount of error (i.e. Euclidean distance) a
208 * possible solution has on projected 2D points.
209 *
210 * @param threshold threshold to be set.
211 * @throws IllegalArgumentException if provided value is equal or less than
212 * zero.
213 * @throws LockedException if robust estimator is locked because an
214 * estimation is already in progress.
215 */
216 public void setThreshold(final double threshold) throws LockedException {
217 if (isLocked()) {
218 throw new LockedException();
219 }
220 if (threshold <= MIN_THRESHOLD) {
221 throw new IllegalArgumentException();
222 }
223 this.threshold = threshold;
224 }
225
226 /**
227 * Returns quality scores corresponding to each provided view.
228 * The larger the score value the better the quality of the sampled view.
229 *
230 * @return quality scores corresponding to each view.
231 */
232 @Override
233 public double[] getQualityScores() {
234 return qualityScores;
235 }
236
237 /**
238 * Sets quality scores corresponding to each provided view.
239 * The larger the score value the better the quality of the sampled view.
240 *
241 * @param qualityScores quality scores corresponding to each view.
242 * @throws LockedException if robust estimator is locked because an
243 * estimation is already in progress.
244 * @throws IllegalArgumentException if provided quality scores length is
245 * smaller than MIN_REQUIRED_VIEWS (i.e. 2 views).
246 */
247 @Override
248 public void setQualityScores(final double[] qualityScores) throws LockedException {
249 if (isLocked()) {
250 throw new LockedException();
251 }
252 internalSetQualityScores(qualityScores);
253 }
254
255 /**
256 * Indicates if triangulator is ready to start the 3D point triangulation.
257 * This is true when input data (i.e. 2D points, cameras and quality scores)
258 * are provided and a minimum of 2 views are available.
259 *
260 * @return true if estimator is ready, false otherwise.
261 */
262 @Override
263 public boolean isReady() {
264 return super.isReady() && qualityScores != null && qualityScores.length == points2D.size();
265 }
266
267
268 /**
269 * Triangulates provided matched 2D points being projected by each
270 * corresponding camera into a single 3D point.
271 * At least 2 matched 2D points and their corresponding 2 cameras are
272 * required to compute triangulation. If more views are provided, an
273 * averaged solution can be found.
274 *
275 * @return computed triangulated 3D point.
276 * @throws LockedException if this instance is locked.
277 * @throws NotReadyException if lists of points and cameras don't have the
278 * same length or less than 2 views are provided.
279 * @throws RobustEstimatorException if estimation fails for any reason
280 * (i.e. numerical instability, no solution available, etc).
281 */
282 @SuppressWarnings("DuplicatedCode")
283 @Override
284 public Point3D triangulate() throws LockedException, NotReadyException, RobustEstimatorException {
285 if (isLocked()) {
286 throw new LockedException();
287 }
288 if (!isReady()) {
289 throw new NotReadyException();
290 }
291
292 final var innerEstimator = new PROSACRobustEstimator<Point3D>(new PROSACRobustEstimatorListener<>() {
293
294 // point to be reused when computing residuals
295 private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
296
297 // non-robust 3D point triangulator
298 private final SinglePoint3DTriangulator triangulator =
299 SinglePoint3DTriangulator.create(useHomogeneousSolution
300 ? Point3DTriangulatorType.LMSE_HOMOGENEOUS_TRIANGULATOR
301 : Point3DTriangulatorType.LMSE_INHOMOGENEOUS_TRIANGULATOR);
302
303 // subset of 2D points
304 private final List<Point2D> subsetPoints = new ArrayList<>();
305
306 // subset of cameras
307 private final List<PinholeCamera> subsetCameras = new ArrayList<>();
308
309 @Override
310 public double getThreshold() {
311 return threshold;
312 }
313
314 @Override
315 public int getTotalSamples() {
316 return points2D.size();
317 }
318
319 @Override
320 public int getSubsetSize() {
321 return MIN_REQUIRED_VIEWS;
322 }
323
324 @Override
325 public void estimatePreliminarSolutions(final int[] samplesIndices, final List<Point3D> solutions) {
326 subsetPoints.clear();
327 subsetPoints.add(points2D.get(samplesIndices[0]));
328 subsetPoints.add(points2D.get(samplesIndices[1]));
329
330 subsetCameras.clear();
331 subsetCameras.add(cameras.get(samplesIndices[0]));
332 subsetCameras.add(cameras.get(samplesIndices[1]));
333
334 try {
335 triangulator.setPointsAndCameras(subsetPoints, subsetCameras);
336 final var triangulated = triangulator.triangulate();
337 solutions.add(triangulated);
338 } catch (final Exception e) {
339 // if anything fails, no solution is added
340 }
341 }
342
343 @Override
344 public double computeResidual(final Point3D currentEstimation, final int i) {
345 final var point2D = points2D.get(i);
346 final var camera = cameras.get(i);
347
348 // project estimated point with camera
349 camera.project(currentEstimation, testPoint);
350
351 // return distance of projected point respect to the original one
352 // as a residual
353 return testPoint.distanceTo(point2D);
354 }
355
356 @Override
357 public boolean isReady() {
358 return PROSACRobustSinglePoint3DTriangulator.this.isReady();
359 }
360
361 @Override
362 public void onEstimateStart(final RobustEstimator<Point3D> estimator) {
363 if (listener != null) {
364 listener.onTriangulateStart(PROSACRobustSinglePoint3DTriangulator.this);
365 }
366 }
367
368 @Override
369 public void onEstimateEnd(final RobustEstimator<Point3D> estimator) {
370 if (listener != null) {
371 listener.onTriangulateEnd(PROSACRobustSinglePoint3DTriangulator.this);
372 }
373 }
374
375 @Override
376 public void onEstimateNextIteration(final RobustEstimator<Point3D> estimator, final int iteration) {
377 if (listener != null) {
378 listener.onTriangulateNextIteration(
379 PROSACRobustSinglePoint3DTriangulator.this, iteration);
380 }
381 }
382
383 @Override
384 public void onEstimateProgressChange(final RobustEstimator<Point3D> estimator, final float progress) {
385 if (listener != null) {
386 listener.onTriangulateProgressChange(
387 PROSACRobustSinglePoint3DTriangulator.this, progress);
388 }
389 }
390
391 @Override
392 public double[] getQualityScores() {
393 return qualityScores;
394 }
395 });
396
397 try {
398 locked = true;
399 innerEstimator.setConfidence(confidence);
400 innerEstimator.setMaxIterations(maxIterations);
401 innerEstimator.setProgressDelta(progressDelta);
402 return innerEstimator.estimate();
403 } catch (final com.irurueta.numerical.LockedException e) {
404 throw new LockedException(e);
405 } catch (final com.irurueta.numerical.NotReadyException e) {
406 throw new NotReadyException(e);
407 } finally {
408 locked = false;
409 }
410 }
411
412 /**
413 * Returns method being used for robust estimation.
414 *
415 * @return method being used for robust estimation.
416 */
417 @Override
418 public RobustEstimatorMethod getMethod() {
419 return RobustEstimatorMethod.PROSAC;
420 }
421
422 /**
423 * Sets quality scores corresponding to each provided view.
424 * This method is used internally and does not check whether instance is
425 * locked or not.
426 *
427 * @param qualityScores quality scores to be set.
428 * @throws IllegalArgumentException if provided quality scores length is
429 * smaller than MINIMUM_SIZE.
430 */
431 private void internalSetQualityScores(final double[] qualityScores) {
432 if (qualityScores.length < MIN_REQUIRED_VIEWS) {
433 throw new IllegalArgumentException();
434 }
435
436 this.qualityScores = qualityScores;
437 }
438 }