1 /*
2 * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.ar.sfm;
17
18 import com.irurueta.geometry.CoordinatesType;
19 import com.irurueta.geometry.PinholeCamera;
20 import com.irurueta.geometry.Point2D;
21 import com.irurueta.geometry.Point3D;
22 import com.irurueta.geometry.estimators.LockedException;
23 import com.irurueta.geometry.estimators.NotReadyException;
24 import com.irurueta.numerical.robust.PROMedSRobustEstimator;
25 import com.irurueta.numerical.robust.PROMedSRobustEstimatorListener;
26 import com.irurueta.numerical.robust.RobustEstimator;
27 import com.irurueta.numerical.robust.RobustEstimatorException;
28 import com.irurueta.numerical.robust.RobustEstimatorMethod;
29
30 import java.util.ArrayList;
31 import java.util.List;
32
33 /**
34 * Robustly triangulates 3D points from matched 2D points and their
35 * corresponding cameras on several views using PROMedS algorithm.
36 */
37 public class PROMedSRobustSinglePoint3DTriangulator extends RobustSinglePoint3DTriangulator {
38
39 /**
40 * Default value to be used for stop threshold. Stop threshold can be used
41 * to keep the algorithm iterating in case that best estimated threshold
42 * using median of residuals is not small enough. Once a solution is found
43 * that generates a threshold below this value, the algorithm will stop.
44 * The stop threshold can be used to prevent the LMedS algorithm iterating
45 * too many times in cases where samples have a very similar accuracy.
46 * For instance, in cases where proportion of outliers is very small (close
47 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
48 * iterate for a long time trying to find the best solution when indeed
49 * there is no need to do that if a reasonable threshold has already been
50 * reached.
51 * Because of this behaviour the stop threshold can be set to a value much
52 * lower than the one typically used in RANSAC, and yet the algorithm could
53 * still produce even smaller thresholds in estimated results.
54 */
55 public static final double DEFAULT_STOP_THRESHOLD = 1e-3;
56
57 /**
58 * Minimum allowed stop threshold value.
59 */
60 public static final double MIN_STOP_THRESHOLD = 0.0;
61
62 /**
63 * Threshold to be used to keep the algorithm iterating in case that best
64 * estimated threshold using median of residuals is not small enough. Once
65 * a solution is found that generates a threshold below this value, the
66 * algorithm will stop.
67 * The stop threshold can be used to prevent the LMedS algorithm iterating
68 * too many times in cases where samples have a very similar accuracy.
69 * For instance, in cases where proportion of outliers is very small (close
70 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
71 * iterate for a long time trying to find the best solution when indeed
72 * there is no need to do that if a reasonable threshold has already been
73 * reached.
74 * Because of this behaviour the stop threshold can be set to a value much
75 * lower than the one typically used in RANSAC, and yet the algorithm could
76 * still produce even smaller thresholds in estimated results.
77 */
78 private double stopThreshold;
79
80 /**
81 * Quality scores corresponding to each provided point.
82 * The larger the score value the better the quality of the sample.
83 */
84 private double[] qualityScores;
85
86 /**
87 * Constructor.
88 */
89 public PROMedSRobustSinglePoint3DTriangulator() {
90 super();
91 stopThreshold = DEFAULT_STOP_THRESHOLD;
92 }
93
94 /**
95 * Constructor.
96 *
97 * @param listener listener to be notified of events such as when estimation
98 * starts, ends or its progress significantly changes.
99 */
100 public PROMedSRobustSinglePoint3DTriangulator(final RobustSinglePoint3DTriangulatorListener listener) {
101 super(listener);
102 stopThreshold = DEFAULT_STOP_THRESHOLD;
103 }
104
105 /**
106 * Constructor.
107 *
108 * @param points Matched 2D points. Each point in the list is assumed to be
109 * projected by the corresponding camera in the list.
110 * @param cameras List of cameras associated to the matched 2D point on the
111 * same position as the camera on the list.
112 * @throws IllegalArgumentException if provided lists don't have the same
113 * length or their length is less than 2 views, which is the minimum
114 * required to compute triangulation.
115 */
116 public PROMedSRobustSinglePoint3DTriangulator(final List<Point2D> points, final List<PinholeCamera> cameras) {
117 super(points, cameras);
118 stopThreshold = DEFAULT_STOP_THRESHOLD;
119 }
120
121 /**
122 * Constructor.
123 *
124 * @param points Matched 2D points. Each point in the list is assumed to be
125 * projected by the corresponding camera in the list.
126 * @param cameras List of cameras associated to the matched 2D point on the
127 * same position as the camera on the list.
128 * @param listener listener to be notified of events such as when estimation
129 * starts, ends or its progress significantly changes.
130 * @throws IllegalArgumentException if provided lists don't have the same
131 * length or their length is less than 2 views, which is the minimum
132 * required to compute triangulation.
133 */
134 public PROMedSRobustSinglePoint3DTriangulator(
135 final List<Point2D> points, final List<PinholeCamera> cameras,
136 final RobustSinglePoint3DTriangulatorListener listener) {
137 super(points, cameras, listener);
138 stopThreshold = DEFAULT_STOP_THRESHOLD;
139 }
140
141 /**
142 * Constructor.
143 *
144 * @param qualityScores quality scores corresponding to each provided view.
145 * @throws IllegalArgumentException if provided quality scores length is
146 * smaller than required size (i.e. 2 views).
147 */
148 public PROMedSRobustSinglePoint3DTriangulator(final double[] qualityScores) {
149 this();
150 internalSetQualityScores(qualityScores);
151 }
152
153 /**
154 * Constructor.
155 *
156 * @param qualityScores quality scores corresponding to each provided view.
157 * @param listener listener to be notified of events such as when estimation
158 * starts, ends or its progress significantly changes.
159 * @throws IllegalArgumentException if provided quality scores length is
160 * smaller than required size (i.e. 2 views).
161 */
162 public PROMedSRobustSinglePoint3DTriangulator(final double[] qualityScores,
163 final RobustSinglePoint3DTriangulatorListener listener) {
164 this(listener);
165 internalSetQualityScores(qualityScores);
166 }
167
168 /**
169 * Constructor.
170 *
171 * @param points Matched 2D points. Each point in the list is assumed to be
172 * projected by the corresponding camera in the list.
173 * @param cameras List of cameras associated to the matched 2D point on the
174 * same position as the camera on the list.
175 * @param qualityScores quality scores corresponding to each provided view.
176 * @throws IllegalArgumentException if provided lists or quality scores
177 * don't have the same length or their length is less than 2 views,
178 * which is the minimum required to compute triangulation.
179 */
180 public PROMedSRobustSinglePoint3DTriangulator(
181 final List<Point2D> points, final List<PinholeCamera> cameras, final double[] qualityScores) {
182 this(points, cameras);
183 internalSetQualityScores(qualityScores);
184 }
185
186 /**
187 * Constructor.
188 *
189 * @param points Matched 2D points. Each point in the list is assumed to be
190 * projected by the corresponding camera in the list.
191 * @param cameras List of cameras associated to the matched 2D point on the
192 * same position as the camera on the list.
193 * @param qualityScores quality scores corresponding to each provided view.
194 * @param listener listener to be notified of events such as when estimation
195 * starts, ends or its progress significantly changes.
196 * @throws IllegalArgumentException if provided lists or quality scores
197 * don't have the same length or their length is less than 2 views,
198 * which is the minimum required to compute triangulation.
199 */
200 public PROMedSRobustSinglePoint3DTriangulator(final List<Point2D> points,
201 final List<PinholeCamera> cameras,
202 final double[] qualityScores,
203 final RobustSinglePoint3DTriangulatorListener listener) {
204 this(points, cameras, listener);
205 internalSetQualityScores(qualityScores);
206 }
207
208 /**
209 * Returns threshold to be used to keep the algorithm iterating in case that
210 * best estimated threshold using median of residuals is not small enough.
211 * Once a solution is found that generates a threshold below this value, the
212 * algorithm will stop.
213 * The stop threshold can be used to prevent the LMedS algorithm iterating
214 * too many times in cases where samples have a very similar accuracy.
215 * For instance, in cases where proportion of outliers is very small (close
216 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
217 * iterate for a long time trying to find the best solution when indeed
218 * there is no need to do that if a reasonable threshold has already been
219 * reached.
220 * Because of this behaviour the stop threshold can be set to a value much
221 * lower than the one typically used in RANSAC, and yet the algorithm could
222 * still produce even smaller thresholds in estimated results.
223 *
224 * @return stop threshold to stop the algorithm prematurely when a certain
225 * accuracy has been reached.
226 */
227 public double getStopThreshold() {
228 return stopThreshold;
229 }
230
231 /**
232 * Sets threshold to be used to keep the algorithm iterating in case that
233 * best estimated threshold using median of residuals is not small enough.
234 * Once a solution is found that generates a threshold below this value, the
235 * algorithm will stop.
236 * The stop threshold can be used to prevent the LMedS algorithm iterating
237 * too many times in cases where samples have a very similar accuracy.
238 * For instance, in cases where proportion of outliers is very small (close
239 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
240 * iterate for a long time trying to find the best solution when indeed
241 * there is no need to do that if a reasonable threshold has already been
242 * reached.
243 * Because of this behaviour the stop threshold can be set to a value much
244 * lower than the one typically used in RANSAC, and yet the algorithm could
245 * still produce even smaller thresholds in estimated results.
246 *
247 * @param stopThreshold stop threshold to stop the algorithm prematurely
248 * when a certain accuracy has been reached.
249 * @throws IllegalArgumentException if provided value is zero or negative.
250 * @throws LockedException if robust estimator is locked because an
251 * estimation is already in progress.
252 */
253 public void setStopThreshold(final double stopThreshold) throws LockedException {
254 if (isLocked()) {
255 throw new LockedException();
256 }
257 if (stopThreshold <= MIN_STOP_THRESHOLD) {
258 throw new IllegalArgumentException();
259 }
260
261 this.stopThreshold = stopThreshold;
262 }
263
264 /**
265 * Returns quality scores corresponding to each provided view.
266 * The larger the score value the better the quality of the sampled view.
267 *
268 * @return quality scores corresponding to each view.
269 */
270 @Override
271 public double[] getQualityScores() {
272 return qualityScores;
273 }
274
275 /**
276 * Sets quality scores corresponding to each provided view.
277 * The larger the score value the better the quality of the sampled view.
278 *
279 * @param qualityScores quality scores corresponding to each view.
280 * @throws LockedException if robust estimator is locked because an
281 * estimation is already in progress.
282 * @throws IllegalArgumentException if provided quality scores length is
283 * smaller than MIN_REQUIRED_VIEWS (i.e. 2 views).
284 */
285 @Override
286 public void setQualityScores(final double[] qualityScores) throws LockedException {
287 if (isLocked()) {
288 throw new LockedException();
289 }
290 internalSetQualityScores(qualityScores);
291 }
292
293 /**
294 * Indicates if triangulator is ready to start the 3D point triangulation.
295 * This is true when input data (i.e. 2D points, cameras and quality scores)
296 * are provided and a minimum of 2 views are available.
297 *
298 * @return true if estimator is ready, false otherwise.
299 */
300 @Override
301 public boolean isReady() {
302 return super.isReady() && qualityScores != null && qualityScores.length == points2D.size();
303 }
304
305
306 /**
307 * Triangulates provided matched 2D points being projected by each
308 * corresponding camera into a single 3D point.
309 * At least 2 matched 2D points and their corresponding 2 cameras are
310 * required to compute triangulation. If more views are provided, an
311 * averaged solution can be found.
312 *
313 * @return computed triangulated 3D point.
314 * @throws LockedException if this instance is locked.
315 * @throws NotReadyException if lists of points and cameras don't have the
316 * same length or less than 2 views are provided.
317 * @throws RobustEstimatorException if estimation fails for any reason
318 * (i.e. numerical instability, no solution available, etc).
319 */
320 @SuppressWarnings("DuplicatedCode")
321 @Override
322 public Point3D triangulate() throws LockedException, NotReadyException, RobustEstimatorException {
323 if (isLocked()) {
324 throw new LockedException();
325 }
326 if (!isReady()) {
327 throw new NotReadyException();
328 }
329
330 final var innerEstimator = new PROMedSRobustEstimator<Point3D>(new PROMedSRobustEstimatorListener<>() {
331
332 // point to be reused when computing residuals
333 private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
334
335 // non-robust 3D point triangulator
336 private final SinglePoint3DTriangulator triangulator =
337 SinglePoint3DTriangulator.create(useHomogeneousSolution
338 ? Point3DTriangulatorType.LMSE_HOMOGENEOUS_TRIANGULATOR
339 : Point3DTriangulatorType.LMSE_INHOMOGENEOUS_TRIANGULATOR);
340
341 // subset of 2D points
342 private final List<Point2D> subsetPoints = new ArrayList<>();
343
344 // subst of cameras
345 private final List<PinholeCamera> subsetCameras = new ArrayList<>();
346
347 @Override
348 public double getThreshold() {
349 return stopThreshold;
350 }
351
352 @Override
353 public int getTotalSamples() {
354 return points2D.size();
355 }
356
357 @Override
358 public int getSubsetSize() {
359 return MIN_REQUIRED_VIEWS;
360 }
361
362 @Override
363 public void estimatePreliminarSolutions(
364 final int[] samplesIndices, final List<Point3D> solutions) {
365 subsetPoints.clear();
366 subsetPoints.add(points2D.get(samplesIndices[0]));
367 subsetPoints.add(points2D.get(samplesIndices[1]));
368
369 subsetCameras.clear();
370 subsetCameras.add(cameras.get(samplesIndices[0]));
371 subsetCameras.add(cameras.get(samplesIndices[1]));
372
373 try {
374 triangulator.setPointsAndCameras(subsetPoints, subsetCameras);
375 final var triangulated = triangulator.triangulate();
376 solutions.add(triangulated);
377 } catch (final Exception e) {
378 // if anything fails, no solution is added
379 }
380 }
381
382 @Override
383 public double computeResidual(final Point3D currentEstimation, final int i) {
384 final var point2D = points2D.get(i);
385 final var camera = cameras.get(i);
386
387 // project estimated point with camera
388 camera.project(currentEstimation, testPoint);
389
390 // return distance of projected point respect to the original one
391 // as a residual
392 return testPoint.distanceTo(point2D);
393 }
394
395 @Override
396 public boolean isReady() {
397 return PROMedSRobustSinglePoint3DTriangulator.this.isReady();
398 }
399
400 @Override
401 public void onEstimateStart(final RobustEstimator<Point3D> estimator) {
402 if (listener != null) {
403 listener.onTriangulateStart(PROMedSRobustSinglePoint3DTriangulator.this);
404 }
405 }
406
407 @Override
408 public void onEstimateEnd(final RobustEstimator<Point3D> estimator) {
409 if (listener != null) {
410 listener.onTriangulateEnd(PROMedSRobustSinglePoint3DTriangulator.this);
411 }
412 }
413
414 @Override
415 public void onEstimateNextIteration(
416 final RobustEstimator<Point3D> estimator, final int iteration) {
417 if (listener != null) {
418 listener.onTriangulateNextIteration(
419 PROMedSRobustSinglePoint3DTriangulator.this, iteration);
420 }
421 }
422
423 @Override
424 public void onEstimateProgressChange(
425 final RobustEstimator<Point3D> estimator, final float progress) {
426 if (listener != null) {
427 listener.onTriangulateProgressChange(
428 PROMedSRobustSinglePoint3DTriangulator.this, progress);
429 }
430 }
431
432 @Override
433 public double[] getQualityScores() {
434 return qualityScores;
435 }
436 });
437
438 try {
439 locked = true;
440 innerEstimator.setConfidence(confidence);
441 innerEstimator.setMaxIterations(maxIterations);
442 innerEstimator.setProgressDelta(progressDelta);
443 return innerEstimator.estimate();
444 } catch (final com.irurueta.numerical.LockedException e) {
445 throw new LockedException(e);
446 } catch (final com.irurueta.numerical.NotReadyException e) {
447 throw new NotReadyException(e);
448 } finally {
449 locked = false;
450 }
451 }
452
453 /**
454 * Returns method being used for robust estimation.
455 *
456 * @return method being used for robust estimation.
457 */
458 @Override
459 public RobustEstimatorMethod getMethod() {
460 return RobustEstimatorMethod.PROMEDS;
461 }
462
463 /**
464 * Sets quality scores corresponding to each provided view.
465 * This method is used internally and does not check whether instance is
466 * locked or not.
467 *
468 * @param qualityScores quality scores to be set.
469 * @throws IllegalArgumentException if provided quality scores length is
470 * smaller than MINIMUM_SIZE.
471 */
472 private void internalSetQualityScores(final double[] qualityScores) {
473 if (qualityScores.length < MIN_REQUIRED_VIEWS) {
474 throw new IllegalArgumentException();
475 }
476
477 this.qualityScores = qualityScores;
478 }
479 }