1 /*
2 * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.geometry.estimators;
17
18 import com.irurueta.geometry.CoordinatesType;
19 import com.irurueta.geometry.PinholeCamera;
20 import com.irurueta.geometry.Point2D;
21 import com.irurueta.geometry.Point3D;
22 import com.irurueta.numerical.robust.PROMedSRobustEstimator;
23 import com.irurueta.numerical.robust.PROMedSRobustEstimatorListener;
24 import com.irurueta.numerical.robust.RobustEstimator;
25 import com.irurueta.numerical.robust.RobustEstimatorException;
26 import com.irurueta.numerical.robust.RobustEstimatorMethod;
27
28 import java.util.ArrayList;
29 import java.util.List;
30
31 /**
32 * Finds the best pinhole camera for provided collections of matched 2D/3D
33 * points using PROMedS algorithm.
34 */
35 @SuppressWarnings("DuplicatedCode")
36 public class PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator extends
37 DLTPointCorrespondencePinholeCameraRobustEstimator {
38
39 /**
40 * Default value to be used for stop threshold. Stop threshold can be used
41 * to keep the algorithm iterating in case that best estimated threshold
42 * using median of residuals is not small enough. Once a solution is found
43 * that generates a threshold below this value, the algorithm will stop.
44 * The stop threshold can be used to prevent the LMedS algorithm iterating
45 * too many times in cases where samples have a very similar accuracy.
46 * For instance, in cases where proportion of outliers is very small (close
47 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
48 * iterate for a long time trying to find the best solution when indeed
49 * there is no need to do that if a reasonable threshold has already been
50 * reached.
51 * Because of this behaviour the stop threshold can be set to a value much
52 * lower than the one typically used in RANSAC, and yet the algorithm could
53 * still produce even smaller thresholds in estimated results.
54 */
55 public static final double DEFAULT_STOP_THRESHOLD = 1.0;
56
57 /**
58 * Minimum allowed stop threshold value.
59 */
60 public static final double MIN_STOP_THRESHOLD = 0.0;
61
62 /**
63 * Threshold to be used to keep the algorithm iterating in case that best
64 * estimated threshold using median of residuals is not small enough. Once
65 * a solution is found that generates a threshold below this value, the
66 * algorithm will stop.
67 * The stop threshold can be used to prevent the LMedS algorithm iterating
68 * too many times in cases where samples have a very similar accuracy.
69 * For instance, in cases where proportion of outliers is very small (close
70 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
71 * iterate for a long time trying to find the best solution when indeed
72 * there is no need to do that if a reasonable threshold has already been
73 * reached.
74 * Because of this behaviour the stop threshold can be set to a value much
75 * lower than the one typically used in RANSAC, and yet the algorithm could
76 * still produce even smaller thresholds in estimated results.
77 */
78 private double stopThreshold;
79
80 /**
81 * Quality scores corresponding to each pair of matched points.
82 * The larger the score value the better the quality of the matching.
83 */
84 private double[] qualityScores;
85
86 /**
87 * Constructor.
88 */
89 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator() {
90 super();
91 stopThreshold = DEFAULT_STOP_THRESHOLD;
92 }
93
94 /**
95 * Constructor with lists of points to be used to estimate a pinhole camera.
96 * Points in the list located at the same position are considered to be
97 * matched. Hence, both lists must have the same size, and their size must
98 * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
99 *
100 * @param points3D list of 3D points used to estimate a pinhole camera.
101 * @param points2D list of corresponding projected 2D points used to
102 * estimate a pinhole camera.
103 * @throws IllegalArgumentException if provided lists of points don't have
104 * the same size or their size is smaller than required minimum size
105 * (6 correspondences).
106 */
107 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
108 final List<Point3D> points3D, final List<Point2D> points2D) {
109 super(points3D, points2D);
110 stopThreshold = DEFAULT_STOP_THRESHOLD;
111 }
112
113 /**
114 * Constructor.
115 *
116 * @param listener listener to be notified of events such as when estimation
117 * starts, ends or its progress significantly changes.
118 */
119 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
120 final PinholeCameraRobustEstimatorListener listener) {
121 super(listener);
122 stopThreshold = DEFAULT_STOP_THRESHOLD;
123 }
124
125 /**
126 * Constructor with listener and lists of points to be used ot estimate a
127 * pinhole camera.
128 * Points in the list located at the same position are considered to be
129 * matched. Hence, both lists must have the same size, and their size must
130 * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
131 *
132 * @param listener listener to be notified of events such as when estimation
133 * starts, ends or its progress significantly changes.
134 * @param points3D list of 3D points used to estimate a pinhole camera.
135 * @param points2D list of corresponding projected 2D points used to
136 * estimate a pinhole camera.
137 * @throws IllegalArgumentException if provided lists of points don't have
138 * the same size or their size is smaller than required minimum size
139 * (6 correspondences).
140 */
141 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
142 final PinholeCameraRobustEstimatorListener listener,
143 final List<Point3D> points3D, final List<Point2D> points2D) {
144 super(listener, points3D, points2D);
145 stopThreshold = DEFAULT_STOP_THRESHOLD;
146 }
147
148 /**
149 * Constructor.
150 *
151 * @param qualityScores quality scores corresponding to each pair of matched
152 * points.
153 * @throws IllegalArgumentException if provided quality scores length is
154 * smaller than MINIMUM_SIZE (i.e. 3 samples).
155 */
156 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(final double[] qualityScores) {
157 super();
158 stopThreshold = DEFAULT_STOP_THRESHOLD;
159 internalSetQualityScores(qualityScores);
160 }
161
162 /**
163 * Constructor with lists of points to be used to estimate a pinhole camera.
164 * Points in the list located at the same position are considered to be
165 * matched. Hence, both lists must have the same size, and their size must
166 * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
167 *
168 * @param points3D list of 3D points used to estimate a pinhole camera.
169 * @param points2D list of corresponding projected 2D points used to
170 * estimate a pinhole camera.
171 * @param qualityScores quality scores corresponding to each pair of matched
172 * points.
173 * @throws IllegalArgumentException if provided lists of points and array
174 * of quality scores don't have the same size or their size is smaller than
175 * 6 correspondences.
176 */
177 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
178 final List<Point3D> points3D, final List<Point2D> points2D, final double[] qualityScores) {
179 super(points3D, points2D);
180
181 if (qualityScores.length != points3D.size()) {
182 throw new IllegalArgumentException();
183 }
184
185 stopThreshold = DEFAULT_STOP_THRESHOLD;
186 internalSetQualityScores(qualityScores);
187 }
188
189 /**
190 * Constructor.
191 *
192 * @param listener listener to be notified of events such as when estimation
193 * starts, ends or its progress significantly changes.
194 * @param qualityScores quality scores corresponding to each pair of matched
195 * points.
196 * @throws IllegalArgumentException if provided quality scores length is
197 * smaller than MINIMUM_SIZE (i.e. 3 samples).
198 */
199 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
200 final PinholeCameraRobustEstimatorListener listener, final double[] qualityScores) {
201 super(listener);
202 stopThreshold = DEFAULT_STOP_THRESHOLD;
203 internalSetQualityScores(qualityScores);
204 }
205
206 /**
207 * Constructor with listener and lists of points to be used ot estimate a
208 * pinhole camera.
209 * Points in the list located at the same position are considered to be
210 * matched. Hence, both lists must have the same size, and their size must
211 * be greater or equal than MIN_NUMBER_OF_POINT_CORRESPONDENCES.
212 *
213 * @param listener listener to be notified of events such as when estimation
214 * starts, ends or its progress significantly changes.
215 * @param points3D list of 3D points used to estimate a pinhole camera.
216 * @param points2D list of corresponding projected 2D points used to
217 * estimate a pinhole camera.
218 * @param qualityScores quality scores corresponding to each pair of matched
219 * points.
220 * @throws IllegalArgumentException if provided lists of points don't have
221 * the same size or their size is smaller than
222 * MIN_NUMBER_OF_POINT_CORRESPONDENCES.
223 */
224 public PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator(
225 final PinholeCameraRobustEstimatorListener listener,
226 final List<Point3D> points3D, final List<Point2D> points2D, final double[] qualityScores) {
227 super(listener, points3D, points2D);
228
229 if (qualityScores.length != points3D.size()) {
230 throw new IllegalArgumentException();
231 }
232
233 stopThreshold = DEFAULT_STOP_THRESHOLD;
234 internalSetQualityScores(qualityScores);
235 }
236
237 /**
238 * Returns threshold to be used to keep the algorithm iterating in case that
239 * best estimated threshold using median of residuals is not small enough.
240 * Once a solution is found that generates a threshold below this value, the
241 * algorithm will stop.
242 * As in LMedS, the stop threshold can be used to prevent the PROMedS
243 * algorithm iterating too many times in cases where samples have a very
244 * similar accuracy.
245 * For instance, in cases where proportion of outliers is very small (close
246 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
247 * iterate for a long time trying to find the best solution when indeed
248 * there is no need to do that if a reasonable threshold has already been
249 * reached.
250 * Because of this behaviour the stop threshold can be set to a value much
251 * lower than the one typically used in RANSAC, and yet the algorithm could
252 * still produce even smaller thresholds in estimated results.
253 *
254 * @return stop threshold to stop the algorithm prematurely when a certain
255 * accuracy has been reached.
256 */
257 public double getStopThreshold() {
258 return stopThreshold;
259 }
260
261 /**
262 * Sets threshold to be used to keep the algorithm iterating in case that
263 * best estimated threshold using median of residuals is not small enough.
264 * Once a solution is found that generates a threshold below this value, the
265 * algorithm will stop.
266 * As in LMedS, the stop threshold can be used to prevent the PROMedS
267 * algorithm iterating too many times in cases where samples have a very
268 * similar accuracy.
269 * For instance, in cases where proportion of outliers is very small (close
270 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
271 * iterate for a long time trying to find the best solution when indeed
272 * there is no need to do that if a reasonable threshold has already been
273 * reached.
274 * Because of this behaviour the stop threshold can be set to a value much
275 * lower than the one typically used in RANSAC, and yet the algorithm could
276 * still produce even smaller thresholds in estimated results.
277 *
278 * @param stopThreshold stop threshold to stop the algorithm prematurely
279 * when a certain accuracy has been reached.
280 * @throws IllegalArgumentException if provided value is zero or negative.
281 * @throws LockedException if robust estimator is locked because an
282 * estimation is already in progress.
283 */
284 public void setStopThreshold(final double stopThreshold) throws LockedException {
285 if (isLocked()) {
286 throw new LockedException();
287 }
288 if (stopThreshold <= MIN_STOP_THRESHOLD) {
289 throw new IllegalArgumentException();
290 }
291
292 this.stopThreshold = stopThreshold;
293 }
294
295 /**
296 * Returns quality scores corresponding to each pair of matched points.
297 * The larger the score value the better the quality of the matching.
298 *
299 * @return quality scores corresponding to each pair of matched points.
300 */
301 @Override
302 public double[] getQualityScores() {
303 return qualityScores;
304 }
305
306 /**
307 * Sets quality scores corresponding to each pair of matched points.
308 * The larger the score value the better the quality of the matching.
309 *
310 * @param qualityScores quality scores corresponding to each pair of matched
311 * points.
312 * @throws LockedException if robust estimator is locked because an
313 * estimation is already in progress.
314 * @throws IllegalArgumentException if provided quality scores length is
315 * smaller than MINIMUM_SIZE (i.e. 3 samples).
316 */
317 @Override
318 public void setQualityScores(final double[] qualityScores) throws LockedException {
319 if (isLocked()) {
320 throw new LockedException();
321 }
322 internalSetQualityScores(qualityScores);
323 }
324
325 /**
326 * Indicates if estimator is ready to start the affine 2D transformation
327 * estimation.
328 * This is true when input data (i.e. lists of matched points and quality
329 * scores) are provided and a minimum of MINIMUM_SIZE points are available.
330 *
331 * @return true if estimator is ready, false otherwise.
332 */
333 @Override
334 public boolean isReady() {
335 return super.isReady() && qualityScores != null && qualityScores.length == points3D.size();
336 }
337
338 /**
339 * Estimates an affine 2D transformation using a robust estimator and
340 * the best set of matched 2D point correspondences found using the robust
341 * estimator.
342 *
343 * @return an affine 2D transformation.
344 * @throws LockedException if robust estimator is locked because an
345 * estimation is already in progress.
346 * @throws NotReadyException if provided input data is not enough to start
347 * the estimation.
348 * @throws RobustEstimatorException if estimation fails for any reason
349 * (i.e. numerical instability, no solution available, etc).
350 */
351 @Override
352 public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
353 if (isLocked()) {
354 throw new LockedException();
355 }
356 if (!isReady()) {
357 throw new NotReadyException();
358 }
359
360 // pinhole camera estimator using DLT (Direct Linear Transform) algorithm
361 final var nonRobustEstimator = new DLTPointCorrespondencePinholeCameraEstimator();
362
363 nonRobustEstimator.setLMSESolutionAllowed(false);
364 nonRobustEstimator.setPointCorrespondencesNormalized(normalizeSubsetPointCorrespondences);
365
366 // suggestions
367 nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
368 nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
369 nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
370 nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
371 nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
372 nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
373 nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
374 nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
375 nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
376 nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
377 nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
378 nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
379 nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
380 nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
381
382 final var innerEstimator = new PROMedSRobustEstimator<>(new PROMedSRobustEstimatorListener<PinholeCamera>() {
383
384 // point to be reused when computing residuals
385 private final Point2D testPoint = Point2D.create(CoordinatesType.HOMOGENEOUS_COORDINATES);
386
387 // 3D points for a subset of samples
388 private final List<Point3D> subset3D = new ArrayList<>();
389
390 // 2D points for a subset of samples
391 private final List<Point2D> subset2D = new ArrayList<>();
392
393 @Override
394 public double getThreshold() {
395 return stopThreshold;
396 }
397
398 @Override
399 public int getTotalSamples() {
400 return points3D.size();
401 }
402
403 @Override
404 public int getSubsetSize() {
405 return PointCorrespondencePinholeCameraRobustEstimator.MIN_NUMBER_OF_POINT_CORRESPONDENCES;
406 }
407
408 @Override
409 public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
410 subset3D.clear();
411 subset3D.add(points3D.get(samplesIndices[0]));
412 subset3D.add(points3D.get(samplesIndices[1]));
413 subset3D.add(points3D.get(samplesIndices[2]));
414 subset3D.add(points3D.get(samplesIndices[3]));
415 subset3D.add(points3D.get(samplesIndices[4]));
416 subset3D.add(points3D.get(samplesIndices[5]));
417
418 subset2D.clear();
419 subset2D.add(points2D.get(samplesIndices[0]));
420 subset2D.add(points2D.get(samplesIndices[1]));
421 subset2D.add(points2D.get(samplesIndices[2]));
422 subset2D.add(points2D.get(samplesIndices[3]));
423 subset2D.add(points2D.get(samplesIndices[4]));
424 subset2D.add(points2D.get(samplesIndices[5]));
425
426 try {
427 nonRobustEstimator.setLists(subset3D, subset2D);
428
429 final var cam = nonRobustEstimator.estimate();
430 solutions.add(cam);
431 } catch (final Exception e) {
432 // if points configuration is degenerate, no solution is added
433 }
434 }
435
436 @Override
437 public double computeResidual(final PinholeCamera currentEstimation, final int i) {
438 // pick i-th points
439 final var point3D = points3D.get(i);
440 final var point2D = points2D.get(i);
441
442 // project point3D into test point
443 currentEstimation.project(point3D, testPoint);
444
445 // compare test point and 2D point
446 return testPoint.distanceTo(point2D);
447 }
448
449 @Override
450 public boolean isReady() {
451 return PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this.isReady();
452 }
453
454 @Override
455 public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
456 if (listener != null) {
457 listener.onEstimateStart(PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this);
458 }
459 }
460
461 @Override
462 public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
463 if (listener != null) {
464 listener.onEstimateEnd(PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this);
465 }
466 }
467
468 @Override
469 public void onEstimateNextIteration(
470 final RobustEstimator<PinholeCamera> estimator, final int iteration) {
471 if (listener != null) {
472 listener.onEstimateNextIteration(
473 PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this, iteration);
474 }
475 }
476
477 @Override
478 public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, final float progress) {
479 if (listener != null) {
480 listener.onEstimateProgressChange(
481 PROMedSDLTPointCorrespondencePinholeCameraRobustEstimator.this, progress);
482 }
483 }
484
485 @Override
486 public double[] getQualityScores() {
487 return qualityScores;
488 }
489 });
490
491 try {
492 locked = true;
493 inliersData = null;
494 innerEstimator.setConfidence(confidence);
495 innerEstimator.setMaxIterations(maxIterations);
496 innerEstimator.setProgressDelta(progressDelta);
497 final var result = innerEstimator.estimate();
498 inliersData = innerEstimator.getInliersData();
499 return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
500 } catch (final com.irurueta.numerical.LockedException e) {
501 throw new LockedException(e);
502 } catch (final com.irurueta.numerical.NotReadyException e) {
503 throw new NotReadyException(e);
504 } finally {
505 locked = false;
506 }
507 }
508
509 /**
510 * Returns method being used for robust estimation.
511 *
512 * @return method being used for robust estimation.
513 */
514 @Override
515 public RobustEstimatorMethod getMethod() {
516 return RobustEstimatorMethod.PROMEDS;
517 }
518
519 /**
520 * Gets standard deviation used for Levenberg-Marquardt fitting during
521 * refinement.
522 * Returned value gives an indication of how much variance each residual
523 * has.
524 * Typically, this value is related to the threshold used on each robust
525 * estimation, since residuals of found inliers are within the range of
526 * such threshold.
527 *
528 * @return standard deviation used for refinement.
529 */
530 @Override
531 protected double getRefinementStandardDeviation() {
532 final var inliersData = (PROMedSRobustEstimator.PROMedSInliersData) getInliersData();
533
534 // avoid setting a threshold too strict
535 final var threshold = inliersData.getEstimatedThreshold();
536 return Math.max(threshold, stopThreshold);
537 }
538
539 /**
540 * Sets quality scores corresponding to each pair of matched points.
541 * This method is used internally and does not check whether instance is
542 * locked or not.
543 *
544 * @param qualityScores quality scores to be set.
545 * @throws IllegalArgumentException if provided quality scores length is
546 * smaller than MINIMUM_SIZE.
547 */
548 private void internalSetQualityScores(double[] qualityScores) {
549 if (qualityScores.length < MIN_NUMBER_OF_POINT_CORRESPONDENCES) {
550 throw new IllegalArgumentException();
551 }
552
553 this.qualityScores = qualityScores;
554 }
555 }