1 /*
2 * Copyright (C) 2015 Alberto Irurueta Carro (alberto@irurueta.com)
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package com.irurueta.geometry.estimators;
17
18 import com.irurueta.geometry.Line2D;
19 import com.irurueta.geometry.PinholeCamera;
20 import com.irurueta.geometry.Plane;
21 import com.irurueta.numerical.robust.*;
22
23 import java.util.ArrayList;
24 import java.util.List;
25
26 /**
27 * Finds the best pinhole camera for provided collections of matched lines and
28 * planes using PROMedS algorithm.
29 */
30 @SuppressWarnings("DuplicatedCode")
31 public class PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator
32 extends DLTLinePlaneCorrespondencePinholeCameraRobustEstimator {
33
34 /**
35 * Default value to be used for stop threshold. Stop threshold can be used
36 * to keep the algorithm iterating in case that best estimated threshold
37 * using median of residuals is not small enough. Once a solution is found
38 * that generates a threshold below this value, the algorithm will stop.
39 * The stop threshold can be used to prevent the LMedS algorithm iterating
40 * too many times in cases where samples have a very similar accuracy.
41 * For instance, in cases where proportion of outliers is very small (close
42 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
43 * iterate for a long time trying to find the best solution when indeed
44 * there is no need to do that if a reasonable threshold has already been
45 * reached.
46 * Because of this behaviour the stop threshold can be set to a value much
47 * lower than the one typically used in RANSAC, and yet the algorithm could
48 * still produce even smaller thresholds in estimated results.
49 */
50 public static final double DEFAULT_STOP_THRESHOLD = 1.0;
51
52 /**
53 * Minimum allowed stop threshold value.
54 */
55 public static final double MIN_STOP_THRESHOLD = 0.0;
56
57 /**
58 * Threshold to be used to keep the algorithm iterating in case that best
59 * estimated threshold using median of residuals is not small enough. Once
60 * a solution is found that generates a threshold below this value, the
61 * algorithm will stop.
62 * The stop threshold can be used to prevent the LMedS algorithm iterating
63 * too many times in cases where samples have a very similar accuracy.
64 * For instance, in cases where proportion of outliers is very small (close
65 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
66 * iterate for a long time trying to find the best solution when indeed
67 * there is no need to do that if a reasonable threshold has already been
68 * reached.
69 * Because of this behaviour the stop threshold can be set to a value much
70 * lower than the one typically used in RANSAC, and yet the algorithm could
71 * still produce even smaller thresholds in estimated results.
72 */
73 private double stopThreshold;
74
75 /**
76 * Quality scores corresponding to each pair of matched points.
77 * The larger the score value the better the quality of the matching.
78 */
79 private double[] qualityScores;
80
81 /**
82 * Constructor.
83 */
84 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator() {
85 super();
86 stopThreshold = DEFAULT_STOP_THRESHOLD;
87 }
88
89 /**
90 * Constructor with lists of matched planes and 2D lines to estimate a
91 * pinhole camera.
92 * Points and lines in the lists located at the same position are considered
93 * to be matched. Hence, both lists must have the same size, and their size
94 * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
95 * (4 matches).
96 *
97 * @param planes list of planes used to estimate a pinhole camera.
98 * @param lines list of corresponding projected 2D lines used to estimate
99 * a pinhole camera.
100 * @throws IllegalArgumentException if provided lists don't have the same
101 * size or their size is smaller than required minimum size (4 matches).
102 */
103 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
104 final List<Plane> planes, final List<Line2D> lines) {
105 super(planes, lines);
106 stopThreshold = DEFAULT_STOP_THRESHOLD;
107 }
108
109 /**
110 * Constructor with listener.
111 *
112 * @param listener listener to be notified of events such as when estimation
113 * starts, ends or its progress significantly changes.
114 */
115 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
116 final PinholeCameraRobustEstimatorListener listener) {
117 super(listener);
118 stopThreshold = DEFAULT_STOP_THRESHOLD;
119 }
120
121 /**
122 * Constructor with listener and lists of matched planes and 2D lines to
123 * estimate a pinhole camera.
124 * Points and lines in the lists located at the same position are considered
125 * to be matched. Hence, both lists must have the same size, and their size
126 * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
127 * (4 matches).
128 *
129 * @param listener listener to be notified of events such as when estimation
130 * starts, ends or its progress significantly changes.
131 * @param planes list of planes used to estimate a pinhole camera.
132 * @param lines list of corresponding projected 2D lines used to estimate
133 * a pinhole camera.
134 * @throws IllegalArgumentException if provided lists don't have the same
135 * size or their size is smaller than required minimum size (4 matches).
136 */
137 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
138 final PinholeCameraRobustEstimatorListener listener, final List<Plane> planes, final List<Line2D> lines) {
139 super(listener, planes, lines);
140 stopThreshold = DEFAULT_STOP_THRESHOLD;
141 }
142
143 /**
144 * Constructor.
145 *
146 * @param qualityScores quality scores corresponding to each pair of matched
147 * points.
148 * @throws IllegalArgumentException if provided quality scores length is
149 * smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
150 */
151 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(final double[] qualityScores) {
152 super();
153 stopThreshold = DEFAULT_STOP_THRESHOLD;
154 internalSetQualityScores(qualityScores);
155 }
156
157 /**
158 * Constructor with lists of matched planes and 2D lines to estimate a
159 * pinhole camera.
160 * Points and lines in the lists located at the same position are considered
161 * to be matched. Hence, both lists must have the same size, and their size
162 * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
163 * (4 matches).
164 *
165 * @param planes list of planes used to estimate a pinhole camera.
166 * @param lines list of corresponding projected 2D lines used to estimate
167 * a pinhole camera.
168 * @param qualityScores quality scores corresponding to each pair of matched
169 * points.
170 * @throws IllegalArgumentException if provided lists or quality scores
171 * don't have the same size or their size is smaller than required minimum
172 * size (4 matches).
173 */
174 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
175 final List<Plane> planes, final List<Line2D> lines, final double[] qualityScores) {
176 super(planes, lines);
177
178 if (qualityScores.length != planes.size()) {
179 throw new IllegalArgumentException();
180 }
181
182 stopThreshold = DEFAULT_STOP_THRESHOLD;
183 internalSetQualityScores(qualityScores);
184 }
185
186 /**
187 * Constructor with listener.
188 *
189 * @param listener listener to be notified of events such as when estimation
190 * starts, ends or its progress significantly changes.
191 * @param qualityScores quality scores corresponding to each pair of matched
192 * points.
193 * @throws IllegalArgumentException if provided quality scores length is
194 * smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
195 */
196 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
197 final PinholeCameraRobustEstimatorListener listener, final double[] qualityScores) {
198 super(listener);
199 stopThreshold = DEFAULT_STOP_THRESHOLD;
200 internalSetQualityScores(qualityScores);
201 }
202
203 /**
204 * Constructor with listener and lists of matched planes and 2D lines to
205 * estimate a pinhole camera.
206 * Points and lines in the lists located at the same position are considered
207 * to be matched. Hence, both lists must have the same size, and their size
208 * must be greater or equal than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES
209 * (4 matches).
210 *
211 * @param listener listener to be notified of events such as when estimation
212 * starts, ends or its progress significantly changes.
213 * @param planes list of planes used to estimate a pinhole camera.
214 * @param lines list of corresponding projected 2D lines used to estimate
215 * a pinhole camera.
216 * @param qualityScores quality scores corresponding to each pair of matched
217 * points.
218 * @throws IllegalArgumentException if provided lists don't have the same
219 * size or their size is smaller than required minimum size (4 matches).
220 */
221 public PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator(
222 final PinholeCameraRobustEstimatorListener listener, final List<Plane> planes, final List<Line2D> lines,
223 final double[] qualityScores) {
224 super(listener, planes, lines);
225
226 if (qualityScores.length != planes.size()) {
227 throw new IllegalArgumentException();
228 }
229
230 stopThreshold = DEFAULT_STOP_THRESHOLD;
231 internalSetQualityScores(qualityScores);
232 }
233
234 /**
235 * Returns threshold to be used to keep the algorithm iterating in case that
236 * best estimated threshold using median of residuals is not small enough.
237 * Once a solution is found that generates a threshold below this value, the
238 * algorithm will stop.
239 * As in LMedS, the stop threshold can be used to prevent the PROMedS
240 * algorithm iterating too many times in cases where samples have a very
241 * similar accuracy.
242 * For instance, in cases where proportion of outliers is very small (close
243 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
244 * iterate for a long time trying to find the best solution when indeed
245 * there is no need to do that if a reasonable threshold has already been
246 * reached.
247 * Because of this behaviour the stop threshold can be set to a value much
248 * lower than the one typically used in RANSAC, and yet the algorithm could
249 * still produce even smaller thresholds in estimated results.
250 *
251 * @return stop threshold to stop the algorithm prematurely when a certain
252 * accuracy has been reached.
253 */
254 public double getStopThreshold() {
255 return stopThreshold;
256 }
257
258 /**
259 * Sets threshold to be used to keep the algorithm iterating in case that
260 * best estimated threshold using median of residuals is not small enough.
261 * Once a solution is found that generates a threshold below this value, the
262 * algorithm will stop.
263 * As in LMedS, the stop threshold can be used to prevent the PROMedS
264 * algorithm iterating too many times in cases where samples have a very
265 * similar accuracy.
266 * For instance, in cases where proportion of outliers is very small (close
267 * to 0%), and samples are very accurate (i.e. 1e-6), the algorithm would
268 * iterate for a long time trying to find the best solution when indeed
269 * there is no need to do that if a reasonable threshold has already been
270 * reached.
271 * Because of this behaviour the stop threshold can be set to a value much
272 * lower than the one typically used in RANSAC, and yet the algorithm could
273 * still produce even smaller thresholds in estimated results.
274 *
275 * @param stopThreshold stop threshold to stop the algorithm prematurely
276 * when a certain accuracy has been reached.
277 * @throws IllegalArgumentException if provided value is zero or negative.
278 * @throws LockedException if robust estimator is locked because an
279 * estimation is already in progress.
280 */
281 public void setStopThreshold(final double stopThreshold) throws LockedException {
282 if (isLocked()) {
283 throw new LockedException();
284 }
285 if (stopThreshold <= MIN_STOP_THRESHOLD) {
286 throw new IllegalArgumentException();
287 }
288
289 this.stopThreshold = stopThreshold;
290 }
291
292 /**
293 * Returns quality scores corresponding to each pair of matched points.
294 * The larger the score value the better the quality of the matching.
295 *
296 * @return quality scores corresponding to each pair of matched points.
297 */
298 @Override
299 public double[] getQualityScores() {
300 return qualityScores;
301 }
302
303 /**
304 * Sets quality scores corresponding to each pair of matched points.
305 * The larger the score value the better the quality of the matching.
306 *
307 * @param qualityScores quality scores corresponding to each pair of matched
308 * points.
309 * @throws LockedException if robust estimator is locked because an
310 * estimation is already in progress.
311 * @throws IllegalArgumentException if provided quality scores length is
312 * smaller than MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES (i.e. 4 samples).
313 */
314 @Override
315 public void setQualityScores(final double[] qualityScores) throws LockedException {
316 if (isLocked()) {
317 throw new LockedException();
318 }
319 internalSetQualityScores(qualityScores);
320 }
321
322 /**
323 * Indicates if estimator is ready to start the affine 2D transformation
324 * estimation.
325 * This is true when input data (i.e. lists of matched points and quality
326 * scores) are provided and a minimum of MINIMUM_SIZE points are available.
327 *
328 * @return true if estimator is ready, false otherwise.
329 */
330 @Override
331 public boolean isReady() {
332 return super.isReady() && qualityScores != null && qualityScores.length == planes.size();
333 }
334
335 /**
336 * Estimates a pinhole camera using a robust estimator and
337 * the best set of matched 2D line/3D plane correspondences found using the
338 * robust estimator.
339 *
340 * @return a pinhole camera.
341 * @throws LockedException if robust estimator is locked because an
342 * estimation is already in progress.
343 * @throws NotReadyException if provided input data is not enough to start
344 * the estimation.
345 * @throws RobustEstimatorException if estimation fails for any reason
346 * (i.e. numerical instability, no solution available, etc).
347 */
348 @Override
349 public PinholeCamera estimate() throws LockedException, NotReadyException, RobustEstimatorException {
350 if (isLocked()) {
351 throw new LockedException();
352 }
353 if (!isReady()) {
354 throw new NotReadyException();
355 }
356
357 // pinhole camera estimator using DLT (Direct Linear Transform) algorithm
358 final var nonRobustEstimator = new DLTLinePlaneCorrespondencePinholeCameraEstimator();
359
360 nonRobustEstimator.setLMSESolutionAllowed(false);
361
362 // suggestions
363 nonRobustEstimator.setSuggestSkewnessValueEnabled(isSuggestSkewnessValueEnabled());
364 nonRobustEstimator.setSuggestedSkewnessValue(getSuggestedSkewnessValue());
365 nonRobustEstimator.setSuggestHorizontalFocalLengthEnabled(isSuggestHorizontalFocalLengthEnabled());
366 nonRobustEstimator.setSuggestedHorizontalFocalLengthValue(getSuggestedHorizontalFocalLengthValue());
367 nonRobustEstimator.setSuggestVerticalFocalLengthEnabled(isSuggestVerticalFocalLengthEnabled());
368 nonRobustEstimator.setSuggestedVerticalFocalLengthValue(getSuggestedVerticalFocalLengthValue());
369 nonRobustEstimator.setSuggestAspectRatioEnabled(isSuggestAspectRatioEnabled());
370 nonRobustEstimator.setSuggestedAspectRatioValue(getSuggestedAspectRatioValue());
371 nonRobustEstimator.setSuggestPrincipalPointEnabled(isSuggestPrincipalPointEnabled());
372 nonRobustEstimator.setSuggestedPrincipalPointValue(getSuggestedPrincipalPointValue());
373 nonRobustEstimator.setSuggestRotationEnabled(isSuggestRotationEnabled());
374 nonRobustEstimator.setSuggestedRotationValue(getSuggestedRotationValue());
375 nonRobustEstimator.setSuggestCenterEnabled(isSuggestCenterEnabled());
376 nonRobustEstimator.setSuggestedCenterValue(getSuggestedCenterValue());
377
378 final var innerEstimator = new PROMedSRobustEstimator<>(new PROMedSRobustEstimatorListener<PinholeCamera>() {
379
380 // 3D planes for a subset of samples
381 private final List<Plane> subsetPlanes = new ArrayList<>();
382
383 // 2D lines for a subset of samples
384 private final List<Line2D> subsetLines = new ArrayList<>();
385
386 @Override
387 public double getThreshold() {
388 return stopThreshold;
389 }
390
391 @Override
392 public int getTotalSamples() {
393 return planes.size();
394 }
395
396 @Override
397 public int getSubsetSize() {
398 return LinePlaneCorrespondencePinholeCameraEstimator.MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES;
399 }
400
401 @Override
402 public void estimatePreliminarSolutions(final int[] samplesIndices, final List<PinholeCamera> solutions) {
403 subsetPlanes.clear();
404 subsetPlanes.add(planes.get(samplesIndices[0]));
405 subsetPlanes.add(planes.get(samplesIndices[1]));
406 subsetPlanes.add(planes.get(samplesIndices[2]));
407 subsetPlanes.add(planes.get(samplesIndices[3]));
408
409 subsetLines.clear();
410 subsetLines.add(lines.get(samplesIndices[0]));
411 subsetLines.add(lines.get(samplesIndices[1]));
412 subsetLines.add(lines.get(samplesIndices[2]));
413 subsetLines.add(lines.get(samplesIndices[3]));
414
415 try {
416 nonRobustEstimator.setLists(subsetPlanes, subsetLines);
417
418 final var cam = nonRobustEstimator.estimate();
419 solutions.add(cam);
420 } catch (final Exception e) {
421 // if lines/planes configuration is degenerate, no solution is added
422 }
423 }
424
425 @Override
426 public double computeResidual(PinholeCamera currentEstimation, int i) {
427 final var inputLine = lines.get(i);
428 final var inputPlane = planes.get(i);
429
430 return singleBackprojectionResidual(currentEstimation, inputLine, inputPlane);
431 }
432
433 @Override
434 public boolean isReady() {
435 return PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this.isReady();
436 }
437
438 @Override
439 public void onEstimateStart(final RobustEstimator<PinholeCamera> estimator) {
440 if (listener != null) {
441 listener.onEstimateStart(
442 PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this);
443 }
444 }
445
446 @Override
447 public void onEstimateEnd(final RobustEstimator<PinholeCamera> estimator) {
448 if (listener != null) {
449 listener.onEstimateEnd(PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this);
450 }
451 }
452
453 @Override
454 public void onEstimateNextIteration(final RobustEstimator<PinholeCamera> estimator, int iteration) {
455 if (listener != null) {
456 listener.onEstimateNextIteration(
457 PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this, iteration);
458 }
459 }
460
461 @Override
462 public void onEstimateProgressChange(final RobustEstimator<PinholeCamera> estimator, float progress) {
463 if (listener != null) {
464 listener.onEstimateProgressChange(
465 PROMedSDLTLinePlaneCorrespondencePinholeCameraRobustEstimator.this, progress);
466 }
467 }
468
469 @Override
470 public double[] getQualityScores() {
471 return qualityScores;
472 }
473 });
474
475 try {
476 locked = true;
477 inliersData = null;
478 innerEstimator.setConfidence(confidence);
479 innerEstimator.setMaxIterations(maxIterations);
480 innerEstimator.setProgressDelta(progressDelta);
481 final var result = innerEstimator.estimate();
482 inliersData = innerEstimator.getInliersData();
483 return attemptRefine(result, nonRobustEstimator.getMaxSuggestionWeight());
484
485 } catch (final com.irurueta.numerical.LockedException e) {
486 throw new LockedException(e);
487 } catch (final com.irurueta.numerical.NotReadyException e) {
488 throw new NotReadyException(e);
489 } finally {
490 locked = false;
491 }
492 }
493
494 /**
495 * Returns method being used for robust estimation.
496 *
497 * @return method being used for robust estimation.
498 */
499 @Override
500 public RobustEstimatorMethod getMethod() {
501 return RobustEstimatorMethod.PROMEDS;
502 }
503
504 /**
505 * Gets standard deviation used for Levenberg-Marquardt fitting during
506 * refinement.
507 * Returned value gives an indication of how much variance each residual
508 * has.
509 * Typically, this value is related to the threshold used on each robust
510 * estimation, since residuals of found inliers are within the range of
511 * such threshold.
512 *
513 * @return standard deviation used for refinement.
514 */
515 @Override
516 protected double getRefinementStandardDeviation() {
517 final var inliersData = (PROMedSRobustEstimator.PROMedSInliersData) getInliersData();
518
519 // avoid setting a threshold too strict
520 final var threshold = inliersData.getEstimatedThreshold();
521 return Math.max(threshold, stopThreshold);
522 }
523
524 /**
525 * Sets quality scores corresponding to each pair of matched points.
526 * This method is used internally and does not check whether instance is
527 * locked or not.
528 *
529 * @param qualityScores quality scores to be set.
530 * @throws IllegalArgumentException if provided quality scores length is
531 * smaller than MINIMUM_SIZE.
532 */
533 private void internalSetQualityScores(final double[] qualityScores) {
534 if (qualityScores.length < MIN_NUMBER_OF_LINE_PLANE_CORRESPONDENCES) {
535 throw new IllegalArgumentException();
536 }
537
538 this.qualityScores = qualityScores;
539 }
540 }