1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package com.irurueta.ar.calibration;
17
18 import com.irurueta.ar.calibration.estimators.LMedSRadialDistortionRobustEstimator;
19 import com.irurueta.ar.calibration.estimators.MSACRadialDistortionRobustEstimator;
20 import com.irurueta.ar.calibration.estimators.PROMedSRadialDistortionRobustEstimator;
21 import com.irurueta.ar.calibration.estimators.PROSACRadialDistortionRobustEstimator;
22 import com.irurueta.ar.calibration.estimators.RANSACRadialDistortionRobustEstimator;
23 import com.irurueta.ar.calibration.estimators.RadialDistortionRobustEstimator;
24 import com.irurueta.ar.calibration.estimators.RadialDistortionRobustEstimatorListener;
25 import com.irurueta.geometry.GeometryException;
26 import com.irurueta.geometry.PinholeCameraIntrinsicParameters;
27 import com.irurueta.geometry.Point2D;
28 import com.irurueta.geometry.estimators.LockedException;
29 import com.irurueta.geometry.estimators.NotReadyException;
30 import com.irurueta.numerical.NumericalException;
31 import com.irurueta.numerical.robust.RobustEstimatorMethod;
32
33 import java.util.ArrayList;
34 import java.util.List;
35 import java.util.logging.Level;
36 import java.util.logging.Logger;
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52 @SuppressWarnings("DuplicatedCode")
53 public class AlternatingCameraCalibrator extends CameraCalibrator {
54
55
56
57
58
59 public static final int DEFAULT_MAX_ITERATIONS = 20;
60
61
62
63
64 public static final int MIN_MAX_ITERATIONS = 1;
65
66
67
68
69
70 public static final double DEFAULT_CONVERGENCE_THRESHOLD = 1e-8;
71
72
73
74
75 public static final double MIN_CONVERGENCE_THRESHOLD = 0.0;
76
77
78
79
80
81 public static final RobustEstimatorMethod DEFAULT_RADIAL_DISTORTION_METHOD = RobustEstimatorMethod.PROSAC;
82
83
84
85
86
87 private int maxIterations;
88
89
90
91
92
93 private double convergenceThreshold;
94
95
96
97
98 private RobustEstimatorMethod distortionMethod;
99
100
101
102
103 private RadialDistortionRobustEstimator distortionEstimator;
104
105
106
107
108 private RadialDistortionRobustEstimatorListener distortionEstimatorListener;
109
110
111
112
113 private float radialDistortionProgress;
114
115
116
117
118 private float iterProgress;
119
120
121
122
123 private float previousNotifiedProgress;
124
125
126
127
128 public AlternatingCameraCalibrator() {
129 super();
130 maxIterations = DEFAULT_MAX_ITERATIONS;
131 convergenceThreshold = DEFAULT_CONVERGENCE_THRESHOLD;
132
133 internalSetDistortionMethod(DEFAULT_RADIAL_DISTORTION_METHOD);
134 }
135
136
137
138
139
140
141
142
143 public AlternatingCameraCalibrator(final Pattern2D pattern, final List<CameraCalibratorSample> samples) {
144 super(pattern, samples);
145 maxIterations = DEFAULT_MAX_ITERATIONS;
146 convergenceThreshold = DEFAULT_CONVERGENCE_THRESHOLD;
147
148 internalSetDistortionMethod(DEFAULT_RADIAL_DISTORTION_METHOD);
149 }
150
151
152
153
154
155
156
157
158
159
160 public AlternatingCameraCalibrator(
161 final Pattern2D pattern, final List<CameraCalibratorSample> samples, final double[] samplesQualityScores) {
162 super(pattern, samples, samplesQualityScores);
163 maxIterations = DEFAULT_MAX_ITERATIONS;
164 convergenceThreshold = DEFAULT_CONVERGENCE_THRESHOLD;
165
166 internalSetDistortionMethod(DEFAULT_RADIAL_DISTORTION_METHOD);
167 }
168
169
170
171
172
173
174
175 public int getMaxIterations() {
176 return maxIterations;
177 }
178
179
180
181
182
183
184
185
186
187
188 public void setMaxIterations(final int maxIterations) throws LockedException {
189 if (isLocked()) {
190 throw new LockedException();
191 }
192 if (maxIterations < MIN_MAX_ITERATIONS) {
193 throw new IllegalArgumentException();
194 }
195
196 this.maxIterations = maxIterations;
197 }
198
199
200
201
202
203
204
205
206 public double getConvergenceThreshold() {
207 return convergenceThreshold;
208 }
209
210
211
212
213
214
215
216
217
218
219 public void setConvergenceThreshold(final double convergenceThreshold) throws LockedException {
220 if (isLocked()) {
221 throw new LockedException();
222 }
223 if (convergenceThreshold < MIN_CONVERGENCE_THRESHOLD) {
224 throw new IllegalArgumentException();
225 }
226
227 this.convergenceThreshold = convergenceThreshold;
228 }
229
230
231
232
233
234
235
236
237 public RobustEstimatorMethod getDistortionMethod() {
238 return distortionMethod;
239 }
240
241
242
243
244
245
246
247
248
249 public void setDistortionMethod(final RobustEstimatorMethod distortionMethod) throws LockedException {
250 if (isLocked()) {
251 throw new LockedException();
252 }
253 internalSetDistortionMethod(distortionMethod);
254 }
255
256
257
258
259
260
261
262
263
264 public RadialDistortionRobustEstimator getDistortionEstimator() {
265 return distortionEstimator;
266 }
267
268
269
270
271
272
273
274
275 public double getDistortionEstimatorThreshold() {
276 return switch (distortionEstimator.getMethod()) {
277 case LMEDS -> ((LMedSRadialDistortionRobustEstimator) distortionEstimator).getStopThreshold();
278 case MSAC -> ((MSACRadialDistortionRobustEstimator) distortionEstimator).getThreshold();
279 case PROSAC -> ((PROSACRadialDistortionRobustEstimator) distortionEstimator).getThreshold();
280 case PROMEDS -> ((PROMedSRadialDistortionRobustEstimator) distortionEstimator).getStopThreshold();
281 default -> ((RANSACRadialDistortionRobustEstimator) distortionEstimator).getThreshold();
282 };
283 }
284
285
286
287
288
289
290
291
292
293
294
295 public void setDistortionEstimatorThreshold(final double distortionEstimatorThreshold) throws LockedException {
296 if (isLocked()) {
297 throw new LockedException();
298 }
299
300 switch (distortionEstimator.getMethod()) {
301 case LMEDS:
302 ((LMedSRadialDistortionRobustEstimator) distortionEstimator)
303 .setStopThreshold(distortionEstimatorThreshold);
304 break;
305 case MSAC:
306 ((MSACRadialDistortionRobustEstimator) distortionEstimator).setThreshold(distortionEstimatorThreshold);
307 break;
308 case PROSAC:
309 ((PROSACRadialDistortionRobustEstimator) distortionEstimator)
310 .setThreshold(distortionEstimatorThreshold);
311 break;
312 case PROMEDS:
313 ((PROMedSRadialDistortionRobustEstimator) distortionEstimator)
314 .setStopThreshold(distortionEstimatorThreshold);
315 break;
316 case RANSAC:
317 default:
318 ((RANSACRadialDistortionRobustEstimator) distortionEstimator)
319 .setThreshold(distortionEstimatorThreshold);
320 break;
321 }
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339 public double getDistortionEstimatorConfidence() {
340 return distortionEstimator.getConfidence();
341 }
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362 public void setDistortionEstimatorConfidence(final double distortionEstimatorConfidence) throws LockedException {
363 if (isLocked()) {
364 throw new LockedException();
365 }
366
367 distortionEstimator.setConfidence(distortionEstimatorConfidence);
368 }
369
370
371
372
373
374
375
376
377
378
379
380
381 public int getDistortionEstimatorMaxIterations() {
382 return distortionEstimator.getMaxIterations();
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398 public void setDistortionEstimatorMaxIterations(final int distortionEstimatorMaxIterations) throws LockedException {
399 if (isLocked()) {
400 throw new LockedException();
401 }
402
403 distortionEstimator.setMaxIterations(distortionEstimatorMaxIterations);
404 }
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420 @Override
421 public void calibrate() throws CalibrationException, LockedException, NotReadyException {
422
423 if (isLocked()) {
424 throw new LockedException();
425 }
426 if (!isReady()) {
427 throw new NotReadyException();
428 }
429
430 locked = true;
431
432 homographyQualityScoresRequired = (distortionEstimator.getMethod() == RobustEstimatorMethod.PROSAC
433 || distortionEstimator.getMethod() == RobustEstimatorMethod.PROMEDS);
434
435 if (listener != null) {
436 listener.onCalibrateStart(this);
437 }
438
439 reset();
440 radialDistortionProgress = iterProgress = previousNotifiedProgress = 0.0f;
441
442 final var idealFallbackPatternMarkers = pattern.getIdealPoints();
443
444 try {
445 double errorDiff;
446 double previousError;
447 var currentError = Double.MAX_VALUE;
448 var bestError = Double.MAX_VALUE;
449 PinholeCameraIntrinsicParameters bestIntrinsic = null;
450 RadialDistortion bestDistortion = null;
451
452
453 var iter = 0;
454 do {
455 previousError = currentError;
456
457
458 estimateIntrinsicParameters(idealFallbackPatternMarkers);
459
460 if (!estimateRadialDistortion) {
461 break;
462 }
463
464
465
466
467 currentError = estimateRadialDistortion(idealFallbackPatternMarkers);
468
469 if (currentError < bestError) {
470 bestError = currentError;
471 bestIntrinsic = intrinsic;
472 bestDistortion = distortion;
473 }
474
475 errorDiff = Math.abs(previousError - currentError);
476 iter++;
477 iterProgress = (float) iter / (float) maxIterations;
478 notifyProgress();
479
480 } while (errorDiff > convergenceThreshold && iter < maxIterations);
481
482 if (bestIntrinsic != null) {
483 intrinsic = bestIntrinsic;
484 }
485 if (bestDistortion != null) {
486 distortion = bestDistortion;
487 }
488
489 if (listener != null) {
490 listener.onCalibrateEnd(this);
491 }
492 } finally {
493 locked = false;
494 }
495 }
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511 protected double estimateRadialDistortion(final List<Point2D> idealFallbackPatternMarkers)
512 throws CalibrationException {
513
514 radialDistortionProgress = 0.0f;
515
516 if (listener != null) {
517 listener.onRadialDistortionEstimationStarts(this);
518 }
519
520 final var distortedPoints = new ArrayList<Point2D>();
521 final var undistortedPoints = new ArrayList<Point2D>();
522
523 var totalPoints = 0;
524 for (final var sample : samples) {
525 if (sample.getHomography() != null) {
526 totalPoints += sample.getSampledMarkers().size();
527 }
528 }
529
530 double[] qualityScores = null;
531 if (distortionMethod == RobustEstimatorMethod.PROSAC || distortionMethod == RobustEstimatorMethod.PROMEDS) {
532 qualityScores = new double[totalPoints];
533 }
534
535
536 var pointCounter = 0;
537 var sampleCounter = 0;
538 for (final var sample : samples) {
539 if (sample.getHomography() == null) {
540
541
542
543 continue;
544 }
545 sample.computeCameraPose(intrinsic);
546
547
548 final List<Point2D> idealPatternMarkers;
549 if (sample.getPattern() != null) {
550
551 idealPatternMarkers = sample.getPattern().getIdealPoints();
552 } else {
553
554 idealPatternMarkers = idealFallbackPatternMarkers;
555 }
556
557 final var transformedIdealPatternMarkers = sample.getHomography()
558 .transformPointsAndReturnNew(idealPatternMarkers);
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574 distortedPoints.addAll(sample.getSampledMarkers());
575 undistortedPoints.addAll(transformedIdealPatternMarkers);
576
577 final var markersSize = transformedIdealPatternMarkers.size();
578
579
580 if (qualityScores != null && (distortionMethod == RobustEstimatorMethod.PROSAC
581 || distortionMethod == RobustEstimatorMethod.PROMEDS)) {
582
583 final var sampleQuality = homographyQualityScores[sampleCounter];
584
585
586
587 for (var i = pointCounter; i < pointCounter + markersSize; i++) {
588 qualityScores[i] = sampleQuality;
589 }
590
591 pointCounter += markersSize;
592 sampleCounter++;
593 }
594 }
595
596
597 var avgError = 0.0;
598 try {
599 distortionEstimator.setIntrinsic(intrinsic);
600 distortionEstimator.setPoints(distortedPoints, undistortedPoints);
601 distortionEstimator.setQualityScores(qualityScores);
602
603 final var distortion = distortionEstimator.estimate();
604
605
606
607 final var distortedPoints2 = distortion.distort(undistortedPoints);
608
609
610
611 for (final var sample : samples) {
612 if (sample.getHomography() == null) {
613 continue;
614 }
615
616
617
618
619 final var undistortedPoints2 = distortion.undistort(sample.getSampledMarkers());
620
621 sample.setUndistortedMarkers(undistortedPoints2);
622 }
623
624
625
626
627
628 Point2D distortedPoint1;
629 Point2D distortedPoint2;
630 totalPoints = distortedPoints.size();
631 var inlierCount = 0;
632 for (var i = 0; i < totalPoints; i++) {
633 distortedPoint1 = distortedPoints.get(i);
634 distortedPoint2 = distortedPoints2.get(i);
635
636 final var distance = distortedPoint1.distanceTo(distortedPoint2);
637 if (distance < getDistortionEstimatorThreshold()) {
638 avgError += distance;
639 inlierCount++;
640 }
641 }
642
643 if (inlierCount == 0) {
644 throw new CalibrationException();
645 }
646
647 avgError /= inlierCount;
648
649 this.distortion = distortion;
650
651 } catch (final GeometryException | NumericalException | DistortionException e) {
652 throw new CalibrationException(e);
653 }
654
655 if (listener != null) {
656 listener.onRadialDistortionEstimationEnds(this, distortion);
657 }
658
659 return avgError;
660 }
661
662
663
664
665
666
667 @Override
668 public CameraCalibratorMethod getMethod() {
669 return CameraCalibratorMethod.ALTERNATING_CALIBRATOR;
670 }
671
672
673
674
675 @Override
676 protected void notifyProgress() {
677 final var lambda = 1.0f / maxIterations;
678 final var partial = 0.5f * intrinsicProgress + 0.5f * radialDistortionProgress;
679
680 final float progress;
681 if (!estimateRadialDistortion) {
682
683
684 progress = partial;
685 } else {
686 progress = iterProgress + lambda * partial;
687 }
688
689 if (listener != null && (progress - previousNotifiedProgress) > progressDelta) {
690 listener.onCalibrateProgressChange(this, progress);
691 previousNotifiedProgress = progress;
692 }
693 }
694
695
696
697
698 protected void refreshDistortionEstimatorListener() {
699 if (distortionEstimatorListener == null) {
700 distortionEstimatorListener = new RadialDistortionRobustEstimatorListener() {
701
702 @Override
703 public void onEstimateStart(final RadialDistortionRobustEstimator estimator) {
704 radialDistortionProgress = 0.0f;
705 notifyProgress();
706 }
707
708 @Override
709 public void onEstimateEnd(final RadialDistortionRobustEstimator estimator) {
710 radialDistortionProgress = 1.0f;
711 notifyProgress();
712 }
713
714 @Override
715 public void onEstimateNextIteration(
716 final RadialDistortionRobustEstimator estimator, final int iteration) {
717
718 }
719
720 @Override
721 public void onEstimateProgressChange(
722 final RadialDistortionRobustEstimator estimator, final float progress) {
723 radialDistortionProgress = progress;
724 notifyProgress();
725 }
726 };
727 }
728
729 try {
730 distortionEstimator.setListener(distortionEstimatorListener);
731 } catch (final LockedException e) {
732 Logger.getLogger(AlternatingCameraCalibrator.class.getName()).log(Level.WARNING,
733 "Could not set radial distortion estimator listener", e);
734 }
735 }
736
737
738
739
740
741
742
743
744
745 private void internalSetDistortionMethod(final RobustEstimatorMethod distortionMethod) {
746
747 if (distortionMethod != this.distortionMethod) {
748 var previousAvailable = this.distortionMethod != null;
749 var threshold = 0.0;
750 var confidence = 0.0;
751 var maxIters = 0;
752 if (previousAvailable) {
753 threshold = getDistortionEstimatorThreshold();
754 confidence = getDistortionEstimatorConfidence();
755 maxIters = getDistortionEstimatorMaxIterations();
756 }
757
758 distortionEstimator = RadialDistortionRobustEstimator.create(distortionMethod);
759
760
761 refreshDistortionEstimatorListener();
762 if (previousAvailable) {
763 try {
764 setDistortionEstimatorThreshold(threshold);
765 setDistortionEstimatorConfidence(confidence);
766 setDistortionEstimatorMaxIterations(maxIters);
767 } catch (final LockedException e) {
768 Logger.getLogger(AlternatingCameraCalibrator.class.getName()).log(Level.WARNING,
769 "Could not reconfigure distortion estimator", e);
770 }
771 }
772 }
773
774 this.distortionMethod = distortionMethod;
775 }
776 }