1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package com.irurueta.ar.sfm;
17
18 import com.irurueta.algebra.Matrix;
19 import com.irurueta.ar.epipolar.Corrector;
20 import com.irurueta.ar.epipolar.CorrectorType;
21 import com.irurueta.ar.epipolar.FundamentalMatrix;
22 import com.irurueta.ar.epipolar.estimators.FundamentalMatrixEstimatorException;
23 import com.irurueta.ar.epipolar.estimators.PlanarFundamentalMatrixEstimator;
24 import com.irurueta.geometry.PinholeCamera;
25 import com.irurueta.geometry.PinholeCameraIntrinsicParameters;
26 import com.irurueta.geometry.Point2D;
27 import com.irurueta.geometry.Point3D;
28 import com.irurueta.geometry.ProjectiveTransformation2D;
29 import com.irurueta.geometry.estimators.LockedException;
30 import com.irurueta.geometry.estimators.NotReadyException;
31 import com.irurueta.geometry.estimators.PROMedSPointCorrespondenceProjectiveTransformation2DRobustEstimator;
32 import com.irurueta.geometry.estimators.PROSACPointCorrespondenceProjectiveTransformation2DRobustEstimator;
33 import com.irurueta.geometry.estimators.PointCorrespondenceProjectiveTransformation2DRobustEstimator;
34 import com.irurueta.geometry.estimators.RANSACPointCorrespondenceProjectiveTransformation2DRobustEstimator;
35 import com.irurueta.numerical.robust.RobustEstimatorException;
36 import com.irurueta.numerical.robust.RobustEstimatorMethod;
37
38 import java.util.ArrayList;
39 import java.util.Arrays;
40 import java.util.BitSet;
41 import java.util.List;
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60 public class PlanarBestFundamentalMatrixEstimatorAndReconstructor {
61
62
63
64
65 public static final int MINIMUM_SIZE = 4;
66
67
68
69
70 private List<Point2D> leftPoints;
71
72
73
74
75 private List<Point2D> rightPoints;
76
77
78
79
80 private PinholeCameraIntrinsicParameters leftIntrinsics;
81
82
83
84
85 private PinholeCameraIntrinsicParameters rightIntrinsics;
86
87
88
89
90 private PlanarBestFundamentalMatrixEstimatorAndReconstructorListener listener;
91
92
93
94
95 private boolean locked;
96
97
98
99
100 private PointCorrespondenceProjectiveTransformation2DRobustEstimator homographyEstimator;
101
102
103
104
105
106 private CorrectorType essentialCameraEstimatorCorrectorType = Corrector.DEFAULT_TYPE;
107
108
109
110
111 private ProjectiveTransformation2D homography;
112
113
114
115
116 private FundamentalMatrix fundamentalMatrix;
117
118
119
120
121 private List<Point3D> triangulatedPoints;
122
123
124
125
126
127 private BitSet validTriangulatedPoints;
128
129
130
131
132 private PinholeCamera estimatedLeftCamera;
133
134
135
136
137 private PinholeCamera estimatedRightCamera;
138
139
140
141
142 public PlanarBestFundamentalMatrixEstimatorAndReconstructor() {
143 homographyEstimator = PointCorrespondenceProjectiveTransformation2DRobustEstimator.create();
144 }
145
146
147
148
149
150
151
152
153
154
155
156 public PlanarBestFundamentalMatrixEstimatorAndReconstructor(
157 final List<Point2D> leftPoints, final List<Point2D> rightPoints,
158 final PinholeCameraIntrinsicParameters leftIntrinsics,
159 final PinholeCameraIntrinsicParameters rightIntrinsics) {
160 this();
161 internalSetLeftAndRightPoints(leftPoints, rightPoints);
162 this.leftIntrinsics = leftIntrinsics;
163 this.rightIntrinsics = rightIntrinsics;
164 }
165
166
167
168
169
170
171
172
173
174
175
176
177 public PlanarBestFundamentalMatrixEstimatorAndReconstructor(
178 final List<Point2D> leftPoints, final List<Point2D> rightPoints,
179 final PinholeCameraIntrinsicParameters leftIntrinsics,
180 final PinholeCameraIntrinsicParameters rightIntrinsics,
181 final PlanarBestFundamentalMatrixEstimatorAndReconstructorListener listener) {
182 this(leftPoints, rightPoints, leftIntrinsics, rightIntrinsics);
183 this.listener = listener;
184 }
185
186
187
188
189
190
191 public List<Point2D> getLeftPoints() {
192 return leftPoints;
193 }
194
195
196
197
198
199
200
201
202
203 public void setLeftPoints(final List<Point2D> leftPoints) throws LockedException {
204 if (isLocked()) {
205 throw new LockedException();
206 }
207 if (leftPoints.size() < MINIMUM_SIZE) {
208 throw new IllegalArgumentException();
209 }
210
211 this.leftPoints = leftPoints;
212 }
213
214
215
216
217
218
219 public List<Point2D> getRightPoints() {
220 return rightPoints;
221 }
222
223
224
225
226
227
228
229
230
231 public void setRightPoints(final List<Point2D> rightPoints) throws LockedException {
232 if (isLocked()) {
233 throw new LockedException();
234 }
235 if (rightPoints.size() < MINIMUM_SIZE) {
236 throw new IllegalArgumentException();
237 }
238
239 this.rightPoints = rightPoints;
240 }
241
242
243
244
245
246
247
248
249
250
251 public void setLeftAndRightPoints(final List<Point2D> leftPoints, final List<Point2D> rightPoints)
252 throws LockedException {
253 if (isLocked()) {
254 throw new LockedException();
255 }
256
257 internalSetLeftAndRightPoints(leftPoints, rightPoints);
258 }
259
260
261
262
263
264
265 public PinholeCameraIntrinsicParameters getLeftIntrinsics() {
266 return leftIntrinsics;
267 }
268
269
270
271
272
273
274
275
276 public void setLeftIntrinsics(final PinholeCameraIntrinsicParameters leftIntrinsics) throws LockedException {
277 if (isLocked()) {
278 throw new LockedException();
279 }
280
281 this.leftIntrinsics = leftIntrinsics;
282 }
283
284
285
286
287
288
289 public PinholeCameraIntrinsicParameters getRightIntrinsics() {
290 return rightIntrinsics;
291 }
292
293
294
295
296
297
298
299
300 public void setRightIntrinsics(final PinholeCameraIntrinsicParameters rightIntrinsics) throws LockedException {
301 if (isLocked()) {
302 throw new LockedException();
303 }
304
305 this.rightIntrinsics = rightIntrinsics;
306 }
307
308
309
310
311
312
313 public PointCorrespondenceProjectiveTransformation2DRobustEstimator getHomographyEstimator() {
314 return homographyEstimator;
315 }
316
317
318
319
320
321
322
323
324 public void setHomographyEstimator(
325 final PointCorrespondenceProjectiveTransformation2DRobustEstimator homographyEstimator)
326 throws LockedException {
327 if (isLocked()) {
328 throw new LockedException();
329 }
330 if (homographyEstimator == null) {
331 throw new NullPointerException();
332 }
333
334 this.homographyEstimator = homographyEstimator;
335 }
336
337
338
339
340
341
342
343 public CorrectorType getEssentialCameraEstimatorCorrectorType() {
344 return essentialCameraEstimatorCorrectorType;
345 }
346
347
348
349
350
351
352
353
354 public void setEssentialCameraEstimatorCorrectorType(final CorrectorType correctorType) throws LockedException {
355 if (isLocked()) {
356 throw new LockedException();
357 }
358
359 essentialCameraEstimatorCorrectorType = correctorType;
360 }
361
362
363
364
365
366
367
368
369
370 public double getHomographyConfidence() {
371 return homographyEstimator.getConfidence();
372 }
373
374
375
376
377
378
379
380
381
382
383
384
385 public void setHomographyConfidence(final double confidence) throws LockedException {
386 if (isLocked()) {
387 throw new LockedException();
388 }
389
390 homographyEstimator.setConfidence(confidence);
391 }
392
393
394
395
396
397
398 public int getHomographyMaxIterations() {
399 return homographyEstimator.getMaxIterations();
400 }
401
402
403
404
405
406
407
408
409 public void setHomographyMaxIterations(final int maxIterations) throws LockedException {
410 if (isLocked()) {
411 throw new LockedException();
412 }
413
414 homographyEstimator.setMaxIterations(maxIterations);
415 }
416
417
418
419
420
421
422
423 public boolean isHomographyRefined() {
424 return homographyEstimator.isResultRefined();
425 }
426
427
428
429
430
431
432
433
434 public void setHomographyRefined(final boolean refineResult) throws LockedException {
435 if (isLocked()) {
436 throw new LockedException();
437 }
438
439 homographyEstimator.setResultRefined(refineResult);
440 }
441
442
443
444
445
446
447
448
449 public boolean isHomographyCovarianceKept() {
450 return homographyEstimator.isCovarianceKept();
451 }
452
453
454
455
456
457
458
459
460
461 public void setHomographyCovarianceKept(final boolean keepCovariance) throws LockedException {
462 if (isLocked()) {
463 throw new LockedException();
464 }
465
466 homographyEstimator.setCovarianceKept(keepCovariance);
467 }
468
469
470
471
472
473
474
475
476 public Matrix getHomographyCovariance() {
477 return homographyEstimator.getCovariance();
478 }
479
480
481
482
483
484
485 public RobustEstimatorMethod getHomographyMethod() {
486 return homographyEstimator.getMethod();
487 }
488
489
490
491
492
493
494
495
496 public double[] getQualityScores() {
497 return homographyEstimator.getQualityScores();
498 }
499
500
501
502
503
504
505
506
507
508
509
510
511 public void setQualityScores(final double[] qualityScore) throws LockedException {
512 if (isLocked()) {
513 throw new LockedException();
514 }
515
516 homographyEstimator.setQualityScores(qualityScore);
517 }
518
519
520
521
522
523
524 public PlanarBestFundamentalMatrixEstimatorAndReconstructorListener getListener() {
525 return listener;
526 }
527
528
529
530
531
532
533 public void setListener(final PlanarBestFundamentalMatrixEstimatorAndReconstructorListener listener) {
534 this.listener = listener;
535 }
536
537
538
539
540
541
542 public boolean isLocked() {
543 return locked;
544 }
545
546
547
548
549
550
551
552 public boolean isReady() {
553 return leftPoints != null && leftPoints.size() >= MINIMUM_SIZE && rightPoints != null
554 && rightPoints.size() >= MINIMUM_SIZE && leftPoints.size() == rightPoints.size()
555 && leftIntrinsics != null && rightIntrinsics != null && homographyEstimator.isReady();
556 }
557
558
559
560
561
562
563 public ProjectiveTransformation2D getHomography() {
564 return homography;
565 }
566
567
568
569
570
571
572 public FundamentalMatrix getFundamentalMatrix() {
573 return fundamentalMatrix;
574 }
575
576
577
578
579
580
581 public List<Point3D> getTriangulatedPoints() {
582 return triangulatedPoints;
583 }
584
585
586
587
588
589
590
591 public BitSet getValidTriangulatedPoints() {
592 return validTriangulatedPoints;
593 }
594
595
596
597
598
599
600 public PinholeCamera getEstimatedLeftCamera() {
601 return estimatedLeftCamera;
602 }
603
604
605
606
607
608
609 public PinholeCamera getEstimatedRightCamera() {
610 return estimatedRightCamera;
611 }
612
613
614
615
616
617
618
619
620
621
622
623 public void estimateAndReconstruct() throws LockedException, NotReadyException,
624 FundamentalMatrixEstimatorException {
625 if (isLocked()) {
626 throw new LockedException();
627 }
628 if (!isReady()) {
629 throw new NotReadyException();
630 }
631
632
633 enableHomographyInliersEstimation();
634
635 try {
636 locked = true;
637
638 if (listener != null) {
639 listener.onEstimateStart(this);
640 }
641
642
643 homography = homographyEstimator.estimate();
644
645 final var homographyInliers = homographyEstimator.getInliersData();
646
647
648 final var fundamentalMatrixEstimator = new PlanarFundamentalMatrixEstimator(homography, leftIntrinsics,
649 rightIntrinsics);
650
651 final var fundamentalMatrices = fundamentalMatrixEstimator.estimate();
652
653 if (fundamentalMatrices == null) {
654 throw new FundamentalMatrixEstimatorException();
655 }
656
657
658 final var lPoints = new ArrayList<Point2D>();
659 final var rPoints = new ArrayList<Point2D>();
660 final var bitset = homographyInliers.getInliers();
661 final var bitsetLength = bitset.length();
662 for (var i = 0; i < bitsetLength; i++) {
663 if (bitset.get(i)) {
664
665 lPoints.add(this.leftPoints.get(i));
666 rPoints.add(this.rightPoints.get(i));
667 }
668 }
669
670
671 final var essentialCamerasEstimator = new EssentialMatrixInitialCamerasEstimator(leftIntrinsics,
672 rightIntrinsics, lPoints, rPoints);
673 essentialCamerasEstimator.setCorrectorType(essentialCameraEstimatorCorrectorType);
674 essentialCamerasEstimator.setPointsTriangulated(true);
675 essentialCamerasEstimator.setValidTriangulatedPointsMarked(true);
676
677 var numBest = 0;
678 for (final var fMatrix : fundamentalMatrices) {
679 essentialCamerasEstimator.setFundamentalMatrix(fMatrix);
680
681 essentialCamerasEstimator.estimate();
682
683 final var validPoints = essentialCamerasEstimator.getValidTriangulatedPoints();
684 final var num = validPoints.cardinality();
685 if (num > numBest) {
686 numBest = num;
687 this.fundamentalMatrix = fMatrix;
688 triangulatedPoints = essentialCamerasEstimator.getTriangulatedPoints();
689 validTriangulatedPoints = validPoints;
690 estimatedLeftCamera = essentialCamerasEstimator.getEstimatedLeftCamera();
691 estimatedRightCamera = essentialCamerasEstimator.getEstimatedRightCamera();
692 }
693 }
694
695 if (listener != null) {
696 listener.onEstimateEnd(this);
697 }
698
699 } catch (final RobustEstimatorException | InitialCamerasEstimationFailedException e) {
700 throw new FundamentalMatrixEstimatorException(e);
701 } finally {
702 locked = false;
703 }
704 }
705
706
707
708
709
710
711
712
713
714
715 private void internalSetLeftAndRightPoints(final List<Point2D> leftPoints, final List<Point2D> rightPoints) {
716 if (leftPoints.size() < MINIMUM_SIZE || rightPoints.size() < MINIMUM_SIZE
717 || leftPoints.size() != rightPoints.size()) {
718 throw new IllegalArgumentException();
719 }
720
721 this.leftPoints = leftPoints;
722 this.rightPoints = rightPoints;
723 try {
724 homographyEstimator.setPoints(leftPoints, rightPoints);
725
726 if (homographyEstimator.getMethod() == RobustEstimatorMethod.PROMEDS) {
727 final var promedsEstimator =
728 (PROMedSPointCorrespondenceProjectiveTransformation2DRobustEstimator) homographyEstimator;
729 if (promedsEstimator.getQualityScores() == null) {
730 final var qualityScores = new double[leftPoints.size()];
731 Arrays.fill(qualityScores, 1.0);
732 promedsEstimator.setQualityScores(qualityScores);
733 }
734 } else if (homographyEstimator.getMethod() == RobustEstimatorMethod.PROSAC) {
735 final var prosacEstimator =
736 (PROSACPointCorrespondenceProjectiveTransformation2DRobustEstimator) homographyEstimator;
737 if (prosacEstimator.getQualityScores() == null) {
738 final var qualityScores = new double[leftPoints.size()];
739 Arrays.fill(qualityScores, 1.0);
740 prosacEstimator.setQualityScores(qualityScores);
741 }
742 }
743 } catch (final LockedException ignore) {
744
745 }
746 }
747
748
749
750
751 private void enableHomographyInliersEstimation() {
752 try {
753 if (homographyEstimator.getMethod() == RobustEstimatorMethod.RANSAC) {
754 final var ransacHomographyEstimator =
755 (RANSACPointCorrespondenceProjectiveTransformation2DRobustEstimator) homographyEstimator;
756 ransacHomographyEstimator.setComputeAndKeepInliersEnabled(true);
757 ransacHomographyEstimator.setComputeAndKeepResidualsEnabled(true);
758 } else if (homographyEstimator.getMethod() == RobustEstimatorMethod.PROSAC) {
759 final var prosacHomographyEstimator =
760 (PROSACPointCorrespondenceProjectiveTransformation2DRobustEstimator) homographyEstimator;
761 prosacHomographyEstimator.setComputeAndKeepInliersEnabled(true);
762 prosacHomographyEstimator.setComputeAndKeepResidualsEnabled(true);
763 }
764 } catch (final LockedException ignore) {
765
766 }
767 }
768 }