Skip to content

Commit 2face9f

Browse files
ahojnnessarlinpe
ahojnnes
and
sarlinpe
authored
Camera models perform valid projection test (#3172)
Towards supporting spherical / large FOV cameras for which a different logic will have to be implemented than for perspective projection models. The logic is now specialized and localized within the camera model and the user just knows about valid or invalid projections. This can also be used to more robustly deal with numerical issues in distortion computation (as we have for some of the fisheye models). The next PRs will do the equivalent for CamFromImg to return camera rays rather than image points as well as remove redundant "HasPointPositiveDepth/Cheirality" checks. --------- Co-authored-by: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com>
1 parent 3b87429 commit 2face9f

22 files changed

Lines changed: 324 additions & 218 deletions

‎.github/workflows/build-ubuntu.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@ jobs:
108108
key: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.cmakeBuildType }}-${{ matrix.config.asanEnabled }}--${{ matrix.config.cudaEnabled }}-${{ github.run_id }}-${{ github.run_number }}
109109
restore-keys: v${{ env.COMPILER_CACHE_VERSION }}-${{ matrix.config.os }}-${{ matrix.config.cmakeBuildType }}-${{ matrix.config.asanEnabled }}--${{ matrix.config.cudaEnabled }}
110110
path: ${{ env.COMPILER_CACHE_DIR }}
111-
112111
- name: Install compiler cache
113112
run: |
114113
mkdir -p "$CCACHE_DIR" "$CTCACHE_DIR"
@@ -248,6 +247,8 @@ jobs:
248247
Xvfb :99 &
249248
sleep 3
250249
cd build
250+
export GLOG_v=1
251+
export GLOG_logtostderr=1
251252
ctest -E "$ctestExclusions" --output-on-failure
252253
253254
- name: Run E2E tests
@@ -295,7 +296,6 @@ jobs:
295296
indicators: true
296297
output: both
297298
thresholds: '75 90'
298-
299299
# TODO: Add code coverage comment to PR. Currently, this action reports
300300
# coverage for the entire repository, not just the changed files in the PR.
301301
# We could manually filter the coverage report to only include the changed

‎src/colmap/estimators/absolute_pose_test.cc‎

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,8 @@ TEST(AbsolutePose, P3P) {
6262

6363
const Camera camera = Camera::CreateFromModelId(
6464
kInvalidCameraId, CameraModelId::kPinhole, 12, 34, 56);
65-
66-
// TODO(jsch): Replace this with camera->ImgFromCam() once the camera class
67-
// returns an optional.
68-
auto img_from_cam_func = [&camera](const Eigen::Vector3d& point3D_in_cam)
69-
-> std::optional<Eigen::Vector2d> {
70-
if (point3D_in_cam.z() < std::numeric_limits<double>::epsilon()) {
71-
return std::nullopt;
72-
} else {
73-
return camera.ImgFromCam(point3D_in_cam.hnormalized());
74-
}
75-
};
65+
ImgFromCamFunc img_from_cam_func =
66+
std::bind(&Camera::ImgFromCam, &camera, std::placeholders::_1);
7667

7768
// NOLINTNEXTLINE(clang-analyzer-security.FloatLoopCounter)
7869
for (double qx = 0; qx < 1; qx += 0.2) {
@@ -200,17 +191,8 @@ TEST(AbsolutePose, EPNP) {
200191

201192
const Camera camera = Camera::CreateFromModelId(
202193
kInvalidCameraId, CameraModelId::kPinhole, 12, 34, 56);
203-
204-
// TODO(jsch): Replace this with camera->ImgFromCam() once the camera class
205-
// returns an optional.
206-
auto img_from_cam_func = [&camera](const Eigen::Vector3d& point3D_in_cam)
207-
-> std::optional<Eigen::Vector2d> {
208-
if (point3D_in_cam.z() < std::numeric_limits<double>::epsilon()) {
209-
return std::nullopt;
210-
} else {
211-
return camera.ImgFromCam(point3D_in_cam.hnormalized());
212-
}
213-
};
194+
auto img_from_cam_func =
195+
std::bind(&Camera::ImgFromCam, &camera, std::placeholders::_1);
214196

215197
// NOLINTNEXTLINE(clang-analyzer-security.FloatLoopCounter)
216198
for (double qx = 0; qx < 1; qx += 0.2) {
@@ -324,17 +306,8 @@ TEST(AbsolutePose, EPNP_BrokenSolveSignCase) {
324306
TEST(ComputeSquaredReprojectionError, Nominal) {
325307
const Camera camera = Camera::CreateFromModelId(
326308
kInvalidCameraId, CameraModelId::kPinhole, 12, 34, 56);
327-
328-
// TODO(jsch): Replace this with camera->ImgFromCam() once the camera class
329-
// returns an optional.
330-
auto img_from_cam_func = [&camera](const Eigen::Vector3d& point3D_in_cam)
331-
-> std::optional<Eigen::Vector2d> {
332-
if (point3D_in_cam.z() < std::numeric_limits<double>::epsilon()) {
333-
return std::nullopt;
334-
} else {
335-
return camera.ImgFromCam(point3D_in_cam.hnormalized());
336-
}
337-
};
309+
auto img_from_cam_func =
310+
std::bind(&Camera::ImgFromCam, &camera, std::placeholders::_1);
338311

339312
std::vector<Eigen::Vector3d> points3D;
340313
points3D.emplace_back(-1, 0, 1);

‎src/colmap/estimators/bundle_adjustment.cc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -701,12 +701,13 @@ class RigBundleAdjuster : public BundleAdjuster {
701701
CameraRig* camera_rig = nullptr;
702702
Eigen::Matrix3x4d cam_from_world_mat = Eigen::Matrix3x4d::Zero();
703703

704-
if (image_id_to_camera_rig_.count(image_id) > 0) {
704+
if (const auto it = image_id_to_camera_rig_.find(image_id);
705+
it != image_id_to_camera_rig_.end()) {
705706
THROW_CHECK(!constant_cam_pose)
706707
<< "Images contained in a camera rig must not have constant pose";
707708
THROW_CHECK(!constant_cam_position)
708709
<< "Images contained in a camera rig must not have constant tvec";
709-
camera_rig = image_id_to_camera_rig_.at(image_id);
710+
camera_rig = it->second;
710711
Rigid3d& rig_from_world = *image_id_to_rig_from_world_.at(image_id);
711712
rig_from_world_rotation = rig_from_world.rotation.coeffs().data();
712713
rig_from_world_translation = rig_from_world.translation.data();

‎src/colmap/estimators/bundle_adjustment_test.cc‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -162,25 +162,21 @@ void GenerateReconstruction(const size_t num_images,
162162
Eigen::Quaterniond::Identity(),
163163
Eigen::Vector3d(
164164
RandomUniformReal(-1.0, 1.0), RandomUniformReal(-1.0, 1.0), 10)));
165-
reconstruction->AddImage(image);
166-
167-
const Eigen::Matrix3x4d cam_from_world_matrix =
168-
image.CamFromWorld().ToMatrix();
169165

170166
std::vector<Eigen::Vector2d> points2D;
171167
for (const auto& point3D : reconstruction->Points3D()) {
172-
EXPECT_TRUE(
173-
HasPointPositiveDepth(cam_from_world_matrix, point3D.second.xyz));
174168
// Get exact projection of 3D point.
175-
Eigen::Vector2d point2D = camera.ImgFromCam(
176-
(image.CamFromWorld() * point3D.second.xyz).hnormalized());
169+
std::optional<Eigen::Vector2d> point2D =
170+
camera.ImgFromCam(image.CamFromWorld() * point3D.second.xyz);
171+
CHECK(point2D.has_value());
177172
// Add some uniform noise.
178-
point2D += Eigen::Vector2d(RandomUniformReal(-2.0, 2.0),
179-
RandomUniformReal(-2.0, 2.0));
180-
points2D.push_back(point2D);
173+
*point2D += Eigen::Vector2d(RandomUniformReal(-2.0, 2.0),
174+
RandomUniformReal(-2.0, 2.0));
175+
points2D.push_back(*point2D);
181176
}
182177

183-
reconstruction->Image(image_id).SetPoints2D(points2D);
178+
image.SetPoints2D(points2D);
179+
reconstruction->AddImage(std::move(image));
184180
}
185181

186182
for (size_t i = 0; i < num_images; ++i) {
@@ -658,7 +654,9 @@ TEST(RigBundleAdjuster, TwoView) {
658654
std::vector<CameraRig> camera_rigs;
659655
camera_rigs.emplace_back();
660656
camera_rigs[0].AddCamera(0, Rigid3d());
661-
camera_rigs[0].AddCamera(1, Rigid3d());
657+
camera_rigs[0].AddCamera(1,
658+
reconstruction.Image(1).CamFromWorld() *
659+
Inverse(reconstruction.Image(0).CamFromWorld()));
662660
camera_rigs[0].AddSnapshot({0, 1});
663661
camera_rigs[0].SetRefCameraId(0);
664662
const auto orig_camera_rigs = camera_rigs;
@@ -713,7 +711,9 @@ TEST(RigBundleAdjuster, FourView) {
713711
std::vector<CameraRig> camera_rigs;
714712
camera_rigs.emplace_back();
715713
camera_rigs[0].AddCamera(0, Rigid3d());
716-
camera_rigs[0].AddCamera(1, Rigid3d());
714+
camera_rigs[0].AddCamera(1,
715+
reconstruction.Image(1).CamFromWorld() *
716+
Inverse(reconstruction.Image(0).CamFromWorld()));
717717
camera_rigs[0].AddSnapshot({0, 1});
718718
camera_rigs[0].AddSnapshot({2, 3});
719719
camera_rigs[0].SetRefCameraId(0);
@@ -769,7 +769,9 @@ TEST(RigBundleAdjuster, ConstantFourView) {
769769
std::vector<CameraRig> camera_rigs;
770770
camera_rigs.emplace_back();
771771
camera_rigs[0].AddCamera(0, Rigid3d());
772-
camera_rigs[0].AddCamera(1, Rigid3d());
772+
camera_rigs[0].AddCamera(1,
773+
reconstruction.Image(1).CamFromWorld() *
774+
Inverse(reconstruction.Image(0).CamFromWorld()));
773775
camera_rigs[0].AddSnapshot({0, 1});
774776
camera_rigs[0].AddSnapshot({2, 3});
775777
camera_rigs[0].SetRefCameraId(0);
@@ -825,7 +827,9 @@ TEST(RigBundleAdjuster, FourViewPartial) {
825827
std::vector<CameraRig> camera_rigs;
826828
camera_rigs.emplace_back();
827829
camera_rigs[0].AddCamera(0, Rigid3d());
828-
camera_rigs[0].AddCamera(1, Rigid3d());
830+
camera_rigs[0].AddCamera(1,
831+
reconstruction.Image(1).CamFromWorld() *
832+
Inverse(reconstruction.Image(0).CamFromWorld()));
829833
camera_rigs[0].AddSnapshot({0, 1});
830834
camera_rigs[0].AddSnapshot({2});
831835
camera_rigs[0].SetRefCameraId(0);

‎src/colmap/estimators/cost_functions.h‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -97,14 +97,18 @@ class ReprojErrorCostFunctor
9797
EigenQuaternionMap<T>(cam_from_world_rotation) *
9898
EigenVector3Map<T>(point3D) +
9999
EigenVector3Map<T>(cam_from_world_translation);
100-
CameraModel::ImgFromCam(camera_params,
101-
point3D_in_cam[0],
102-
point3D_in_cam[1],
103-
point3D_in_cam[2],
104-
&residuals[0],
105-
&residuals[1]);
106-
residuals[0] -= T(observed_x_);
107-
residuals[1] -= T(observed_y_);
100+
if (CameraModel::ImgFromCam(camera_params,
101+
point3D_in_cam[0],
102+
point3D_in_cam[1],
103+
point3D_in_cam[2],
104+
&residuals[0],
105+
&residuals[1])) {
106+
residuals[0] -= T(observed_x_);
107+
residuals[1] -= T(observed_y_);
108+
} else {
109+
residuals[0] = T(0);
110+
residuals[1] = T(0);
111+
}
108112
return true;
109113
}
110114

@@ -214,14 +218,18 @@ class RigReprojErrorCostFunctor
214218
EigenVector3Map<T>(point3D) +
215219
EigenVector3Map<T>(rig_from_world_translation)) +
216220
EigenVector3Map<T>(cam_from_rig_translation);
217-
CameraModel::ImgFromCam(camera_params,
218-
point3D_in_cam[0],
219-
point3D_in_cam[1],
220-
point3D_in_cam[2],
221-
&residuals[0],
222-
&residuals[1]);
223-
residuals[0] -= T(observed_x_);
224-
residuals[1] -= T(observed_y_);
221+
if (CameraModel::ImgFromCam(camera_params,
222+
point3D_in_cam[0],
223+
point3D_in_cam[1],
224+
point3D_in_cam[2],
225+
&residuals[0],
226+
&residuals[1])) {
227+
residuals[0] -= T(observed_x_);
228+
residuals[1] -= T(observed_y_);
229+
} else {
230+
residuals[0] = T(0);
231+
residuals[1] = T(0);
232+
}
225233
return true;
226234
}
227235

‎src/colmap/estimators/pose.cc‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,17 +80,8 @@ bool EstimateAbsolutePose(const AbsolutePoseEstimationOptions& options,
8080
camera->CamFromImg(points2D[i]).homogeneous().normalized();
8181
}
8282

83-
// TODO(jsch): Replace this with camera->ImgFromCam() once the camera class
84-
// returns an optional.
85-
auto img_from_cam_func = [&camera](const Eigen::Vector3d& point3D_in_cam)
86-
-> std::optional<Eigen::Vector2d> {
87-
if (point3D_in_cam.z() < std::numeric_limits<double>::epsilon()) {
88-
return std::nullopt;
89-
} else {
90-
return camera->ImgFromCam(point3D_in_cam.hnormalized());
91-
}
92-
};
93-
83+
ImgFromCamFunc img_from_cam_func =
84+
std::bind(&Camera::ImgFromCam, camera, std::placeholders::_1);
9485
LORANSAC<P3PEstimator, EPNPEstimator> ransac(
9586
options.ransac_options,
9687
P3PEstimator(img_from_cam_func),

‎src/colmap/image/undistortion.cc‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -853,17 +853,21 @@ Camera UndistortCamera(const UndistortCameraOptions& options,
853853
// Left border.
854854
const Eigen::Vector2d point1_in_cam =
855855
camera.CamFromImg(Eigen::Vector2d(0.5, y + 0.5));
856-
const Eigen::Vector2d undistorted_point1 =
857-
undistorted_camera.ImgFromCam(point1_in_cam);
858-
left_min_x = std::min(left_min_x, undistorted_point1(0));
859-
left_max_x = std::max(left_max_x, undistorted_point1(0));
856+
const std::optional<Eigen::Vector2d> undistorted_point1 =
857+
undistorted_camera.ImgFromCam(point1_in_cam.homogeneous());
858+
if (undistorted_point1) {
859+
left_min_x = std::min(left_min_x, undistorted_point1->x());
860+
left_max_x = std::max(left_max_x, undistorted_point1->x());
861+
}
860862
// Right border.
861863
const Eigen::Vector2d point2_in_cam =
862864
camera.CamFromImg(Eigen::Vector2d(camera.width - 0.5, y + 0.5));
863-
const Eigen::Vector2d undistorted_point2 =
864-
undistorted_camera.ImgFromCam(point2_in_cam);
865-
right_min_x = std::min(right_min_x, undistorted_point2(0));
866-
right_max_x = std::max(right_max_x, undistorted_point2(0));
865+
const std::optional<Eigen::Vector2d> undistorted_point2 =
866+
undistorted_camera.ImgFromCam(point2_in_cam.homogeneous());
867+
if (undistorted_point2) {
868+
right_min_x = std::min(right_min_x, undistorted_point2->x());
869+
right_max_x = std::max(right_max_x, undistorted_point2->x());
870+
}
867871
}
868872

869873
// Determine min, max coordinates along left / right image border.
@@ -877,17 +881,21 @@ Camera UndistortCamera(const UndistortCameraOptions& options,
877881
// Top border.
878882
const Eigen::Vector2d point1_in_cam =
879883
camera.CamFromImg(Eigen::Vector2d(x + 0.5, 0.5));
880-
const Eigen::Vector2d undistorted_point1 =
881-
undistorted_camera.ImgFromCam(point1_in_cam);
882-
top_min_y = std::min(top_min_y, undistorted_point1(1));
883-
top_max_y = std::max(top_max_y, undistorted_point1(1));
884+
const std::optional<Eigen::Vector2d> undistorted_point1 =
885+
undistorted_camera.ImgFromCam(point1_in_cam.homogeneous());
886+
if (undistorted_point1) {
887+
top_min_y = std::min(top_min_y, undistorted_point1->y());
888+
top_max_y = std::max(top_max_y, undistorted_point1->y());
889+
}
884890
// Bottom border.
885891
const Eigen::Vector2d point2_in_cam =
886892
camera.CamFromImg(Eigen::Vector2d(x + 0.5, camera.height - 0.5));
887-
const Eigen::Vector2d undistorted_point2 =
888-
undistorted_camera.ImgFromCam(point2_in_cam);
889-
bottom_min_y = std::min(bottom_min_y, undistorted_point2(1));
890-
bottom_max_y = std::max(bottom_max_y, undistorted_point2(1));
893+
const std::optional<Eigen::Vector2d> undistorted_point2 =
894+
undistorted_camera.ImgFromCam(point2_in_cam.homogeneous());
895+
if (undistorted_point2) {
896+
bottom_min_y = std::min(bottom_min_y, undistorted_point2->y());
897+
bottom_max_y = std::max(bottom_max_y, undistorted_point2->y());
898+
}
891899
}
892900

893901
const double cx = undistorted_camera.PrincipalPointX();
@@ -990,8 +998,15 @@ void UndistortReconstruction(const UndistortCameraOptions& options,
990998
for (point2D_t point2D_idx = 0; point2D_idx < image.NumPoints2D();
991999
++point2D_idx) {
9921000
auto& point2D = image.Point2D(point2D_idx);
993-
point2D.xy = undistorted_camera.ImgFromCam(
994-
distorted_camera.CamFromImg(point2D.xy));
1001+
const std::optional<Eigen::Vector2d> undistorted_point =
1002+
undistorted_camera.ImgFromCam(
1003+
distorted_camera.CamFromImg(point2D.xy).homogeneous());
1004+
if (undistorted_point) {
1005+
point2D.xy = *undistorted_point;
1006+
} else {
1007+
point2D.xy =
1008+
Eigen::Vector2d::Constant(std::numeric_limits<double>::quiet_NaN());
1009+
}
9951010
}
9961011
}
9971012
}

‎src/colmap/image/warp.cc‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,13 @@ void WarpImageBetweenCameras(const Camera& source_camera,
8282
// Camera models assume that the upper left pixel center is (0.5, 0.5).
8383
const Eigen::Vector2d cam_point =
8484
scaled_target_camera.CamFromImg(image_point);
85-
const Eigen::Vector2d source_point = source_camera.ImgFromCam(cam_point);
85+
const std::optional<Eigen::Vector2d> source_point =
86+
source_camera.ImgFromCam(cam_point.homogeneous());
8687

8788
BitmapColor<float> color;
88-
if (source_image.InterpolateBilinear(
89-
source_point.x() - 0.5, source_point.y() - 0.5, &color)) {
89+
if (source_point &&
90+
source_image.InterpolateBilinear(
91+
source_point->x() - 0.5, source_point->y() - 0.5, &color)) {
9092
target_image->SetPixel(x, y, color.Cast<uint8_t>());
9193
} else {
9294
target_image->SetPixel(x, y, BitmapColor<uint8_t>(0));
@@ -158,11 +160,13 @@ void WarpImageWithHomographyBetweenCameras(const Eigen::Matrix3d& H,
158160
const Eigen::Vector3d warped_point = H * image_point;
159161
const Eigen::Vector2d cam_point =
160162
target_camera.CamFromImg(warped_point.hnormalized());
161-
const Eigen::Vector2d source_point = source_camera.ImgFromCam(cam_point);
163+
const std::optional<Eigen::Vector2d> source_point =
164+
source_camera.ImgFromCam(cam_point.homogeneous());
162165

163166
BitmapColor<float> color;
164-
if (source_image.InterpolateBilinear(
165-
source_point.x() - 0.5, source_point.y() - 0.5, &color)) {
167+
if (source_point &&
168+
source_image.InterpolateBilinear(
169+
source_point->x() - 0.5, source_point->y() - 0.5, &color)) {
166170
target_image->SetPixel(x, y, color.Cast<uint8_t>());
167171
} else {
168172
target_image->SetPixel(x, y, BitmapColor<uint8_t>(0));

‎src/colmap/mvs/meshing.cc‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -395,14 +395,15 @@ class DelaunayMeshingInput {
395395
}
396396

397397
// Check reprojection error between the two points.
398-
const Eigen::Vector2f point_proj =
399-
camera.ImgFromCam(point_local.hnormalized().cast<double>())
400-
.cast<float>();
401-
const Eigen::Vector2f cell_point_proj =
402-
camera.ImgFromCam(cell_point_local.hnormalized().cast<double>())
403-
.cast<float>();
398+
const std::optional<Eigen::Vector2d> point_proj =
399+
camera.ImgFromCam(point_local.cast<double>());
400+
const std::optional<Eigen::Vector2d> cell_point_proj =
401+
camera.ImgFromCam(cell_point_local.cast<double>());
402+
if (!point_proj || !cell_point_proj) {
403+
continue;
404+
}
404405
const float squared_proj_dist =
405-
(point_proj - cell_point_proj).squaredNorm();
406+
(*point_proj - *cell_point_proj).squaredNorm();
406407
if (squared_proj_dist > max_squared_proj_dist) {
407408
insert_point = true;
408409
break;

0 commit comments

Comments
 (0)