10 #include "vision-precomp.h" 11 #include <mrpt/config.h> 15 #include <mrpt/config.h> 16 #if MRPT_HAS_OPENCV && MRPT_OPENCV_VERSION_NUM<0x240 17 # undef MRPT_HAS_OPENCV 18 # define MRPT_HAS_OPENCV 0 27 mrpt::vision::pnp::epnp::epnp(
const cv::Mat& cameraMatrix,
const cv::Mat& opoints,
const cv::Mat& ipoints)
29 if (cameraMatrix.depth() == CV_32F)
30 init_camera_parameters<float>(cameraMatrix);
32 init_camera_parameters<double>(cameraMatrix);
34 number_of_correspondences =
std::max(opoints.checkVector(3, CV_32F), opoints.checkVector(3, CV_64F));
36 pws.resize(3 * number_of_correspondences);
37 us.resize(2 * number_of_correspondences);
39 if (opoints.depth() == ipoints.depth())
41 if (opoints.depth() == CV_32F)
42 init_points<cv::Point3f,cv::Point2f>(opoints, ipoints);
44 init_points<cv::Point3d,cv::Point2d>(opoints, ipoints);
46 else if (opoints.depth() == CV_32F)
47 init_points<cv::Point3f,cv::Point2d>(opoints, ipoints);
49 init_points<cv::Point3d,cv::Point2f>(opoints, ipoints);
51 alphas.resize(4 * number_of_correspondences);
52 pcs.resize(3 * number_of_correspondences);
59 mrpt::vision::pnp::epnp::~epnp()
67 void mrpt::vision::pnp::epnp::choose_control_points(
void)
70 cws[0][0] = cws[0][1] = cws[0][2] = 0;
71 for(
int i = 0; i < number_of_correspondences; i++)
72 for(
int j = 0; j < 3; j++)
73 cws[0][j] += pws[3 * i + j];
75 for(
int j = 0; j < 3; j++)
76 cws[0][j] /= number_of_correspondences;
80 CvMat * PW0 = cvCreateMat(number_of_correspondences, 3, CV_64F);
82 double pw0tpw0[3 * 3], dc[3], uct[3 * 3];
83 CvMat PW0tPW0 = cvMat(3, 3, CV_64F, pw0tpw0);
84 CvMat DC = cvMat(3, 1, CV_64F, dc);
85 CvMat UCt = cvMat(3, 3, CV_64F, uct);
87 for(
int i = 0; i < number_of_correspondences; i++)
88 for(
int j = 0; j < 3; j++)
89 PW0->data.db[3 * i + j] = pws[3 * i + j] - cws[0][j];
91 cvMulTransposed(PW0, &PW0tPW0, 1);
92 cvSVD(&PW0tPW0, &DC, &UCt, 0, CV_SVD_MODIFY_A | CV_SVD_U_T);
96 for(
int i = 1; i < 4; i++) {
97 double k = sqrt(dc[i - 1] / number_of_correspondences);
98 for(
int j = 0; j < 3; j++)
99 cws[i][j] = cws[0][j] + k * uct[3 * (i - 1) + j];
103 void mrpt::vision::pnp::epnp::compute_barycentric_coordinates(
void)
105 double cc[3 * 3], cc_inv[3 * 3];
106 CvMat CC = cvMat(3, 3, CV_64F, cc);
107 CvMat CC_inv = cvMat(3, 3, CV_64F, cc_inv);
109 for(
int i = 0; i < 3; i++)
110 for(
int j = 1; j < 4; j++)
111 cc[3 * i + j - 1] = cws[j][i] - cws[0][i];
113 cvInvert(&CC, &CC_inv, CV_SVD);
114 double * ci = cc_inv;
115 for(
int i = 0; i < number_of_correspondences; i++) {
116 double *
pi = &pws[0] + 3 * i;
117 double * a = &alphas[0] + 4 * i;
119 for(
int j = 0; j < 3; j++)
121 ci[3 * j ] * (pi[0] - cws[0][0]) +
122 ci[3 * j + 1] * (pi[1] - cws[0][1]) +
123 ci[3 * j + 2] * (pi[2] - cws[0][2]);
124 a[0] = 1.0f - a[1] - a[2] - a[3];
128 void mrpt::vision::pnp::epnp::fill_M(CvMat * M,
129 const int row,
const double * as,
const double u,
const double v)
131 double * M1 = M->data.db + row * 12;
132 double * M2 = M1 + 12;
134 for(
int i = 0; i < 4; i++) {
135 M1[3 * i ] = as[i] * fu;
137 M1[3 * i + 2] = as[i] * (uc - u);
140 M2[3 * i + 1] = as[i] * fv;
141 M2[3 * i + 2] = as[i] * (vc - v);
145 void mrpt::vision::pnp::epnp::compute_ccs(
const double * betas,
const double * ut)
147 for(
int i = 0; i < 4; i++)
148 ccs[i][0] = ccs[i][1] = ccs[i][2] = 0.0f;
150 for(
int i = 0; i < 4; i++) {
151 const double * v = ut + 12 * (11 - i);
152 for(
int j = 0; j < 4; j++)
153 for(
int k = 0; k < 3; k++)
154 ccs[j][k] += betas[i] * v[3 * j + k];
158 void mrpt::vision::pnp::epnp::compute_pcs(
void)
160 for(
int i = 0; i < number_of_correspondences; i++) {
161 double * a = &alphas[0] + 4 * i;
162 double * pc = &pcs[0] + 3 * i;
164 for(
int j = 0; j < 3; j++)
165 pc[j] = a[0] * ccs[0][j] + a[1] * ccs[1][j] + a[2] * ccs[2][j] + a[3] * ccs[3][j];
169 void mrpt::vision::pnp::epnp::compute_pose(cv::Mat& R, cv::Mat&
t)
171 choose_control_points();
172 compute_barycentric_coordinates();
174 CvMat * M = cvCreateMat(2 * number_of_correspondences, 12, CV_64F);
176 for(
int i = 0; i < number_of_correspondences; i++)
177 fill_M(M, 2 * i, &alphas[0] + 4 * i, us[2 * i], us[2 * i + 1]);
179 double mtm[12 * 12], d[12], ut[12 * 12];
180 CvMat MtM = cvMat(12, 12, CV_64F, mtm);
181 CvMat D = cvMat(12, 1, CV_64F, d);
182 CvMat Ut = cvMat(12, 12, CV_64F, ut);
184 cvMulTransposed(M, &MtM, 1);
185 cvSVD(&MtM, &D, &Ut, 0, CV_SVD_MODIFY_A | CV_SVD_U_T);
188 double l_6x10[6 * 10], rho[6];
189 CvMat L_6x10 = cvMat(6, 10, CV_64F, l_6x10);
190 CvMat Rho = cvMat(6, 1, CV_64F, rho);
192 compute_L_6x10(ut, l_6x10);
195 double Betas[4][4], rep_errors[4];
196 double Rs[4][3][3], ts[4][3];
198 find_betas_approx_1(&L_6x10, &Rho, Betas[1]);
199 gauss_newton(&L_6x10, &Rho, Betas[1]);
200 rep_errors[1] = compute_R_and_t(ut, Betas[1], Rs[1], ts[1]);
202 find_betas_approx_2(&L_6x10, &Rho, Betas[2]);
203 gauss_newton(&L_6x10, &Rho, Betas[2]);
204 rep_errors[2] = compute_R_and_t(ut, Betas[2], Rs[2], ts[2]);
206 find_betas_approx_3(&L_6x10, &Rho, Betas[3]);
207 gauss_newton(&L_6x10, &Rho, Betas[3]);
208 rep_errors[3] = compute_R_and_t(ut, Betas[3], Rs[3], ts[3]);
211 if (rep_errors[2] < rep_errors[1]) N = 2;
212 if (rep_errors[3] < rep_errors[N]) N = 3;
214 cv::Mat(3, 1, CV_64F, ts[N]).copyTo(t);
215 cv::Mat(3, 3, CV_64F, Rs[N]).copyTo(R);
218 void mrpt::vision::pnp::epnp::copy_R_and_t(
const double R_src[3][3],
const double t_src[3],
219 double R_dst[3][3],
double t_dst[3])
221 for(
int i = 0; i < 3; i++) {
222 for(
int j = 0; j < 3; j++)
223 R_dst[i][j] = R_src[i][j];
228 double mrpt::vision::pnp::epnp::dist2(
const double * p1,
const double * p2)
231 (p1[0] - p2[0]) * (p1[0] - p2[0]) +
232 (p1[1] - p2[1]) * (p1[1] - p2[1]) +
233 (p1[2] - p2[2]) * (p1[2] - p2[2]);
236 double mrpt::vision::pnp::epnp::dot(
const double * v1,
const double * v2)
238 return v1[0] * v2[0] + v1[1] * v2[1] + v1[2] * v2[2];
241 void mrpt::vision::pnp::epnp::estimate_R_and_t(
double R[3][3],
double t[3])
243 double pc0[3], pw0[3];
245 pc0[0] = pc0[1] = pc0[2] = 0.0;
246 pw0[0] = pw0[1] = pw0[2] = 0.0;
248 for(
int i = 0; i < number_of_correspondences; i++) {
249 const double * pc = &pcs[3 * i];
250 const double * pw = &pws[3 * i];
252 for(
int j = 0; j < 3; j++) {
257 for(
int j = 0; j < 3; j++) {
258 pc0[j] /= number_of_correspondences;
259 pw0[j] /= number_of_correspondences;
262 double abt[3 * 3], abt_d[3], abt_u[3 * 3], abt_v[3 * 3];
263 CvMat ABt = cvMat(3, 3, CV_64F, abt);
264 CvMat ABt_D = cvMat(3, 1, CV_64F, abt_d);
265 CvMat ABt_U = cvMat(3, 3, CV_64F, abt_u);
266 CvMat ABt_V = cvMat(3, 3, CV_64F, abt_v);
269 for(
int i = 0; i < number_of_correspondences; i++) {
270 double * pc = &pcs[3 * i];
271 double * pw = &pws[3 * i];
273 for(
int j = 0; j < 3; j++) {
274 abt[3 * j ] += (pc[j] - pc0[j]) * (pw[0] - pw0[0]);
275 abt[3 * j + 1] += (pc[j] - pc0[j]) * (pw[1] - pw0[1]);
276 abt[3 * j + 2] += (pc[j] - pc0[j]) * (pw[2] - pw0[2]);
280 cvSVD(&ABt, &ABt_D, &ABt_U, &ABt_V, CV_SVD_MODIFY_A);
282 for(
int i = 0; i < 3; i++)
283 for(
int j = 0; j < 3; j++)
284 R[i][j] = dot(abt_u + 3 * i, abt_v + 3 * j);
287 R[0][0] * R[1][1] * R[2][2] + R[0][1] * R[1][2] * R[2][0] + R[0][2] * R[1][0] * R[2][1] -
288 R[0][2] * R[1][1] * R[2][0] - R[0][1] * R[1][0] * R[2][2] - R[0][0] * R[1][2] * R[2][1];
296 t[0] = pc0[0] - dot(R[0], pw0);
297 t[1] = pc0[1] - dot(R[1], pw0);
298 t[2] = pc0[2] - dot(R[2], pw0);
301 void mrpt::vision::pnp::epnp::solve_for_sign(
void)
304 for(
int i = 0; i < 4; i++)
305 for(
int j = 0; j < 3; j++)
306 ccs[i][j] = -ccs[i][j];
308 for(
int i = 0; i < number_of_correspondences; i++) {
309 pcs[3 * i ] = -pcs[3 * i];
310 pcs[3 * i + 1] = -pcs[3 * i + 1];
311 pcs[3 * i + 2] = -pcs[3 * i + 2];
316 double mrpt::vision::pnp::epnp::compute_R_and_t(
const double * ut,
const double * betas,
317 double R[3][3],
double t[3])
319 compute_ccs(betas, ut);
324 estimate_R_and_t(R, t);
326 return reprojection_error(R, t);
329 double mrpt::vision::pnp::epnp::reprojection_error(
const double R[3][3],
const double t[3])
333 for(
int i = 0; i < number_of_correspondences; i++) {
334 double * pw = &pws[3 * i];
335 double Xc = dot(R[0], pw) + t[0];
336 double Yc = dot(R[1], pw) + t[1];
337 double inv_Zc = 1.0 / (dot(R[2], pw) + t[2]);
338 double ue = uc + fu * Xc * inv_Zc;
339 double ve = vc + fv * Yc * inv_Zc;
340 double u = us[2 * i], v = us[2 * i + 1];
342 sum2 += sqrt( (u - ue) * (u - ue) + (v - ve) * (v - ve) );
345 return sum2 / number_of_correspondences;
351 void mrpt::vision::pnp::epnp::find_betas_approx_1(
const CvMat * L_6x10,
const CvMat * Rho,
354 double l_6x4[6 * 4], b4[4];
355 CvMat L_6x4 = cvMat(6, 4, CV_64F, l_6x4);
356 CvMat B4 = cvMat(4, 1, CV_64F, b4);
358 for(
int i = 0; i < 6; i++) {
359 cvmSet(&L_6x4, i, 0, cvmGet(L_6x10, i, 0));
360 cvmSet(&L_6x4, i, 1, cvmGet(L_6x10, i, 1));
361 cvmSet(&L_6x4, i, 2, cvmGet(L_6x10, i, 3));
362 cvmSet(&L_6x4, i, 3, cvmGet(L_6x10, i, 6));
365 cvSolve(&L_6x4, Rho, &B4, CV_SVD);
368 betas[0] = sqrt(-b4[0]);
369 betas[1] = -b4[1] / betas[0];
370 betas[2] = -b4[2] / betas[0];
371 betas[3] = -b4[3] / betas[0];
373 betas[0] = sqrt(b4[0]);
374 betas[1] = b4[1] / betas[0];
375 betas[2] = b4[2] / betas[0];
376 betas[3] = b4[3] / betas[0];
383 void mrpt::vision::pnp::epnp::find_betas_approx_2(
const CvMat * L_6x10,
const CvMat * Rho,
386 double l_6x3[6 * 3],
b3[3];
387 CvMat L_6x3 = cvMat(6, 3, CV_64F, l_6x3);
388 CvMat B3 = cvMat(3, 1, CV_64F, b3);
390 for(
int i = 0; i < 6; i++) {
391 cvmSet(&L_6x3, i, 0, cvmGet(L_6x10, i, 0));
392 cvmSet(&L_6x3, i, 1, cvmGet(L_6x10, i, 1));
393 cvmSet(&L_6x3, i, 2, cvmGet(L_6x10, i, 2));
396 cvSolve(&L_6x3, Rho, &B3, CV_SVD);
399 betas[0] = sqrt(-b3[0]);
400 betas[1] = (b3[2] < 0) ? sqrt(-b3[2]) : 0.0;
402 betas[0] = sqrt(b3[0]);
403 betas[1] = (b3[2] > 0) ? sqrt(b3[2]) : 0.0;
406 if (b3[1] < 0) betas[0] = -betas[0];
415 void mrpt::vision::pnp::epnp::find_betas_approx_3(
const CvMat * L_6x10,
const CvMat * Rho,
418 double l_6x5[6 * 5], b5[5];
419 CvMat L_6x5 = cvMat(6, 5, CV_64F, l_6x5);
420 CvMat B5 = cvMat(5, 1, CV_64F, b5);
422 for(
int i = 0; i < 6; i++) {
423 cvmSet(&L_6x5, i, 0, cvmGet(L_6x10, i, 0));
424 cvmSet(&L_6x5, i, 1, cvmGet(L_6x10, i, 1));
425 cvmSet(&L_6x5, i, 2, cvmGet(L_6x10, i, 2));
426 cvmSet(&L_6x5, i, 3, cvmGet(L_6x10, i, 3));
427 cvmSet(&L_6x5, i, 4, cvmGet(L_6x10, i, 4));
430 cvSolve(&L_6x5, Rho, &B5, CV_SVD);
433 betas[0] = sqrt(-b5[0]);
434 betas[1] = (b5[2] < 0) ? sqrt(-b5[2]) : 0.0;
436 betas[0] = sqrt(b5[0]);
437 betas[1] = (b5[2] > 0) ? sqrt(b5[2]) : 0.0;
439 if (b5[1] < 0) betas[0] = -betas[0];
440 betas[2] = b5[3] / betas[0];
444 void mrpt::vision::pnp::epnp::compute_L_6x10(
const double * ut,
double * l_6x10)
455 for(
int i = 0; i < 4; i++) {
457 for(
int j = 0; j < 6; j++) {
458 dv[i][j][0] = v[i][3 * a ] - v[i][3 * b];
459 dv[i][j][1] = v[i][3 * a + 1] - v[i][3 * b + 1];
460 dv[i][j][2] = v[i][3 * a + 2] - v[i][3 * b + 2];
470 for(
int i = 0; i < 6; i++) {
471 double * row = l_6x10 + 10 * i;
473 row[0] = dot(dv[0][i], dv[0][i]);
474 row[1] = 2.0f * dot(dv[0][i], dv[1][i]);
475 row[2] = dot(dv[1][i], dv[1][i]);
476 row[3] = 2.0f * dot(dv[0][i], dv[2][i]);
477 row[4] = 2.0f * dot(dv[1][i], dv[2][i]);
478 row[5] = dot(dv[2][i], dv[2][i]);
479 row[6] = 2.0f * dot(dv[0][i], dv[3][i]);
480 row[7] = 2.0f * dot(dv[1][i], dv[3][i]);
481 row[8] = 2.0f * dot(dv[2][i], dv[3][i]);
482 row[9] = dot(dv[3][i], dv[3][i]);
486 void mrpt::vision::pnp::epnp::compute_rho(
double * rho)
488 rho[0] = dist2(cws[0], cws[1]);
489 rho[1] = dist2(cws[0], cws[2]);
490 rho[2] = dist2(cws[0], cws[3]);
491 rho[3] = dist2(cws[1], cws[2]);
492 rho[4] = dist2(cws[1], cws[3]);
493 rho[5] = dist2(cws[2], cws[3]);
496 void mrpt::vision::pnp::epnp::compute_A_and_b_gauss_newton(
const double * l_6x10,
const double * rho,
497 const double betas[4], CvMat * A, CvMat * b)
499 for(
int i = 0; i < 6; i++) {
500 const double * rowL = l_6x10 + i * 10;
501 double * rowA = A->data.db + i * 4;
503 rowA[0] = 2 * rowL[0] * betas[0] + rowL[1] * betas[1] + rowL[3] * betas[2] + rowL[6] * betas[3];
504 rowA[1] = rowL[1] * betas[0] + 2 * rowL[2] * betas[1] + rowL[4] * betas[2] + rowL[7] * betas[3];
505 rowA[2] = rowL[3] * betas[0] + rowL[4] * betas[1] + 2 * rowL[5] * betas[2] + rowL[8] * betas[3];
506 rowA[3] = rowL[6] * betas[0] + rowL[7] * betas[1] + rowL[8] * betas[2] + 2 * rowL[9] * betas[3];
508 cvmSet(b, i, 0, rho[i] -
510 rowL[0] * betas[0] * betas[0] +
511 rowL[1] * betas[0] * betas[1] +
512 rowL[2] * betas[1] * betas[1] +
513 rowL[3] * betas[0] * betas[2] +
514 rowL[4] * betas[1] * betas[2] +
515 rowL[5] * betas[2] * betas[2] +
516 rowL[6] * betas[0] * betas[3] +
517 rowL[7] * betas[1] * betas[3] +
518 rowL[8] * betas[2] * betas[3] +
519 rowL[9] * betas[3] * betas[3]
524 void mrpt::vision::pnp::epnp::gauss_newton(
const CvMat * L_6x10,
const CvMat * Rho,
double betas[4])
526 const int iterations_number = 5;
528 double a[6*4], b[6],
x[4];
529 CvMat A = cvMat(6, 4, CV_64F, a);
530 CvMat B = cvMat(6, 1, CV_64F, b);
531 CvMat X = cvMat(4, 1, CV_64F, x);
533 for(
int k = 0; k < iterations_number; k++)
535 compute_A_and_b_gauss_newton(L_6x10->data.db, Rho->data.db,
537 qr_solve(&A, &B, &X);
538 for(
int i = 0; i < 4; i++)
543 void mrpt::vision::pnp::epnp::qr_solve(CvMat * A, CvMat * b, CvMat * X)
545 const int nr = A->rows;
546 const int nc = A->cols;
548 if (max_nr != 0 && max_nr < nr)
560 double * pA = A->data.db, * ppAkk = pA;
561 for(
int k = 0; k < nc; k++)
563 double * ppAik1 = ppAkk, eta = fabs(*ppAik1);
564 for(
int i = k + 1; i < nr; i++)
566 double elt = fabs(*ppAik1);
567 if (eta < elt) eta = elt;
578 double * ppAik2 = ppAkk, sum2 = 0.0, inv_eta = 1. / eta;
579 for(
int i = k; i < nr; i++)
582 sum2 += *ppAik2 * *ppAik2;
585 double sigma = sqrt(sum2);
589 A1[k] = sigma * *ppAkk;
590 A2[k] = -eta * sigma;
591 for(
int j = k + 1; j < nc; j++)
593 double * ppAik = ppAkk,
sum = 0;
594 for(
int i = k; i < nr; i++)
596 sum += *ppAik * ppAik[j - k];
599 double tau = sum /
A1[k];
601 for(
int i = k; i < nr; i++)
603 ppAik[j - k] -= tau * *ppAik;
612 double * ppAjj = pA, * pb = b->data.db;
613 for(
int j = 0; j < nc; j++)
615 double * ppAij = ppAjj, tau = 0;
616 for(
int i = j; i < nr; i++)
618 tau += *ppAij * pb[i];
623 for(
int i = j; i < nr; i++)
625 pb[i] -= tau * *ppAij;
632 double * pX = X->data.db;
633 pX[nc - 1] = pb[nc - 1] / A2[nc - 1];
634 for(
int i = nc - 2; i >= 0; i--)
636 double * ppAij = pA + i * nc + (i + 1),
sum = 0;
638 for(
int j = i + 1; j < nc; j++)
640 sum += *ppAij * pX[j];
643 pX[i] = (pb[i] -
sum) / A2[i];
EIGEN_STRONG_INLINE Scalar det() const
EIGEN_STRONG_INLINE const AdjointReturnType t() const
Transpose.
T max(const T v0, const T v1)
CONTAINER::Scalar sum(const CONTAINER &v)
Computes the sum of all the elements.
double A1
UTC constant and 1st order terms.
x y t t *t x y t t t x y t t t x *y t *t t x *y t *t t x y t t t x y t t t x(y+z)