From 755546643a671420fae0c20ec5d4cd24a228da8e Mon Sep 17 00:00:00 2001 From: Arne Baeyens Date: Sat, 29 Aug 2026 10:52:17 +0200 Subject: [PATCH] Merge pull request #29779 from abaeyens:abaeyens/speed-up-warp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Speed up imgproc warpAffine and warpPerspective for BORDER_TRANSPARENT - #29779 ### Pull Request Readiness Checklist - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake ## Why I was using `warpPerspective` to draw several source images on a large destination image in mode `BORDER_TRANSPARENT` and ran into `warpPerspective` being surprisingly slow. Upon reading the code, it turns out that `warpPerspective`, as well as `warpAffine`, iterates over all the destination image's pixels even if the source image gets projected to only a small part of the destination image, resulting in considerable overhead for my use case. I believe other users would also benefit from making this case more efficient. ## Changes d19c3437045ecc654de08a18417488a868fe0122 calculates the ROI of the source image in the destination image and then limits the destination image walk to that area. Given that the existing tests didn't cover `BORDER_TRANSPARENT`, I extended that in e451f59cdabf88660119410b4117c46488ec02a6. Next to that, I added a small performance test dedicated to this use case (c4e271c2dc09c287a1940b3e57b93742bc9abf03). ## Performance improvement The following table show the timing difference before and after, generated using the added perf test (source image projects to 64x64, drawn in a 512x512 destination image): | function | type | interp | base [ms] | opt [ms] | speedup | | --- | --- | --- | --- | --- | --- | | warpAffine | 8UC1 | NEAREST | 0.204 | 0.010 | 19.8× | | warpAffine | 8UC1 | LINEAR | 0.428 | 0.026 | 16.6× | | warpAffine | 8UC4 | NEAREST | 0.239 | 0.021 | 11.5× | | warpAffine | 8UC4 | LINEAR | 0.433 | 0.031 | 14.0× | | warpPerspective | 8UC1 | NEAREST | 0.759 | 0.033 | 23.2× | | warpPerspective | 8UC1 | LINEAR | 1.125 | 0.067 | 16.8× | | warpPerspective | 8UC4 | NEAREST | 0.786 | 0.040 | 19.5× | | warpPerspective | 8UC4 | LINEAR | 1.110 | 0.101 | 11.0× | In short, a 10 to 20x speedup. ## Notes - This is my first PR for the OpenCV project, I'm sorry in case I didn't respect all contribution guidelines. - If relevant, Clause Opus 4.8 was used for exploring the codebase, some code and style suggestions and review. --- modules/imgproc/perf/perf_warp.cpp | 59 +++++++++++++ modules/imgproc/src/imgwarp.cpp | 90 +++++++++++++++++--- modules/imgproc/test/test_imgwarp_strict.cpp | 9 ++ 3 files changed, 144 insertions(+), 14 deletions(-) diff --git a/modules/imgproc/perf/perf_warp.cpp b/modules/imgproc/perf/perf_warp.cpp index cb4c32c905..91c24e4699 100644 --- a/modules/imgproc/perf/perf_warp.cpp +++ b/modules/imgproc/perf/perf_warp.cpp @@ -16,6 +16,8 @@ typedef TestBaseWithParam< tuple > TestWar typedef TestBaseWithParam< tuple > TestWarpPerspective; typedef TestBaseWithParam< tuple > TestWarpPerspectiveNear_t; typedef TestBaseWithParam< tuple > TestRemap; +typedef TestBaseWithParam< tuple > TestWarpAffineSmallSrc; +typedef TestBaseWithParam< tuple > TestWarpPerspectiveSmallSrc; void update_map(const Mat& src, Mat& map_x, Mat& map_y, const int remapMode, bool relative = false ); @@ -307,4 +309,61 @@ PERF_TEST(Transform, getPerspectiveTransform_QR_1000) SANITY_CHECK_NOTHING(); } +PERF_TEST_P( TestWarpAffineSmallSrc, WarpAffine_SmallSrc, + Combine( + Values(CV_8UC1, CV_8UC4), + Values( Size(512, 512) ), + InterType::all() + ) +) +{ + const Size szSrc(64, 64); + const int dataType = get<0>(GetParam()); + const Size sz = get<1>(GetParam()); + const int interType = get<2>(GetParam()); + const int borderMode = BORDER_TRANSPARENT; + + Mat src(szSrc, dataType), dst(sz, dataType, Scalar::all(0)); + cvtest::fillGradient(src); + + const double offset = (sz.width - szSrc.width) / 2.0; + const Mat warpMat = (Mat_(2, 3) << 1, 0, offset, + 0, 1, offset); + declare.in(src).out(dst); + + TEST_CYCLE() warpAffine( src, dst, warpMat, sz, interType, borderMode ); + + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P( TestWarpPerspectiveSmallSrc, WarpPerspective_SmallSrc, + Combine( + Values(CV_8UC1, CV_8UC4), + Values( Size(512, 512) ), + InterType::all() + ) +) +{ + const Size szSrc(64, 64); + const int dataType = get<0>(GetParam()); + const Size sz = get<1>(GetParam()); + const int interType = get<2>(GetParam()); + const int borderMode = BORDER_TRANSPARENT; + + Mat src(szSrc, dataType), dst(sz, dataType, Scalar::all(0)); + cvtest::fillGradient(src); + + const float w = (float)szSrc.width, h = (float)szSrc.height; + const float off = (sz.width - szSrc.width) / 2.0f; + const Point2f sp[] = { Point2f(0, 0), Point2f(w, 0), Point2f(0, h), Point2f(w, h) }; + const Point2f dp[] = { Point2f(off, off), Point2f(off + w, off), + Point2f(off, off + h), Point2f(off + w, off + h) }; + const Mat warpMat = getPerspectiveTransform(sp, dp); + declare.in(src).out(dst); + + TEST_CYCLE() warpPerspective( src, dst, warpMat, sz, interType, borderMode ); + + SANITY_CHECK_NOTHING(); +} + } // namespace diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 17f77c1599..d5fd16e2f6 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -2168,15 +2168,57 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, namespace cv { +static Rect warpSourceFootprintRoi(Size srcSize, Size dstSize, const double* M, bool isPerspective) +{ + const Rect fullRoi(0, 0, dstSize.width, dstSize.height); + + // Invert M because it maps destination to source while we need the reverse + const Matx33d fullM = isPerspective + ? Matx33d(M[0], M[1], M[2], M[3], M[4], M[5], M[6], M[7], M[8]) + : Matx33d(M[0], M[1], M[2], M[3], M[4], M[5], 0.0, 0.0, 1.0); + bool invertible = false; + const Matx33d invM = fullM.inv(DECOMP_LU, &invertible); + if (!invertible) + return fullRoi; + + const int INTERPOLATION_PADDING = 4; // assume worst case: INTER_LANCZOS4 + const double x0 = -INTERPOLATION_PADDING, x1 = srcSize.width + INTERPOLATION_PADDING; + const double y0 = -INTERPOLATION_PADDING, y1 = srcSize.height + INTERPOLATION_PADDING; + const Point2d corners[4] = { {x0, y0}, {x1, y0}, {x1, y1}, {x0, y1} }; + double minX = DBL_MAX, maxX = -DBL_MAX, minY = DBL_MAX, maxY = -DBL_MAX; + int zsign = 0; + for (const Point2d& c : corners) + { + const double z = invM(2, 0) * c.x + invM(2, 1) * c.y + invM(2, 2); + if (std::abs(z) < 1e-9) + return fullRoi; // z small => xy range close to inf + const int s = z > 0 ? 1 : -1; + if (zsign == 0) + zsign = s; + else if (s != zsign) + return fullRoi; // sign changed => corners front and rear + const double x = (invM(0, 0) * c.x + invM(0, 1) * c.y + invM(0, 2)) / z; + const double y = (invM(1, 0) * c.x + invM(1, 1) * c.y + invM(1, 2)) / z; + minX = std::min(minX, x); maxX = std::max(maxX, x); + minY = std::min(minY, y); maxY = std::max(maxY, y); + } + const int x0i = cvFloor(std::min(std::max(minX, -1.0), dstSize.width + 1.0)); + const int x1i = cvCeil (std::min(std::max(maxX, -1.0), dstSize.width + 1.0)); + const int y0i = cvFloor(std::min(std::max(minY, -1.0), dstSize.height + 1.0)); + const int y1i = cvCeil (std::min(std::max(maxY, -1.0), dstSize.height + 1.0)); + return Rect(x0i, y0i, x1i - x0i, y1i - y0i) & fullRoi; +} + class WarpAffineInvoker : public ParallelLoopBody { public: WarpAffineInvoker(const Mat &_src, Mat &_dst, int _interpolation, int _borderType, - const Scalar &_borderValue, int *_adelta, int *_bdelta, const double *_M) : + const Scalar &_borderValue, int *_adelta, int *_bdelta, const double *_M, + const Range &_colRange) : ParallelLoopBody(), src(_src), dst(_dst), interpolation(_interpolation), borderType(_borderType), borderValue(_borderValue), adelta(_adelta), bdelta(_bdelta), - M(_M) + M(_M), colRange(_colRange) { } @@ -2195,9 +2237,9 @@ public: for( y = range.start; y < range.end; y += bh0 ) { - for( x = 0; x < dst.cols; x += bw0 ) + for( x = colRange.start; x < colRange.end; x += bw0 ) { - int bw = std::min( bw0, dst.cols - x); + int bw = std::min( bw0, colRange.end - x); int bh = std::min( bh0, range.end - y); Mat _XY(bh, bw, CV_16SC2, XY); @@ -2233,6 +2275,7 @@ private: Scalar borderValue; int *adelta, *bdelta; const double *M; + Range colRange; }; #ifdef HAVE_OPENCL @@ -2448,11 +2491,20 @@ void warpAffine(int src_type, bdelta[x] = saturate_cast(M[3]*x*AB_SCALE); } - Range range(0, dst.rows); + Range rowRange(0, dst.rows); + Range colRange(0, dst.cols); + double nbPixels = (double)dst.total(); + if ((borderType & ~BORDER_ISOLATED) == BORDER_TRANSPARENT) + { + const Rect dstRoi = warpSourceFootprintRoi(src.size(), dst.size(), M, false); + rowRange = Range(dstRoi.y, dstRoi.y + dstRoi.height); + colRange = Range(dstRoi.x, dstRoi.x + dstRoi.width); + nbPixels = dstRoi.area(); + } WarpAffineInvoker invoker(src, dst, interpolation, borderType, Scalar(borderValue[0], borderValue[1], borderValue[2], borderValue[3]), - adelta, bdelta, M); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); + adelta, bdelta, M, colRange); + parallel_for_(rowRange, invoker, nbPixels/(1<<16)); } CV_DISABLE_UBSAN @@ -2854,9 +2906,9 @@ class WarpPerspectiveInvoker : { public: WarpPerspectiveInvoker(const Mat &_src, Mat &_dst, const double *_M, int _interpolation, - int _borderType, const Scalar &_borderValue) : + int _borderType, const Scalar &_borderValue, const Range &_colRange) : ParallelLoopBody(), src(_src), dst(_dst), M(_M), interpolation(_interpolation), - borderType(_borderType), borderValue(_borderValue) + borderType(_borderType), borderValue(_borderValue), colRange(_colRange) { #if defined(_MSC_VER) && _MSC_VER == 1800 /* MSVS 2013 */ && CV_AVX // details: https://github.com/opencv/opencv/issues/11026 @@ -2877,9 +2929,9 @@ public: for( y = range.start; y < range.end; y += bh0 ) { - for( x = 0; x < width; x += bw0 ) + for( x = colRange.start; x < colRange.end; x += bw0 ) { - int bw = std::min( bw0, width - x); + int bw = std::min( bw0, colRange.end - x); int bh = std::min( bh0, range.end - y); // height Mat _XY(bh, bw, CV_16SC2, XY); @@ -2915,6 +2967,7 @@ private: const double* M; int interpolation, borderType; Scalar borderValue; + Range colRange; }; @@ -2929,9 +2982,18 @@ void warpPerspective(int src_type, Mat src(Size(src_width, src_height), src_type, const_cast(src_data), src_step); Mat dst(Size(dst_width, dst_height), src_type, dst_data, dst_step); - Range range(0, dst.rows); - WarpPerspectiveInvoker invoker(src, dst, M, interpolation, borderType, Scalar(borderValue[0], borderValue[1], borderValue[2], borderValue[3])); - parallel_for_(range, invoker, dst.total()/(double)(1<<16)); + Range rowRange(0, dst.rows); + Range colRange(0, dst.cols); + double nbPixels = (double)dst.total(); + if ((borderType & ~BORDER_ISOLATED) == BORDER_TRANSPARENT) + { + const Rect dstRoi = warpSourceFootprintRoi(src.size(), dst.size(), M, true); + rowRange = Range(dstRoi.y, dstRoi.y + dstRoi.height); + colRange = Range(dstRoi.x, dstRoi.x + dstRoi.width); + nbPixels = dstRoi.area(); + } + WarpPerspectiveInvoker invoker(src, dst, M, interpolation, borderType, Scalar(borderValue[0], borderValue[1], borderValue[2], borderValue[3]), colRange); + parallel_for_(rowRange, invoker, nbPixels/(1<<16)); } void warpPerspectiveBlocklineNN(const double *M, short* xy, double X0, double Y0, double W0, int bw) diff --git a/modules/imgproc/test/test_imgwarp_strict.cpp b/modules/imgproc/test/test_imgwarp_strict.cpp index 673c6f03e6..53d3f9bc71 100644 --- a/modules/imgproc/test/test_imgwarp_strict.cpp +++ b/modules/imgproc/test/test_imgwarp_strict.cpp @@ -1079,6 +1079,10 @@ void CV_WarpAffine_Test::generate_test_data() // warp_matrix is inverse if (rng.uniform(0., 1.) > 0) interpolation |= cv::WARP_INVERSE_MAP; + + // base sets only REPLICATE and REFLECT while want to test TRANSPARENT as well + static const int borderTypes[] = { BORDER_REPLICATE, BORDER_REFLECT, BORDER_TRANSPARENT }; + borderType = borderTypes[rng.uniform(0, sizeof(borderTypes) / sizeof(borderTypes[0]))]; } void CV_WarpAffine_Test::run_func() @@ -1201,6 +1205,11 @@ void CV_WarpPerspective_Test::generate_test_data() static const int depths[] = { CV_32F, CV_64F }; int depth = depths[rng.uniform(0, 2)]; M.clone().convertTo(M, depth); + + // BUG: https://github.com/opencv/opencv/issues/29816 + // BORDER_REPLICATE disabled due to KleidiCV accuracy issue on ARM + static const int borderTypes[] = { /*BORDER_REPLICATE,*/ BORDER_REFLECT, BORDER_TRANSPARENT }; + borderType = borderTypes[rng.uniform(0, sizeof(borderTypes) / sizeof(borderTypes[0]))]; } void CV_WarpPerspective_Test::run_func()