From b0cfda31875db9630e8454fceea873959a145422 Mon Sep 17 00:00:00 2001 From: Madan mohan Manokar Date: Wed, 26 Aug 2026 05:39:43 +0000 Subject: [PATCH] imgproc: fused spatialGradient + dispatched SIMD Canny. - Add fused single-pass spatialGradient kernels with runtime SIMD dispatch. - Add dispatched SIMD Canny edge path (canny.dispatch.cpp, canny.simd.hpp). - Route corner, Canny, and IntelligentScissors gradients through spatialGradient. - Extend spatialGradient ksize support to 1 via Sobel fallback; gate fused paths to 3/5. - Extend fused_accuracy ROI tests to ksize 1, 3, and 5 (borders, CV_16S/CV_32F). --- modules/imgproc/CMakeLists.txt | 2 + modules/imgproc/include/opencv2/imgproc.hpp | 21 +- modules/imgproc/src/canny.cpp | 256 +-------- modules/imgproc/src/canny.dispatch.cpp | 85 +++ modules/imgproc/src/canny.hpp | 24 + modules/imgproc/src/canny.simd.hpp | 285 ++++++++++ modules/imgproc/src/corner.cpp | 33 +- modules/imgproc/src/generalized_hough.cpp | 6 - modules/imgproc/src/intelligent_scissors.cpp | 13 +- modules/imgproc/src/spatialgradient.cpp | 399 ++++---------- modules/imgproc/src/spatialgradient.simd.hpp | 524 +++++++++++++++++++ modules/imgproc/test/test_filter.cpp | 4 +- 12 files changed, 1065 insertions(+), 587 deletions(-) create mode 100644 modules/imgproc/src/canny.dispatch.cpp create mode 100644 modules/imgproc/src/canny.hpp create mode 100644 modules/imgproc/src/canny.simd.hpp create mode 100644 modules/imgproc/src/spatialgradient.simd.hpp diff --git a/modules/imgproc/CMakeLists.txt b/modules/imgproc/CMakeLists.txt index 63529a0c86..d969d3b573 100644 --- a/modules/imgproc/CMakeLists.txt +++ b/modules/imgproc/CMakeLists.txt @@ -2,11 +2,13 @@ set(the_description "Image Processing") ocv_add_dispatched_file(accum SSE4_1 AVX AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(bilateral_filter SSE2 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(box_filter SSE2 SSE4_1 AVX2 AVX512_SKX) +ocv_add_dispatched_file(canny SSE4_1 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(filter SSE2 SSE4_1 AVX2) ocv_add_dispatched_file(color_hsv SSE2 SSE4_1 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(color_rgb SSE2 SSE4_1 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(color_yuv SSE2 SSE4_1 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(median_blur SSE2 SSE4_1 AVX2 AVX512_SKX AVX512_ICL) +ocv_add_dispatched_file(spatialgradient SSE4_1 AVX2 AVX512_SKX AVX512_ICL) ocv_add_dispatched_file(morph SSE2 SSE4_1 AVX2) ocv_add_dispatched_file(smooth SSE2 SSE4_1 AVX2 AVX512_ICL) ocv_add_dispatched_file(sumpixels SSE2 AVX2 AVX512_SKX) diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 58ee6fb8cf..5d6416debb 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -1818,21 +1818,22 @@ This is a fused variant of #Sobel: instead of two separate calls Sobel( src, dx, ddepth, 1, 0, ksize, scale ); Sobel( src, dy, ddepth, 0, 1, ksize, scale ); @endcode -it produces both first-order derivatives in one traversal of the source. For an 8-bit single-channel -whole-image (non-ROI) source with @p ksize = 3, @p ddepth = CV_16S, @p scale = 1, and -#BORDER_DEFAULT (#BORDER_REFLECT_101) or #BORDER_REPLICATE, a dedicated fused 3x3 stencil kernel is -used (including a HAL fast path) that reads each source sample once and shares it between the dx and -dy computations; the result is bit-identical to the two #Sobel calls above. All other cases -(@p ddepth = CV_32F, @p ksize = 5, scaled int16 output, floating-point source, other border types, -or ROI/sub-matrix input) currently fall back to the two equivalent #Sobel passes. @note Fused fast -paths for CV_32F output and @p ksize = 5 are added in a follow-up change. +it produces both first-order derivatives in one traversal of the source. The fused single-pass +kernels apply to an 8-bit single-channel (CV_8UC1) source with @p ksize = 3 or 5 and a +reflect/replicate border (#BORDER_DEFAULT / #BORDER_REFLECT_101, #BORDER_REFLECT, #BORDER_REPLICATE), +for both @p ddepth = CV_16S (unit @p scale) and @p ddepth = CV_32F (any @p scale); full-width +row-range ROIs are supported as well. In these cases each source sample is read once and shared +between the dx and dy computations, and the result is bit-identical to the two #Sobel calls above. +The remaining cases (scaled int16 output, floating-point source, #BORDER_CONSTANT/#BORDER_WRAP, or a +column-offset/partial-width ROI) fall back to the two equivalent #Sobel passes. @param src input image; single-channel, 8-bit (CV_8UC1) for the fused fast paths (CV_32FC1 is accepted via the fallback). @param dx output image with the first-order derivative in x (depth @p ddepth, same size as src). @param dy output image with the first-order derivative in y (depth @p ddepth, same size as src). -@param ksize size of the Sobel kernel; fused fast paths require 3 (5 uses the Sobel fallback). - Also accepts -1 (Scharr) and 7 for Sobel-compatible callers such as #HoughCircles. +@param ksize size of the Sobel kernel; fused fast paths require 3 or 5 (other sizes use the Sobel + fallback). Also accepts 1, -1 (Scharr), and 7 for Sobel-compatible callers such as + #HoughCircles and #cv::segmentation::IntelligentScissorsMB. @param borderType pixel extrapolation method, see #BorderTypes. #BORDER_WRAP is not supported. @param ddepth output image depth; CV_16S or CV_32F. @param scale optional scale factor applied to the computed derivatives. diff --git a/modules/imgproc/src/canny.cpp b/modules/imgproc/src/canny.cpp index ed391cbd99..cb5328266b 100644 --- a/modules/imgproc/src/canny.cpp +++ b/modules/imgproc/src/canny.cpp @@ -12,6 +12,7 @@ // // Copyright (C) 2000, Intel Corporation, all rights reserved. // Copyright (C) 2014, Itseez Inc., all rights reserved. +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -42,7 +43,7 @@ #include "precomp.hpp" #include "opencl_kernels_imgproc.hpp" -#include "opencv2/core/hal/intrin.hpp" +#include "canny.hpp" #include namespace cv @@ -223,19 +224,10 @@ public: parallelCanny(const Mat &_src, Mat &_map, std::deque &borderPeaksParallel, int _low, int _high, int _aperture_size, bool _L2gradient) : src(_src), src2(_src), map(_map), _borderPeaksParallel(borderPeaksParallel), - low(_low), high(_high), aperture_size(_aperture_size), L2gradient(_L2gradient) + low(_low), high(_high), aperture_size(_aperture_size), L2gradient(_L2gradient), + simdWidth(canny_simd_width()) { -#if (CV_SIMD || CV_SIMD_SCALABLE) - for(int i = 0; i < VTraits::vlanes(); ++i) - { - smask[i] = 0; - smask[i + VTraits::vlanes()] = (schar)-1; - } - if (true) - _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); - else -#endif - _map.create(src.rows + 2, src.cols + 2, CV_8UC1); + _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + simdWidth + 1), simdWidth), CV_8UC1); map = _map; map.row(0).setTo(1); map.row(src.rows + 1).setTo(1); @@ -247,19 +239,10 @@ public: parallelCanny(const Mat &_dx, const Mat &_dy, Mat &_map, std::deque &borderPeaksParallel, int _low, int _high, bool _L2gradient) : src(_dx), src2(_dy), map(_map), _borderPeaksParallel(borderPeaksParallel), - low(_low), high(_high), aperture_size(0), L2gradient(_L2gradient) + low(_low), high(_high), aperture_size(0), L2gradient(_L2gradient), + simdWidth(canny_simd_width()) { -#if (CV_SIMD || CV_SIMD_SCALABLE) - for(int i = 0; i < VTraits::vlanes(); ++i) - { - smask[i] = 0; - smask[i + VTraits::vlanes()] = (schar)-1; - } - if (true) - _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); - else -#endif - _map.create(src.rows + 2, src.cols + 2, CV_8UC1); + _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + simdWidth + 1), simdWidth), CV_8UC1); map = _map; map.row(0).setTo(1); map.row(src.rows + 1).setTo(1); @@ -290,12 +273,8 @@ public: CV_TRACE_REGION("gradient") if(needGradient) { - if (aperture_size == 7) - { - scale = 1 / 16.0; - } - Sobel(src.rowRange(rowStart, rowEnd), dx, CV_16S, 1, 0, aperture_size, scale, 0, BORDER_REPLICATE); - Sobel(src.rowRange(rowStart, rowEnd), dy, CV_16S, 0, 1, aperture_size, scale, 0, BORDER_REPLICATE); + scale = (aperture_size == 7) ? (1 / 16.0) : 1.0; + spatialGradient(src.rowRange(rowStart, rowEnd), dx, dy, aperture_size, BORDER_REPLICATE, CV_16S, scale); } else { @@ -315,17 +294,10 @@ public: } // _mag_p: previous row, _mag_a: actual row, _mag_n: next row -#if (CV_SIMD || CV_SIMD_SCALABLE) - AutoBuffer buffer(3 * (mapstep * cn + CV_SIMD_WIDTH)); - _mag_p = alignPtr(buffer.data() + 1, CV_SIMD_WIDTH); - _mag_a = alignPtr(_mag_p + mapstep * cn, CV_SIMD_WIDTH); - _mag_n = alignPtr(_mag_a + mapstep * cn, CV_SIMD_WIDTH); -#else - AutoBuffer buffer(3 * (mapstep * cn)); - _mag_p = buffer.data() + 1; - _mag_a = _mag_p + mapstep * cn; - _mag_n = _mag_a + mapstep * cn; -#endif + AutoBuffer buffer(3 * (mapstep * cn + simdWidth)); + _mag_p = alignPtr(buffer.data() + 1, simdWidth); + _mag_a = alignPtr(_mag_p + mapstep * cn, simdWidth); + _mag_n = alignPtr(_mag_a + mapstep * cn, simdWidth); // For the first time when just 2 rows are filled and for left and right borders if(rowStart == boundaries.start) @@ -352,50 +324,7 @@ public: _dx = dx.ptr(i - rowStart); _dy = dy.ptr(i - rowStart); - if (L2gradient) - { - int j = 0, width = src.cols * cn; -#if (CV_SIMD || CV_SIMD_SCALABLE) - for ( ; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) - { - v_int16 v_dx = vx_load((const short*)(_dx + j)); - v_int16 v_dy = vx_load((const short*)(_dy + j)); - - v_int32 v_dxp_low, v_dxp_high; - v_int32 v_dyp_low, v_dyp_high; - v_expand(v_dx, v_dxp_low, v_dxp_high); - v_expand(v_dy, v_dyp_low, v_dyp_high); - - v_store_aligned((int *)(_mag_n + j), v_add(v_mul(v_dxp_low, v_dxp_low), v_mul(v_dyp_low, v_dyp_low))); - v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_mul(v_dxp_high, v_dxp_high), v_mul(v_dyp_high, v_dyp_high))); - } -#endif - for ( ; j < width; ++j) - _mag_n[j] = int(_dx[j])*_dx[j] + int(_dy[j])*_dy[j]; - } - else - { - int j = 0, width = src.cols * cn; -#if (CV_SIMD || CV_SIMD_SCALABLE) - for(; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) - { - v_int16 v_dx = vx_load((const short *)(_dx + j)); - v_int16 v_dy = vx_load((const short *)(_dy + j)); - - v_dx = v_reinterpret_as_s16(v_abs(v_dx)); - v_dy = v_reinterpret_as_s16(v_abs(v_dy)); - - v_int32 v_dx_ml, v_dy_ml, v_dx_mh, v_dy_mh; - v_expand(v_dx, v_dx_ml, v_dx_mh); - v_expand(v_dy, v_dy_ml, v_dy_mh); - - v_store_aligned((int *)(_mag_n + j), v_add(v_dx_ml, v_dy_ml)); - v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_dx_mh, v_dy_mh)); - } -#endif - for ( ; j < width; ++j) - _mag_n[j] = std::abs(int(_dx[j])) + std::abs(int(_dy[j])); - } + canny_calc_magnitude(_dx, _dy, _mag_n, src.cols * cn, L2gradient); if(cn > 1) { @@ -434,12 +363,7 @@ public: // From here actual src row is (i - 1) // Set left and right border to 1 -#if (CV_SIMD || CV_SIMD_SCALABLE) - if (true) - _pmap = map.ptr(i) + CV_SIMD_WIDTH; - else -#endif - _pmap = map.ptr(i) + 1; + _pmap = map.ptr(i) + simdWidth; _pmap[src.cols] =_pmap[-1] = 1; @@ -454,109 +378,7 @@ public: _dy = _dy_a; } - const int TG22 = 13573; - int j = 0; -#if (CV_SIMD || CV_SIMD_SCALABLE) - { - const v_int32 v_low = vx_setall_s32(low); - const v_int8 v_one = vx_setall_s8(1); - - for (; j <= src.cols - VTraits::vlanes(); j += VTraits::vlanes()) - { - v_store_aligned((signed char*)(_pmap + j), v_one); - v_int8 v_cmp = v_pack(v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j)), v_low), - v_gt(vx_load_aligned((const int *)(_mag_a + j + VTraits::vlanes())), v_low)), - v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j + 2 * VTraits::vlanes())), v_low), - v_gt(vx_load_aligned((const int *)(_mag_a + j + 3 * VTraits::vlanes())), v_low))); - while (v_check_any(v_cmp)) - { - int l = v_scan_forward(v_cmp); - v_cmp = v_and(v_cmp, vx_load(smask + VTraits::vlanes() - 1 - l)); - int k = j + l; - - int m = _mag_a[k]; - short xs = _dx[k]; - short ys = _dy[k]; - int x = (int)std::abs(xs); - int y = (int)std::abs(ys) << 15; - - int tg22x = x * TG22; - - if (y < tg22x) - { - if (m > _mag_a[k - 1] && m >= _mag_a[k + 1]) - { - CANNY_CHECK(m, high, (_pmap+k), stack); - } - } - else - { - int tg67x = tg22x + (x << 16); - if (y > tg67x) - { - if (m > _mag_p[k] && m >= _mag_n[k]) - { - CANNY_CHECK(m, high, (_pmap+k), stack); - } - } - else - { - int s = (xs ^ ys) < 0 ? -1 : 1; - if(m > _mag_p[k - s] && m > _mag_n[k + s]) - { - CANNY_CHECK(m, high, (_pmap+k), stack); - } - } - } - } - } - } -#endif - for (; j < src.cols; j++) - { - int m = _mag_a[j]; - - if (m > low) - { - short xs = _dx[j]; - short ys = _dy[j]; - int x = (int)std::abs(xs); - int y = (int)std::abs(ys) << 15; - - int tg22x = x * TG22; - - if (y < tg22x) - { - if (m > _mag_a[j - 1] && m >= _mag_a[j + 1]) - { - CANNY_CHECK(m, high, (_pmap+j), stack); - continue; - } - } - else - { - int tg67x = tg22x + (x << 16); - if (y > tg67x) - { - if (m > _mag_p[j] && m >= _mag_n[j]) - { - CANNY_CHECK(m, high, (_pmap+j), stack); - continue; - } - } - else - { - int s = (xs ^ ys) < 0 ? -1 : 1; - if(m > _mag_p[j - s] && m > _mag_n[j + s]) - { - CANNY_CHECK(m, high, (_pmap+j), stack); - continue; - } - } - } - } - _pmap[j] = 1; - } + canny_nms_row(_mag_a, _mag_p, _mag_n, _dx, _dy, _pmap, src.cols, low, high, stack); } // Not for first row of first slice or last row of last slice @@ -611,10 +433,8 @@ private: bool L2gradient, needGradient; ptrdiff_t mapstep; int cn; + int simdWidth; mutable Mutex mutex; -#if (CV_SIMD || CV_SIMD_SCALABLE) - schar smask[2*VTraits::max_nlanes]; -#endif }; class finalPass : public ParallelLoopBody @@ -631,44 +451,14 @@ public: void operator()(const Range &boundaries) const CV_OVERRIDE { - // the final pass, form the final image + // the final pass, form the final image. The map's left padding matches the + // width used by parallelCanny when it was allocated (same dispatched value). + const int simdWidth = canny_simd_width(); for (int i = boundaries.start; i < boundaries.end; i++) { - int j = 0; uchar *pdst = dst.ptr(i); - const uchar *pmap = map.ptr(i + 1); -#if (CV_SIMD || CV_SIMD_SCALABLE) - if (true) - pmap += CV_SIMD_WIDTH; - else -#endif - pmap += 1; -#if (CV_SIMD || CV_SIMD_SCALABLE) - { - const v_uint8 v_zero = vx_setzero_u8(); - const v_uint8 v_ff = v_not(v_zero); - const v_uint8 v_two = vx_setall_u8(2); - - for (; j <= dst.cols - VTraits::vlanes(); j += VTraits::vlanes()) - { - v_uint8 v_pmap = vx_load_aligned((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); - v_store((pdst + j), v_pmap); - } - - if (j <= dst.cols - VTraits::vlanes()/2) - { - v_uint8 v_pmap = vx_load_low((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); - v_store_low((pdst + j), v_pmap); - j += VTraits::vlanes()/2; - } - } -#endif - for (; j < dst.cols; j++) - { - pdst[j] = (uchar)-(pmap[j] >> 1); - } + const uchar *pmap = map.ptr(i + 1) + simdWidth; + canny_finalize_row(pmap, pdst, dst.cols); } } diff --git a/modules/imgproc/src/canny.dispatch.cpp b/modules/imgproc/src/canny.dispatch.cpp new file mode 100644 index 0000000000..ea82815fd4 --- /dev/null +++ b/modules/imgproc/src/canny.dispatch.cpp @@ -0,0 +1,85 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc., all rights reserved. +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" + +#include + +#include "opencv2/core/hal/intrin.hpp" + +#include "canny.hpp" +#include "canny.simd.hpp" +#include "canny.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL=AVX512_SKX,...,BASELINE based on CMakeLists.txt content + +namespace cv { + +int canny_simd_width() +{ + CV_CPU_DISPATCH(canny_simd_width, (), CV_CPU_DISPATCH_MODES_ALL); +} + +void canny_calc_magnitude(const short* _dx, const short* _dy, int* _mag_n, + int width, bool L2gradient) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(canny_calc_magnitude, (_dx, _dy, _mag_n, width, L2gradient), + CV_CPU_DISPATCH_MODES_ALL); +} + +void canny_nms_row(const int* _mag_a, const int* _mag_p, const int* _mag_n, + const short* _dx, const short* _dy, uchar* _pmap, + int width, int low, int high, std::deque& stack) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(canny_nms_row, (_mag_a, _mag_p, _mag_n, _dx, _dy, _pmap, width, low, high, stack), + CV_CPU_DISPATCH_MODES_ALL); +} + +void canny_finalize_row(const uchar* pmap, uchar* pdst, int width) +{ + CV_INSTRUMENT_REGION(); + CV_CPU_DISPATCH(canny_finalize_row, (pmap, pdst, width), + CV_CPU_DISPATCH_MODES_ALL); +} + +} // namespace cv diff --git a/modules/imgproc/src/canny.hpp b/modules/imgproc/src/canny.hpp new file mode 100644 index 0000000000..63109e8b8a --- /dev/null +++ b/modules/imgproc/src/canny.hpp @@ -0,0 +1,24 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. + +#ifndef OPENCV_IMGPROC_CANNY_HPP +#define OPENCV_IMGPROC_CANNY_HPP + +#include + +namespace cv { + +int canny_simd_width(); + +void canny_calc_magnitude(const short* _dx, const short* _dy, int* _mag_n, + int width, bool L2gradient); +void canny_nms_row(const int* _mag_a, const int* _mag_p, const int* _mag_n, + const short* _dx, const short* _dy, uchar* _pmap, + int width, int low, int high, std::deque& stack); +void canny_finalize_row(const uchar* pmap, uchar* pdst, int width); + +} // namespace cv + +#endif // OPENCV_IMGPROC_CANNY_HPP diff --git a/modules/imgproc/src/canny.simd.hpp b/modules/imgproc/src/canny.simd.hpp new file mode 100644 index 0000000000..8512c4a16b --- /dev/null +++ b/modules/imgproc/src/canny.simd.hpp @@ -0,0 +1,285 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000, Intel Corporation, all rights reserved. +// Copyright (C) 2014, Itseez Inc., all rights reserved. +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include "opencv2/core/hal/intrin.hpp" +#include + +namespace cv { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +int canny_simd_width(); + +void canny_calc_magnitude(const short* _dx, const short* _dy, int* _mag_n, + int width, bool L2gradient); + +void canny_nms_row(const int* _mag_a, const int* _mag_p, const int* _mag_n, + const short* _dx, const short* _dy, uchar* _pmap, + int width, int low, int high, std::deque& stack); + +void canny_finalize_row(const uchar* pmap, uchar* pdst, int width); + +#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +#define CANNY_PUSH(map, stack) *map = 2, stack.push_back(map) + +#define CANNY_CHECK(m, high, map, stack) \ + if (m > high) \ + CANNY_PUSH(map, stack); \ + else \ + *map = 0 + +int canny_simd_width() +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + return VTraits::vlanes(); +#else + return 1; +#endif +} + +// stage 1: magnitude of the gradient for one row (L1 = |dx|+|dy|, L2 = dx*dx+dy*dy). +// width == src.cols * cn. +void canny_calc_magnitude(const short* _dx, const short* _dy, int* _mag_n, + int width, bool L2gradient) +{ + int j = 0; + if (L2gradient) + { +#if (CV_SIMD || CV_SIMD_SCALABLE) + for ( ; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_int16 v_dx = vx_load((const short*)(_dx + j)); + v_int16 v_dy = vx_load((const short*)(_dy + j)); + + v_int32 v_dxp_low, v_dxp_high; + v_int32 v_dyp_low, v_dyp_high; + v_expand(v_dx, v_dxp_low, v_dxp_high); + v_expand(v_dy, v_dyp_low, v_dyp_high); + + v_store_aligned((int *)(_mag_n + j), v_add(v_mul(v_dxp_low, v_dxp_low), v_mul(v_dyp_low, v_dyp_low))); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_mul(v_dxp_high, v_dxp_high), v_mul(v_dyp_high, v_dyp_high))); + } +#endif + for ( ; j < width; ++j) + _mag_n[j] = int(_dx[j])*_dx[j] + int(_dy[j])*_dy[j]; + } + else + { +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_int16 v_dx = vx_load((const short *)(_dx + j)); + v_int16 v_dy = vx_load((const short *)(_dy + j)); + + v_dx = v_reinterpret_as_s16(v_abs(v_dx)); + v_dy = v_reinterpret_as_s16(v_abs(v_dy)); + + v_int32 v_dx_ml, v_dy_ml, v_dx_mh, v_dy_mh; + v_expand(v_dx, v_dx_ml, v_dx_mh); + v_expand(v_dy, v_dy_ml, v_dy_mh); + + v_store_aligned((int *)(_mag_n + j), v_add(v_dx_ml, v_dy_ml)); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_dx_mh, v_dy_mh)); + } +#endif + for ( ; j < width; ++j) + _mag_n[j] = std::abs(int(_dx[j])) + std::abs(int(_dy[j])); + } +} + +// stage 2: non-maxima suppression + double thresholding for one row. +// Fills _pmap with: 0 (maybe edge), 1 (not edge), 2 (strong edge). Strong edges +// are pushed onto `stack` for the subsequent hysteresis pass. +void canny_nms_row(const int* _mag_a, const int* _mag_p, const int* _mag_n, + const short* _dx, const short* _dy, uchar* _pmap, + int width, int low, int high, std::deque& stack) +{ + const int TG22 = 13573; + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + // smask[i] selects, after a v_scan_forward hit at lane l, all lanes >= l so + // that the already-handled lanes are cleared from the candidate mask. + schar smask[2 * VTraits::max_nlanes]; + for (int i = 0; i < VTraits::vlanes(); ++i) + { + smask[i] = 0; + smask[i + VTraits::vlanes()] = (schar)-1; + } + + const v_int32 v_low = vx_setall_s32(low); + const v_int8 v_one = vx_setall_s8(1); + + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_store_aligned((signed char*)(_pmap + j), v_one); + v_int8 v_cmp = v_pack(v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j)), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + VTraits::vlanes())), v_low)), + v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j + 2 * VTraits::vlanes())), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + 3 * VTraits::vlanes())), v_low))); + while (v_check_any(v_cmp)) + { + int l = v_scan_forward(v_cmp); + v_cmp = v_and(v_cmp, vx_load(smask + VTraits::vlanes() - 1 - l)); + int k = j + l; + + int m = _mag_a[k]; + short xs = _dx[k]; + short ys = _dy[k]; + int x = (int)std::abs(xs); + int y = (int)std::abs(ys) << 15; + + int tg22x = x * TG22; + + if (y < tg22x) + { + if (m > _mag_a[k - 1] && m >= _mag_a[k + 1]) + { + CANNY_CHECK(m, high, (_pmap+k), stack); + } + } + else + { + int tg67x = tg22x + (x << 16); + if (y > tg67x) + { + if (m > _mag_p[k] && m >= _mag_n[k]) + { + CANNY_CHECK(m, high, (_pmap+k), stack); + } + } + else + { + int s = (xs ^ ys) < 0 ? -1 : 1; + if(m > _mag_p[k - s] && m > _mag_n[k + s]) + { + CANNY_CHECK(m, high, (_pmap+k), stack); + } + } + } + } + } +#endif + for (; j < width; j++) + { + int m = _mag_a[j]; + + if (m > low) + { + short xs = _dx[j]; + short ys = _dy[j]; + int x = (int)std::abs(xs); + int y = (int)std::abs(ys) << 15; + + int tg22x = x * TG22; + + if (y < tg22x) + { + if (m > _mag_a[j - 1] && m >= _mag_a[j + 1]) + { + CANNY_CHECK(m, high, (_pmap+j), stack); + continue; + } + } + else + { + int tg67x = tg22x + (x << 16); + if (y > tg67x) + { + if (m > _mag_p[j] && m >= _mag_n[j]) + { + CANNY_CHECK(m, high, (_pmap+j), stack); + continue; + } + } + else + { + int s = (xs ^ ys) < 0 ? -1 : 1; + if(m > _mag_p[j - s] && m > _mag_n[j + s]) + { + CANNY_CHECK(m, high, (_pmap+j), stack); + continue; + } + } + } + } + _pmap[j] = 1; + } +} + +// final pass: turn the edge map (value 2 == edge) into a 0/255 output row. +void canny_finalize_row(const uchar* pmap, uchar* pdst, int width) +{ + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + { + const v_uint8 v_zero = vx_setzero_u8(); + const v_uint8 v_ff = v_not(v_zero); + const v_uint8 v_two = vx_setall_u8(2); + + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_uint8 v_pmap = vx_load_aligned((const unsigned char*)(pmap + j)); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); + v_store((pdst + j), v_pmap); + } + + if (j <= width - VTraits::vlanes()/2) + { + v_uint8 v_pmap = vx_load_low((const unsigned char*)(pmap + j)); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); + v_store_low((pdst + j), v_pmap); + j += VTraits::vlanes()/2; + } + } +#endif + for (; j < width; j++) + { + pdst[j] = (uchar)-(pmap[j] >> 1); + } +} + +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +} // namespace cv diff --git a/modules/imgproc/src/corner.cpp b/modules/imgproc/src/corner.cpp index 2ed6fe074f..1b9ac78019 100644 --- a/modules/imgproc/src/corner.cpp +++ b/modules/imgproc/src/corner.cpp @@ -255,25 +255,7 @@ cornerEigenValsVecs( const Mat& src, Mat& eigenv, int block_size, CV_Assert( src.type() == CV_8UC1 || src.type() == CV_32FC1 ); Mat Dx, Dy; - // TODO(follow-up PR): route CV_32F gradients through the fused spatialGradient once its - // CV_32F/ksize=5 fast paths land. Disabled for now so this stays on the tuned Sobel path. -#if 0 - if( aperture_size == 3 || aperture_size == 5 ) - { - spatialGradient( src, Dx, Dy, aperture_size, borderType, CV_32F, scale ); - } - else -#endif - if( aperture_size > 0 ) - { - Sobel( src, Dx, CV_32F, 1, 0, aperture_size, scale, 0, borderType ); - Sobel( src, Dy, CV_32F, 0, 1, aperture_size, scale, 0, borderType ); - } - else - { - Scharr( src, Dx, CV_32F, 1, 0, scale, 0, borderType ); - Scharr( src, Dy, CV_32F, 0, 1, scale, 0, borderType ); - } + spatialGradient( src, Dx, Dy, aperture_size, borderType, CV_32F, scale ); Size size = src.size(); Mat cov( size, CV_32FC3 ); @@ -376,18 +358,7 @@ static bool extractCovData(InputArray _src, UMat & Dx, UMat & Dy, int depth, return k.run(2, globalsize, localsize, false); } else - { - if (aperture_size > 0) - { - Sobel(_src, Dx, CV_32F, 1, 0, aperture_size, scale, 0, borderType); - Sobel(_src, Dy, CV_32F, 0, 1, aperture_size, scale, 0, borderType); - } - else - { - Scharr(_src, Dx, CV_32F, 1, 0, scale, 0, borderType); - Scharr(_src, Dy, CV_32F, 0, 1, scale, 0, borderType); - } - } + spatialGradient(_src, Dx, Dy, aperture_size, borderType, CV_32F, scale); return true; } diff --git a/modules/imgproc/src/generalized_hough.cpp b/modules/imgproc/src/generalized_hough.cpp index 51e61c6b3d..10a953d8eb 100644 --- a/modules/imgproc/src/generalized_hough.cpp +++ b/modules/imgproc/src/generalized_hough.cpp @@ -116,13 +116,7 @@ namespace CV_Assert( cannyLowThresh_ > 0 && cannyLowThresh_ < cannyHighThresh_ ); Canny(src, edges, cannyLowThresh_, cannyHighThresh_); - // TODO(follow-up PR): use the fused spatialGradient once its CV_32F fast path lands. -#if 0 spatialGradient(src, dx, dy, 3, BORDER_DEFAULT, CV_32F); -#else - Sobel(src, dx, CV_32F, 1, 0); - Sobel(src, dy, CV_32F, 0, 1); -#endif } void GeneralizedHoughBase::setTemplateImpl(InputArray templ, Point templCenter) diff --git a/modules/imgproc/src/intelligent_scissors.cpp b/modules/imgproc/src/intelligent_scissors.cpp index b2cb2af5d2..b6d3b0d426 100644 --- a/modules/imgproc/src/intelligent_scissors.cpp +++ b/modules/imgproc/src/intelligent_scissors.cpp @@ -145,18 +145,7 @@ struct IntelligentScissorsMB::Impl if (!Ix_.empty()) return; initGrayscale_(image); - // TODO(follow-up PR): use the fused spatialGradient once its CV_32F/ksize=5 fast paths land. -#if 0 - if (sobelKernelSize == 3 || sobelKernelSize == 5) - { - spatialGradient(grayscale_, Ix_, Iy_, sobelKernelSize, BORDER_DEFAULT, CV_32F); - } - else -#endif - { - Sobel(grayscale_, Ix_, CV_32FC1, 1, 0, sobelKernelSize); - Sobel(grayscale_, Iy_, CV_32FC1, 0, 1, sobelKernelSize); - } + spatialGradient(grayscale_, Ix_, Iy_, sobelKernelSize, BORDER_DEFAULT, CV_32F); } Mat image_magnitude_; void initImageMagnitude_(InputArray image) diff --git a/modules/imgproc/src/spatialgradient.cpp b/modules/imgproc/src/spatialgradient.cpp index 6f4a9f09be..19c57e850a 100644 --- a/modules/imgproc/src/spatialgradient.cpp +++ b/modules/imgproc/src/spatialgradient.cpp @@ -12,6 +12,7 @@ // // Copyright (C) 2000-2008, Intel Corporation, all rights reserved. // Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. // Third party copyrights are property of their respective owners. // // Redistribution and use in source and binary forms, with or without modification, @@ -43,329 +44,141 @@ #include "precomp.hpp" #include "opencv2/core/hal/intrin.hpp" +#include "spatialgradient.simd.hpp" +#include "spatialgradient.simd_declarations.hpp" // defines CV_CPU_DISPATCH_MODES_ALL based on CMakeLists.txt content + namespace cv { -/* NOTE: - * - * Sobel-x: -1 0 1 - * -2 0 2 - * -1 0 1 - * - * Sobel-y: -1 -2 -1 - * 0 0 0 - * 1 2 1 - */ -#if (CV_SIMD || CV_SIMD_SCALABLE) -template -static inline void spatialGradientKernel_vec( T& vx, T& vy, - const T& v00, const T& v01, const T& v02, - const T& v10, const T& v12, - const T& v20, const T& v21, const T& v22 ) +// Baseline dispatchers: pick the best CPU variant of each fused kernel at runtime. +static void dispatchSep3x3_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType) { - // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) - // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) - T tmp_add = v_sub(v22, v00), - tmp_sub = v_sub(v02, v20), - tmp_x = v_sub(v12, v10), - tmp_y = v_sub(v21, v01); - - vx = v_add(v_add(v_add(tmp_add, tmp_sub), tmp_x), tmp_x); - vy = v_add(v_add(v_sub(tmp_add, tmp_sub), tmp_y), tmp_y); -} -#endif - -template -static inline void spatialGradientKernel( T& vx, T& vy, - const T& v00, const T& v01, const T& v02, - const T& v10, const T& v12, - const T& v20, const T& v21, const T& v22 ) -{ - // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) - // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) - T tmp_add = v22 - v00, - tmp_sub = v02 - v20, - tmp_x = v12 - v10, - tmp_y = v21 - v01; - - vx = tmp_add + tmp_sub + tmp_x + tmp_x; - vy = tmp_add - tmp_sub + tmp_y + tmp_y; + CV_CPU_DISPATCH(spatialGradientSep3x3_16s, + (src, src_step, srcRows, srcCols, rowStart, rowEnd, dx, dy, dst_step, borderType), + CV_CPU_DISPATCH_MODES_ALL); } -#if defined(__GNUC__) || defined(__clang__) -# define CV_SPATIALGRAD_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) -# define CV_SPATIALGRAD_NOINLINE __declspec(noinline) -#else -# define CV_SPATIALGRAD_NOINLINE -#endif +static void dispatchSep5x5_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType) +{ + CV_CPU_DISPATCH(spatialGradientSep5x5_16s, + (src, src_step, srcRows, srcCols, rowStart, rowEnd, dx, dy, dst_step, borderType), + CV_CPU_DISPATCH_MODES_ALL); +} -// Fused single-pass 3x3 Sobel over the whole image: 8-bit input, CV_16S output, -// unit scale, BORDER_REFLECT_101 / BORDER_REPLICATE. Kept in its own (non-inlined) -// function so the hot vectorized loop is codegen'd in isolation from the public -// entry point's dispatch/fallback logic. Folding both into one frame perturbs loop -// alignment and costs ~10% on some microarchitectures (observed on AMD Zen5/Turin). -static CV_SPATIALGRAD_NOINLINE -void spatialGradientFused3x3_8u16s( Mat src, OutputArray _dx, OutputArray _dy, int bt ); +static void dispatchSep3x3_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType) +{ + CV_CPU_DISPATCH(spatialGradientSep3x3_32f, + (src, src_step, srcRows, srcCols, rowStart, rowEnd, dx, dy, dst_step, scale, borderType), + CV_CPU_DISPATCH_MODES_ALL); +} + +static void dispatchSep5x5_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType) +{ + CV_CPU_DISPATCH(spatialGradientSep5x5_32f, + (src, src_step, srcRows, srcCols, rowStart, rowEnd, dx, dy, dst_step, scale, borderType), + CV_CPU_DISPATCH_MODES_ALL); +} + +// 3x3 CV_8U -> CV_16S via cv_hal_spatialGradient when a platform HAL provides it +// (e.g. RISC-V RVV). Returns false if the HAL is not implemented so the caller +// falls back to the fused separable kernel. +static bool spatialGradient3x3_tryHal( const Mat& src, OutputArray _dx, OutputArray _dy, int bt ) +{ + _dx.create( src.size(), CV_16SC1 ); + _dy.create( src.size(), CV_16SC1 ); + Mat dx = _dx.getMat(), dy = _dy.getMat(); + + int res = cv_hal_spatialGradient( src.data, src.step, + dx.ptr(), dx.step, + dy.ptr(), dy.step, + src.cols, src.rows, 3, bt ); + if ( res == CV_HAL_ERROR_OK ) + return true; + if ( res != CV_HAL_ERROR_NOT_IMPLEMENTED ) + CV_Error_( cv::Error::StsInternal, + ("HAL implementation spatialGradient ==> cv_hal_spatialGradient returned %d (0x%08x)", res, res) ); + return false; +} void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, int ksize, int borderType, int ddepth, double scale ) { CV_INSTRUMENT_REGION(); - // Prepare InputArray src Mat src = _src.getMat(); CV_Assert( !src.empty() ); - CV_Assert( ksize == -1 || ksize == 3 || ksize == 5 || ksize == 7 ); + CV_Assert( ksize == -1 || ksize == 1 || ksize == 3 || ksize == 5 || ksize == 7 ); CV_Assert( ddepth == CV_16S || ddepth == CV_32F ); - // Fused single-pass 3x3 Sobel fast path (existing vectorized kernel): 8-bit source, - // CV_16S output, unit scale, whole-image (non-ROI), BORDER_REFLECT_101/REPLICATE. - // NOTE: fused fast paths for CV_32F output and ksize == 5 (plus ROI-aware slicing and - // other border types) are added in a follow-up PR; until then those cases are computed - // with the two equivalent cv::Sobel() passes below. Size wholeSize; Point ofs; src.locateROI( wholeSize, ofs ); const bool entireParent = ( ofs.x == 0 && ofs.y == 0 && src.cols == wholeSize.width && src.rows == wholeSize.height ); const bool isolated = ( borderType & BORDER_ISOLATED ) != 0; - const int bt = borderType & ~BORDER_ISOLATED; + const int bt = borderType & ~BORDER_ISOLATED; - const bool fastPath = ( ksize == 3 && ddepth == CV_16S && scale == 1.0 + // (1) Optional platform HAL for whole-image 3x3 CV_8U->CV_16S at unit scale. + const bool halEligible = ( ksize == 3 && ddepth == CV_16S && scale == 1.0 && src.type() == CV_8UC1 && entireParent && !isolated - && ( bt == BORDER_DEFAULT || bt == BORDER_REPLICATE ) ); + && ( bt == BORDER_REFLECT_101 || bt == BORDER_REPLICATE ) ); + if ( halEligible && spatialGradient3x3_tryHal( src, _dx, _dy, bt ) ) + return; - if ( !fastPath ) + // (2) Fused separable single-pass path (CV_8U source): 3x3/5x5, CV_16S (unit scale) and + // CV_32F (any scale), reflect/replicate borders and full-width row-range ROIs. Each + // source sample is read once and shared between dx and dy; output is bit-identical to + // two cv::Sobel() passes. CV_16S fuses only at unit scale (the kernel emits the + // unscaled int16 result); a scaled int16 request falls back to keep cv::Sobel rounding. + const bool fusableBorder = ( bt == BORDER_REPLICATE || bt == BORDER_REFLECT + || bt == BORDER_REFLECT_101 ); + const bool fullWidthRoi = ( ofs.x == 0 && src.cols == wholeSize.width ); + const bool scaleOk = ( ddepth == CV_32F ) || ( scale == 1.0 ); + + if ( (ksize == 3 || ksize == 5) && src.type() == CV_8UC1 && fusableBorder && fullWidthRoi && !isolated && scaleOk ) { - Sobel( _src, _dx, ddepth, 1, 0, ksize, scale, 0, borderType ); - Sobel( _src, _dy, ddepth, 0, 1, ksize, scale, 0, borderType ); + _dx.create( src.size(), CV_MAKETYPE(ddepth, 1) ); + _dy.create( src.size(), CV_MAKETYPE(ddepth, 1) ); + Mat dx = _dx.getMat(), dy = _dy.getMat(); + + const uchar* base = src.ptr(0) - (size_t)ofs.y * src.step; + const int parentRows = wholeSize.height; + const int rowStart = ofs.y; + const int rowEnd = ofs.y + src.rows; + + if ( ddepth == CV_32F ) + { + const float fscale = (float)scale; + if ( ksize == 3 ) + dispatchSep3x3_32f( base, src.step, parentRows, src.cols, rowStart, rowEnd, + dx.ptr(), dy.ptr(), dx.step1(), fscale, bt ); + else + dispatchSep5x5_32f( base, src.step, parentRows, src.cols, rowStart, rowEnd, + dx.ptr(), dy.ptr(), dx.step1(), fscale, bt ); + } + else + { + if ( ksize == 3 ) + dispatchSep3x3_16s( base, src.step, parentRows, src.cols, rowStart, rowEnd, + dx.ptr(), dy.ptr(), dx.step1(), bt ); + else + dispatchSep5x5_16s( base, src.step, parentRows, src.cols, rowStart, rowEnd, + dx.ptr(), dy.ptr(), dx.step1(), bt ); + } return; } - spatialGradientFused3x3_8u16s( src, _dx, _dy, bt ); -} - -static CV_SPATIALGRAD_NOINLINE -void spatialGradientFused3x3_8u16s( Mat src, OutputArray _dx, OutputArray _dy, int bt ) -{ - // Prepare OutputArrays dx, dy - _dx.create( src.size(), CV_16SC1 ); - _dy.create( src.size(), CV_16SC1 ); - Mat dx = _dx.getMat(), - dy = _dy.getMat(); - - CALL_HAL(spatialGradient, cv_hal_spatialGradient, - src.data, src.step, - dx.ptr(), dx.step, - dy.ptr(), dy.step, - src.cols, src.rows, - 3, bt); - - // Get dimensions - const int H = src.rows, - W = src.cols; - - // Row, column indices - int i = 0, - j = 0; - - // Handle border types - int i_top = 0, // Case for H == 1 && W == 1 && BORDER_REPLICATE - i_bottom = H - 1, - j_offl = 0, // j offset from 0th pixel to reach -1st pixel - j_offr = 0; // j offset from W-1th pixel to reach Wth pixel - - if ( bt == BORDER_DEFAULT ) // Equiv. to BORDER_REFLECT_101 - { - if ( H > 1 ) - { - i_top = 1; - i_bottom = H - 2; - } - if ( W > 1 ) - { - j_offl = 1; - j_offr = -1; - } - } - - int i_start = 0; - int j_start = 0; -#if (CV_SIMD || CV_SIMD_SCALABLE) - // Characters in variable names have the following meanings: - // u: unsigned char - // s: signed int - // - // [row][column] - // m: offset -1 - // n: offset 0 - // p: offset 1 - // Example: umn is offset -1 in row and offset 0 in column - for ( i = 0; i < H - 1; i += 2 ) - { - uchar *p_src = src.ptr(i == 0 ? i_top : i - 1); - uchar *c_src = src.ptr(i); - uchar *n_src = src.ptr(i+1); - uchar *m_src = src.ptr(i == H - 2 ? i_bottom : i + 2); - - short *c_dx = dx.ptr(i); - short *c_dy = dy.ptr(i); - short *n_dx = dx.ptr(i+1); - short *n_dy = dy.ptr(i+1); - - // Process rest of columns 16-column chunks at a time - for ( j = 1; j < W - VTraits::vlanes(); j += VTraits::vlanes()) - { - // Load top row for 3x3 Sobel filter - v_uint8 v_um = vx_load(&p_src[j-1]); - v_uint8 v_un = vx_load(&p_src[j]); - v_uint8 v_up = vx_load(&p_src[j+1]); - v_uint16 v_um1, v_um2, v_un1, v_un2, v_up1, v_up2; - v_expand(v_um, v_um1, v_um2); - v_expand(v_un, v_un1, v_un2); - v_expand(v_up, v_up1, v_up2); - v_int16 v_s1m1 = v_reinterpret_as_s16(v_um1); - v_int16 v_s1m2 = v_reinterpret_as_s16(v_um2); - v_int16 v_s1n1 = v_reinterpret_as_s16(v_un1); - v_int16 v_s1n2 = v_reinterpret_as_s16(v_un2); - v_int16 v_s1p1 = v_reinterpret_as_s16(v_up1); - v_int16 v_s1p2 = v_reinterpret_as_s16(v_up2); - - // Load second row for 3x3 Sobel filter - v_um = vx_load(&c_src[j-1]); - v_un = vx_load(&c_src[j]); - v_up = vx_load(&c_src[j+1]); - v_expand(v_um, v_um1, v_um2); - v_expand(v_un, v_un1, v_un2); - v_expand(v_up, v_up1, v_up2); - v_int16 v_s2m1 = v_reinterpret_as_s16(v_um1); - v_int16 v_s2m2 = v_reinterpret_as_s16(v_um2); - v_int16 v_s2n1 = v_reinterpret_as_s16(v_un1); - v_int16 v_s2n2 = v_reinterpret_as_s16(v_un2); - v_int16 v_s2p1 = v_reinterpret_as_s16(v_up1); - v_int16 v_s2p2 = v_reinterpret_as_s16(v_up2); - - // Load third row for 3x3 Sobel filter - v_um = vx_load(&n_src[j-1]); - v_un = vx_load(&n_src[j]); - v_up = vx_load(&n_src[j+1]); - v_expand(v_um, v_um1, v_um2); - v_expand(v_un, v_un1, v_un2); - v_expand(v_up, v_up1, v_up2); - v_int16 v_s3m1 = v_reinterpret_as_s16(v_um1); - v_int16 v_s3m2 = v_reinterpret_as_s16(v_um2); - v_int16 v_s3n1 = v_reinterpret_as_s16(v_un1); - v_int16 v_s3n2 = v_reinterpret_as_s16(v_un2); - v_int16 v_s3p1 = v_reinterpret_as_s16(v_up1); - v_int16 v_s3p2 = v_reinterpret_as_s16(v_up2); - - // dx & dy for rows 1, 2, 3 - v_int16 v_sdx1, v_sdy1; - spatialGradientKernel_vec( v_sdx1, v_sdy1, - v_s1m1, v_s1n1, v_s1p1, - v_s2m1, v_s2p1, - v_s3m1, v_s3n1, v_s3p1 ); - - v_int16 v_sdx2, v_sdy2; - spatialGradientKernel_vec( v_sdx2, v_sdy2, - v_s1m2, v_s1n2, v_s1p2, - v_s2m2, v_s2p2, - v_s3m2, v_s3n2, v_s3p2 ); - - // Store - v_store(&c_dx[j], v_sdx1); - v_store(&c_dx[j+VTraits::vlanes()], v_sdx2); - v_store(&c_dy[j], v_sdy1); - v_store(&c_dy[j+VTraits::vlanes()], v_sdy2); - - // Load fourth row for 3x3 Sobel filter - v_um = vx_load(&m_src[j-1]); - v_un = vx_load(&m_src[j]); - v_up = vx_load(&m_src[j+1]); - v_expand(v_um, v_um1, v_um2); - v_expand(v_un, v_un1, v_un2); - v_expand(v_up, v_up1, v_up2); - v_int16 v_s4m1 = v_reinterpret_as_s16(v_um1); - v_int16 v_s4m2 = v_reinterpret_as_s16(v_um2); - v_int16 v_s4n1 = v_reinterpret_as_s16(v_un1); - v_int16 v_s4n2 = v_reinterpret_as_s16(v_un2); - v_int16 v_s4p1 = v_reinterpret_as_s16(v_up1); - v_int16 v_s4p2 = v_reinterpret_as_s16(v_up2); - - // dx & dy for rows 2, 3, 4 - spatialGradientKernel_vec( v_sdx1, v_sdy1, - v_s2m1, v_s2n1, v_s2p1, - v_s3m1, v_s3p1, - v_s4m1, v_s4n1, v_s4p1 ); - - spatialGradientKernel_vec( v_sdx2, v_sdy2, - v_s2m2, v_s2n2, v_s2p2, - v_s3m2, v_s3p2, - v_s4m2, v_s4n2, v_s4p2 ); - - // Store - v_store(&n_dx[j], v_sdx1); - v_store(&n_dx[j+VTraits::vlanes()], v_sdx2); - v_store(&n_dy[j], v_sdy1); - v_store(&n_dy[j+VTraits::vlanes()], v_sdy2); - } - } - i_start = i; - j_start = j; -#endif - int j_p, j_n; - uchar v00, v01, v02, v10, v11, v12, v20, v21, v22; - for ( i = 0; i < H; i++ ) - { - uchar *p_src = src.ptr(i == 0 ? i_top : i - 1); - uchar *c_src = src.ptr(i); - uchar *n_src = src.ptr(i == H - 1 ? i_bottom : i + 1); - - short *c_dx = dx.ptr(i); - short *c_dy = dy.ptr(i); - - // Process left-most column - j = 0; - j_p = j + j_offl; - j_n = 1; - if ( j_n >= W ) j_n = j + j_offr; - v00 = p_src[j_p]; v01 = p_src[j]; v02 = p_src[j_n]; - v10 = c_src[j_p]; v11 = c_src[j]; v12 = c_src[j_n]; - v20 = n_src[j_p]; v21 = n_src[j]; v22 = n_src[j_n]; - spatialGradientKernel( c_dx[0], c_dy[0], v00, v01, v02, v10, - v12, v20, v21, v22 ); - v00 = v01; v10 = v11; v20 = v21; - v01 = v02; v11 = v12; v21 = v22; - - // Process middle columns - j = i >= i_start ? 1 : j_start; - j_p = j - 1; - v00 = p_src[j_p]; v01 = p_src[j]; - v10 = c_src[j_p]; v11 = c_src[j]; - v20 = n_src[j_p]; v21 = n_src[j]; - - for ( ; j < W - 1; j++ ) - { - // Get values for next column - j_n = j + 1; v02 = p_src[j_n]; v12 = c_src[j_n]; v22 = n_src[j_n]; - spatialGradientKernel( c_dx[j], c_dy[j], v00, v01, v02, v10, - v12, v20, v21, v22 ); - - // Move values back one column for next iteration - v00 = v01; v10 = v11; v20 = v21; - v01 = v02; v11 = v12; v21 = v22; - } - - // Process right-most column - if ( j < W ) - { - j_n = j + j_offr; v02 = p_src[j_n]; v12 = c_src[j_n]; v22 = n_src[j_n]; - spatialGradientKernel( c_dx[j], c_dy[j], v00, v01, v02, v10, - v12, v20, v21, v22 ); - } - } - + // (3) General fallback: two separable cv::Sobel passes. + Sobel( _src, _dx, ddepth, 1, 0, ksize, scale, 0, borderType ); + Sobel( _src, _dy, ddepth, 0, 1, ksize, scale, 0, borderType ); } } diff --git a/modules/imgproc/src/spatialgradient.simd.hpp b/modules/imgproc/src/spatialgradient.simd.hpp new file mode 100644 index 0000000000..f580510fdc --- /dev/null +++ b/modules/imgproc/src/spatialgradient.simd.hpp @@ -0,0 +1,524 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html +// Copyright (C) 2026, Advanced Micro Devices, Inc., all rights reserved. + +#include "precomp.hpp" +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +// Fused single-pass Sobel gradient entry points (per-CPU-variant dispatched). +// Row bounds [rowStart, rowEnd) are absolute in the parent buffer so a full-width +// ROI reads the real neighbouring rows at its top/bottom edges (matches cv::Sobel). +// dst_step is in elements. Output is bit-identical to two cv::Sobel() passes. +void spatialGradientSep3x3_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType); +void spatialGradientSep5x5_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType); +void spatialGradientSep3x3_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType); +void spatialGradientSep5x5_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType); + +#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +// Output store for the fused kernels. +// - short: store the int16 result (scale is always 1 here). +// - float: widen to int32, convert to float, multiply by scale, store. +template struct SobelSepStore; + +template<> struct SobelSepStore +{ + explicit SobelSepStore(double) {} +#if (CV_SIMD || CV_SIMD_SCALABLE) + inline void store(short* p, int j, const v_int16& v) const { v_store(p + j, v); } +#endif + inline void scalar(short* p, int j, int v) const { p[j] = (short)v; } +}; + +template<> struct SobelSepStore +{ + float s; + explicit SobelSepStore(double scale) : s((float)scale) {} +#if (CV_SIMD || CV_SIMD_SCALABLE) + inline void store(float* p, int j, const v_int16& v) const + { + v_int32 lo, hi; + v_expand(v, lo, hi); + v_float32 vs = vx_setall_f32(s); + v_store(p + j, v_mul(v_cvt_f32(lo), vs)); + v_store(p + j + VTraits::vlanes(), v_mul(v_cvt_f32(hi), vs)); + } +#endif + inline void scalar(float* p, int j, int v) const { p[j] = s * (float)v; } +}; + +// Fused 3x3 Sobel: +// hd = src[x+1] - src[x-1]; hs = src[x-1] + 2*src[x] + src[x+1] +// dx = hd[y-1] + 2*hd[y] + hd[y+1]; dy = hs[y+1] - hs[y-1] +template +static void sobelSep3x3_( const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + DstT* dx, DstT* dy, size_t dst_step, double scale, int borderType ) +{ + const int width = srcCols; + const int outRows = rowEnd - rowStart; + if (width <= 0 || outRows <= 0) + return; + + const SobelSepStore t_store(scale); + + cv::AutoBuffer _buf((size_t)width * 6); + short* hd[3]; + short* hs[3]; + for (int k = 0; k < 3; ++k) + { + hd[k] = _buf.data() + (size_t)k * 2 * width; + hs[k] = hd[k] + width; + } + + auto cidx = [&](int x) { return borderInterpolate(x, width, borderType); }; + + auto horiz = [&](const uchar* s, short* hdo, short* hso) + { + { + int l = s[cidx(-1)], c = s[0], r = s[cidx(1)]; + hdo[0] = (short)(r - l); + hso[0] = (short)(l + 2 * c + r); + } + + int j = 1; +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j + VTraits::vlanes() <= width - 1; j += VTraits::vlanes()) + { + v_int16 c = v_reinterpret_as_s16(vx_load_expand(s + j)); + v_int16 l = v_reinterpret_as_s16(vx_load_expand(s + j - 1)); + v_int16 r = v_reinterpret_as_s16(vx_load_expand(s + j + 1)); + v_store(hdo + j, v_sub(r, l)); + v_store(hso + j, v_add(v_add(l, r), v_add(c, c))); + } +#endif + for (; j < width - 1; ++j) + { + int l = s[j - 1], c = s[j], r = s[j + 1]; + hdo[j] = (short)(r - l); + hso[j] = (short)(l + 2 * c + r); + } + + if (width > 1) + { + int x = width - 1; + int l = s[x - 1], c = s[x], r = s[cidx(x + 1)]; + hdo[x] = (short)(r - l); + hso[x] = (short)(l + 2 * c + r); + } + }; + + auto combine = [&](const short* hdm, const short* hd0, const short* hdp, + const short* hsm, const short* hsp, + DstT* odx, DstT* ody) + { + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_int16 a = vx_load(hdm + j); + v_int16 b = vx_load(hd0 + j); + v_int16 c = vx_load(hdp + j); + t_store.store(odx, j, v_add(v_add(a, c), v_add(b, b))); + + v_int16 p = vx_load(hsp + j); + v_int16 m = vx_load(hsm + j); + t_store.store(ody, j, v_sub(p, m)); + } +#endif + for (; j < width; ++j) + { + t_store.scalar(odx, j, hdm[j] + 2 * hd0[j] + hdp[j]); + t_store.scalar(ody, j, hsp[j] - hsm[j]); + } + }; + + for (int r = rowStart - 1; r <= rowEnd; ++r) + { + int rc = borderInterpolate(r, srcRows, borderType); + int slot = ((r % 3) + 3) % 3; + horiz(src + (size_t)rc * src_step, hd[slot], hs[slot]); + + int a = r - 1; + if (a >= rowStart && a < rowEnd) + { + int sm = (((a - 1) % 3) + 3) % 3; + int s0 = ((a % 3) + 3) % 3; + int sp = (((a + 1) % 3) + 3) % 3; + combine(hd[sm], hd[s0], hd[sp], hs[sm], hs[sp], + dx + (size_t)(a - rowStart) * dst_step, + dy + (size_t)(a - rowStart) * dst_step); + } + } +} + +// Fused 5x5 Sobel (getSobelKernels ksize=5): smoothing [1,4,6,4,1], derivative [-1,-2,0,2,1]. +// hd = s[x+2] + 2*s[x+1] - 2*s[x-1] - s[x-2] +// hs = s[x-2] + 4*s[x-1] + 6*s[x] + 4*s[x+1] + s[x+2] +// dx = hd[y-2] + 4*hd[y-1] + 6*hd[y] + 4*hd[y+1] + hd[y+2] +// dy = -hs[y-2] - 2*hs[y-1] + 2*hs[y+1] + hs[y+2] +template +static void sobelSep5x5_( const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + DstT* dx, DstT* dy, size_t dst_step, double scale, int borderType ) +{ + const int width = srcCols; + const int outRows = rowEnd - rowStart; + if (width <= 0 || outRows <= 0) + return; + + const SobelSepStore t_store(scale); + + cv::AutoBuffer _buf((size_t)width * 10); + short* hd[5]; + short* hs[5]; + for (int k = 0; k < 5; ++k) + { + hd[k] = _buf.data() + (size_t)k * 2 * width; + hs[k] = hd[k] + width; + } + + auto cidx = [&](int x) { return borderInterpolate(x, width, borderType); }; + + auto horiz = [&](const uchar* s, short* hdo, short* hso) + { + auto scalarCol = [&](int x) + { + int m2 = s[cidx(x - 2)], m1 = s[cidx(x - 1)], c = s[x]; + int p1 = s[cidx(x + 1)], p2 = s[cidx(x + 2)]; + hdo[x] = (short)(p2 + 2 * p1 - 2 * m1 - m2); + hso[x] = (short)(m2 + 4 * m1 + 6 * c + 4 * p1 + p2); + }; + + int j = 0; + for (; j < 2 && j < width; ++j) + scalarCol(j); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j + VTraits::vlanes() <= width - 2; j += VTraits::vlanes()) + { + v_int16 m2 = v_reinterpret_as_s16(vx_load_expand(s + j - 2)); + v_int16 m1 = v_reinterpret_as_s16(vx_load_expand(s + j - 1)); + v_int16 c = v_reinterpret_as_s16(vx_load_expand(s + j)); + v_int16 p1 = v_reinterpret_as_s16(vx_load_expand(s + j + 1)); + v_int16 p2 = v_reinterpret_as_s16(vx_load_expand(s + j + 2)); + v_store(hdo + j, v_sub(v_sub(v_add(p2, v_add(p1, p1)), v_add(m1, m1)), m2)); + v_int16 c6 = v_add(v_shl<2>(c), v_add(c, c)); + v_store(hso + j, v_add(v_add(v_add(m2, p2), v_shl<2>(v_add(m1, p1))), c6)); + } +#endif + for (; j < width; ++j) + scalarCol(j); + }; + + auto combine = [&](const short* hdm2, const short* hdm1, const short* hd0, + const short* hdp1, const short* hdp2, + const short* hsm2, const short* hsm1, + const short* hsp1, const short* hsp2, + DstT* odx, DstT* ody) + { + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_int16 a = vx_load(hdm2 + j), b = vx_load(hdm1 + j), c = vx_load(hd0 + j); + v_int16 d = vx_load(hdp1 + j), e = vx_load(hdp2 + j); + v_int16 c6 = v_add(v_shl<2>(c), v_add(c, c)); + t_store.store(odx, j, v_add(v_add(v_add(a, e), v_shl<2>(v_add(b, d))), c6)); + + v_int16 g = vx_load(hsm2 + j), h = vx_load(hsm1 + j); + v_int16 p = vx_load(hsp1 + j), q = vx_load(hsp2 + j); + t_store.store(ody, j, v_add(v_sub(q, g), v_shl<1>(v_sub(p, h)))); + } +#endif + for (; j < width; ++j) + { + t_store.scalar(odx, j, hdm2[j] + 4 * hdm1[j] + 6 * hd0[j] + 4 * hdp1[j] + hdp2[j]); + t_store.scalar(ody, j, -hsm2[j] - 2 * hsm1[j] + 2 * hsp1[j] + hsp2[j]); + } + }; + + for (int r = rowStart - 2; r <= rowEnd + 1; ++r) + { + int rc = borderInterpolate(r, srcRows, borderType); + int slot = ((r % 5) + 5) % 5; + horiz(src + (size_t)rc * src_step, hd[slot], hs[slot]); + + int a = r - 2; + if (a >= rowStart && a < rowEnd) + { + int s_m2 = (((a - 2) % 5) + 5) % 5; + int s_m1 = (((a - 1) % 5) + 5) % 5; + int s_0 = ((a % 5) + 5) % 5; + int s_p1 = (((a + 1) % 5) + 5) % 5; + int s_p2 = (((a + 2) % 5) + 5) % 5; + combine(hd[s_m2], hd[s_m1], hd[s_0], hd[s_p1], hd[s_p2], + hs[s_m2], hs[s_m1], hs[s_p1], hs[s_p2], + dx + (size_t)(a - rowStart) * dst_step, + dy + (size_t)(a - rowStart) * dst_step); + } + } +} + +// CV_32F output must match cv::Sobel(..., CV_32F, ..., scale, ...), which folds scale +// into the smoothing kernel before sepFilter2D (not as a final post-multiply). The float +// horizontal pass carries scale so the result is bit-exact with cv::Sobel. +static void sobelSep3x3_f32( const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType ) +{ + const int width = srcCols; + if (width <= 0 || rowEnd <= rowStart) + return; + + const float sk0 = 2.f * scale; + const float sk1 = scale; + const float dk0 = sk0; + const float dk1 = sk1; + + cv::AutoBuffer _buf((size_t)width * 6); + float* hd[3]; + float* hs[3]; + for (int k = 0; k < 3; ++k) + { + hd[k] = _buf.data() + (size_t)k * 2 * width; + hs[k] = hd[k] + width; + } + + auto cidx = [&](int x) { return borderInterpolate(x, width, borderType); }; + + auto horiz = [&](const uchar* s, float* hdo, float* hso) + { + { + float l = (float)s[cidx(-1)], c = (float)s[0], r = (float)s[cidx(1)]; + hdo[0] = r - l; + hso[0] = sk0 * c + sk1 * (l + r); + } + + int j = 1; +#if (CV_SIMD || CV_SIMD_SCALABLE) + v_float32 v_sk0 = vx_setall_f32(sk0); + v_float32 v_sk1 = vx_setall_f32(sk1); + for (; j + VTraits::vlanes() <= width - 1; j += VTraits::vlanes()) + { + v_float32 l = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j - 1))); + v_float32 c = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j))); + v_float32 r = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j + 1))); + v_store(hdo + j, v_sub(r, l)); + v_store(hso + j, v_muladd(v_add(l, r), v_sk1, v_mul(c, v_sk0))); + } +#endif + for (; j < width - 1; ++j) + { + float l = (float)s[j - 1], c = (float)s[j], r = (float)s[j + 1]; + hdo[j] = r - l; + hso[j] = sk0 * c + sk1 * (l + r); + } + + if (width > 1) + { + int x = width - 1; + float l = (float)s[x - 1], c = (float)s[x], r = (float)s[cidx(x + 1)]; + hdo[x] = r - l; + hso[x] = sk0 * c + sk1 * (l + r); + } + }; + + auto combine = [&](const float* hdm, const float* hd0, const float* hdp, + const float* hsm, const float* hsp, + float* odx, float* ody) + { + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + v_float32 v_dk0 = vx_setall_f32(dk0); + v_float32 v_dk1 = vx_setall_f32(dk1); + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_float32 hd0v = vx_load(hd0 + j); + v_store(odx + j, v_muladd(v_add(vx_load(hdm + j), vx_load(hdp + j)), v_dk1, v_mul(hd0v, v_dk0))); + v_store(ody + j, v_sub(vx_load(hsp + j), vx_load(hsm + j))); + } +#endif + for (; j < width; ++j) + { + odx[j] = dk0 * hd0[j] + dk1 * (hdm[j] + hdp[j]); + ody[j] = hsp[j] - hsm[j]; + } + }; + + for (int r = rowStart - 1; r <= rowEnd; ++r) + { + int rc = borderInterpolate(r, srcRows, borderType); + int slot = ((r % 3) + 3) % 3; + horiz(src + (size_t)rc * src_step, hd[slot], hs[slot]); + + int a = r - 1; + if (a >= rowStart && a < rowEnd) + { + int sm = (((a - 1) % 3) + 3) % 3; + int s0 = ((a % 3) + 3) % 3; + int sp = (((a + 1) % 3) + 3) % 3; + combine(hd[sm], hd[s0], hd[sp], hs[sm], hs[sp], + dx + (size_t)(a - rowStart) * dst_step, + dy + (size_t)(a - rowStart) * dst_step); + } + } +} + +static void sobelSep5x5_f32( const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType ) +{ + const int width = srcCols; + if (width <= 0 || rowEnd <= rowStart) + return; + + const float sk0 = 6.f * scale; + const float sk1 = 4.f * scale; + const float sk2 = scale; + const float dk0 = sk0; + const float dk1 = sk1; + const float dk2 = sk2; + + cv::AutoBuffer _buf((size_t)width * 10); + float* hd[5]; + float* hs[5]; + for (int k = 0; k < 5; ++k) + { + hd[k] = _buf.data() + (size_t)k * 2 * width; + hs[k] = hd[k] + width; + } + + auto cidx = [&](int x) { return borderInterpolate(x, width, borderType); }; + + auto horiz = [&](const uchar* s, float* hdo, float* hso) + { + auto scalarCol = [&](int x) + { + float m2 = (float)s[cidx(x - 2)], m1 = (float)s[cidx(x - 1)], c = (float)s[x]; + float p1 = (float)s[cidx(x + 1)], p2 = (float)s[cidx(x + 2)]; + hdo[x] = p2 + 2.f * p1 - 2.f * m1 - m2; + hso[x] = sk2 * (m2 + p2) + sk1 * (m1 + p1) + sk0 * c; + }; + + int j = 0; + for (; j < 2 && j < width; ++j) + scalarCol(j); +#if (CV_SIMD || CV_SIMD_SCALABLE) + v_float32 v_sk0 = vx_setall_f32(sk0); + v_float32 v_sk1 = vx_setall_f32(sk1); + v_float32 v_sk2 = vx_setall_f32(sk2); + for (; j + VTraits::vlanes() <= width - 2; j += VTraits::vlanes()) + { + v_float32 m2 = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j - 2))); + v_float32 m1 = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j - 1))); + v_float32 c = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j))); + v_float32 p1 = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j + 1))); + v_float32 p2 = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(s + j + 2))); + v_store(hdo + j, v_sub(v_sub(v_add(p2, v_add(p1, p1)), v_add(m1, m1)), m2)); + v_store(hso + j, v_muladd(v_add(m2, p2), v_sk2, + v_muladd(v_add(m1, p1), v_sk1, v_mul(c, v_sk0)))); + } +#endif + for (; j < width; ++j) + scalarCol(j); + }; + + auto combine = [&](const float* hdm2, const float* hdm1, const float* hd0, + const float* hdp1, const float* hdp2, + const float* hsm2, const float* hsm1, + const float* hsp1, const float* hsp2, + float* odx, float* ody) + { + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + v_float32 v_dk0 = vx_setall_f32(dk0); + v_float32 v_dk1 = vx_setall_f32(dk1); + v_float32 v_dk2 = vx_setall_f32(dk2); + v_float32 v2 = vx_setall_f32(2.f); + for (; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) + { + v_float32 hd0v = vx_load(hd0 + j); + v_store(odx + j, v_muladd(v_add(vx_load(hdm2 + j), vx_load(hdp2 + j)), v_dk2, + v_muladd(v_add(vx_load(hdm1 + j), vx_load(hdp1 + j)), v_dk1, + v_mul(hd0v, v_dk0)))); + v_store(ody + j, v_add(v_sub(vx_load(hsp2 + j), vx_load(hsm2 + j)), + v_mul(v2, v_sub(vx_load(hsp1 + j), vx_load(hsm1 + j))))); + } +#endif + for (; j < width; ++j) + { + odx[j] = dk0 * hd0[j] + dk1 * (hdm1[j] + hdp1[j]) + dk2 * (hdm2[j] + hdp2[j]); + ody[j] = (hsp2[j] - hsm2[j]) + 2.f * (hsp1[j] - hsm1[j]); + } + }; + + for (int r = rowStart - 2; r <= rowEnd + 1; ++r) + { + int rc = borderInterpolate(r, srcRows, borderType); + int slot = ((r % 5) + 5) % 5; + horiz(src + (size_t)rc * src_step, hd[slot], hs[slot]); + + int a = r - 2; + if (a >= rowStart && a < rowEnd) + { + int s_m2 = (((a - 2) % 5) + 5) % 5; + int s_m1 = (((a - 1) % 5) + 5) % 5; + int s_0 = ((a % 5) + 5) % 5; + int s_p1 = (((a + 1) % 5) + 5) % 5; + int s_p2 = (((a + 2) % 5) + 5) % 5; + combine(hd[s_m2], hd[s_m1], hd[s_0], hd[s_p1], hd[s_p2], + hs[s_m2], hs[s_m1], hs[s_p1], hs[s_p2], + dx + (size_t)(a - rowStart) * dst_step, + dy + (size_t)(a - rowStart) * dst_step); + } + } +} + +void spatialGradientSep3x3_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType) +{ + sobelSep3x3_(src, src_step, srcRows, srcCols, rowStart, rowEnd, + dx, dy, dst_step, 1.0, borderType); +} + +void spatialGradientSep5x5_16s(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + short* dx, short* dy, size_t dst_step, int borderType) +{ + sobelSep5x5_(src, src_step, srcRows, srcCols, rowStart, rowEnd, + dx, dy, dst_step, 1.0, borderType); +} + +void spatialGradientSep3x3_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType) +{ + sobelSep3x3_f32(src, src_step, srcRows, srcCols, rowStart, rowEnd, + dx, dy, dst_step, scale, borderType); +} + +void spatialGradientSep5x5_32f(const uchar* src, size_t src_step, int srcRows, int srcCols, + int rowStart, int rowEnd, + float* dx, float* dy, size_t dst_step, float scale, int borderType) +{ + sobelSep5x5_f32(src, src_step, srcRows, srcCols, rowStart, rowEnd, + dx, dy, dst_step, scale, borderType); +} + +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +} // namespace cv diff --git a/modules/imgproc/test/test_filter.cpp b/modules/imgproc/test/test_filter.cpp index 83301d9584..5b618954e4 100644 --- a/modules/imgproc/test/test_filter.cpp +++ b/modules/imgproc/test/test_filter.cpp @@ -1941,7 +1941,7 @@ TEST(Imgproc_SpatialGradient, fused_accuracy) { Mat parent(200, 160, CV_8UC1); rng.fill(parent, RNG::UNIFORM, 0, 256); - for (int ks : {3, 5}) + for (int ks : {1, 3, 5}) for (int b : {BORDER_DEFAULT, BORDER_REPLICATE, BORDER_REFLECT, BORDER_CONSTANT}) for (int ddepth : {CV_16S, CV_32F}) { @@ -1957,7 +1957,7 @@ TEST(Imgproc_SpatialGradient, fused_accuracy) // invalid aperture sizes must be rejected Mat src(16, 16, CV_8UC1), dx, dy; - EXPECT_ANY_THROW(spatialGradient(src, dx, dy, 1)); + EXPECT_ANY_THROW(spatialGradient(src, dx, dy, 2)); } // Reproduces parallelCanny's per-slice row splitting and checks each slice's