Files
opencv-MIRROR/modules/core/src/sum.simd.hpp
Alexander Smorkalov 0044047782 Merge pull request #25598 from asmorkalov:as/tables_range_check_core
Check range for type-dependant function tables #25598

Address https://github.com/opencv/opencv/issues/24703

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
2024-05-17 10:48:40 +03:00

410 lines
13 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
#include "precomp.hpp"
#include "stat.hpp"
namespace cv {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
SumFunc getSumFunc(int depth);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
template <typename T, typename ST>
struct Sum_SIMD
{
Sum_SIMD(int) {}
int operator () (const T*, const uchar*, ST*, int, int) const
{
return 0;
}
};
#if (CV_SIMD || CV_SIMD_SCALABLE)
#undef REDUCE_PARTIAL_SUMS
#define REDUCE_PARTIAL_SUMS() \
if (cn == 1) \
dst[0] += v_reduce_sum(v_add(v_add(s0, s1), s2)); \
else if (cn == 2) { \
s0 = v_add(v_add(s0, s1), s2); \
dst[0] += v_reduce_sum(v_and(s0, m0)); \
dst[1] += v_reduce_sum(v_and(s0, m1)); \
} else if (cn == 3) { \
dst[0] += v_reduce_sum(v_add(v_add(v_and(s0, m0), v_and(s1, m1)), v_and(s2, m2))); \
dst[1] += v_reduce_sum(v_add(v_add(v_and(s0, m3), v_and(s1, m4)), v_and(s2, m5))); \
dst[2] += v_reduce_sum(v_add(v_add(v_and(s0, m6), v_and(s1, m7)), v_and(s2, m8))); \
} else if (cn == 4) { \
s0 = v_add(v_add(s0, s1), s2); \
dst[0] += v_reduce_sum(v_and(s0, m0)); \
dst[1] += v_reduce_sum(v_and(s0, m1)); \
dst[2] += v_reduce_sum(v_and(s0, m2)); \
dst[3] += v_reduce_sum(v_and(s0, m3)); \
}
template<typename ST>
static void init_maskbuf(ST* maskbuf, int cn, int simd_width)
{
memset(maskbuf, 0, simd_width*9*sizeof(maskbuf[0]));
if (cn == 1)
;
else if (cn == 2)
for (int i = 0; i < simd_width; i += 2) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width] = (ST)-1;
}
else if (cn == 3)
for (int i = 0; i < simd_width*3; i += 3) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width*3] = (ST)-1;
maskbuf[i+2+simd_width*6] = (ST)-1;
}
else if (cn == 4 && simd_width >= 4) {
for (int i = 0; i < simd_width; i += 4) {
maskbuf[i] = (ST)-1;
maskbuf[i+1+simd_width] = (ST)-1;
maskbuf[i+2+simd_width*2] = (ST)-1;
maskbuf[i+3+simd_width*3] = (ST)-1;
}
}
}
#undef DEFINE_SUM_SIMD_8
#define DEFINE_SUM_SIMD_8(T, ST, iST, VecT, load_op) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
if (mask || (cn < 1 || cn > 4)) \
return 0; \
len *= cn; \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*6; x += simd_width*6) { \
auto v0 = load_op(src + x); \
auto v1 = load_op(src + x + simd_width*2); \
auto v2 = load_op(src + x + simd_width*4); \
s0 = v_add(s0, v_expand_low(v0)); \
s1 = v_add(s1, v_expand_high(v0)); \
s2 = v_add(s2, v_expand_low(v1)); \
s0 = v_add(s0, v_expand_high(v1)); \
s1 = v_add(s1, v_expand_low(v2)); \
s2 = v_add(s2, v_expand_high(v2)); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
#undef DEFINE_SUM_SIMD_16
#define DEFINE_SUM_SIMD_16(T, ST, iST, VecT, load_op) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
if (mask || (cn < 1 || cn > 4)) \
return 0; \
len *= cn; \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*3; x += simd_width*3) { \
auto v0 = load_op(src + x); \
auto v1 = load_op(src + x + simd_width); \
auto v2 = load_op(src + x + simd_width*2); \
s0 = v_add(s0, v0); \
s1 = v_add(s1, v1); \
s2 = v_add(s2, v2); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
#undef load_u8_as_s16
#undef load_u16_as_s32
#define load_u8_as_s16(addr) v_reinterpret_as_s16(vx_load_expand(addr))
#define load_u16_as_s32(addr) v_reinterpret_as_s32(vx_load_expand(addr))
DEFINE_SUM_SIMD_8(uchar, int, int, v_int32, load_u8_as_s16)
DEFINE_SUM_SIMD_8(schar, int, int, v_int32, vx_load_expand)
DEFINE_SUM_SIMD_16(ushort, int, int, v_int32, load_u16_as_s32)
DEFINE_SUM_SIMD_16(short, int, int, v_int32, vx_load_expand)
DEFINE_SUM_SIMD_16(hfloat, float, int, v_float32, vx_load_expand)
DEFINE_SUM_SIMD_16(bfloat, float, int, v_float32, vx_load_expand)
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
#undef DEFINE_SUM_SIMD_32
#define DEFINE_SUM_SIMD_32(T, ST, iST, VecT) \
template<> struct Sum_SIMD<T, ST> \
{ \
Sum_SIMD(int cn) \
{ \
init_maskbuf((iST*)maskbuf, cn, VTraits<VecT>::vlanes()); \
} \
int operator ()(const T* src, const uchar* mask, ST* dst, int len, int cn) const \
{ \
int x = 0, simd_width = VTraits<VecT>::vlanes(); \
if (mask || (cn < 1 || cn > 3+(simd_width>=4))) \
return 0; \
len *= cn; \
VecT m0 = vx_load(maskbuf), m1, m2, m3, m4, m5, m6, m7, m8; \
if (cn == 1) { \
m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m0; \
} else { \
m1 = vx_load(maskbuf + simd_width); \
m2 = vx_load(maskbuf + simd_width*2); \
m3 = vx_load(maskbuf + simd_width*3); \
m4 = vx_load(maskbuf + simd_width*4); \
m5 = vx_load(maskbuf + simd_width*5); \
m6 = vx_load(maskbuf + simd_width*6); \
m7 = vx_load(maskbuf + simd_width*7); \
m8 = vx_load(maskbuf + simd_width*8); \
} \
VecT s0 = v_xor(m0, m0), s1 = s0, s2 = s0; \
for (; x <= len - simd_width*6; x += simd_width*6) { \
auto v0 = vx_load(src + x); \
auto v1 = vx_load(src + x + simd_width*2); \
auto v2 = vx_load(src + x + simd_width*4); \
s0 = v_add(s0, v_cvt_f64(v0)); \
s1 = v_add(s1, v_cvt_f64_high(v0)); \
s2 = v_add(s2, v_cvt_f64(v1)); \
s0 = v_add(s0, v_cvt_f64_high(v1)); \
s1 = v_add(s1, v_cvt_f64(v2)); \
s2 = v_add(s2, v_cvt_f64_high(v2)); \
} \
REDUCE_PARTIAL_SUMS(); \
vx_cleanup(); \
return x / cn; \
} \
ST maskbuf[VTraits<VecT>::max_nlanes*9]; \
};
DEFINE_SUM_SIMD_32(int, double, int64, v_float64)
DEFINE_SUM_SIMD_32(float, double, int64, v_float64)
#endif
#endif
template<typename T, typename ST, typename WT=T>
static int sum_(const T* src0, const uchar* mask, ST* dst, int len, int cn )
{
const T* src = src0;
if( !mask )
{
Sum_SIMD<T, ST> vop(cn);
int i0 = vop(src0, mask, dst, len, cn), i = i0, k = cn % 4;
src += i0 * cn;
if( k == 1 )
{
ST s0 = dst[0];
#if CV_ENABLE_UNROLLED
for(; i <= len - 4; i += 4, src += cn*4 )
s0 += (WT)src[0] + (WT)src[cn] + (WT)src[cn*2] + (WT)src[cn*3];
#endif
for( ; i < len; i++, src += cn )
s0 += (WT)src[0];
dst[0] = s0;
}
else if( k == 2 )
{
ST s0 = dst[0], s1 = dst[1];
for( ; i < len; i++, src += cn )
{
s0 += (WT)src[0];
s1 += (WT)src[1];
}
dst[0] = s0;
dst[1] = s1;
}
else if( k == 3 )
{
ST s0 = dst[0], s1 = dst[1], s2 = dst[2];
for( ; i < len; i++, src += cn )
{
s0 += (WT)src[0];
s1 += (WT)src[1];
s2 += (WT)src[2];
}
dst[0] = s0;
dst[1] = s1;
dst[2] = s2;
}
for( ; k < cn; k += 4 )
{
src = src0 + i0*cn + k;
ST s0 = dst[k], s1 = dst[k+1], s2 = dst[k+2], s3 = dst[k+3];
for( i = i0; i < len; i++, src += cn )
{
s0 += (WT)src[0]; s1 += (WT)src[1];
s2 += (WT)src[2]; s3 += (WT)src[3];
}
dst[k] = s0;
dst[k+1] = s1;
dst[k+2] = s2;
dst[k+3] = s3;
}
return len;
}
int i, nzm = 0;
if( cn == 1 )
{
ST s = dst[0];
for( i = 0; i < len; i++ )
if( mask[i] )
{
s += (WT)src[i];
nzm++;
}
dst[0] = s;
}
else if( cn == 3 )
{
ST s0 = dst[0], s1 = dst[1], s2 = dst[2];
for( i = 0; i < len; i++, src += 3 )
if( mask[i] )
{
s0 += (WT)src[0];
s1 += (WT)src[1];
s2 += (WT)src[2];
nzm++;
}
dst[0] = s0;
dst[1] = s1;
dst[2] = s2;
}
else
{
for( i = 0; i < len; i++, src += cn )
if( mask[i] )
{
int k = 0;
#if CV_ENABLE_UNROLLED
for( ; k <= cn - 4; k += 4 )
{
ST s0, s1;
s0 = dst[k] + (WT)src[k];
s1 = dst[k+1] + (WT)src[k+1];
dst[k] = s0; dst[k+1] = s1;
s0 = dst[k+2] + (WT)src[k+2];
s1 = dst[k+3] + (WT)src[k+3];
dst[k+2] = s0; dst[k+3] = s1;
}
#endif
for( ; k < cn; k++ )
dst[k] += (WT)src[k];
nzm++;
}
}
return nzm;
}
static int sum8u( const uchar* src, const uchar* mask, int* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum8s( const schar* src, const uchar* mask, int* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum16u( const ushort* src, const uchar* mask, int* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum16s( const short* src, const uchar* mask, int* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32u( const unsigned* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32s( const int* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64u( const uint64* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64s( const int64* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum32f( const float* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum64f( const double* src, const uchar* mask, double* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_(src, mask, dst, len, cn); }
static int sum16f( const hfloat* src, const uchar* mask, float* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_<hfloat, float, float>(src, mask, dst, len, cn); }
static int sum16bf( const bfloat* src, const uchar* mask, float* dst, int len, int cn )
{ CV_INSTRUMENT_REGION(); return sum_<bfloat, float, float>(src, mask, dst, len, cn); }
SumFunc getSumFunc(int depth)
{
static SumFunc sumTab[CV_DEPTH_MAX] =
{
(SumFunc)GET_OPTIMIZED(sum8u),
(SumFunc)sum8s,
(SumFunc)sum16u,
(SumFunc)sum16s,
(SumFunc)sum32s,
(SumFunc)GET_OPTIMIZED(sum32f),
(SumFunc)sum64f,
(SumFunc)sum16f,
(SumFunc)sum16bf,
0,
(SumFunc)sum64u,
(SumFunc)sum64s,
(SumFunc)sum32u,
0
};
return sumTab[depth];
}
#endif
CV_CPU_OPTIMIZATION_NAMESPACE_END
} // namespace