From a427f0be446c4f315f2655cf9d05eac584c949bc Mon Sep 17 00:00:00 2001 From: Abhishek Gola Date: Sun, 6 Sep 2026 12:50:41 +0530 Subject: [PATCH] Merge pull request #29832 from abhishek-gola:simd-fp8-support SIMD support for FP8 - #29832 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/perf/perf_convertTo.cpp | 47 +++++++++++++ modules/core/src/convert.hpp | 86 ++++++++++++++++++++++++ modules/core/src/convert.simd.hpp | 35 +++++++++- modules/core/src/convert_scale.simd.hpp | 64 +++++++++++++++++- modules/core/test/test_fp8.cpp | 89 +++++++++++++++++++++++++ 5 files changed, 319 insertions(+), 2 deletions(-) diff --git a/modules/core/perf/perf_convertTo.cpp b/modules/core/perf/perf_convertTo.cpp index 344d81cb8a..68e55c6301 100644 --- a/modules/core/perf/perf_convertTo.cpp +++ b/modules/core/perf/perf_convertTo.cpp @@ -38,4 +38,51 @@ PERF_TEST_P( Size_DepthSrc_DepthDst_Channels_alpha, convertTo, SANITY_CHECK(dst, eps); } +// alpha param covers both the identity and scale FP8 kernels. +// Two distributions tracked: well-scaled (fast path) vs near-zero (fallback path). +typedef tuple Size_Fp8Depth_Alpha_t; +typedef perf::TestBaseWithParam Size_Fp8Depth_Alpha; + +PERF_TEST_P( Size_Fp8Depth_Alpha, convertToFp8_wellScaled, + testing::Combine + ( + testing::Values(szVGA, sz1080p), + testing::Values(CV_8F_E4M3FN, CV_8F_E4M3FNUZ), + testing::Values(1.0, 0.1) + ) + ) +{ + Size sz = get<0>(GetParam()); + int fp8depth = get<1>(GetParam()); + double alpha = get<2>(GetParam()); + + Mat src(sz, CV_32FC1); + randu(src, -8.0, 8.0); // mostly normal-range -> mostly the fast path + Mat dst(sz, fp8depth); + + TEST_CYCLE() src.convertTo(dst, fp8depth, alpha, 0.0); + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P( Size_Fp8Depth_Alpha, convertToFp8_nearZero, + testing::Combine + ( + testing::Values(szVGA, sz1080p), + testing::Values(CV_8F_E4M3FN, CV_8F_E4M3FNUZ), + testing::Values(1.0, 0.1) + ) + ) +{ + Size sz = get<0>(GetParam()); + int fp8depth = get<1>(GetParam()); + double alpha = get<2>(GetParam()); + + Mat src(sz, CV_32FC1); + randu(src, -0.006, 0.006); // below E4M3's smallest normal (2^-6) -> mostly the fallback path + Mat dst(sz, fp8depth); + + TEST_CYCLE() src.convertTo(dst, fp8depth, alpha, 0.0); + SANITY_CHECK_NOTHING(); +} + } // namespace diff --git a/modules/core/src/convert.hpp b/modules/core/src/convert.hpp index b2c2e398f7..f619385653 100644 --- a/modules/core/src/convert.hpp +++ b/modules/core/src/convert.hpp @@ -475,6 +475,92 @@ static inline void v_store_pair_as(uint64_t* ptr, const v_uint64& a, const v_uin v_store(ptr + VTraits::vlanes(), b); } +// float32 -> FP8 encode, shared by both scale paths. Fallback: no portable per-lane variable shift. + +// fallback test plus the pieces encodeFp8Finish needs, to skip costly math when unused +template static inline void +fp8Prepare(const v_float32& vf, v_uint32& full, v_uint32& sbit, v_int32& newexpRaw, v_int32& fallbackMask) +{ + v_uint32 u = v_reinterpret_as_u32(vf); + v_uint32 e = v_and(v_shr<23>(u), vx_setall_u32(0xFFu)); + v_uint32 m = v_and(u, vx_setall_u32(0x7FFFFFu)); + sbit = v_and(v_shr<24>(u), vx_setall_u32(0x80u)); + full = v_or(m, vx_setall_u32(1u << 23)); + newexpRaw = v_sub(v_reinterpret_as_s32(e), vx_setall_s32(127 - bias)); + v_int32 isNan = v_reinterpret_as_s32(v_eq(e, vx_setall_u32(0xFFu))); + fallbackMask = v_or(isNan, v_le(newexpRaw, vx_setall_s32(0))); +} + +// round-half-up + carry + overflow-to-NaN; valid only when fp8Prepare's fallbackMask is false +template static inline v_int32 +encodeFp8Finish(v_uint32 full, v_uint32 sbit, v_int32 newexpRaw) +{ + v_uint32 q = v_shr<20>(full); + v_uint32 rem = v_and(full, vx_setall_u32((1u << 20) - 1)); + v_uint32 inc = v_and(v_ge(rem, vx_setall_u32(1u << 19)), vx_setall_u32(1u)); + v_uint32 rounded = v_add(q, inc); + + v_uint32 carry = v_ne(v_and(rounded, vx_setall_u32(16u)), vx_setall_u32(0u)); + rounded = v_select(carry, v_shr<1>(rounded), rounded); + v_int32 newexp = v_add(newexpRaw, v_and(v_reinterpret_as_s32(carry), vx_setall_s32(1))); + + v_uint32 mant = v_and(rounded, vx_setall_u32(7u)); + + v_int32 gt15 = v_gt(newexp, vx_setall_s32(15)); + v_int32 overflow; + if constexpr (fnuz) + overflow = gt15; + else + overflow = v_or(gt15, v_and(v_eq(newexp, vx_setall_s32(15)), + v_eq(v_reinterpret_as_s32(mant), vx_setall_s32(7)))); + + v_uint32 nanCode; + if constexpr (fnuz) + nanCode = vx_setall_u32(0x80u); + else + nanCode = v_or(sbit, vx_setall_u32(0x7Fu)); + + v_uint32 normal = v_or(v_or(sbit, v_shl<3>(v_reinterpret_as_u32(newexp))), mant); + v_uint32 result = v_select(v_reinterpret_as_u32(overflow), nanCode, normal); + return v_reinterpret_as_s32(result); +} + +// encodes vf0..vf3 to FP8 bytes; scalarSrc must hold the same values, contiguous, for fallback +template static inline void +encodeFp8Vec4(const v_float32& vf0, const v_float32& vf1, const v_float32& vf2, const v_float32& vf3, + const float* scalarSrc, int vecsz, uchar* dst) +{ + v_uint32 full0, full1, full2, full3, sbit0, sbit1, sbit2, sbit3; + v_int32 newexpRaw0, newexpRaw1, newexpRaw2, newexpRaw3; + v_int32 pre0, pre1, pre2, pre3; + fp8Prepare(vf0, full0, sbit0, newexpRaw0, pre0); + fp8Prepare(vf1, full1, sbit1, newexpRaw1, pre1); + fp8Prepare(vf2, full2, sbit2, newexpRaw2, pre2); + fp8Prepare(vf3, full3, sbit3, newexpRaw3, pre3); + + if (v_check_all(v_and(v_and(pre0, pre1), v_and(pre2, pre3)))) + { + // whole group is subnormal/zero/NaN — skip the costlier fast-path math + for (int k = 0; k < vecsz; k++) + dst[k] = fp8_detail::encodeE4M3(scalarSrc[k], bias, fnuz); + return; + } + + v_int32 byte0 = encodeFp8Finish(full0, sbit0, newexpRaw0); + v_int32 byte1 = encodeFp8Finish(full1, sbit1, newexpRaw1); + v_int32 byte2 = encodeFp8Finish(full2, sbit2, newexpRaw2); + v_int32 byte3 = encodeFp8Finish(full3, sbit3, newexpRaw3); + v_store(dst, v_pack_u(v_pack(byte0, byte1), v_pack(byte2, byte3))); + + v_int32 one = vx_setall_s32(1); + uchar fallback[VTraits::max_nlanes]; + v_store(fallback, v_pack_u(v_pack(v_and(pre0, one), v_and(pre1, one)), + v_pack(v_and(pre2, one), v_and(pre3, one)))); + for (int k = 0; k < vecsz; k++) + if (fallback[k]) + dst[k] = fp8_detail::encodeE4M3(scalarSrc[k], bias, fnuz); +} + #if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) static inline void vx_load_as(const uint64_t* ptr, v_float32& a) diff --git a/modules/core/src/convert.simd.hpp b/modules/core/src/convert.simd.hpp index 85235aa566..e6a20be002 100644 --- a/modules/core/src/convert.simd.hpp +++ b/modules/core/src/convert.simd.hpp @@ -517,7 +517,6 @@ static void cvt64s(const uchar* src, size_t sstep, const uchar*, size_t, uchar* DEF_CVT_SCALAR_FUNC(S##16s, T, short) DEF_CVT_SCALAR_FUNC(16s##S, short, T) \ DEF_CVT_SCALAR_FUNC(S##32u, T, unsigned) DEF_CVT_SCALAR_FUNC(32u##S, unsigned, T) \ DEF_CVT_SCALAR_FUNC(S##32s, T, int) DEF_CVT_SCALAR_FUNC(32s##S, int, T) \ - DEF_CVT_SCALAR_FUNC(32f##S, float, T) \ DEF_CVT_SCALAR_FUNC(S##64f, T, double) DEF_CVT_SCALAR_FUNC(64f##S, double, T) \ DEF_CVT_SCALAR_FUNC(S##16f, T, hfloat) DEF_CVT_SCALAR_FUNC(16f##S, hfloat, T) \ DEF_CVT_SCALAR_FUNC(S##16bf, T, bfloat) DEF_CVT_SCALAR_FUNC(16bf##S, bfloat, T) \ @@ -532,6 +531,40 @@ DEF_CVT_FP8(8fe4m3u, fp8a_t) DEF_CVT_SCALAR_FUNC(8fe4m38fe4m3u, fp8_t, fp8a_t) DEF_CVT_SCALAR_FUNC(8fe4m3u8fe4m3, fp8a_t, fp8_t) +// float32 -> FP8 encode via the shared building blocks in convert.hpp. +template static void +cvtF32ToFp8(const uchar* src_, size_t sstep, const uchar*, size_t, + uchar* dst, size_t dstep, Size size, void*) +{ + CV_INSTRUMENT_REGION(); + const float* src = (const float*)src_; + sstep /= sizeof(src[0]); + + for (int i = 0; i < size.height; i++, src += sstep, dst += dstep) + { + int j = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int LANES32 = VTraits::vlanes(); + const int VECSZ = LANES32 * 4; + for (; j <= size.width - VECSZ; j += VECSZ) + { + v_float32 vf0 = vx_load(src + j); + v_float32 vf1 = vx_load(src + j + LANES32); + v_float32 vf2 = vx_load(src + j + 2 * LANES32); + v_float32 vf3 = vx_load(src + j + 3 * LANES32); + encodeFp8Vec4(vf0, vf1, vf2, vf3, src + j, VECSZ, dst + j); + } + vx_cleanup(); +#endif + for (; j < size.width; j++) + dst[j] = fp8_detail::encodeE4M3(src[j], bias, fnuz); + } +} +static void cvt32f8fe4m3(const uchar* s, size_t ss, const uchar* p, size_t ps, uchar* d, size_t ds, Size sz, void* x) +{ cvtF32ToFp8<7, false>(s, ss, p, ps, d, ds, sz, x); } +static void cvt32f8fe4m3u(const uchar* s, size_t ss, const uchar* p, size_t ps, uchar* d, size_t ds, Size sz, void* x) +{ cvtF32ToFp8<8, true>(s, ss, p, ps, d, ds, sz, x); } + // FP8 -> float32: 256-entry decode table gathered via universal intrinsics (same table as scalar) template static void cvtFp8ToF32(const uchar* src, size_t sstep, const uchar*, size_t, diff --git a/modules/core/src/convert_scale.simd.hpp b/modules/core/src/convert_scale.simd.hpp index 9bf092f8af..374184e5f6 100644 --- a/modules/core/src/convert_scale.simd.hpp +++ b/modules/core/src/convert_scale.simd.hpp @@ -492,7 +492,7 @@ static void cvtScale##suffix( const uchar* src_, size_t sstep, const uchar*, siz DEF_CVT_SCALE_SCALAR_FUNC(S##16s, T, short, double) DEF_CVT_SCALE_SCALAR_FUNC(16s##S, short, T, double) \ DEF_CVT_SCALE_SCALAR_FUNC(S##32u, T, unsigned, double) DEF_CVT_SCALE_SCALAR_FUNC(32u##S, unsigned, T, double) \ DEF_CVT_SCALE_SCALAR_FUNC(S##32s, T, int, double) DEF_CVT_SCALE_SCALAR_FUNC(32s##S, int, T, double) \ - DEF_CVT_SCALE_SCALAR_FUNC(S##32f, T, float, double) DEF_CVT_SCALE_SCALAR_FUNC(32f##S, float, T, double) \ + DEF_CVT_SCALE_SCALAR_FUNC(S##32f, T, float, double) \ DEF_CVT_SCALE_SCALAR_FUNC(S##64f, T, double, double) DEF_CVT_SCALE_SCALAR_FUNC(64f##S, double, T, double) \ DEF_CVT_SCALE_SCALAR_FUNC(S##16f, T, hfloat, double) DEF_CVT_SCALE_SCALAR_FUNC(16f##S, hfloat, T, double) \ DEF_CVT_SCALE_SCALAR_FUNC(S##16bf, T, bfloat, double) DEF_CVT_SCALE_SCALAR_FUNC(16bf##S, bfloat, T, double) \ @@ -504,6 +504,68 @@ static void cvtScale##suffix( const uchar* src_, size_t sstep, const uchar*, siz DEF_CVT_SCALE_FP8(8fe4m3, fp8_t) DEF_CVT_SCALE_FP8(8fe4m3u, fp8a_t) +// convertTo(dst, depth, alpha, beta) path -- real quantization. Reuses convert.hpp's fast path. +// Double precision here matches the scalar reference (wtype)x*a+b bit-for-bit. +template static void +cvtScaleF32ToFp8(const uchar* src_, size_t sstep, const uchar*, size_t, + uchar* dst, size_t dstep, Size size, void* scale_) +{ + CV_INSTRUMENT_REGION(); + const float* src = (const float*)src_; + const double* scale = (const double*)scale_; + double a = scale[0], b = scale[1]; + sstep /= sizeof(src[0]); + + for (int i = 0; i < size.height; i++, src += sstep, dst += dstep) + { + int j = 0; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + v_float64 va = vx_setall_f64(a), vb = vx_setall_f64(b); + const int LANES32 = VTraits::vlanes(); + const int VECSZ = LANES32 * 4; + float scaled[VTraits::max_nlanes * 4]; + for (; j <= size.width - VECSZ; j += VECSZ) + { + v_float32 vf0, vf1, vf2, vf3; + v_float64 d0, d1; + + vx_load_pair_as(src + j + 0 * LANES32, d0, d1); + d0 = v_add(v_mul(d0, va), vb); + d1 = v_add(v_mul(d1, va), vb); + vf0 = v_cvt_f32(d0, d1); + v_store(scaled + 0 * LANES32, vf0); + + vx_load_pair_as(src + j + 1 * LANES32, d0, d1); + d0 = v_add(v_mul(d0, va), vb); + d1 = v_add(v_mul(d1, va), vb); + vf1 = v_cvt_f32(d0, d1); + v_store(scaled + 1 * LANES32, vf1); + + vx_load_pair_as(src + j + 2 * LANES32, d0, d1); + d0 = v_add(v_mul(d0, va), vb); + d1 = v_add(v_mul(d1, va), vb); + vf2 = v_cvt_f32(d0, d1); + v_store(scaled + 2 * LANES32, vf2); + + vx_load_pair_as(src + j + 3 * LANES32, d0, d1); + d0 = v_add(v_mul(d0, va), vb); + d1 = v_add(v_mul(d1, va), vb); + vf3 = v_cvt_f32(d0, d1); + v_store(scaled + 3 * LANES32, vf3); + + encodeFp8Vec4(vf0, vf1, vf2, vf3, scaled, VECSZ, dst + j); + } + vx_cleanup(); +#endif + for (; j < size.width; j++) + dst[j] = fp8_detail::encodeE4M3((float)((double)src[j]*a + b), bias, fnuz); + } +} +static void cvtScale32f8fe4m3(const uchar* s, size_t ss, const uchar* p, size_t ps, uchar* d, size_t ds, Size sz, void* x) +{ cvtScaleF32ToFp8<7, false>(s, ss, p, ps, d, ds, sz, x); } +static void cvtScale32f8fe4m3u(const uchar* s, size_t ss, const uchar* p, size_t ps, uchar* d, size_t ds, Size sz, void* x) +{ cvtScaleF32ToFp8<8, true>(s, ss, p, ps, d, ds, sz, x); } + BinaryFunc getConvertScaleFunc(int sdepth_, int ddepth_) { int sdepth = CV_MAT_DEPTH(sdepth_); diff --git a/modules/core/test/test_fp8.cpp b/modules/core/test/test_fp8.cpp index 2083411c54..5bf12b0564 100644 --- a/modules/core/test/test_fp8.cpp +++ b/modules/core/test/test_fp8.cpp @@ -165,4 +165,93 @@ TEST(Core_FP8, convert_all_depths) } } +// Wide stress buffer to exercise the SIMD encode path and its scalar tail. +static std::vector fp8EncodeStressValues() +{ + std::vector vals; + + // every representable FP8 value in both formats, round-tripped through float32 + for (int b = 0; b < 256; b++) vals.push_back(fp8_t::decodeLUT()[b]); + for (int b = 0; b < 256; b++) vals.push_back(fp8a_t::decodeLUT()[b]); + + // dense sweep across the normal range, both signs, crossing every rounding boundary + for (int i = -20000; i <= 20000; i++) + vals.push_back(i * 0.031f); + + // geometric sweep from subnormal-FP8 through overflow-to-NaN + for (int e = -30; e <= 30; e++) + for (int m = 0; m < 8; m++) + vals.push_back((float)(std::ldexp(1.0 + m / 8.0, e))); + + float specials[] = { + 0.f, -0.f, + std::numeric_limits::infinity(), -std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN(), + std::numeric_limits::denorm_min(), -std::numeric_limits::denorm_min(), + std::numeric_limits::max(), -std::numeric_limits::max(), + std::numeric_limits::min(), 1e-40f, -1e-40f, + }; + vals.insert(vals.end(), std::begin(specials), std::end(specials)); + return vals; +} + +template +static void checkFp8EncodeMatchesScalar(const std::vector& vals) +{ + Mat f(1, (int)vals.size(), CV_32F, (void*)vals.data()); + Mat q; + f.convertTo(q, DataType::depth); + ASSERT_EQ(q.total(), vals.size()); + const uchar* qd = q.ptr(); + for (size_t i = 0; i < vals.size(); i++) + { + FP8 ref(vals[i]); + uchar refByte = *reinterpret_cast(&ref); + ASSERT_EQ(qd[i], refByte) << "value " << vals[i] << " (idx " << i << ")"; + } +} + +TEST(Core_FP8, simd_encode_matches_scalar) +{ + std::vector vals = fp8EncodeStressValues(); + checkFp8EncodeMatchesScalar(vals); + checkFp8EncodeMatchesScalar(vals); +} + +// Scale path uses a different kernel than the identity path above; check separately. +template +static void checkFp8ScaleEncodeMatchesScalar(const std::vector& vals, double alpha, double beta) +{ + Mat f(1, (int)vals.size(), CV_32F, (void*)vals.data()); + Mat q; + f.convertTo(q, DataType::depth, alpha, beta); + ASSERT_EQ(q.total(), vals.size()); + const uchar* qd = q.ptr(); + for (size_t i = 0; i < vals.size(); i++) + { + float scaled = (float)((double)vals[i]*alpha + beta); + FP8 ref(scaled); + uchar refByte = *reinterpret_cast(&ref); + ASSERT_EQ(qd[i], refByte) << "value " << vals[i] << " * " << alpha << " + " << beta + << " = " << scaled << " (idx " << i << ")"; + } +} + +TEST(Core_FP8, simd_scale_encode_matches_scalar) +{ + std::vector vals = fp8EncodeStressValues(); + // representative quantization scales: shrink, grow, shift-only, negate + double alphas[] = { 1.0, 0.015625, 64.0, -1.0 }; + double betas[] = { 0.0, 0.5, -3.25 }; + for (double alpha : alphas) + for (double beta : betas) + { + // alpha=1,beta=0 is convertTo's noScale case; skips the scale kernel entirely. + if (alpha == 1.0 && beta == 0.0) + continue; + checkFp8ScaleEncodeMatchesScalar(vals, alpha, beta); + checkFp8ScaleEncodeMatchesScalar(vals, alpha, beta); + } +} + }} // namespace