From c7bf418d06072b6b0e5ef87261ef3477dab22c82 Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Sun, 1 Nov 2026 13:25:31 +0300 Subject: [PATCH] core(dft): radix-8 stages and cache-set padding for the power-of-two lengths The power-of-two part is now factored as r0 * 8^a * 4^b (r0 = 8 whenever the rest can still be covered, radix-8 stages first, radix-4 last): 256 runs 2 middle stages instead of 3, 1024 3 instead of 4, 4096 3 instead of 5, 65536 5 instead of 7. The radix-8 butterfly follows FFTW's t1fv_8: the odd legs are combined first (4-point DFT + the W8 rotations, kept as sums and folded into the outputs with FMAs by +-1/sqrt(2)), then the even legs, whose pure additions go straight into the outputs. GCC's AVX2 code for the kernel has 5 register spills per butterfly, so the 16-register budget is not the issue that made an earlier radix-8 attempt slower. The last stage of C2C / even C2R can be radix 8 too (radix8Last writes the interleaved scaled output). Static tables gained the radix-8 stage twiddles for spans 4..128. What did make the earlier attempt (and the radix-4 engine) slow at large n is 4K aliasing: with a span of a 4 KB multiple the r data legs, the r-1 twiddle leg tables and the re/im arrays of one butterfly all map to one L1 set (12 ways on Intel, 8 on Zen), and 16-30 streams thrash it. One cache line of padding between the twiddle legs (tw_stride) and between re and im (im_ofs) fixes it: on its own it gives 1.3-1.4x at n >= 16384 for radix 4 already. Apply time of one cv::dft() (Core Ultra 7 255H, AVX2, no IPP), speedup of radix-8 + padding over the previous engine: 32FC2 64FC2 32FC1 64FC1 1024 1.17 1.07 1.09 1.09 4096 1.37 1.32 1.04 1.04 16384 1.63 1.51 1.22 1.32 65536 1.76 1.38 1.39 1.33 262144 1.70 1.46 1.35 1.30 Co-Authored-By: Claude Fable 5.1 --- modules/core/src/dxt.cpp | 73 +++++++++++----- modules/core/src/dxt.hpp | 16 ++-- modules/core/src/dxt.simd.hpp | 159 ++++++++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+), 29 deletions(-) diff --git a/modules/core/src/dxt.cpp b/modules/core/src/dxt.cpp index 333c876789..5aab7c222b 100644 --- a/modules/core/src/dxt.cpp +++ b/modules/core/src/dxt.cpp @@ -352,18 +352,24 @@ static void fillRealTwiddles(const DftTwiddleGen& gen, int n, int nc, double sca } // The radix sequence of a length nc: the fused first stage r0 (1, 2, 3, 4, 5 or 8), then the -// radix-4 stages of the remaining 2^k part, then 3s, 5s and the other odd factors ascending. -// Even nc: r0 from the power of two. Odd nc: r0 = 5 or 3 when available (a vectorized, +// radix-8 and radix-4 stages of the remaining 2^k part, then 3s, 5s and the other odd factors +// ascending. Even nc: r0 from the power of two. Odd nc: r0 = 5 or 3 when available (a vectorized, // twiddle-free first stage; it also makes the spans of the following stages larger), else 1. -// (A radix-8 middle stage was tried and measured slower on NEON: with per-leg twiddle tables it -// does not save loads over two radix-4 stages and keeps 16 data vectors live.) static int dftFactorize(int nc, int radices[DFT_MAX_STAGES], int& nst) { int k2 = 0, rest = nc; while ((rest & 1) == 0) { rest >>= 1; k2++; } - int r0 = k2 == 0 ? (rest % 5 == 0 ? 5 : rest % 3 == 0 ? 3 : 1) : k2 == 1 ? 2 : (k2 & 1) ? 8 : 4; + // 2^k2 = r0 * 8^a * 4^b: the twiddle-free first stage takes 8 whenever the rest can still be + // covered (k2 - 3 is 0 or >= 2), then as many radix-8 stages as possible, radix-4 stages last + int r0 = k2 == 0 ? (rest % 5 == 0 ? 5 : rest % 3 == 0 ? 3 : 1) : k2 == 1 ? 2 : (k2 == 2 || k2 == 4) ? 4 : 8; nst = 0; - for (int m = k2 - (r0 == 2 ? 1 : r0 == 4 ? 2 : r0 == 8 ? 3 : 0); m > 0; m -= 2) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 4; } + { + int m = k2 - (r0 == 2 ? 1 : r0 == 4 ? 2 : r0 == 8 ? 3 : 0); + int b = m % 3 == 0 ? 0 : m % 3 == 2 ? 1 : 2, a = (m - 2*b)/3; + CV_Assert(m - 2*b >= 0 && nst + a + b <= DFT_MAX_STAGES); + for (int i = 0; i < a; i++) radices[nst++] = 8; + for (int i = 0; i < b; i++) radices[nst++] = 4; + } if (r0 == 3 || r0 == 5) rest /= r0; while (rest % 3 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 3; rest /= 3; } while (rest % 5 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 5; rest /= 5; } @@ -483,8 +489,9 @@ static void dftBuildPermutation(int nc, const int* seq, int nseq, bool real_inpu // // The power-of-two lengths are the common case (matchTemplate/filter2D tiles, 256/512/1024 rows), // and cv::dft() builds a plan per call, so their tables are built once per process and shared: -// - radix-4 stage twiddles W_{4L}^{q*j} for every span L = 2^k, 4 <= L <= 256 (they depend on -// (radix, span) only, not on the transform length); +// - radix-4 stage twiddles W_{4L}^{q*j} for every span L = 2^k, 4 <= L <= 256, and radix-8 +// stage twiddles W_{8L}^{q*j} for 4 <= L <= 128 (they depend on (radix, span) only, not on +// the transform length); // - the permutation gtab of nc = 2^k, 4 <= nc <= 1024 (r0 in {4, 8}); // - W_n^k, k in [0, nc], of the even real kinds (R2C/C2R) with nc = 2^k <= 1024. // ~90 KB for both depths. DCT tables are kind- and length-specific and stay per plan. @@ -494,8 +501,10 @@ template struct DftStaticTables { AutoBuffer tw; // stage tables, then the real twiddles AutoBuffer itab; - const T* tw_re[DFT_STATIC_MAX_LOG2]; // [log2(L)], L in [4, 256] + const T* tw_re[DFT_STATIC_MAX_LOG2]; // radix 4: [log2(L)], L in [4, 256] const T* tw_im[DFT_STATIC_MAX_LOG2]; + const T* tw8_re[DFT_STATIC_MAX_LOG2]; // radix 8: [log2(L)], L in [4, 128] + const T* tw8_im[DFT_STATIC_MAX_LOG2]; const T* rtw_re[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024] const T* rtw_im[DFT_STATIC_MAX_LOG2 + 1]; const int* gtab[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024] @@ -507,8 +516,10 @@ template struct DftStaticTables { int nc = 1 << k; if (k <= 8) ntw += 6*(size_t)nc; // radix-4 stage with L = nc: 3 legs x (re, im) x L + if (k <= 7) ntw += 14*(size_t)nc; // radix-8 stage with L = nc: 7 legs x (re, im) x L ntw += 2*(size_t)(nc + 1); // W_{2nc}^k, k in [0, nc] - nitab += nc/(k & 1 ? 8 : 4); + int radices[DFT_MAX_STAGES], nst; + nitab += nc/dftFactorize(nc, radices, nst); } tw.allocate(ntw); itab.allocate(nitab); @@ -527,6 +538,15 @@ template struct DftStaticTables fillStageTwiddles(gen, 4, L, L, cur, cur + 3*L, twd.data()); cur += 6*L; } + if (k <= 7) + { + int L = nc; + DftTwiddleGen gen; + gen.init(8*L); + tw8_re[k] = cur; tw8_im[k] = cur + 7*L; + fillStageTwiddles(gen, 8, L, L, cur, cur + 7*L, twd.data()); + cur += 14*L; + } { DftTwiddleGen gen; gen.init(2*nc); @@ -582,9 +602,10 @@ void DftPlan::fillTables(T* dst, size_t ntw, bool need_rtw, bool need_dct, bool st.cs = st.sn = 0; if (static_stages) { - CV_DbgAssert(r == 4 && dftIsPow2(L) && L >= 4 && L <= 256); + CV_DbgAssert((r == 4 || r == 8) && dftIsPow2(L) && L >= 4 && L*r <= 1024 && ts == L); int k = dftLog2(L); - st.tw_re = st_tables->tw_re[k]; st.tw_im = st_tables->tw_im[k]; + if (r == 4) { st.tw_re = st_tables->tw_re[k]; st.tw_im = st_tables->tw_im[k]; } + else { st.tw_re = st_tables->tw8_re[k]; st.tw_im = st_tables->tw8_im[k]; } continue; } T* twr = cur; T* twi = cur + (r - 1)*ts; @@ -684,7 +705,10 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit { DftStage& st = p.stages[s]; int r = radices[s]; - st.radix = r; st.span = L; st.ngroups = nc/(r*L); st.tw_stride = L; + st.radix = r; st.span = L; st.ngroups = nc/(r*L); + // 4K-aliasing: with a span of a 4 KB multiple all r-1 leg tables (and the r data legs) + // of one butterfly map to the same L1 set; one line of padding per leg spreads them + st.tw_stride = L + ((L*esz & 4095) == 0 ? (int)(64/esz) : 0); if (vl > 1 && L >= vl) { st.mode = DFT_STAGE_FULL; @@ -715,8 +739,8 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit } CV_Assert(L == nc); - // the static tables cover the stage twiddles of nc = 2^k <= 1024 (all stages radix 4, - // L <= 256), the permutation of 4 <= nc = 2^k <= 1024 and W_n^k of the even real kinds + // the static tables cover the stage twiddles of nc = 2^k <= 1024 (stages of radix 4 and 8, + // r*L <= 1024), the permutation of 4 <= nc = 2^k <= 1024 and W_n^k of the even real kinds bool static_pow2 = dftIsPow2(nc) && nc >= 4 && nc <= DFT_STATIC_MAX_NC; bool need_rtw = real_kind && n % 2 == 0; bool static_stages = static_pow2; @@ -728,7 +752,9 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit // workspace layout (from the 64-byte aligned base): pair0 (re, im), optional pair1 (re, im) or // the interleaved temp, radix-odd scratch (a[h], b[h] complex per lane). - size_t pair_bytes = 2*(size_t)nc*esz; + size_t pad = ((size_t)nc*esz & 4095) == 0 ? 64 : 0; // see im_ofs + p.im_ofs = (size_t)nc*esz + pad; + size_t pair_bytes = alignSize(2*(size_t)nc*esz + 2*pad, 64); bool two_pairs = p.pingpong || p.need_tmp; p.pair1_ofs = two_pairs ? pair_bytes : 0; p.scratch_ofs = pair_bytes*(two_pairs ? 2 : 1); @@ -765,8 +791,8 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit else fillTables((double*)base, ntw, need_rtw, dct_kind, static_stages, static_rtw); - // the last radix-4 stage of C2C / even C2R can write the interleaved scaled output directly - p.fused_last = nst > 0 && radices[nst - 1] == 4 && + // the last radix-4/8 stage of C2C / even C2R can write the interleaved scaled output directly + p.fused_last = nst > 0 && (radices[nst - 1] == 4 || radices[nst - 1] == 8) && (kind == DFT_KIND_C2C || (kind == DFT_KIND_C2R && nc < n)); // depth-first chunking of the leading stages (see DFT_CHUNK_BYTES) @@ -791,7 +817,7 @@ static inline void runStage(const DFTKernels& k, const DftStage& st0, int i0, in DftStage st = st0; st.ngroups = len/(st.radix*st.span); st.nsteps = st.ngroups*st.steps_per_group; - DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 3 ? k.radix3 : + DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 8 ? k.radix8 : st.radix == 3 ? k.radix3 : st.radix == 5 ? k.radix5 : k.radixOdd; size_t ofs = (size_t)i0*esz; fn(st, sre + ofs, sim + ofs, dre + ofs, dim + ofs, scratch); @@ -808,9 +834,9 @@ static void runDft(const OcvDftOptions& c, const void* src, size_t sstep, void* size_t esz = p.depth == CV_32F ? sizeof(float) : sizeof(double); uchar* base = alignPtr(c.workspace, 64); uchar* re0 = base; - uchar* im0 = re0 + p.nc*esz; + uchar* im0 = re0 + p.im_ofs; uchar* re1 = base + p.pair1_ofs; - uchar* im1 = re1 + p.nc*esz; + uchar* im1 = re1 + p.im_ofs; uchar* scratch = base + p.scratch_ofs; const void* asrc = src; @@ -882,8 +908,9 @@ static void runDft(const OcvDftOptions& c, const void* src, size_t sstep, void* double scale = c.scale; if (p.fused_last) { - // C2C or even C2R: last radix-4 stage + interleave + scale, straight into dst - k.radix4Last(p.stages[p.nstages - 1], sre, sim, dst, scale, c.isInverse || p.kind == DFT_KIND_C2R ? -scale : scale); + // C2C or even C2R: last radix-4/8 stage + interleave + scale, straight into dst + const DftStage& last = p.stages[p.nstages - 1]; + (last.radix == 8 ? k.radix8Last : k.radix4Last)(last, sre, sim, dst, scale, c.isInverse || p.kind == DFT_KIND_C2R ? -scale : scale); return; } switch (p.kind) diff --git a/modules/core/src/dxt.hpp b/modules/core/src/dxt.hpp index 1c5ae5347b..43d6d9e17a 100644 --- a/modules/core/src/dxt.hpp +++ b/modules/core/src/dxt.hpp @@ -14,7 +14,7 @@ // For long transforms the leading stages run chunk by chunk (depth-first) so // that their data stays in L2. // C (postproc): interleave / CCS packing / DCT ordering with fused scaling into the user output. -// For C2C (and even C2R) the last radix-4 stage writes the interleaved, scaled +// For C2C (and even C2R) the last radix-4/8 stage writes the interleaved, scaled // output itself (fused last stage), saving one pass over the data. // // Inverse transforms use the conjugation trick: conj() on the gather, im_scale = -scale on output. @@ -55,13 +55,14 @@ enum { DFT_MAX_STAGES = 40 }; struct DftStage { - int radix; // 3, 4, 5 or an odd number >= 7 + int radix; // 3, 4, 5, 8 or an odd number >= 7 int span; // L: length of the sub-DFTs combined by this stage int ngroups; // G = nc / (radix*span) int nsteps; // number of (vector) iterations of the flat loop, precomputed by the plan int steps_per_group; // nsteps / ngroups (the driver re-derives nsteps for chunked runs) int mode; // DftStageMode - int tw_stride; // per-leg twiddle table length (== span for now) + int tw_stride; // per-leg twiddle table length: span, plus one cache line of padding when + // span*esz is a multiple of 4 KB (the legs must not share one cache set) // Per-stage twiddles, split layout, leg-major: tw_re[(q-1)*tw_stride + j] = cos(-2*pi*q*j/(radix*span)), // q in [1, radix), j in [0, span). Type T of the plan. const void* tw_re; @@ -83,7 +84,7 @@ struct DftPlan bool real_input; // R2C with odd n: gather real samples, im = 0 bool pingpong; // stages must alternate between two workspace pairs bool need_tmp; // C2R / DCT / IDCT: a temp of 2*nc elements (interleaved complex or the Makhoul-permuted reals) - bool fused_last; // the last stage is radix 4 and writes the interleaved scaled output itself (C2C, even C2R) + bool fused_last; // the last stage is radix 4 or 8 and writes the interleaved scaled output itself (C2C, even C2R) int nchunk_stages; // leading stages run chunk by chunk (chunk = stages[nchunk_stages].span); 0 = none int nstages; DftStage stages[DFT_MAX_STAGES]; @@ -100,13 +101,15 @@ struct DftPlan const void* dct_re; // s*W_4n^k, k in [0, n/2]; DCT/IDCT (s = DCT normalization) const void* dct_im; size_t ws_bytes; // workspace size (incl. 64 bytes for alignment) + size_t im_ofs; // byte offset of im from re inside a pair: nc*esz plus one cache line when + // nc*esz is a multiple of 4 KB (keeps the re and im streams out of one cache set) size_t pair1_ofs; // byte offset of the second (re, im) pair or of the temp, from the aligned base size_t scratch_ofs; // byte offset of the radix-odd scratch area DftPlan() : kind(0), n(0), nc(0), depth(0), vl(0), first_radix(1), real_input(false), pingpong(false), need_tmp(false), fused_last(false), nchunk_stages(0), nstages(0), itab(0), workspace(0), rtw_re(0), rtw_im(0), dct_re(0), dct_im(0), ws_bytes(0), - pair1_ofs(0), scratch_ofs(0) {} + im_ofs(0), pair1_ofs(0), scratch_ofs(0) {} // Builds the plan for a 1D transform of the given kind (DftKind) and length. // @@ -151,8 +154,9 @@ struct DFTKernels // IDCT: pre-twiddle + CCS untangle -> interleaved temp void (*preprocIDCT)(const DftPlan& plan, const void* src, size_t sstep, void* tmp); // group B - DftStageFunc radix3, radix4, radix5, radixOdd; + DftStageFunc radix3, radix4, radix5, radix8, radixOdd; DftLastStageFunc radix4Last; // radix-4 stage + interleave + scale into the user output + DftLastStageFunc radix8Last; // the same for a radix-8 last stage // group C void (*postprocDFT)(const DftPlan& plan, const void* re, const void* im, void* dst, double re_scale, double im_scale); // split -> interleaved complex diff --git a/modules/core/src/dxt.simd.hpp b/modules/core/src/dxt.simd.hpp index 1165e46f26..033eeecc96 100644 --- a/modules/core/src/dxt.simd.hpp +++ b/modules/core/src/dxt.simd.hpp @@ -796,6 +796,58 @@ void radix4Last_scalar(const DftStage& st, const T* sre, const T* sim, T* dst, T } } +// radix-8 DIT stage, scalar. The odd legs (1,3,5,7) are combined first: their 4-point DFT needs +// the W8 rotations (the costly part); the even legs (0,2,4,6) are pure additions and are merged +// into the outputs as they are produced. LAST: write the interleaved, scaled output (fused last). +template +void radix8_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim, T* dst, T rs, T is) +{ + const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im; + int L = st.span, G = st.ngroups, ts = st.tw_stride; + const T c = DftConst::sin45; + for (int g = 0; g < G; g++) + { + int base = g*8*L; + for (int j = 0; j < L; j++) + { + int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L, i4 = i3 + L, i5 = i4 + L, i6 = i5 + L, i7 = i6 + L; + T x1r, x1i, x3r, x3i, x5r, x5i, x7r, x7i; + DFT_CMUL(x1r, x1i, sre[i1], sim[i1], twr[j], twi[j]); + DFT_CMUL(x5r, x5i, sre[i5], sim[i5], twr[ts*4 + j], twi[ts*4 + j]); + DFT_CMUL(x3r, x3i, sre[i3], sim[i3], twr[ts*2 + j], twi[ts*2 + j]); + DFT_CMUL(x7r, x7i, sre[i7], sim[i7], twr[ts*6 + j], twi[ts*6 + j]); + T o0r, o0i, o1r, o1i, o2r, o2i, o3r, o3i; + DFT4_SCALAR(x1r,x1i,x3r,x3i,x5r,x5i,x7r,x7i, o0r,o0i,o1r,o1i,o2r,o2i,o3r,o3i); + // O1*W8 = c*((o1r + o1i) + i(o1i - o1r)), O3*W8^3 = c*((o3i - o3r) - i(o3r + o3i)), O2*W8^2 = (o2i, -o2r) + T s1r = o1r + o1i, s1i = o1i - o1r, s3r = o3i - o3r, s3i = o3r + o3i; + T x0r = sre[i0], x0i = sim[i0], x2r, x2i, x4r, x4i, x6r, x6i; + DFT_CMUL(x4r, x4i, sre[i4], sim[i4], twr[ts*3 + j], twi[ts*3 + j]); + DFT_CMUL(x2r, x2i, sre[i2], sim[i2], twr[ts + j], twi[ts + j]); + DFT_CMUL(x6r, x6i, sre[i6], sim[i6], twr[ts*5 + j], twi[ts*5 + j]); + T e0r, e0i, e1r, e1i, e2r, e2i, e3r, e3i; + DFT4_SCALAR(x0r,x0i,x2r,x2i,x4r,x4i,x6r,x6i, e0r,e0i,e1r,e1i,e2r,e2i,e3r,e3i); + T y0r = e0r + o0r, y0i = e0i + o0i, y4r = e0r - o0r, y4i = e0i - o0i; + T y2r = e2r + o2i, y2i = e2i - o2r, y6r = e2r - o2i, y6i = e2i + o2r; + T y1r = e1r + c*s1r, y1i = e1i + c*s1i, y5r = e1r - c*s1r, y5i = e1i - c*s1i; + T y3r = e3r + c*s3r, y3i = e3i - c*s3i, y7r = e3r - c*s3r, y7i = e3i + c*s3i; + if constexpr (LAST) + { + dst[i0*2] = y0r*rs; dst[i0*2+1] = y0i*is; dst[i1*2] = y1r*rs; dst[i1*2+1] = y1i*is; + dst[i2*2] = y2r*rs; dst[i2*2+1] = y2i*is; dst[i3*2] = y3r*rs; dst[i3*2+1] = y3i*is; + dst[i4*2] = y4r*rs; dst[i4*2+1] = y4i*is; dst[i5*2] = y5r*rs; dst[i5*2+1] = y5i*is; + dst[i6*2] = y6r*rs; dst[i6*2+1] = y6i*is; dst[i7*2] = y7r*rs; dst[i7*2+1] = y7i*is; + } + else + { + dre[i0] = y0r; dim[i0] = y0i; dre[i1] = y1r; dim[i1] = y1i; + dre[i2] = y2r; dim[i2] = y2i; dre[i3] = y3r; dim[i3] = y3i; + dre[i4] = y4r; dim[i4] = y4i; dre[i5] = y5r; dim[i5] = y5i; + dre[i6] = y6r; dim[i6] = y6i; dre[i7] = y7r; dim[i7] = y7i; + } + } + } +} + template void radix5_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim) { @@ -1000,6 +1052,68 @@ void radix4Last_vec(const DftStage& st, const T* sre, const T* sim, T* dst, T rs DFT_VLOOP_END(4) } +// radix-8 DIT stage, vector (see radix8_scalar for the structure). Odd legs first (4-point DFT + +// W8 rotations: the rotated values are kept as the sums s = (o1r+o1i, ...) and folded into the +// outputs with FMAs by +-sin45, as in FFTW's t1fv_8), then the even legs, whose pure additions +// go straight into the outputs. Peak live set is ~18 vectors (odd results 8 + even inputs 8 + +// temporaries), so a couple of spills are expected on 16-register ISAs; still one pass over the +// data instead of two radix-4 passes. LAST: fused interleave + scale into the user output. +template +void radix8_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim, T* dst, T rs, T is) +{ + const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im; + const int L = st.span, ts = st.tw_stride, VL = VTraits::vlanes(); + const VT c = VSETALL((T)DftConst::sin45), nc = VSETALL((T)-DftConst::sin45); + const VT vrs = VSETALL(rs), vis = VSETALL(is); + auto store = [&](int i, const VT& yr, const VT& yi) + { + if constexpr (LAST) + v_store_interleave(dst + i*2, v_mul(yr, vrs), v_mul(yi, vis)); + else + { + v_store(dre + i, yr); v_store(dim + i, yi); + } + }; + DFT_VLOOP_BEGIN(8) + int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L, i4 = i3 + L, i5 = i4 + L, i6 = i5 + L, i7 = i6 + L; + // odd legs: b1 = x1 + x5, b5 = x1 - x5, b3 = x3 + x7, b7 = x3 - x7 + VT x1r, x1i, x5r, x5i, x3r, x3i, x7r, x7i; + DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j)); + DFT_VCMUL(x5r, x5i, VLOAD(sre + i5), VLOAD(sim + i5), VLOAD(twr + ts*4 + j), VLOAD(twi + ts*4 + j)); + VT b1r = v_add(x1r, x5r), b1i = v_add(x1i, x5i), b5r = v_sub(x1r, x5r), b5i = v_sub(x1i, x5i); + DFT_VCMUL(x3r, x3i, VLOAD(sre + i3), VLOAD(sim + i3), VLOAD(twr + ts*2 + j), VLOAD(twi + ts*2 + j)); + DFT_VCMUL(x7r, x7i, VLOAD(sre + i7), VLOAD(sim + i7), VLOAD(twr + ts*6 + j), VLOAD(twi + ts*6 + j)); + VT b3r = v_add(x3r, x7r), b3i = v_add(x3i, x7i), b7r = v_sub(x3r, x7r), b7i = v_sub(x3i, x7i); + // 4-point DFT of the odd legs: O0 = b1 + b3, O2 = b1 - b3, O1 = b5 - i*b7, O3 = b5 + i*b7; + // O1*W8 = c*(s1r + i*s1i), O3*W8^3 = c*(s3r - i*s3i), O2*W8^2 = (o2i, -o2r) + VT o0r = v_add(b1r, b3r), o0i = v_add(b1i, b3i), o2r = v_sub(b1r, b3r), o2i = v_sub(b1i, b3i); + VT o1r = v_add(b5r, b7i), o1i = v_sub(b5i, b7r), o3r = v_sub(b5r, b7i), o3i = v_add(b5i, b7r); + VT s1r = v_add(o1r, o1i), s1i = v_sub(o1i, o1r), s3r = v_sub(o3i, o3r), s3i = v_add(o3r, o3i); + // even legs: a0 = x0 + x4, a4 = x0 - x4, a2 = x2 + x6, a6 = x2 - x6 + VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x4r, x4i; + DFT_VCMUL(x4r, x4i, VLOAD(sre + i4), VLOAD(sim + i4), VLOAD(twr + ts*3 + j), VLOAD(twi + ts*3 + j)); + VT a0r = v_add(x0r, x4r), a0i = v_add(x0i, x4i), a4r = v_sub(x0r, x4r), a4i = v_sub(x0i, x4i); + VT x2r, x2i, x6r, x6i; + DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j)); + DFT_VCMUL(x6r, x6i, VLOAD(sre + i6), VLOAD(sim + i6), VLOAD(twr + ts*5 + j), VLOAD(twi + ts*5 + j)); + VT a2r = v_add(x2r, x6r), a2i = v_add(x2i, x6i), a6r = v_sub(x2r, x6r), a6i = v_sub(x2i, x6i); + // E0 = a0 + a2 -> y0, y4; E2 = a0 - a2 -> y2, y6 (with O2*W8^2) + VT e0r = v_add(a0r, a2r), e0i = v_add(a0i, a2i); + store(i0, v_add(e0r, o0r), v_add(e0i, o0i)); + store(i4, v_sub(e0r, o0r), v_sub(e0i, o0i)); + VT e2r = v_sub(a0r, a2r), e2i = v_sub(a0i, a2i); + store(i2, v_add(e2r, o2i), v_sub(e2i, o2r)); + store(i6, v_sub(e2r, o2i), v_add(e2i, o2r)); + // E1 = a4 - i*a6 -> y1, y5 (+- c*s1); E3 = a4 + i*a6 -> y3, y7 (+- c*(s3r, -s3i)) + VT e1r = v_add(a4r, a6i), e1i = v_sub(a4i, a6r); + store(i1, v_fma(s1r, c, e1r), v_fma(s1i, c, e1i)); + store(i5, v_fma(s1r, nc, e1r), v_fma(s1i, nc, e1i)); + VT e3r = v_sub(a4r, a6i), e3i = v_add(a4i, a6r); + store(i3, v_fma(s3r, c, e3r), v_fma(s3i, nc, e3i)); + store(i7, v_fma(s3r, nc, e3r), v_fma(s3i, c, e3i)); + DFT_VLOOP_END(8) +} + template void radix5_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim) { @@ -1191,6 +1305,49 @@ void radix4Last_(const DftStage& st, const void* sre, const void* sim, void* dst } +template +void radix8_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void*) +{ +#if (CV_SIMD || CV_SIMD_SCALABLE) + if constexpr (haveSIMD) + { + if (st.mode == DFT_STAGE_FULL) + { + radix8_vec(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0); + return; + } + if (st.mode == DFT_STAGE_SIMD128) + { + radix8_vec(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0); + return; + } + } +#endif + radix8_scalar(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0); +} + +template +void radix8Last_(const DftStage& st, const void* sre, const void* sim, void* dst, double re_scale, double im_scale) +{ + T rs = (T)re_scale, is = (T)im_scale; +#if (CV_SIMD || CV_SIMD_SCALABLE) + if constexpr (haveSIMD) + { + if (st.mode == DFT_STAGE_FULL) + { + radix8_vec(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is); + return; + } + if (st.mode == DFT_STAGE_SIMD128) + { + radix8_vec(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is); + return; + } + } +#endif + radix8_scalar(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is); +} + //////////////////////////////// group C: postprocessing //////////////////////////////// // // Pure map passes over the split arrays (plus the C2R / IDCT input untangles, which are the @@ -1459,6 +1616,8 @@ DFTKernels makeKernels() k.radix5 = radix5_; k.radixOdd = radixOdd_; k.radix4Last = radix4Last_; + k.radix8 = radix8_; + k.radix8Last = radix8Last_; k.postprocDFT = postprocDFT_; k.postprocRealDFT = postprocRealDFT_; k.postprocReal = postprocReal_;