core(dft): radix-8 stages and cache-set padding for the power-of-two lengths

The power-of-two part is now factored as r0 * 8^a * 4^b (r0 = 8 whenever the
rest can still be covered, radix-8 stages first, radix-4 last): 256 runs 2
middle stages instead of 3, 1024 3 instead of 4, 4096 3 instead of 5, 65536 5
instead of 7. The radix-8 butterfly follows FFTW's t1fv_8: the odd legs are
combined first (4-point DFT + the W8 rotations, kept as sums and folded into
the outputs with FMAs by +-1/sqrt(2)), then the even legs, whose pure
additions go straight into the outputs. GCC's AVX2 code for the kernel has 5
register spills per butterfly, so the 16-register budget is not the issue
that made an earlier radix-8 attempt slower. The last stage of C2C / even C2R
can be radix 8 too (radix8Last writes the interleaved scaled output).
Static tables gained the radix-8 stage twiddles for spans 4..128.

What did make the earlier attempt (and the radix-4 engine) slow at large n is
4K aliasing: with a span of a 4 KB multiple the r data legs, the r-1 twiddle
leg tables and the re/im arrays of one butterfly all map to one L1 set (12
ways on Intel, 8 on Zen), and 16-30 streams thrash it. One cache line of
padding between the twiddle legs (tw_stride) and between re and im (im_ofs)
fixes it: on its own it gives 1.3-1.4x at n >= 16384 for radix 4 already.

Apply time of one cv::dft() (Core Ultra 7 255H, AVX2, no IPP), speedup of
radix-8 + padding over the previous engine:
             32FC2  64FC2  32FC1  64FC1
   1024       1.17   1.07   1.09   1.09
   4096       1.37   1.32   1.04   1.04
  16384       1.63   1.51   1.22   1.32
  65536       1.76   1.38   1.39   1.33
 262144       1.70   1.46   1.35   1.30

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Vadim Pisarevsky
2026-11-01 13:25:31 +03:00
parent 02c6f39c82
commit c7bf418d06
3 changed files with 219 additions and 29 deletions

View File

@@ -352,18 +352,24 @@ static void fillRealTwiddles(const DftTwiddleGen& gen, int n, int nc, double sca
}
// The radix sequence of a length nc: the fused first stage r0 (1, 2, 3, 4, 5 or 8), then the
// radix-4 stages of the remaining 2^k part, then 3s, 5s and the other odd factors ascending.
// Even nc: r0 from the power of two. Odd nc: r0 = 5 or 3 when available (a vectorized,
// radix-8 and radix-4 stages of the remaining 2^k part, then 3s, 5s and the other odd factors
// ascending. Even nc: r0 from the power of two. Odd nc: r0 = 5 or 3 when available (a vectorized,
// twiddle-free first stage; it also makes the spans of the following stages larger), else 1.
// (A radix-8 middle stage was tried and measured slower on NEON: with per-leg twiddle tables it
// does not save loads over two radix-4 stages and keeps 16 data vectors live.)
static int dftFactorize(int nc, int radices[DFT_MAX_STAGES], int& nst)
{
int k2 = 0, rest = nc;
while ((rest & 1) == 0) { rest >>= 1; k2++; }
int r0 = k2 == 0 ? (rest % 5 == 0 ? 5 : rest % 3 == 0 ? 3 : 1) : k2 == 1 ? 2 : (k2 & 1) ? 8 : 4;
// 2^k2 = r0 * 8^a * 4^b: the twiddle-free first stage takes 8 whenever the rest can still be
// covered (k2 - 3 is 0 or >= 2), then as many radix-8 stages as possible, radix-4 stages last
int r0 = k2 == 0 ? (rest % 5 == 0 ? 5 : rest % 3 == 0 ? 3 : 1) : k2 == 1 ? 2 : (k2 == 2 || k2 == 4) ? 4 : 8;
nst = 0;
for (int m = k2 - (r0 == 2 ? 1 : r0 == 4 ? 2 : r0 == 8 ? 3 : 0); m > 0; m -= 2) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 4; }
{
int m = k2 - (r0 == 2 ? 1 : r0 == 4 ? 2 : r0 == 8 ? 3 : 0);
int b = m % 3 == 0 ? 0 : m % 3 == 2 ? 1 : 2, a = (m - 2*b)/3;
CV_Assert(m - 2*b >= 0 && nst + a + b <= DFT_MAX_STAGES);
for (int i = 0; i < a; i++) radices[nst++] = 8;
for (int i = 0; i < b; i++) radices[nst++] = 4;
}
if (r0 == 3 || r0 == 5) rest /= r0;
while (rest % 3 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 3; rest /= 3; }
while (rest % 5 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 5; rest /= 5; }
@@ -483,8 +489,9 @@ static void dftBuildPermutation(int nc, const int* seq, int nseq, bool real_inpu
//
// The power-of-two lengths are the common case (matchTemplate/filter2D tiles, 256/512/1024 rows),
// and cv::dft() builds a plan per call, so their tables are built once per process and shared:
// - radix-4 stage twiddles W_{4L}^{q*j} for every span L = 2^k, 4 <= L <= 256 (they depend on
// (radix, span) only, not on the transform length);
// - radix-4 stage twiddles W_{4L}^{q*j} for every span L = 2^k, 4 <= L <= 256, and radix-8
// stage twiddles W_{8L}^{q*j} for 4 <= L <= 128 (they depend on (radix, span) only, not on
// the transform length);
// - the permutation gtab of nc = 2^k, 4 <= nc <= 1024 (r0 in {4, 8});
// - W_n^k, k in [0, nc], of the even real kinds (R2C/C2R) with nc = 2^k <= 1024.
// ~90 KB for both depths. DCT tables are kind- and length-specific and stay per plan.
@@ -494,8 +501,10 @@ template<typename T> struct DftStaticTables
{
AutoBuffer<T> tw; // stage tables, then the real twiddles
AutoBuffer<int> itab;
const T* tw_re[DFT_STATIC_MAX_LOG2]; // [log2(L)], L in [4, 256]
const T* tw_re[DFT_STATIC_MAX_LOG2]; // radix 4: [log2(L)], L in [4, 256]
const T* tw_im[DFT_STATIC_MAX_LOG2];
const T* tw8_re[DFT_STATIC_MAX_LOG2]; // radix 8: [log2(L)], L in [4, 128]
const T* tw8_im[DFT_STATIC_MAX_LOG2];
const T* rtw_re[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024]
const T* rtw_im[DFT_STATIC_MAX_LOG2 + 1];
const int* gtab[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024]
@@ -507,8 +516,10 @@ template<typename T> struct DftStaticTables
{
int nc = 1 << k;
if (k <= 8) ntw += 6*(size_t)nc; // radix-4 stage with L = nc: 3 legs x (re, im) x L
if (k <= 7) ntw += 14*(size_t)nc; // radix-8 stage with L = nc: 7 legs x (re, im) x L
ntw += 2*(size_t)(nc + 1); // W_{2nc}^k, k in [0, nc]
nitab += nc/(k & 1 ? 8 : 4);
int radices[DFT_MAX_STAGES], nst;
nitab += nc/dftFactorize(nc, radices, nst);
}
tw.allocate(ntw);
itab.allocate(nitab);
@@ -527,6 +538,15 @@ template<typename T> struct DftStaticTables
fillStageTwiddles<T>(gen, 4, L, L, cur, cur + 3*L, twd.data());
cur += 6*L;
}
if (k <= 7)
{
int L = nc;
DftTwiddleGen gen;
gen.init(8*L);
tw8_re[k] = cur; tw8_im[k] = cur + 7*L;
fillStageTwiddles<T>(gen, 8, L, L, cur, cur + 7*L, twd.data());
cur += 14*L;
}
{
DftTwiddleGen gen;
gen.init(2*nc);
@@ -582,9 +602,10 @@ void DftPlan::fillTables(T* dst, size_t ntw, bool need_rtw, bool need_dct, bool
st.cs = st.sn = 0;
if (static_stages)
{
CV_DbgAssert(r == 4 && dftIsPow2(L) && L >= 4 && L <= 256);
CV_DbgAssert((r == 4 || r == 8) && dftIsPow2(L) && L >= 4 && L*r <= 1024 && ts == L);
int k = dftLog2(L);
st.tw_re = st_tables->tw_re[k]; st.tw_im = st_tables->tw_im[k];
if (r == 4) { st.tw_re = st_tables->tw_re[k]; st.tw_im = st_tables->tw_im[k]; }
else { st.tw_re = st_tables->tw8_re[k]; st.tw_im = st_tables->tw8_im[k]; }
continue;
}
T* twr = cur; T* twi = cur + (r - 1)*ts;
@@ -684,7 +705,10 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit
{
DftStage& st = p.stages[s];
int r = radices[s];
st.radix = r; st.span = L; st.ngroups = nc/(r*L); st.tw_stride = L;
st.radix = r; st.span = L; st.ngroups = nc/(r*L);
// 4K-aliasing: with a span of a 4 KB multiple all r-1 leg tables (and the r data legs)
// of one butterfly map to the same L1 set; one line of padding per leg spreads them
st.tw_stride = L + ((L*esz & 4095) == 0 ? (int)(64/esz) : 0);
if (vl > 1 && L >= vl)
{
st.mode = DFT_STAGE_FULL;
@@ -715,8 +739,8 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit
}
CV_Assert(L == nc);
// the static tables cover the stage twiddles of nc = 2^k <= 1024 (all stages radix 4,
// L <= 256), the permutation of 4 <= nc = 2^k <= 1024 and W_n^k of the even real kinds
// the static tables cover the stage twiddles of nc = 2^k <= 1024 (stages of radix 4 and 8,
// r*L <= 1024), the permutation of 4 <= nc = 2^k <= 1024 and W_n^k of the even real kinds
bool static_pow2 = dftIsPow2(nc) && nc >= 4 && nc <= DFT_STATIC_MAX_NC;
bool need_rtw = real_kind && n % 2 == 0;
bool static_stages = static_pow2;
@@ -728,7 +752,9 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit
// workspace layout (from the 64-byte aligned base): pair0 (re, im), optional pair1 (re, im) or
// the interleaved temp, radix-odd scratch (a[h], b[h] complex per lane).
size_t pair_bytes = 2*(size_t)nc*esz;
size_t pad = ((size_t)nc*esz & 4095) == 0 ? 64 : 0; // see im_ofs
p.im_ofs = (size_t)nc*esz + pad;
size_t pair_bytes = alignSize(2*(size_t)nc*esz + 2*pad, 64);
bool two_pairs = p.pingpong || p.need_tmp;
p.pair1_ofs = two_pairs ? pair_bytes : 0;
p.scratch_ofs = pair_bytes*(two_pairs ? 2 : 1);
@@ -765,8 +791,8 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool wit
else
fillTables<double>((double*)base, ntw, need_rtw, dct_kind, static_stages, static_rtw);
// the last radix-4 stage of C2C / even C2R can write the interleaved scaled output directly
p.fused_last = nst > 0 && radices[nst - 1] == 4 &&
// the last radix-4/8 stage of C2C / even C2R can write the interleaved scaled output directly
p.fused_last = nst > 0 && (radices[nst - 1] == 4 || radices[nst - 1] == 8) &&
(kind == DFT_KIND_C2C || (kind == DFT_KIND_C2R && nc < n));
// depth-first chunking of the leading stages (see DFT_CHUNK_BYTES)
@@ -791,7 +817,7 @@ static inline void runStage(const DFTKernels& k, const DftStage& st0, int i0, in
DftStage st = st0;
st.ngroups = len/(st.radix*st.span);
st.nsteps = st.ngroups*st.steps_per_group;
DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 3 ? k.radix3 :
DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 8 ? k.radix8 : st.radix == 3 ? k.radix3 :
st.radix == 5 ? k.radix5 : k.radixOdd;
size_t ofs = (size_t)i0*esz;
fn(st, sre + ofs, sim + ofs, dre + ofs, dim + ofs, scratch);
@@ -808,9 +834,9 @@ static void runDft(const OcvDftOptions& c, const void* src, size_t sstep, void*
size_t esz = p.depth == CV_32F ? sizeof(float) : sizeof(double);
uchar* base = alignPtr(c.workspace, 64);
uchar* re0 = base;
uchar* im0 = re0 + p.nc*esz;
uchar* im0 = re0 + p.im_ofs;
uchar* re1 = base + p.pair1_ofs;
uchar* im1 = re1 + p.nc*esz;
uchar* im1 = re1 + p.im_ofs;
uchar* scratch = base + p.scratch_ofs;
const void* asrc = src;
@@ -882,8 +908,9 @@ static void runDft(const OcvDftOptions& c, const void* src, size_t sstep, void*
double scale = c.scale;
if (p.fused_last)
{
// C2C or even C2R: last radix-4 stage + interleave + scale, straight into dst
k.radix4Last(p.stages[p.nstages - 1], sre, sim, dst, scale, c.isInverse || p.kind == DFT_KIND_C2R ? -scale : scale);
// C2C or even C2R: last radix-4/8 stage + interleave + scale, straight into dst
const DftStage& last = p.stages[p.nstages - 1];
(last.radix == 8 ? k.radix8Last : k.radix4Last)(last, sre, sim, dst, scale, c.isInverse || p.kind == DFT_KIND_C2R ? -scale : scale);
return;
}
switch (p.kind)

View File

@@ -14,7 +14,7 @@
// For long transforms the leading stages run chunk by chunk (depth-first) so
// that their data stays in L2.
// C (postproc): interleave / CCS packing / DCT ordering with fused scaling into the user output.
// For C2C (and even C2R) the last radix-4 stage writes the interleaved, scaled
// For C2C (and even C2R) the last radix-4/8 stage writes the interleaved, scaled
// output itself (fused last stage), saving one pass over the data.
//
// Inverse transforms use the conjugation trick: conj() on the gather, im_scale = -scale on output.
@@ -55,13 +55,14 @@ enum { DFT_MAX_STAGES = 40 };
struct DftStage
{
int radix; // 3, 4, 5 or an odd number >= 7
int radix; // 3, 4, 5, 8 or an odd number >= 7
int span; // L: length of the sub-DFTs combined by this stage
int ngroups; // G = nc / (radix*span)
int nsteps; // number of (vector) iterations of the flat loop, precomputed by the plan
int steps_per_group; // nsteps / ngroups (the driver re-derives nsteps for chunked runs)
int mode; // DftStageMode
int tw_stride; // per-leg twiddle table length (== span for now)
int tw_stride; // per-leg twiddle table length: span, plus one cache line of padding when
// span*esz is a multiple of 4 KB (the legs must not share one cache set)
// Per-stage twiddles, split layout, leg-major: tw_re[(q-1)*tw_stride + j] = cos(-2*pi*q*j/(radix*span)),
// q in [1, radix), j in [0, span). Type T of the plan.
const void* tw_re;
@@ -83,7 +84,7 @@ struct DftPlan
bool real_input; // R2C with odd n: gather real samples, im = 0
bool pingpong; // stages must alternate between two workspace pairs
bool need_tmp; // C2R / DCT / IDCT: a temp of 2*nc elements (interleaved complex or the Makhoul-permuted reals)
bool fused_last; // the last stage is radix 4 and writes the interleaved scaled output itself (C2C, even C2R)
bool fused_last; // the last stage is radix 4 or 8 and writes the interleaved scaled output itself (C2C, even C2R)
int nchunk_stages; // leading stages run chunk by chunk (chunk = stages[nchunk_stages].span); 0 = none
int nstages;
DftStage stages[DFT_MAX_STAGES];
@@ -100,13 +101,15 @@ struct DftPlan
const void* dct_re; // s*W_4n^k, k in [0, n/2]; DCT/IDCT (s = DCT normalization)
const void* dct_im;
size_t ws_bytes; // workspace size (incl. 64 bytes for alignment)
size_t im_ofs; // byte offset of im from re inside a pair: nc*esz plus one cache line when
// nc*esz is a multiple of 4 KB (keeps the re and im streams out of one cache set)
size_t pair1_ofs; // byte offset of the second (re, im) pair or of the temp, from the aligned base
size_t scratch_ofs; // byte offset of the radix-odd scratch area
DftPlan() : kind(0), n(0), nc(0), depth(0), vl(0), first_radix(1), real_input(false),
pingpong(false), need_tmp(false), fused_last(false), nchunk_stages(0), nstages(0),
itab(0), workspace(0), rtw_re(0), rtw_im(0), dct_re(0), dct_im(0), ws_bytes(0),
pair1_ofs(0), scratch_ofs(0) {}
im_ofs(0), pair1_ofs(0), scratch_ofs(0) {}
// Builds the plan for a 1D transform of the given kind (DftKind) and length.
//
@@ -151,8 +154,9 @@ struct DFTKernels
// IDCT: pre-twiddle + CCS untangle -> interleaved temp
void (*preprocIDCT)(const DftPlan& plan, const void* src, size_t sstep, void* tmp);
// group B
DftStageFunc radix3, radix4, radix5, radixOdd;
DftStageFunc radix3, radix4, radix5, radix8, radixOdd;
DftLastStageFunc radix4Last; // radix-4 stage + interleave + scale into the user output
DftLastStageFunc radix8Last; // the same for a radix-8 last stage
// group C
void (*postprocDFT)(const DftPlan& plan, const void* re, const void* im, void* dst,
double re_scale, double im_scale); // split -> interleaved complex

View File

@@ -796,6 +796,58 @@ void radix4Last_scalar(const DftStage& st, const T* sre, const T* sim, T* dst, T
}
}
// radix-8 DIT stage, scalar. The odd legs (1,3,5,7) are combined first: their 4-point DFT needs
// the W8 rotations (the costly part); the even legs (0,2,4,6) are pure additions and are merged
// into the outputs as they are produced. LAST: write the interleaved, scaled output (fused last).
template<typename T, bool LAST>
void radix8_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim, T* dst, T rs, T is)
{
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
int L = st.span, G = st.ngroups, ts = st.tw_stride;
const T c = DftConst<T>::sin45;
for (int g = 0; g < G; g++)
{
int base = g*8*L;
for (int j = 0; j < L; j++)
{
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L, i4 = i3 + L, i5 = i4 + L, i6 = i5 + L, i7 = i6 + L;
T x1r, x1i, x3r, x3i, x5r, x5i, x7r, x7i;
DFT_CMUL(x1r, x1i, sre[i1], sim[i1], twr[j], twi[j]);
DFT_CMUL(x5r, x5i, sre[i5], sim[i5], twr[ts*4 + j], twi[ts*4 + j]);
DFT_CMUL(x3r, x3i, sre[i3], sim[i3], twr[ts*2 + j], twi[ts*2 + j]);
DFT_CMUL(x7r, x7i, sre[i7], sim[i7], twr[ts*6 + j], twi[ts*6 + j]);
T o0r, o0i, o1r, o1i, o2r, o2i, o3r, o3i;
DFT4_SCALAR(x1r,x1i,x3r,x3i,x5r,x5i,x7r,x7i, o0r,o0i,o1r,o1i,o2r,o2i,o3r,o3i);
// O1*W8 = c*((o1r + o1i) + i(o1i - o1r)), O3*W8^3 = c*((o3i - o3r) - i(o3r + o3i)), O2*W8^2 = (o2i, -o2r)
T s1r = o1r + o1i, s1i = o1i - o1r, s3r = o3i - o3r, s3i = o3r + o3i;
T x0r = sre[i0], x0i = sim[i0], x2r, x2i, x4r, x4i, x6r, x6i;
DFT_CMUL(x4r, x4i, sre[i4], sim[i4], twr[ts*3 + j], twi[ts*3 + j]);
DFT_CMUL(x2r, x2i, sre[i2], sim[i2], twr[ts + j], twi[ts + j]);
DFT_CMUL(x6r, x6i, sre[i6], sim[i6], twr[ts*5 + j], twi[ts*5 + j]);
T e0r, e0i, e1r, e1i, e2r, e2i, e3r, e3i;
DFT4_SCALAR(x0r,x0i,x2r,x2i,x4r,x4i,x6r,x6i, e0r,e0i,e1r,e1i,e2r,e2i,e3r,e3i);
T y0r = e0r + o0r, y0i = e0i + o0i, y4r = e0r - o0r, y4i = e0i - o0i;
T y2r = e2r + o2i, y2i = e2i - o2r, y6r = e2r - o2i, y6i = e2i + o2r;
T y1r = e1r + c*s1r, y1i = e1i + c*s1i, y5r = e1r - c*s1r, y5i = e1i - c*s1i;
T y3r = e3r + c*s3r, y3i = e3i - c*s3i, y7r = e3r - c*s3r, y7i = e3i + c*s3i;
if constexpr (LAST)
{
dst[i0*2] = y0r*rs; dst[i0*2+1] = y0i*is; dst[i1*2] = y1r*rs; dst[i1*2+1] = y1i*is;
dst[i2*2] = y2r*rs; dst[i2*2+1] = y2i*is; dst[i3*2] = y3r*rs; dst[i3*2+1] = y3i*is;
dst[i4*2] = y4r*rs; dst[i4*2+1] = y4i*is; dst[i5*2] = y5r*rs; dst[i5*2+1] = y5i*is;
dst[i6*2] = y6r*rs; dst[i6*2+1] = y6i*is; dst[i7*2] = y7r*rs; dst[i7*2+1] = y7i*is;
}
else
{
dre[i0] = y0r; dim[i0] = y0i; dre[i1] = y1r; dim[i1] = y1i;
dre[i2] = y2r; dim[i2] = y2i; dre[i3] = y3r; dim[i3] = y3i;
dre[i4] = y4r; dim[i4] = y4i; dre[i5] = y5r; dim[i5] = y5i;
dre[i6] = y6r; dim[i6] = y6i; dre[i7] = y7r; dim[i7] = y7i;
}
}
}
}
template<typename T>
void radix5_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
{
@@ -1000,6 +1052,68 @@ void radix4Last_vec(const DftStage& st, const T* sre, const T* sim, T* dst, T rs
DFT_VLOOP_END(4)
}
// radix-8 DIT stage, vector (see radix8_scalar for the structure). Odd legs first (4-point DFT +
// W8 rotations: the rotated values are kept as the sums s = (o1r+o1i, ...) and folded into the
// outputs with FMAs by +-sin45, as in FFTW's t1fv_8), then the even legs, whose pure additions
// go straight into the outputs. Peak live set is ~18 vectors (odd results 8 + even inputs 8 +
// temporaries), so a couple of spills are expected on 16-register ISAs; still one pass over the
// data instead of two radix-4 passes. LAST: fused interleave + scale into the user output.
template<typename T, typename VT, bool LAST>
void radix8_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim, T* dst, T rs, T is)
{
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
const int L = st.span, ts = st.tw_stride, VL = VTraits<VT>::vlanes();
const VT c = VSETALL((T)DftConst<T>::sin45), nc = VSETALL((T)-DftConst<T>::sin45);
const VT vrs = VSETALL(rs), vis = VSETALL(is);
auto store = [&](int i, const VT& yr, const VT& yi)
{
if constexpr (LAST)
v_store_interleave(dst + i*2, v_mul(yr, vrs), v_mul(yi, vis));
else
{
v_store(dre + i, yr); v_store(dim + i, yi);
}
};
DFT_VLOOP_BEGIN(8)
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L, i4 = i3 + L, i5 = i4 + L, i6 = i5 + L, i7 = i6 + L;
// odd legs: b1 = x1 + x5, b5 = x1 - x5, b3 = x3 + x7, b7 = x3 - x7
VT x1r, x1i, x5r, x5i, x3r, x3i, x7r, x7i;
DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j));
DFT_VCMUL(x5r, x5i, VLOAD(sre + i5), VLOAD(sim + i5), VLOAD(twr + ts*4 + j), VLOAD(twi + ts*4 + j));
VT b1r = v_add(x1r, x5r), b1i = v_add(x1i, x5i), b5r = v_sub(x1r, x5r), b5i = v_sub(x1i, x5i);
DFT_VCMUL(x3r, x3i, VLOAD(sre + i3), VLOAD(sim + i3), VLOAD(twr + ts*2 + j), VLOAD(twi + ts*2 + j));
DFT_VCMUL(x7r, x7i, VLOAD(sre + i7), VLOAD(sim + i7), VLOAD(twr + ts*6 + j), VLOAD(twi + ts*6 + j));
VT b3r = v_add(x3r, x7r), b3i = v_add(x3i, x7i), b7r = v_sub(x3r, x7r), b7i = v_sub(x3i, x7i);
// 4-point DFT of the odd legs: O0 = b1 + b3, O2 = b1 - b3, O1 = b5 - i*b7, O3 = b5 + i*b7;
// O1*W8 = c*(s1r + i*s1i), O3*W8^3 = c*(s3r - i*s3i), O2*W8^2 = (o2i, -o2r)
VT o0r = v_add(b1r, b3r), o0i = v_add(b1i, b3i), o2r = v_sub(b1r, b3r), o2i = v_sub(b1i, b3i);
VT o1r = v_add(b5r, b7i), o1i = v_sub(b5i, b7r), o3r = v_sub(b5r, b7i), o3i = v_add(b5i, b7r);
VT s1r = v_add(o1r, o1i), s1i = v_sub(o1i, o1r), s3r = v_sub(o3i, o3r), s3i = v_add(o3r, o3i);
// even legs: a0 = x0 + x4, a4 = x0 - x4, a2 = x2 + x6, a6 = x2 - x6
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x4r, x4i;
DFT_VCMUL(x4r, x4i, VLOAD(sre + i4), VLOAD(sim + i4), VLOAD(twr + ts*3 + j), VLOAD(twi + ts*3 + j));
VT a0r = v_add(x0r, x4r), a0i = v_add(x0i, x4i), a4r = v_sub(x0r, x4r), a4i = v_sub(x0i, x4i);
VT x2r, x2i, x6r, x6i;
DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j));
DFT_VCMUL(x6r, x6i, VLOAD(sre + i6), VLOAD(sim + i6), VLOAD(twr + ts*5 + j), VLOAD(twi + ts*5 + j));
VT a2r = v_add(x2r, x6r), a2i = v_add(x2i, x6i), a6r = v_sub(x2r, x6r), a6i = v_sub(x2i, x6i);
// E0 = a0 + a2 -> y0, y4; E2 = a0 - a2 -> y2, y6 (with O2*W8^2)
VT e0r = v_add(a0r, a2r), e0i = v_add(a0i, a2i);
store(i0, v_add(e0r, o0r), v_add(e0i, o0i));
store(i4, v_sub(e0r, o0r), v_sub(e0i, o0i));
VT e2r = v_sub(a0r, a2r), e2i = v_sub(a0i, a2i);
store(i2, v_add(e2r, o2i), v_sub(e2i, o2r));
store(i6, v_sub(e2r, o2i), v_add(e2i, o2r));
// E1 = a4 - i*a6 -> y1, y5 (+- c*s1); E3 = a4 + i*a6 -> y3, y7 (+- c*(s3r, -s3i))
VT e1r = v_add(a4r, a6i), e1i = v_sub(a4i, a6r);
store(i1, v_fma(s1r, c, e1r), v_fma(s1i, c, e1i));
store(i5, v_fma(s1r, nc, e1r), v_fma(s1i, nc, e1i));
VT e3r = v_sub(a4r, a6i), e3i = v_add(a4i, a6r);
store(i3, v_fma(s3r, c, e3r), v_fma(s3i, nc, e3i));
store(i7, v_fma(s3r, nc, e3r), v_fma(s3i, c, e3i));
DFT_VLOOP_END(8)
}
template<typename T, typename VT>
void radix5_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
{
@@ -1191,6 +1305,49 @@ void radix4Last_(const DftStage& st, const void* sre, const void* sim, void* dst
}
template<typename T, typename VT, typename VT128, bool haveSIMD>
void radix8_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void*)
{
#if (CV_SIMD || CV_SIMD_SCALABLE)
if constexpr (haveSIMD)
{
if (st.mode == DFT_STAGE_FULL)
{
radix8_vec<T, VT, false>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0);
return;
}
if (st.mode == DFT_STAGE_SIMD128)
{
radix8_vec<T, VT128, false>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0);
return;
}
}
#endif
radix8_scalar<T, false>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, 0, (T)0, (T)0);
}
template<typename T, typename VT, typename VT128, bool haveSIMD>
void radix8Last_(const DftStage& st, const void* sre, const void* sim, void* dst, double re_scale, double im_scale)
{
T rs = (T)re_scale, is = (T)im_scale;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if constexpr (haveSIMD)
{
if (st.mode == DFT_STAGE_FULL)
{
radix8_vec<T, VT, true>(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is);
return;
}
if (st.mode == DFT_STAGE_SIMD128)
{
radix8_vec<T, VT128, true>(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is);
return;
}
}
#endif
radix8_scalar<T, true>(st, (const T*)sre, (const T*)sim, 0, 0, (T*)dst, rs, is);
}
//////////////////////////////// group C: postprocessing ////////////////////////////////
//
// Pure map passes over the split arrays (plus the C2R / IDCT input untangles, which are the
@@ -1459,6 +1616,8 @@ DFTKernels makeKernels()
k.radix5 = radix5_<T, VT, VT128, haveSIMD>;
k.radixOdd = radixOdd_<T, VT, VT128, haveSIMD>;
k.radix4Last = radix4Last_<T, VT, VT128, haveSIMD>;
k.radix8 = radix8_<T, VT, VT128, haveSIMD>;
k.radix8Last = radix8Last_<T, VT, VT128, haveSIMD>;
k.postprocDFT = postprocDFT_<T, VT, haveSIMD>;
k.postprocRealDFT = postprocRealDFT_<T, VT, haveSIMD>;
k.postprocReal = postprocReal_<T, VT, haveSIMD>;