mirror of
https://github.com/opencv/opencv.git
synced 2026-11-17 06:47:47 -06:00
core(dft): follow-up to the new 1D DFT engine: memory, plan overhead, vectorization
Fixes the 64F 1D regressions measured on x86 after #30061 and reduces the memory of the generic odd radix. Measured without IPP (AVX2 kernels). Root cause of the 1D regressions: cv::dft() builds a fresh plan per call, and at n = 65536 the engine allocated three MB-sized blocks per call (twiddles, permutation, workspace). glibc then trims the heap after every free (dynamic M_TRIM_THRESHOLD = 2 x the largest block), so every call page-faulted the buffers again: 65536 64FC2 ran at 620 us, of which ~215 us were outside the transform itself. The old code made one ~1.5 MB allocation and escaped. Changes: - The plan's tables, the permutation and the owner's workspace are one allocation (DftPlan::build(..., with_workspace)): one malloc per cv::dft() call instead of three, and glibc's trimming is not triggered by a single freed block (its threshold is twice the largest block seen). No memory is retained between calls, no plan cache. - Static, lazily built tables for power-of-two lengths nc <= 1024: radix-4 stage twiddles (shared by all plans: they depend on (radix, span) only), digit-reversal tables and the real-FFT twiddles. Plan building for 256..2048 points is now table lookups only. - Generic odd radix: cs/sn tables of r entries with a cyclic index p*k mod r instead of h*h entries each (h = (r-1)/2). n = 4093 in double: peak RSS 72 MB -> 7.6 MB, max error 5e-13 -> 7e-15. - DFT_STAGE_SIMD128: stages whose span is shorter than the full vector but at least 128 bits run on v_float32x4 / v_float64x2 instead of scalar code (span 4 on AVX2 floats, i.e. every 4^k length; span 2 on AVX-512 doubles). - Fused twiddle-free first stage of radix 3 and 5 for odd nc (vectorized gather + DFT3/DFT5), replacing the scalar table gather and two scalar stages for lengths like 4725. - Fused last radix-4 stage writing the interleaved, scaled output (C2C and even C2R): one pass over the data less. - Leading stages run chunk by chunk (depth-first, 32 KB per array) for long transforms so that their data stays in L2. - perf: 1D dft1d/dct1d cases and 256x256 / 512x512 2D sizes; test: prime and 2*prime lengths (257, 1009, 2018, ...) in the sweep. Single thread, Core Ultra 7 255H (AVX2 kernels, no IPP), min of two perf runs, geomean over flags and row/column; speedup of the optimized build vs 5.x head and vs the pre-#30061 code: 32FC1 64FC1 32FC2 64FC2 dft1d 256 2.0 / 2.8 1.8 / 2.2 2.1 / 3.5 1.7 / 2.4 dft1d 1000 1.1 / 2.7 1.0 / 1.9 1.0 / 4.2 1.0 / 2.3 dft1d 1024 2.4 / 5.8 1.9 / 3.4 2.3 / 6.0 1.6 / 3.3 dft1d 4096 1.0 / 2.8 1.0 / 1.8 1.2 / 3.2 1.1 / 1.7 dft1d 4725 1.7 / 3.2 1.7 / 1.8 1.7 / 3.0 1.7 / 1.7 dft1d 65536 1.0 / 2.7 1.15/ 1.75 1.2 / 2.8 1.0 / 1.65 dct1d 256..65536: 1.03-1.22 / 1.8-2.6 2D dft 256x256..2048x2048: 1.05-1.15 / 2.2-3.8; 2D dct 1.04 / 2.6; matchTemplate 1.01-1.06 / 2.1-3.4; filter2D unchanged. In a plain cv::dft() loop (outside the perf harness) 65536 64FC2 goes from 620 us (5.x head) to 287 us, as the page-fault storm is gone. No case is slower than 5.x head beyond run-to-run noise (checked by alternating runs of the suspicious cases). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@ using namespace perf;
|
||||
///////////////////////////////////////////////////////dft//////////////////////////////////////////////////////////////
|
||||
|
||||
#define MAT_TYPES_DFT CV_32FC1, CV_32FC2, CV_64FC1
|
||||
#define MAT_SIZES_DFT cv::Size(320, 480), cv::Size(800, 600), cv::Size(1280, 1024), sz1080p, sz2K
|
||||
#define MAT_SIZES_DFT cv::Size(256, 256), cv::Size(512, 512), cv::Size(320, 480), cv::Size(800, 600), cv::Size(1280, 1024), sz1080p, sz2K
|
||||
CV_ENUM(FlagsType, 0, DFT_INVERSE, DFT_SCALE, DFT_COMPLEX_OUTPUT, DFT_ROWS, DFT_INVERSE|DFT_COMPLEX_OUTPUT)
|
||||
#define TEST_MATS_DFT testing::Combine(testing::Values(MAT_SIZES_DFT), testing::Values(MAT_TYPES_DFT), FlagsType::all(), testing::Values(true, false))
|
||||
|
||||
|
||||
@@ -232,6 +232,8 @@ static const DFTKernels& getDFTKernels(int depth)
|
||||
return depth == CV_32F ? k32 : k64;
|
||||
}
|
||||
|
||||
/////////////////////////////// twiddle generation ///////////////////////////////
|
||||
|
||||
// exp(-2*pi*i*m/M) in double precision, m in [0, M): m = B*a + b -> hi[a]*lo[b] with B a power of
|
||||
// two ~ sqrt(M). Both small tables are built by the rotation recurrence, re-seeded with exact
|
||||
// sin/cos every 16 entries, so the error stays within a few ulps while only ~(B + M/B)/16 + 4
|
||||
@@ -292,157 +294,77 @@ struct DftTwiddleGen
|
||||
}
|
||||
};
|
||||
|
||||
// Stage twiddles of a radix-r (r <= 8) stage with span L, leg-major: twr/twi[(q-1)*ts + j] =
|
||||
// W_{rL}^{q*j} = gen(q*j*step), step = gen.M/(r*L). Leg 1 (W_{rL}^j) by K independent interleaved
|
||||
// rotation recurrences (a single chain is latency-bound), each re-seeded from the exact generator
|
||||
// every 16 of its steps; legs q >= 2 as t_q(j) = t_{q-1}(j)*t_1(j), independent per entry
|
||||
// (error <= q ulps). twd is a scratch of 2*L doubles.
|
||||
template<typename T>
|
||||
void DftPlan::fillTables(size_t ntw, bool need_rtw, bool need_dct)
|
||||
static void fillStageTwiddles(const DftTwiddleGen& gen, int r, int L, int ts, T* twr, T* twi, double* twd)
|
||||
{
|
||||
DftPlan& p = *this;
|
||||
// all angles come from one base table exp(-2*pi*i*m/M), M a multiple of every angle grid used:
|
||||
// C2C: M = nc; real kinds with even n: M = n (=2nc) or 4n for DCT (W_4n^k); odd real: M = n = nc.
|
||||
int M = nc*(need_dct ? 8 : need_rtw ? 2 : 1);
|
||||
DftTwiddleGen gen;
|
||||
gen.init(M);
|
||||
p.tw.allocate((ntw*sizeof(T) + sizeof(double) - 1)/sizeof(double) + 1);
|
||||
T* cur = (T*)p.tw.data();
|
||||
int maxL = 1;
|
||||
for (int s = 0; s < p.nstages; s++)
|
||||
if (p.stages[s].radix <= 8) maxL = std::max(maxL, p.stages[s].span);
|
||||
AutoBuffer<double, 2048> twd_buf(2*maxL); // leg-1 twiddles of the current stage in double
|
||||
double* twd = twd_buf.data();
|
||||
for (int s = 0; s < p.nstages; s++)
|
||||
int step = gen.M/(r*L);
|
||||
constexpr int K = 8;
|
||||
double cr[K], ci[K], wr, wi;
|
||||
gen.get(K*step, wr, wi);
|
||||
double* t1r = twd; double* t1i = twd + L; // leg 1 kept in double for the products
|
||||
for (int j0 = 0; j0 < L; j0 += 16*K) // blocks of 16 steps per chain
|
||||
{
|
||||
DftStage& st = p.stages[s];
|
||||
int r = st.radix, L = st.span, ts = st.tw_stride;
|
||||
T* twr = cur; T* twi = cur + (r - 1)*ts;
|
||||
cur = twi + (r - 1)*ts;
|
||||
int step = M/(r*L);
|
||||
if (r <= 8)
|
||||
{
|
||||
// Leg 1 (W_{rL}^j) by K independent interleaved rotation recurrences (a single chain is
|
||||
// latency-bound), each re-seeded from the exact generator every 16 of its steps; legs
|
||||
// q >= 2 as t_q(j) = t_{q-1}(j)*t_1(j), independent per entry (error <= q ulps).
|
||||
constexpr int K = 8;
|
||||
double cr[K], ci[K], wr, wi;
|
||||
gen.get(K*step, wr, wi);
|
||||
double* t1r = twd; double* t1i = twd + L; // leg 1 kept in double for the products
|
||||
for (int j0 = 0; j0 < L; j0 += 16*K) // blocks of 16 steps per chain
|
||||
for (int k = 0; k < K; k++)
|
||||
gen.get(std::min(j0 + k, L - 1)*step, cr[k], ci[k]);
|
||||
int j1 = std::min(j0 + 16*K, L);
|
||||
for (int j = j0; j < j1; j += K)
|
||||
for (int k = 0; k < K && j + k < L; k++)
|
||||
{
|
||||
for (int k = 0; k < K; k++)
|
||||
gen.get(std::min(j0 + k, L - 1)*step, cr[k], ci[k]);
|
||||
int j1 = std::min(j0 + 16*K, L);
|
||||
for (int j = j0; j < j1; j += K)
|
||||
for (int k = 0; k < K && j + k < L; k++)
|
||||
{
|
||||
t1r[j+k] = cr[k]; t1i[j+k] = ci[k];
|
||||
double t = cr[k]*wr - ci[k]*wi;
|
||||
ci[k] = cr[k]*wi + ci[k]*wr;
|
||||
cr[k] = t;
|
||||
}
|
||||
t1r[j+k] = cr[k]; t1i[j+k] = ci[k];
|
||||
double t = cr[k]*wr - ci[k]*wi;
|
||||
ci[k] = cr[k]*wi + ci[k]*wr;
|
||||
cr[k] = t;
|
||||
}
|
||||
for (int j = 0; j < L; j++) // separate pass: vectorizes, the recurrence loop does not
|
||||
{
|
||||
twr[j] = (T)t1r[j]; twi[j] = (T)t1i[j];
|
||||
}
|
||||
for (int q = 2; q < r; q++)
|
||||
{
|
||||
const T* pr = twr + (q-2)*ts; const T* pi = twi + (q-2)*ts;
|
||||
T* tr = twr + (q-1)*ts; T* ti = twi + (q-1)*ts;
|
||||
for (int j = 0; j < L; j++)
|
||||
{
|
||||
double ar = pr[j], ai = pi[j], br = t1r[j], bi = t1i[j];
|
||||
tr[j] = (T)(ar*br - ai*bi);
|
||||
ti[j] = (T)(ar*bi + ai*br);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// large odd radix: exact-ish generator lookup per entry
|
||||
for (int q = 1; q < r; q++)
|
||||
for (int j = 0; j < L; j++)
|
||||
{
|
||||
double re, im;
|
||||
gen.get(q*j*step, re, im);
|
||||
twr[(q-1)*ts + j] = (T)re;
|
||||
twi[(q-1)*ts + j] = (T)im;
|
||||
}
|
||||
}
|
||||
st.tw_re = twr; st.tw_im = twi;
|
||||
st.cs = st.sn = 0;
|
||||
if (r >= 7)
|
||||
{
|
||||
int h = (r - 1)/2;
|
||||
T* cs = cur; T* sn = cur + h*h;
|
||||
cur = sn + h*h;
|
||||
for (int pp = 1; pp <= h; pp++)
|
||||
for (int k = 1; k <= h; k++)
|
||||
{
|
||||
double t = 2*CV_PI*((double)pp*k/r);
|
||||
cs[(pp-1)*h + (k-1)] = (T)cos(t);
|
||||
sn[(pp-1)*h + (k-1)] = (T)sin(t);
|
||||
}
|
||||
st.cs = cs; st.sn = sn;
|
||||
}
|
||||
}
|
||||
p.rtw_re = p.rtw_im = p.dct_re = p.dct_im = 0;
|
||||
if (need_rtw)
|
||||
for (int j = 0; j < L; j++) // separate pass: vectorizes, the recurrence loop does not
|
||||
{
|
||||
// W_n^k, k in [0, nc]
|
||||
T* wr = cur; T* wi = cur + (nc + 1);
|
||||
cur = wi + (nc + 1);
|
||||
int step = M/n;
|
||||
for (int k = 0; k <= nc; k++)
|
||||
{
|
||||
double re, im;
|
||||
gen.get(k*step, re, im);
|
||||
wr[k] = (T)re; wi[k] = (T)im;
|
||||
}
|
||||
p.rtw_re = wr; p.rtw_im = wi;
|
||||
twr[j] = (T)t1r[j]; twi[j] = (T)t1i[j];
|
||||
}
|
||||
if (need_dct)
|
||||
for (int q = 2; q < r; q++)
|
||||
{
|
||||
// s*W_4n^k, k in [0, nc]; s = 2*sqrt(1/(2n)) forward, sqrt(1/(2n)) inverse (the original normalization)
|
||||
T* wr = cur; T* wi = cur + (nc + 1);
|
||||
cur = wi + (nc + 1);
|
||||
double scale = (p.kind == DFT_KIND_DCT ? 2 : 1)*std::sqrt(1./(2*n));
|
||||
int step = M/(4*n);
|
||||
for (int k = 0; k <= nc; k++)
|
||||
const T* pr = twr + (q-2)*ts; const T* pi = twi + (q-2)*ts;
|
||||
T* tr = twr + (q-1)*ts; T* ti = twi + (q-1)*ts;
|
||||
for (int j = 0; j < L; j++)
|
||||
{
|
||||
double re, im;
|
||||
gen.get(k*step, re, im);
|
||||
wr[k] = (T)(re*scale); wi[k] = (T)(im*scale);
|
||||
double ar = pr[j], ai = pi[j], br = t1r[j], bi = t1i[j];
|
||||
tr[j] = (T)(ar*br - ai*bi);
|
||||
ti[j] = (T)(ar*bi + ai*br);
|
||||
}
|
||||
p.dct_re = wr; p.dct_im = wi;
|
||||
}
|
||||
}
|
||||
|
||||
// See dxt.hpp for the contract (in particular where `vl` must come from).
|
||||
void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
// W_n^k, k in [0, nc] (n = 2*nc), scaled by `scale`, from gen with M a multiple of n
|
||||
template<typename T>
|
||||
static void fillRealTwiddles(const DftTwiddleGen& gen, int n, int nc, double scale, T* wr, T* wi)
|
||||
{
|
||||
DftPlan& p = *this;
|
||||
CV_Assert(_n >= 1 && _vl >= 1);
|
||||
CV_Assert(_depth == CV_32F || _depth == CV_64F);
|
||||
bool real_kind = _kind != DFT_KIND_C2C;
|
||||
bool dct_kind = _kind == DFT_KIND_DCT || _kind == DFT_KIND_IDCT;
|
||||
if (dct_kind)
|
||||
CV_Assert(_n % 2 == 0); // cv::dct() supports even n only (n == 1 is handled by the caller)
|
||||
p.kind = _kind; p.n = _n; p.depth = _depth; p.vl = _vl;
|
||||
p.nc = real_kind && _n % 2 == 0 ? _n/2 : _n;
|
||||
CV_Assert(nc < (1 << 28));
|
||||
size_t esz = depth == CV_32F ? sizeof(float) : sizeof(double);
|
||||
p.real_input = kind == DFT_KIND_R2C && (n & 1) != 0;
|
||||
p.need_tmp = kind == DFT_KIND_C2R || kind == DFT_KIND_DCT || kind == DFT_KIND_IDCT;
|
||||
p.pingpong = false;
|
||||
int step = gen.M/n;
|
||||
for (int k = 0; k <= nc; k++)
|
||||
{
|
||||
double re, im;
|
||||
gen.get(k*step, re, im);
|
||||
wr[k] = (T)(re*scale); wi[k] = (T)(im*scale);
|
||||
}
|
||||
}
|
||||
|
||||
// factorization: 2^k part first (fused first stage r0 + radix-4 stages), then 3s, 5s, other odd.
|
||||
// (A radix-8 middle stage was tried and measured slower on NEON: with per-leg twiddle tables it
|
||||
// does not save loads over two radix-4 stages and keeps 16 data vectors live.)
|
||||
// The radix sequence of a length nc: the fused first stage r0 (1, 2, 3, 4, 5 or 8), then the
|
||||
// radix-4 stages of the remaining 2^k part, then 3s, 5s and the other odd factors ascending.
|
||||
// Even nc: r0 from the power of two. Odd nc: r0 = 5 or 3 when available (a vectorized,
|
||||
// twiddle-free first stage; it also makes the spans of the following stages larger), else 1.
|
||||
// (A radix-8 middle stage was tried and measured slower on NEON: with per-leg twiddle tables it
|
||||
// does not save loads over two radix-4 stages and keeps 16 data vectors live.)
|
||||
static int dftFactorize(int nc, int radices[DFT_MAX_STAGES], int& nst)
|
||||
{
|
||||
int k2 = 0, rest = nc;
|
||||
while ((rest & 1) == 0) { rest >>= 1; k2++; }
|
||||
int r0 = k2 == 0 ? 1 : k2 == 1 ? 2 : (k2 & 1) ? 8 : 4;
|
||||
p.first_radix = r0;
|
||||
int radices[DFT_MAX_STAGES], nst = 0;
|
||||
for (int m = k2 - (r0 == 1 ? 0 : r0 == 2 ? 1 : r0 == 4 ? 2 : 3); m > 0; m -= 2) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 4; }
|
||||
rest = nc >> k2;
|
||||
int r0 = k2 == 0 ? (rest % 5 == 0 ? 5 : rest % 3 == 0 ? 3 : 1) : k2 == 1 ? 2 : (k2 & 1) ? 8 : 4;
|
||||
nst = 0;
|
||||
for (int m = k2 - (r0 == 2 ? 1 : r0 == 4 ? 2 : r0 == 8 ? 3 : 0); m > 0; m -= 2) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 4; }
|
||||
if (r0 == 3 || r0 == 5) rest /= r0;
|
||||
while (rest % 3 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 3; rest /= 3; }
|
||||
while (rest % 5 == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = 5; rest /= 5; }
|
||||
for (int f = 7; rest > 1; f += 2)
|
||||
@@ -455,48 +377,18 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
}
|
||||
while (rest % f == 0) { CV_Assert(nst < DFT_MAX_STAGES); radices[nst++] = f; rest /= f; }
|
||||
}
|
||||
p.nstages = nst;
|
||||
return r0;
|
||||
}
|
||||
|
||||
// stages
|
||||
int L = r0, max_h = 0;
|
||||
size_t ntw = 0;
|
||||
for (int s = 0; s < nst; s++)
|
||||
{
|
||||
DftStage& st = p.stages[s];
|
||||
int r = radices[s];
|
||||
st.radix = r; st.span = L; st.ngroups = nc/(r*L); st.tw_stride = L;
|
||||
if (vl > 1 && L >= vl)
|
||||
{
|
||||
st.mode = DFT_STAGE_FULL;
|
||||
st.nsteps = st.ngroups*((L + vl - 1)/vl);
|
||||
if (L % vl != 0)
|
||||
p.pingpong = true; // back-off clamp => overlapping stores => must be out-of-place
|
||||
}
|
||||
else
|
||||
{
|
||||
st.mode = DFT_STAGE_SCALAR;
|
||||
st.nsteps = st.ngroups*L;
|
||||
}
|
||||
ntw += 2*(size_t)(r - 1)*st.tw_stride;
|
||||
if (r >= 7)
|
||||
{
|
||||
int h = (r - 1)/2;
|
||||
ntw += 2*(size_t)h*h;
|
||||
max_h = std::max(max_h, h);
|
||||
}
|
||||
L *= r;
|
||||
}
|
||||
CV_Assert(L == nc);
|
||||
|
||||
// digit-reversal: position p = sum_s d_s*L_s holds input element sum_s d_s*M_s, M_s = nc/(L_s*r_s);
|
||||
// the radix sequence is r0 (if > 1) followed by the stage radices.
|
||||
// - r0 == 1 (odd nc): itab holds two element indices (re, im) per complex element for the
|
||||
// scalar gather (im index unused when real_input);
|
||||
// - r0 > 1: the r0 inputs of group g are the column x[base(g) + q*nc/r0]; itab[base(g)] = g*r0
|
||||
// tells the transposed first stage where the outputs of column base(g) go.
|
||||
int seq[DFT_MAX_STAGES + 1], nseq = 0;
|
||||
if (r0 > 1) seq[nseq++] = r0;
|
||||
for (int s = 0; s < nst; s++) seq[nseq++] = radices[s];
|
||||
// Digit-reversal permutation for the radix sequence seq[0..nseq) (seq[0] = r0 when r0 > 1):
|
||||
// position p = sum_s d_s*L_s holds input element sum_s d_s*M_s, M_s = nc/(L_s*r_s).
|
||||
// - r0 == 1 (odd nc without 3 or 5): itab holds two element indices (re, im) per complex element
|
||||
// for the scalar gather (im index == re index when real_input, i.e. a real source);
|
||||
// - r0 > 1: the r0 inputs of group g are the column x[base(g) + q*nc/r0]; gtab[base(g)] = g*r0
|
||||
// tells the transposed first stage where the outputs of column base(g) go. nc/r0 entries.
|
||||
static void dftBuildPermutation(int nc, const int* seq, int nseq, bool real_input, int* itab)
|
||||
{
|
||||
int r0 = nseq > 0 ? seq[0] : 1;
|
||||
int digits[DFT_MAX_STAGES + 1], weights[DFT_MAX_STAGES + 1];
|
||||
{
|
||||
int Ls = 1;
|
||||
@@ -507,10 +399,10 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
Ls *= seq[s];
|
||||
}
|
||||
}
|
||||
if (r0 == 1)
|
||||
bool first_fused = r0 == 2 || r0 == 3 || r0 == 4 || r0 == 5 || r0 == 8;
|
||||
if (!first_fused)
|
||||
{
|
||||
p.itab.allocate(nc*2);
|
||||
int* it_all = p.itab.data();
|
||||
int* it_all = itab;
|
||||
// the two lowest digits are enumerated by plain nested loops, the higher ones by a counter
|
||||
int n_in = std::min(nseq, 2);
|
||||
int r_0 = n_in > 0 ? seq[0] : 1, w_0 = n_in > 0 ? weights[0] : 0;
|
||||
@@ -521,7 +413,7 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
{
|
||||
int rev = rev_hi + d1*w_1;
|
||||
int* it = it_all + pos*2;
|
||||
if (p.real_input)
|
||||
if (real_input)
|
||||
{
|
||||
for (int d0 = 0; d0 < r_0; d0++, rev += w_0)
|
||||
{
|
||||
@@ -554,8 +446,7 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
// g follows incrementally with the digit weights gw[s] = L_s/r0. The two fastest digits
|
||||
// are enumerated by plain nested loops.
|
||||
int M = nc/r0;
|
||||
p.itab.allocate(M);
|
||||
int* gtab = p.itab.data();
|
||||
int* gtab = itab;
|
||||
int gw[DFT_MAX_STAGES + 1];
|
||||
{
|
||||
int Ls = 1;
|
||||
@@ -586,15 +477,254 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// twiddle tables
|
||||
/////////////////////////////// static tables for 2^k lengths <= 1024 ///////////////////////////////
|
||||
//
|
||||
// The power-of-two lengths are the common case (matchTemplate/filter2D tiles, 256/512/1024 rows),
|
||||
// and cv::dft() builds a plan per call, so their tables are built once per process and shared:
|
||||
// - radix-4 stage twiddles W_{4L}^{q*j} for every span L = 2^k, 4 <= L <= 256 (they depend on
|
||||
// (radix, span) only, not on the transform length);
|
||||
// - the permutation gtab of nc = 2^k, 4 <= nc <= 1024 (r0 in {4, 8});
|
||||
// - W_n^k, k in [0, nc], of the even real kinds (R2C/C2R) with nc = 2^k <= 1024.
|
||||
// ~90 KB for both depths. DCT tables are kind- and length-specific and stay per plan.
|
||||
enum { DFT_STATIC_MAX_LOG2 = 10, DFT_STATIC_MAX_NC = 1 << DFT_STATIC_MAX_LOG2 };
|
||||
|
||||
template<typename T> struct DftStaticTables
|
||||
{
|
||||
AutoBuffer<T> tw; // stage tables, then the real twiddles
|
||||
AutoBuffer<int> itab;
|
||||
const T* tw_re[DFT_STATIC_MAX_LOG2]; // [log2(L)], L in [4, 256]
|
||||
const T* tw_im[DFT_STATIC_MAX_LOG2];
|
||||
const T* rtw_re[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024]
|
||||
const T* rtw_im[DFT_STATIC_MAX_LOG2 + 1];
|
||||
const int* gtab[DFT_STATIC_MAX_LOG2 + 1]; // [log2(nc)], nc in [4, 1024]
|
||||
|
||||
DftStaticTables()
|
||||
{
|
||||
size_t ntw = 0, nitab = 0;
|
||||
for (int k = 2; k <= DFT_STATIC_MAX_LOG2; k++)
|
||||
{
|
||||
int nc = 1 << k;
|
||||
if (k <= 8) ntw += 6*(size_t)nc; // radix-4 stage with L = nc: 3 legs x (re, im) x L
|
||||
ntw += 2*(size_t)(nc + 1); // W_{2nc}^k, k in [0, nc]
|
||||
nitab += nc/(k & 1 ? 8 : 4);
|
||||
}
|
||||
tw.allocate(ntw);
|
||||
itab.allocate(nitab);
|
||||
T* cur = tw.data();
|
||||
int* icur = itab.data();
|
||||
AutoBuffer<double> twd(2*256);
|
||||
for (int k = 2; k <= DFT_STATIC_MAX_LOG2; k++)
|
||||
{
|
||||
int nc = 1 << k;
|
||||
if (k <= 8)
|
||||
{
|
||||
int L = nc;
|
||||
DftTwiddleGen gen;
|
||||
gen.init(4*L);
|
||||
tw_re[k] = cur; tw_im[k] = cur + 3*L;
|
||||
fillStageTwiddles<T>(gen, 4, L, L, cur, cur + 3*L, twd.data());
|
||||
cur += 6*L;
|
||||
}
|
||||
{
|
||||
DftTwiddleGen gen;
|
||||
gen.init(2*nc);
|
||||
rtw_re[k] = cur; rtw_im[k] = cur + nc + 1;
|
||||
fillRealTwiddles<T>(gen, 2*nc, nc, 1., cur, cur + nc + 1);
|
||||
cur += 2*(nc + 1);
|
||||
}
|
||||
{
|
||||
int radices[DFT_MAX_STAGES], nst;
|
||||
int r0 = dftFactorize(nc, radices, nst);
|
||||
int seq[DFT_MAX_STAGES + 1], nseq = 0;
|
||||
seq[nseq++] = r0;
|
||||
for (int s = 0; s < nst; s++) seq[nseq++] = radices[s];
|
||||
gtab[k] = icur;
|
||||
dftBuildPermutation(nc, seq, nseq, false, icur);
|
||||
icur += nc/r0;
|
||||
}
|
||||
}
|
||||
CV_Assert(cur == tw.data() + ntw && icur == itab.data() + nitab);
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T> static const DftStaticTables<T>& getDftStaticTables()
|
||||
{
|
||||
static const DftStaticTables<T> tables;
|
||||
return tables;
|
||||
}
|
||||
|
||||
static inline bool dftIsPow2(int x) { return x > 0 && (x & (x - 1)) == 0; }
|
||||
static inline int dftLog2(int x) { int k = 0; while ((1 << k) < x) k++; return k; }
|
||||
|
||||
template<typename T>
|
||||
void DftPlan::fillTables(T* dst, size_t ntw, bool need_rtw, bool need_dct, bool static_stages, bool static_rtw)
|
||||
{
|
||||
DftPlan& p = *this;
|
||||
// all angles come from one base table exp(-2*pi*i*m/M), M a multiple of every angle grid used:
|
||||
// C2C: M = nc; real kinds with even n: M = n (=2nc) or 4n for DCT (W_4n^k); odd real: M = n = nc.
|
||||
int M = nc*(need_dct ? 8 : need_rtw ? 2 : 1);
|
||||
DftTwiddleGen gen;
|
||||
if (ntw > 0)
|
||||
gen.init(M);
|
||||
T* cur = dst;
|
||||
const DftStaticTables<T>* st_tables = static_stages || static_rtw ? &getDftStaticTables<T>() : 0;
|
||||
int maxL = 1;
|
||||
for (int s = 0; s < p.nstages; s++)
|
||||
if (p.stages[s].radix <= 8 && !static_stages) maxL = std::max(maxL, p.stages[s].span);
|
||||
AutoBuffer<double, 2048> twd_buf(2*maxL); // leg-1 twiddles of the current stage in double
|
||||
double* twd = twd_buf.data();
|
||||
for (int s = 0; s < p.nstages; s++)
|
||||
{
|
||||
DftStage& st = p.stages[s];
|
||||
int r = st.radix, L = st.span, ts = st.tw_stride;
|
||||
st.cs = st.sn = 0;
|
||||
if (static_stages)
|
||||
{
|
||||
CV_DbgAssert(r == 4 && dftIsPow2(L) && L >= 4 && L <= 256);
|
||||
int k = dftLog2(L);
|
||||
st.tw_re = st_tables->tw_re[k]; st.tw_im = st_tables->tw_im[k];
|
||||
continue;
|
||||
}
|
||||
T* twr = cur; T* twi = cur + (r - 1)*ts;
|
||||
cur = twi + (r - 1)*ts;
|
||||
if (r <= 8)
|
||||
fillStageTwiddles<T>(gen, r, L, ts, twr, twi, twd);
|
||||
else
|
||||
{
|
||||
// large odd radix: exact-ish generator lookup per entry
|
||||
int step = M/(r*L);
|
||||
for (int q = 1; q < r; q++)
|
||||
for (int j = 0; j < L; j++)
|
||||
{
|
||||
double re, im;
|
||||
gen.get(q*j*step, re, im);
|
||||
twr[(q-1)*ts + j] = (T)re;
|
||||
twi[(q-1)*ts + j] = (T)im;
|
||||
}
|
||||
}
|
||||
st.tw_re = twr; st.tw_im = twi;
|
||||
if (r >= 7)
|
||||
{
|
||||
// r entries (not h*h): the butterfly walks them with the cyclic index p*k mod r
|
||||
T* cs = cur; T* sn = cur + r;
|
||||
cur = sn + r;
|
||||
for (int m = 0; m < r; m++)
|
||||
{
|
||||
double t = 2*CV_PI*((double)m/r);
|
||||
cs[m] = (T)cos(t);
|
||||
sn[m] = (T)sin(t);
|
||||
}
|
||||
st.cs = cs; st.sn = sn;
|
||||
}
|
||||
}
|
||||
p.rtw_re = p.rtw_im = p.dct_re = p.dct_im = 0;
|
||||
if (need_rtw)
|
||||
{
|
||||
if (static_rtw)
|
||||
{
|
||||
int k = dftLog2(nc);
|
||||
p.rtw_re = st_tables->rtw_re[k]; p.rtw_im = st_tables->rtw_im[k];
|
||||
}
|
||||
else
|
||||
{
|
||||
// W_n^k, k in [0, nc]
|
||||
T* wr = cur; T* wi = cur + (nc + 1);
|
||||
cur = wi + (nc + 1);
|
||||
fillRealTwiddles<T>(gen, n, nc, 1., wr, wi);
|
||||
p.rtw_re = wr; p.rtw_im = wi;
|
||||
}
|
||||
}
|
||||
if (need_dct)
|
||||
{
|
||||
// s*W_4n^k, k in [0, nc]; s = 2*sqrt(1/(2n)) forward, sqrt(1/(2n)) inverse (the original normalization)
|
||||
T* wr = cur; T* wi = cur + (nc + 1);
|
||||
cur = wi + (nc + 1);
|
||||
double scale = (p.kind == DFT_KIND_DCT ? 2 : 1)*std::sqrt(1./(2*n));
|
||||
fillRealTwiddles<T>(gen, 4*n, nc, scale, wr, wi);
|
||||
p.dct_re = wr; p.dct_im = wi;
|
||||
}
|
||||
CV_Assert((size_t)(cur - dst) == ntw);
|
||||
}
|
||||
|
||||
// Leading stages whose groups (radix*span elements) fit into a chunk of DFT_CHUNK_BYTES per array
|
||||
// are run chunk by chunk (depth-first) by the driver: the chunk stays in L2 across those stages
|
||||
// instead of streaming the whole array through the cache once per stage.
|
||||
enum { DFT_CHUNK_BYTES = 1 << 15 };
|
||||
|
||||
// See dxt.hpp for the contract (in particular where `vl` and `vl128` must come from).
|
||||
void DftPlan::build(int _kind, int _n, int _depth, int _vl, int _vl128, bool with_workspace)
|
||||
{
|
||||
DftPlan& p = *this;
|
||||
CV_Assert(_n >= 1 && _vl >= 1 && _vl128 >= 1 && _vl128 <= _vl);
|
||||
CV_Assert(_depth == CV_32F || _depth == CV_64F);
|
||||
bool real_kind = _kind != DFT_KIND_C2C;
|
||||
bool dct_kind = _kind == DFT_KIND_DCT || _kind == DFT_KIND_IDCT;
|
||||
if (dct_kind)
|
||||
CV_Assert(_n % 2 == 0); // cv::dct() supports even n only (n == 1 is handled by the caller)
|
||||
p.kind = _kind; p.n = _n; p.depth = _depth; p.vl = _vl;
|
||||
p.nc = real_kind && _n % 2 == 0 ? _n/2 : _n;
|
||||
CV_Assert(nc < (1 << 28));
|
||||
size_t esz = depth == CV_32F ? sizeof(float) : sizeof(double);
|
||||
p.real_input = kind == DFT_KIND_R2C && (n & 1) != 0;
|
||||
p.need_tmp = kind == DFT_KIND_C2R || kind == DFT_KIND_DCT || kind == DFT_KIND_IDCT;
|
||||
p.pingpong = false;
|
||||
|
||||
int radices[DFT_MAX_STAGES], nst = 0;
|
||||
int r0 = dftFactorize(nc, radices, nst);
|
||||
p.first_radix = r0;
|
||||
p.nstages = nst;
|
||||
|
||||
// stages
|
||||
int L = r0, max_h = 0;
|
||||
size_t ntw = 0;
|
||||
int vl128 = _vl128;
|
||||
for (int s = 0; s < nst; s++)
|
||||
{
|
||||
DftStage& st = p.stages[s];
|
||||
int r = radices[s];
|
||||
st.radix = r; st.span = L; st.ngroups = nc/(r*L); st.tw_stride = L;
|
||||
if (vl > 1 && L >= vl)
|
||||
{
|
||||
st.mode = DFT_STAGE_FULL;
|
||||
st.steps_per_group = (L + vl - 1)/vl;
|
||||
if (L % vl != 0)
|
||||
p.pingpong = true; // back-off clamp => overlapping stores => must be out-of-place
|
||||
}
|
||||
else if (vl128 > 1 && vl128 < vl && L >= vl128)
|
||||
{
|
||||
st.mode = DFT_STAGE_SIMD128;
|
||||
st.steps_per_group = (L + vl128 - 1)/vl128;
|
||||
if (L % vl128 != 0)
|
||||
p.pingpong = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
st.mode = DFT_STAGE_SCALAR;
|
||||
st.steps_per_group = L;
|
||||
}
|
||||
st.nsteps = st.ngroups*st.steps_per_group;
|
||||
ntw += 2*(size_t)(r - 1)*st.tw_stride;
|
||||
if (r >= 7)
|
||||
{
|
||||
ntw += 2*(size_t)r;
|
||||
max_h = std::max(max_h, (r - 1)/2);
|
||||
}
|
||||
L *= r;
|
||||
}
|
||||
CV_Assert(L == nc);
|
||||
|
||||
// the static tables cover the stage twiddles of nc = 2^k <= 1024 (all stages radix 4,
|
||||
// L <= 256), the permutation of 4 <= nc = 2^k <= 1024 and W_n^k of the even real kinds
|
||||
bool static_pow2 = dftIsPow2(nc) && nc >= 4 && nc <= DFT_STATIC_MAX_NC;
|
||||
bool need_rtw = real_kind && n % 2 == 0;
|
||||
if (need_rtw) ntw += 2*(size_t)(nc + 1);
|
||||
bool static_stages = static_pow2;
|
||||
bool static_rtw = static_pow2 && need_rtw && !dct_kind;
|
||||
if (static_stages)
|
||||
ntw = 0;
|
||||
if (need_rtw && !static_rtw) ntw += 2*(size_t)(nc + 1);
|
||||
if (dct_kind) ntw += 2*(size_t)(nc + 1);
|
||||
if (depth == CV_32F)
|
||||
fillTables<float>(ntw, need_rtw, dct_kind);
|
||||
else
|
||||
fillTables<double>(ntw, need_rtw, dct_kind);
|
||||
|
||||
// workspace layout (from the 64-byte aligned base): pair0 (re, im), optional pair1 (re, im) or
|
||||
// the interleaved temp, radix-odd scratch (a[h], b[h] complex per lane).
|
||||
@@ -604,6 +734,67 @@ void DftPlan::build(int _kind, int _n, int _depth, int _vl)
|
||||
p.scratch_ofs = pair_bytes*(two_pairs ? 2 : 1);
|
||||
size_t scratch_bytes = 4*(size_t)max_h*std::max(vl, 1)*esz;
|
||||
p.ws_bytes = p.scratch_ofs + scratch_bytes + 64;
|
||||
|
||||
// ONE allocation for the generated tables, the permutation and (optionally) the owner's
|
||||
// workspace (see the note in dxt.hpp)
|
||||
size_t nitab = static_pow2 ? 0 : r0 == 1 ? 2*(size_t)nc : (size_t)nc/r0;
|
||||
size_t itab_ofs = ntw*esz;
|
||||
size_t ws_ofs = alignSize(itab_ofs + nitab*sizeof(int), 64);
|
||||
size_t total_bytes = ws_ofs + (with_workspace ? p.ws_bytes : 0);
|
||||
p.buf.allocate(total_bytes/sizeof(double) + 1);
|
||||
uchar* base = (uchar*)p.buf.data();
|
||||
p.workspace = with_workspace ? base + ws_ofs : 0;
|
||||
|
||||
// digit-reversal permutation (see dftBuildPermutation)
|
||||
int seq[DFT_MAX_STAGES + 1], nseq = 0;
|
||||
if (r0 > 1) seq[nseq++] = r0;
|
||||
for (int s = 0; s < nst; s++) seq[nseq++] = radices[s];
|
||||
if (static_pow2)
|
||||
p.itab = depth == CV_32F ? getDftStaticTables<float>().gtab[dftLog2(nc)]
|
||||
: getDftStaticTables<double>().gtab[dftLog2(nc)];
|
||||
else
|
||||
{
|
||||
int* itab_ = (int*)(base + itab_ofs);
|
||||
dftBuildPermutation(nc, seq, nseq, p.real_input, itab_);
|
||||
p.itab = itab_;
|
||||
}
|
||||
|
||||
// twiddle tables
|
||||
if (depth == CV_32F)
|
||||
fillTables<float>((float*)base, ntw, need_rtw, dct_kind, static_stages, static_rtw);
|
||||
else
|
||||
fillTables<double>((double*)base, ntw, need_rtw, dct_kind, static_stages, static_rtw);
|
||||
|
||||
// the last radix-4 stage of C2C / even C2R can write the interleaved scaled output directly
|
||||
p.fused_last = nst > 0 && radices[nst - 1] == 4 &&
|
||||
(kind == DFT_KIND_C2C || (kind == DFT_KIND_C2R && nc < n));
|
||||
|
||||
// depth-first chunking of the leading stages (see DFT_CHUNK_BYTES)
|
||||
p.nchunk_stages = 0;
|
||||
{
|
||||
int chunk = (int)(DFT_CHUNK_BYTES/esz);
|
||||
if (nc > chunk)
|
||||
{
|
||||
int m = 0, avail = nst - (p.fused_last ? 1 : 0); // the fused last stage never runs chunked
|
||||
while (m < avail && p.stages[m].radix*p.stages[m].span <= chunk) m++;
|
||||
// chunking pays off only if at least two stages are grouped
|
||||
if (m >= 2)
|
||||
p.nchunk_stages = m;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// one stage over [i0, i0 + len) elements of the arrays (len a multiple of radix*span)
|
||||
static inline void runStage(const DFTKernels& k, const DftStage& st0, int i0, int len, size_t esz,
|
||||
const uchar* sre, const uchar* sim, uchar* dre, uchar* dim, uchar* scratch)
|
||||
{
|
||||
DftStage st = st0;
|
||||
st.ngroups = len/(st.radix*st.span);
|
||||
st.nsteps = st.ngroups*st.steps_per_group;
|
||||
DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 3 ? k.radix3 :
|
||||
st.radix == 5 ? k.radix5 : k.radixOdd;
|
||||
size_t ofs = (size_t)i0*esz;
|
||||
fn(st, sre + ofs, sim + ofs, dre + ofs, dim + ofs, scratch);
|
||||
}
|
||||
|
||||
// Runs the transform described by c.plan: A (preproc) -> B stages (ping-pong) -> C (postproc).
|
||||
@@ -645,25 +836,56 @@ static void runDft(const OcvDftOptions& c, const void* src, size_t sstep, void*
|
||||
CV_DbgAssert(astep == 1 || p.first_radix == 1);
|
||||
|
||||
DftPreprocFunc pre = p.first_radix == 1 ? k.preprocRadix0 : p.first_radix == 2 ? k.preprocRadix2 :
|
||||
p.first_radix == 4 ? k.preprocRadix4 : k.preprocRadix8;
|
||||
p.first_radix == 3 ? k.preprocRadix3 : p.first_radix == 4 ? k.preprocRadix4 :
|
||||
p.first_radix == 5 ? k.preprocRadix5 : k.preprocRadix8;
|
||||
pre(p, asrc, astep, re0, im0, conj);
|
||||
|
||||
uchar *sre = re0, *sim = im0, *dre = re1, *dim = im1;
|
||||
for (int i = 0; i < p.nstages; i++)
|
||||
int nstages = p.nstages - (p.fused_last ? 1 : 0);
|
||||
int s = 0;
|
||||
if (p.nchunk_stages > 0)
|
||||
{
|
||||
// the leading stages chunk by chunk; every chunk runs the same number of stages, so the
|
||||
// ping-pong parity after them is the same for all chunks
|
||||
int chunk = p.stages[p.nchunk_stages].span;
|
||||
for (int i0 = 0; i0 < p.nc; i0 += chunk)
|
||||
{
|
||||
uchar *csre = sre, *csim = sim, *cdre = dre, *cdim = dim;
|
||||
for (int i = 0; i < p.nchunk_stages; i++)
|
||||
{
|
||||
if (p.pingpong)
|
||||
{
|
||||
runStage(k, p.stages[i], i0, chunk, esz, csre, csim, cdre, cdim, scratch);
|
||||
std::swap(csre, cdre); std::swap(csim, cdim);
|
||||
}
|
||||
else
|
||||
runStage(k, p.stages[i], i0, chunk, esz, csre, csim, csre, csim, scratch);
|
||||
}
|
||||
}
|
||||
if (p.pingpong && (p.nchunk_stages & 1))
|
||||
{
|
||||
std::swap(sre, dre); std::swap(sim, dim);
|
||||
}
|
||||
s = p.nchunk_stages;
|
||||
}
|
||||
for (; s < nstages; s++)
|
||||
{
|
||||
const DftStage& st = p.stages[i];
|
||||
DftStageFunc fn = st.radix == 4 ? k.radix4 : st.radix == 3 ? k.radix3 :
|
||||
st.radix == 5 ? k.radix5 : k.radixOdd;
|
||||
if (p.pingpong)
|
||||
{
|
||||
fn(st, sre, sim, dre, dim, scratch);
|
||||
runStage(k, p.stages[s], 0, p.nc, esz, sre, sim, dre, dim, scratch);
|
||||
std::swap(sre, dre); std::swap(sim, dim);
|
||||
}
|
||||
else
|
||||
fn(st, sre, sim, sre, sim, scratch);
|
||||
runStage(k, p.stages[s], 0, p.nc, esz, sre, sim, sre, sim, scratch);
|
||||
}
|
||||
|
||||
double scale = c.scale;
|
||||
if (p.fused_last)
|
||||
{
|
||||
// C2C or even C2R: last radix-4 stage + interleave + scale, straight into dst
|
||||
k.radix4Last(p.stages[p.nstages - 1], sre, sim, dst, scale, c.isInverse || p.kind == DFT_KIND_C2R ? -scale : scale);
|
||||
return;
|
||||
}
|
||||
switch (p.kind)
|
||||
{
|
||||
case DFT_KIND_C2C:
|
||||
@@ -2016,10 +2238,9 @@ class OcvDftBasicImpl CV_FINAL : public hal::DFT1D
|
||||
{
|
||||
public:
|
||||
OcvDftOptions opt;
|
||||
// the plan is built once in init(), the workspace is allocated once here and reused by
|
||||
// the plan (with the workspace inside its block) is built once in init() and reused by
|
||||
// every apply() (zero allocations per transform)
|
||||
DftPlan plan;
|
||||
AutoBuffer<uchar> ws;
|
||||
#ifdef USE_IPP_DFT
|
||||
AutoBuffer<uchar> ippbuf;
|
||||
AutoBuffer<uchar> ippworkbuf;
|
||||
@@ -2104,11 +2325,10 @@ public:
|
||||
const DFTKernels& kernels = getDFTKernels(depth);
|
||||
int kind = stage == 0 && real_transform ? (opt.isInverse ? DFT_KIND_C2R : DFT_KIND_R2C)
|
||||
: DFT_KIND_C2C;
|
||||
plan.build(kind, len, depth, kernels.vlanes);
|
||||
ws.allocate(plan.ws_bytes);
|
||||
plan.build(kind, len, depth, kernels.vlanes, kernels.vlanes128, true);
|
||||
opt.plan = &plan;
|
||||
opt.kernels = &kernels;
|
||||
opt.workspace = ws.data();
|
||||
opt.workspace = plan.workspace;
|
||||
if (needBuffer)
|
||||
*needBuffer = false; // the engine supports in-place for every kind
|
||||
}
|
||||
@@ -3641,9 +3861,8 @@ class OcvDctImpl CV_FINAL : public hal::DCT2D
|
||||
public:
|
||||
OcvDftOptions opt;
|
||||
DCTFunc dct_func;
|
||||
// per-stage plan + workspace, (re)built once per stage length, never per row
|
||||
// per-stage plan (+ workspace inside), (re)built once per stage length, never per row
|
||||
DftPlan plan;
|
||||
AutoBuffer<uchar> ws;
|
||||
bool isRowTransform;
|
||||
bool isInverse;
|
||||
bool isContinuous;
|
||||
@@ -3727,13 +3946,10 @@ public:
|
||||
if( len > 1 && (len & 1) )
|
||||
CV_Error( cv::Error::StsNotImplemented, "Odd-size DCT\'s are not implemented" );
|
||||
if( len > 1 )
|
||||
{
|
||||
plan.build(isInverse ? DFT_KIND_IDCT : DFT_KIND_DCT, len, depth, kernels.vlanes);
|
||||
ws.allocate(plan.ws_bytes);
|
||||
}
|
||||
plan.build(isInverse ? DFT_KIND_IDCT : DFT_KIND_DCT, len, depth, kernels.vlanes, kernels.vlanes128, true);
|
||||
opt.plan = &plan;
|
||||
opt.kernels = &kernels;
|
||||
opt.workspace = ws.data();
|
||||
opt.workspace = plan.workspace;
|
||||
prev_len = len;
|
||||
}
|
||||
// otherwise reuse the plan built on the previous stage (same length, only the steps differ)
|
||||
|
||||
@@ -7,18 +7,26 @@
|
||||
// The engine runs a 1D transform in three groups of passes:
|
||||
// A (preproc): table-driven gather from the user input (or from an interleaved temp) into
|
||||
// split re[]/im[] workspace, fused with the first twiddle-free radix-r0 stage,
|
||||
// r0 in {1,2,4,8}. Scalar.
|
||||
// r0 in {1,2,3,4,5,8} (vectorized for r0 > 1, scalar table gather for r0 == 1).
|
||||
// B (stages): mixed-radix decimation-in-time stages with growing span, direction-free
|
||||
// (forward twiddles only), out-of-place (src pair -> dst pair, ping-pong) or
|
||||
// in-place when the plan says it is legal. Vectorized with universal intrinsics.
|
||||
// For long transforms the leading stages run chunk by chunk (depth-first) so
|
||||
// that their data stays in L2.
|
||||
// C (postproc): interleave / CCS packing / DCT ordering with fused scaling into the user output.
|
||||
// For C2C (and even C2R) the last radix-4 stage writes the interleaved, scaled
|
||||
// output itself (fused last stage), saving one pass over the data.
|
||||
//
|
||||
// Inverse transforms use the conjugation trick: conj() on the gather, im_scale = -scale on output.
|
||||
// Buffers keep fixed semantics (re is always re).
|
||||
//
|
||||
// Plans are immutable after construction and shared read-only by all rows of a batch; the
|
||||
// workspace is a separate, per-executor, mutable buffer allocated ONCE by the owner (never per
|
||||
// call).
|
||||
// call). The owner's workspace can live in the plan's single allocation (build(..., true)):
|
||||
// one malloc per cv::dft() call instead of three, which also matters for glibc, whose heap
|
||||
// trimming (and the page faults that follow on every call) is triggered by two freed MB-sized
|
||||
// blocks but not by one twice as big. The tables of the power-of-two lengths up to 1024 are
|
||||
// shared, static tables (see dxt.cpp).
|
||||
|
||||
#ifndef OPENCV_CORE_SRC_DXT_HPP
|
||||
#define OPENCV_CORE_SRC_DXT_HPP
|
||||
@@ -38,8 +46,9 @@ enum DftKind
|
||||
|
||||
enum DftStageMode
|
||||
{
|
||||
DFT_STAGE_SCALAR = 0, // scalar reference loop (span < VL, no SIMD for this depth, ...)
|
||||
DFT_STAGE_FULL = 1 // span >= VL: vector loop with back-off clamp (brief, section 5)
|
||||
DFT_STAGE_SCALAR = 0, // scalar reference loop (span < any vector width, no SIMD for this depth, ...)
|
||||
DFT_STAGE_FULL = 1, // span >= VL: vector loop with back-off clamp (brief, section 5)
|
||||
DFT_STAGE_SIMD128 = 2 // 128-bit lanes < span < VL (wide ISAs only): the same loop on 128-bit vectors
|
||||
};
|
||||
|
||||
enum { DFT_MAX_STAGES = 40 };
|
||||
@@ -50,13 +59,15 @@ struct DftStage
|
||||
int span; // L: length of the sub-DFTs combined by this stage
|
||||
int ngroups; // G = nc / (radix*span)
|
||||
int nsteps; // number of (vector) iterations of the flat loop, precomputed by the plan
|
||||
int steps_per_group; // nsteps / ngroups (the driver re-derives nsteps for chunked runs)
|
||||
int mode; // DftStageMode
|
||||
int tw_stride; // per-leg twiddle table length (== span for now)
|
||||
// Per-stage twiddles, split layout, leg-major: tw_re[(q-1)*tw_stride + j] = cos(-2*pi*q*j/(radix*span)),
|
||||
// q in [1, radix), j in [0, span). Type T of the plan.
|
||||
const void* tw_re;
|
||||
const void* tw_im;
|
||||
// radix-odd (>=7) only: cs[p*h + k] = cos(2*pi*(p+1)*(k+1)/radix), sn[...] = sin(...), h = (radix-1)/2
|
||||
// radix-odd (>=7) only: cs[m] = cos(2*pi*m/radix), sn[m] = sin(2*pi*m/radix), m in [0, radix);
|
||||
// the butterfly addresses them with the cyclic index m = p*k mod radix
|
||||
const void* cs;
|
||||
const void* sn;
|
||||
};
|
||||
@@ -68,16 +79,22 @@ struct DftPlan
|
||||
int nc; // complex length actually run through the stages (n/2 for even real kinds)
|
||||
int depth; // CV_32F or CV_64F
|
||||
int vl; // vector lanes of the kernels this plan was built for (DFTKernels::vlanes)
|
||||
int first_radix; // 1, 2, 4 or 8: the fused first stage done in preproc
|
||||
int first_radix; // 1, 2, 3, 4, 5 or 8: the fused first stage done in preproc
|
||||
bool real_input; // R2C with odd n: gather real samples, im = 0
|
||||
bool pingpong; // stages must alternate between two workspace pairs
|
||||
bool need_tmp; // C2R / DCT / IDCT: a temp of 2*nc elements (interleaved complex or the Makhoul-permuted reals)
|
||||
bool fused_last; // the last stage is radix 4 and writes the interleaved scaled output itself (C2C, even C2R)
|
||||
int nchunk_stages; // leading stages run chunk by chunk (chunk = stages[nchunk_stages].span); 0 = none
|
||||
int nstages;
|
||||
DftStage stages[DFT_MAX_STAGES];
|
||||
// first_radix == 1: 2*nc element indices into the (real-typed) source (re index, im index);
|
||||
// first_radix > 1: nc/first_radix output positions (g*first_radix) of the input columns j
|
||||
AutoBuffer<int> itab;
|
||||
AutoBuffer<double> tw; // storage for all the tables below (type T, 8-byte aligned)
|
||||
// first_radix > 1: nc/first_radix output positions (g*first_radix) of the input columns j.
|
||||
// Points into `buf` or into the static tables.
|
||||
const int* itab;
|
||||
AutoBuffer<double> buf; // ONE block: generated tables (type T, 8-byte aligned), itab and,
|
||||
// when requested, the owner's workspace; the stage/rtw tables of
|
||||
// 2^k lengths <= 1024 are static and not stored here
|
||||
uchar* workspace; // ws_bytes inside `buf` when built with with_workspace, else 0
|
||||
const void* rtw_re; // W_n^k = exp(-2*pi*i*k/n), k in [0, n/2]; R2C/C2R/DCT/IDCT with even n
|
||||
const void* rtw_im;
|
||||
const void* dct_re; // s*W_4n^k, k in [0, n/2]; DCT/IDCT (s = DCT normalization)
|
||||
@@ -87,22 +104,24 @@ struct DftPlan
|
||||
size_t scratch_ofs; // byte offset of the radix-odd scratch area
|
||||
|
||||
DftPlan() : kind(0), n(0), nc(0), depth(0), vl(0), first_radix(1), real_input(false),
|
||||
pingpong(false), need_tmp(false), nstages(0), rtw_re(0), rtw_im(0),
|
||||
dct_re(0), dct_im(0), ws_bytes(0), pair1_ofs(0), scratch_ofs(0) {}
|
||||
pingpong(false), need_tmp(false), fused_last(false), nchunk_stages(0), nstages(0),
|
||||
itab(0), workspace(0), rtw_re(0), rtw_im(0), dct_re(0), dct_im(0), ws_bytes(0),
|
||||
pair1_ofs(0), scratch_ofs(0) {}
|
||||
|
||||
// Builds the plan for a 1D transform of the given kind (DftKind) and length.
|
||||
//
|
||||
// IMPORTANT: `vl` (vector lanes) MUST come from the dispatched kernel table
|
||||
// (DFTKernels::vlanes), never from VTraits<> in dxt.cpp, which is compiled for the baseline
|
||||
// ISA and reports the wrong lane count on AVX2/AVX512/RVV builds. Stage modes and step counts
|
||||
// depend on it.
|
||||
void build(int kind, int n, int depth, int vl);
|
||||
// IMPORTANT: `vl` and `vl128` (vector lanes of the widest and of the 128-bit vectors) MUST
|
||||
// come from the dispatched kernel table (DFTKernels::vlanes, ::vlanes128), never from
|
||||
// VTraits<> in dxt.cpp, which is compiled for the baseline ISA and reports the wrong lane
|
||||
// count on AVX2/AVX512/RVV builds. Stage modes and step counts depend on them.
|
||||
// with_workspace: also reserve ws_bytes for the owner's workspace in the same block (`workspace`).
|
||||
void build(int kind, int n, int depth, int vl, int vl128, bool with_workspace = false);
|
||||
|
||||
private:
|
||||
// fills the twiddle tables (stage tables, W_n^k, DCT) into `tw`; ntw = total number of T entries
|
||||
template<typename T> void fillTables(size_t ntw, bool need_rtw, bool need_dct);
|
||||
// fills the twiddle tables (stage tables, W_n^k, DCT) at `dst`; ntw = total number of T entries
|
||||
template<typename T> void fillTables(T* dst, size_t ntw, bool need_rtw, bool need_dct, bool static_stages, bool static_rtw);
|
||||
|
||||
DftPlan(const DftPlan&); // tables hold raw pointers into the AutoBuffers: no copies
|
||||
DftPlan(const DftPlan&); // tables hold raw pointers into `buf`: no copies
|
||||
DftPlan& operator=(const DftPlan&);
|
||||
};
|
||||
|
||||
@@ -112,13 +131,18 @@ typedef void (*DftPreprocFunc)(const DftPlan& plan, const void* src, size_t sste
|
||||
// (stage, src_re, src_im, dst_re, dst_im, scratch); src == dst allowed only when !plan.pingpong
|
||||
typedef void (*DftStageFunc)(const DftStage& st, const void* sre, const void* sim,
|
||||
void* dre, void* dim, void* scratch);
|
||||
// fused last stage: (stage, src_re, src_im, interleaved dst, re_scale, im_scale)
|
||||
typedef void (*DftLastStageFunc)(const DftStage& st, const void* sre, const void* sim,
|
||||
void* dst, double re_scale, double im_scale);
|
||||
|
||||
struct DFTKernels
|
||||
{
|
||||
// group A
|
||||
DftPreprocFunc preprocRadix0; // gather + deinterleave only
|
||||
DftPreprocFunc preprocRadix2; // + fused twiddle-free radix-2
|
||||
DftPreprocFunc preprocRadix3; // + fused twiddle-free radix-3 (odd nc)
|
||||
DftPreprocFunc preprocRadix4; // + fused twiddle-free radix-4
|
||||
DftPreprocFunc preprocRadix5; // + fused twiddle-free radix-5 (odd nc)
|
||||
DftPreprocFunc preprocRadix8; // + fused twiddle-free radix-8
|
||||
// C2R: CCS (or full complex, complex_input) spectrum -> interleaved "2Z" temp (2*nc elements)
|
||||
void (*preprocCCS)(const DftPlan& plan, const void* src, void* tmp, bool complex_input);
|
||||
@@ -128,6 +152,7 @@ struct DFTKernels
|
||||
void (*preprocIDCT)(const DftPlan& plan, const void* src, size_t sstep, void* tmp);
|
||||
// group B
|
||||
DftStageFunc radix3, radix4, radix5, radixOdd;
|
||||
DftLastStageFunc radix4Last; // radix-4 stage + interleave + scale into the user output
|
||||
// group C
|
||||
void (*postprocDFT)(const DftPlan& plan, const void* re, const void* im, void* dst,
|
||||
double re_scale, double im_scale); // split -> interleaved complex
|
||||
@@ -137,6 +162,7 @@ struct DFTKernels
|
||||
void (*postprocDCT)(const DftPlan& plan, const void* re, const void* im, void* dst, size_t dstep);
|
||||
void (*postprocIDCT)(const DftPlan& plan, const void* re, const void* im, void* dst, size_t dstep);
|
||||
int vlanes; // VTraits<v_T>::vlanes() of the ISA these pointers belong to (1 = no SIMD)
|
||||
int vlanes128; // lanes of the 128-bit vectors used by DFT_STAGE_SIMD128 (== vlanes when not wider)
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -66,23 +66,25 @@ inline v_float64 v_signmask_(bool neg, double) { return v_reinterpret_as_f64(vx_
|
||||
//////////////////////////////// group A: gather + first stage ////////////////////////////////
|
||||
//
|
||||
// Two flavours:
|
||||
// - preprocRadix0 (first radix 1, odd nc): table-driven scalar gather, plan.itab holds two
|
||||
// element indices (re, im) per complex element; im = 0 gather when plan.real_input;
|
||||
// - preprocRadix2/4/8: "transposed" first stage. The r0 inputs of group g are the elements
|
||||
// - preprocRadix0 (first radix 1: odd nc without a factor 3 or 5): table-driven scalar gather,
|
||||
// plan.itab holds two element indices (re, im) per complex element; im = 0 gather when
|
||||
// plan.real_input;
|
||||
// - preprocRadix2/3/4/5/8: "transposed" first stage. The r0 inputs of group g are the elements
|
||||
// x[j + q*M], M = nc/r0, q = 0..r0-1, where j = base(g) runs over [0, M) in digit-reversed
|
||||
// order. Iterating over j instead of g makes the loads contiguous: VL lanes load the r0 legs
|
||||
// with v_load_deinterleave, the r0-point DFT runs in registers, and the r0 outputs of every
|
||||
// lane are stored contiguously at plan.itab[j] (= g*r0) through a 4-channel
|
||||
// lane are stored contiguously at plan.itab[j] (= g*r0) through a multi-channel
|
||||
// v_store_interleave into a small stack buffer + short copies. plan.itab has M entries.
|
||||
// The source must be interleaved complex with sstep == 1; DCT input goes through
|
||||
// preprocDCT (Makhoul permutation into the temp) first.
|
||||
// The source must be interleaved complex with sstep == 1 (or real samples with sstep == 1
|
||||
// when plan.real_input: R2C with odd n and r0 in {3, 5}); DCT input goes through preprocDCT
|
||||
// (Makhoul permutation into the temp) first.
|
||||
|
||||
template<typename T>
|
||||
void preprocRadix0_(const DftPlan& plan, const void* _src, size_t sstep, void* _re, void* _im, bool conj)
|
||||
{
|
||||
const T* src = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* itab = plan.itab.data();
|
||||
const int* itab = plan.itab;
|
||||
int nc = plan.nc;
|
||||
if (plan.real_input)
|
||||
{
|
||||
@@ -119,6 +121,60 @@ static inline void firstStage2_scalar(const T* x, int M, int j, T* re, T* im, T
|
||||
re[1] = x0r - x1r; im[1] = x0i - x1i;
|
||||
}
|
||||
|
||||
// radix-3 and radix-5 first stages also serve R2C with odd n (real samples: x[j + q*M], im = 0)
|
||||
template<typename T>
|
||||
static inline void firstStage3_scalar(const T* x, int M, int j, T* re, T* im, T conjsign, bool realInput)
|
||||
{
|
||||
const T s120 = DftConst<T>::sin120;
|
||||
T x0r, x0i, x1r, x1i, x2r, x2i;
|
||||
if (realInput)
|
||||
{
|
||||
x0r = x[j]; x1r = x[j+M]; x2r = x[j+2*M]; x0i = x1i = x2i = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
x0r = x[j*2]; x0i = conjsign*x[j*2+1];
|
||||
x1r = x[(j+M)*2]; x1i = conjsign*x[(j+M)*2+1];
|
||||
x2r = x[(j+2*M)*2]; x2i = conjsign*x[(j+2*M)*2+1];
|
||||
}
|
||||
T sr = x1r + x2r, si = x1i + x2i;
|
||||
T dr = s120*(x1r - x2r), di = s120*(x1i - x2i);
|
||||
T mr = x0r - (T)0.5*sr, mi = x0i - (T)0.5*si;
|
||||
re[0] = x0r + sr; im[0] = x0i + si;
|
||||
re[1] = mr + di; im[1] = mi - dr;
|
||||
re[2] = mr - di; im[2] = mi + dr;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static inline void firstStage5_scalar(const T* x, int M, int j, T* re, T* im, T conjsign, bool realInput)
|
||||
{
|
||||
const T c5 = DftConst<T>::fft5_2, s1 = DftConst<T>::sin72, s2 = DftConst<T>::sin144;
|
||||
T xr[5], xi[5];
|
||||
for (int q = 0; q < 5; q++)
|
||||
{
|
||||
if (realInput)
|
||||
{
|
||||
xr[q] = x[j + q*M]; xi[q] = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
xr[q] = x[(j + q*M)*2]; xi[q] = conjsign*x[(j + q*M)*2 + 1];
|
||||
}
|
||||
}
|
||||
T a1r = xr[1] + xr[4], a1i = xi[1] + xi[4], b1r = xr[1] - xr[4], b1i = xi[1] - xi[4];
|
||||
T a2r = xr[2] + xr[3], a2i = xi[2] + xi[3], b2r = xr[2] - xr[3], b2i = xi[2] - xi[3];
|
||||
T tr = xr[0] - (T)0.25*(a1r + a2r), ti = xi[0] - (T)0.25*(a1i + a2i);
|
||||
T ur = c5*(a1r - a2r), ui = c5*(a1i - a2i);
|
||||
T v1r = s1*b1r + s2*b2r, v1i = s1*b1i + s2*b2i;
|
||||
T v2r = s2*b1r - s1*b2r, v2i = s2*b1i - s1*b2i;
|
||||
T pr = tr + ur, pi = ti + ui, qr = tr - ur, qi = ti - ui;
|
||||
re[0] = xr[0] + a1r + a2r; im[0] = xi[0] + a1i + a2i;
|
||||
re[1] = pr + v1i; im[1] = pi - v1r;
|
||||
re[4] = pr - v1i; im[4] = pi + v1r;
|
||||
re[2] = qr + v2i; im[2] = qi - v2r;
|
||||
re[3] = qr - v2i; im[3] = qi + v2r;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static inline void firstStage4_scalar(const T* x, int M, int j, T* re, T* im, T conjsign)
|
||||
{
|
||||
@@ -202,6 +258,98 @@ static void firstStage2_vec(const T* x, int M, const int* gtab, T* re, T* im, bo
|
||||
firstStage2_scalar(x, M, j, re + gtab[j], im + gtab[j], conjsign);
|
||||
}
|
||||
|
||||
// real samples (plan.real_input): leg q of lanes j..j+VL-1 is x[j + q*M], im = 0
|
||||
#define DFT_LOAD_LEG_REAL(q, xr, xi) \
|
||||
xr = vx_load(x + j + (q)*M); xi = zero;
|
||||
|
||||
template<typename T, typename VT, bool realInput>
|
||||
static void firstStage3_vec(const T* x, int M, const int* gtab, T* re, T* im, bool conj)
|
||||
{
|
||||
const int VL = VTraits<VT>::vlanes();
|
||||
const VT imsign = v_signmask_(conj, (T)0), zero = v_setall_((T)0);
|
||||
const VT s120 = v_setall_((T)DftConst<T>::sin120), half = v_setall_((T)0.5);
|
||||
T bufr[3*VTraits<VT>::max_nlanes], bufi[3*VTraits<VT>::max_nlanes];
|
||||
int j = 0;
|
||||
for (; j + VL <= M; j += VL)
|
||||
{
|
||||
VT x0r, x0i, x1r, x1i, x2r, x2i;
|
||||
if constexpr (realInput)
|
||||
{
|
||||
DFT_LOAD_LEG_REAL(0, x0r, x0i)
|
||||
DFT_LOAD_LEG_REAL(1, x1r, x1i)
|
||||
DFT_LOAD_LEG_REAL(2, x2r, x2i)
|
||||
}
|
||||
else
|
||||
{
|
||||
DFT_LOAD_LEG(0, x0r, x0i)
|
||||
DFT_LOAD_LEG(1, x1r, x1i)
|
||||
DFT_LOAD_LEG(2, x2r, x2i)
|
||||
}
|
||||
VT sr = v_add(x1r, x2r), si = v_add(x1i, x2i);
|
||||
VT dr = v_mul(s120, v_sub(x1r, x2r)), di = v_mul(s120, v_sub(x1i, x2i));
|
||||
VT mr = v_sub(x0r, v_mul(half, sr)), mi = v_sub(x0i, v_mul(half, si));
|
||||
v_store_interleave(bufr, v_add(x0r, sr), v_add(mr, di), v_sub(mr, di));
|
||||
v_store_interleave(bufi, v_add(x0i, si), v_sub(mi, dr), v_add(mi, dr));
|
||||
scatterLanes<T, 3>(bufr, bufi, re, im, gtab + j, VL);
|
||||
}
|
||||
T conjsign = conj ? (T)-1 : (T)1;
|
||||
for (; j < M; j++)
|
||||
firstStage3_scalar(x, M, j, re + gtab[j], im + gtab[j], conjsign, realInput);
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool realInput>
|
||||
static void firstStage5_vec(const T* x, int M, const int* gtab, T* re, T* im, bool conj)
|
||||
{
|
||||
const int VL = VTraits<VT>::vlanes();
|
||||
const VT imsign = v_signmask_(conj, (T)0), zero = v_setall_((T)0);
|
||||
const VT c5 = v_setall_((T)DftConst<T>::fft5_2), s1 = v_setall_((T)DftConst<T>::sin72);
|
||||
const VT s2 = v_setall_((T)DftConst<T>::sin144), quarter = v_setall_((T)0.25);
|
||||
// outputs 0..3 lane-major in buf (4 per lane), output 4 in buf4
|
||||
T bufr[4*VTraits<VT>::max_nlanes], bufi[4*VTraits<VT>::max_nlanes];
|
||||
T buf4r[VTraits<VT>::max_nlanes], buf4i[VTraits<VT>::max_nlanes];
|
||||
int j = 0;
|
||||
for (; j + VL <= M; j += VL)
|
||||
{
|
||||
VT x0r, x0i, x1r, x1i, x2r, x2i, x3r, x3i, x4r, x4i;
|
||||
if constexpr (realInput)
|
||||
{
|
||||
DFT_LOAD_LEG_REAL(0, x0r, x0i)
|
||||
DFT_LOAD_LEG_REAL(1, x1r, x1i)
|
||||
DFT_LOAD_LEG_REAL(2, x2r, x2i)
|
||||
DFT_LOAD_LEG_REAL(3, x3r, x3i)
|
||||
DFT_LOAD_LEG_REAL(4, x4r, x4i)
|
||||
}
|
||||
else
|
||||
{
|
||||
DFT_LOAD_LEG(0, x0r, x0i)
|
||||
DFT_LOAD_LEG(1, x1r, x1i)
|
||||
DFT_LOAD_LEG(2, x2r, x2i)
|
||||
DFT_LOAD_LEG(3, x3r, x3i)
|
||||
DFT_LOAD_LEG(4, x4r, x4i)
|
||||
}
|
||||
VT a1r = v_add(x1r, x4r), a1i = v_add(x1i, x4i), b1r = v_sub(x1r, x4r), b1i = v_sub(x1i, x4i);
|
||||
VT a2r = v_add(x2r, x3r), a2i = v_add(x2i, x3i), b2r = v_sub(x2r, x3r), b2i = v_sub(x2i, x3i);
|
||||
VT sar = v_add(a1r, a2r), sai = v_add(a1i, a2i);
|
||||
VT tr = v_sub(x0r, v_mul(quarter, sar)), ti = v_sub(x0i, v_mul(quarter, sai));
|
||||
VT ur = v_mul(c5, v_sub(a1r, a2r)), ui = v_mul(c5, v_sub(a1i, a2i));
|
||||
VT v1r = v_fma(s1, b1r, v_mul(s2, b2r)), v1i = v_fma(s1, b1i, v_mul(s2, b2i));
|
||||
VT v2r = v_sub(v_mul(s2, b1r), v_mul(s1, b2r)), v2i = v_sub(v_mul(s2, b1i), v_mul(s1, b2i));
|
||||
VT pr = v_add(tr, ur), pi = v_add(ti, ui), qr = v_sub(tr, ur), qi = v_sub(ti, ui);
|
||||
v_store_interleave(bufr, v_add(x0r, sar), v_add(pr, v1i), v_add(qr, v2i), v_sub(qr, v2i));
|
||||
v_store_interleave(bufi, v_add(x0i, sai), v_sub(pi, v1r), v_sub(qi, v2r), v_add(qi, v2r));
|
||||
v_store(buf4r, v_sub(pr, v1i)); v_store(buf4i, v_add(pi, v1r));
|
||||
for (int l = 0; l < VL; l++)
|
||||
{
|
||||
int base = gtab[j + l];
|
||||
memcpy(re + base, bufr + l*4, 4*sizeof(T)); re[base + 4] = buf4r[l];
|
||||
memcpy(im + base, bufi + l*4, 4*sizeof(T)); im[base + 4] = buf4i[l];
|
||||
}
|
||||
}
|
||||
T conjsign = conj ? (T)-1 : (T)1;
|
||||
for (; j < M; j++)
|
||||
firstStage5_scalar(x, M, j, re + gtab[j], im + gtab[j], conjsign, realInput);
|
||||
}
|
||||
|
||||
template<typename T, typename VT>
|
||||
static void firstStage4_vec(const T* x, int M, const int* gtab, T* re, T* im, bool conj)
|
||||
{
|
||||
@@ -272,6 +420,7 @@ static void firstStage8_vec(const T* x, int M, const int* gtab, T* re, T* im, bo
|
||||
}
|
||||
|
||||
#undef DFT_LOAD_LEG
|
||||
#undef DFT_LOAD_LEG_REAL
|
||||
#undef DFT4_VEC
|
||||
|
||||
#endif // CV_SIMD || CV_SIMD_SCALABLE
|
||||
@@ -282,7 +431,7 @@ void preprocRadix2_(const DftPlan& plan, const void* _src, size_t sstep, void* _
|
||||
CV_DbgAssert(sstep == 1); CV_UNUSED(sstep);
|
||||
const T* x = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* gtab = plan.itab.data();
|
||||
const int* gtab = plan.itab;
|
||||
int M = plan.nc/2;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
@@ -296,13 +445,63 @@ void preprocRadix2_(const DftPlan& plan, const void* _src, size_t sstep, void* _
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
void preprocRadix3_(const DftPlan& plan, const void* _src, size_t sstep, void* _re, void* _im, bool conj)
|
||||
{
|
||||
CV_DbgAssert(sstep == 1); CV_UNUSED(sstep);
|
||||
const T* x = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* gtab = plan.itab;
|
||||
int M = plan.nc/3;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
{
|
||||
if (plan.real_input)
|
||||
firstStage3_vec<T, VT, true>(x, M, gtab, re, im, conj);
|
||||
else
|
||||
firstStage3_vec<T, VT, false>(x, M, gtab, re, im, conj);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
T conjsign = conj ? (T)-1 : (T)1;
|
||||
for (int j = 0; j < M; j++)
|
||||
firstStage3_scalar(x, M, j, re + gtab[j], im + gtab[j], conjsign, plan.real_input);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
void preprocRadix5_(const DftPlan& plan, const void* _src, size_t sstep, void* _re, void* _im, bool conj)
|
||||
{
|
||||
CV_DbgAssert(sstep == 1); CV_UNUSED(sstep);
|
||||
const T* x = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* gtab = plan.itab;
|
||||
int M = plan.nc/5;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
{
|
||||
if (plan.real_input)
|
||||
firstStage5_vec<T, VT, true>(x, M, gtab, re, im, conj);
|
||||
else
|
||||
firstStage5_vec<T, VT, false>(x, M, gtab, re, im, conj);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
T conjsign = conj ? (T)-1 : (T)1;
|
||||
for (int j = 0; j < M; j++)
|
||||
firstStage5_scalar(x, M, j, re + gtab[j], im + gtab[j], conjsign, plan.real_input);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
void preprocRadix4_(const DftPlan& plan, const void* _src, size_t sstep, void* _re, void* _im, bool conj)
|
||||
{
|
||||
CV_DbgAssert(sstep == 1); CV_UNUSED(sstep);
|
||||
const T* x = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* gtab = plan.itab.data();
|
||||
const int* gtab = plan.itab;
|
||||
int M = plan.nc/4;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
@@ -322,7 +521,7 @@ void preprocRadix8_(const DftPlan& plan, const void* _src, size_t sstep, void* _
|
||||
CV_DbgAssert(sstep == 1); CV_UNUSED(sstep);
|
||||
const T* x = (const T*)_src;
|
||||
T* re = (T*)_re; T* im = (T*)_im;
|
||||
const int* gtab = plan.itab.data();
|
||||
const int* gtab = plan.itab;
|
||||
int M = plan.nc/8;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
@@ -571,6 +770,32 @@ void radix4_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* di
|
||||
}
|
||||
}
|
||||
|
||||
// last radix-4 stage fused with the output interleave and scaling (C2C, even C2R)
|
||||
template<typename T>
|
||||
void radix4Last_scalar(const DftStage& st, const T* sre, const T* sim, T* dst, T rs, T is)
|
||||
{
|
||||
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
|
||||
int L = st.span, G = st.ngroups, ts = st.tw_stride;
|
||||
for (int g = 0; g < G; g++)
|
||||
{
|
||||
int base = g*4*L;
|
||||
for (int j = 0; j < L; j++)
|
||||
{
|
||||
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L;
|
||||
T x0r = sre[i0], x0i = sim[i0], x1r, x1i, x2r, x2i, x3r, x3i;
|
||||
T y0r, y0i, y1r, y1i, y2r, y2i, y3r, y3i;
|
||||
DFT_CMUL(x1r, x1i, sre[i1], sim[i1], twr[j], twi[j]);
|
||||
DFT_CMUL(x2r, x2i, sre[i2], sim[i2], twr[ts + j], twi[ts + j]);
|
||||
DFT_CMUL(x3r, x3i, sre[i3], sim[i3], twr[ts*2 + j], twi[ts*2 + j]);
|
||||
DFT4_SCALAR(x0r,x0i,x1r,x1i,x2r,x2i,x3r,x3i, y0r,y0i,y1r,y1i,y2r,y2i,y3r,y3i);
|
||||
dst[i0*2] = y0r*rs; dst[i0*2+1] = y0i*is;
|
||||
dst[i1*2] = y1r*rs; dst[i1*2+1] = y1i*is;
|
||||
dst[i2*2] = y2r*rs; dst[i2*2+1] = y2i*is;
|
||||
dst[i3*2] = y3r*rs; dst[i3*2+1] = y3i*is;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void radix5_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
|
||||
{
|
||||
@@ -635,9 +860,10 @@ void radixOdd_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T*
|
||||
for (int k = 1; k <= h; k++)
|
||||
{
|
||||
T sr = x0r, si = x0i, tr = 0, ti = 0;
|
||||
for (int p = 1; p <= h; p++)
|
||||
for (int p = 1, m = 0; p <= h; p++)
|
||||
{
|
||||
T c = cs[(p-1)*h + (k-1)], s = sn[(p-1)*h + (k-1)];
|
||||
m += k; m -= m >= r ? r : 0; // m = p*k mod r
|
||||
T c = cs[m], s = sn[m];
|
||||
sr += c*ar[p-1]; si += c*ai[p-1];
|
||||
tr += s*br[p-1]; ti += s*bi[p-1];
|
||||
}
|
||||
@@ -652,9 +878,47 @@ void radixOdd_scalar(const DftStage& st, const T* sre, const T* sim, T* dre, T*
|
||||
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
|
||||
// The stage kernels are instantiated for the widest vectors (v_float32/v_float64, DFT_STAGE_FULL)
|
||||
// and, on ISAs wider than 128 bits, for the 128-bit vectors too (DFT_STAGE_SIMD128: spans that
|
||||
// are shorter than the wide vector but not than the narrow one). Load/broadcast by vector type:
|
||||
#if CV_SIMD128 && !CV_SIMD_SCALABLE
|
||||
inline v_float32x4 dft_vload(const float* p, const v_float32x4*) { return v_load(p); }
|
||||
inline v_float32x4 dft_vsetall(float x, const v_float32x4*) { return v_setall_f32(x); }
|
||||
#if CV_SIMD128_64F
|
||||
inline v_float64x2 dft_vload(const double* p, const v_float64x2*) { return v_load(p); }
|
||||
inline v_float64x2 dft_vsetall(double x, const v_float64x2*) { return v_setall_f64(x); }
|
||||
#endif
|
||||
#endif
|
||||
#if CV_SIMD256
|
||||
inline v_float32x8 dft_vload(const float* p, const v_float32x8*) { return v256_load(p); }
|
||||
inline v_float32x8 dft_vsetall(float x, const v_float32x8*) { return v256_setall_f32(x); }
|
||||
#if CV_SIMD256_64F
|
||||
inline v_float64x4 dft_vload(const double* p, const v_float64x4*) { return v256_load(p); }
|
||||
inline v_float64x4 dft_vsetall(double x, const v_float64x4*) { return v256_setall_f64(x); }
|
||||
#endif
|
||||
#endif
|
||||
#if CV_SIMD512
|
||||
inline v_float32x16 dft_vload(const float* p, const v_float32x16*) { return v512_load(p); }
|
||||
inline v_float32x16 dft_vsetall(float x, const v_float32x16*) { return v512_setall_f32(x); }
|
||||
#if CV_SIMD512_64F
|
||||
inline v_float64x8 dft_vload(const double* p, const v_float64x8*) { return v512_load(p); }
|
||||
inline v_float64x8 dft_vsetall(double x, const v_float64x8*) { return v512_setall_f64(x); }
|
||||
#endif
|
||||
#endif
|
||||
#if CV_SIMD_SCALABLE
|
||||
inline v_float32 dft_vload(const float* p, const v_float32*) { return vx_load(p); }
|
||||
inline v_float32 dft_vsetall(float x, const v_float32*) { return vx_setall_f32(x); }
|
||||
#if CV_SIMD_SCALABLE_64F
|
||||
inline v_float64 dft_vload(const double* p, const v_float64*) { return vx_load(p); }
|
||||
inline v_float64 dft_vsetall(double x, const v_float64*) { return vx_setall_f64(x); }
|
||||
#endif
|
||||
#endif
|
||||
#define VLOAD(p) dft_vload((p), (const VT*)0)
|
||||
#define VSETALL(x) dft_vsetall((x), (const VT*)0)
|
||||
|
||||
// dst = a*s + c, s is a scalar constant of the stage (the compiler hoists the broadcast)
|
||||
template<typename VT, typename T>
|
||||
inline VT v_fma_n(const VT& a, T s, const VT& c) { return v_fma(a, v_setall_(s), c); }
|
||||
inline VT v_fma_n(const VT& a, T s, const VT& c) { return v_fma(a, dft_vsetall(s, (const VT*)0), c); }
|
||||
|
||||
// (xr, xi) = (ar, ai) * (wr, wi)
|
||||
#define DFT_VCMUL(xr, xi, ar, ai, wr, wi) \
|
||||
@@ -678,12 +942,12 @@ void radix3_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
|
||||
{
|
||||
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
|
||||
const int L = st.span, ts = st.tw_stride, VL = VTraits<VT>::vlanes();
|
||||
const VT s120 = v_setall_((T)DftConst<T>::sin120), half = v_setall_((T)0.5);
|
||||
const VT s120 = VSETALL((T)DftConst<T>::sin120), half = VSETALL((T)0.5);
|
||||
DFT_VLOOP_BEGIN(3)
|
||||
int i0 = base + j, i1 = i0 + L, i2 = i1 + L;
|
||||
VT x0r = vx_load(sre + i0), x0i = vx_load(sim + i0), x1r, x1i, x2r, x2i;
|
||||
DFT_VCMUL(x1r, x1i, vx_load(sre + i1), vx_load(sim + i1), vx_load(twr + j), vx_load(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, vx_load(sre + i2), vx_load(sim + i2), vx_load(twr + ts + j), vx_load(twi + ts + j));
|
||||
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x1r, x1i, x2r, x2i;
|
||||
DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j));
|
||||
VT sr = v_add(x1r, x2r), si = v_add(x1i, x2i);
|
||||
VT dr = v_mul(s120, v_sub(x1r, x2r)), di = v_mul(s120, v_sub(x1i, x2i));
|
||||
VT mr = v_sub(x0r, v_mul(half, sr)), mi = v_sub(x0i, v_mul(half, si));
|
||||
@@ -700,10 +964,10 @@ void radix4_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
|
||||
const int L = st.span, ts = st.tw_stride, VL = VTraits<VT>::vlanes();
|
||||
DFT_VLOOP_BEGIN(4)
|
||||
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L;
|
||||
VT x0r = vx_load(sre + i0), x0i = vx_load(sim + i0), x1r, x1i, x2r, x2i, x3r, x3i;
|
||||
DFT_VCMUL(x1r, x1i, vx_load(sre + i1), vx_load(sim + i1), vx_load(twr + j), vx_load(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, vx_load(sre + i2), vx_load(sim + i2), vx_load(twr + ts + j), vx_load(twi + ts + j));
|
||||
DFT_VCMUL(x3r, x3i, vx_load(sre + i3), vx_load(sim + i3), vx_load(twr + ts*2 + j), vx_load(twi + ts*2 + j));
|
||||
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x1r, x1i, x2r, x2i, x3r, x3i;
|
||||
DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j));
|
||||
DFT_VCMUL(x3r, x3i, VLOAD(sre + i3), VLOAD(sim + i3), VLOAD(twr + ts*2 + j), VLOAD(twi + ts*2 + j));
|
||||
VT y0r = v_add(x0r, x2r), y0i = v_add(x0i, x2i), y1r = v_sub(x0r, x2r), y1i = v_sub(x0i, x2i);
|
||||
VT y2r = v_add(x1r, x3r), y2i = v_add(x1i, x3i), y3r = v_sub(x1r, x3r), y3i = v_sub(x1i, x3i);
|
||||
v_store(dre + i0, v_add(y0r, y2r)); v_store(dim + i0, v_add(y0i, y2i));
|
||||
@@ -713,27 +977,50 @@ void radix4_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
|
||||
DFT_VLOOP_END(4)
|
||||
}
|
||||
|
||||
// last radix-4 stage + interleave + scale into the (interleaved complex) output; always
|
||||
// out-of-place, so the back-off clamp is harmless (overlapping lanes store identical values)
|
||||
template<typename T, typename VT>
|
||||
void radix4Last_vec(const DftStage& st, const T* sre, const T* sim, T* dst, T rs, T is)
|
||||
{
|
||||
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
|
||||
const int L = st.span, ts = st.tw_stride, VL = VTraits<VT>::vlanes();
|
||||
const VT vrs = VSETALL(rs), vis = VSETALL(is);
|
||||
DFT_VLOOP_BEGIN(4)
|
||||
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L;
|
||||
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x1r, x1i, x2r, x2i, x3r, x3i;
|
||||
DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j));
|
||||
DFT_VCMUL(x3r, x3i, VLOAD(sre + i3), VLOAD(sim + i3), VLOAD(twr + ts*2 + j), VLOAD(twi + ts*2 + j));
|
||||
VT y0r = v_add(x0r, x2r), y0i = v_add(x0i, x2i), y1r = v_sub(x0r, x2r), y1i = v_sub(x0i, x2i);
|
||||
VT y2r = v_add(x1r, x3r), y2i = v_add(x1i, x3i), y3r = v_sub(x1r, x3r), y3i = v_sub(x1i, x3i);
|
||||
v_store_interleave(dst + i0*2, v_mul(v_add(y0r, y2r), vrs), v_mul(v_add(y0i, y2i), vis));
|
||||
v_store_interleave(dst + i2*2, v_mul(v_sub(y0r, y2r), vrs), v_mul(v_sub(y0i, y2i), vis));
|
||||
v_store_interleave(dst + i1*2, v_mul(v_add(y1r, y3i), vrs), v_mul(v_sub(y1i, y3r), vis));
|
||||
v_store_interleave(dst + i3*2, v_mul(v_sub(y1r, y3i), vrs), v_mul(v_add(y1i, y3r), vis));
|
||||
DFT_VLOOP_END(4)
|
||||
}
|
||||
|
||||
template<typename T, typename VT>
|
||||
void radix5_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim)
|
||||
{
|
||||
const T* twr = (const T*)st.tw_re; const T* twi = (const T*)st.tw_im;
|
||||
const int L = st.span, ts = st.tw_stride, VL = VTraits<VT>::vlanes();
|
||||
const T c5 = DftConst<T>::fft5_2, s1 = DftConst<T>::sin72, s2 = DftConst<T>::sin144;
|
||||
const VT quarter = v_setall_((T)0.25);
|
||||
const VT quarter = VSETALL((T)0.25);
|
||||
DFT_VLOOP_BEGIN(5)
|
||||
int i0 = base + j, i1 = i0 + L, i2 = i1 + L, i3 = i2 + L, i4 = i3 + L;
|
||||
VT x0r = vx_load(sre + i0), x0i = vx_load(sim + i0), x1r, x1i, x2r, x2i, x3r, x3i, x4r, x4i;
|
||||
DFT_VCMUL(x1r, x1i, vx_load(sre + i1), vx_load(sim + i1), vx_load(twr + j), vx_load(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, vx_load(sre + i2), vx_load(sim + i2), vx_load(twr + ts + j), vx_load(twi + ts + j));
|
||||
DFT_VCMUL(x3r, x3i, vx_load(sre + i3), vx_load(sim + i3), vx_load(twr + ts*2 + j), vx_load(twi + ts*2 + j));
|
||||
DFT_VCMUL(x4r, x4i, vx_load(sre + i4), vx_load(sim + i4), vx_load(twr + ts*3 + j), vx_load(twi + ts*3 + j));
|
||||
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0), x1r, x1i, x2r, x2i, x3r, x3i, x4r, x4i;
|
||||
DFT_VCMUL(x1r, x1i, VLOAD(sre + i1), VLOAD(sim + i1), VLOAD(twr + j), VLOAD(twi + j));
|
||||
DFT_VCMUL(x2r, x2i, VLOAD(sre + i2), VLOAD(sim + i2), VLOAD(twr + ts + j), VLOAD(twi + ts + j));
|
||||
DFT_VCMUL(x3r, x3i, VLOAD(sre + i3), VLOAD(sim + i3), VLOAD(twr + ts*2 + j), VLOAD(twi + ts*2 + j));
|
||||
DFT_VCMUL(x4r, x4i, VLOAD(sre + i4), VLOAD(sim + i4), VLOAD(twr + ts*3 + j), VLOAD(twi + ts*3 + j));
|
||||
VT a1r = v_add(x1r, x4r), a1i = v_add(x1i, x4i), b1r = v_sub(x1r, x4r), b1i = v_sub(x1i, x4i);
|
||||
VT a2r = v_add(x2r, x3r), a2i = v_add(x2i, x3i), b2r = v_sub(x2r, x3r), b2i = v_sub(x2i, x3i);
|
||||
VT sar = v_add(a1r, a2r), sai = v_add(a1i, a2i);
|
||||
VT tr = v_sub(x0r, v_mul(quarter, sar)), ti = v_sub(x0i, v_mul(quarter, sai));
|
||||
VT ur = v_mul(v_setall_(c5), v_sub(a1r, a2r)), ui = v_mul(v_setall_(c5), v_sub(a1i, a2i));
|
||||
VT v1r = v_fma_n(b1r, s1, v_mul(v_setall_(s2), b2r)), v1i = v_fma_n(b1i, s1, v_mul(v_setall_(s2), b2i));
|
||||
VT v2r = v_fma_n(b1r, s2, v_mul(v_setall_(-s1), b2r)), v2i = v_fma_n(b1i, s2, v_mul(v_setall_(-s1), b2i));
|
||||
VT ur = v_mul(VSETALL(c5), v_sub(a1r, a2r)), ui = v_mul(VSETALL(c5), v_sub(a1i, a2i));
|
||||
VT v1r = v_fma_n(b1r, s1, v_mul(VSETALL(s2), b2r)), v1i = v_fma_n(b1i, s1, v_mul(VSETALL(s2), b2i));
|
||||
VT v2r = v_fma_n(b1r, s2, v_mul(VSETALL(-s1), b2r)), v2i = v_fma_n(b1i, s2, v_mul(VSETALL(-s1), b2i));
|
||||
VT pr = v_add(tr, ur), pi = v_add(ti, ui), qr = v_sub(tr, ur), qi = v_sub(ti, ui);
|
||||
v_store(dre + i0, v_add(x0r, sar)); v_store(dim + i0, v_add(x0i, sai));
|
||||
v_store(dre + i1, v_add(pr, v1i)); v_store(dim + i1, v_sub(pi, v1r));
|
||||
@@ -754,16 +1041,16 @@ void radixOdd_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim
|
||||
T* ar = scratch; T* ai = ar + h*VL; T* br = ai + h*VL; T* bi = br + h*VL;
|
||||
DFT_VLOOP_BEGIN(r)
|
||||
int i0 = base + j;
|
||||
VT x0r = vx_load(sre + i0), x0i = vx_load(sim + i0);
|
||||
VT x0r = VLOAD(sre + i0), x0i = VLOAD(sim + i0);
|
||||
VT X0r = x0r, X0i = x0i;
|
||||
for (int p = 1; p <= h; p++)
|
||||
{
|
||||
int ip = i0 + p*L, iq = i0 + (r-p)*L;
|
||||
VT xpr, xpi, xqr, xqi;
|
||||
DFT_VCMUL(xpr, xpi, vx_load(sre + ip), vx_load(sim + ip),
|
||||
vx_load(twr + (p-1)*ts + j), vx_load(twi + (p-1)*ts + j));
|
||||
DFT_VCMUL(xqr, xqi, vx_load(sre + iq), vx_load(sim + iq),
|
||||
vx_load(twr + (r-p-1)*ts + j), vx_load(twi + (r-p-1)*ts + j));
|
||||
DFT_VCMUL(xpr, xpi, VLOAD(sre + ip), VLOAD(sim + ip),
|
||||
VLOAD(twr + (p-1)*ts + j), VLOAD(twi + (p-1)*ts + j));
|
||||
DFT_VCMUL(xqr, xqi, VLOAD(sre + iq), VLOAD(sim + iq),
|
||||
VLOAD(twr + (r-p-1)*ts + j), VLOAD(twi + (r-p-1)*ts + j));
|
||||
VT apr = v_add(xpr, xqr), api = v_add(xpi, xqi);
|
||||
v_store(ar + (p-1)*VL, apr); v_store(ai + (p-1)*VL, api);
|
||||
v_store(br + (p-1)*VL, v_sub(xpr, xqr)); v_store(bi + (p-1)*VL, v_sub(xpi, xqi));
|
||||
@@ -772,12 +1059,13 @@ void radixOdd_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim
|
||||
v_store(dre + i0, X0r); v_store(dim + i0, X0i);
|
||||
for (int k = 1; k <= h; k++)
|
||||
{
|
||||
VT sr = x0r, si = x0i, tr = v_setall_((T)0), ti = tr;
|
||||
for (int p = 1; p <= h; p++)
|
||||
VT sr = x0r, si = x0i, tr = VSETALL((T)0), ti = tr;
|
||||
for (int p = 1, m = 0; p <= h; p++)
|
||||
{
|
||||
T c = cs[(p-1)*h + (k-1)], s = sn[(p-1)*h + (k-1)];
|
||||
sr = v_fma_n(vx_load(ar + (p-1)*VL), c, sr); si = v_fma_n(vx_load(ai + (p-1)*VL), c, si);
|
||||
tr = v_fma_n(vx_load(br + (p-1)*VL), s, tr); ti = v_fma_n(vx_load(bi + (p-1)*VL), s, ti);
|
||||
m += k; m -= m >= r ? r : 0; // m = p*k mod r
|
||||
T c = cs[m], s = sn[m];
|
||||
sr = v_fma_n(VLOAD(ar + (p-1)*VL), c, sr); si = v_fma_n(VLOAD(ai + (p-1)*VL), c, si);
|
||||
tr = v_fma_n(VLOAD(br + (p-1)*VL), s, tr); ti = v_fma_n(VLOAD(bi + (p-1)*VL), s, ti);
|
||||
}
|
||||
v_store(dre + i0 + k*L, v_add(sr, ti)); v_store(dim + i0 + k*L, v_sub(si, tr));
|
||||
v_store(dre + i0 + (r-k)*L, v_sub(sr, ti)); v_store(dim + i0 + (r-k)*L, v_add(si, tr));
|
||||
@@ -788,11 +1076,13 @@ void radixOdd_vec(const DftStage& st, const T* sre, const T* sim, T* dre, T* dim
|
||||
#undef DFT_VLOOP_BEGIN
|
||||
#undef DFT_VLOOP_END
|
||||
#undef DFT_VCMUL
|
||||
#undef VLOAD
|
||||
#undef VSETALL
|
||||
|
||||
#endif // CV_SIMD || CV_SIMD_SCALABLE
|
||||
|
||||
// entry points: the plan's stage mode selects the body; VT = T (no SIMD for this type) means scalar only
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
void radix3_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void*)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
@@ -803,12 +1093,17 @@ void radix3_(const DftStage& st, const void* sre, const void* sim, void* dre, vo
|
||||
radix3_vec<T, VT>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
if (st.mode == DFT_STAGE_SIMD128)
|
||||
{
|
||||
radix3_vec<T, VT128>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
radix3_scalar<T>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
void radix4_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void*)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
@@ -819,12 +1114,17 @@ void radix4_(const DftStage& st, const void* sre, const void* sim, void* dre, vo
|
||||
radix4_vec<T, VT>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
if (st.mode == DFT_STAGE_SIMD128)
|
||||
{
|
||||
radix4_vec<T, VT128>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
radix4_scalar<T>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
}
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
void radix5_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void*)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
@@ -835,13 +1135,18 @@ void radix5_(const DftStage& st, const void* sre, const void* sim, void* dre, vo
|
||||
radix5_vec<T, VT>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
if (st.mode == DFT_STAGE_SIMD128)
|
||||
{
|
||||
radix5_vec<T, VT128>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
radix5_scalar<T>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim);
|
||||
}
|
||||
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
void radixOdd_(const DftStage& st, const void* sre, const void* sim, void* dre, void* dim, void* scratch)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
@@ -852,11 +1157,39 @@ void radixOdd_(const DftStage& st, const void* sre, const void* sim, void* dre,
|
||||
radixOdd_vec<T, VT>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, (T*)scratch);
|
||||
return;
|
||||
}
|
||||
if (st.mode == DFT_STAGE_SIMD128)
|
||||
{
|
||||
// the scratch is sized for the wide lanes, which is enough for the narrow ones
|
||||
radixOdd_vec<T, VT128>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, (T*)scratch);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
radixOdd_scalar<T>(st, (const T*)sre, (const T*)sim, (T*)dre, (T*)dim, (T*)scratch);
|
||||
}
|
||||
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
void radix4Last_(const DftStage& st, const void* sre, const void* sim, void* dst, double re_scale, double im_scale)
|
||||
{
|
||||
T rs = (T)re_scale, is = (T)im_scale;
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
if constexpr (haveSIMD)
|
||||
{
|
||||
if (st.mode == DFT_STAGE_FULL)
|
||||
{
|
||||
radix4Last_vec<T, VT>(st, (const T*)sre, (const T*)sim, (T*)dst, rs, is);
|
||||
return;
|
||||
}
|
||||
if (st.mode == DFT_STAGE_SIMD128)
|
||||
{
|
||||
radix4Last_vec<T, VT128>(st, (const T*)sre, (const T*)sim, (T*)dst, rs, is);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
radix4Last_scalar<T>(st, (const T*)sre, (const T*)sim, (T*)dst, rs, is);
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////// group C: postprocessing ////////////////////////////////
|
||||
//
|
||||
@@ -1108,30 +1441,36 @@ void postprocIDCT_(const DftPlan& plan, const void* _re, const void* _im, void*
|
||||
|
||||
#undef DFT4_SCALAR
|
||||
|
||||
template<typename T, typename VT, bool haveSIMD>
|
||||
template<typename T, typename VT, typename VT128, bool haveSIMD>
|
||||
DFTKernels makeKernels()
|
||||
{
|
||||
DFTKernels k;
|
||||
k.preprocRadix0 = preprocRadix0_<T>;
|
||||
k.preprocRadix2 = preprocRadix2_<T, VT, haveSIMD>;
|
||||
k.preprocRadix3 = preprocRadix3_<T, VT, haveSIMD>;
|
||||
k.preprocRadix4 = preprocRadix4_<T, VT, haveSIMD>;
|
||||
k.preprocRadix5 = preprocRadix5_<T, VT, haveSIMD>;
|
||||
k.preprocRadix8 = preprocRadix8_<T, VT, haveSIMD>;
|
||||
k.preprocDCT = preprocDCT_<T, VT, haveSIMD>;
|
||||
k.preprocCCS = preprocCCS_<T, VT, haveSIMD>;
|
||||
k.preprocIDCT = preprocIDCT_<T, VT, haveSIMD>;
|
||||
k.radix3 = radix3_<T, VT, haveSIMD>;
|
||||
k.radix4 = radix4_<T, VT, haveSIMD>;
|
||||
k.radix5 = radix5_<T, VT, haveSIMD>;
|
||||
k.radixOdd = radixOdd_<T, VT, haveSIMD>;
|
||||
k.radix3 = radix3_<T, VT, VT128, haveSIMD>;
|
||||
k.radix4 = radix4_<T, VT, VT128, haveSIMD>;
|
||||
k.radix5 = radix5_<T, VT, VT128, haveSIMD>;
|
||||
k.radixOdd = radixOdd_<T, VT, VT128, haveSIMD>;
|
||||
k.radix4Last = radix4Last_<T, VT, VT128, haveSIMD>;
|
||||
k.postprocDFT = postprocDFT_<T, VT, haveSIMD>;
|
||||
k.postprocRealDFT = postprocRealDFT_<T, VT, haveSIMD>;
|
||||
k.postprocReal = postprocReal_<T, VT, haveSIMD>;
|
||||
k.postprocDCT = postprocDCT_<T, VT, haveSIMD>;
|
||||
k.postprocIDCT = postprocIDCT_<T, VT, haveSIMD>;
|
||||
if constexpr (haveSIMD)
|
||||
{
|
||||
k.vlanes = VTraits<VT>::vlanes();
|
||||
k.vlanes128 = VTraits<VT128>::vlanes();
|
||||
}
|
||||
else
|
||||
k.vlanes = 1;
|
||||
k.vlanes = k.vlanes128 = 1;
|
||||
return k;
|
||||
}
|
||||
|
||||
@@ -1139,19 +1478,29 @@ DFTKernels makeKernels()
|
||||
|
||||
DFTKernels getDFTKernels_(int depth)
|
||||
{
|
||||
// the 128-bit types exist alongside the wide ones on all fixed-width ISAs (not with scalable
|
||||
// vectors); they are the same type as the wide ones on 128-bit ISAs
|
||||
if (depth == CV_32F)
|
||||
{
|
||||
#if (CV_SIMD || CV_SIMD_SCALABLE)
|
||||
return makeKernels<float, v_float32, true>();
|
||||
#if CV_SIMD128 && !CV_SIMD_SCALABLE
|
||||
return makeKernels<float, v_float32, v_float32x4, true>();
|
||||
#else
|
||||
return makeKernels<float, float, false>();
|
||||
return makeKernels<float, v_float32, v_float32, true>();
|
||||
#endif
|
||||
#else
|
||||
return makeKernels<float, float, float, false>();
|
||||
#endif
|
||||
}
|
||||
CV_Assert(depth == CV_64F);
|
||||
#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F)
|
||||
return makeKernels<double, v_float64, true>();
|
||||
#if CV_SIMD128_64F && !CV_SIMD_SCALABLE
|
||||
return makeKernels<double, v_float64, v_float64x2, true>();
|
||||
#else
|
||||
return makeKernels<double, double, false>();
|
||||
return makeKernels<double, v_float64, v_float64, true>();
|
||||
#endif
|
||||
#else
|
||||
return makeKernels<double, double, double, false>();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -1311,7 +1311,8 @@ static std::vector<int> dxtSweepSizes()
|
||||
if (m == 1)
|
||||
sizes.push_back(n);
|
||||
}
|
||||
static const int extra[] = { 7, 11, 13, 14, 21, 22, 26, 49, 77, 98, 121, 143, 154, 242, 338, 1001, 2018 };
|
||||
// odd and prime lengths (generic odd radix, r0 = 1/3/5 first stages, radix >= 127 tables)
|
||||
static const int extra[] = { 7, 11, 13, 14, 21, 22, 26, 49, 77, 98, 121, 143, 154, 242, 257, 338, 1001, 1009, 2018 };
|
||||
for (size_t i = 0; i < sizeof(extra)/sizeof(extra[0]); i++)
|
||||
sizes.push_back(extra[i]);
|
||||
return sizes;
|
||||
|
||||
Reference in New Issue
Block a user