Files
opencv-MIRROR/modules/dnn/src/layers/recurrent2_layers.cpp
Savya Sanchi Sharma d1f25babff Merge pull request #30309 from SavyaSanchi-Sharma:opti/lstm
dnn: speed up LSTM2 (gate math, input projection, recurrence) - #30309

Speeds up the ONNX LSTM layer (`LSTM2`) in `recurrent2_layers.cpp`:
- Default sigmoid/tanh gates are computed on raw rows with the dispatched activation kernels. Peephole, clip and custom activations keep the old path.
- The input projection runs before the per-direction loop, so the GEMM uses all threads. It goes through MLAS when available.
- Batch 1 (hidden size >= 128): hidden units are split across threads, with one `parallel_for_` per timestep.
- Batch > 1: one packed GEMM per step replaces one GEMV per row.

Ryzen 9 9950X, default threads, 5.x vs this PR:

| Model | 5.x, ms | PR, ms | x-factor |
|---|:-:|:-:|:-:|
| `DNNTestNetwork.CRNN` | 2.693 | 1.562 | 1.72 |
| CRNN EN 2021sep | 2.498 | 1.472 | 1.70 |
| crnn.onnx | 2.650 | 1.434 | 1.85 |
| LSTM2 bidirectional, batch 1, in 512, hidden 256, 24 steps (CRNN layer 1) | 0.710 | 0.212 | 3.36 
### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
2026-10-19 09:41:52 +03:00

867 lines
38 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include <cmath>
#include <opencv2/dnn/shape_utils.hpp>
#include "layers_common.hpp"
#include "cpu_kernels/fast_gemm.hpp"
#include "cpu_kernels/mlas_gemm.hpp"
#include "../net_impl.hpp"
namespace cv
{
namespace dnn
{
template<typename Dtype>
static void tanh(const Mat &src, Mat &dst)
{
MatConstIterator_<Dtype> itSrc = src.begin<Dtype>();
MatIterator_<Dtype> itDst = dst.begin<Dtype>();
for (; itSrc != src.end<Dtype>(); itSrc++, itDst++)
*itDst = std::tanh(*itSrc);
}
static void tanh(const Mat &src, Mat &dst)
{
dst.create(src.size, src.type());
if (src.type() == CV_32F)
tanh<float>(src, dst);
else if (src.type() == CV_64F)
tanh<double>(src, dst);
else
CV_Error(Error::StsUnsupportedFormat, "Function supports only floating point types");
}
static void sigmoid(const Mat &src, Mat &dst)
{
cv::exp(-src, dst);
cv::pow(1 + dst, -1, dst);
}
// ONNX gate order is I,O,F,C but the recurrence wants I,F,O,C: swap the O/F blocks.
template<typename T>
static void reorderGatesIOFCtoIFOC_(Mat& in)
{
int first = in.size[0];
int rest = in.total() / first / 4;
Mat m = in.reshape(1, {first, 4, rest});
Mat outputGate = m.col(1), forgetGate = m.col(2);
std::swap_ranges(outputGate.begin<T>(), outputGate.end<T>(), forgetGate.begin<T>());
}
static void reorderGatesIOFCtoIFOC(Mat& in)
{
if (in.depth() == CV_64F)
reorderGatesIOFCtoIFOC_<double>(in);
else
reorderGatesIOFCtoIFOC_<float>(in);
}
typedef void (*ActivationFunction)(const Mat &src, Mat &dst);
static ActivationFunction get_activation_function(const String& activation) {
if (activation == "Tanh"){
return tanh;
}
else if (activation == "Sigmoid"){
return sigmoid;
}
else
{
CV_Error(Error::StsNotImplemented,
cv::format("Activation function [%s] for layer LSTM is not supported", activation.c_str()));
}
}
// One LSTM cell step on the [i | f | o | g] gate row (blocks of H) for hidden units [j0, j0+nj); c and h already point at unit j0.
static void lstmCellFp32(float* g, int H, int j0, int nj, float forgetBias,
ActivationFunc sigmoidFn, ActivationFunc tanhFn,
float* c, float* h, float* tanhC)
{
float *gi = g + j0, *gf = g + H + j0, *go = g + 2*H + j0, *gc = g + 3*H + j0;
if (forgetBias)
for (int j = 0; j < nj; j++)
gf[j] += forgetBias;
sigmoidFn(gi, gi, nj, nullptr);
sigmoidFn(gf, gf, nj, nullptr);
sigmoidFn(go, go, nj, nullptr);
tanhFn(gc, gc, nj, nullptr);
for (int j = 0; j < nj; j++)
c[j] = gf[j]*c[j] + gi[j]*gc[j]; // c_t = f_t * c_{t-1} + i_t * g_t
tanhFn(c, tanhC, nj, nullptr);
for (int j = 0; j < nj; j++)
h[j] = go[j]*tanhC[j]; // h_t = o_t * tanh(c_t)
}
class LSTM2LayerImpl CV_FINAL : public LSTM2Layer
{
int seqLenth, batchSize, numHidden;
MatShape outTailShape; //shape of single output sample
enum layout_t : int {
SEQ_BATCH_HID = 0,
BATCH_SEQ_HID = 1
};
bool useTimestampDim;
bool produceCellOutput, produceOutputYh;
bool useCellClip, usePeephole;
bool reverse; // If true, go in negative direction along the time axis
bool bidirectional; // If true, produces both forward and reversed directions along time axis
float forgetBias, cellClip;
layout_t layout; // If layout == BATCH_SEQ_HID, uses batch_size x seq_length x num_hidden for input and output
// else uses seq_length x batch_size x num_hidden
ActivationFunction f_activation;
ActivationFunction g_activation;
ActivationFunction h_activation;
bool defaultActivations; // sigmoid/tanh/tanh: the gate math can take the fast path
ActivationFunc sigmoidFn, tanhFn; // dispatched kernels used by the fast gate path
bool useAVX, useAVX2, useSVE, useNEON;
bool constWeights; // W/R/B are graph constants -> transform them once
bool weightsPacked; // whether the cached transform has been built
std::vector<Mat> weightBlobs; // cached transformed weights: [Wx, Wh, bias, (pI,pF,pO)]
FastGemmOpt gemmOpt; // MLAS GEMM options for the batched input projection
std::vector<std::vector<float>> packedWx; // per-direction prepacked Wx (fp32) for fastGemm
std::vector<std::vector<float>> packedWh; // per-direction prepacked Wh (fp32) for the batched recurrent GEMM
#ifdef HAVE_MLAS
std::vector<Mat> packedWxMlas; // per-direction MLAS-packed Wx; preferred over packedWx when present
#endif
public:
LSTM2LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
numHidden = params.get<int>("hidden_size", 1);
layout = (layout_t) params.get<int>("layout", SEQ_BATCH_HID);
produceCellOutput = params.get<bool>("produce_cell_output", false);
produceOutputYh = params.get<bool>("produce_output_yh", false);
bidirectional = params.get<bool>("bidirectional", false);
reverse = params.get<bool>("reverse", false);
useTimestampDim = params.get<bool>("use_timestamp_dim", true);
usePeephole = params.get<bool>("use_peephole", false);
useCellClip = params.get<bool>("use_cell_clip", false);
forgetBias = params.get<float>("forget_bias", 0.0f);
cellClip = params.get<float>("cell_clip", 0.0f);
CV_Assert(!reverse || !bidirectional);
// read activations
DictValue activations = params.get<DictValue>("activations", DictValue(String()));
if (activations.size() == 1) // if activations wasn't specified use default
{
f_activation = sigmoid;
g_activation = tanh;
h_activation = tanh;
defaultActivations = true;
} else {
CV_Assert(activations.size() == 3);
f_activation = get_activation_function(activations.getStringValue(0));
g_activation = get_activation_function(activations.getStringValue(1));
h_activation = get_activation_function(activations.getStringValue(2));
defaultActivations = activations.getStringValue(0) == "Sigmoid" &&
activations.getStringValue(1) == "Tanh" &&
activations.getStringValue(2) == "Tanh";
}
constWeights = params.get<bool>("const_weights", false);
weightsPacked = false;
useAVX = checkHardwareSupport(CPU_AVX);
useAVX2 = checkHardwareSupport(CPU_AVX2);
useSVE = checkHardwareSupport(CPU_SVE);
useNEON = checkHardwareSupport(CPU_NEON);
gemmOpt.init();
sigmoidFn = getActivationFunc(ACTIV_SIGMOID);
tanhFn = getActivationFunc(ACTIV_TANH);
outTailShape.clear();
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE
{
const MatShape& inp0 = inputs[0];
const MatShape& Wx = inputs[1];
const MatShape& Wh = inputs[2];
int _hidSize = Wh[2];
int _inpSize = Wx[2];
MatShape outTailShape_(outTailShape), outResShape;
if (!outTailShape_.empty())
CV_Assert(total(outTailShape_) == _hidSize);
else
outTailShape_.assign(1, _hidSize);
// compute output shape of y
// figure out batch size
int _batchSize;
int _seqLen;
if (useTimestampDim)
{
CV_Assert(inp0.size() >= 2 && total(inp0, 2) == _inpSize);
if (layout == SEQ_BATCH_HID) {
_batchSize = inp0[1];
_seqLen = inp0[0];
outResShape.push_back(_seqLen);
outResShape.push_back(1 + static_cast<int>(bidirectional));
outResShape.push_back(_batchSize);
} else {
// ONNX layout=1: Y is (batch, seq, dirs, hid) - this must match what forward()
// actually writes, the graph engine preallocates the output by this shape
_batchSize = inp0[0];
_seqLen = inp0[1];
outResShape.push_back(_batchSize);
outResShape.push_back(_seqLen);
outResShape.push_back(1 + static_cast<int>(bidirectional));
}
}
else
{
CV_Assert(inp0.size() >= 2 && total(inp0, 1) == _inpSize);
_batchSize = inp0[0];
outResShape.push_back(1 + static_cast<int>(bidirectional));
outResShape.push_back(_batchSize);
}
outResShape.push_back(_hidSize);
outputs.assign(1, outResShape);
// Yh / Yc: ONNX layout=0 -> (dirs, batch, hid), layout=1 -> (batch, dirs, hid)
int shp[] = {1 + static_cast<int>(bidirectional), _batchSize, numHidden};
if (layout == BATCH_SEQ_HID)
std::swap(shp[0], shp[1]);
MatShape newShape(shp, shp + sizeof(shp)/sizeof(shp[0]));
// compute output shape of yc
if (produceCellOutput)
{
outputs.push_back(newShape);
}
// compute output shape of yh
if (produceOutputYh)
{
outputs.push_back(newShape);
}
// forward() allocates its own per-direction scratch, so no engine internals are needed
internals.clear();
return false;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(inputs[0] == CV_32F || inputs[0] == CV_64F); // Only floating-point types are supported currently
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const CV_OVERRIDE
{
// LSTM: 4 gates, each gate = input_size*hidden_size + hidden_size*hidden_size MACs
const MatShape& inp0 = inputs[0];
const MatShape& Wx = inputs[1];
int _hidSize = numHidden;
int _inpSize = Wx[2];
int _seqLen = inp0[0];
int _batchSize = inp0[1];
int numDirs = 1 + static_cast<int>(bidirectional);
// Per timestep: 4 gates * (2*input_size*hidden_size + 2*hidden_size*hidden_size + hidden_size)
int64 flopsPerStep = CV_BIG_INT(4) * (CV_BIG_INT(2) * _inpSize * _hidSize +
CV_BIG_INT(2) * _hidSize * _hidSize +
_hidSize);
return (int64)numDirs * _seqLen * _batchSize * flopsPerStep;
}
// Batched projection Wx*x + bias for one direction's whole sequence; runs outside the direction loop so the GEMM can use all threads.
Mat projectInput(int i, int numDirs, const Mat& xTs) const
{
const int batchSizeTotal = seqLenth * batchSize;
const int dtype = xTs.type();
Mat Wx = weightBlobs[0].rowRange(i * weightBlobs[0].rows / numDirs, (i + 1) * weightBlobs[0].rows / numDirs);
Mat bias = weightBlobs[2].colRange(i * weightBlobs[2].cols / numDirs, (i + 1) * weightBlobs[2].cols / numDirs);
const int gateN = 4 * numHidden, projK = xTs.cols;
Mat gatesAll;
if (dtype == CV_32F && (int)packedWx.size() == numDirs)
{
repeat(bias, batchSizeTotal, 1, gatesAll); // each row starts as the bias
#ifdef HAVE_MLAS
if (!packedWxMlas[i].empty() &&
mlasSgemmPacked(false, true, batchSizeTotal, gateN, projK, 1.f, xTs.ptr<float>(), projK,
packedWxMlas[i].data, 1.f, gatesAll.ptr<float>(), gateN))
return gatesAll;
#endif
if (!packedWx[i].empty())
{
FastGemmOpt opt = gemmOpt;
fastGemm(false, batchSizeTotal, gateN, projK, 1.f, xTs.ptr<float>(), projK,
packedWx[i].data(), 1.f, gatesAll.ptr<float>(), gateN, opt);
return gatesAll;
}
}
// fp64 / unpacked fallback
Mat onesCol(batchSizeTotal, 1, dtype, Scalar(1));
gatesAll.create(batchSizeTotal, gateN, dtype);
gemm(onesCol, bias, 1.0, noArray(), 0.0, gatesAll);
gemm(xTs, Wx, 1.0, gatesAll, 1.0, gatesAll, GEMM_2_T);
return gatesAll;
}
// dst[r] += Wh.row(row0 + r) . h for r in [0, nrows): fastGEMM1T for the CPU, cv::gemm otherwise.
void recurrentGemv(const float* h, const Mat& Wh, int row0, int nrows, float* dst) const
{
const float* w = Wh.ptr<float>(row0);
const size_t wstep = Wh.step1();
const int K = Wh.cols;
#if CV_TRY_AVX2
if (useAVX2 && K >= 8) { opt_AVX2::fastGEMM1T(h, w, wstep, dst, dst, nrows, K); return; }
#endif
#if CV_TRY_AVX
if (useAVX && K >= 8) { opt_AVX::fastGEMM1T(h, w, wstep, dst, dst, nrows, K); return; }
#endif
#if CV_TRY_SVE
if (useSVE) { opt_SVE::fastGEMM1T(h, w, wstep, dst, dst, nrows, K); return; }
#endif
#if CV_TRY_NEON
if (useNEON && K >= 4) { opt_NEON::fastGEMM1T(h, w, wstep, dst, dst, nrows, K); return; }
#endif
Mat hrow(1, K, CV_32F, (void*)h), wrows(nrows, K, CV_32F, (void*)w, wstep * sizeof(float));
Mat drow(1, nrows, CV_32F, dst);
gemm(hrow, wrows, 1, drow, 1, drow, GEMM_2_T);
}
// Batch-1 fp32 recurrence, one parallel_for_ per timestep; each task owns a hidden-unit chunk of one direction (its four gate rows, c and h).
void forwardStepParallel(int numDirs, int chunksPerDir, std::vector<Mat>& gatesAll, const Mat& h0All,
const Mat& c0All, Mat& hOutAll, Mat& cOut) const
{
const int H = numHidden, T = seqLenth;
const int chunk = ((H + chunksPerDir - 1) / chunksPerDir + 7) & ~7; // multiple of fastGEMM1T's 8-row block
const int ntasks = numDirs * chunksPerDir;
Mat cState = c0All.clone(); // numDirs x H, each task updates its own slice
Mat cOutRows = produceCellOutput ? cOut.reshape(1, T) : Mat(); // T x (numDirs*H)
for (int t = 0; t < T; t++)
{
parallel_for_(Range(0, ntasks), [&](const Range& r)
{
AutoBuffer<float, 1024> tanhC(chunk);
for (int task = r.start; task < r.end; task++)
{
const int d = task / chunksPerDir, j0 = (task % chunksPerDir) * chunk;
const int nj = std::min(chunk, H - j0);
if (nj <= 0)
continue;
const bool rev = reverse || d == 1;
const int ts = rev ? T - 1 - t : t;
// h_{t-1} is the previous step's row of hOutAll, complete since the last parallel_for_ returned
const float* hPrev = t == 0 ? h0All.ptr<float>(d) : hOutAll.ptr<float>(rev ? ts + 1 : ts - 1) + d*H;
Mat Wh = weightBlobs[1].rowRange(d * 4*H, (d + 1) * 4*H);
float* g = gatesAll[d].ptr<float>(ts);
for (int k = 0; k < 4; k++)
recurrentGemv(hPrev, Wh, k*H + j0, nj, g + k*H + j0);
lstmCellFp32(g, H, j0, nj, forgetBias, sigmoidFn, tanhFn,
cState.ptr<float>(d) + j0, hOutAll.ptr<float>(ts) + d*H + j0, tanhC.data());
if (produceCellOutput)
memcpy(cOutRows.ptr<float>(ts) + d*H + j0, cState.ptr<float>(d) + j0, nj * sizeof(float));
}
}, ntasks);
}
}
// Run one direction's recurrence into its own scratch so both directions run concurrently.
// Writes hOutAll columns [i*H, (i+1)*H) and the matching cOut slice.
void forwardDirection(int i, int numDirs, Mat gatesAll, const Mat& h0All, const Mat& c0All,
Mat& hOutAll, Mat& cOut, bool parallelInside) const
{
const int batchSizeTotal = seqLenth * batchSize;
const int dtype = gatesAll.type();
Mat Wh = weightBlobs[1].rowRange(i * weightBlobs[1].rows / numDirs, (i + 1) * weightBlobs[1].rows / numDirs);
Mat pI, pF, pO;
if (usePeephole)
{
pI = weightBlobs[3].rowRange(i * weightBlobs[3].rows / numDirs, (i + 1) * weightBlobs[3].rows / numDirs);
pF = weightBlobs[4].rowRange(i * weightBlobs[4].rows / numDirs, (i + 1) * weightBlobs[4].rows / numDirs);
pO = weightBlobs[5].rowRange(i * weightBlobs[5].rows / numDirs, (i + 1) * weightBlobs[5].rows / numDirs);
}
Mat hInternal(batchSize, numHidden, dtype);
Mat cInternal(batchSize, numHidden, dtype);
h0All.rowRange(i * batchSize, (i + 1) * batchSize).copyTo(hInternal);
c0All.rowRange(i * batchSize, (i + 1) * batchSize).copyTo(cInternal);
Mat hOutTs(batchSizeTotal, numHidden, dtype);
Mat cOutTs;
if (produceCellOutput)
{
cOutTs = cOut.reshape(1, batchSizeTotal);
cOutTs = cOutTs.colRange(i * cOutTs.cols / numDirs, (i + 1) * cOutTs.cols / numDirs);
}
// sigmoid/tanh/tanh without peephole or clip: gate math on raw rows with the dispatched kernels
const bool fastGates = dtype == CV_32F && defaultActivations && !usePeephole && !useCellClip;
// batch > 1: one GEMM per step reads Wh once instead of once per batch row
const bool batchedH = dtype == CV_32F && batchSize > 1 && (int)packedWh.size() == numDirs
&& hInternal.isContinuous() && gatesAll.isContinuous();
// outside a parallel region the per-step GEMM and the gate rows can use all threads
const bool parallelRows = parallelInside && batchSize > 1;
int tsStart, tsEnd, tsInc;
if (reverse || i == 1) {
tsStart = seqLenth - 1;
tsEnd = -1;
tsInc = -1;
}
else {
tsStart = 0;
tsEnd = seqLenth;
tsInc = 1;
}
for (int ts = tsStart; ts != tsEnd; ts += tsInc)
{
Range curRowRange(ts*batchSize, (ts + 1)*batchSize);
Mat gates = gatesAll.rowRange(curRowRange); // already holds Wx * x_t + b
// gates += Wh * h_{t-1}
if (batchedH)
{
FastGemmOpt opt = gemmOpt;
opt.multi_thread = parallelInside;
fastGemm(false, batchSize, Wh.rows, Wh.cols, 1.f, hInternal.ptr<float>(), Wh.cols,
packedWh[i].data(), 1.f, gates.ptr<float>(), Wh.rows, opt);
}
else if (dtype == CV_32F)
{
for (int n = 0; n < hInternal.rows; n++)
recurrentGemv(hInternal.ptr<float>(n), Wh, 0, Wh.rows, gates.ptr<float>(n));
}
else
{
gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T);
}
if (fastGates)
{
const int H = numHidden;
auto gateRows = [&](const Range& r)
{
AutoBuffer<float, 1024> tanhC(H);
for (int n = r.start; n < r.end; n++)
lstmCellFp32(gates.ptr<float>(n), H, 0, H, forgetBias, sigmoidFn, tanhFn,
cInternal.ptr<float>(n), hInternal.ptr<float>(n), tanhC.data());
};
if (parallelRows)
parallel_for_(Range(0, batchSize), gateRows);
else
gateRows(Range(0, batchSize));
hInternal.copyTo(hOutTs.rowRange(curRowRange));
if (produceCellOutput)
cInternal.copyTo(cOutTs.rowRange(curRowRange));
continue;
}
Mat gateI = gates.colRange(0*numHidden, 1*numHidden);
Mat gateF = gates.colRange(1*numHidden, 2*numHidden);
Mat gateO = gates.colRange(2*numHidden, 3*numHidden);
Mat gateG = gates.colRange(3*numHidden, 4*numHidden);
if (forgetBias){
add(gateF, forgetBias, gateF);
}
if (usePeephole)
{
Mat gatesIF = gates.colRange(0, 2*numHidden);
gemm(cInternal, pI, 1, gateI, 1, gateI);
gemm(cInternal, pF, 1, gateF, 1, gateF);
f_activation(gatesIF, gatesIF);
}
else
{
Mat gatesIFO = gates.colRange(0, 3*numHidden);
f_activation(gatesIFO, gatesIFO);
}
g_activation(gateG, gateG);
//compute c_t
multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1}
multiply(gateI, gateG, gateI); // i_t (*) g_t
add(gateF, gateI, cInternal); // c_t = f_t (*) c_{t-1} + i_t (*) g_t
if (useCellClip)
{
min(cInternal, cellClip, cInternal);
max(cInternal, -cellClip, cInternal);
}
if (usePeephole)
{
gemm(cInternal, pO, 1, gateO, 1, gateO);
f_activation(gateO, gateO);
}
//compute h_t
h_activation(cInternal, hInternal);
multiply(gateO, hInternal, hInternal);
hInternal.copyTo(hOutTs.rowRange(curRowRange));
if (produceCellOutput)
cInternal.copyTo(cOutTs.rowRange(curRowRange));
}
// slice this direction's result into the assembly buffer
hOutTs.copyTo(hOutAll.colRange(i * numHidden, (i + 1) * numHidden));
}
void forward(InputArrayOfArrays inputs_arr,
OutputArrayOfArrays outputs_arr,
OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
std::vector<Mat> input, output;
inputs_arr.getMatVector(input);
outputs_arr.getMatVector(output);
int numInputs = input.size();
int inpSize = input[0].size[2];
// determine seqLen and batchSize
if (useTimestampDim)
{
CV_Assert(input[0].dims >= 2 && (int)input[0].total(2) == inpSize);
if (layout == SEQ_BATCH_HID){
seqLenth = input[0].size[0];
batchSize = input[0].size[1];
}else{
seqLenth = input[0].size[1];
batchSize = input[0].size[0];
}
} else {
CV_Assert(input[0].dims >= 2 && (int)input[0].total(1) == inpSize);
seqLenth = 1;
batchSize = input[0].size[0];
}
// ONNX LSTM inputs 0..7: X,W,R,B,sequence_lens,initial_h,initial_c,P; slots 3..7 optional.
// Test non-emptiness, not count: some exports keep empty slots for unused inputs.
auto hasInput = [&](int idx) {
return idx < numInputs && !input[idx].empty();
};
CV_Assert(numInputs >= 3); // X, W, R are mandatory
const int numDirs = 1 + static_cast<int>(bidirectional);
const int batchSizeTotal = seqLenth * batchSize;
// Weight transform is shape-independent: once for const weights, else every forward.
if (!constWeights || !weightsPacked)
{
packWeights(input);
weightsPacked = true;
}
// Initial states are shape dependent, so (re)build them every forward.
Mat h0All = hasInput(5) ? input[5].reshape(1, input[5].size[0] * input[5].size[1])
: Mat::zeros(numDirs * batchSize, numHidden, input[0].type());
Mat c0All = hasInput(6) ? input[6].reshape(1, input[6].size[0] * input[6].size[1])
: Mat::zeros(numDirs * batchSize, numHidden, input[0].type());
// set outputs to 0
for (auto& out : output)
out.setTo(0);
// seq-major cell-state scratch: (seq, batch, dirs, hid), matching the recurrence.
int cOutShape[] = {seqLenth, batchSize, numDirs, numHidden};
Mat cOut = produceCellOutput ? Mat::zeros(4, cOutShape, output[0].type()) : Mat();
// the recurrence below slices X by timestep, so it needs the seq-major order;
// under ONNX layout=1 the input arrives as (batch, seq, ...)
Mat xSeqFirst = input[0];
if (layout == BATCH_SEQ_HID)
{
std::vector<int> perm(input[0].dims);
std::iota(perm.begin(), perm.end(), 0);
std::swap(perm[0], perm[1]);
cv::transpose(input[0], perm, xSeqFirst);
}
Mat xTs = xSeqFirst.reshape(1, batchSizeTotal);
// seq-major Y assembly buffer; transposed into output[0] below.
// Never reallocate output[0]'s header or it detaches from the graph.
Mat hOutAll(batchSizeTotal, numDirs * numHidden, output[0].type());
// Project every direction first, outside the direction loop, so the GEMM gets all threads.
std::vector<Mat> gatesAll(numDirs);
for (int i = 0; i < numDirs; i++)
gatesAll[i] = projectInput(i, numDirs, xTs);
const bool fp32 = input[0].type() == CV_32F && h0All.isContinuous() && c0All.isContinuous();
const bool fastGates = fp32 && defaultActivations && !usePeephole && !useCellClip;
// Batch 1 with enough hidden units: split each direction across threads at every timestep.
int chunksPerDir = 1;
if (fastGates && batchSize == 1 && numHidden >= 128)
chunksPerDir = std::min(numHidden / 64, std::max(1, std::min(getNumThreads(), 8) / numDirs));
if (chunksPerDir >= 2)
forwardStepParallel(numDirs, chunksPerDir, gatesAll, h0All, c0All, hOutAll, cOut);
else if (batchSize > 1)
{
// Pack Wh once for the batched recurrent GEMM; packWeights clears it when the weights change.
if (fp32 && packedWh.empty())
{
const int N = weightBlobs[1].rows / numDirs; // 4 * numHidden
packedWh.resize(numDirs);
for (int d = 0; d < numDirs; d++)
fastGemmPackB(weightBlobs[1].rowRange(d * N, (d + 1) * N), packedWh[d], true, gemmOpt);
}
// One direction at a time, so each step's GEMM and gate rows can use all threads.
for (int i = 0; i < numDirs; i++)
forwardDirection(i, numDirs, gatesAll[i], h0All, c0All, hOutAll, cOut, true);
}
else
{
// Directions are independent (disjoint output columns): run them in parallel.
parallel_for_(Range(0, numDirs), [&](const Range& r)
{
for (int i = r.start; i < r.end; i++)
forwardDirection(i, numDirs, gatesAll[i], h0All, c0All, hOutAll, cOut, false);
}, numDirs);
}
// Reshape to (seq, batch, dirs, hid), then transpose into output[0] per `layout`.
int shp1[] = {seqLenth, batchSize, numDirs, numHidden};
Mat y4d = hOutAll.reshape(1, sizeof(shp1)/sizeof(shp1[0]), shp1);
Mat ySeqFirst; // (seq, dirs, batch, hid): the layout=0 Y; Yh sliced from it
if (layout == SEQ_BATCH_HID) {
cv::transpose(y4d, {0, 2, 1, 3}, output[0]);
ySeqFirst = output[0];
} else {
cv::transpose(y4d, {0, 2, 1, 3}, ySeqFirst);
cv::transpose(y4d, {1, 0, 2, 3}, output[0]); // (batch, seq, dirs, hid)
}
if (produceOutputYh){
getCellStateYh(ySeqFirst, output[1], numDirs);
}
if (produceCellOutput){
getCellStateYc(cOut, output[2], numDirs);
}
}
void getCellStateYh(Mat& scr, Mat& dst, int numDirs)
{
// TODO: implement
if (numDirs == 1){
// Last step processed: Y[-1] going forward, Y[0] going in reverse.
const int lastTs = reverse ? 0 : scr.size[0] - 1;
Mat hOut = scr.rowRange(lastTs, lastTs + 1);
// reshape 1x1xBxH -> 1xBxH
int shp[] = {1, batchSize, numHidden};
hOut = hOut.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
if (layout == BATCH_SEQ_HID){
cv::transpose(hOut, {1, 0, 2}, dst);
}
else{
hOut.copyTo(dst);
}
} else {
// Slice: SxDxBxH -> last sequence, first direction
Range ranges1[] = {cv::Range(scr.size[0] - 1, scr.size[0]), cv::Range(0, 1), cv::Range::all(), cv::Range::all()};
Mat part1 = scr(ranges1);
// Slice: SxDxBxH -> first sequence, last direction
Range ranges2[] = {cv::Range(0, 1), cv::Range(scr.size[1] - 1, scr.size[1]), cv::Range::all(), cv::Range::all()};
Mat part2 = scr(ranges2);
int shp[] = {1, part1.size[2] * part1.size[3]};
part1 = part1.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
part2 = part2.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
// build into a temp, then write into the preallocated dst in place (vconcat straight
// into dst would replace its header and detach it from the graph's output tensor)
Mat tmp;
vconcat(part1, part2, tmp);
int finalShape[] = {2, batchSize, numHidden};
tmp = tmp.reshape(1, sizeof(finalShape)/sizeof(finalShape[0]), finalShape);
if (layout == BATCH_SEQ_HID){
cv::transpose(tmp, {1, 0, 2}, dst);
} else {
tmp.copyTo(dst);
}
}
}
void getCellStateYc(Mat& cOut, Mat& dst, int numDirs)
{
// seq, batch, dirs, hidden
int shp[] = {0, batchSize, numDirs, numHidden};
cOut = cOut.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
// permute to (seq, dirs, batch, hidden); the `layout` only affects the FINAL Yc order
// below, the last-timestep/last-direction slicing is layout-independent
cv::Mat newCellState;
cv::transpose(cOut, {0, 2, 1, 3}, newCellState);
cOut = newCellState;
if (numDirs == 1)
{
// Last step processed: Y[-1] going forward, Y[0] going in reverse.
const int lastTs = reverse ? 0 : cOut.size[0] - 1;
Range ranges[] = {cv::Range(lastTs, lastTs + 1), cv::Range::all(), cv::Range::all(), cv::Range::all()};
cOut = cOut(ranges);
// Reshape: 1x1xBxH -> 1xBxH
int shp[] = {1, batchSize, numHidden};
cOut = cOut.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
}
else
{
// Slice: SxDxBxH -> last sequence, first direction
Range ranges1[] = {cv::Range(cOut.size[0] - 1, cOut.size[0]), cv::Range(0, 1), cv::Range::all(), cv::Range::all()};
Mat part1 = cOut(ranges1);
// Slice: SxDxBxH -> first sequence, last direction
Range ranges2[] = {cv::Range(0, 1), cv::Range(cOut.size[1] - 1, cOut.size[1]), cv::Range::all(), cv::Range::all()};
Mat part2 = cOut(ranges2);
int shp[] = {1, part1.size[2] * part1.size[3]};
part1 = part1.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
part2 = part2.reshape(1, sizeof(shp)/sizeof(shp[0]), shp);
vconcat(part1, part2, cOut);
// Reshape: 1x2xBxH -> 2xBxH
int finalShape[] = {2, batchSize, numHidden};
cOut = cOut.reshape(1, sizeof(finalShape)/sizeof(finalShape[0]), finalShape);
}
// (dirs, batch, hid), or (batch, dirs, hid) for the batch-first layout - written into the
// preallocated dst in place
if (layout == BATCH_SEQ_HID){
cv::transpose(cOut, {1, 0, 2}, dst);
} else {
cOut.copyTo(dst);
}
}
// Fill weightBlobs: [0]=Wx, [1]=Wh, [2]=bias, and (peephole) [3]=pI, [4]=pF, [5]=pO.
// Shape-independent, so it runs once for constant weights.
void packWeights(const std::vector<Mat>& input)
{
int biasShape[] = {1 + static_cast<int>(bidirectional), 8 * numHidden};
weightBlobs.clear();
weightBlobs.push_back(input[1].clone()); // W -> Wx
weightBlobs.push_back(input[2].clone()); // R -> Wh
weightBlobs.push_back((input.size() > 3 && !input[3].empty())
? input[3].clone()
: Mat::zeros(2, biasShape, input[0].type())); // B
bool hasP = usePeephole && input.size() > 7 && !input[7].empty();
if (hasP)
weightBlobs.push_back(input[7].clone()); // P
Mat& Wx = weightBlobs[0];
Mat& Wh = weightBlobs[1];
Mat& b = weightBlobs[2];
b = b.reshape(1, b.size[0]);
Mat bx = b.colRange(0, b.cols / 2);
Mat bh = b.colRange(b.cols / 2, b.cols);
b = bx + bh;
reorderGatesIOFCtoIFOC(Wx);
reorderGatesIOFCtoIFOC(Wh);
reorderGatesIOFCtoIFOC(b);
weightBlobs[0] = Wx.reshape(1, Wx.size[0] * Wx.size[1]);
weightBlobs[1] = Wh.reshape(1, Wh.size[0] * Wh.size[1]);
weightBlobs[2] = b.reshape(1, 1);
// Prepack Wx per direction (fp32) so the projection is one batched GEMM; MLAS when available.
packedWx.clear();
packedWh.clear(); // packed on demand in forward() for batches > 1
#ifdef HAVE_MLAS
packedWxMlas.clear();
#endif
if (weightBlobs[0].type() == CV_32F)
{
const int numDirs = 1 + static_cast<int>(bidirectional);
const int N = weightBlobs[0].rows / numDirs; // 4 * numHidden
packedWx.resize(numDirs);
#ifdef HAVE_MLAS
packedWxMlas.resize(numDirs);
#endif
for (int d = 0; d < numDirs; d++)
{
Mat WxDir = weightBlobs[0].rowRange(d * N, (d + 1) * N);
#ifdef HAVE_MLAS
const int K = WxDir.cols; // input size
const size_t bytes = gemmOpt.use_rvv ? 0 : mlasSgemmPackBSize(false, true, N, K);
if (bytes > 0 && bytes <= static_cast<size_t>(INT_MAX))
{
packedWxMlas[d].create(1, static_cast<int>(bytes), CV_8U);
if (mlasSgemmPackB(false, true, N, K, WxDir.ptr<float>(), K, packedWxMlas[d].data))
continue;
packedWxMlas[d].release();
}
#endif
fastGemmPackB(WxDir, packedWx[d], true, gemmOpt);
}
}
if (!hasP)
return;
// ONNX stores P as Pi, Po, Pf; the gate loop wants pI, pF, pO.
Mat P = weightBlobs[3];
weightBlobs[3] = packPeephole(P, 0, numHidden); // Pi
weightBlobs.push_back(packPeephole(P, 2 * numHidden, numHidden)); // Pf
weightBlobs.push_back(packPeephole(P, numHidden, numHidden)); // Po
}
// One H x H diagonal per direction, so the per-direction rowRange stays square.
static Mat packPeephole(const Mat& P, int col0, int numHidden)
{
const int numDirs = P.rows;
Mat packed = Mat::zeros(numDirs * numHidden, numHidden, P.type());
for (int d = 0; d < numDirs; d++)
{
Mat vals = P.row(d).colRange(col0, col0 + numHidden).clone().reshape(1, numHidden);
Mat::diag(vals).copyTo(packed.rowRange(d * numHidden, (d + 1) * numHidden));
}
return packed;
}
};
Ptr<LSTM2Layer> LSTM2Layer::create(const LayerParams& params)
{
return Ptr<LSTM2Layer>(new LSTM2LayerImpl(params));
};
}}