mirror of
https://github.com/opencv/opencv.git
synced 2026-09-11 04:43:22 -05:00
0627765f01
Support ONNX Cast/CastLike for FP8/FP4/INT4/UINT4/E8M0 dtypes - #29360 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
707 lines
30 KiB
C++
707 lines
30 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
|
|
#include "../precomp.hpp"
|
|
|
|
#include <type_traits>
|
|
#include <opencv2/dnn/shape_utils.hpp>
|
|
#include "cpu_kernels/fast_gemm.hpp"
|
|
#include "cpu_kernels/mlas_gemm.hpp"
|
|
|
|
// OpenVINO backend
|
|
#include "../op_inf_engine.hpp"
|
|
#include "../ie_ngraph.hpp"
|
|
|
|
// Vulkan backend
|
|
#include "../op_vkcom.hpp"
|
|
|
|
// CUDA backend
|
|
#ifdef HAVE_CUDA
|
|
#include "../cuda4dnn/primitives/matmul_broadcast.hpp"
|
|
using namespace cv::dnn::cuda4dnn;
|
|
#endif
|
|
|
|
// CANN backend
|
|
#include "../op_cann.hpp"
|
|
|
|
namespace cv { namespace dnn {
|
|
|
|
class MatMulLayerImpl CV_FINAL : public MatMulLayer {
|
|
#ifdef HAVE_OPENCL
|
|
UMat weight_umat, bias_umat;
|
|
#endif
|
|
|
|
public:
|
|
MatMulLayerImpl(const LayerParams& params) {
|
|
setParamsFrom(params);
|
|
|
|
trans_a = params.get<bool>("transA", false);
|
|
trans_b = params.get<bool>("transB", false);
|
|
alpha = params.get<float>("alpha", 1.f);
|
|
beta = params.get<float>("beta", 1.f);
|
|
|
|
real_ndims_C = params.get<int>("real_ndims_C", -1);
|
|
|
|
for (Mat& blob : blobs) {
|
|
if (blob.type() == CV_16F || blob.type() == CV_16BF) {
|
|
Mat widened;
|
|
blob.convertTo(widened, CV_32F);
|
|
blob = widened;
|
|
}
|
|
}
|
|
}
|
|
|
|
virtual bool supportBackend(int backendId) CV_OVERRIDE {
|
|
return backendId == DNN_BACKEND_OPENCV ||
|
|
backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ||
|
|
(backendId == DNN_BACKEND_VKCOM && haveVulkan() && !trans_a && !trans_b) ||
|
|
backendId == DNN_BACKEND_CUDA ||
|
|
backendId == DNN_BACKEND_CANN;
|
|
}
|
|
|
|
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
|
const int requiredOutputs,
|
|
std::vector<MatShape> &outputs,
|
|
std::vector<MatShape> &internals) const CV_OVERRIDE {
|
|
int num_inputs = inputs.size() + blobs.size();
|
|
CV_CheckGE(num_inputs, 2, "DNN/MatMul: two inputs at least");
|
|
CV_CheckLE(num_inputs, 3, "DNN/MatMul: three inputs at most");
|
|
|
|
const auto shape_A = inputs[0], shape_B = blobs.empty() ? inputs[1] : shape(blobs[0]);
|
|
CV_CheckGE(shape_A.size(), static_cast<size_t>(2), "DNN/MatMul: invalid shape of input A");
|
|
CV_CheckGE(shape_B.size(), static_cast<size_t>(2), "DNN/MatMul: invalid shape of input B");
|
|
|
|
// Check legal matrix multiplication
|
|
int mA = shape_A[shape_A.size() - 2], nA = shape_A.back();
|
|
int mB = shape_B[shape_B.size() - 2], nB = shape_B.back();
|
|
int M = trans_a ? nA : mA;
|
|
int N = trans_b ? mB : nB;
|
|
int K_A = trans_a ? mA : nA;
|
|
int K_B = trans_b ? nB : mB;
|
|
CV_CheckEQ(K_A, K_B, "DNN/MatMul: invalid dimension K");
|
|
|
|
// Check if inputs are broadcastable.
|
|
MatShape common_shape;
|
|
if (shape_A.size() != 2 || shape_B.size() != 2) {
|
|
const auto &shape_more_dims = shape_A.size() > shape_B.size() ? shape_A : shape_B;
|
|
const auto &shape_less_dims = shape_A.size() > shape_B.size() ? shape_B : shape_A;
|
|
size_t diff_dims = shape_more_dims.size() - shape_less_dims.size();
|
|
common_shape = shape_more_dims;
|
|
for (size_t i = 0; i < shape_less_dims.size() - 2; i++) {
|
|
const auto dl = shape_less_dims[i], dm = shape_more_dims[i + diff_dims];
|
|
if (dl != 1 && dm != 1 && dl != dm) {
|
|
CV_Error(Error::StsBadSize, format("DNN/MatMul: invalid shape for broadcasting, shape_A[%zu]=%d, shape_B[%zu]=%d\n", i, shape_less_dims[i], i, shape_more_dims[i + diff_dims]));
|
|
}
|
|
|
|
if (dm == 1) {
|
|
common_shape[i + diff_dims] = dl;
|
|
}
|
|
}
|
|
common_shape[common_shape.size() - 2] = M;
|
|
common_shape[common_shape.size() - 1] = N;
|
|
} else {
|
|
common_shape.resize(2);
|
|
common_shape[0] = M;
|
|
common_shape[1] = N;
|
|
}
|
|
|
|
// Check if bias is broadcastable
|
|
if (num_inputs == 3) {
|
|
const auto shape_C = blobs.empty() ? inputs.back() : shape(blobs.back());
|
|
if (real_ndims_C == 1) { // (1) or (N)
|
|
CV_Check(shape_C[0], shape_C[0] == 1 || shape_C[0] == N, "DNN/MatMul: invalid dimension of C");
|
|
} else if (real_ndims_C >= 2) {
|
|
const auto &shape_large = common_shape.size() > shape_C.size() ? common_shape : shape_C;
|
|
const auto &shape_small = common_shape.size() > shape_C.size() ? shape_C : common_shape;
|
|
size_t diff_dims = shape_large.size() - shape_small.size();
|
|
for (size_t i = 0; i < shape_small.size(); i++) {
|
|
const auto dl = shape_small[i], dm = shape_large[i + diff_dims];
|
|
if (dl != 1 && dm != 1 && dl != dm) {
|
|
CV_Error(Error::StsBadSize, "DNN/MatMul: invalid shape of C");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
outputs.assign(1, common_shape);
|
|
return false;
|
|
}
|
|
|
|
// Only types forward() actually dispatches; the default gate's CV_8S/CV_8U had no kernel and corrupted memory.
|
|
void getTypes(const std::vector<MatType>& inputs,
|
|
const int requiredOutputs,
|
|
const int requiredInternals,
|
|
std::vector<MatType>& outputs,
|
|
std::vector<MatType>& internals) const CV_OVERRIDE
|
|
{
|
|
CV_Assert(inputs.size());
|
|
for (auto input : inputs)
|
|
CV_CheckType(input, input == CV_32F || input == CV_64F ||
|
|
input == CV_32S || input == CV_64S || input == CV_32U || input == CV_64U, "");
|
|
|
|
outputs.assign(requiredOutputs, inputs[0]);
|
|
internals.assign(requiredInternals, inputs[0]);
|
|
}
|
|
|
|
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
|
|
const std::vector<MatShape> &outputs) const CV_OVERRIDE
|
|
{
|
|
CV_Assert(!inputs.empty());
|
|
const auto shape_A = inputs[0], shape_B = blobs.empty() ? inputs[1] : shape(blobs[0]);
|
|
int mA = shape_A[shape_A.size() - 2], nA = shape_A.back();
|
|
int mB = shape_B[shape_B.size() - 2], nB = shape_B.back();
|
|
int M = trans_a ? nA : mA;
|
|
int N = trans_b ? mB : nB;
|
|
int K = trans_a ? mA : nA;
|
|
|
|
int64 batch = 1;
|
|
for (size_t i = 0; i + 2 < outputs[0].size(); i++)
|
|
batch *= outputs[0][i];
|
|
|
|
// 2*M*N*K multiply-adds per batch element, +M*N for bias if present
|
|
int64 flops = batch * (CV_BIG_INT(2) * M * N * K);
|
|
int num_inputs = (int)inputs.size() + (int)blobs.size();
|
|
if (num_inputs == 3)
|
|
flops += batch * M * N;
|
|
return flops;
|
|
}
|
|
|
|
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE {
|
|
opt.init();
|
|
|
|
std::vector<Mat> inputs, outputs;
|
|
inputs_arr.getMatVector(inputs);
|
|
outputs_arr.getMatVector(outputs);
|
|
|
|
const auto A_shape = shape(inputs[0]),
|
|
B_shape = blobs.empty() ? shape(inputs[1]) : shape(blobs[0]),
|
|
C_shape = shape(outputs[0]);
|
|
helper.compute(trans_a, trans_b, A_shape, B_shape, C_shape);
|
|
|
|
// These five types skip the float-only packed-B/MLAS caching below.
|
|
{
|
|
int depth0 = inputs[0].depth();
|
|
if (depth0 == CV_64F || depth0 == CV_32S || depth0 == CV_64S || depth0 == CV_32U || depth0 == CV_64U)
|
|
return;
|
|
}
|
|
|
|
// Pack only 2D weight matrices; skip higher-dim tensors (e.g. Q@K^T in attention).
|
|
const Mat* B_mat = !blobs.empty() ? &blobs[0] :
|
|
(inputs.size() >= 2 && inputs[1].dims == 2 ? &inputs[1] : nullptr);
|
|
if (B_mat && B_mat->data != last_packed_input_B_data) {
|
|
packed_input_B.clear();
|
|
packed_input_B.shrink_to_fit();
|
|
thin_packed_B.clear();
|
|
|
|
if (helper.batch == 1 && B_mat->type() == CV_32F &&
|
|
fastGemmThinEligible(helper.M, helper.N, helper.K)) {
|
|
thin_packed_B.resize(fastGemmThinPackBSize(helper.N, helper.K));
|
|
fastGemmThinPackB(helper.N, helper.K,
|
|
B_mat->ptr<const float>(),
|
|
(size_t)helper.ldb0, (size_t)helper.ldb1,
|
|
thin_packed_B.data());
|
|
} else {
|
|
fastGemmPackB(*B_mat, packed_input_B, trans_b, opt);
|
|
helper.updatePackedBOffsets(packed_input_B.size());
|
|
}
|
|
last_packed_input_B_data = B_mat->data;
|
|
}
|
|
|
|
// broadcast bias if needed
|
|
if ((inputs.size() + blobs.size()) >= 3 && blobs.size() >= 2) {
|
|
const auto bias_mat = blobs.back();
|
|
const auto bias_shape = shape(bias_mat);
|
|
bool is_broadcast_needed = real_ndims_C == 0 || real_ndims_C == 1 || (total(bias_shape) != total(C_shape) || bias_shape.size() != C_shape.size());
|
|
|
|
if (is_broadcast_needed) {
|
|
broadcast_bias = Mat(C_shape, CV_32F);
|
|
auto *broadcast_bias_ptr = broadcast_bias.ptr<float>();
|
|
|
|
const auto *bias = bias_mat.ptr<const float>();
|
|
if (bias_mat.total() == 1) { // [], [1], [1, ...]
|
|
float b = (*bias) * beta;
|
|
for (size_t i = 0; i < broadcast_bias.total(); i++) {
|
|
broadcast_bias_ptr[i] = b;
|
|
}
|
|
} else if (real_ndims_C == 1) { // [n]
|
|
size_t inner_size = C_shape.back(),
|
|
loops = total(C_shape) / inner_size;
|
|
for (size_t i = 0; i < loops; i++) {
|
|
size_t step = i * inner_size;
|
|
for (size_t j = 0; j < inner_size; j++) {
|
|
broadcast_bias_ptr[step + j] = beta * bias[j];
|
|
}
|
|
}
|
|
} else {
|
|
broadcast(bias_mat, C_shape, broadcast_bias);
|
|
}
|
|
} else {
|
|
broadcast_bias = blobs.back();
|
|
}
|
|
}
|
|
|
|
#ifdef HAVE_OPENCL
|
|
weight_umat.release();
|
|
bias_umat.release();
|
|
#endif
|
|
}
|
|
|
|
// works like Y = numpy.matmul(A, B)
|
|
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE {
|
|
CV_TRACE_FUNCTION();
|
|
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
|
|
|
|
CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget),
|
|
forward_ocl(inputs_arr, outputs_arr, internals_arr))
|
|
|
|
if (inputs_arr.depth() == CV_16F)
|
|
{
|
|
forward_fallback(inputs_arr, outputs_arr, internals_arr);
|
|
return;
|
|
}
|
|
|
|
std::vector<Mat> inputs, outputs;
|
|
inputs_arr.getMatVector(inputs);
|
|
outputs_arr.getMatVector(outputs);
|
|
|
|
switch (inputs[0].depth()) {
|
|
case CV_64F: forwardDouble(inputs, outputs); return;
|
|
case CV_32S: forwardInt<int32_t>(inputs, outputs); return;
|
|
case CV_64S: forwardInt<int64_t>(inputs, outputs); return;
|
|
case CV_32U: forwardInt<uint32_t>(inputs, outputs); return;
|
|
case CV_64U: forwardInt<uint64_t>(inputs, outputs); return;
|
|
default: break;
|
|
}
|
|
|
|
const auto &A = inputs[0];
|
|
auto &Y = outputs[0];
|
|
|
|
const auto *a = A.ptr<const float>();
|
|
auto *y = Y.ptr<float>();
|
|
// add bias if existed
|
|
if ((inputs.size() + blobs.size()) >= 3) {
|
|
const auto &shape_Y = shape(Y);
|
|
if (blobs.empty()) { // bias from input
|
|
const auto &bias_mat = inputs.back();
|
|
const auto *bias = bias_mat.ptr<const float>();
|
|
if (bias_mat.total() == 1) { // [], [1], [1, ...]
|
|
float b = (*bias) * beta;
|
|
for (size_t i = 0; i < Y.total(); i++) {
|
|
y[i] = b;
|
|
}
|
|
} else if (real_ndims_C == 1) { // [n]
|
|
const size_t inner_size = shape_Y.back(),
|
|
batches = total(Y) / inner_size;
|
|
parallel_for_(Range(0, batches), [&] (const Range &r) {
|
|
for (int i = r.start; i < r.end; i++) {
|
|
const size_t output_offset = i * inner_size;
|
|
for (size_t j = 0; j < inner_size; j++) {
|
|
y[output_offset + j] = beta * bias[j];
|
|
}
|
|
}
|
|
}, double(batches * inner_size * (1 / 1024.0)));
|
|
} else {
|
|
broadcast(bias_mat, shape_Y, Y);
|
|
}
|
|
} else { // bias from constant
|
|
const auto *bias = broadcast_bias.ptr<const float>();
|
|
std::memcpy(y, bias, total(shape_Y) * sizeof(float));
|
|
}
|
|
} else {
|
|
std::memset(y, 0, Y.total() * sizeof(float));
|
|
}
|
|
|
|
if (blobs.empty()) {
|
|
const auto &B = inputs[1];
|
|
const auto *b = B.ptr<const float>();
|
|
bool done = false;
|
|
if (mlasAvailable() && !opt.use_rvv && helper.M > 0 && helper.N > 0 && helper.K > 0) {
|
|
const auto A_shape = shape(A);
|
|
const auto B_shape = shape(B);
|
|
const int lda_mem = A_shape.back();
|
|
const int ldb_mem = B_shape.back();
|
|
done = mlasSgemmBatch(helper.batch,
|
|
helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
|
trans_a, trans_b, helper.M, helper.N, helper.K,
|
|
alpha, a, lda_mem, b, ldb_mem, beta, y, helper.ldc);
|
|
}
|
|
if (!done) {
|
|
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
|
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
|
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
|
|
}
|
|
} else if (!thin_packed_B.empty()) {
|
|
fastGemmThin(helper.M, helper.N, helper.K, alpha,
|
|
a, helper.lda0, helper.lda1,
|
|
thin_packed_B.data(), beta,
|
|
y, helper.ldc, opt.multi_thread);
|
|
} else if (!packed_input_B.empty()) {
|
|
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.packed_B_offsets.data(), helper.C_offsets.data(),
|
|
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
|
packed_input_B.data(), beta, y, helper.ldc, opt);
|
|
} else {
|
|
// truly dynamic B (changes every call — no packing cache available)
|
|
const auto &B = inputs[1];
|
|
const auto *b = B.ptr<const float>();
|
|
fastGemmBatch(helper.batch, helper.A_offsets.data(), helper.B_offsets.data(), helper.C_offsets.data(),
|
|
helper.M, helper.N, helper.K, alpha, a, helper.lda0, helper.lda1,
|
|
b, helper.ldb0, helper.ldb1, beta, y, helper.ldc, opt);
|
|
}
|
|
}
|
|
|
|
// CV_64F: one cv::gemm call per batch slice (batches don't collapse like Gemm's).
|
|
void forwardDouble(const std::vector<Mat>& inputs, std::vector<Mat>& outputs)
|
|
{
|
|
const Mat &A = inputs[0];
|
|
Mat &Y = outputs[0];
|
|
const Mat &B = blobs.empty() ? inputs[1] : blobs[0];
|
|
|
|
CV_CheckTypeEQ(B.depth(), CV_64F, "DNN/MatMul: B must be CV_64F to match A");
|
|
CV_Assert(A.isContinuous() && B.isContinuous() && Y.isContinuous());
|
|
|
|
const auto shape_A = shape(A), shape_B = shape(B), shape_Y = shape(Y);
|
|
const int ma = shape_A[shape_A.size() - 2], na = shape_A.back();
|
|
const int mb = shape_B[shape_B.size() - 2], nb = shape_B.back();
|
|
const int M = helper.M, N = helper.N;
|
|
|
|
double* yptr0 = Y.ptr<double>();
|
|
const bool haveBias = (inputs.size() + blobs.size()) >= 3;
|
|
if (haveBias) {
|
|
const Mat& bias_mat = blobs.size() >= 2 ? blobs.back() : inputs.back();
|
|
CV_CheckTypeEQ(bias_mat.depth(), CV_64F, "DNN/MatMul: bias must be CV_64F to match A");
|
|
if (bias_mat.total() == 1) {
|
|
std::fill_n(yptr0, Y.total(), (double)beta * bias_mat.ptr<double>()[0]);
|
|
} else {
|
|
Mat biasBroadcast(shape_Y, CV_64F);
|
|
cv::broadcast(bias_mat, shape_Y, biasBroadcast);
|
|
const double* bb = biasBroadcast.ptr<double>();
|
|
double b = (double)beta;
|
|
for (size_t i = 0; i < Y.total(); i++)
|
|
yptr0[i] = b * bb[i];
|
|
}
|
|
} else {
|
|
std::fill_n(yptr0, Y.total(), 0.0);
|
|
}
|
|
|
|
const int flags = (trans_a ? GEMM_1_T : 0) | (trans_b ? GEMM_2_T : 0);
|
|
const double* aptr0 = A.ptr<double>();
|
|
const double* bptr0 = B.ptr<double>();
|
|
|
|
for (size_t i = 0; i < helper.batch; i++) {
|
|
Mat Aview(ma, na, CV_64F, (void*)(aptr0 + helper.A_offsets[i]));
|
|
Mat Bview(mb, nb, CV_64F, (void*)(bptr0 + helper.B_offsets[i]));
|
|
Mat Yview(M, N, CV_64F, (void*)(yptr0 + helper.C_offsets[i]));
|
|
cv::gemm(Aview, Bview, (double)alpha, Yview, 1.0, Yview, flags);
|
|
}
|
|
}
|
|
|
|
// No BLAS routine handles integers; accumulates in 64-bit Acc and wraps on cast.
|
|
template<typename T>
|
|
void forwardInt(const std::vector<Mat>& inputs, std::vector<Mat>& outputs)
|
|
{
|
|
typedef typename std::conditional<std::is_unsigned<T>::value, uint64_t, int64_t>::type Acc;
|
|
|
|
const Mat &A = inputs[0];
|
|
Mat &Y = outputs[0];
|
|
const Mat &B = blobs.empty() ? inputs[1] : blobs[0];
|
|
|
|
CV_CheckTypeEQ(B.depth(), A.depth(), "DNN/MatMul: B must match A's type");
|
|
CV_Assert(A.isContinuous() && B.isContinuous() && Y.isContinuous());
|
|
|
|
const auto shape_Y = shape(Y);
|
|
const int M = helper.M, N = helper.N, K = helper.K;
|
|
const int lda0 = helper.lda0, lda1 = helper.lda1;
|
|
const int ldb0 = helper.ldb0, ldb1 = helper.ldb1;
|
|
const int ldc = helper.ldc;
|
|
|
|
const T* aptr0 = A.ptr<T>();
|
|
const T* bptr0 = B.ptr<T>();
|
|
T* yptr0 = Y.ptr<T>();
|
|
|
|
// Reject non-integer alpha/beta rather than round: rounding 0.4/0.2 to 0 would silently zero the result.
|
|
const bool haveBias = (inputs.size() + blobs.size()) >= 3;
|
|
double alpha_d = (double)alpha, beta_d = haveBias ? (double)beta : 0.0;
|
|
CV_CheckTrue(std::floor(alpha_d) == alpha_d, "DNN/MatMul: alpha must be an integer value for integer types");
|
|
CV_CheckTrue(std::floor(beta_d) == beta_d, "DNN/MatMul: beta must be an integer value for integer types");
|
|
const Acc alphaScale = (Acc)alpha_d;
|
|
const Acc betaScale = (Acc)beta_d;
|
|
|
|
Mat biasBroadcast;
|
|
if (haveBias) {
|
|
const Mat& bias_mat = blobs.size() >= 2 ? blobs.back() : inputs.back();
|
|
CV_CheckTypeEQ(bias_mat.depth(), A.depth(), "DNN/MatMul: bias must match A's type");
|
|
if ((size_t)bias_mat.total() != Y.total() || shape(bias_mat).size() != shape_Y.size()) {
|
|
biasBroadcast = Mat(shape_Y, A.depth());
|
|
cv::broadcast(bias_mat, shape_Y, biasBroadcast);
|
|
} else {
|
|
biasBroadcast = bias_mat;
|
|
}
|
|
}
|
|
const T* biasPtr0 = haveBias ? biasBroadcast.ptr<T>() : nullptr;
|
|
|
|
parallel_for_(Range(0, (int)(helper.batch * (size_t)M)), [&](const Range& r) {
|
|
for (int idx = r.start; idx < r.end; idx++) {
|
|
int bi = idx / M;
|
|
int m = idx % M;
|
|
const T* aBase = aptr0 + helper.A_offsets[bi];
|
|
const T* bBase = bptr0 + helper.B_offsets[bi];
|
|
T* yRow = yptr0 + helper.C_offsets[bi] + (size_t)m * ldc;
|
|
const T* biasRow = haveBias ? biasPtr0 + helper.C_offsets[bi] + (size_t)m * ldc : nullptr;
|
|
|
|
for (int n = 0; n < N; n++) {
|
|
Acc sum = 0;
|
|
for (int k = 0; k < K; k++) {
|
|
Acc av = (Acc)aBase[(size_t)m * lda0 + (size_t)k * lda1];
|
|
Acc bv = (Acc)bBase[(size_t)k * ldb0 + (size_t)n * ldb1];
|
|
sum += av * bv;
|
|
}
|
|
Acc result = sum * alphaScale;
|
|
if (haveBias)
|
|
result += betaScale * (Acc)biasRow[n];
|
|
yRow[n] = (T)result;
|
|
}
|
|
}
|
|
}, (double)helper.batch * M * N * (1 / 1024.0));
|
|
}
|
|
|
|
#ifdef HAVE_OPENCL
|
|
bool forward_ocl(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, InputArrayOfArrays internals) {
|
|
std::vector<UMat> inputs;
|
|
std::vector<UMat> outputs;
|
|
|
|
bool use_half = (inputs_arr.depth() == CV_16F);
|
|
inputs_arr.getUMatVector(inputs);
|
|
outputs_arr.getUMatVector(outputs);
|
|
|
|
// does not support bias as input
|
|
if (inputs.size() >= 3) {
|
|
return false;
|
|
}
|
|
|
|
const auto &input_A = inputs[0];
|
|
auto &output = outputs[0];
|
|
const auto output_shape = shape(output);
|
|
|
|
if (blobs.empty()) {
|
|
weight_umat = inputs[1];
|
|
if ((inputs.size() + blobs.size() >= 3)) {
|
|
bias_umat = UMat::zeros(output_shape.size(), output_shape.data(), CV_32F);
|
|
}
|
|
} else {
|
|
if (weight_umat.empty()) {
|
|
blobs.front().copyTo(weight_umat);
|
|
}
|
|
if ((inputs.size() + blobs.size() >= 3)) {
|
|
if (bias_umat.empty()) {
|
|
broadcast_bias.copyTo(bias_umat);
|
|
}
|
|
} else {
|
|
if (bias_umat.empty()) {
|
|
bias_umat = UMat::zeros(output_shape.size(), output_shape.data(), CV_32F);
|
|
}
|
|
}
|
|
}
|
|
|
|
auto &input_B = weight_umat;
|
|
|
|
int M = static_cast<int>(helper.M),
|
|
N = static_cast<int>(helper.N),
|
|
K = static_cast<int>(helper.K),
|
|
batch = static_cast<int>(helper.batch);
|
|
int batch_A = total(shape(input_A)) / (M * K),
|
|
batch_B = total(shape(input_B)) / (N * K);
|
|
MatShape new_shape_A{batch_A, M * K}, new_shape_B{batch_B, N * K}, new_shape_output{batch, M * N};
|
|
|
|
const auto input_A_2d = input_A.reshape(1, new_shape_A.size(), &new_shape_A[0]),
|
|
input_B_2d = input_B.reshape(1, new_shape_B.size(), &new_shape_B[0]);
|
|
auto output_2d = output.reshape(1, new_shape_output.size(), &new_shape_output[0]);
|
|
UMat A, B, C, A_fp32, B_fp32, C_fp32;
|
|
for (int i = 0; i < batch; i++) {
|
|
A = input_A_2d.row(helper.A_rows[i]).reshape(1, trans_a ? K : M);
|
|
B = input_B_2d.row(helper.B_rows[i]).reshape(1, trans_b ? N : K);
|
|
C = output_2d.row(helper.C_rows[i]).reshape(1, M);
|
|
|
|
if (trans_a) {
|
|
A = A.t();
|
|
}
|
|
if (trans_b) {
|
|
B = B.t();
|
|
}
|
|
|
|
if (use_half) {
|
|
A.convertTo(A_fp32, CV_32F);
|
|
B.convertTo(B_fp32, CV_32F);
|
|
C.convertTo(C_fp32, CV_32F);
|
|
} else {
|
|
A_fp32 = A;
|
|
B_fp32 = B;
|
|
C_fp32 = C;
|
|
}
|
|
cv::gemm(A_fp32, B_fp32, 1.f, noArray(), 0.f, C_fp32);
|
|
if (use_half) {
|
|
A_fp32.convertTo(A, CV_16F);
|
|
B_fp32.convertTo(B, CV_16F);
|
|
C_fp32.convertTo(C, CV_16F);
|
|
}
|
|
}
|
|
|
|
// add bias
|
|
if (!bias_umat.empty()) {
|
|
cv::add(output, bias_umat, output);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
#endif // HAVE_OPENCL
|
|
|
|
#ifdef HAVE_DNN_NGRAPH
|
|
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs,
|
|
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
|
|
auto& input_A_node = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
|
|
std::shared_ptr<ov::Node> result;
|
|
ov::Output<ov::Node> bias;
|
|
|
|
if (blobs.empty()) {
|
|
auto &input_B_node = nodes[1].dynamicCast<InfEngineNgraphNode>()->node;
|
|
result = std::make_shared<ov::op::v0::MatMul>(input_A_node, input_B_node, trans_a, trans_b);
|
|
if (nodes.size() >= 3) {
|
|
bias = nodes[2].dynamicCast<InfEngineNgraphNode>()->node;
|
|
result = std::make_shared<ov::op::v1::Add>(result, bias);
|
|
}
|
|
} else {
|
|
auto input_B_shape = getShape<size_t>(blobs[0]);
|
|
auto input_B_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, input_B_shape, blobs[0].data);
|
|
result = std::make_shared<ov::op::v0::MatMul>(input_A_node, input_B_node, trans_a, trans_b);
|
|
if ((nodes.size() + blobs.size()) >= 3) {
|
|
const auto bias_shape = shape(broadcast_bias);
|
|
bias = std::make_shared<ov::op::v0::Constant>(ov::element::f32, std::vector<size_t>(bias_shape.begin(), bias_shape.end()), broadcast_bias.data);
|
|
result = std::make_shared<ov::op::v1::Add>(result, bias);
|
|
}
|
|
}
|
|
|
|
return Ptr<BackendNode>(new InfEngineNgraphNode(result));
|
|
}
|
|
#endif // HAVE_DNN_NGRAPH
|
|
|
|
#ifdef HAVE_VULKAN
|
|
virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs,
|
|
std::vector<Ptr<BackendWrapper> > &outputs) CV_OVERRIDE {
|
|
auto input_A_wrapper = inputs[0].dynamicCast<VkComBackendWrapper>();
|
|
auto output_wrapper = outputs[0].dynamicCast<VkComBackendWrapper>();
|
|
|
|
const auto input_A_shape = shape(*input_A_wrapper->getMat());
|
|
const auto output_shape = shape(*output_wrapper->getMat());
|
|
if ((inputs.size() + blobs.size()) >= 3 || output_shape.size() != 2) {
|
|
return Ptr<BackendNode>();
|
|
}
|
|
|
|
std::vector<Mat> constants;
|
|
|
|
if (!blobs.empty()) {
|
|
constants.push_back(blobs[0]);
|
|
}
|
|
|
|
Ptr<vkcom::OpBase> op = new vkcom::OpMatMul(constants, input_A_shape[0], input_A_shape[1], output_shape[1]);
|
|
return Ptr<BackendNode>(new VkComBackendNode(inputs, op, outputs));
|
|
}
|
|
#endif
|
|
|
|
#ifdef HAVE_CUDA
|
|
Ptr<BackendNode> initCUDA(void *context_,
|
|
const std::vector<Ptr<BackendWrapper>>& inputs,
|
|
const std::vector<Ptr<BackendWrapper>>& outputs) override {
|
|
auto context = reinterpret_cast<csl::CSLContext*>(context_);
|
|
auto input_B = Mat(), bias = Mat();
|
|
if (!blobs.empty()) {
|
|
input_B = blobs.front();
|
|
if (blobs.size() >= 2) {
|
|
bias = broadcast_bias;
|
|
}
|
|
}
|
|
|
|
CV_CheckFalse(helper.empty(), "DNN/MatMul/CUDA: MatMulHelper is not initialized");
|
|
|
|
return make_cuda_node<cuda4dnn::MatMulBroadcastOp>(preferableTarget, std::move(context->stream), std::move(context->cublas_handle), input_B, bias, trans_a, trans_b, helper.A_offsets, helper.B_offsets, helper.C_offsets, helper.batch);
|
|
}
|
|
#endif // HAVE_CUDA
|
|
|
|
#ifdef HAVE_CANN
|
|
virtual Ptr<BackendNode> initCann(const std::vector<Ptr<BackendWrapper> > &inputs,
|
|
const std::vector<Ptr<BackendWrapper> > &outputs,
|
|
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE {
|
|
auto input_A_wrapper = inputs[0].dynamicCast<CannBackendWrapper>();
|
|
auto input_A_desc = input_A_wrapper->getTensorDesc();
|
|
auto input_A_node = nodes[0].dynamicCast<CannBackendNode>()->getOp();
|
|
|
|
auto op = std::make_shared<ge::op::BatchMatMulV2>(name);
|
|
|
|
// set attributes
|
|
op->set_attr_adj_x1(trans_a);
|
|
op->set_attr_adj_x2(trans_b);
|
|
|
|
// set inputs
|
|
// set inputs : x1
|
|
op->set_input_x1_by_name(*input_A_node, input_A_wrapper->name.c_str());
|
|
op->update_input_desc_x1(*input_A_desc);
|
|
// set inputs : x2
|
|
if (blobs.empty()) { // variable input B
|
|
auto input_B_wrapper = inputs[1].dynamicCast<CannBackendWrapper>();
|
|
auto input_B_desc = input_B_wrapper->getTensorDesc();
|
|
auto input_B_node = nodes[1].dynamicCast<CannBackendNode>()->getOp();
|
|
op->set_input_x2_by_name(*input_B_node, "y");
|
|
op->update_input_desc_x2(*input_B_desc);
|
|
if (inputs.size() >= 3) {
|
|
auto input_bias_wrapper = inputs[2].dynamicCast<CannBackendWrapper>();
|
|
auto input_bias_desc = input_bias_wrapper->getTensorDesc();
|
|
auto input_bias_node = nodes[2].dynamicCast<CannBackendNode>()->getOp();
|
|
op->set_input_bias_by_name(*input_bias_node, "y");
|
|
op->update_input_desc_bias(*input_bias_desc);
|
|
}
|
|
} else { // constant input B
|
|
auto B = blobs[0];
|
|
auto const_B_node = std::make_shared<CannConstOp>(B.data, B.type(), shape(B), cv::format("%s_B", name.c_str()));
|
|
op->set_input_x2_by_name(*(const_B_node->getOp()), "y");
|
|
op->update_input_desc_x2(*(const_B_node->getTensorDesc()));
|
|
if ((inputs.size() + blobs.size()) >= 3) { // does not support broadcast bias
|
|
auto bias_mat = blobs.back();
|
|
auto bias_shape = shape(bias_mat);
|
|
|
|
// reshape if 1d
|
|
if (real_ndims_C == 1 && bias_shape.front() != 1) {
|
|
bias_shape = std::vector<int>{bias_shape.front()};
|
|
}
|
|
|
|
auto const_bias_node = std::make_shared<CannConstOp>(bias_mat.data, bias_mat.type(), bias_shape, cv::format("%s_bias", name.c_str()));
|
|
op->set_input_bias_by_name(*(const_bias_node->getOp()), "y");
|
|
op->update_input_desc_bias(*(const_bias_node->getTensorDesc()));
|
|
}
|
|
}
|
|
|
|
// set outputs
|
|
auto output_desc = std::make_shared<ge::TensorDesc>(ge::Shape(), ge::FORMAT_NCHW, ge::DT_FLOAT);
|
|
op->update_output_desc_y(*output_desc);
|
|
return Ptr<BackendNode>(new CannBackendNode(op));
|
|
}
|
|
#endif // HAVE_CANN
|
|
|
|
private:
|
|
int real_ndims_C;
|
|
|
|
std::vector<float> packed_input_B;
|
|
std::vector<float> thin_packed_B;
|
|
Mat broadcast_bias;
|
|
|
|
const uchar* last_packed_input_B_data = nullptr;
|
|
|
|
FastGemmOpt opt;
|
|
MatMulHelper helper;
|
|
};
|
|
|
|
Ptr<MatMulLayer> MatMulLayer::create(const LayerParams& params)
|
|
{
|
|
return makePtr<MatMulLayerImpl>(params);
|
|
}
|
|
|
|
}} // cv::dnn
|