mirror of
https://github.com/opencv/opencv.git
synced 2026-11-21 00:35:35 -06:00
dnn(cuda): transformer layer coverage and CUDA Graph replay on the new engine - #30439 Follow-up to #30415 (ResNet layer coverage and cuDNN JIT conv/pooling on the new engine). Adds CUDA support on the new DNN engine for the layers transformer models need, and brings the CUDA path to parity with ONNX Runtime. **Companion PR** opencv/opencv_extra#1448 **Changes** - CUDA execs for Attention, Cast, ConstantOfShape, Expand, Gather, Not, Reduce, Shape, Slice, Softmax, Tile, Transpose, Concat, Gelu, LayerNormalization, MatMul, Split, Squeeze, Unsqueeze and the element-wise activations. - A graph with static shapes is recorded as one CUDA Graph on the second `forward()` and replayed from then on. - The cuDNN JIT convolution reads NCHW buffers in place; NHWC staging is kept only for FP16 grouped convolutions. - TF32 for cuBLAS GEMMs (`OPENCV_DNN_CUDA_ALLOW_TF32`), FP32 accumulation for FP16 GEMMs (`OPENCV_DNN_CUDA_FP16_HALF_ACCUMULATE`), and fused MatMul/Gemm post-ops. - Batch-broadcast MatMul and Shape are safe to capture in a CUDA Graph; the unused pointer-array `cublas::gemmBatched` is removed. - Tests: `forward()` runs three times on batch-broadcast MatMul and Shape models; `MatMul_init_bcast` runs on CUDA again; `AttentionSharedShapeReshape` uses the FP16 default tolerance on CUDA FP16. - Perf: new CUDA-only `*_CUDA_pinned` tests pin the input shapes. **Performance** (ms, median of 50, FP32 with TF32, RTX PRO 6000 Blackwell, idle GPU) | Model | CPU | CUDA (cuDNN JIT) | CUDA (full cuDNN) | ORT 1.31 CUDA EP | |---|---|---|---|---| | ResNet-50 | 8.02 | 0.80 | 0.80 | 0.88 | | ResNet-18 | 3.27 | 0.44 | 0.44 | 0.48 | | MobileNetV2 | 1.99 | 0.28 | 0.27 | 0.40 | | DenseNet-121 | 17.42 | 1.48 | 1.58 | 1.58 | | EfficientNet-Lite4 | 5.03 | 0.53 | 0.54 | 0.73 | | ViT-B/32 | 11.06 | 1.02 | 1.02 | 0.95 | | ViT-Base/16 | 24.38 | 1.16 | 1.17 | 1.13 | | DeiT-Tiny | 4.58 | 0.65 | 0.65 | 0.68 | | MobileViT-XS | 6.93 | 0.72 | 0.72 | 0.77 | | YOLOv3 | 47.21 | 2.60 | 2.59 | 2.72 | | YOLOv4 | 109.38 | 3.58 | 3.61 | 4.18 | | YOLOv4-tiny | 7.31 | 0.64 | 0.64 | 0.66 | | BEiT-Base | 25.30 | 1.24 | 1.24 | 1.19 | | BERT | 10.00 | 0.97 | 0.97 | 0.99 | | SigLIP | 41.14 | 2.09 | 2.09 | 2.44 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake
330 lines
12 KiB
C++
330 lines
12 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
// Copyright (C) 2025, BigVision LLC, all rights reserved.
|
|
// Third party copyrights are property of their respective owners.
|
|
|
|
#include "../precomp.hpp"
|
|
|
|
#include <opencv2/dnn/shape_utils.hpp>
|
|
#include "../net_impl.hpp"
|
|
#include "../op_cann.hpp"
|
|
#include "../op_cuda.hpp"
|
|
#ifdef HAVE_CUDA
|
|
#include "../cuda4dnn/primitives/reduce.hpp"
|
|
#endif
|
|
#include "layers_common.hpp"
|
|
#include "../dnn_common.hpp"
|
|
|
|
namespace cv {
|
|
namespace dnn {
|
|
|
|
class Reduce2LayerImpl CV_FINAL : public Reduce2Layer
|
|
{
|
|
public:
|
|
static const char* reduceTypeToString(ReduceType t)
|
|
{
|
|
switch (t) {
|
|
case ReduceType::MAX: return "MAX";
|
|
case ReduceType::MIN: return "MIN";
|
|
case ReduceType::MEAN: return "MEAN";
|
|
case ReduceType::SUM: return "SUM";
|
|
case ReduceType::L1: return "L1";
|
|
case ReduceType::L2: return "L2";
|
|
case ReduceType::PROD: return "PROD";
|
|
case ReduceType::SUM_SQUARE: return "SUM_SQUARE";
|
|
case ReduceType::LOG_SUM: return "LOG_SUM";
|
|
case ReduceType::LOG_SUM_EXP: return "LOG_SUM_EXP";
|
|
}
|
|
return "UNKNOWN";
|
|
}
|
|
Reduce2LayerImpl(const LayerParams& params) {
|
|
setParamsFrom(params);
|
|
|
|
CV_Assert(params.has("reduce"));
|
|
String reduce_type_str = toLowerCase(params.get<String>("reduce"));
|
|
if (reduce_type_str == "max")
|
|
reduce_type = ReduceType::MAX;
|
|
else if (reduce_type_str == "min")
|
|
reduce_type = ReduceType::MIN;
|
|
else if (reduce_type_str == "mean")
|
|
reduce_type = ReduceType::MEAN;
|
|
else if (reduce_type_str == "sum")
|
|
reduce_type = ReduceType::SUM;
|
|
else if (reduce_type_str == "sum_square")
|
|
reduce_type = ReduceType::SUM_SQUARE;
|
|
else if (reduce_type_str == "l1")
|
|
reduce_type = ReduceType::L1;
|
|
else if (reduce_type_str == "l2")
|
|
reduce_type = ReduceType::L2;
|
|
else if (reduce_type_str == "log_sum")
|
|
reduce_type = ReduceType::LOG_SUM;
|
|
else if (reduce_type_str == "log_sum_exp")
|
|
reduce_type = ReduceType::LOG_SUM_EXP;
|
|
else if (reduce_type_str == "prod")
|
|
reduce_type = ReduceType::PROD;
|
|
else
|
|
CV_Error(Error::StsBadArg, "Unknown reduce type\"" + reduce_type_str + "\"");
|
|
|
|
keepdims = params.get<bool>("keepdims", true);
|
|
noop_with_empty_axes = params.get<bool>("noop_with_empty_axes", false);
|
|
|
|
if (params.has("axes")) {
|
|
auto param_axes = params.get("axes");
|
|
int num_axes = param_axes.size();
|
|
axes.resize(num_axes);
|
|
for (int i = 0; i < num_axes; ++i)
|
|
axes[i] = param_axes.get<int>(i);
|
|
}
|
|
}
|
|
|
|
bool dynamicOutputShapes() const CV_OVERRIDE
|
|
{
|
|
if (inputs.size() < 2)
|
|
return false;
|
|
Net::Impl* netimpl_ = getNetImpl(this);
|
|
if (!netimpl_)
|
|
return true;
|
|
return !netimpl_->isConstArg(inputs[1]);
|
|
}
|
|
|
|
bool getMemoryShapes(const std::vector<MatShape> &inps,
|
|
const int /*requiredOutputs*/,
|
|
std::vector<MatShape> &outs,
|
|
std::vector<MatShape> &/*internals*/) const CV_OVERRIDE
|
|
{
|
|
CV_Assert(!inps.empty());
|
|
outs.resize(1);
|
|
const MatShape& inp0 = inps[0];
|
|
if (inp0.empty()) {
|
|
outs[0] = MatShape();
|
|
return false;
|
|
}
|
|
|
|
std::vector<int> axes;
|
|
if (!this->axes.empty()) {
|
|
axes = this->axes;
|
|
} else if (inps.size() >= 2) {
|
|
Net::Impl* netimpl_ = getNetImpl(this);
|
|
if (netimpl_ && netimpl_->isConstArg(inputs[1])) {
|
|
Mat axesTensor = netimpl_->argTensor(inputs[1]);
|
|
tensorToIntVec(axesTensor, axes);
|
|
}
|
|
}
|
|
|
|
if (axes.empty()) {
|
|
if (noop_with_empty_axes) {
|
|
outs[0] = inp0;
|
|
} else {
|
|
if (keepdims) {
|
|
MatShape shape_out = inp0;
|
|
std::fill(shape_out.begin(), shape_out.end(), 1);
|
|
outs[0] = shape_out;
|
|
} else {
|
|
outs[0] = MatShape::scalar();
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
std::vector<int> norm_axes = axes;
|
|
for (size_t i = 0; i < norm_axes.size(); ++i)
|
|
norm_axes[i] = normalize_axis(norm_axes[i], inp0);
|
|
|
|
auto shape_output_ = inp0;
|
|
for (int axis : norm_axes) shape_output_[axis] = -1;
|
|
MatShape shape_output;
|
|
for (size_t i = 0; i < shape_output_.size(); ++i) {
|
|
if (shape_output_[i] == -1) {
|
|
if (keepdims) shape_output.push_back(1);
|
|
} else {
|
|
shape_output.push_back(shape_output_[i]);
|
|
}
|
|
}
|
|
if (shape_output.empty()) shape_output = MatShape::scalar();
|
|
outs[0] = shape_output;
|
|
return false;
|
|
}
|
|
|
|
virtual bool supportBackend(int backendId) CV_OVERRIDE {
|
|
#ifdef HAVE_CUDA
|
|
if (backendId == DNN_BACKEND_CUDA) {
|
|
Net::Impl* netimpl_ = getNetImpl(this);
|
|
if (!netimpl_ || inputs.empty())
|
|
return false;
|
|
// an intermediate arg has no type until the first forward; refuse only a known unsupported one
|
|
const int t = netimpl_->argData(inputs[0]).type;
|
|
return t < 0 || isCudaNodeType(t, false);
|
|
}
|
|
#endif
|
|
return backendId == DNN_BACKEND_OPENCV;
|
|
}
|
|
|
|
#ifdef HAVE_CUDA
|
|
Ptr<BackendNode> initCUDA(void* context_,
|
|
InputArrayOfArrays inputs_arr,
|
|
InputArrayOfArrays) CV_OVERRIDE
|
|
{
|
|
auto context = reinterpret_cast<cuda4dnn::csl::CSLContext*>(context_);
|
|
const int rank = inputs_arr.shape(0).dims;
|
|
|
|
// a non-const axes input makes dynamicOutputShapes() true, which keeps the graph off CUDA
|
|
std::vector<int> resolved_axes = axes;
|
|
if (resolved_axes.empty() && inputs.size() >= 2) {
|
|
Net::Impl* netimpl_ = getNetImpl(this);
|
|
CV_Assert(netimpl_ && netimpl_->isConstArg(inputs[1]));
|
|
tensorToIntVec(netimpl_->argTensor(inputs[1]), resolved_axes);
|
|
}
|
|
std::vector<std::int64_t> norm_axes;
|
|
for (int a : resolved_axes)
|
|
norm_axes.push_back(normalize_axis(a, rank));
|
|
|
|
cuda4dnn::ReduceOpType cudaOp = cuda4dnn::ReduceOpType::SUM;
|
|
switch (reduce_type) {
|
|
case ReduceType::MEAN: cudaOp = cuda4dnn::ReduceOpType::MEAN; break;
|
|
case ReduceType::MAX: cudaOp = cuda4dnn::ReduceOpType::MAX; break;
|
|
case ReduceType::MIN: cudaOp = cuda4dnn::ReduceOpType::MIN; break;
|
|
case ReduceType::SUM: cudaOp = cuda4dnn::ReduceOpType::SUM; break;
|
|
case ReduceType::PROD: cudaOp = cuda4dnn::ReduceOpType::PROD; break;
|
|
case ReduceType::L1: cudaOp = cuda4dnn::ReduceOpType::L1; break;
|
|
case ReduceType::L2: cudaOp = cuda4dnn::ReduceOpType::L2; break;
|
|
case ReduceType::SUM_SQUARE: cudaOp = cuda4dnn::ReduceOpType::SUM_SQUARE; break;
|
|
case ReduceType::LOG_SUM: cudaOp = cuda4dnn::ReduceOpType::LOG_SUM; break;
|
|
case ReduceType::LOG_SUM_EXP: cudaOp = cuda4dnn::ReduceOpType::LOG_SUM_EXP; break;
|
|
}
|
|
return make_cuda_node_with_type<cuda4dnn::ReduceOp>(preferableTarget, inputs_arr.type(0),
|
|
std::move(context->stream), cudaOp,
|
|
norm_axes, noop_with_empty_axes);
|
|
}
|
|
#endif
|
|
|
|
virtual void getTypes(const std::vector<MatType>& inputs,
|
|
const int requiredOutputs,
|
|
const int requiredInternals,
|
|
std::vector<MatType>& outputs,
|
|
std::vector<MatType>& internals) const CV_OVERRIDE
|
|
{
|
|
CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_64F || inputs[0] == CV_32S || inputs[0] == CV_64S || inputs[0] == CV_16F || inputs[0] == CV_16BF || inputs[0] == CV_8U || inputs[0] == CV_8S || inputs[0] == CV_Bool, "");
|
|
outputs.assign(1, inputs[0]);
|
|
}
|
|
|
|
static int toCoreReduceOp(ReduceType t)
|
|
{
|
|
switch (t) {
|
|
case ReduceType::MAX: return REDUCE_MAX;
|
|
case ReduceType::MIN: return REDUCE_MIN;
|
|
case ReduceType::MEAN: return REDUCE_AVG;
|
|
case ReduceType::SUM: return REDUCE_SUM;
|
|
case ReduceType::L1: return REDUCE_L1;
|
|
case ReduceType::L2: return REDUCE_L2;
|
|
case ReduceType::PROD: return REDUCE_PROD;
|
|
case ReduceType::SUM_SQUARE: return REDUCE_SUM2;
|
|
case ReduceType::LOG_SUM: return REDUCE_LOG_SUM;
|
|
case ReduceType::LOG_SUM_EXP: return REDUCE_LOG_SUM_EXP;
|
|
}
|
|
CV_Error(Error::StsBadArg, "DNN/Reduce: Unsupported operation.");
|
|
}
|
|
|
|
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
|
|
{
|
|
CV_TRACE_FUNCTION();
|
|
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
|
|
|
|
std::vector<Mat> inputs, outputs;
|
|
inputs_arr.getMatVector(inputs);
|
|
|
|
CV_Assert(!inputs.empty());
|
|
Mat& src = inputs[0];
|
|
std::vector<int> axes;
|
|
if (!this->axes.empty()) {
|
|
axes = this->axes;
|
|
} else if (inputs.size() >= 2) {
|
|
tensorToIntVec(inputs[1], axes);
|
|
}
|
|
|
|
MatShape inpShape = shape(src);
|
|
MatShape outShape;
|
|
if (axes.empty()) {
|
|
if (noop_with_empty_axes) {
|
|
outShape = inpShape;
|
|
} else {
|
|
if (keepdims) {
|
|
outShape.assign(inpShape.size(), 1);
|
|
} else {
|
|
outShape = MatShape::scalar();
|
|
}
|
|
}
|
|
} else {
|
|
std::vector<int> norm_axes = axes;
|
|
for (size_t i = 0; i < norm_axes.size(); ++i)
|
|
norm_axes[i] = normalize_axis(norm_axes[i], inpShape);
|
|
MatShape tmp = inpShape;
|
|
for (int a : norm_axes) tmp[a] = -1;
|
|
for (size_t i = 0; i < tmp.size(); ++i) {
|
|
if (tmp[i] == -1) {
|
|
if (keepdims) outShape.push_back(1);
|
|
} else {
|
|
outShape.push_back(tmp[i]);
|
|
}
|
|
}
|
|
if (outShape.size() == 0) outShape = MatShape::scalar();
|
|
axes = norm_axes;
|
|
}
|
|
|
|
auto kind = outputs_arr.kind();
|
|
if (kind == _InputArray::STD_VECTOR_MAT) {
|
|
outputs_arr.getMatVecRef()[0].fit(outShape, src.type());
|
|
} else {
|
|
CV_Assert(kind == _InputArray::STD_VECTOR_UMAT);
|
|
outputs_arr.getUMatVecRef()[0].fit(outShape, src.type());
|
|
}
|
|
outputs_arr.getMatVector(outputs);
|
|
Mat& dst = outputs[0];
|
|
|
|
if (axes.empty() && noop_with_empty_axes) {
|
|
src.copyTo(dst);
|
|
return;
|
|
}
|
|
if (src.depth() == CV_Bool)
|
|
CV_Assert(reduce_type == ReduceType::MAX || reduce_type == ReduceType::MIN);
|
|
|
|
// reduce into a keepdims view of the output, which has the same layout
|
|
MatShape keepShape = inpShape;
|
|
if (axes.empty())
|
|
std::fill(keepShape.begin(), keepShape.end(), 1);
|
|
for (int a : axes)
|
|
keepShape[a] = 1;
|
|
CV_Assert(dst.isContinuous() && dst.total() == keepShape.total());
|
|
Mat dstKeep(keepShape, dst.type(), dst.data);
|
|
cv::reduce(src, dstKeep, ReduceParams(toCoreReduceOp(reduce_type), axes));
|
|
}
|
|
|
|
virtual std::ostream& dumpAttrs(std::ostream& strm, int indent) const CV_OVERRIDE
|
|
{
|
|
prindent(strm, indent);
|
|
strm << "reduce_type: \"" << reduceTypeToString(reduce_type) << "\",\n";
|
|
|
|
prindent(strm, indent);
|
|
strm << "keepdims: " << (keepdims ? "true" : "false") << ",\n";
|
|
|
|
prindent(strm, indent);
|
|
strm << "noop_with_empty_axes: " << (noop_with_empty_axes ? "true" : "false") << ",\n";
|
|
|
|
prindent(strm, indent);
|
|
strm << "axes: [";
|
|
for (size_t i = 0; i < axes.size(); ++i) {
|
|
if (i > 0) strm << ", ";
|
|
strm << axes[i];
|
|
}
|
|
strm << "]\n";
|
|
return strm;
|
|
}
|
|
};
|
|
|
|
Ptr<Reduce2Layer> Reduce2Layer::create(const LayerParams& params)
|
|
{
|
|
return Ptr<Reduce2Layer>(new Reduce2LayerImpl(params));
|
|
}
|
|
|
|
}} // cv::dnn
|