Files
opencv-MIRROR/modules/dnn/src/layers/reduce2_layer.cpp
Savya Sanchi Sharma 0f51622bac Merge pull request #30439 from SavyaSanchi-Sharma:layers
dnn(cuda): transformer layer coverage and CUDA Graph replay on the new engine - #30439

  Follow-up to #30415 (ResNet layer coverage and cuDNN JIT conv/pooling on the new engine). Adds CUDA support on the new DNN engine
  for the layers transformer models need, and brings the CUDA path to parity with ONNX Runtime.
**Companion PR** opencv/opencv_extra#1448

  **Changes**
  - CUDA execs for Attention, Cast, ConstantOfShape, Expand, Gather, Not, Reduce, Shape, Slice, Softmax, Tile, Transpose, Concat,
  Gelu, LayerNormalization, MatMul, Split, Squeeze, Unsqueeze and the element-wise activations.
  - A graph with static shapes is recorded as one CUDA Graph on the second `forward()` and replayed from then on.
  - The cuDNN JIT convolution reads NCHW buffers in place; NHWC staging is kept only for FP16 grouped convolutions.
  - TF32 for cuBLAS GEMMs (`OPENCV_DNN_CUDA_ALLOW_TF32`), FP32 accumulation for FP16 GEMMs
  (`OPENCV_DNN_CUDA_FP16_HALF_ACCUMULATE`), and fused MatMul/Gemm post-ops.
  - Batch-broadcast MatMul and Shape are safe to capture in a CUDA Graph; the unused pointer-array `cublas::gemmBatched` is
  removed.
  - Tests: `forward()` runs three times on batch-broadcast MatMul and Shape models; `MatMul_init_bcast` runs on CUDA again;
  `AttentionSharedShapeReshape` uses the FP16 default tolerance on CUDA FP16.
  - Perf: new CUDA-only `*_CUDA_pinned` tests pin the input shapes.

  **Performance** (ms, median of 50, FP32 with TF32, RTX PRO 6000 Blackwell, idle GPU)

| Model | CPU | CUDA (cuDNN JIT) | CUDA (full cuDNN) | ORT 1.31 CUDA EP |
  |---|---|---|---|---|
  | ResNet-50 | 8.02 | 0.80 | 0.80 | 0.88 |
  | ResNet-18 | 3.27 | 0.44 | 0.44 | 0.48 |
  | MobileNetV2 | 1.99 | 0.28 | 0.27 | 0.40 |
  | DenseNet-121 | 17.42 | 1.48 | 1.58 | 1.58 |
  | EfficientNet-Lite4 | 5.03 | 0.53 | 0.54 | 0.73 |
  | ViT-B/32 | 11.06 | 1.02 | 1.02 | 0.95 |
  | ViT-Base/16 | 24.38 | 1.16 | 1.17 | 1.13 |
  | DeiT-Tiny | 4.58 | 0.65 | 0.65 | 0.68 |
  | MobileViT-XS | 6.93 | 0.72 | 0.72 | 0.77 |
  | YOLOv3 | 47.21 | 2.60 | 2.59 | 2.72 |
  | YOLOv4 | 109.38 | 3.58 | 3.61 | 4.18 |
  | YOLOv4-tiny | 7.31 | 0.64 | 0.64 | 0.66 |
  | BEiT-Base | 25.30 | 1.24 | 1.24 | 1.19 |
  | BERT | 10.00 | 0.97 | 0.97 | 0.99 |
  | SigLIP | 41.14 | 2.09 | 2.09 | 2.44 
### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
2026-11-03 10:29:32 +03:00

330 lines
12 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2025, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include "../net_impl.hpp"
#include "../op_cann.hpp"
#include "../op_cuda.hpp"
#ifdef HAVE_CUDA
#include "../cuda4dnn/primitives/reduce.hpp"
#endif
#include "layers_common.hpp"
#include "../dnn_common.hpp"
namespace cv {
namespace dnn {
class Reduce2LayerImpl CV_FINAL : public Reduce2Layer
{
public:
static const char* reduceTypeToString(ReduceType t)
{
switch (t) {
case ReduceType::MAX: return "MAX";
case ReduceType::MIN: return "MIN";
case ReduceType::MEAN: return "MEAN";
case ReduceType::SUM: return "SUM";
case ReduceType::L1: return "L1";
case ReduceType::L2: return "L2";
case ReduceType::PROD: return "PROD";
case ReduceType::SUM_SQUARE: return "SUM_SQUARE";
case ReduceType::LOG_SUM: return "LOG_SUM";
case ReduceType::LOG_SUM_EXP: return "LOG_SUM_EXP";
}
return "UNKNOWN";
}
Reduce2LayerImpl(const LayerParams& params) {
setParamsFrom(params);
CV_Assert(params.has("reduce"));
String reduce_type_str = toLowerCase(params.get<String>("reduce"));
if (reduce_type_str == "max")
reduce_type = ReduceType::MAX;
else if (reduce_type_str == "min")
reduce_type = ReduceType::MIN;
else if (reduce_type_str == "mean")
reduce_type = ReduceType::MEAN;
else if (reduce_type_str == "sum")
reduce_type = ReduceType::SUM;
else if (reduce_type_str == "sum_square")
reduce_type = ReduceType::SUM_SQUARE;
else if (reduce_type_str == "l1")
reduce_type = ReduceType::L1;
else if (reduce_type_str == "l2")
reduce_type = ReduceType::L2;
else if (reduce_type_str == "log_sum")
reduce_type = ReduceType::LOG_SUM;
else if (reduce_type_str == "log_sum_exp")
reduce_type = ReduceType::LOG_SUM_EXP;
else if (reduce_type_str == "prod")
reduce_type = ReduceType::PROD;
else
CV_Error(Error::StsBadArg, "Unknown reduce type\"" + reduce_type_str + "\"");
keepdims = params.get<bool>("keepdims", true);
noop_with_empty_axes = params.get<bool>("noop_with_empty_axes", false);
if (params.has("axes")) {
auto param_axes = params.get("axes");
int num_axes = param_axes.size();
axes.resize(num_axes);
for (int i = 0; i < num_axes; ++i)
axes[i] = param_axes.get<int>(i);
}
}
bool dynamicOutputShapes() const CV_OVERRIDE
{
if (inputs.size() < 2)
return false;
Net::Impl* netimpl_ = getNetImpl(this);
if (!netimpl_)
return true;
return !netimpl_->isConstArg(inputs[1]);
}
bool getMemoryShapes(const std::vector<MatShape> &inps,
const int /*requiredOutputs*/,
std::vector<MatShape> &outs,
std::vector<MatShape> &/*internals*/) const CV_OVERRIDE
{
CV_Assert(!inps.empty());
outs.resize(1);
const MatShape& inp0 = inps[0];
if (inp0.empty()) {
outs[0] = MatShape();
return false;
}
std::vector<int> axes;
if (!this->axes.empty()) {
axes = this->axes;
} else if (inps.size() >= 2) {
Net::Impl* netimpl_ = getNetImpl(this);
if (netimpl_ && netimpl_->isConstArg(inputs[1])) {
Mat axesTensor = netimpl_->argTensor(inputs[1]);
tensorToIntVec(axesTensor, axes);
}
}
if (axes.empty()) {
if (noop_with_empty_axes) {
outs[0] = inp0;
} else {
if (keepdims) {
MatShape shape_out = inp0;
std::fill(shape_out.begin(), shape_out.end(), 1);
outs[0] = shape_out;
} else {
outs[0] = MatShape::scalar();
}
}
return false;
}
std::vector<int> norm_axes = axes;
for (size_t i = 0; i < norm_axes.size(); ++i)
norm_axes[i] = normalize_axis(norm_axes[i], inp0);
auto shape_output_ = inp0;
for (int axis : norm_axes) shape_output_[axis] = -1;
MatShape shape_output;
for (size_t i = 0; i < shape_output_.size(); ++i) {
if (shape_output_[i] == -1) {
if (keepdims) shape_output.push_back(1);
} else {
shape_output.push_back(shape_output_[i]);
}
}
if (shape_output.empty()) shape_output = MatShape::scalar();
outs[0] = shape_output;
return false;
}
virtual bool supportBackend(int backendId) CV_OVERRIDE {
#ifdef HAVE_CUDA
if (backendId == DNN_BACKEND_CUDA) {
Net::Impl* netimpl_ = getNetImpl(this);
if (!netimpl_ || inputs.empty())
return false;
// an intermediate arg has no type until the first forward; refuse only a known unsupported one
const int t = netimpl_->argData(inputs[0]).type;
return t < 0 || isCudaNodeType(t, false);
}
#endif
return backendId == DNN_BACKEND_OPENCV;
}
#ifdef HAVE_CUDA
Ptr<BackendNode> initCUDA(void* context_,
InputArrayOfArrays inputs_arr,
InputArrayOfArrays) CV_OVERRIDE
{
auto context = reinterpret_cast<cuda4dnn::csl::CSLContext*>(context_);
const int rank = inputs_arr.shape(0).dims;
// a non-const axes input makes dynamicOutputShapes() true, which keeps the graph off CUDA
std::vector<int> resolved_axes = axes;
if (resolved_axes.empty() && inputs.size() >= 2) {
Net::Impl* netimpl_ = getNetImpl(this);
CV_Assert(netimpl_ && netimpl_->isConstArg(inputs[1]));
tensorToIntVec(netimpl_->argTensor(inputs[1]), resolved_axes);
}
std::vector<std::int64_t> norm_axes;
for (int a : resolved_axes)
norm_axes.push_back(normalize_axis(a, rank));
cuda4dnn::ReduceOpType cudaOp = cuda4dnn::ReduceOpType::SUM;
switch (reduce_type) {
case ReduceType::MEAN: cudaOp = cuda4dnn::ReduceOpType::MEAN; break;
case ReduceType::MAX: cudaOp = cuda4dnn::ReduceOpType::MAX; break;
case ReduceType::MIN: cudaOp = cuda4dnn::ReduceOpType::MIN; break;
case ReduceType::SUM: cudaOp = cuda4dnn::ReduceOpType::SUM; break;
case ReduceType::PROD: cudaOp = cuda4dnn::ReduceOpType::PROD; break;
case ReduceType::L1: cudaOp = cuda4dnn::ReduceOpType::L1; break;
case ReduceType::L2: cudaOp = cuda4dnn::ReduceOpType::L2; break;
case ReduceType::SUM_SQUARE: cudaOp = cuda4dnn::ReduceOpType::SUM_SQUARE; break;
case ReduceType::LOG_SUM: cudaOp = cuda4dnn::ReduceOpType::LOG_SUM; break;
case ReduceType::LOG_SUM_EXP: cudaOp = cuda4dnn::ReduceOpType::LOG_SUM_EXP; break;
}
return make_cuda_node_with_type<cuda4dnn::ReduceOp>(preferableTarget, inputs_arr.type(0),
std::move(context->stream), cudaOp,
norm_axes, noop_with_empty_axes);
}
#endif
virtual void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_64F || inputs[0] == CV_32S || inputs[0] == CV_64S || inputs[0] == CV_16F || inputs[0] == CV_16BF || inputs[0] == CV_8U || inputs[0] == CV_8S || inputs[0] == CV_Bool, "");
outputs.assign(1, inputs[0]);
}
static int toCoreReduceOp(ReduceType t)
{
switch (t) {
case ReduceType::MAX: return REDUCE_MAX;
case ReduceType::MIN: return REDUCE_MIN;
case ReduceType::MEAN: return REDUCE_AVG;
case ReduceType::SUM: return REDUCE_SUM;
case ReduceType::L1: return REDUCE_L1;
case ReduceType::L2: return REDUCE_L2;
case ReduceType::PROD: return REDUCE_PROD;
case ReduceType::SUM_SQUARE: return REDUCE_SUM2;
case ReduceType::LOG_SUM: return REDUCE_LOG_SUM;
case ReduceType::LOG_SUM_EXP: return REDUCE_LOG_SUM_EXP;
}
CV_Error(Error::StsBadArg, "DNN/Reduce: Unsupported operation.");
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
CV_Assert(!inputs.empty());
Mat& src = inputs[0];
std::vector<int> axes;
if (!this->axes.empty()) {
axes = this->axes;
} else if (inputs.size() >= 2) {
tensorToIntVec(inputs[1], axes);
}
MatShape inpShape = shape(src);
MatShape outShape;
if (axes.empty()) {
if (noop_with_empty_axes) {
outShape = inpShape;
} else {
if (keepdims) {
outShape.assign(inpShape.size(), 1);
} else {
outShape = MatShape::scalar();
}
}
} else {
std::vector<int> norm_axes = axes;
for (size_t i = 0; i < norm_axes.size(); ++i)
norm_axes[i] = normalize_axis(norm_axes[i], inpShape);
MatShape tmp = inpShape;
for (int a : norm_axes) tmp[a] = -1;
for (size_t i = 0; i < tmp.size(); ++i) {
if (tmp[i] == -1) {
if (keepdims) outShape.push_back(1);
} else {
outShape.push_back(tmp[i]);
}
}
if (outShape.size() == 0) outShape = MatShape::scalar();
axes = norm_axes;
}
auto kind = outputs_arr.kind();
if (kind == _InputArray::STD_VECTOR_MAT) {
outputs_arr.getMatVecRef()[0].fit(outShape, src.type());
} else {
CV_Assert(kind == _InputArray::STD_VECTOR_UMAT);
outputs_arr.getUMatVecRef()[0].fit(outShape, src.type());
}
outputs_arr.getMatVector(outputs);
Mat& dst = outputs[0];
if (axes.empty() && noop_with_empty_axes) {
src.copyTo(dst);
return;
}
if (src.depth() == CV_Bool)
CV_Assert(reduce_type == ReduceType::MAX || reduce_type == ReduceType::MIN);
// reduce into a keepdims view of the output, which has the same layout
MatShape keepShape = inpShape;
if (axes.empty())
std::fill(keepShape.begin(), keepShape.end(), 1);
for (int a : axes)
keepShape[a] = 1;
CV_Assert(dst.isContinuous() && dst.total() == keepShape.total());
Mat dstKeep(keepShape, dst.type(), dst.data);
cv::reduce(src, dstKeep, ReduceParams(toCoreReduceOp(reduce_type), axes));
}
virtual std::ostream& dumpAttrs(std::ostream& strm, int indent) const CV_OVERRIDE
{
prindent(strm, indent);
strm << "reduce_type: \"" << reduceTypeToString(reduce_type) << "\",\n";
prindent(strm, indent);
strm << "keepdims: " << (keepdims ? "true" : "false") << ",\n";
prindent(strm, indent);
strm << "noop_with_empty_axes: " << (noop_with_empty_axes ? "true" : "false") << ",\n";
prindent(strm, indent);
strm << "axes: [";
for (size_t i = 0; i < axes.size(); ++i) {
if (i > 0) strm << ", ";
strm << axes[i];
}
strm << "]\n";
return strm;
}
};
Ptr<Reduce2Layer> Reduce2Layer::create(const LayerParams& params)
{
return Ptr<Reduce2Layer>(new Reduce2LayerImpl(params));
}
}} // cv::dnn