mirror of
https://github.com/opencv/opencv.git
synced 2026-11-19 15:55:28 -06:00
Extended graph fusion for transformer models #28931 ### Pull Request Readiness Checklist Performance numbers on Intel(R) Core(TM) i9-14900KS: ``` AlexNet::DNNTestNetwork::OCV/CPU 7.143 6.871 1.04 AlexNet::DNNTestNetwork::OCV/OCL 7.471 7.047 1.06 AlexNet::DNNTestNetwork::OCV/OCL_FP16 7.157 6.907 1.04 BERT::DNNTestNetwork::OCV/CPU 9.454 9.154 1.03 BERT::DNNTestNetwork::OCV/OCL 9.657 9.138 1.06 BERT::DNNTestNetwork::OCV/OCL_FP16 9.590 9.271 1.03 CRNN::DNNTestNetwork::OCV/CPU 18.194 17.020 1.07 CRNN::DNNTestNetwork::OCV/OCL 17.855 17.433 1.02 CRNN::DNNTestNetwork::OCV/OCL_FP16 17.629 17.277 1.02 DenseNet_121::DNNTestNetwork::OCV/CPU 24.735 23.863 1.04 DenseNet_121::DNNTestNetwork::OCV/OCL 24.991 25.140 0.99 DenseNet_121::DNNTestNetwork::OCV/OCL_FP16 24.598 24.019 1.02 EAST_text_detection::DNNTestNetwork::OCV/CPU 26.929 27.084 0.99 EAST_text_detection::DNNTestNetwork::OCV/OCL 26.863 27.531 0.98 EAST_text_detection::DNNTestNetwork::OCV/OCL_FP16 26.884 27.082 0.99 EfficientDet::DNNTestNetwork::OCV/CPU 62.667 62.716 1.00 EfficientDet_int8::DNNTestNetwork::OCV/CPU 34.446 34.796 0.99 EfficientNet::DNNTestNetwork::OCV/CPU 11.875 11.786 1.01 EfficientNet::DNNTestNetwork::OCV/OCL 12.434 11.806 1.05 EfficientNet::DNNTestNetwork::OCV/OCL_FP16 11.886 11.899 1.00 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/CPU 15.104 14.327 1.05 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL 15.071 14.670 1.03 FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL_FP16 15.262 14.072 1.08 GoogLeNet::DNNTestNetwork::OCV/CPU 5.030 5.031 1.00 GoogLeNet::DNNTestNetwork::OCV/OCL 5.075 5.017 1.01 GoogLeNet::DNNTestNetwork::OCV/OCL_FP16 5.053 5.055 1.00 Inception_5h::DNNTestNetwork::OCV/CPU 6.777 6.430 1.05 Inception_5h::DNNTestNetwork::OCV/OCL 7.309 6.683 1.09 Inception_5h::DNNTestNetwork::OCV/OCL_FP16 6.624 6.549 1.01 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/CPU 86.676 82.918 1.05 Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/OCL 85.460 84.873 1.01 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/CPU 16.007 15.723 1.02 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL 15.974 16.015 1.00 Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 16.219 15.593 1.04 MPHand::DNNTestNetwork::OCV/CPU 3.273 3.313 0.99 MPHand::DNNTestNetwork::OCV/OCL 3.466 3.318 1.04 MPHand::DNNTestNetwork::OCV/OCL_FP16 3.314 3.258 1.02 MPPalm::DNNTestNetwork::OCV/CPU 2.839 1.973 1.44 MPPalm::DNNTestNetwork::OCV/OCL 2.773 2.132 1.30 MPPalm::DNNTestNetwork::OCV/OCL_FP16 2.783 2.133 1.30 MPPose::DNNTestNetwork::OCV/CPU 7.644 8.711 0.88 MPPose::DNNTestNetwork::OCV/OCL 7.555 7.210 1.05 MPPose::DNNTestNetwork::OCV/OCL_FP16 7.564 7.559 1.00 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/CPU 9.040 8.813 1.03 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL 8.692 9.481 0.92 MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL_FP16 8.792 8.884 0.99 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/CPU 7.893 7.735 1.02 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL 8.233 7.816 1.05 MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 7.949 7.817 1.02 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/CPU 14.373 15.282 0.94 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL 14.218 15.788 0.90 MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 14.473 14.853 0.97 MobileNetv2_ONNX::DNNTestNetwork::OCV/CPU 1.507 1.440 1.05 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL 1.529 1.394 1.10 MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL_FP16 1.448 1.433 1.01 PPHumanSeg::DNNTestNetwork::OCV/CPU 2.302 1.919 1.20 PPHumanSeg::DNNTestNetwork::OCV/OCL 2.097 1.865 1.12 PPHumanSeg::DNNTestNetwork::OCV/OCL_FP16 2.129 1.908 1.12 PPOCRv3::DNNTestNetwork::OCV/CPU 20.612 20.858 0.99 PPOCRv3::DNNTestNetwork::OCV/OCL 21.672 21.444 1.01 PPOCRv3::DNNTestNetwork::OCV/OCL_FP16 22.660 20.857 1.09 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/CPU 6.443 6.718 0.96 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL 6.842 6.823 1.00 ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL_FP16 6.895 6.579 1.05 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/CPU 7.669 7.629 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL 7.514 7.473 1.01 ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL_FP16 7.598 7.513 1.01 SFace::DNNTestNetwork::OCV/CPU 3.427 3.334 1.03 SFace::DNNTestNetwork::OCV/OCL 3.420 3.337 1.02 SFace::DNNTestNetwork::OCV/OCL_FP16 3.497 3.375 1.04 SSD::DNNTestNetwork::OCV/CPU 83.165 83.476 1.00 SSD::DNNTestNetwork::OCV/OCL 83.531 84.121 0.99 SSD::DNNTestNetwork::OCV/OCL_FP16 82.447 83.234 0.99 SqueezeNet_v1_1::DNNTestNetwork::OCV/CPU 1.334 1.308 1.02 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL 1.279 1.313 0.97 SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL_FP16 1.429 1.290 1.11 VIT_Base_Patch16_224::DNNTestNetwork::OCV/CPU 71.211 75.013 0.95 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL 71.988 71.218 1.01 VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL_FP16 73.700 82.285 0.90 VitTrack::DNNTestNetwork::OCV/CPU 4.096 2.091 1.96 VitTrack::DNNTestNetwork::OCV/OCL 3.830 2.056 1.86 VitTrack::DNNTestNetwork::OCV/OCL_FP16 3.796 2.110 1.80 YOLOX::DNNTestNetwork::OCV/CPU 24.346 23.079 1.05 YOLOX::DNNTestNetwork::OCV/OCL 24.442 23.459 1.04 YOLOX::DNNTestNetwork::OCV/OCL_FP16 23.947 23.327 1.03 YOLOv3::DNNTestNetwork::OCV/CPU 59.140 59.140 1.00 YOLOv3::DNNTestNetwork::OCV/OCL 46.725 45.292 1.03 YOLOv3::DNNTestNetwork::OCV/OCL_FP16 46.939 45.382 1.03 YOLOv4::DNNTestNetwork::OCV/CPU 70.062 68.145 1.03 YOLOv4::DNNTestNetwork::OCV/OCL 219.570 214.823 1.02 YOLOv4::DNNTestNetwork::OCV/OCL_FP16 219.276 213.472 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/CPU 8.362 8.221 1.02 YOLOv4_tiny::DNNTestNetwork::OCV/OCL 8.373 8.095 1.03 YOLOv4_tiny::DNNTestNetwork::OCV/OCL_FP16 8.830 8.088 1.09 YOLOv5::DNNTestNetwork::OCV/CPU 9.237 8.484 1.09 YOLOv5::DNNTestNetwork::OCV/OCL 8.776 8.537 1.03 YOLOv5::DNNTestNetwork::OCV/OCL_FP16 8.747 8.536 1.02 YOLOv8::DNNTestNetwork::OCV/CPU 11.019 11.463 0.96 YOLOv8::DNNTestNetwork::OCV/OCL 11.204 10.766 1.04 YOLOv8::DNNTestNetwork::OCV/OCL_FP16 11.355 10.764 1.05 YuNet::DNNTestNetwork::OCV/CPU 3.371 3.466 0.97 YuNet::DNNTestNetwork::OCV/OCL 3.396 3.105 1.09 YuNet::DNNTestNetwork::OCV/OCL_FP16 3.150 3.130 1.01 ``` See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
152 lines
4.9 KiB
C++
152 lines
4.9 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
|
// Third party copyrights are property of their respective owners.
|
|
|
|
// Folds a scalar Mul / Div immediately preceding Softmax into the Softmax
|
|
// layer's `scale` parameter. The fused softmax bakes the scale into the
|
|
// per-element load before max/exp/sum/divide, removing one full pass over
|
|
// the (often large) [B, H, S, S] attention-logits tensor.
|
|
//
|
|
// Pattern:
|
|
// X -> Mul(X, c) -> Softmax => X -> Softmax(scale=c)
|
|
// X -> Div(X, c) -> Softmax => X -> Softmax(scale=1/c)
|
|
|
|
#include "precomp.hpp"
|
|
#include "net_impl.hpp"
|
|
|
|
namespace cv { namespace dnn {
|
|
CV__DNN_INLINE_NS_BEGIN
|
|
|
|
using std::vector;
|
|
using std::string;
|
|
|
|
struct ModelFusionScaleSoftmax
|
|
{
|
|
explicit ModelFusionScaleSoftmax(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
|
|
|
bool extractScalar(Arg a, float& out) const
|
|
{
|
|
if (!netimpl->isConstArg(a)) return false;
|
|
Mat t = netimpl->argTensor(a);
|
|
if (t.total() != 1) return false;
|
|
if (t.type() == CV_32F) { out = t.ptr<float>()[0]; return true; }
|
|
if (t.type() == CV_64F) { out = (float)t.ptr<double>()[0]; return true; }
|
|
return false;
|
|
}
|
|
|
|
void fuse() { fuseGraph(netimpl->mainGraph); }
|
|
|
|
bool fuseGraph(Ptr<Graph>& graph)
|
|
{
|
|
const vector<Ptr<Layer>>& prog = graph->prog();
|
|
size_t nops = prog.size();
|
|
bool modified = false;
|
|
|
|
for (size_t i = 0; i < nops; i++) {
|
|
if (!prog[i]) continue;
|
|
vector<Ptr<Graph>>* subgraphs = prog[i]->subgraphs();
|
|
if (subgraphs) {
|
|
for (Ptr<Graph>& g : *subgraphs)
|
|
if (fuseGraph(g)) modified = true;
|
|
}
|
|
}
|
|
|
|
vector<int> usecounts;
|
|
netimpl->useCounts(usecounts);
|
|
|
|
std::set<int> externalArgs;
|
|
for (Arg out : graph->outputs())
|
|
externalArgs.insert(out.idx);
|
|
|
|
std::map<int, int> producer;
|
|
for (size_t i = 0; i < nops; i++) {
|
|
if (!prog[i]) continue;
|
|
for (Arg out : prog[i]->outputs)
|
|
producer[out.idx] = (int)i;
|
|
}
|
|
|
|
vector<bool> dropped(nops, false);
|
|
|
|
for (size_t i = 0; i < nops; i++) {
|
|
const Ptr<Layer>& layer = prog[i];
|
|
if (!layer || dropped[i]) continue;
|
|
|
|
SoftmaxLayer* sm = dynamic_cast<SoftmaxLayer*>(layer.get());
|
|
if (!sm || sm->logSoftMax) continue;
|
|
if (layer->inputs.size() != 1) continue;
|
|
|
|
Arg sm_in = layer->inputs[0];
|
|
auto it = producer.find(sm_in.idx);
|
|
if (it == producer.end()) continue;
|
|
int prod_idx = it->second;
|
|
if (prod_idx < 0 || dropped[prod_idx]) continue;
|
|
|
|
const Ptr<Layer>& pl = prog[prod_idx];
|
|
NaryEltwiseLayer* elt = dynamic_cast<NaryEltwiseLayer*>(pl.get());
|
|
if (!elt) continue;
|
|
const auto op = elt->op;
|
|
const bool is_mul = (op == NaryEltwiseLayer::OPERATION::PROD);
|
|
const bool is_div = (op == NaryEltwiseLayer::OPERATION::DIV);
|
|
if (!is_mul && !is_div) continue;
|
|
if (pl->inputs.size() != 2 || pl->outputs.size() != 1) continue;
|
|
|
|
// Identify which operand is the scalar constant.
|
|
float scalar = 0.f;
|
|
int x_slot = -1;
|
|
if (extractScalar(pl->inputs[1], scalar)) {
|
|
x_slot = 0;
|
|
} else if (is_mul && extractScalar(pl->inputs[0], scalar)) {
|
|
// Mul is commutative; Div with the data on the right doesn't
|
|
// simplify (1 / scale*x ≠ softmax-fold) so only handle Mul here.
|
|
x_slot = 1;
|
|
} else {
|
|
continue;
|
|
}
|
|
|
|
if (is_div) {
|
|
if (scalar == 0.f) continue;
|
|
scalar = 1.f / scalar;
|
|
}
|
|
if (!std::isfinite(scalar)) continue;
|
|
|
|
bool single_consumer = usecounts[sm_in.idx] == 1
|
|
&& externalArgs.count(sm_in.idx) == 0;
|
|
if (!single_consumer) continue;
|
|
|
|
sm->scale *= scalar;
|
|
layer->inputs[0] = pl->inputs[x_slot];
|
|
dropped[prod_idx] = true;
|
|
usecounts[sm_in.idx] = 0;
|
|
modified = true;
|
|
}
|
|
|
|
if (modified) {
|
|
vector<Ptr<Layer>> newprog;
|
|
newprog.reserve(nops);
|
|
for (size_t i = 0; i < nops; i++) {
|
|
if (!dropped[i] && prog[i])
|
|
newprog.push_back(prog[i]);
|
|
}
|
|
graph->setProg(newprog);
|
|
}
|
|
|
|
return modified;
|
|
}
|
|
|
|
Net::Impl* netimpl;
|
|
};
|
|
|
|
void Net::Impl::fuseScaleSoftmax()
|
|
{
|
|
if (preferableBackend != DNN_BACKEND_OPENCV ||
|
|
preferableTarget != DNN_TARGET_CPU)
|
|
return;
|
|
ModelFusionScaleSoftmax pass(this);
|
|
pass.fuse();
|
|
}
|
|
|
|
CV__DNN_INLINE_NS_END
|
|
}}
|