Files
opencv-MIRROR/modules/dnn/src/graph_fusion_scale_softmax.cpp
Abhishek Gola 2760c0c08a Merge pull request #28931 from abhishek-gola:extended_fusions
Extended graph fusion for transformer models #28931

### Pull Request Readiness Checklist

Performance numbers on Intel(R) Core(TM) i9-14900KS:
```
AlexNet::DNNTestNetwork::OCV/CPU                           7.143   6.871     1.04   
AlexNet::DNNTestNetwork::OCV/OCL                           7.471   7.047     1.06   
AlexNet::DNNTestNetwork::OCV/OCL_FP16                      7.157   6.907     1.04   
BERT::DNNTestNetwork::OCV/CPU                              9.454   9.154     1.03   
BERT::DNNTestNetwork::OCV/OCL                              9.657   9.138     1.06   
BERT::DNNTestNetwork::OCV/OCL_FP16                         9.590   9.271     1.03   
CRNN::DNNTestNetwork::OCV/CPU                             18.194  17.020     1.07   
CRNN::DNNTestNetwork::OCV/OCL                             17.855  17.433     1.02   
CRNN::DNNTestNetwork::OCV/OCL_FP16                        17.629  17.277     1.02   
DenseNet_121::DNNTestNetwork::OCV/CPU                     24.735  23.863     1.04   
DenseNet_121::DNNTestNetwork::OCV/OCL                     24.991  25.140     0.99   
DenseNet_121::DNNTestNetwork::OCV/OCL_FP16                24.598  24.019     1.02   
EAST_text_detection::DNNTestNetwork::OCV/CPU              26.929  27.084     0.99   
EAST_text_detection::DNNTestNetwork::OCV/OCL              26.863  27.531     0.98   
EAST_text_detection::DNNTestNetwork::OCV/OCL_FP16         26.884  27.082     0.99   
EfficientDet::DNNTestNetwork::OCV/CPU                     62.667  62.716     1.00   
EfficientDet_int8::DNNTestNetwork::OCV/CPU                34.446  34.796     0.99   
EfficientNet::DNNTestNetwork::OCV/CPU                     11.875  11.786     1.01   
EfficientNet::DNNTestNetwork::OCV/OCL                     12.434  11.806     1.05   
EfficientNet::DNNTestNetwork::OCV/OCL_FP16                11.886  11.899     1.00   
FastNeuralStyle_eccv16::DNNTestNetwork::OCV/CPU           15.104  14.327     1.05   
FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL           15.071  14.670     1.03   
FastNeuralStyle_eccv16::DNNTestNetwork::OCV/OCL_FP16      15.262  14.072     1.08   
GoogLeNet::DNNTestNetwork::OCV/CPU                         5.030   5.031     1.00   
GoogLeNet::DNNTestNetwork::OCV/OCL                         5.075   5.017     1.01   
GoogLeNet::DNNTestNetwork::OCV/OCL_FP16                    5.053   5.055     1.00   
Inception_5h::DNNTestNetwork::OCV/CPU                      6.777   6.430     1.05   
Inception_5h::DNNTestNetwork::OCV/OCL                      7.309   6.683     1.09   
Inception_5h::DNNTestNetwork::OCV/OCL_FP16                 6.624   6.549     1.01   
Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/CPU         86.676  82.918     1.05   
Inception_v2_Faster_RCNN::DNNTestNetwork::OCV/OCL         85.460  84.873     1.01   
Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/CPU      16.007  15.723     1.02   
Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL      15.974  16.015     1.00   
Inception_v2_SSD_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 16.219  15.593     1.04   
MPHand::DNNTestNetwork::OCV/CPU                            3.273   3.313     0.99   
MPHand::DNNTestNetwork::OCV/OCL                            3.466   3.318     1.04   
MPHand::DNNTestNetwork::OCV/OCL_FP16                       3.314   3.258     1.02   
MPPalm::DNNTestNetwork::OCV/CPU                            2.839   1.973     1.44   
MPPalm::DNNTestNetwork::OCV/OCL                            2.773   2.132     1.30   
MPPalm::DNNTestNetwork::OCV/OCL_FP16                       2.783   2.133     1.30   
MPPose::DNNTestNetwork::OCV/CPU                            7.644   8.711     0.88   
MPPose::DNNTestNetwork::OCV/OCL                            7.555   7.210     1.05   
MPPose::DNNTestNetwork::OCV/OCL_FP16                       7.564   7.559     1.00   
MobileNet_SSD_Caffe::DNNTestNetwork::OCV/CPU               9.040   8.813     1.03   
MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL               8.692   9.481     0.92   
MobileNet_SSD_Caffe::DNNTestNetwork::OCV/OCL_FP16          8.792   8.884     0.99   
MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/CPU       7.893   7.735     1.02   
MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL       8.233   7.816     1.05   
MobileNet_SSD_v1_TensorFlow::DNNTestNetwork::OCV/OCL_FP16  7.949   7.817     1.02   
MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/CPU      14.373  15.282     0.94   
MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL      14.218  15.788     0.90   
MobileNet_SSD_v2_TensorFlow::DNNTestNetwork::OCV/OCL_FP16 14.473  14.853     0.97   
MobileNetv2_ONNX::DNNTestNetwork::OCV/CPU                  1.507   1.440     1.05   
MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL                  1.529   1.394     1.10   
MobileNetv2_ONNX::DNNTestNetwork::OCV/OCL_FP16             1.448   1.433     1.01   
PPHumanSeg::DNNTestNetwork::OCV/CPU                        2.302   1.919     1.20   
PPHumanSeg::DNNTestNetwork::OCV/OCL                        2.097   1.865     1.12   
PPHumanSeg::DNNTestNetwork::OCV/OCL_FP16                   2.129   1.908     1.12   
PPOCRv3::DNNTestNetwork::OCV/CPU                          20.612  20.858     0.99   
PPOCRv3::DNNTestNetwork::OCV/OCL                          21.672  21.444     1.01   
PPOCRv3::DNNTestNetwork::OCV/OCL_FP16                     22.660  20.857     1.09   
ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/CPU                 6.443   6.718     0.96   
ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL                 6.842   6.823     1.00   
ResNet50_QDQ_ONNX::DNNTestNetwork::OCV/OCL_FP16            6.895   6.579     1.05   
ResNet_50_v1_ONNX::DNNTestNetwork::OCV/CPU                 7.669   7.629     1.01   
ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL                 7.514   7.473     1.01   
ResNet_50_v1_ONNX::DNNTestNetwork::OCV/OCL_FP16            7.598   7.513     1.01   
SFace::DNNTestNetwork::OCV/CPU                             3.427   3.334     1.03   
SFace::DNNTestNetwork::OCV/OCL                             3.420   3.337     1.02   
SFace::DNNTestNetwork::OCV/OCL_FP16                        3.497   3.375     1.04   
SSD::DNNTestNetwork::OCV/CPU                              83.165  83.476     1.00   
SSD::DNNTestNetwork::OCV/OCL                              83.531  84.121     0.99   
SSD::DNNTestNetwork::OCV/OCL_FP16                         82.447  83.234     0.99   
SqueezeNet_v1_1::DNNTestNetwork::OCV/CPU                   1.334   1.308     1.02   
SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL                   1.279   1.313     0.97   
SqueezeNet_v1_1::DNNTestNetwork::OCV/OCL_FP16              1.429   1.290     1.11   
VIT_Base_Patch16_224::DNNTestNetwork::OCV/CPU             71.211  75.013     0.95   
VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL             71.988  71.218     1.01   
VIT_Base_Patch16_224::DNNTestNetwork::OCV/OCL_FP16        73.700  82.285     0.90   
VitTrack::DNNTestNetwork::OCV/CPU                          4.096   2.091     1.96   
VitTrack::DNNTestNetwork::OCV/OCL                          3.830   2.056     1.86   
VitTrack::DNNTestNetwork::OCV/OCL_FP16                     3.796   2.110     1.80   
YOLOX::DNNTestNetwork::OCV/CPU                            24.346  23.079     1.05   
YOLOX::DNNTestNetwork::OCV/OCL                            24.442  23.459     1.04   
YOLOX::DNNTestNetwork::OCV/OCL_FP16                       23.947  23.327     1.03   
YOLOv3::DNNTestNetwork::OCV/CPU                           59.140  59.140     1.00   
YOLOv3::DNNTestNetwork::OCV/OCL                           46.725  45.292     1.03   
YOLOv3::DNNTestNetwork::OCV/OCL_FP16                      46.939  45.382     1.03   
YOLOv4::DNNTestNetwork::OCV/CPU                           70.062  68.145     1.03   
YOLOv4::DNNTestNetwork::OCV/OCL                           219.570 214.823    1.02   
YOLOv4::DNNTestNetwork::OCV/OCL_FP16                      219.276 213.472    1.03   
YOLOv4_tiny::DNNTestNetwork::OCV/CPU                       8.362   8.221     1.02   
YOLOv4_tiny::DNNTestNetwork::OCV/OCL                       8.373   8.095     1.03   
YOLOv4_tiny::DNNTestNetwork::OCV/OCL_FP16                  8.830   8.088     1.09   
YOLOv5::DNNTestNetwork::OCV/CPU                            9.237   8.484     1.09   
YOLOv5::DNNTestNetwork::OCV/OCL                            8.776   8.537     1.03   
YOLOv5::DNNTestNetwork::OCV/OCL_FP16                       8.747   8.536     1.02   
YOLOv8::DNNTestNetwork::OCV/CPU                           11.019  11.463     0.96   
YOLOv8::DNNTestNetwork::OCV/OCL                           11.204  10.766     1.04   
YOLOv8::DNNTestNetwork::OCV/OCL_FP16                      11.355  10.764     1.05   
YuNet::DNNTestNetwork::OCV/CPU                             3.371   3.466     0.97   
YuNet::DNNTestNetwork::OCV/OCL                             3.396   3.105     1.09   
YuNet::DNNTestNetwork::OCV/OCL_FP16                        3.150   3.130     1.01   
```

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2026-05-06 11:54:40 +03:00

152 lines
4.9 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
// Folds a scalar Mul / Div immediately preceding Softmax into the Softmax
// layer's `scale` parameter. The fused softmax bakes the scale into the
// per-element load before max/exp/sum/divide, removing one full pass over
// the (often large) [B, H, S, S] attention-logits tensor.
//
// Pattern:
// X -> Mul(X, c) -> Softmax => X -> Softmax(scale=c)
// X -> Div(X, c) -> Softmax => X -> Softmax(scale=1/c)
#include "precomp.hpp"
#include "net_impl.hpp"
namespace cv { namespace dnn {
CV__DNN_INLINE_NS_BEGIN
using std::vector;
using std::string;
struct ModelFusionScaleSoftmax
{
explicit ModelFusionScaleSoftmax(Net::Impl* netimpl_) : netimpl(netimpl_) {}
bool extractScalar(Arg a, float& out) const
{
if (!netimpl->isConstArg(a)) return false;
Mat t = netimpl->argTensor(a);
if (t.total() != 1) return false;
if (t.type() == CV_32F) { out = t.ptr<float>()[0]; return true; }
if (t.type() == CV_64F) { out = (float)t.ptr<double>()[0]; return true; }
return false;
}
void fuse() { fuseGraph(netimpl->mainGraph); }
bool fuseGraph(Ptr<Graph>& graph)
{
const vector<Ptr<Layer>>& prog = graph->prog();
size_t nops = prog.size();
bool modified = false;
for (size_t i = 0; i < nops; i++) {
if (!prog[i]) continue;
vector<Ptr<Graph>>* subgraphs = prog[i]->subgraphs();
if (subgraphs) {
for (Ptr<Graph>& g : *subgraphs)
if (fuseGraph(g)) modified = true;
}
}
vector<int> usecounts;
netimpl->useCounts(usecounts);
std::set<int> externalArgs;
for (Arg out : graph->outputs())
externalArgs.insert(out.idx);
std::map<int, int> producer;
for (size_t i = 0; i < nops; i++) {
if (!prog[i]) continue;
for (Arg out : prog[i]->outputs)
producer[out.idx] = (int)i;
}
vector<bool> dropped(nops, false);
for (size_t i = 0; i < nops; i++) {
const Ptr<Layer>& layer = prog[i];
if (!layer || dropped[i]) continue;
SoftmaxLayer* sm = dynamic_cast<SoftmaxLayer*>(layer.get());
if (!sm || sm->logSoftMax) continue;
if (layer->inputs.size() != 1) continue;
Arg sm_in = layer->inputs[0];
auto it = producer.find(sm_in.idx);
if (it == producer.end()) continue;
int prod_idx = it->second;
if (prod_idx < 0 || dropped[prod_idx]) continue;
const Ptr<Layer>& pl = prog[prod_idx];
NaryEltwiseLayer* elt = dynamic_cast<NaryEltwiseLayer*>(pl.get());
if (!elt) continue;
const auto op = elt->op;
const bool is_mul = (op == NaryEltwiseLayer::OPERATION::PROD);
const bool is_div = (op == NaryEltwiseLayer::OPERATION::DIV);
if (!is_mul && !is_div) continue;
if (pl->inputs.size() != 2 || pl->outputs.size() != 1) continue;
// Identify which operand is the scalar constant.
float scalar = 0.f;
int x_slot = -1;
if (extractScalar(pl->inputs[1], scalar)) {
x_slot = 0;
} else if (is_mul && extractScalar(pl->inputs[0], scalar)) {
// Mul is commutative; Div with the data on the right doesn't
// simplify (1 / scale*x ≠ softmax-fold) so only handle Mul here.
x_slot = 1;
} else {
continue;
}
if (is_div) {
if (scalar == 0.f) continue;
scalar = 1.f / scalar;
}
if (!std::isfinite(scalar)) continue;
bool single_consumer = usecounts[sm_in.idx] == 1
&& externalArgs.count(sm_in.idx) == 0;
if (!single_consumer) continue;
sm->scale *= scalar;
layer->inputs[0] = pl->inputs[x_slot];
dropped[prod_idx] = true;
usecounts[sm_in.idx] = 0;
modified = true;
}
if (modified) {
vector<Ptr<Layer>> newprog;
newprog.reserve(nops);
for (size_t i = 0; i < nops; i++) {
if (!dropped[i] && prog[i])
newprog.push_back(prog[i]);
}
graph->setProg(newprog);
}
return modified;
}
Net::Impl* netimpl;
};
void Net::Impl::fuseScaleSoftmax()
{
if (preferableBackend != DNN_BACKEND_OPENCV ||
preferableTarget != DNN_TARGET_CPU)
return;
ModelFusionScaleSoftmax pass(this);
pass.fuse();
}
CV__DNN_INLINE_NS_END
}}