Files
opencv-MIRROR/modules/dnn/perf/perf_layer.cpp
Prasad Ayush Kumar f7ca3a382a Merge pull request #30416 from Prasadayus:dnn-int8-layer-support
support dynamic int8 ConvInteger and MatMulInteger - #30416

Requires: https://github.com/opencv/opencv_extra/pull/1446

Models quantized with ONNX Runtime's dynamic quantization use DynamicQuantizeLinear followed by ConvInteger or MatMulInteger. 5.x has no importer for either op, so none of these models load.

### Models this enables
The int8 ONNX files of these models load and run with this PR:
- Segmentation: [SAM 3 tracker](https://huggingface.co/onnx-community/sam3-tracker-ONNX), [SAM 2.1 hiera-tiny](https://huggingface.co/onnx-community/sam2.1-hiera-tiny-ONNX), [SegFormer-B3 ADE20K](https://huggingface.co/onnx-community/segformer-b3-finetuned-ade-512-512-ONNX)
- Detection: [Grounding DINO tiny](https://huggingface.co/onnx-community/grounding-dino-tiny-ONNX), [OWLv2 base](https://huggingface.co/onnx-community/owlv2-base-patch16-ensemble-ONNX), [RT-DETR R18vd](https://huggingface.co/onnx-community/rtdetr_r18vd), [RF-DETR base](https://huggingface.co/onnx-community/rfdetr_base-ONNX), [YOLO26n](https://huggingface.co/onnx-community/yolo26n-ONNX)
- Depth and features: [Depth Anything V2 small](https://huggingface.co/onnx-community/depth-anything-v2-small), [DINOv2 small](https://huggingface.co/onnx-community/dinov2-small), [ViT-B/16](https://huggingface.co/onnx-community/vit-base-patch16-224-ONNX)
- Text: [TrOCR base printed](https://huggingface.co/onnx-community/trocr-base-printed-ONNX), [BERT base uncased](https://huggingface.co/Xenova/bert-base-uncased)

### Changes
- **ConvInteger** ([opset 10](https://onnx.ai/onnx/operators/onnx__ConvInteger.html)): imported as `Conv2Int8` in a new int32-output mode. The int8 conv kernels (AVX-VNNI, AVX2, NEON, RVV and the scalar path) can now store the int32 accumulators instead of requantizing. Weights and zero points stay inputs, so constant and runtime weights both work. The first commit also fixes the NEON 1x1 fast path, which ignored padding and assumed at least 8 output positions; those cases now take the general path.
- **MatMulInteger** ([opset 10](https://onnx.ai/onnx/operators/onnx__MatMulInteger.html)): imported as `MatMulInt8` in a new int32-output mode, backed by a dedicated int8 GEMM in `int8layers/matmul_int8_kernels` (dispatched for AVX2, AVX_VNNI and NEON_DOTPROD, with a universal-intrinsics baseline). A constant B is packed once, and zero points are applied as a correction term so the inner loop is a plain int8 dot product. B can be N-D with broadcast batch dimensions; b_zero_point can be a scalar, one value per column, or per batch and column. A per-row a_zero_point is rejected, as in ONNX Runtime.
- ONNX conformance tests `test_basic_convinteger`, `test_convinteger_with_padding`, `test_convinteger_without_padding` and `test_matmulinteger` are enabled (skipped for the OpenVINO backend).
### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2026-10-26 18:52:12 +03:00

1514 lines
46 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "perf_precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace opencv_test {
struct Layer_Slice : public TestBaseWithParam<tuple<Backend, Target> >
{
template<int DIMS>
void test_slice(const int* inputShape, const int* begin, const int* end)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat input(DIMS, inputShape, CV_32FC1, Scalar::all(0));
for (int i = 0; i < (int)input.total(); ++i)
input.ptr<float>()[i] = (float)(i & 4095);
std::vector<Range> range(DIMS);
for (int i = 0; i < DIMS; ++i)
range[i] = Range(begin[i], end[i]);
Net net;
LayerParams lp;
lp.type = "Slice";
lp.name = "testLayer";
lp.set("begin", DictValue::arrayInt<int*>((int*)&begin[0], DIMS));
lp.set("end", DictValue::arrayInt<int*>((int*)&end[0], DIMS));
net.addLayerToPrev(lp.name, lp.type, lp);
// warmup
{
net.setInput(input);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
EXPECT_GT(cv::norm(out, NORM_INF), 0);
#if 0
//normAssert(out, input(range));
cout << input(range).clone().reshape(1, 1) << endl;
cout << out.reshape(1, 1) << endl;
#endif
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
static std::set<std::string> nary_eltwise_cuda_deny_ops = {"equal", "greater", "less", "mean", "pow", "sub"};
struct Layer_NaryEltwise : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& a_shape, const std::vector<int>& b_shape, const String op, bool isRef = false)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
if (!isRef && backendId == DNN_BACKEND_CUDA)
{
if (a_shape.size() != b_shape.size())
throw SkipTestException("The test is skipped because inputs with different shape size are not supported.");
for(int i = 0; i < a_shape.size(); i++)
if (a_shape[i] != b_shape[i] && a_shape[i] != 1 && b_shape[i] != 1)
throw SkipTestException("The test is skipped because inputs are not supported.");
if (nary_eltwise_cuda_deny_ops.find(op) != nary_eltwise_cuda_deny_ops.end())
throw SkipTestException("The operator '" + op + "' is skipped because is not support with cuda currently.");
}
Mat a(a_shape, CV_32FC1);
Mat b(b_shape, CV_32FC1);
Scalar mean = 0.f;
Scalar std = 1.f;
randn(a, mean, std);
randn(b, mean, std);
Net net;
LayerParams lp;
if (isRef)
lp.type = "Eltwise";
else
lp.type = "NaryEltwise";
lp.name = "testLayer";
lp.set("operation", op);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 1, id, 1);
// warmup
{
std::vector<String> inpNames(2);
inpNames[0] = "a";
inpNames[1] = "b";
net.setInputsNames(inpNames);
net.setInput(a, inpNames[0]);
net.setInput(b, inpNames[1]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 8;
int C = 256;
int H = 128;
int W = 100;
};
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_add)
{
test_layer({N, C, H, W}, {N, C, H, W}, "add");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_div)
{
test_layer({N, C, H, W}, {N, C, H, W}, "div");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_ref_div)
{
test_layer({N, C, H, W}, {N, C, H, W}, "div", true);
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_equal)
{
test_layer({N, C, H, W}, {N, C, H, W}, "equal");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_greater)
{
test_layer({N, C, H, W}, {N, C, H, W}, "greater");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_less)
{
test_layer({N, C, H, W}, {N, C, H, W}, "less");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_max)
{
test_layer({N, C, H, W}, {N, C, H, W}, "max");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_ref_max)
{
test_layer({N, C, H, W}, {N, C, H, W}, "max", true);
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_mean)
{
test_layer({N, C, H, W}, {N, C, H, W}, "mean");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_min)
{
test_layer({N, C, H, W}, {N, C, H, W}, "min");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_ref_min)
{
test_layer({N, C, H, W}, {N, C, H, W}, "min", true);
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_mul)
{
test_layer({N, C, H, W}, {N, C, H, W}, "mul");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_ref_mul)
{
test_layer({N, C, H, W}, {N, C, H, W}, "prod", true);
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_pow)
{
test_layer({N, C, H, W}, {N, C, H, W}, "pow");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_sub)
{
test_layer({N, C, H, W}, {N, C, H, W}, "sub");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_sum)
{
test_layer({N, C, H, W}, {N, C, H, W}, "sum");
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_NCHW_ref_sum)
{
test_layer({N, C, H, W}, {N, C, H, W}, "sum", true);
}
PERF_TEST_P_(Layer_NaryEltwise, NCHW_C_sum)
{
test_layer({N, C, H, W}, {C, 1, 1}, "sum");
}
PERF_TEST_P_(Layer_NaryEltwise, NHWC_C)
{
test_layer({N, H, W, C}, {1, C}, "sum");
}
PERF_TEST_P_(Layer_NaryEltwise, NHWC_H)
{
test_layer({N, H, W, C}, {1, H, 1, 1}, "sum");
}
PERF_TEST_P_(Layer_Slice, YOLOv4_tiny_1)
{
const int inputShape[4] = {1, 64, 104, 104};
const int begin[] = {0, 32, 0, 0};
const int end[] = {1, 64, 104, 104};
test_slice<4>(inputShape, begin, end);
}
PERF_TEST_P_(Layer_Slice, YOLOv4_tiny_2)
{
const int inputShape[4] = {1, 128, 52, 52};
const int begin[] = {0, 64, 0, 0};
const int end[] = {1, 128, 52, 52};
test_slice<4>(inputShape, begin, end);
}
PERF_TEST_P_(Layer_Slice, YOLOv4_tiny_3)
{
const int inputShape[4] = {1, 256, 26, 26};
const int begin[] = {0, 128, 0, 0};
const int end[] = {1, 256, 26, 26};
test_slice<4>(inputShape, begin, end);
}
PERF_TEST_P_(Layer_Slice, FastNeuralStyle_eccv16)
{
const int inputShape[4] = {1, 128, 80, 100};
const int begin[] = {0, 0, 2, 2};
const int end[] = {1, 128, 76, 96};
test_slice<4>(inputShape, begin, end);
}
using Layer_Scatter = TestBaseWithParam<tuple<std::vector<int>, std::string, int, tuple<Backend, Target>>>;
PERF_TEST_P_(Layer_Scatter, scatter) {
std::vector<int> shape = get<0>(GetParam());
std::string reduction = get<1>(GetParam());
int axis = get<2>(GetParam());
int backend_id = get<0>(get<3>(GetParam()));
int target_id = get<1>(get<3>(GetParam()));
Mat data(shape, CV_32FC1);
Mat indices(shape, CV_64SC1);
Mat updates(shape, CV_32FC1);
randn(data, 0.f, 1.f);
randu(indices, 0, shape[axis]);
randn(updates, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "Scatter";
lp.name = "testLayer";
lp.set("reduction", reduction);
lp.set("axis", axis);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> input_names{"data", "indices", "updates"};
net.setInputsNames(input_names);
net.setInput(data, input_names[0]);
net.setInput(indices, input_names[1]);
net.setInput(updates, input_names[2]);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
Mat out = net.forward();
}
// perf
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, Combine(
Values(std::vector<int>{2, 128, 64, 50}),
Values(std::string("none"), std::string("add")),
Values(0), // use Values(0, 1, 2, 3) for more details
dnnBackendsAndTargets(/* withInferenceEngine= */ false,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ false,
/* withNgraph= */ false,
/* withWebnn= */ false,
/* withCann= */ false) // only test on CPU
));
using Layer_ScatterND = TestBaseWithParam<tuple<std::vector<int>, std::string, tuple<Backend, Target>>>;
PERF_TEST_P_(Layer_ScatterND, scatterND) {
std::vector<int> shape = get<0>(GetParam());
std::string reduction = get<1>(GetParam());
int backend_id = get<0>(get<2>(GetParam()));
int target_id = get<1>(get<2>(GetParam()));
std::vector<int> indices_shape(shape);
indices_shape.push_back(int(shape.size()));
Mat data(shape, CV_32FC1);
Mat indices(indices_shape, CV_32SC1);
Mat updates(shape, CV_32FC1);
randn(data, 0.f, 1.f);
randn(updates, 0.f, 1.f);
// Create indices such that indices[n_i, c_j, h_k, w_l, :4] = [i, j, k, l]
std::vector<int> current_index_tuple(shape.size());
int total = data.total();
std::vector<int> indices_step;
for (int i = 0; i < indices.dims; i++)
{
int step = indices.step.p[i] / sizeof(int32_t);
indices_step.push_back(step);
}
int t, j, idx, offset_at_idx, offset;
auto *indices_ptr = indices.ptr<int32_t>();
for (int i = 0; i < total; i++)
{
t = i;
for (j = shape.size() - 1; j >= 0; j--)
{
idx = t / shape[j];
offset_at_idx = (int)(t - idx * shape[j]);
current_index_tuple[j] = offset_at_idx;
t = idx;
}
offset = 0;
for (j = 0; j < shape.size(); j++)
offset += current_index_tuple[j] * indices_step[j];
for (j = 0; j < shape.size(); j++)
indices_ptr[offset + j] = current_index_tuple[j];
}
Net net;
LayerParams lp;
lp.type = "ScatterND";
lp.name = "testLayer";
lp.set("reduction", reduction);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> input_names{"data", "indices", "updates"};
net.setInputsNames(input_names);
net.setInput(data, input_names[0]);
net.setInput(indices, input_names[1]);
net.setInput(updates, input_names[2]);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, Combine(
Values(std::vector<int>{2, 128, 64, 50}),
Values(std::string("none"), std::string("add")),
dnnBackendsAndTargets(/* withInferenceEngine= */ false,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ false,
/* withNgraph= */ false,
/* withWebnn= */ false,
/* withCann= */ false) // only test on CPU
));
struct Layer_LayerNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(x_shape.back(), 1, CV_32FC1);
Mat b(x_shape.back(), 1, CV_32FC1);
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "LayerNormalization";
lp.name = "testLayer";
lp.set("axis", 2);
lp.set("hasBias", true);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "x";
inpNames[1] = "scale";
inpNames[2] = "b";
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 1;
int H = 50;
int W = 768;
};
PERF_TEST_P_(Layer_LayerNorm, LayerNorm)
{
test_layer({N, H ,W});
}
// Transformer-sized sequence: the small shape above is dominated by threading overhead.
PERF_TEST_P_(Layer_LayerNorm, LayerNorm_Large)
{
test_layer({1, 512, 768});
}
struct Layer_LayerNormExpanded : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(1, x_shape.back(), CV_32FC1); // transpose to pass shape check
Mat b(1, x_shape.back(), CV_32FC1); // transpose to pass shape check
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);
// sub graph structure:
// -> ReduceMean -> -> Pow(2) -> ReduceMean -> Add(epsilon) -> Sqrt ->
// x Sub Div -> Mul(scale) -> Add(bias)
// ---------------> ------------------------------------------------->
Net net;
LayerParams lp_rm;
lp_rm.type = "Reduce";
lp_rm.name = "reducemean1";
lp_rm.set("reduce", "AVE");
std::vector<int> deleteDims(1, x_shape.back());
lp_rm.set("deleted_dims", DictValue::arrayInt(&deleteDims[0], deleteDims.size()));
std::vector<int> targetDims(x_shape.begin(), x_shape.end());
targetDims[x_shape.size() - 1] = 1;
lp_rm.set("target_dims", DictValue::arrayInt(&targetDims[0], targetDims.size()));
int id_rm = net.addLayerToPrev(lp_rm.name, lp_rm.type, lp_rm);
net.connect(0, 0, id_rm, 0);
LayerParams lp_sub;
lp_sub.type = "NaryEltwise";
lp_sub.name = "sub1";
lp_sub.set("operation", "sub");
int id_sub = net.addLayer(lp_sub.name, lp_sub.type, lp_sub);
net.connect(0, 0, id_sub, 0);
net.connect(id_rm, 0, id_sub, 1);
Mat pow_const(1, 1, CV_32FC1);
pow_const.at<float>(0) = 2.f;
LayerParams lp_pow_const;
lp_pow_const.type = "Const";
lp_pow_const.name = "const1";
lp_pow_const.blobs.push_back(pow_const);
int id_pow_const = net.addLayer(lp_pow_const.name, lp_pow_const.type, lp_pow_const);
LayerParams lp_pow;
lp_pow.type = "NaryEltwise";
lp_pow.name = "pow1";
lp_pow.set("operation", "pow");
int id_pow = net.addLayer(lp_pow.name, lp_pow.type, lp_pow);
net.connect(id_sub, 0, id_pow, 0);
net.connect(id_pow_const, 0, id_pow, 1);
LayerParams lp_rm1;
lp_rm1.type = "Reduce";
lp_rm1.name = "reducemean2";
lp_rm1.set("reduce", "AVE");
lp_rm1.set("deleted_dims", DictValue::arrayInt(&deleteDims[0], deleteDims.size()));
lp_rm1.set("target_dims", DictValue::arrayInt(&targetDims[0], targetDims.size()));
int id_rm1 = net.addLayer(lp_rm1.name, lp_rm1.type, lp_rm1);
net.connect(id_pow, 0, id_rm1, 0);
Mat add_const(1, 1, CV_32F);
add_const.at<float>(0) = 1e-5;
LayerParams lp_add_const;
lp_add_const.type = "Const";
lp_add_const.name = "const2";
lp_add_const.blobs.push_back(add_const);
int id_add_const = net.addLayer(lp_add_const.name, lp_add_const.type, lp_add_const);
LayerParams lp_add;
lp_add.type = "NaryEltwise";
lp_add.name = "add1";
lp_add.set("operation", "add");
int id_add = net.addLayer(lp_add.name, lp_add.type, lp_add);
net.connect(id_rm1, 0, id_add, 0);
net.connect(id_add_const, 0, id_add, 1);
LayerParams lp_sqrt;
lp_sqrt.type = "Sqrt";
lp_sqrt.name = "sqrt1";
int id_sqrt = net.addLayer(lp_sqrt.name, lp_sqrt.type, lp_sqrt);
net.connect(id_add, 0, id_sqrt, 0);
LayerParams lp_div;
lp_div.type = "NaryEltwise";
lp_div.name = "div1";
lp_div.set("operation", "div");
int id_div = net.addLayer(lp_div.name, lp_div.type, lp_div);
net.connect(id_sub, 0, id_div, 0);
net.connect(id_sqrt, 0, id_div, 1);
LayerParams lp_mul;
lp_mul.type = "NaryEltwise";
lp_mul.name = "mul1";
lp_mul.set("operation", "mul");
int id_mul = net.addLayer(lp_mul.name, lp_mul.type, lp_mul);
net.connect(id_div, 0, id_mul, 0);
net.connect(0, 1, id_mul, 1);
LayerParams lp_add1;
lp_add1.type = "NaryEltwise";
lp_add1.name = "add2";
lp_add1.set("operation", "add");
int id_add1 = net.addLayer(lp_add1.name, lp_add1.type, lp_add1);
net.connect(id_mul, 0, id_add1, 0);
net.connect(0, 2, id_add1, 1);
// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "x";
inpNames[1] = "scale";
inpNames[2] = "b";
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 1;
int H = 50;
int W = 768;
};
PERF_TEST_P_(Layer_LayerNormExpanded, DISABLED_LayerNormExpanded)
{
test_layer({N, H ,W});
}
struct Layer_GatherElements : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& data_shape, const std::vector<int>& indices_shape, int axis = 0)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat data(data_shape, CV_32FC1);
Mat indices(indices_shape, CV_64SC1);
randu(data, 0.f, 1.f);
randu(indices, 0, data_shape[axis]);
Net net;
LayerParams lp;
lp.type = "GatherElements";
lp.name = "testLayer";
lp.set("axis", axis);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "data";
inpNames[1] = "indices";
net.setInputsNames(inpNames);
net.setInput(data, inpNames[0]);
net.setInput(indices, inpNames[1]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_GatherElements, GatherElements)
{
test_layer({2700, 1, 2914}, {2700, 1, 81}, 2);
}
struct Layer_InstanceNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(x_shape[1], 1, CV_32FC1);
Mat b(x_shape[1], 1, CV_32FC1);
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "InstanceNormalization";
lp.name = "testLayer";
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> inpNames{"x", "scale", "b"};
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 2;
int C = 64;
int H = 180;
int W = 240;
};
PERF_TEST_P_(Layer_InstanceNorm, InstanceNorm)
{
test_layer({N, C, H, W});
}
struct Layer_Attention : public TestBaseWithParam<tuple<Backend, Target>> {
void test_layer(const std::vector<int> x_shape, const std::vector<int> qkv_hidden_sizes, const int num_heads,
bool const_weights = false) {
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
auto qk_hidden_size = qkv_hidden_sizes[0];
auto v_hidden_size = qkv_hidden_sizes[2];
auto input_hidden_size = x_shape[2];
auto hidden_size = qk_hidden_size + qk_hidden_size + v_hidden_size;
Mat x(x_shape, CV_32F);
Mat weight(std::vector<int>{input_hidden_size, hidden_size}, CV_32F);
Mat bias(std::vector<int>{hidden_size}, CV_32F);
randu(x, 0.f, 1.f);
randu(weight, 0.f, 1.f);
randu(bias, 0.f, 1.f);
LayerParams lp;
lp.type = "Attention";
lp.name = "testLayer";
lp.set("num_heads", num_heads);
lp.set("qkv_hidden_sizes", DictValue::arrayInt(qkv_hidden_sizes.data(), qkv_hidden_sizes.size()));
Net net;
if (const_weights) {
// Weights as blobs: packed once, then every forward() takes the prepacked path.
lp.blobs.push_back(weight);
lp.blobs.push_back(bias);
net.addLayerToPrev(lp.name, lp.type, lp);
net.setInput(x);
} else {
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
std::vector<std::string> input_names{"x", "weight", "bias"};
net.setInputsNames(input_names);
net.setInput(x, input_names[0]);
net.setInput(weight, input_names[1]);
net.setInput(bias, input_names[2]);
}
{
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat out = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_Attention, VisionTransformer) {
test_layer({1, 197, 768}, {768, 768, 768}, 12);
}
PERF_TEST_P_(Layer_Attention, BERT_ConstWeights) {
test_layer({1, 128, 768}, {768, 768, 768}, 12, /*const_weights=*/true);
}
PERF_TEST_P_(Layer_Attention, LongSequence_257) {
test_layer({1, 257, 192}, {192, 192, 192}, 12, /*const_weights=*/true);
}
PERF_TEST_P_(Layer_Attention, LongSequence_1024) {
test_layer({1, 1024, 192}, {192, 192, 192}, 12, /*const_weights=*/true);
}
PERF_TEST_P_(Layer_Attention, LongSequence_4096) {
test_layer({1, 4096, 192}, {192, 192, 192}, 12, /*const_weights=*/true);
}
// GroupQueryAttention runs on AttentionOnnxAi; these flags are what the ONNX importer sets.
struct Layer_GroupQueryAttention : public TestBaseWithParam<tuple<Backend, Target>> {
void test_layer(int B, int S, int Sp, int num_heads, int kv_num_heads, int D) {
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
int Skv = Sp + S;
Mat query(std::vector<int>{B, S, num_heads * D}, CV_32F);
Mat key(std::vector<int>{B, S, kv_num_heads * D}, CV_32F);
Mat value(std::vector<int>{B, S, kv_num_heads * D}, CV_32F);
Mat pastKey(std::vector<int>{B, kv_num_heads, Sp, D}, CV_32F);
Mat pastValue(std::vector<int>{B, kv_num_heads, Sp, D}, CV_32F);
Mat seqlensK(std::vector<int>{B}, CV_32S, Scalar(Skv - 1));
randu(query, 0.f, 1.f);
randu(key, 0.f, 1.f);
randu(value, 0.f, 1.f);
randu(pastKey, 0.f, 1.f);
randu(pastValue, 0.f, 1.f);
LayerParams lp;
lp.type = "AttentionOnnxAi";
lp.name = "testLayer";
lp.set("q_num_heads", num_heads);
lp.set("kv_num_heads", kv_num_heads);
lp.set("is_causal", true);
lp.set("has_attn_mask", 0);
lp.set("has_past", 1);
lp.set("has_seqlens_k", 1);
lp.set("has_rotary_cache", 0);
Net net;
int id = net.addLayerToPrev(lp.name, lp.type, lp);
for (int i = 0; i < 6; ++i)
net.connect(0, i, id, i);
std::vector<std::string> input_names{
"query", "key", "value", "past_key", "past_value", "seqlens_k"};
net.setInputsNames(input_names);
net.setInput(query, input_names[0]);
net.setInput(key, input_names[1]);
net.setInput(value, input_names[2]);
net.setInput(pastKey, input_names[3]);
net.setInput(pastValue, input_names[4]);
net.setInput(seqlensK, input_names[5]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_GroupQueryAttention, MHA_ShortCache) {
test_layer(/*B*/1, /*S*/512, /*Sp*/1, /*num_heads*/32, /*kv_num_heads*/32, /*D*/128);
}
PERF_TEST_P_(Layer_GroupQueryAttention, Grouped_WithCache) {
test_layer(/*B*/1, /*S*/1, /*Sp*/2048, /*num_heads*/32, /*kv_num_heads*/8, /*D*/128);
}
struct Layer_AttentionOnnxAi : public TestBaseWithParam<int>
{
void decode_step(const std::string& layout, int nq, int nkv)
{
int past = GetParam();
bool is3d = layout == "3d";
std::string model = "dnn/onnx/models/test_attention_kv_cache_" + layout + ".onnx";
Net net = readNetFromONNX(findDataFile(model, true), ENGINE_OPENCV);
auto qkv = [&](int n, int t) {
return is3d ? std::vector<int>{1, t, n * D} : std::vector<int>{1, n, t, D};
};
Mat qp(qkv(nq, past), CV_32F), kp(qkv(nkv, past), CV_32F), vp(qkv(nkv, past), CV_32F);
Mat qd(qkv(nq, 1), CV_32F), kd(qkv(nkv, 1), CV_32F), vd(qkv(nkv, 1), CV_32F);
for (Mat* m : {&qp, &kp, &vp, &qd, &kd, &vd})
randu(*m, -1.f, 1.f);
std::vector<int> mp_sz{1, nq, past, past}, md_sz{1, nq, 1, past + 1};
Mat mp(mp_sz, CV_32S, Scalar(1)), md(md_sz, CV_32S, Scalar(1));
net.enableKVCache();
while (next())
{
// Reset, reserve and prefill stay outside the timer, so the measurement is one
// decode step against `past` cached tokens with the page pool already allocated.
net.resetKVCache();
net.reserveKVCache(past + 1);
net.setInput(qp, "Q");
net.setInput(kp, "K");
net.setInput(vp, "V");
net.setInput(mp, "Mask");
net.forward();
net.setInput(qd, "Q");
net.setInput(kd, "K");
net.setInput(vd, "V");
net.setInput(md, "Mask");
startTimer();
net.forward();
stopTimer();
}
SANITY_CHECK_NOTHING();
}
const int D = 256;
};
// 3D takes its head counts from the model attributes (q_num_heads=8, kv_num_heads=4);
// 4D takes them from the [batch, heads, seq, head_dim] layout, so both ratios are drivable.
PERF_TEST_P_(Layer_AttentionOnnxAi, decode_gqa_3d) { decode_step("3d", 8, 4); }
PERF_TEST_P_(Layer_AttentionOnnxAi, decode_gqa_4d) { decode_step("4d", 8, 2); }
PERF_TEST_P_(Layer_AttentionOnnxAi, decode_mha_4d) { decode_step("4d", 8, 8); }
struct Layer_GroupNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape, int num_groups)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(x_shape[1], 1, CV_32FC1);
Mat b(x_shape[1], 1, CV_32FC1);
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "GroupNormalization";
lp.name = "testLayer";
lp.set("num_groups", num_groups);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);
// warmup
{
std::vector<String> inpNames{"x", "scale", "b"};
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 2;
int C = 64;
int H = 180;
int W = 240;
int num_groups = 16;
};
PERF_TEST_P_(Layer_GroupNorm, GroupNorm)
{
test_layer({N, C, H, W}, num_groups);
}
struct Layer_MVN : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape, bool across_channels, bool normalize_variance)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
randu(x, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "MVN";
lp.name = "testLayer";
lp.set("across_channels", across_channels);
lp.set("normalize_variance", normalize_variance);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
// warmup
{
std::vector<String> inpNames{"x"};
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 2;
int C = 64;
int H = 180;
int W = 240;
};
PERF_TEST_P_(Layer_MVN, MVN)
{
test_layer({N, C, H, W}, false, true);
}
struct Layer_RMSNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat x(x_shape, CV_32FC1);
Mat scale(std::vector<int>{x_shape.back()}, CV_32FC1);
randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
Net net;
LayerParams lp;
lp.type = "RMSNormalization";
lp.name = "testLayer";
lp.set("axis", -1);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
// warmup
{
std::vector<String> inpNames{"x", "scale"};
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 1;
int H = 50;
int W = 768;
};
PERF_TEST_P_(Layer_RMSNorm, RMSNorm)
{
test_layer({N, H, W});
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(false, false));
INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
#ifdef HAVE_CUDA
INSTANTIATE_TEST_CASE_P(CUDA, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_CUDA, DNN_TARGET_CUDA)));
#endif
#ifdef HAVE_VULKAN
INSTANTIATE_TEST_CASE_P(VULKAN, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_VKCOM, DNN_TARGET_VULKAN)));
#endif
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_InstanceNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_Attention, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_GroupQueryAttention, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_AttentionOnnxAi, testing::Values(1, 64, 512));
INSTANTIATE_TEST_CASE_P(/**/, Layer_GroupNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_MVN, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_RMSNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
typedef TestBaseWithParam<tuple<Vec4i, int, bool, tuple<Backend, Target> > > Layer_FullyConnected;
PERF_TEST_P_(Layer_FullyConnected, fc)
{
std::vector<int> inpShape;
inpShape.reserve(4);
for (int i = 0; i < 4; ++i) {
int dim = get<0>(GetParam())[i];
if (dim == 0)
break;
inpShape.push_back(dim);
}
Mat input(inpShape, CV_32F);
randn(input, 0, 1);
int axis = input.dims - 1;
int outDims = get<1>(GetParam());
bool isMatMul = get<2>(GetParam());
int backendId = get<0>(get<3>(GetParam()));
int targetId = get<1>(get<3>(GetParam()));
if (inpShape.size() == 4 && inpShape[0] == 5 && inpShape[1] == 16 && inpShape[2] == 512 && inpShape[3] == 128 && outDims >= 512)
applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);
std::vector<int> weightShape;
if (isMatMul) {
weightShape = inpShape;
weightShape[weightShape.size() - 2] = outDims;
} else {
weightShape = {outDims, (int)input.total(axis, input.dims)};
}
Mat weights(weightShape, CV_32F);
randn(weights, 0, 1);
LayerParams lp;
lp.set("axis", input.dims - 1);
lp.set("is_matmul", weights.dims > 2);
lp.set("bias_term", false);
lp.set("num_output", (int)weights.total(0, weights.dims - 1));
lp.blobs.resize(1, weights);
Net net;
net.addLayerToPrev("matmul", "InnerProduct", lp);
net.setInput(input);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
// warmup
Mat output = net.forward();
TEST_CYCLE()
{
net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_FullyConnected, Combine(
Values( // input size
Vec4i(5, 512, 384),
Vec4i(5, 16, 512, 128)
),
Values(256, 512, 1024), // output dimension
testing::Bool(), // is_matmul
dnnBackendsAndTargets()
));
typedef TestBaseWithParam<tuple<std::vector<int>, int, tuple<Backend, Target> > > Layer_Softmax;
PERF_TEST_P_(Layer_Softmax, softmax_3d) {
std::vector<int> shape = get<0>(GetParam());
int axis = get<1>(GetParam());
int backendId = get<0>(get<2>(GetParam()));
int targetId = get<1>(get<2>(GetParam()));
Mat data(shape, CV_32FC1);
Scalar mean = 0.f;
Scalar std = 1.f;
randn(data, mean, std);
Net net;
LayerParams lp;
lp.type = "Softmax";
lp.name = "testLayer";
lp.set("axis", axis);
net.addLayerToPrev(lp.name, lp.type, lp);
// warmup
{
net.setInput(data);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE() {
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Softmax, Combine(
Values( // input size
std::vector<int>({16, 50, 50}),
std::vector<int>({16, 197, 197}),
std::vector<int>({16, 1024, 1024})
),
Values(0, 1, 2), // axis
dnnBackendsAndTargets(/* withInferenceEngine= */ false,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ false,
/* withNgraph= */ false,
/* withWebnn= */ false,
/* withCann= */ false) // only test on CPU
));
struct Layer_Elementwise : public TestBaseWithParam<tuple<Backend, Target>> {
void test_layer(const std::string &op_type, const std::vector<int> &input_shape) {
int backend_id = get<0>(GetParam());
int target_id = get<1>(GetParam());
Mat input(input_shape, CV_32F);
randu(input, -10.0f, 10.f);
LayerParams lp;
lp.type = op_type;
lp.name = cv::format("PerfLayer/%s", op_type.c_str());
Net net;
net.addLayerToPrev(lp.name, lp.type, lp);
// Warmup
{
net.setInput(input);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
net.forward();
}
TEST_CYCLE() {
net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 2;
int C = 32;
int H = 416;
int W = 416;
};
PERF_TEST_P_(Layer_Elementwise, Gelu) {
test_layer("Gelu", std::vector<int>{1, 50, 3072});
}
PERF_TEST_P_(Layer_Elementwise, Swish) {
test_layer("Swish", std::vector<int>{N, C, H, W});
}
PERF_TEST_P_(Layer_Elementwise, Mish) {
test_layer("Mish", std::vector<int>{N, C, H, W});
}
PERF_TEST_P_(Layer_Elementwise, Elu) {
test_layer("ELU", std::vector<int>{N, C, H, W});
}
PERF_TEST_P_(Layer_Elementwise, Celu) {
test_layer("Celu", std::vector<int>{N, C, H, W});
}
PERF_TEST_P_(Layer_Elementwise, Selu) {
test_layer("Selu", std::vector<int>{N, C, H, W});
}
PERF_TEST_P_(Layer_Elementwise, HardSwish) {
test_layer("HardSwish", std::vector<int>{N, C, H, W});
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Elementwise,
dnnBackendsAndTargets(/* withInferenceEngine= */ true,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ true,
/* withNgraph= */ true,
/* withWebnn= */ false,
/* withCann= */ false));
struct Layer_Sign : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(int matType, const std::vector<int>& input_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
if (backendId == DNN_BACKEND_CUDA && matType != CV_32F)
throw SkipTestException("The CUDA SignOp only supports floating point tensors.");
// OpenVINO narrows custom-layer outputs from int64 to int32; test_int.cpp skips
// the same CV_64S/NGRAPH pairing.
if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && matType == CV_64S)
throw SkipTestException("OpenVINO narrows int64 to int32 for custom layers.");
Mat input(input_shape, matType);
randu(input, -100, 100);
LayerParams lp;
lp.type = "Sign";
lp.name = "PerfLayer/Sign";
Net net;
net.addLayerToPrev(lp.name, lp.type, lp);
{
net.setInput(input);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
net.forward();
}
TEST_CYCLE()
{
net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_Sign, Float) {
test_layer(CV_32F, {2, 32, 416, 416});
}
PERF_TEST_P_(Layer_Sign, Int32) {
test_layer(CV_32S, {2, 32, 416, 416});
}
PERF_TEST_P_(Layer_Sign, Int64) {
test_layer(CV_64S, {2, 32, 416, 416});
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Sign, dnnBackendsAndTargets());
struct Layer_TopK : public TestBaseWithParam<tuple<Backend, Target>> {
void test_layer(const std::vector<int> &input_shape, const int K, const int axis) {
int backend_id = get<0>(GetParam());
int target_id = get<1>(GetParam());
Mat input_data(input_shape, CV_32F);
randn(input_data, -1.f, 1.f);
Net net;
LayerParams lp;
lp.type = "TopK";
lp.name = "testLayer";
lp.set("k", K);
lp.set("axis", axis);
net.addLayerToPrev(lp.name, lp.type, lp);
// Warmup
{
net.setInput(input_data);
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
net.forward();
}
TEST_CYCLE() {
net.forward();
}
SANITY_CHECK_NOTHING();
}
std::vector<int> input_shape_2d{1000, 100};
std::vector<int> input_shape_3d{100, 100, 100};
};
PERF_TEST_P_(Layer_TopK, TopK_2D_Axis0) {
test_layer(input_shape_2d, input_shape_2d[0] / 2, 0);
}
PERF_TEST_P_(Layer_TopK, TopK_2D_Axis0_K5) {
test_layer(input_shape_2d, 5, 0);
}
PERF_TEST_P_(Layer_TopK, TopK_2D_Axis1) {
test_layer(input_shape_2d, input_shape_2d[1] / 2, 1);
}
PERF_TEST_P_(Layer_TopK, TopK_3D_Axis0) {
test_layer(input_shape_3d, input_shape_3d[0] / 2, 0);
}
PERF_TEST_P_(Layer_TopK, TopK_3D_Axis1) {
test_layer(input_shape_3d, input_shape_3d[1] / 2, 1);
}
PERF_TEST_P_(Layer_TopK, TopK_3D_Axis2) {
test_layer(input_shape_3d, input_shape_3d[2] / 2, 2);
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_TopK,
dnnBackendsAndTargets(/* withInferenceEngine= */ false,
/* withHalide= */ false,
/* withCpuOCV= */ true,
/* withVkCom= */ false,
/* withCUDA= */ false,
/* withNgraph= */ false,
/* withWebnn= */ false,
/* withCann= */ false));
// Single-input elementwise activation throughput (the transcendental functors
// vectorized in elementwise_layers.cpp via the activation_kernels registry).
// Inputs are drawn from each function's valid domain so the output stays finite.
struct Layer_Activation : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_activation(const String& type, float lo, float hi)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat input({N, C, H, W}, CV_32F);
randu(input, lo, hi);
Net net;
LayerParams lp;
lp.type = type;
lp.name = "testLayer";
net.addLayerToPrev(lp.name, lp.type, lp);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
net.setInput(input);
Mat out = net.forward(); // warmup
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
int N = 8;
int C = 256;
int H = 128;
int W = 100;
};
PERF_TEST_P_(Layer_Activation, Log) { test_activation("Log", 0.1f, 10.f); }
PERF_TEST_P_(Layer_Activation, Erf) { test_activation("Erf", -3.f, 3.f); }
PERF_TEST_P_(Layer_Activation, Exp) { test_activation("Exp", -5.f, 5.f); }
PERF_TEST_P_(Layer_Activation, Sin) { test_activation("Sin", -3.f, 3.f); }
PERF_TEST_P_(Layer_Activation, Cos) { test_activation("Cos", -3.f, 3.f); }
PERF_TEST_P_(Layer_Activation, Sinh) { test_activation("Sinh", -5.f, 5.f); }
PERF_TEST_P_(Layer_Activation, Cosh) { test_activation("Cosh", -5.f, 5.f); }
PERF_TEST_P_(Layer_Activation, Tan) { test_activation("Tan", -1.4f, 1.4f); }
PERF_TEST_P_(Layer_Activation, Softplus) { test_activation("Softplus", -5.f, 5.f); }
PERF_TEST_P_(Layer_Activation, BNLL) { test_activation("BNLL", -5.f, 5.f); }
PERF_TEST_P_(Layer_Activation, Asinh) { test_activation("Asinh", -10.f, 10.f); }
PERF_TEST_P_(Layer_Activation, Acosh) { test_activation("Acosh", 1.f, 20.f); }
PERF_TEST_P_(Layer_Activation, Atanh) { test_activation("Atanh", -0.95f, 0.95f); }
INSTANTIATE_TEST_CASE_P(/**/, Layer_Activation, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
struct Layer_DynamicInt8 : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_model(const std::string& name, const std::vector<int>& inp_shape)
{
std::string model = findDataFile("dnn/onnx/models/" + name + ".onnx", true);
Net net = readNetFromONNX(model, ENGINE_OPENCV);
net.setPreferableBackend(get<0>(GetParam()));
net.setPreferableTarget(get<1>(GetParam()));
Mat inp(inp_shape, CV_32F);
randu(inp, -1.f, 1.f);
net.setInput(inp);
net.forward();
TEST_CYCLE()
{
net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_DynamicInt8, ConvInteger)
{
test_model("convinteger_dynamic_block", {1, 3, 256, 256});
}
PERF_TEST_P_(Layer_DynamicInt8, MatMulInteger)
{
test_model("matmulinteger_dynamic_block", {1, 256, 256});
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_DynamicInt8, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
} // namespace