diff --git a/modules/dnn/src/layers/recurrent2_layers.cpp b/modules/dnn/src/layers/recurrent2_layers.cpp index 6eb149fe22..124c53c11a 100644 --- a/modules/dnn/src/layers/recurrent2_layers.cpp +++ b/modules/dnn/src/layers/recurrent2_layers.cpp @@ -491,7 +491,18 @@ class LSTM2LayerImpl CV_FINAL : public LSTM2Layer // seq-major cell-state scratch: (seq, batch, dirs, hid), matching the recurrence. int cOutShape[] = {seqLenth, batchSize, numDirs, numHidden}; Mat cOut = produceCellOutput ? Mat::zeros(4, cOutShape, output[0].type()) : Mat(); - Mat xTs = input[0].reshape(1, batchSizeTotal); + + // the recurrence below slices X by timestep, so it needs the seq-major order; + // under ONNX layout=1 the input arrives as (batch, seq, ...) + Mat xSeqFirst = input[0]; + if (layout == BATCH_SEQ_HID) + { + std::vector perm(input[0].dims); + std::iota(perm.begin(), perm.end(), 0); + std::swap(perm[0], perm[1]); + cv::transposeND(input[0], perm, xSeqFirst); + } + Mat xTs = xSeqFirst.reshape(1, batchSizeTotal); // seq-major Y assembly buffer; transposed into output[0] below. // Never reallocate output[0]'s header or it detaches from the graph. diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index b30e07ba22..238374cad5 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -115,7 +115,7 @@ public: Net net; }; -TEST_P(DNNTestNetwork, DISABLED_YOLOv8n) { +TEST_P(DNNTestNetwork, YOLOv8n) { processNet("dnn/onnx/models/yolov8n.onnx", "", Size(640, 640), "output0"); expectNoFallbacksFromIE(net); expectNoFallbacksFromCUDA(net); diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 3638f22400..f9898f3c7c 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -52,37 +52,6 @@ static std::string _tf(TString filename) return findDataFile(std::string("dnn/") + filename); } -class Test_Caffe_nets : public DNNTestLayer -{ -public: - void testFaster(const std::string& proto, const std::string& model, const Mat& ref, - double scoreDiff = 0.0, double iouDiff = 0.0) - { - checkBackend(); - Net net = readNet(findDataFile("dnn/" + proto), - findDataFile("dnn/" + model, false)); - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - if (target == DNN_TARGET_CPU_FP16) - net.enableWinograd(false); - - Mat img = imread(findDataFile("dnn/dog416.png")); - resize(img, img, Size(800, 600)); - Mat blob = blobFromImage(img, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); - Mat imInfo = (Mat_(1, 3) << img.rows, img.cols, 1.6f); - - net.setInput(blob); - net.setInput(imInfo, "im_info"); - // Output has shape 1x1xNx7 where N - number of detections. - // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom] - Mat out = net.forward(); - scoreDiff = scoreDiff ? scoreDiff : default_l1; - iouDiff = iouDiff ? iouDiff : default_lInf; - normAssertDetections(ref, out, ("model name: " + model).c_str(), 0.8, scoreDiff, iouDiff); - } -}; - TEST(Reproducibility_SSD, Accuracy) { applyTestTag( @@ -137,6 +106,4 @@ TEST(Test_Caffe, multiple_inputs) normAssert(out, first_image + second_image); } -INSTANTIATE_TEST_CASE_P(/**/, Test_Caffe_nets, dnnBackendsAndTargets()); - }} // namespace diff --git a/modules/dnn/test/test_graph_simplifier.cpp b/modules/dnn/test/test_graph_simplifier.cpp index 99814d970d..04fee7fa49 100644 --- a/modules/dnn/test/test_graph_simplifier.cpp +++ b/modules/dnn/test/test_graph_simplifier.cpp @@ -26,8 +26,9 @@ class Test_Graph_Simplifier : public ::testing::Test { std::vector layers; net.getLayerTypes(layers); - // remove Const, Identity (output layer), __NetInputLayer__ (input layer) - layers.erase(std::remove_if(layers.begin(), layers.end(), [] (const std::string l) { return l == "Const" || l == "Identity" || l == "__NetInputLayer__"; }), layers.end()); + // remove Const, Identity (output layer), __NetInputLayer__ (input layer), + // TransformLayout (inserted by the block layout pass) + layers.erase(std::remove_if(layers.begin(), layers.end(), [] (const std::string l) { return l == "Const" || l == "Identity" || l == "__NetInputLayer__" || l == "TransformLayout"; }), layers.end()); // Instead of 'Tile', 'Expand' etc. we may now have 'Tile2', 'Expand2' etc. // We should correctly match them with the respective patterns for (auto& l: layers) { @@ -57,19 +58,18 @@ TEST_F(Test_Graph_Simplifier, LayerNormNoFusionSubGraph) { test("layer_norm_no_fusion", std::vector{"NaryEltwise", "Reduce", "Sqrt"}); } -TEST_F(Test_Graph_Simplifier, DISABLED_ResizeSubgraph) { - /* Test for 6 subgraphs: - - GatherCastSubgraph - - MulCastSubgraph +TEST_F(Test_Graph_Simplifier, ResizeSubgraph) { + /* Test for 4 subgraphs: - UpsampleSubgraph - ResizeSubgraph1 - ResizeSubgraph2 - ResizeSubgraph3 */ - test("upsample_unfused_torch1.2", std::vector{"BatchNorm", "Resize"}); - test("resize_nearest_unfused_opset11_torch1.3", std::vector{"BatchNorm", "Convolution", "Resize"}); - test("resize_nearest_unfused_opset11_torch1.4", std::vector{"BatchNorm", "Convolution", "Resize"}); - test("upsample_unfused_opset9_torch1.4", std::vector{"BatchNorm", "Convolution", "Resize"}); + test("upsample_unfused_torch1.2", std::vector{"BatchNorm", "Cast", "Concat", "Floor", "Gather", "NaryEltwise", "Resize", "Shape", "Slice", "Unsqueeze"}); + // In the models below the BatchNorm is folded into the preceding convolution by fuseBN(). + test("resize_nearest_unfused_opset11_torch1.3", std::vector{"Cast", "Concat", "Conv", "Floor", "Gather", "NaryEltwise", "Resize", "Shape", "Unsqueeze"}); + test("resize_nearest_unfused_opset11_torch1.4", std::vector{"Cast", "Concat", "Conv", "Floor", "Gather", "NaryEltwise", "Resize", "Shape", "Slice", "Unsqueeze"}); + test("upsample_unfused_opset9_torch1.4", std::vector{"Cast", "Concat", "Conv", "Floor", "Gather", "NaryEltwise", "Resize", "Shape", "Slice", "Unsqueeze"}); test("two_resizes_with_shared_subgraphs", std::vector{"NaryEltwise", "Resize"}); } diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp deleted file mode 100644 index 8eee0d5ccc..0000000000 --- a/modules/dnn/test/test_int8_layers.cpp +++ /dev/null @@ -1,1118 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. - -// The tests are disabled, because on-fly quantization was removed in https://github.com/opencv/opencv/pull/24980 -// To be restored, when test models are quantized outsize of OpenCV -#if 0 - -#include "test_precomp.hpp" -#include "npy_blob.hpp" -#include -#include -namespace opencv_test { namespace { - -testing::internal::ParamGenerator< tuple > dnnBackendsAndTargetsInt8() -{ - std::vector< tuple > targets; - targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)); -#ifdef HAVE_TIMVX - targets.push_back(make_tuple(DNN_BACKEND_TIMVX, DNN_TARGET_NPU)); -#endif -#ifdef HAVE_INF_ENGINE - targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, DNN_TARGET_CPU)); -#endif - return testing::ValuesIn(targets); -} - -template -static std::string _tf(TString filename) -{ - return (getOpenCVExtraDir() + "dnn/") + filename; -} - -class Test_Int8_layers : public DNNTestLayer -{ -public: - void testLayer(const String& basename, const String& importer, double l1, double lInf, - int numInps = 1, int numOuts = 1, bool useCaffeModel = false, - bool useCommonInputBlob = true, bool hasText = false, bool perChannel = true) - { - CV_Assert_N(numInps >= 1, numInps <= 10, numOuts >= 1, numOuts <= 10); - std::vector inps(numInps), inps_int8(numInps); - std::vector refs(numOuts), outs_int8(numOuts), outs_dequantized(numOuts); - std::vector inputScale, outputScale; - std::vector inputZp, outputZp; - String inpPath, outPath; - Net net, qnet; - - if (importer == "TensorFlow") - { - String netPath = _tf("tensorflow/" + basename + "_net.pb"); - String netConfig = hasText ? _tf("tensorflow/" + basename + "_net.pbtxt") : ""; - net = readNetFromTensorflow(netPath, netConfig); - - inpPath = _tf("tensorflow/" + basename + "_in"); - outPath = _tf("tensorflow/" + basename + "_out"); - } - else if (importer == "ONNX") - { - String onnxmodel = _tf("onnx/models/" + basename + ".onnx"); - net = readNetFromONNX(onnxmodel); - - inpPath = _tf("onnx/data/input_" + basename); - outPath = _tf("onnx/data/output_" + basename); - } - ASSERT_FALSE(net.empty()); - - for (int i = 0; i < numInps; i++) - inps[i] = blobFromNPY(inpPath + ((numInps > 1) ? cv::format("_%d.npy", i) : ".npy")); - - for (int i = 0; i < numOuts; i++) - refs[i] = blobFromNPY(outPath + ((numOuts > 1) ? cv::format("_%d.npy", i) : ".npy")); - - qnet = net.quantize(inps, CV_8S, CV_8S, perChannel); - qnet.getInputDetails(inputScale, inputZp); - qnet.getOutputDetails(outputScale, outputZp); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - - // Quantize inputs to int8 - // int8_value = float_value/scale + zero-point - for (int i = 0; i < numInps; i++) - { - inps[i].convertTo(inps_int8[i], CV_8S, 1.f/inputScale[i], inputZp[i]); - String inp_name = numInps > 1 ? (importer == "Caffe" ? cv::format("input_%d", i) : cv::format("%d", i)) : ""; - qnet.setInput(inps_int8[i], inp_name); - } - qnet.forward(outs_int8); - - // Dequantize outputs and compare with reference outputs - // float_value = scale*(int8_value - zero-point) - for (int i = 0; i < numOuts; i++) - { - outs_int8[i].convertTo(outs_dequantized[i], CV_32F, outputScale[i], -(outputScale[i] * outputZp[i])); - Mat out_i = outs_dequantized[i], ref_i = refs[i]; - if (out_i.dims == 2 && ref_i.dims == 1) { - ref_i = ref_i.reshape(1, 1); - } - normAssert(ref_i, out_i, basename.c_str(), l1, lInf); - } - } -}; - -TEST_P(Test_Int8_layers, Convolution1D) -{ - testLayer("conv1d", "ONNX", 0.00302, 0.00909); - testLayer("conv1d_bias", "ONNX", 0.00306, 0.00948); - - { - SCOPED_TRACE("Per-tensor quantize"); - testLayer("conv1d", "ONNX", 0.00302, 0.00909, 1, 1, false, true, false, false); - testLayer("conv1d_bias", "ONNX", 0.00319, 0.00948, 1, 1, false, true, false, false); - } -} - -TEST_P(Test_Int8_layers, Convolution2D) -{ - if(backend == DNN_BACKEND_TIMVX) - testLayer("single_conv", "TensorFlow", 0.00424, 0.02201); - else - testLayer("single_conv", "TensorFlow", 0.00413, 0.02201); - - testLayer("atrous_conv2d_valid", "TensorFlow", 0.0193, 0.0633); - testLayer("atrous_conv2d_same", "TensorFlow", 0.0185, 0.1322); - testLayer("keras_atrous_conv2d_same", "TensorFlow", 0.0056, 0.0244); - - if(backend == DNN_BACKEND_TIMVX) - testLayer("convolution", "ONNX", 0.00534, 0.01516); - else - testLayer("convolution", "ONNX", 0.0052, 0.01516); - - if(backend == DNN_BACKEND_TIMVX) - testLayer("two_convolution", "ONNX", 0.0033, 0.01); - else - testLayer("two_convolution", "ONNX", 0.00295, 0.00840); - - if(backend == DNN_BACKEND_TIMVX) - applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); - testLayer("layer_convolution", "Caffe", 0.0174, 0.0758, 1, 1, true); - testLayer("depthwise_conv2d", "TensorFlow", 0.0388, 0.169); - - { - SCOPED_TRACE("Per-tensor quantize"); - testLayer("single_conv", "TensorFlow", 0.00413, 0.02301, 1, 1, false, true, false, false); - testLayer("atrous_conv2d_valid", "TensorFlow", 0.027967, 0.07808, 1, 1, false, true, false, false); - testLayer("atrous_conv2d_same", "TensorFlow", 0.01945, 0.1322, 1, 1, false, true, false, false); - testLayer("keras_atrous_conv2d_same", "TensorFlow", 0.005677, 0.03327, 1, 1, false, true, false, false); - testLayer("convolution", "ONNX", 0.00538, 0.01517, 1, 1, false, true, false, false); - testLayer("two_convolution", "ONNX", 0.00295, 0.00926, 1, 1, false, true, false, false); - testLayer("layer_convolution", "Caffe", 0.0175, 0.0759, 1, 1, true, true, false, false); - testLayer("depthwise_conv2d", "TensorFlow", 0.041847, 0.18744, 1, 1, false, true, false, false); - } -} - -TEST_P(Test_Int8_layers, Convolution3D) -{ - testLayer("conv3d", "TensorFlow", 0.00734, 0.02434); - testLayer("conv3d", "ONNX", 0.00353, 0.00941); - testLayer("conv3d_bias", "ONNX", 0.00129, 0.00249); -} - -TEST_P(Test_Int8_layers, Flatten) -{ - testLayer("flatten", "TensorFlow", 0.0036, 0.0069, 1, 1, false, true, true); - testLayer("unfused_flatten", "TensorFlow", 0.0014, 0.0028); - testLayer("unfused_flatten_unknown_batch", "TensorFlow", 0.0043, 0.0051); - - { - SCOPED_TRACE("Per-tensor quantize"); - testLayer("conv3d", "TensorFlow", 0.00734, 0.02434, 1, 1, false, true, false, false); - testLayer("conv3d", "ONNX", 0.00377, 0.01362, 1, 1, false, true, false, false); - testLayer("conv3d_bias", "ONNX", 0.00201, 0.0039, 1, 1, false, true, false, false); - } -} - -TEST_P(Test_Int8_layers, Padding) -{ - if (backend == DNN_BACKEND_TIMVX) - testLayer("padding_valid", "TensorFlow", 0.0292, 0.0105); - else - testLayer("padding_valid", "TensorFlow", 0.0026, 0.0064); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("padding_same", "TensorFlow", 0.0085, 0.032); - else - testLayer("padding_same", "TensorFlow", 0.0081, 0.032); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("spatial_padding", "TensorFlow", 0.0079, 0.028); - else - testLayer("spatial_padding", "TensorFlow", 0.0078, 0.028); - - testLayer("mirror_pad", "TensorFlow", 0.0064, 0.013); - testLayer("pad_and_concat", "TensorFlow", 0.0021, 0.0098); - testLayer("padding", "ONNX", 0.0005, 0.0069); - testLayer("ReflectionPad2d", "ONNX", 0.00062, 0.0018); - testLayer("ZeroPad2d", "ONNX", 0.00037, 0.0018); -} - -TEST_P(Test_Int8_layers, AvePooling) -{ - // Some tests failed with OpenVINO due to wrong padded area calculation - if (backend != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - testLayer("layer_pooling_ave", "Caffe", 0.0021, 0.0075); - testLayer("ave_pool_same", "TensorFlow", 0.00153, 0.0041); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2025030000) - if (backend != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) -#endif - testLayer("average_pooling_1d", "ONNX", 0.002, 0.0048); - if (backend != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - testLayer("average_pooling", "ONNX", 0.0014, 0.0032); - testLayer("average_pooling_dynamic_axes", "ONNX", 0.0014, 0.006); - - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); - testLayer("ave_pool3d", "TensorFlow", 0.00175, 0.0047); - testLayer("ave_pool3d", "ONNX", 0.00063, 0.0016); -} - -TEST_P(Test_Int8_layers, MaxPooling) -{ - testLayer("pool_conv_1d", "ONNX", 0.0006, 0.0015); - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); - testLayer("pool_conv_3d", "ONNX", 0.0033, 0.0124); - - testLayer("layer_pooling_max", "Caffe", 0.0021, 0.004); - testLayer("max_pool_even", "TensorFlow", 0.0048, 0.0139); - testLayer("max_pool_odd_valid", "TensorFlow", 0.0043, 0.012); - testLayer("conv_pool_nchw", "TensorFlow", 0.007, 0.025); - testLayer("max_pool3d", "TensorFlow", 0.0025, 0.0058); - testLayer("maxpooling_1d", "ONNX", 0.0018, 0.0037); - testLayer("two_maxpooling_1d", "ONNX", 0.0037, 0.0052); - testLayer("maxpooling", "ONNX", 0.0034, 0.0065); - testLayer("two_maxpooling", "ONNX", 0.0025, 0.0052); - testLayer("max_pool3d", "ONNX", 0.0028, 0.0069); -} - -TEST_P(Test_Int8_layers, Reduce) -{ - testLayer("reduce_mean", "TensorFlow", 0.0005, 0.0014); - testLayer("reduce_mean", "ONNX", 0.00062, 0.0014); - testLayer("reduce_mean_axis1", "ONNX", 0.00032, 0.0007); - testLayer("reduce_mean_axis2", "ONNX", 0.00033, 0.001); - - testLayer("reduce_sum", "TensorFlow", 0.015, 0.031); - testLayer("reduce_sum_channel", "TensorFlow", 0.008, 0.019); - testLayer("sum_pool_by_axis", "TensorFlow", 0.012, 0.032); - testLayer("reduce_sum", "ONNX", 0.0025, 0.0048); - - testLayer("reduce_max", "ONNX", 0, 0); - testLayer("reduce_max_axis_0", "ONNX", 0.0042, 0.007); - testLayer("reduce_max_axis_1", "ONNX", 0.0018, 0.0036); - - if (target != DNN_TARGET_CPU) - throw SkipTestException("Only CPU is supported"); - testLayer("reduce_mean3d", "ONNX", 0.00048, 0.0016); -} - -TEST_P(Test_Int8_layers, ReLU) -{ - testLayer("layer_relu", "Caffe", 0.0005, 0.002); - testLayer("ReLU", "ONNX", 0.0012, 0.0047); -} - -TEST_P(Test_Int8_layers, LeakyReLU) -{ - testLayer("leaky_relu", "TensorFlow", 0.0002, 0.0004); -} - -TEST_P(Test_Int8_layers, ReLU6) -{ - testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062); - testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062, 1, 1, false, true, true); - testLayer("clip_by_value", "TensorFlow", 0.0009, 0.002); - testLayer("clip", "ONNX", 0.00006, 0.00037); -} - -TEST_P(Test_Int8_layers, Sigmoid) -{ - testLayer("maxpooling_sigmoid", "ONNX", 0.0011, 0.0032); -} - -TEST_P(Test_Int8_layers, Sigmoid_dynamic_axes) -{ - testLayer("maxpooling_sigmoid_dynamic_axes", "ONNX", 0.002, 0.0032); -} - -TEST_P(Test_Int8_layers, Sigmoid_1d) -{ - testLayer("maxpooling_sigmoid_1d", "ONNX", 0.002, 0.0037); -} - -TEST_P(Test_Int8_layers, Mish) -{ - testLayer("mish", "ONNX", 0.0015, 0.0025); -} - -TEST_P(Test_Int8_layers, Softmax_Caffe) -{ - testLayer("layer_softmax", "Caffe", 0.0011, 0.0036); -} -TEST_P(Test_Int8_layers, Softmax_keras_TF) -{ - testLayer("keras_softmax", "TensorFlow", 0.00093, 0.0027); -} -TEST_P(Test_Int8_layers, Softmax_slim_TF) -{ - testLayer("slim_softmax", "TensorFlow", 0.0016, 0.0034); -} -TEST_P(Test_Int8_layers, Softmax_slim_v2_TF) -{ - testLayer("slim_softmax_v2", "TensorFlow", 0.0029, 0.017); -} -TEST_P(Test_Int8_layers, Softmax_ONNX) -{ - testLayer("softmax", "ONNX", 0.0016, 0.0028); -} -TEST_P(Test_Int8_layers, Softmax_log_ONNX) -{ - testLayer("log_softmax", "ONNX", 0.014, 0.025); -} -TEST_P(Test_Int8_layers, DISABLED_Softmax_unfused_ONNX) // FIXIT Support 'Identity' layer for outputs (#22022) -{ - testLayer("softmax_unfused", "ONNX", 0.0009, 0.0021); -} - -TEST_P(Test_Int8_layers, Concat) -{ - testLayer("layer_concat_shared_input", "Caffe", 0.0076, 0.029, 1, 1, true, false); - if (backend != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { - // Crashes with segfault - testLayer("concat_axis_1", "TensorFlow", 0.0056, 0.017); - } - testLayer("keras_pad_concat", "TensorFlow", 0.0032, 0.0089); - testLayer("concat_3d", "TensorFlow", 0.005, 0.014); - testLayer("concatenation", "ONNX", 0.0032, 0.009); -} - -TEST_P(Test_Int8_layers, BatchNorm) -{ - testLayer("layer_batch_norm", "Caffe", 0.0061, 0.019, 1, 1, true); - testLayer("fused_batch_norm", "TensorFlow", 0.0063, 0.02); - testLayer("batch_norm_text", "TensorFlow", 0.0048, 0.013, 1, 1, false, true, true); - testLayer("unfused_batch_norm", "TensorFlow", 0.0076, 0.019); - testLayer("fused_batch_norm_no_gamma", "TensorFlow", 0.0067, 0.015); - testLayer("unfused_batch_norm_no_gamma", "TensorFlow", 0.0123, 0.044); - testLayer("switch_identity", "TensorFlow", 0.0035, 0.011); - testLayer("batch_norm3d", "TensorFlow", 0.0077, 0.02); - testLayer("batch_norm", "ONNX", 0.0012, 0.0049); - testLayer("batch_norm_3d", "ONNX", 0.0039, 0.012); - testLayer("frozenBatchNorm2d", "ONNX", 0.001, 0.0018); - testLayer("batch_norm_subgraph", "ONNX", 0.0049, 0.0098); -} - -TEST_P(Test_Int8_layers, Scale) -{ - testLayer("batch_norm", "TensorFlow", 0.0028, 0.0098); - testLayer("scale", "ONNX", 0.0025, 0.0071); - testLayer("expand_hw", "ONNX", 0.0012, 0.0012); - testLayer("flatten_const", "ONNX", 0.0024, 0.0048); -} - -TEST_P(Test_Int8_layers, InnerProduct) -{ - testLayer("layer_inner_product", "Caffe", 0.005, 0.02, 1, 1, true); - testLayer("matmul", "TensorFlow", 0.0061, 0.019); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0018, 0.0175); - else - testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0009, 0.0091); - - testLayer("nhwc_reshape_matmul", "TensorFlow", 0.03, 0.071); - testLayer("matmul_layout", "TensorFlow", 0.035, 0.06); - testLayer("tf2_dense", "TensorFlow", 0, 0); - testLayer("matmul_add", "ONNX", 0.041, 0.082); - testLayer("linear", "ONNX", 0.0027, 0.0046); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("constant", "ONNX", 0.00048, 0.0013); - else - testLayer("constant", "ONNX", 0.00021, 0.0006); - - testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016); - - { - SCOPED_TRACE("Per-tensor quantize"); - testLayer("layer_inner_product", "Caffe", 0.0055, 0.02, 1, 1, true, true, false, false); - testLayer("matmul", "TensorFlow", 0.0075, 0.019, 1, 1, false, true, false, false); - testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0009, 0.0091, 1, 1, false, true, false, false); - testLayer("nhwc_reshape_matmul", "TensorFlow", 0.037, 0.071, 1, 1, false, true, false, false); - testLayer("matmul_layout", "TensorFlow", 0.035, 0.095, 1, 1, false, true, false, false); - testLayer("tf2_dense", "TensorFlow", 0, 0, 1, 1, false, true, false, false); - testLayer("matmul_add", "ONNX", 0.041, 0.082, 1, 1, false, true, false, false); - testLayer("linear", "ONNX", 0.0027, 0.005, 1, 1, false, true, false, false); - testLayer("constant", "ONNX", 0.00038, 0.0012, 1, 1, false, true, false, false); - testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016, 1, 1, false, true, false, false); - } -} - -TEST_P(Test_Int8_layers, Reshape) -{ - testLayer("reshape_layer", "TensorFlow", 0.0032, 0.0082); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("reshape_nchw", "TensorFlow", 0.0092, 0.0495); - else - testLayer("reshape_nchw", "TensorFlow", 0.0089, 0.029); - - testLayer("reshape_conv", "TensorFlow", 0.035, 0.054); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - testLayer("reshape_reduce", "TensorFlow", 0.0053, 0.011); - else - testLayer("reshape_reduce", "TensorFlow", 0.0042, 0.0078); - testLayer("reshape_as_shape", "TensorFlow", 0.0014, 0.0028); - testLayer("reshape_no_reorder", "TensorFlow", 0.0014, 0.0028); - testLayer("shift_reshape_no_reorder", "TensorFlow", 0.0063, backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ? 0.016 : 0.014); - testLayer("dynamic_reshape", "ONNX", 0.0047, 0.0079); - testLayer("dynamic_reshape_opset_11", "ONNX", 0.0048, 0.0081); - testLayer("flatten_by_prod", "ONNX", 0.0048, 0.0081); - testLayer("squeeze", "ONNX", 0.0048, 0.0081); - testLayer("unsqueeze", "ONNX", 0.0033, 0.0053); - - if (backend == DNN_BACKEND_TIMVX) - testLayer("squeeze_and_conv_dynamic_axes", "ONNX", 0.006, 0.0212); - else - testLayer("squeeze_and_conv_dynamic_axes", "ONNX", 0.0054, 0.0154); - - testLayer("unsqueeze_and_conv_dynamic_axes", "ONNX", 0.0037, 0.0151); -} - -TEST_P(Test_Int8_layers, Permute) -{ - testLayer("tf2_permute_nhwc_ncwh", "TensorFlow", 0.0028, 0.006); - testLayer("transpose", "ONNX", 0.0015, 0.0046); -} - -TEST_P(Test_Int8_layers, Identity) -{ - testLayer("expand_batch", "ONNX", 0.0027, 0.0036); - testLayer("expand_channels", "ONNX", 0.0013, 0.0019); - testLayer("expand_neg_batch", "ONNX", 0.00071, 0.0019); -} - -TEST_P(Test_Int8_layers, Slice_split_tf) -{ - testLayer("split", "TensorFlow", 0.0033, 0.0056); -} - -TEST_P(Test_Int8_layers, Slice_4d_tf) -{ - testLayer("slice_4d", "TensorFlow", 0.003, 0.0073); -} - -TEST_P(Test_Int8_layers, Slice_strided_tf) -{ - testLayer("strided_slice", "TensorFlow", 0.008, 0.0142); -} - -TEST_P(Test_Int8_layers, DISABLED_Slice_onnx) // FIXIT Support 'Identity' layer for outputs (#22022) -{ - testLayer("slice", "ONNX", 0.0046, 0.0077); -} - -TEST_P(Test_Int8_layers, Slice_dynamic_axes_onnx) -{ - testLayer("slice_dynamic_axes", "ONNX", 0.0039, 0.02); -} - -TEST_P(Test_Int8_layers, Slice_steps_2d_onnx11) -{ - testLayer("slice_opset_11_steps_2d", "ONNX", 0.01, 0.0124); -} - -TEST_P(Test_Int8_layers, Slice_steps_3d_onnx11) -{ - testLayer("slice_opset_11_steps_3d", "ONNX", 0.0068, 0.014); -} - -TEST_P(Test_Int8_layers, Slice_steps_4d_onnx11) -{ - testLayer("slice_opset_11_steps_4d", "ONNX", 0.0041, 0.008); -} - -TEST_P(Test_Int8_layers, Slice_steps_5d_onnx11) -{ - testLayer("slice_opset_11_steps_5d", "ONNX", 0.0085, 0.021); -} - -TEST_P(Test_Int8_layers, Dropout) -{ - testLayer("layer_dropout", "Caffe", 0.0021, 0.004); - testLayer("dropout", "ONNX", 0.0029, 0.004); -} - -TEST_P(Test_Int8_layers, Eltwise) -{ - testLayer("layer_eltwise", "Caffe", 0.062, 0.15); - - if (backend == DNN_BACKEND_TIMVX) - applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); - - testLayer("conv_2_inps", "Caffe", 0.0086, 0.0232, 2, 1, true, false); - testLayer("eltwise_sub", "TensorFlow", 0.015, 0.047); - testLayer("eltwise_add_vec", "TensorFlow", 0.037, backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ? 0.24 : 0.21); // tflite 0.0095, 0.0365 - testLayer("eltwise_mul_vec", "TensorFlow", 0.173, 1.14); // tflite 0.0028, 0.017 - testLayer("channel_broadcast", "TensorFlow", 0.0025, 0.0063); - testLayer("split_equals", "TensorFlow", backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH ? 0.021 : 0.02, 0.065); - testLayer("mul", "ONNX", 0.0039, 0.014); - testLayer("split_max", "ONNX", 0.004, 0.012); -} - -TEST_P(Test_Int8_layers, DepthSpaceOps) { - auto test_layer_with_onnx_conformance_models = [&](const std::string &model_name, double l1, double lInf) { - std::string model_path = _tf("onnx/conformance/node/test_" + model_name + "/model.onnx"); - auto net = readNet(model_path); - - // load reference inputs and outputs - std::string data_base_path = _tf("onnx/conformance/node/test_" + model_name + "/test_data_set_0"); - Mat input = readTensorFromONNX(data_base_path + "/input_0.pb"); - Mat ref_output = readTensorFromONNX(data_base_path + "/output_0.pb"); - - std::vector input_scales, output_scales; - std::vector input_zeropoints, output_zeropoints; - auto qnet = net.quantize(std::vector{input}, CV_8S, CV_8S, false); - qnet.getInputDetails(input_scales, input_zeropoints); - qnet.getOutputDetails(output_scales, output_zeropoints); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - - Mat quantized_input, quantized_output; - input.convertTo(quantized_input, CV_8S, 1.f / input_scales.front(), input_zeropoints.front()); - qnet.setInput(quantized_input); - quantized_output = qnet.forward(); - - Mat output; - quantized_output.convertTo(output, CV_32F, output_scales.front(), -(output_scales.front() * output_zeropoints.front())); - normAssert(ref_output, output, model_name.c_str(), l1, lInf); - }; - - double l1 = default_l1, lInf = default_lInf; - { - l1 = 0.001; lInf = 0.002; - if (backend == DNN_BACKEND_TIMVX) { l1 = 0.001; lInf = 0.002; } - test_layer_with_onnx_conformance_models("spacetodepth", l1, lInf); - } - { - l1 = 0.022; lInf = 0.044; - if (backend == DNN_BACKEND_TIMVX) { l1 = 0.022; lInf = 0.044; } - test_layer_with_onnx_conformance_models("spacetodepth_example", l1, lInf); - } - { - l1 = 0.001; lInf = 0.002; - if (backend == DNN_BACKEND_TIMVX) { l1 = 0.24; lInf = 0.99; } - test_layer_with_onnx_conformance_models("depthtospace_crd_mode", l1, lInf); - } - test_layer_with_onnx_conformance_models("depthtospace_dcr_mode", 0.001, 0.002); - test_layer_with_onnx_conformance_models("depthtospace_example", 0.07, 0.14); - - { - l1 = 0.07; lInf = 0.14; - if (backend == DNN_BACKEND_TIMVX) // diff too huge, l1 = 13.6; lInf = 27.2 - applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); - test_layer_with_onnx_conformance_models("depthtospace_crd_mode_example", l1, lInf); - } -} - -INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargetsInt8()); - -class Test_Int8_nets : public DNNTestLayer -{ -public: - void testClassificationNet(Net baseNet, const Mat& blob, const Mat& ref, double l1, double lInf, bool perChannel = true) - { - Net qnet = baseNet.quantize(blob, CV_32F, CV_32F, perChannel); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - - qnet.setInput(blob); - Mat out = qnet.forward(); - normAssert(ref, out, "", l1, lInf); - } - - void testDetectionNet(Net baseNet, const Mat& blob, const Mat& ref, - double confThreshold, double scoreDiff, double iouDiff, bool perChannel = true) - { - Net qnet = baseNet.quantize(blob, CV_32F, CV_32F, perChannel); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - - qnet.setInput(blob); - Mat out = qnet.forward(); - normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); - } - - void testFaster(Net baseNet, const Mat& ref, double confThreshold, double scoreDiff, double iouDiff, bool perChannel = true) - { - Mat inp = imread(_tf("dog416.png")); - resize(inp, inp, Size(800, 600)); - Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); - Mat imInfo = (Mat_(1, 3) << inp.rows, inp.cols, 1.6f); - - Net qnet = baseNet.quantize(std::vector{blob, imInfo}, CV_32F, CV_32F, perChannel); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - - qnet.setInput(blob, "data"); - qnet.setInput(imInfo, "im_info"); - Mat out = qnet.forward(); - normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); - } - - void testONNXNet(const String& basename, double l1, double lInf, bool useSoftmax = false, bool perChannel = true) - { - String onnxmodel = findDataFile("dnn/onnx/models/" + basename + ".onnx", false); - - Mat blob = readTensorFromONNX(findDataFile("dnn/onnx/data/input_" + basename + ".pb")); - Mat ref = readTensorFromONNX(findDataFile("dnn/onnx/data/output_" + basename + ".pb")); - Net baseNet = readNetFromONNX(onnxmodel); - - Net qnet = baseNet.quantize(blob, CV_32F, CV_32F, perChannel); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - qnet.setInput(blob); - Mat out = qnet.forward(); - - if (useSoftmax) - { - LayerParams lp; - Net netSoftmax; - netSoftmax.addLayerToPrev("softmaxLayer", "Softmax", lp); - netSoftmax.setPreferableBackend(DNN_BACKEND_OPENCV); - - netSoftmax.setInput(out); - out = netSoftmax.forward(); - - netSoftmax.setInput(ref); - ref = netSoftmax.forward(); - } - - normAssert(ref, out, "", l1, lInf); - } - - void testYOLOModel(const std::string& model, - const cv::Mat& ref, double scoreDiff, double iouDiff, - float confThreshold = 0.24, float nmsThreshold = 0.4, bool perChannel = true) - { - CV_Assert(ref.cols == 7); - std::vector > refClassIds; - std::vector > refScores; - std::vector > refBoxes; - for (int i = 0; i < ref.rows; ++i) - { - int batchId = static_cast(ref.at(i, 0)); - int classId = static_cast(ref.at(i, 1)); - float score = ref.at(i, 2); - float left = ref.at(i, 3); - float top = ref.at(i, 4); - float right = ref.at(i, 5); - float bottom = ref.at(i, 6); - Rect2d box(left, top, right - left, bottom - top); - if (batchId >= refClassIds.size()) - { - refClassIds.resize(batchId + 1); - refScores.resize(batchId + 1); - refBoxes.resize(batchId + 1); - } - refClassIds[batchId].push_back(classId); - refScores[batchId].push_back(score); - refBoxes[batchId].push_back(box); - } - - Mat img1 = imread(_tf("dog416.png")); - Mat img2 = imread(_tf("street.png")); - std::vector samples(2); - samples[0] = img1; samples[1] = img2; - - // determine test type, whether batch or single img - int batch_size = refClassIds.size(); - CV_Assert(batch_size == 1 || batch_size == 2); - samples.resize(batch_size); - - Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false); - - Net baseNet = readNet(findDataFile("dnn/" + model, false)); - Net qnet = baseNet.quantize(inp, CV_32F, CV_32F, perChannel); - qnet.setPreferableBackend(backend); - qnet.setPreferableTarget(target); - qnet.setInput(inp); - std::vector outs; - qnet.forward(outs, qnet.getUnconnectedOutLayersNames()); - - for (int b = 0; b < batch_size; ++b) - { - std::vector classIds; - std::vector confidences; - std::vector boxes; - for (int i = 0; i < outs.size(); ++i) - { - Mat out; - if (batch_size > 1){ - // get the sample slice from 3D matrix (batch, box, classes+5) - Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()}; - out = outs[i](ranges).reshape(1, outs[i].size[1]); - }else{ - out = outs[i]; - } - for (int j = 0; j < out.rows; ++j) - { - Mat scores = out.row(j).colRange(5, out.cols); - double confidence; - Point maxLoc; - minMaxLoc(scores, 0, &confidence, 0, &maxLoc); - - if (confidence > confThreshold) { - float* detection = out.ptr(j); - double centerX = detection[0]; - double centerY = detection[1]; - double width = detection[2]; - double height = detection[3]; - boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height, - width, height)); - confidences.push_back(confidence); - classIds.push_back(maxLoc.x); - } - } - } - - // here we need NMS of boxes - std::vector indices; - NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); - - std::vector nms_classIds; - std::vector nms_confidences; - std::vector nms_boxes; - - for (size_t i = 0; i < indices.size(); ++i) - { - int idx = indices[i]; - Rect2d box = boxes[idx]; - float conf = confidences[idx]; - int class_id = classIds[idx]; - nms_boxes.push_back(box); - nms_confidences.push_back(conf); - nms_classIds.push_back(class_id); - } - - if (cvIsNaN(iouDiff)) - { - if (b == 0) - std::cout << "Skip accuracy checks" << std::endl; - continue; - } - - normAssertDetections(refClassIds[b], refScores[b], refBoxes[b], nms_classIds, nms_confidences, nms_boxes, - format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff); - } - } -}; - -TEST_P(Test_Int8_nets, CaffeNet) -{ -#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - float l1 = 4e-5, lInf = 0.0025; - testONNXNet("caffenet", l1, lInf); -} - -TEST_P(Test_Int8_nets, RCNN_ILSVRC13) -{ -#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - float l1 = 0.02, lInf = 0.047; - testONNXNet("rcnn_ilsvrc13", l1, lInf); -} - -TEST_P(Test_Int8_nets, Inception_v2) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - testONNXNet("inception_v2", default_l1, default_lInf, true); -} - -TEST_P(Test_Int8_nets, MobileNet_v2) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - testONNXNet("mobilenetv2", default_l1, default_lInf, true); -} - -TEST_P(Test_Int8_nets, Shufflenet) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - testONNXNet("shufflenet", default_l1, default_lInf); -} - -TEST_P(Test_Int8_nets, MobileNet_v1_SSD) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), - findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); - Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy")); - - float confThreshold = 0.5, scoreDiff = 0.034, iouDiff = 0.14; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false), - findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt")); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); - Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); - - float confThreshold = 0.51, scoreDiff = 0.05, iouDiff = 0.07; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, Inception_v2_SSD) -{ - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); - - Net net = readNetFromTensorflow(findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false), - findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt")); - - Mat inp = imread(_tf("street.png")); - Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); - Mat ref = (Mat_(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729, - 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131, - 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015, - 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527, - 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384); - - float confThreshold = 0.5, scoreDiff = 0.0114, iouDiff = 0.22; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, EfficientDet) -{ - if (cvtest::skipUnstableTests) - throw SkipTestException("Skip unstable test"); // detail: https://github.com/opencv/opencv/pull/23167 - - applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG); - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_TIMVX) - applyTestTag(CV_TEST_TAG_DNN_SKIP_TIMVX); - - if (target != DNN_TARGET_CPU) - { - if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); - } - Net net = readNetFromTensorflow(findDataFile("dnn/efficientdet-d0.pb", false), - findDataFile("dnn/efficientdet-d0.pbtxt")); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0/255, Size(512, 512), Scalar(123.675, 116.28, 103.53)); - Mat ref = (Mat_(3, 7) << 0, 1, 0.8437444, 0.153996080160141, 0.20534580945968628, 0.7463544607162476, 0.7414066195487976, - 0, 17, 0.8245924, 0.16657517850399017, 0.3996818959712982, 0.4111558794975281, 0.9306337833404541, - 0, 7, 0.8039304, 0.6118435263633728, 0.13175517320632935, 0.9065558314323425, 0.2943994700908661); - - float confThreshold = 0.65, scoreDiff = 0.3, iouDiff = 0.18; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); - - { - SCOPED_TRACE("Per-tensor quantize"); - testDetectionNet(net, blob, ref, 0.85, scoreDiff, iouDiff, false); - } -} - -TEST_P(Test_Int8_nets, FasterRCNN_resnet50) -{ - applyTestTag( - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - - Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pb", false), - findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pbtxt")); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); - Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); - - float confThreshold = 0.8, scoreDiff = 0.05, iouDiff = 0.15; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) -{ - applyTestTag( - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_LONG, - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - - Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false), - findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt")); - - Mat inp = imread(_tf("dog416.png")); - Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); - Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy")); - - float confThreshold = 0.5, scoreDiff = 0.21, iouDiff = 0.1; - testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); -} - -TEST_P(Test_Int8_nets, YOLOv3) -{ - applyTestTag( - CV_TEST_TAG_LONG, - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - const int N0 = 3; - const int N1 = 6; - static const float ref_[/* (N0 + N1) * 7 */] = { -0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f, -0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f, -0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f, - -1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f, -1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f, -1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f, -1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f, -1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f, -1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f, - }; - Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); - - std::string model_file = "yolov3.onnx"; - - double scoreDiff = 0.08, iouDiff = 0.21, confThreshold = 0.28; - { - SCOPED_TRACE("batch size 1"); - testYOLOModel(model_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); - } - - { - SCOPED_TRACE("batch size 2"); - testYOLOModel(model_file, ref, scoreDiff, iouDiff, confThreshold); - } -} - -TEST_P(Test_Int8_nets, YOLOv4) -{ - applyTestTag( - CV_TEST_TAG_LONG, - (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), - CV_TEST_TAG_DEBUG_VERYLONG - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - - const int N0 = 3; - const int N1 = 5; - static const float ref_[/* (N0 + N1) * 7 */] = { -0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f, -0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f, -0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f, - -1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f, -1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f, -1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f, -1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f, -1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f, - }; - Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); - - std::string model_file = "yolov4.onnx"; - double scoreDiff = 0.15, iouDiff = 0.2; - { - SCOPED_TRACE("batch size 1"); - testYOLOModel(model_file, ref.rowRange(0, N0), scoreDiff, iouDiff, 0.5); - } - - { - SCOPED_TRACE("batch size 2"); - - testYOLOModel(model_file, ref, scoreDiff, iouDiff, 0.5); - } -} - -TEST_P(Test_Int8_nets, YOLOv4_tiny) -{ - applyTestTag( - target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB - ); - - if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); - if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) - applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - const float confThreshold = 0.6; - - const int N0 = 2; - const int N1 = 3; - static const float ref_[/* (N0 + N1) * 7 */] = { -0, 16, 0.912199f, 0.169926f, 0.350896f, 0.422704f, 0.941837f, -0, 7, 0.845388f, 0.617568f, 0.13961f, 0.9008f, 0.29315f, - -1, 2, 0.997789f, 0.657455f, 0.459714f, 0.809122f, 0.656829f, -1, 2, 0.924423f, 0.442872f, 0.470127f, 0.49816f, 0.516516f, -1, 0, 0.728307f, 0.202607f, 0.369828f, 0.259445f, 0.613846f, - }; - Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); - - std::string model_file = "yolov4-tiny.onnx"; - double scoreDiff = 0.12; - double iouDiff = target == DNN_TARGET_OPENCL_FP16 ? 0.2 : 0.118; - - { - SCOPED_TRACE("batch size 1"); - testYOLOModel(model_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); - - { - SCOPED_TRACE("Per-tensor quantize"); - testYOLOModel(model_file, ref.rowRange(0, N0), scoreDiff, 0.224, 0.7, 0.4, false); - } - } - - throw SkipTestException("batch2: bad accuracy on second image"); - /* bad accuracy on second image - { - SCOPED_TRACE("batch size 2"); - testYOLOModel(model_file, ref, scoreDiff, iouDiff, confThreshold); - } - */ -} - -INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargetsInt8()); - -}} // namespace - -#endif // #if 0 diff --git a/modules/dnn/test/test_layers_1d.cpp b/modules/dnn/test/test_layers_1d.cpp index 16e6fc6ac0..c5b8957544 100644 --- a/modules/dnn/test/test_layers_1d.cpp +++ b/modules/dnn/test/test_layers_1d.cpp @@ -78,9 +78,9 @@ TEST_P(Layer_Test_01D, Clip) lp.type = "Clip"; lp.name = "ClipLayer"; - lp.set("min_value", 0.0); - lp.set("max_value", 1.0); - Ptr layer = ReLU6Layer::create(lp); + lp.set("min", 0.0); + lp.set("max", 1.0); + Ptr layer = ClipLayer::create(lp); Mat output_ref(output_shape.size(), output_shape.data(), CV_32F, 1.0); std::vector inputs{input}; @@ -725,7 +725,6 @@ int arg_op(const std::vector& vec, const std::string& operation) { CV_Error(Error::StsAssert, "Provided operation: " + operation + " is not supported. Please check the test instantiation."); } } -// Test for ArgLayer is disabled because there problem in runLayer function related to type assignment typedef testing::TestWithParam, std::string>> Layer_Arg_Test; TEST_P(Layer_Arg_Test, Accuracy_01D) { std::vector input_shape = get<0>(GetParam()); @@ -774,7 +773,7 @@ TEST_P(Layer_Arg_Test, Accuracy_01D) { runLayer(layer, inputs, outputs); ASSERT_EQ(1, outputs.size()); ASSERT_EQ(shape(output_ref), shape(outputs[0])); - // convert output_ref to float to match the output type + // ArgLayer::getTypes() reports CV_64S; match it before comparing output_ref.convertTo(output_ref, CV_64SC1); normAssert(output_ref, outputs[0]); } diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 5b9e85a02d..caf2cc02ee 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -735,8 +735,8 @@ TEST_P(Test_ONNX_layers, Elementwise_Sqrt) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - testONNXModels("sqrt"); #endif + testONNXModels("sqrt"); } TEST_P(Test_ONNX_layers, Elementwise_not) @@ -1374,8 +1374,7 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_neg_axis"); } -// Mul inside with 0-d tensor, output should be A x 1, but is 1 x A. PR #22652 -TEST_P(Test_ONNX_layers, DISABLED_Split_sizes_0d) +TEST_P(Test_ONNX_layers, Split_sizes_0d) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); @@ -1551,14 +1550,12 @@ TEST_P(Test_ONNX_layers, LSTM_Activations) testONNXModels("lstm_cntk_tanh", pb, 0, 0, false, false); } -// disabled due to poor handling of 1-d mats -TEST_P(Test_ONNX_layers, DISABLED_LSTM) +TEST_P(Test_ONNX_layers, LSTM) { testONNXModels("lstm", npy, 0, 0, false, false); } -// disabled due to poor handling of 1-d mats -TEST_P(Test_ONNX_layers, DISABLED_LSTM_bidirectional) +TEST_P(Test_ONNX_layers, LSTM_bidirectional) { testONNXModels("lstm_bidirectional", npy, 0, 0, false, false); } @@ -1721,20 +1718,14 @@ TEST_P(Test_ONNX_layers, LSTM_init_h0_c0) testONNXModels("lstm_init_h0_c0", npy, 0, 0, false, false, 3); } -// epsilon is larger because onnx does not match with torch/opencv exactly -// Test uses incorrect ONNX and test data with 3 dims instead of 4. -// ONNNRuntime does not support layout=1 attiribute inference. See a detailed issue #26456 -TEST_P(Test_ONNX_layers, DISABLED_LSTM_layout_seq) +TEST_P(Test_ONNX_layers, LSTM_layout_seq) { if(backend == DNN_BACKEND_CUDA) applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); testONNXModels("lstm_layout_0", npy, 0.005, 0.005, false, false, 3); } -// epsilon is larger because onnx does not match with torch/opencv exactly -// Test uses incorrect ONNX and test data with 3 dims instead of 4. -// ONNNRuntime does not support layout=1 attiribute inference. See a detailed issue #26456 -TEST_P(Test_ONNX_layers, DISABLED_LSTM_layout_batch) +TEST_P(Test_ONNX_layers, LSTM_layout_batch) { if(backend == DNN_BACKEND_CUDA) applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); @@ -2516,11 +2507,6 @@ TEST_P(Test_ONNX_nets, RAFT) normAssert(ref0, outs[0], "", 1.5e-3, 3.2e-2); } -TEST_P(Test_ONNX_nets, Squeezenet) -{ - testONNXModels("squeezenet", pb); -} - TEST_P(Test_ONNX_nets, Googlenet) { #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) @@ -2568,48 +2554,6 @@ TEST_P(Test_ONNX_nets, Googlenet) expectNoFallbacksFromIE(net); } -TEST_P(Test_ONNX_nets, CaffeNet) -{ -#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - testONNXModels("caffenet", pb); -} - -TEST_P(Test_ONNX_nets, RCNN_ILSVRC13) -{ -#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) - applyTestTag(CV_TEST_TAG_MEMORY_2GB); -#else - applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - // Reference output values are in range [-4.992, -1.161] - testONNXModels("rcnn_ilsvrc13", pb, 0.0046); -} - -TEST_P(Test_ONNX_nets, VGG16_bn) -{ - applyTestTag(CV_TEST_TAG_MEMORY_6GB); // > 2.3Gb - - // output range: [-16; 27], after Softmax [0; 0.67] - const double lInf = (target == DNN_TARGET_MYRIAD) ? 0.038 : default_lInf; - testONNXModels("vgg16-bn", pb, default_l1, lInf, true); -} - TEST_P(Test_ONNX_nets, ZFNet) { applyTestTag(CV_TEST_TAG_MEMORY_2GB); @@ -2836,16 +2780,6 @@ TEST_P(Test_ONNX_nets, DenseNet121) testONNXModels("densenet121", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD); } -TEST_P(Test_ONNX_nets, Inception_v1) -{ -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif - testONNXModels("inception_v1", pb); -} - TEST_P(Test_ONNX_nets, Shufflenet) { #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 8b93269c74..c21af7cd6d 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -27,30 +27,6 @@ static std::string _tf(TString filename) return (getOpenCVExtraDir() + "/dnn/") + filename; } -TEST(Test_TensorFlow, read_inception) -{ - Net net; - { - const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false); - net = readNetFromTensorflow(model); - ASSERT_FALSE(net.empty()); - } - net.setPreferableBackend(DNN_BACKEND_OPENCV); - - Mat sample = imread(_tf("grace_hopper_227.png")); - ASSERT_TRUE(!sample.empty()); - Mat input; - resize(sample, input, Size(224, 224)); - input -= Scalar::all(117); // mean sub - - Mat inputBlob = blobFromImage(input); - - net.setInput(inputBlob, "input"); - Mat out = net.forward(); - - std::cout << out.dims << std::endl; -} - TEST(Test_TensorFlow, inception_accuracy) { Net net; diff --git a/modules/dnn/test/test_tokenizer.cpp b/modules/dnn/test/test_tokenizer.cpp index 38e3005df7..dc734db671 100644 --- a/modules/dnn/test/test_tokenizer.cpp +++ b/modules/dnn/test/test_tokenizer.cpp @@ -52,14 +52,6 @@ TEST(Tokenizer_BPE, Tokenizer_GPT2) { std::cout << word << std::endl; } -TEST(Tokenizer_BPE, Tokenizer_GPT2_Model) { - std::string gpt2_model = _tf("gpt2/config.json"); - Tokenizer tok = Tokenizer::load(gpt2_model); - auto ids = tok.encode("hello world"); - auto text = tok.decode(ids); - EXPECT_EQ(text, "hello world"); -} - TEST(Tokenizer_BPE, SimpleRepeated_GPT2) { Tokenizer gpt2_tok = Tokenizer::load(_tf("gpt2/config.json")); EXPECT_EQ(gpt2_tok.encode("0"), std::vector({15}));