mirror of
https://github.com/opencv/opencv.git
synced 2026-11-21 00:35:35 -06:00
Added support for Granite Docling 258M and PaddleOCR-VL - #29618 Adds `com.microsoft::GroupQueryAttention` to the engine-new ONNX importer, required by GraniteDocling-258M's ,end-to-end samples for GraniteDocling-258M and PaddleOCR-VL-1.5. Verified on Linux x86_64 / GCC / CPU **companion opencv_extra PR** [Vlm ocr support engine new opencv/opencv_extra#1412](https://github.com/opencv/opencv_extra/pull/1412) for test data upstream/5.x already lowers `GroupQueryAttention` to `AttentionOnnxAi`, but that path rejects `do_rotary=1`: ```cpp CV_CheckEQ(params.get<int>("do_rotary", 0), 0, "GroupQueryAttention: do_rotary=1 is not supported"); ``` GraniteDocling-258M's export sets it. Confirmed two ways: building a Llama checkpoint from its published `text_config` (9 heads / 3 KV heads, `rope_theta=100000`) and running onnxruntime-genai's `model_builder`; and reading the community export at `onnx-community/granite-docling-258M-ONNX`, which carries 30 `GroupQueryAttention` nodes with `do_rotary=1`, `num_heads=9`, `kv_num_heads=3`, `head_dim=64`. ## Changes **Importer** (`onnx_importer2.cpp`, +49) Routing as above, plus the input validation the op needs: packed QKV rejected, past_key / past_value required as a pair, `do_rotary=1` requires cos/sin, unsupported trailing inputs rejected, and `seqlens_k` shorter than the buffer recognised as a shared-buffer export rather than silently mis-attended. **Layer** (`group_query_attention_layer.cpp`, +380) Attention is built on `fastGemmBatch` and `fused_softmax_softcap_mask`, the same kernels `attention_onnxai_layer.cpp` uses. Grouping falls out of the batched-GEMM offset table (query head → `h / groupSize`), so no broadcast copy is needed, and the second GEMM writes straight into the `[B, S, num_heads*D]` output. Supports rotary (full and partial), sliding window, softcap, growing and preallocated caches, and optional `present_*` outputs. **Core** (`persistence_json.cpp`, +92) `\uXXXX` escapes are UTF-16 code units, so anything above the BMP arrives as a surrogate pair. These were encoded as two separate 3-byte sequences CESU-8, which is not valid UTF-8 - for keys and values alike. Now recombined into one 4-byte sequence, with unpaired surrogates rejected. **Tests / perf / samples** `test_layer_group_query_attention.cpp` (+93), `test_tokenizer.cpp` (+28), `test_tokenizer.py` (+42), `perf_layer.cpp` (+125: `Layer_GroupQueryAttention` MHA_ShortCache and Grouped_WithCache, plus `Layer_Sign`), and two samples (`granite_docling_inference.py`, `paddleocr_vl_inference.py`). ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
186 lines
7.6 KiB
C++
186 lines
7.6 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
|
// Third party copyrights are property of their respective owners.
|
|
|
|
#include "test_precomp.hpp"
|
|
#include "npy_blob.hpp"
|
|
#include <opencv2/dnn/shape_utils.hpp>
|
|
#include <opencv2/dnn/all_layers.hpp>
|
|
|
|
namespace opencv_test { namespace {
|
|
|
|
static void runGQAModel(const std::string& basename, const std::vector<std::string>& inputNames,
|
|
std::vector<Mat>& outs)
|
|
{
|
|
Net net = readNetFromONNX(findDataFile("dnn/onnx/models/" + basename + ".onnx", true));
|
|
ASSERT_FALSE(net.empty());
|
|
|
|
for (size_t i = 0; i < inputNames.size(); ++i)
|
|
net.setInput(blobFromNPY(findDataFile(format("dnn/onnx/data/input_%s_%d.npy", basename.c_str(), (int)i))),
|
|
inputNames[i]);
|
|
|
|
net.forward(outs, std::vector<String>{"output", "present_key", "present_value"});
|
|
}
|
|
|
|
static void checkGQAOutputs(const std::string& basename, const std::vector<Mat>& outs)
|
|
{
|
|
ASSERT_EQ(outs.size(), (size_t)3);
|
|
Mat refOutput = blobFromNPY(findDataFile("dnn/onnx/data/output_" + basename + "_0.npy"));
|
|
Mat refPresentKey = blobFromNPY(findDataFile("dnn/onnx/data/output_" + basename + "_1.npy"));
|
|
Mat refPresentValue = blobFromNPY(findDataFile("dnn/onnx/data/output_" + basename + "_2.npy"));
|
|
|
|
normAssert(refOutput, outs[0], "output", 1e-4, 1e-3);
|
|
normAssert(refPresentKey, outs[1], "present_key", 1e-4, 1e-3);
|
|
normAssert(refPresentValue, outs[2], "present_value", 1e-4, 1e-3);
|
|
}
|
|
|
|
// cos_cache / sin_cache exist only on a node that sets do_rotary, as in real exports,
|
|
// so the non-rotary models stop at total_sequence_length.
|
|
static const std::vector<std::string> GQA_INPUT_NAMES = {
|
|
"query", "key", "value", "past_key", "past_value",
|
|
"seqlens_k", "total_sequence_length"
|
|
};
|
|
|
|
static const std::vector<std::string> GQA_ROTARY_INPUT_NAMES = {
|
|
"query", "key", "value", "past_key", "past_value",
|
|
"seqlens_k", "total_sequence_length", "cos_cache", "sin_cache"
|
|
};
|
|
|
|
// (model basename, whether the node sets do_rotary and so carries cos_cache/sin_cache)
|
|
typedef std::tuple<std::string, bool> GQAModelParam;
|
|
typedef testing::TestWithParam<GQAModelParam> Test_GQA_Model;
|
|
|
|
TEST_P(Test_GQA_Model, Accuracy)
|
|
{
|
|
const std::string basename = std::get<0>(GetParam());
|
|
const bool rotary = std::get<1>(GetParam());
|
|
SCOPED_TRACE(basename);
|
|
|
|
std::vector<Mat> outs;
|
|
runGQAModel(basename, rotary ? GQA_ROTARY_INPUT_NAMES : GQA_INPUT_NAMES, outs);
|
|
checkGQAOutputs(basename, outs);
|
|
}
|
|
|
|
// A GroupQueryAttention node must reach exactly one AttentionOnnxAi layer and no layer of
|
|
// the old standalone type. Naming that type here is deliberate: if the forked lowering ever
|
|
// comes back, this fails instead of passing quietly.
|
|
TEST_P(Test_GQA_Model, LowersToOneAttentionLayer)
|
|
{
|
|
const std::string basename = std::get<0>(GetParam());
|
|
SCOPED_TRACE(basename);
|
|
|
|
Net net = readNetFromONNX(findDataFile("dnn/onnx/models/" + basename + ".onnx", true));
|
|
ASSERT_FALSE(net.empty());
|
|
|
|
int unified = 0, forked = 0;
|
|
const std::vector<String> names = net.getLayerNames();
|
|
for (size_t i = 0; i < names.size(); ++i)
|
|
{
|
|
const String type = net.getLayer(net.getLayerId(names[i]))->type;
|
|
if (type == "AttentionOnnxAi")
|
|
unified++;
|
|
else if (type == "GroupQueryAttention")
|
|
forked++;
|
|
}
|
|
EXPECT_EQ(unified, 1);
|
|
EXPECT_EQ(forked, 0);
|
|
}
|
|
|
|
INSTANTIATE_TEST_CASE_P(/**/, Test_GQA_Model, testing::Values(
|
|
GQAModelParam("group_query_attention_causal", false), // causal self-attention, no cache
|
|
GQAModelParam("group_query_attention_grouped_heads", false), // grouped heads map to the right KV head
|
|
GQAModelParam("group_query_attention_past_kv", false), // present = past concatenated with new
|
|
GQAModelParam("group_query_attention_local_window", false), // sliding window restricts the range
|
|
GQAModelParam("group_query_attention_softcap", false), // softcap clamps the scores
|
|
GQAModelParam("group_query_attention_rotary", true) // rotary applies to the new token only
|
|
));
|
|
|
|
// ---- guards on the GroupQueryAttention path ----
|
|
// Built through LayerParams with the flags the importer sets, so these are the same code
|
|
// paths a real export takes without needing a malformed model on disk.
|
|
|
|
static const int GQA_NUM_HEADS = 4, GQA_KV_NUM_HEADS = 2, GQA_HEAD_SIZE = 8;
|
|
|
|
static Net makeGQANet()
|
|
{
|
|
LayerParams lp;
|
|
lp.type = "AttentionOnnxAi";
|
|
lp.name = "gqa";
|
|
lp.set("q_num_heads", GQA_NUM_HEADS);
|
|
lp.set("kv_num_heads", GQA_KV_NUM_HEADS);
|
|
lp.set("is_causal", true);
|
|
lp.set("has_attn_mask", 0);
|
|
lp.set("has_past", 1);
|
|
lp.set("has_seqlens_k", 1);
|
|
lp.set("has_rotary_cache", 0);
|
|
|
|
Net net;
|
|
int id = net.addLayerToPrev(lp.name, lp.type, lp);
|
|
for (int i = 0; i < 6; ++i)
|
|
net.connect(0, i, id, i);
|
|
net.setInputsNames(std::vector<String>{"query", "key", "value",
|
|
"past_key", "past_value", "seqlens_k"});
|
|
return net;
|
|
}
|
|
|
|
// The fed blob shapes, not any declared shape, are what the layer infers from, so a
|
|
// malformed case has to be malformed here.
|
|
static void feedGQANet(Net& net, int S, int keyS, int pastLen, int pastHeads, int pastD,
|
|
int seqlens)
|
|
{
|
|
const int B = 1, D = GQA_HEAD_SIZE;
|
|
net.setInput(Mat(std::vector<int>{B, S, GQA_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "query");
|
|
net.setInput(Mat(std::vector<int>{B, keyS, GQA_KV_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "key");
|
|
net.setInput(Mat(std::vector<int>{B, keyS, GQA_KV_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "value");
|
|
net.setInput(Mat(std::vector<int>{B, pastHeads, pastLen, pastD}, CV_32F, Scalar(0.1f)), "past_key");
|
|
net.setInput(Mat(std::vector<int>{B, pastHeads, pastLen, pastD}, CV_32F, Scalar(0.1f)), "past_value");
|
|
net.setInput(Mat(std::vector<int>{B}, CV_32S, Scalar(seqlens)), "seqlens_k");
|
|
}
|
|
|
|
// Control: an over-strict guard would fail here rather than in the rejection tests below.
|
|
TEST(GroupQueryAttentionLayer, Guard_AcceptsWellFormedNode)
|
|
{
|
|
Net net = makeGQANet();
|
|
feedGQANet(net, /*S*/4, /*keyS*/4, /*pastLen*/2, /*pastHeads*/GQA_KV_NUM_HEADS,
|
|
/*pastD*/GQA_HEAD_SIZE, /*seqlens*/5);
|
|
EXPECT_NO_THROW(net.forward());
|
|
}
|
|
|
|
// A shorter key is an out-of-bounds read, not just a wrong answer: the shared-buffer write
|
|
// and the rotary pass both step through key/value a query length at a time.
|
|
TEST(GroupQueryAttentionLayer, Guard_RejectsKeyShorterThanQuery)
|
|
{
|
|
Net net = makeGQANet();
|
|
feedGQANet(net, /*S*/4, /*keyS*/2, /*pastLen*/2, /*pastHeads*/GQA_KV_NUM_HEADS,
|
|
/*pastD*/GQA_HEAD_SIZE, /*seqlens*/5);
|
|
EXPECT_ANY_THROW(net.forward());
|
|
}
|
|
|
|
TEST(GroupQueryAttentionLayer, Guard_RejectsWrongPastHeadCount)
|
|
{
|
|
Net net = makeGQANet();
|
|
feedGQANet(net, /*S*/4, /*keyS*/4, /*pastLen*/2, /*pastHeads*/GQA_KV_NUM_HEADS + 1,
|
|
/*pastD*/GQA_HEAD_SIZE, /*seqlens*/5);
|
|
EXPECT_ANY_THROW(net.forward());
|
|
}
|
|
|
|
TEST(GroupQueryAttentionLayer, Guard_RejectsWrongPastHeadSize)
|
|
{
|
|
Net net = makeGQANet();
|
|
feedGQANet(net, /*S*/4, /*keyS*/4, /*pastLen*/2, /*pastHeads*/GQA_KV_NUM_HEADS,
|
|
/*pastD*/GQA_HEAD_SIZE * 2, /*seqlens*/5);
|
|
EXPECT_ANY_THROW(net.forward());
|
|
}
|
|
|
|
TEST(GroupQueryAttentionLayer, Guard_RejectsSeqlensSmallerThanQuery)
|
|
{
|
|
Net net = makeGQANet();
|
|
feedGQANet(net, /*S*/4, /*keyS*/4, /*pastLen*/2, /*pastHeads*/GQA_KV_NUM_HEADS,
|
|
/*pastD*/GQA_HEAD_SIZE, /*seqlens*/1);
|
|
EXPECT_ANY_THROW(net.forward());
|
|
}
|
|
|
|
}} // namespace opencv_test::(anonymous)
|