From 78a5bba29d04736cff90f7f60d1a0d9c3aa1ff89 Mon Sep 17 00:00:00 2001 From: sanjay Date: Thu, 15 Oct 2026 18:25:58 +0530 Subject: [PATCH] dnn(onnx): support bias and key_padding_mask in com.microsoft MultiHeadAttention Adds bias (packed Q/K/V projection bias, split at import time when constant, at runtime otherwise) and key_padding_mask (dense (batch, kv_seq) form, cast to the query's runtime dtype, summed with attention_bias when both are present) support to the importer. AttentionOnnxAi::getMemoryShapes gains a shared attn_mask broadcast shape check, used by ONNX Attention/GroupQueryAttention/ MultiHeadAttention alike. --- .../dnn/src/layers/attention_onnxai_layer.cpp | 15 ++ modules/dnn/src/onnx/onnx_importer2.cpp | 188 +++++++++++++++++- .../test/test_layer_group_query_attention.cpp | 53 +++++ modules/dnn/test/test_onnx_importer.cpp | 70 +++++++ 4 files changed, 318 insertions(+), 8 deletions(-) diff --git a/modules/dnn/src/layers/attention_onnxai_layer.cpp b/modules/dnn/src/layers/attention_onnxai_layer.cpp index 5a29d30e5d..d6f81520c7 100644 --- a/modules/dnn/src/layers/attention_onnxai_layer.cpp +++ b/modules/dnn/src/layers/attention_onnxai_layer.cpp @@ -236,6 +236,21 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer { const int nhq = input_dims == 4 ? inputs[0][1] : q_num_heads; const int total_seq_kv = seq_len_k; + // The kernel broadcasts attn_mask right-aligned with no bounds checks of its own, so the + // ONNX-legal form with a last dim shorter than total_kv_seq (-inf padded) is rejected + // here as unsupported, not as a malformed model. + if (map.mask >= 0) { + const MatShape& ms = inputs[map.mask]; + CV_CheckLE(ms.dims, 4, "attn_mask must have at most 4 dims"); + const int full[4] = {batch_size, nhq, seq_len_q, total_seq_kv}; + for (int i = 0; i < ms.dims; i++) { + const int d = ms[i], f = full[4 - ms.dims + i]; + CV_Check(d, d == 1 || d == f, + "attn_mask must broadcast to [batch, q_num_heads, q_seq, total_kv_seq]; " + "a last dim shorter than total_kv_seq is not supported"); + } + } + CV_CheckTrue(q_hn % kv_hn == 0, "q_num_heads must be divisible by kv_num_heads"); diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index ce5fe66c18..4e33cf59e1 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -312,6 +312,13 @@ protected: const std::vector& scale_zp, const std::vector& outputs, const opencv_onnx::NodeProto& node_proto, int output_onnx_dtype = -1); + Arg addElemAdd(const std::string& name, const opencv_onnx::NodeProto& node_proto, + Arg a, Arg b); + void addMultiHeadAttentionBias(const std::string& bn, const opencv_onnx::NodeProto& node_proto, + Arg bias, Arg& q, Arg& k, Arg& v); + Arg addKeyPaddingMaskBias(const std::string& bn, const opencv_onnx::NodeProto& node_proto, + Arg key_padding_mask, Arg like, float mask_filter_value); + std::map onnx_opset_map; // map from OperatorSetIdProto void parseOperatorSet(); @@ -2926,40 +2933,205 @@ static bool hasOutput(const opencv_onnx::NodeProto& node, int i) { return i < node.output_size() && !node.output(i).empty(); } -// com.microsoft MultiHeadAttention (unpacked Q/K/V) -> AttentionOnnxAi. +Arg ONNXImporter2::addElemAdd(const std::string& name, const opencv_onnx::NodeProto& node_proto, + Arg a, Arg b) { + LayerParams lp; + lp.name = name; + lp.type = "NaryEltwise"; + lp.set("operation", "add"); + Arg out = net.getArg(name); + node_inputs = {a, b}; + node_outputs = {out}; + addLayer(lp, node_proto, 2); + return out; +} + +// bias = [q(H), k(H), v(rest)]; H = query's hidden size, read at runtime (Shape layer) since +// Q/K/V are usually in-graph projections, not graph inputs with a known shape at import time. +void ONNXImporter2::addMultiHeadAttentionBias(const std::string& bn, + const opencv_onnx::NodeProto& node_proto, + Arg bias, Arg& q, Arg& k, Arg& v) { + const MatShape& qs = netimpl->args.at(q.idx).shape; + const int H = qs.dims >= 1 ? qs[qs.dims - 1] : -1; + if (H > 0 && net.isConstArg(bias)) { + // Static split: no Shape/Slice2 at runtime, so the net stays shape-inferable past + // this node (a runtime Slice2 would make tryInferGraphShapes give up here on, costing + // getLayerShapes()/getFLOPS() every layer downstream - see the fallback below). + const Mat b = net.argTensor(bias); + const int L = (int)b.total(); + CV_CheckGT(L, 2 * H, "MultiHeadAttention: bias is shorter than 2*hidden_size"); + const int beg[3] = {0, H, 2 * H}, len[3] = {H, H, L - 2 * H}; + Arg* dst[3] = {&q, &k, &v}; + const char* tag[3] = {"q", "k", "v"}; + const Mat b1d = b.reshape(1, std::vector{L}); + for (int i = 0; i < 3; i++) { + const Mat part = Mat(b1d, std::vector{Range(beg[i], beg[i] + len[i])}).clone(); + Arg c = netimpl->newConstArg(bn + "/" + tag[i] + "_bias", part); + *dst[i] = addElemAdd(bn + "/" + tag[i] + "_bias_add", node_proto, *dst[i], c); + } + return; + } + + // Fallback: H unknown at import (Q is an in-graph projection) or bias isn't constant, so + // it's read at runtime instead; the runtime Slice2 bounds below make the net shape-dynamic + // from here on (see the static path above). + LayerParams shapeLp; + shapeLp.name = bn + "/q_hidden"; + shapeLp.type = "Shape"; + shapeLp.set("start", -1); + Arg h = net.getArg(shapeLp.name); + node_inputs = {q}; + node_outputs = {h}; + addLayer(shapeLp, node_proto, 1); + + Arg h2 = addElemAdd(bn + "/q_hidden_x2", node_proto, h, h); + Arg zero = netimpl->newConstArg(bn + "/bias_begin", Mat(std::vector{1}, CV_64S, Scalar(0))); + Arg end = netimpl->newConstArg(bn + "/bias_end", Mat(std::vector{1}, CV_64S, Scalar(INT_MAX))); + + const Arg bounds[3][2] = {{zero, h}, {h, h2}, {h2, end}}; + Arg* dst[3] = {&q, &k, &v}; + const char* tag[3] = {"q", "k", "v"}; + for (int i = 0; i < 3; i++) { + LayerParams sliceLp; + sliceLp.name = bn + "/" + tag[i] + "_bias"; + sliceLp.type = "Slice2"; + Arg b = net.getArg(sliceLp.name); + node_inputs = {bias, bounds[i][0], bounds[i][1]}; + node_outputs = {b}; + addLayer(sliceLp, node_proto, 3); + *dst[i] = addElemAdd(bn + "/" + tag[i] + "_bias_add", node_proto, *dst[i], b); + } +} + +// key_padding_mask (0 = pad, 1 = keep), (batch, kv_seq) form only, lowered to an additive +// (batch, 1, 1, kv_seq) attn_mask so its batch axis is not read as the query axis. +Arg ONNXImporter2::addKeyPaddingMaskBias(const std::string& bn, + const opencv_onnx::NodeProto& node_proto, + Arg key_padding_mask, Arg like, float mask_filter_value) { + // Rank is only known here for shaped graph inputs. The 1D forms (batch_size) / + // (3*batch_size+2) hold key lengths, not 0/1 flags; unsqueezed to (B,1,1) they become a + // per-row additive constant that softmax is invariant to, so masking would silently do + // nothing instead of failing - reject a rank-unknown mask at runtime instead of guessing. + const MatShape& mShape = netimpl->args.at(key_padding_mask.idx).shape; + if (mShape.dims >= 0) { + CV_CheckEQ(mShape.dims, 2, + "MultiHeadAttention: only a (batch_size, kv_sequence_length) key_padding_mask " + "is supported"); + } else { + // Reshape2 with a constant {0, 0} spec throws at runtime on a rank-1 input (MatShape:: + // reshape: "0 refers to an axis the source does not have") and passes a 2D mask through + // unchanged, without needing the rank at import time. + LayerParams rankLp; + rankLp.name = bn + "/key_padding_mask_2d"; + rankLp.type = "Reshape2"; + const int spec[2] = {0, 0}; + rankLp.set("shape", DictValue::arrayInt(spec, 2)); + Arg mask_2d = net.getArg(rankLp.name); + node_inputs = {key_padding_mask}; + node_outputs = {mask_2d}; + addLayer(rankLp, node_proto, 1); + key_padding_mask = mask_2d; + } + + // Two-input Cast2 (CastLike): the output follows `like`'s (query's) runtime type, so the + // mask stays CV_16F under fp16 inference instead of being hardcoded to CV_32F. + LayerParams castLp; + castLp.name = bn + "/key_padding_mask_cast"; + castLp.type = "Cast2"; + Arg mask_cast = net.getArg(castLp.name); + node_inputs = {key_padding_mask, like}; + node_outputs = {mask_cast}; + addLayer(castLp, node_proto, 2); + + // mask*(-mfv) + mfv: 1 -> 0, 0 -> mask_filter_value. + LayerParams powLp; + powLp.name = bn + "/key_padding_mask_additive"; + powLp.type = "Power"; + powLp.set("power", 1.f); + powLp.set("scale", -mask_filter_value); + powLp.set("shift", mask_filter_value); + Arg mask_additive = net.getArg(powLp.name); + node_inputs = {mask_cast}; + node_outputs = {mask_additive}; + addLayer(powLp, node_proto, 1); + + LayerParams unsqLp; + unsqLp.name = bn + "/key_padding_mask_4d"; + unsqLp.type = "Unsqueeze"; + std::vector axes{1, 2}; + unsqLp.set("axes", DictValue::arrayInt(axes.data(), axes.size())); + Arg mask_4d = net.getArg(unsqLp.name); + node_inputs = {mask_additive}; + node_outputs = {mask_4d}; + addLayer(unsqLp, node_proto, 1); + + return mask_4d; +} + +// Operator spec: https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.MultiHeadAttention +// Supported opsets: com.microsoft 1. Inputs 0..7 (query..past_value); key_padding_mask only +// in (batch_size, kv_sequence_length) form. Lowers to AttentionOnnxAi. void ONNXImporter2::parseMultiHeadAttention(LayerParams& params, const opencv_onnx::NodeProto& node_proto) { CV_CheckTrue(params.has("num_heads"), "MultiHeadAttention: num_heads is required"); CV_CheckTrue(hasInput(node_proto, 1) && hasInput(node_proto, 2), "MultiHeadAttention: separate key and value are required"); - CV_CheckFalse(hasInput(node_proto, 3) || hasInput(node_proto, 4), - "MultiHeadAttention: bias / key_padding_mask inputs are not supported"); for (int i = 8; i < node_proto.input_size(); i++) // past_sequence_length / cache_indirection CV_CheckFalse(hasInput(node_proto, i), - "MultiHeadAttention: only query/key/value/attention_bias/past_key/past_value are supported"); + "MultiHeadAttention: only query/key/value/bias/key_padding_mask/attention_bias/" + "past_key/past_value are supported"); // inputs: query, key, value, bias, key_padding_mask, attention_bias, past_key, past_value, ... const bool has_past_k = hasInput(node_proto, 6), has_past_v = hasInput(node_proto, 7); CV_CheckTrue(has_past_k == has_past_v, "MultiHeadAttention: past_key and past_value must be provided as a pair"); - std::vector ins{node_inputs[0], node_inputs[1], node_inputs[2]}; - if (hasInput(node_proto, 5)) ins.push_back(node_inputs[5]); // attention_bias -> mask + // The helpers below call addLayer() themselves, which overwrites node_inputs/node_outputs; + // save and restore them for AttentionOnnxAi's own addLayer() call at the end. + const std::vector orig_node_inputs = node_inputs, orig_node_outputs = node_outputs; + const std::string bn = params.name; + Arg q = orig_node_inputs[0], k = orig_node_inputs[1], v = orig_node_inputs[2]; + + // bias is part of the input projection, not the attention math, so it's split and added + // into Q/K/V before they reach AttentionOnnxAi. + if (hasInput(node_proto, 3)) + addMultiHeadAttentionBias(bn, node_proto, orig_node_inputs[3], q, k, v); + + Arg mask; + bool has_mask = false; + if (hasInput(node_proto, 4)) { + const float mask_filter_value = params.get("mask_filter_value", -10000.f); + mask = addKeyPaddingMaskBias(bn, node_proto, orig_node_inputs[4], q, mask_filter_value); + has_mask = true; + } + if (hasInput(node_proto, 5)) { + // attention_bias is already additive; combine it with the key_padding_mask bias when + // both are present instead of dropping one on the floor. + mask = has_mask ? addElemAdd(bn + "/mask_add", node_proto, mask, orig_node_inputs[5]) + : orig_node_inputs[5]; + has_mask = true; + } + + std::vector ins{q, k, v}; + if (has_mask) ins.push_back(mask); // attn_mask if (has_past_k) { // The past length is read back off past_key's shape. A fixed seq dim means a shared-buffer // cache whose real length lives in past_sequence_length, so the unwritten tail would be // taken as history. - const MatShape& pk = netimpl->args.at(node_inputs[6].idx).shape; + const MatShape& pk = netimpl->args.at(orig_node_inputs[6].idx).shape; if (pk.dims >= 2) CV_CheckLE(pk[pk.dims - 2], 0, "MultiHeadAttention: past_key has a fixed sequence length (shared-buffer / static " "cache export); only dynamic-cache exports are supported"); - ins.push_back(node_inputs[6]); ins.push_back(node_inputs[7]); + ins.push_back(orig_node_inputs[6]); ins.push_back(orig_node_inputs[7]); } node_inputs = ins; + node_outputs = orig_node_outputs; params.type = "AttentionOnnxAi"; params.set("q_num_heads", params.get("num_heads")); // MHA is not grouped params.set("kv_num_heads", params.get("num_heads")); + params.set("has_attn_mask", has_mask ? 1 : 0); + params.set("has_past", has_past_k ? 1 : 0); if (params.get("unidirectional", 0)) params.set("is_causal", true); diff --git a/modules/dnn/test/test_layer_group_query_attention.cpp b/modules/dnn/test/test_layer_group_query_attention.cpp index f80be52e27..d2f5a77edb 100644 --- a/modules/dnn/test/test_layer_group_query_attention.cpp +++ b/modules/dnn/test/test_layer_group_query_attention.cpp @@ -182,4 +182,57 @@ TEST(GroupQueryAttentionLayer, Guard_RejectsSeqlensSmallerThanQuery) EXPECT_ANY_THROW(net.forward()); } +// ---- guard on the shared attn_mask broadcast check (AttentionOnnxAi::getMemoryShapes) ---- +// Same pattern as the GQA guards above: this check is shared by every AttentionOnnxAi caller +// (ONNX Attention, GroupQueryAttention, MultiHeadAttention), so it's tested directly rather +// than only through one importer's models. + +static const int ATTN_NUM_HEADS = 2, ATTN_HEAD_SIZE = 8; + +static Net makeMaskedAttentionNet() +{ + LayerParams lp; + lp.type = "AttentionOnnxAi"; + lp.name = "attn"; + lp.set("q_num_heads", ATTN_NUM_HEADS); + lp.set("kv_num_heads", ATTN_NUM_HEADS); + lp.set("has_attn_mask", 1); + lp.set("has_past", 0); + + Net net; + int id = net.addLayerToPrev(lp.name, lp.type, lp); + for (int i = 0; i < 4; ++i) + net.connect(0, i, id, i); + net.setInputsNames(std::vector{"query", "key", "value", "attn_mask"}); + return net; +} + +static void feedMaskedAttentionNet(Net& net, int S, const std::vector& maskShape) +{ + const int B = 1, D = ATTN_HEAD_SIZE; + net.setInput(Mat(std::vector{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "query"); + net.setInput(Mat(std::vector{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "key"); + net.setInput(Mat(std::vector{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "value"); + net.setInput(Mat(maskShape, CV_32F, Scalar(0.f)), "attn_mask"); +} + +// Control: an over-strict guard would fail here rather than in the rejection test below. +TEST(AttentionOnnxAiLayer, Guard_AcceptsBroadcastableMask) +{ + const int S = 4, B = 1; + Net net = makeMaskedAttentionNet(); + feedMaskedAttentionNet(net, S, {B, 1, 1, S}); + EXPECT_NO_THROW(net.forward()); +} + +// A last dim that's neither 1 nor total_kv_seq is a legal-but-unsupported ONNX form (a +// last dim shorter than total_kv_seq, meant to be -inf padded) - see attention_onnxai_layer.cpp. +TEST(AttentionOnnxAiLayer, Guard_RejectsNonBroadcastableMask) +{ + const int S = 4, B = 1; + Net net = makeMaskedAttentionNet(); + feedMaskedAttentionNet(net, S, {B, 1, 1, S - 1}); + EXPECT_ANY_THROW(net.forward()); +} + }} // namespace opencv_test::(anonymous) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index c197dd0e75..7fc8b2ae6d 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -3534,6 +3534,41 @@ TEST_P(Test_ONNX_layers, AttentionSharedShapeReshape) { testONNXModels("attention_shared_shape_reshape", npy, 1e-4, 5e-4); } +// MultiHeadAttention `bias`/`key_padding_mask`, alone and together; BiasProjected feeds +// Q/K/V from in-graph MatMul projections (the real HF-export layout), not graph inputs. +TEST_P(Test_ONNX_layers, MultiHeadAttention_Bias) { + testONNXModels("multi_head_attention_bias", npy, 1e-4, 1e-3, false, true, 4); +} +TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMask) { + testONNXModels("multi_head_attention_key_padding_mask", npy, 1e-4, 1e-3, false, true, 4); +} +TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasAndKeyPaddingMask) { + testONNXModels("multi_head_attention_bias_key_padding_mask", npy, 1e-4, 1e-3, false, true, 5); +} +TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasProjected) { + testONNXModels("multi_head_attention_bias_projected", npy, 1e-4, 1e-3, false, true, 3); +} +// Q/K/V cast to fp16 inside the graph; key_padding_mask's own cast must follow, not stay +// hardcoded float32. Fixed fp16-scale tolerances, since this runs fp16 math on every target. +TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskFp16) { + testONNXModels("multi_head_attention_key_padding_mask_fp16", npy, 1e-3, 5e-3, false, true, 4); +} +// Bias as a true ONNX initializer: the static bias-split path should fire, keeping the net +// shape-inferable (no runtime Shape/Slice2) - see addMultiHeadAttentionBias. +TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasConst) { + testONNXModels("multi_head_attention_bias_const", npy, 1e-4, 1e-3, false, true, 3); +} +// key_padding_mask arrives as int64 and is Cast to int32 in-graph, so its rank is unknown at +// import (dims == -1) - the case the importer's rank check used to reject outright. +TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskInGraph) { + testONNXModels("multi_head_attention_key_padding_mask_in_graph", npy, 1e-4, 1e-3, false, true, 4); +} +// key_padding_mask and attention_bias both set: the importer must sum the two additive masks +// instead of one silently overriding the other - see the mask_add branch in parseMultiHeadAttention. +TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskAndAttentionBias) { + testONNXModels("multi_head_attention_key_padding_mask_attention_bias", npy, 1e-4, 1e-3, false, true, 5); +} + // Batch + multi-head, square attention (B=2, H=4, S_q=S_kv=16, D=D_v=32). TEST_P(Test_ONNX_layers, SDPA_MultiHead) { testSDPAModel("sdpa_multi_head", 1e-4, 5e-4); @@ -3827,4 +3862,39 @@ TEST(Test_ONNX_importer, parse_resize_too_few_inputs) EXPECT_TRUE(net.empty()); } +// bias/key_padding_mask must still lower to one AttentionOnnxAi, never the standalone type, +// across every model this PR adds - not just the bias+mask one. +typedef testing::TestWithParam Test_ONNX_MHA_Lowering; +TEST_P(Test_ONNX_MHA_Lowering, lowers_to_one_layer) +{ + const std::string basename = GetParam(); + SCOPED_TRACE(basename); + + Net net = readNetFromONNX(_tf("models/" + basename + ".onnx")); + ASSERT_FALSE(net.empty()); + + int unified = 0; + const std::vector names = net.getLayerNames(); + for (size_t i = 0; i < names.size(); ++i) + { + const String type = net.getLayer(net.getLayerId(names[i]))->type; + if (type == "AttentionOnnxAi") + unified++; + else + EXPECT_NE(type, "MultiHeadAttention"); + } + EXPECT_EQ(unified, 1); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_MHA_Lowering, testing::Values( + "multi_head_attention_bias", + "multi_head_attention_key_padding_mask", + "multi_head_attention_bias_key_padding_mask", + "multi_head_attention_bias_projected", + "multi_head_attention_key_padding_mask_fp16", + "multi_head_attention_bias_const", + "multi_head_attention_key_padding_mask_in_graph", + "multi_head_attention_key_padding_mask_attention_bias" +)); + }} // namespace