Merge pull request #30311 from sanjayk9:dnn-mha-bias-keypadmask

dnn(onnx): support bias and key_padding_mask in com.microsoft MultiHeadAttention
This commit is contained in:
Alexander Smorkalov
2026-10-19 17:43:46 +03:00
committed by GitHub
4 changed files with 318 additions and 8 deletions

View File

@@ -236,6 +236,21 @@ class AttentionOnnxAiLayerImpl CV_FINAL : public AttentionOnnxAiLayer {
const int nhq = input_dims == 4 ? inputs[0][1] : q_num_heads;
const int total_seq_kv = seq_len_k;
// The kernel broadcasts attn_mask right-aligned with no bounds checks of its own, so the
// ONNX-legal form with a last dim shorter than total_kv_seq (-inf padded) is rejected
// here as unsupported, not as a malformed model.
if (map.mask >= 0) {
const MatShape& ms = inputs[map.mask];
CV_CheckLE(ms.dims, 4, "attn_mask must have at most 4 dims");
const int full[4] = {batch_size, nhq, seq_len_q, total_seq_kv};
for (int i = 0; i < ms.dims; i++) {
const int d = ms[i], f = full[4 - ms.dims + i];
CV_Check(d, d == 1 || d == f,
"attn_mask must broadcast to [batch, q_num_heads, q_seq, total_kv_seq]; "
"a last dim shorter than total_kv_seq is not supported");
}
}
CV_CheckTrue(q_hn % kv_hn == 0,
"q_num_heads must be divisible by kv_num_heads");

View File

@@ -312,6 +312,13 @@ protected:
const std::vector<Arg>& scale_zp, const std::vector<Arg>& outputs,
const opencv_onnx::NodeProto& node_proto, int output_onnx_dtype = -1);
Arg addElemAdd(const std::string& name, const opencv_onnx::NodeProto& node_proto,
Arg a, Arg b);
void addMultiHeadAttentionBias(const std::string& bn, const opencv_onnx::NodeProto& node_proto,
Arg bias, Arg& q, Arg& k, Arg& v);
Arg addKeyPaddingMaskBias(const std::string& bn, const opencv_onnx::NodeProto& node_proto,
Arg key_padding_mask, Arg like, float mask_filter_value);
std::map<std::string, int> onnx_opset_map; // map from OperatorSetIdProto
void parseOperatorSet();
@@ -2926,40 +2933,205 @@ static bool hasOutput(const opencv_onnx::NodeProto& node, int i) {
return i < node.output_size() && !node.output(i).empty();
}
// com.microsoft MultiHeadAttention (unpacked Q/K/V) -> AttentionOnnxAi.
Arg ONNXImporter2::addElemAdd(const std::string& name, const opencv_onnx::NodeProto& node_proto,
Arg a, Arg b) {
LayerParams lp;
lp.name = name;
lp.type = "NaryEltwise";
lp.set("operation", "add");
Arg out = net.getArg(name);
node_inputs = {a, b};
node_outputs = {out};
addLayer(lp, node_proto, 2);
return out;
}
// bias = [q(H), k(H), v(rest)]; H = query's hidden size, read at runtime (Shape layer) since
// Q/K/V are usually in-graph projections, not graph inputs with a known shape at import time.
void ONNXImporter2::addMultiHeadAttentionBias(const std::string& bn,
const opencv_onnx::NodeProto& node_proto,
Arg bias, Arg& q, Arg& k, Arg& v) {
const MatShape& qs = netimpl->args.at(q.idx).shape;
const int H = qs.dims >= 1 ? qs[qs.dims - 1] : -1;
if (H > 0 && net.isConstArg(bias)) {
// Static split: no Shape/Slice2 at runtime, so the net stays shape-inferable past
// this node (a runtime Slice2 would make tryInferGraphShapes give up here on, costing
// getLayerShapes()/getFLOPS() every layer downstream - see the fallback below).
const Mat b = net.argTensor(bias);
const int L = (int)b.total();
CV_CheckGT(L, 2 * H, "MultiHeadAttention: bias is shorter than 2*hidden_size");
const int beg[3] = {0, H, 2 * H}, len[3] = {H, H, L - 2 * H};
Arg* dst[3] = {&q, &k, &v};
const char* tag[3] = {"q", "k", "v"};
const Mat b1d = b.reshape(1, std::vector<int>{L});
for (int i = 0; i < 3; i++) {
const Mat part = Mat(b1d, std::vector<Range>{Range(beg[i], beg[i] + len[i])}).clone();
Arg c = netimpl->newConstArg(bn + "/" + tag[i] + "_bias", part);
*dst[i] = addElemAdd(bn + "/" + tag[i] + "_bias_add", node_proto, *dst[i], c);
}
return;
}
// Fallback: H unknown at import (Q is an in-graph projection) or bias isn't constant, so
// it's read at runtime instead; the runtime Slice2 bounds below make the net shape-dynamic
// from here on (see the static path above).
LayerParams shapeLp;
shapeLp.name = bn + "/q_hidden";
shapeLp.type = "Shape";
shapeLp.set("start", -1);
Arg h = net.getArg(shapeLp.name);
node_inputs = {q};
node_outputs = {h};
addLayer(shapeLp, node_proto, 1);
Arg h2 = addElemAdd(bn + "/q_hidden_x2", node_proto, h, h);
Arg zero = netimpl->newConstArg(bn + "/bias_begin", Mat(std::vector<int>{1}, CV_64S, Scalar(0)));
Arg end = netimpl->newConstArg(bn + "/bias_end", Mat(std::vector<int>{1}, CV_64S, Scalar(INT_MAX)));
const Arg bounds[3][2] = {{zero, h}, {h, h2}, {h2, end}};
Arg* dst[3] = {&q, &k, &v};
const char* tag[3] = {"q", "k", "v"};
for (int i = 0; i < 3; i++) {
LayerParams sliceLp;
sliceLp.name = bn + "/" + tag[i] + "_bias";
sliceLp.type = "Slice2";
Arg b = net.getArg(sliceLp.name);
node_inputs = {bias, bounds[i][0], bounds[i][1]};
node_outputs = {b};
addLayer(sliceLp, node_proto, 3);
*dst[i] = addElemAdd(bn + "/" + tag[i] + "_bias_add", node_proto, *dst[i], b);
}
}
// key_padding_mask (0 = pad, 1 = keep), (batch, kv_seq) form only, lowered to an additive
// (batch, 1, 1, kv_seq) attn_mask so its batch axis is not read as the query axis.
Arg ONNXImporter2::addKeyPaddingMaskBias(const std::string& bn,
const opencv_onnx::NodeProto& node_proto,
Arg key_padding_mask, Arg like, float mask_filter_value) {
// Rank is only known here for shaped graph inputs. The 1D forms (batch_size) /
// (3*batch_size+2) hold key lengths, not 0/1 flags; unsqueezed to (B,1,1) they become a
// per-row additive constant that softmax is invariant to, so masking would silently do
// nothing instead of failing - reject a rank-unknown mask at runtime instead of guessing.
const MatShape& mShape = netimpl->args.at(key_padding_mask.idx).shape;
if (mShape.dims >= 0) {
CV_CheckEQ(mShape.dims, 2,
"MultiHeadAttention: only a (batch_size, kv_sequence_length) key_padding_mask "
"is supported");
} else {
// Reshape2 with a constant {0, 0} spec throws at runtime on a rank-1 input (MatShape::
// reshape: "0 refers to an axis the source does not have") and passes a 2D mask through
// unchanged, without needing the rank at import time.
LayerParams rankLp;
rankLp.name = bn + "/key_padding_mask_2d";
rankLp.type = "Reshape2";
const int spec[2] = {0, 0};
rankLp.set("shape", DictValue::arrayInt(spec, 2));
Arg mask_2d = net.getArg(rankLp.name);
node_inputs = {key_padding_mask};
node_outputs = {mask_2d};
addLayer(rankLp, node_proto, 1);
key_padding_mask = mask_2d;
}
// Two-input Cast2 (CastLike): the output follows `like`'s (query's) runtime type, so the
// mask stays CV_16F under fp16 inference instead of being hardcoded to CV_32F.
LayerParams castLp;
castLp.name = bn + "/key_padding_mask_cast";
castLp.type = "Cast2";
Arg mask_cast = net.getArg(castLp.name);
node_inputs = {key_padding_mask, like};
node_outputs = {mask_cast};
addLayer(castLp, node_proto, 2);
// mask*(-mfv) + mfv: 1 -> 0, 0 -> mask_filter_value.
LayerParams powLp;
powLp.name = bn + "/key_padding_mask_additive";
powLp.type = "Power";
powLp.set("power", 1.f);
powLp.set("scale", -mask_filter_value);
powLp.set("shift", mask_filter_value);
Arg mask_additive = net.getArg(powLp.name);
node_inputs = {mask_cast};
node_outputs = {mask_additive};
addLayer(powLp, node_proto, 1);
LayerParams unsqLp;
unsqLp.name = bn + "/key_padding_mask_4d";
unsqLp.type = "Unsqueeze";
std::vector<int> axes{1, 2};
unsqLp.set("axes", DictValue::arrayInt(axes.data(), axes.size()));
Arg mask_4d = net.getArg(unsqLp.name);
node_inputs = {mask_additive};
node_outputs = {mask_4d};
addLayer(unsqLp, node_proto, 1);
return mask_4d;
}
// Operator spec: https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.MultiHeadAttention
// Supported opsets: com.microsoft 1. Inputs 0..7 (query..past_value); key_padding_mask only
// in (batch_size, kv_sequence_length) form. Lowers to AttentionOnnxAi.
void ONNXImporter2::parseMultiHeadAttention(LayerParams& params, const opencv_onnx::NodeProto& node_proto) {
CV_CheckTrue(params.has("num_heads"), "MultiHeadAttention: num_heads is required");
CV_CheckTrue(hasInput(node_proto, 1) && hasInput(node_proto, 2),
"MultiHeadAttention: separate key and value are required");
CV_CheckFalse(hasInput(node_proto, 3) || hasInput(node_proto, 4),
"MultiHeadAttention: bias / key_padding_mask inputs are not supported");
for (int i = 8; i < node_proto.input_size(); i++) // past_sequence_length / cache_indirection
CV_CheckFalse(hasInput(node_proto, i),
"MultiHeadAttention: only query/key/value/attention_bias/past_key/past_value are supported");
"MultiHeadAttention: only query/key/value/bias/key_padding_mask/attention_bias/"
"past_key/past_value are supported");
// inputs: query, key, value, bias, key_padding_mask, attention_bias, past_key, past_value, ...
const bool has_past_k = hasInput(node_proto, 6), has_past_v = hasInput(node_proto, 7);
CV_CheckTrue(has_past_k == has_past_v,
"MultiHeadAttention: past_key and past_value must be provided as a pair");
std::vector<Arg> ins{node_inputs[0], node_inputs[1], node_inputs[2]};
if (hasInput(node_proto, 5)) ins.push_back(node_inputs[5]); // attention_bias -> mask
// The helpers below call addLayer() themselves, which overwrites node_inputs/node_outputs;
// save and restore them for AttentionOnnxAi's own addLayer() call at the end.
const std::vector<Arg> orig_node_inputs = node_inputs, orig_node_outputs = node_outputs;
const std::string bn = params.name;
Arg q = orig_node_inputs[0], k = orig_node_inputs[1], v = orig_node_inputs[2];
// bias is part of the input projection, not the attention math, so it's split and added
// into Q/K/V before they reach AttentionOnnxAi.
if (hasInput(node_proto, 3))
addMultiHeadAttentionBias(bn, node_proto, orig_node_inputs[3], q, k, v);
Arg mask;
bool has_mask = false;
if (hasInput(node_proto, 4)) {
const float mask_filter_value = params.get<float>("mask_filter_value", -10000.f);
mask = addKeyPaddingMaskBias(bn, node_proto, orig_node_inputs[4], q, mask_filter_value);
has_mask = true;
}
if (hasInput(node_proto, 5)) {
// attention_bias is already additive; combine it with the key_padding_mask bias when
// both are present instead of dropping one on the floor.
mask = has_mask ? addElemAdd(bn + "/mask_add", node_proto, mask, orig_node_inputs[5])
: orig_node_inputs[5];
has_mask = true;
}
std::vector<Arg> ins{q, k, v};
if (has_mask) ins.push_back(mask); // attn_mask
if (has_past_k) {
// The past length is read back off past_key's shape. A fixed seq dim means a shared-buffer
// cache whose real length lives in past_sequence_length, so the unwritten tail would be
// taken as history.
const MatShape& pk = netimpl->args.at(node_inputs[6].idx).shape;
const MatShape& pk = netimpl->args.at(orig_node_inputs[6].idx).shape;
if (pk.dims >= 2)
CV_CheckLE(pk[pk.dims - 2], 0,
"MultiHeadAttention: past_key has a fixed sequence length (shared-buffer / static "
"cache export); only dynamic-cache exports are supported");
ins.push_back(node_inputs[6]); ins.push_back(node_inputs[7]);
ins.push_back(orig_node_inputs[6]); ins.push_back(orig_node_inputs[7]);
}
node_inputs = ins;
node_outputs = orig_node_outputs;
params.type = "AttentionOnnxAi";
params.set("q_num_heads", params.get<int>("num_heads")); // MHA is not grouped
params.set("kv_num_heads", params.get<int>("num_heads"));
params.set("has_attn_mask", has_mask ? 1 : 0);
params.set("has_past", has_past_k ? 1 : 0);
if (params.get<int>("unidirectional", 0))
params.set("is_causal", true);

View File

@@ -182,4 +182,57 @@ TEST(GroupQueryAttentionLayer, Guard_RejectsSeqlensSmallerThanQuery)
EXPECT_ANY_THROW(net.forward());
}
// ---- guard on the shared attn_mask broadcast check (AttentionOnnxAi::getMemoryShapes) ----
// Same pattern as the GQA guards above: this check is shared by every AttentionOnnxAi caller
// (ONNX Attention, GroupQueryAttention, MultiHeadAttention), so it's tested directly rather
// than only through one importer's models.
static const int ATTN_NUM_HEADS = 2, ATTN_HEAD_SIZE = 8;
static Net makeMaskedAttentionNet()
{
LayerParams lp;
lp.type = "AttentionOnnxAi";
lp.name = "attn";
lp.set("q_num_heads", ATTN_NUM_HEADS);
lp.set("kv_num_heads", ATTN_NUM_HEADS);
lp.set("has_attn_mask", 1);
lp.set("has_past", 0);
Net net;
int id = net.addLayerToPrev(lp.name, lp.type, lp);
for (int i = 0; i < 4; ++i)
net.connect(0, i, id, i);
net.setInputsNames(std::vector<String>{"query", "key", "value", "attn_mask"});
return net;
}
static void feedMaskedAttentionNet(Net& net, int S, const std::vector<int>& maskShape)
{
const int B = 1, D = ATTN_HEAD_SIZE;
net.setInput(Mat(std::vector<int>{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "query");
net.setInput(Mat(std::vector<int>{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "key");
net.setInput(Mat(std::vector<int>{B, S, ATTN_NUM_HEADS * D}, CV_32F, Scalar(0.1f)), "value");
net.setInput(Mat(maskShape, CV_32F, Scalar(0.f)), "attn_mask");
}
// Control: an over-strict guard would fail here rather than in the rejection test below.
TEST(AttentionOnnxAiLayer, Guard_AcceptsBroadcastableMask)
{
const int S = 4, B = 1;
Net net = makeMaskedAttentionNet();
feedMaskedAttentionNet(net, S, {B, 1, 1, S});
EXPECT_NO_THROW(net.forward());
}
// A last dim that's neither 1 nor total_kv_seq is a legal-but-unsupported ONNX form (a
// last dim shorter than total_kv_seq, meant to be -inf padded) - see attention_onnxai_layer.cpp.
TEST(AttentionOnnxAiLayer, Guard_RejectsNonBroadcastableMask)
{
const int S = 4, B = 1;
Net net = makeMaskedAttentionNet();
feedMaskedAttentionNet(net, S, {B, 1, 1, S - 1});
EXPECT_ANY_THROW(net.forward());
}
}} // namespace opencv_test::(anonymous)

View File

@@ -3534,6 +3534,41 @@ TEST_P(Test_ONNX_layers, AttentionSharedShapeReshape) {
testONNXModels("attention_shared_shape_reshape", npy, 1e-4, 5e-4);
}
// MultiHeadAttention `bias`/`key_padding_mask`, alone and together; BiasProjected feeds
// Q/K/V from in-graph MatMul projections (the real HF-export layout), not graph inputs.
TEST_P(Test_ONNX_layers, MultiHeadAttention_Bias) {
testONNXModels("multi_head_attention_bias", npy, 1e-4, 1e-3, false, true, 4);
}
TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMask) {
testONNXModels("multi_head_attention_key_padding_mask", npy, 1e-4, 1e-3, false, true, 4);
}
TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasAndKeyPaddingMask) {
testONNXModels("multi_head_attention_bias_key_padding_mask", npy, 1e-4, 1e-3, false, true, 5);
}
TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasProjected) {
testONNXModels("multi_head_attention_bias_projected", npy, 1e-4, 1e-3, false, true, 3);
}
// Q/K/V cast to fp16 inside the graph; key_padding_mask's own cast must follow, not stay
// hardcoded float32. Fixed fp16-scale tolerances, since this runs fp16 math on every target.
TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskFp16) {
testONNXModels("multi_head_attention_key_padding_mask_fp16", npy, 1e-3, 5e-3, false, true, 4);
}
// Bias as a true ONNX initializer: the static bias-split path should fire, keeping the net
// shape-inferable (no runtime Shape/Slice2) - see addMultiHeadAttentionBias.
TEST_P(Test_ONNX_layers, MultiHeadAttention_BiasConst) {
testONNXModels("multi_head_attention_bias_const", npy, 1e-4, 1e-3, false, true, 3);
}
// key_padding_mask arrives as int64 and is Cast to int32 in-graph, so its rank is unknown at
// import (dims == -1) - the case the importer's rank check used to reject outright.
TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskInGraph) {
testONNXModels("multi_head_attention_key_padding_mask_in_graph", npy, 1e-4, 1e-3, false, true, 4);
}
// key_padding_mask and attention_bias both set: the importer must sum the two additive masks
// instead of one silently overriding the other - see the mask_add branch in parseMultiHeadAttention.
TEST_P(Test_ONNX_layers, MultiHeadAttention_KeyPaddingMaskAndAttentionBias) {
testONNXModels("multi_head_attention_key_padding_mask_attention_bias", npy, 1e-4, 1e-3, false, true, 5);
}
// Batch + multi-head, square attention (B=2, H=4, S_q=S_kv=16, D=D_v=32).
TEST_P(Test_ONNX_layers, SDPA_MultiHead) {
testSDPAModel("sdpa_multi_head", 1e-4, 5e-4);
@@ -3827,4 +3862,39 @@ TEST(Test_ONNX_importer, parse_resize_too_few_inputs)
EXPECT_TRUE(net.empty());
}
// bias/key_padding_mask must still lower to one AttentionOnnxAi, never the standalone type,
// across every model this PR adds - not just the bias+mask one.
typedef testing::TestWithParam<std::string> Test_ONNX_MHA_Lowering;
TEST_P(Test_ONNX_MHA_Lowering, lowers_to_one_layer)
{
const std::string basename = GetParam();
SCOPED_TRACE(basename);
Net net = readNetFromONNX(_tf("models/" + basename + ".onnx"));
ASSERT_FALSE(net.empty());
int unified = 0;
const std::vector<String> names = net.getLayerNames();
for (size_t i = 0; i < names.size(); ++i)
{
const String type = net.getLayer(net.getLayerId(names[i]))->type;
if (type == "AttentionOnnxAi")
unified++;
else
EXPECT_NE(type, "MultiHeadAttention");
}
EXPECT_EQ(unified, 1);
}
INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_MHA_Lowering, testing::Values(
"multi_head_attention_bias",
"multi_head_attention_key_padding_mask",
"multi_head_attention_bias_key_padding_mask",
"multi_head_attention_bias_projected",
"multi_head_attention_key_padding_mask_fp16",
"multi_head_attention_bias_const",
"multi_head_attention_key_padding_mask_in_graph",
"multi_head_attention_key_padding_mask_attention_bias"
));
}} // namespace