diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index c66af459f5..df8b3f4409 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1971,6 +1971,16 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS LinearAttentionLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS FlexAttentionLayer : public Layer { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS GroupNormLayer : public Layer { public: static Ptr create(const LayerParams ¶ms); diff --git a/modules/dnn/misc/python/test/test_onnx_conformance.py b/modules/dnn/misc/python/test/test_onnx_conformance.py index f5ddfd6a93..991f9d3616 100644 --- a/modules/dnn/misc/python/test/test_onnx_conformance.py +++ b/modules/dnn/misc/python/test/test_onnx_conformance.py @@ -27,8 +27,12 @@ TOLERANCE_OVERRIDES = { "test_attention_4d_gqa_with_past_and_present_fp16_expanded": (0.0002, 0.001), "test_causal_conv_with_state_fp16": (0.0002, 0.002), "test_causal_conv_with_state_silu_fp16": (0.0002, 0.002), + "test_flexattention_fp16": (0.0002, 0.001), + "test_flexattention_fp16_expanded_ver26": (0.0002, 0.001), "test_gelu_tanh_1": (0.00011, 0.00016), "test_gelu_tanh_2": (9e-05, 0.0005), + "test_linear_attention_fp16": (0.0002, 0.001), + "test_linear_attention_fp16_expanded": (0.0002, 0.001), "test_nllloss_NCd1d2_reduction_sum_expanded": (2e-05, 0.0001), "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded": (2e-05, 0.0001), "test_reduce_prod_default_axes_keepdims_random": (0.002, 0.002), diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 1880099858..44e8c30883 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -235,6 +235,8 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(SDPA, SDPALayer); CV_DNN_REGISTER_LAYER_CLASS(AttentionOnnxAi, AttentionOnnxAiLayer); CV_DNN_REGISTER_LAYER_CLASS(CausalConvWithState, CausalConvWithStateLayer); + CV_DNN_REGISTER_LAYER_CLASS(LinearAttention, LinearAttentionLayer); + CV_DNN_REGISTER_LAYER_CLASS(FlexAttention, FlexAttentionLayer); CV_DNN_REGISTER_LAYER_CLASS(RotaryEmbedding, RotaryEmbeddingLayer); CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Cast, CastLayer); diff --git a/modules/dnn/src/layers/flex_attention_layer.cpp b/modules/dnn/src/layers/flex_attention_layer.cpp new file mode 100644 index 0000000000..773bab5ec0 --- /dev/null +++ b/modules/dnn/src/layers/flex_attention_layer.cpp @@ -0,0 +1,291 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include "cpu_kernels/fast_gemm.hpp" +#include "cpu_kernels/softmax.hpp" +#include + +#include + +namespace cv { +namespace dnn { + +/* + Implementation of FlexAttention (domain ai.onnx.preview, opset 1). + Spec: https://github.com/onnx/onnx/blob/main/docs/Operators-preview.md#aionnxpreviewFlexAttention + Supported: score_mod / prob_mod sub-graphs (inlined by the importer). + An explicit softmax_precision is rejected; only the spec default is implemented. +*/ +// Scaled dot-product attention with GQA and optional score_mod / prob_mod +// sub-graphs that rewrite the whole [B, H, L, S] score / probability tensor: +// scores = scale * (Q . expand_kv(K)^T); scores = score_mod(scores) +// probs = softmax(scores, axis=-1); probs = prob_mod(probs) +// Y = probs . expand_kv(V) +// The sub-graphs carry no control flow, so the importer inlines them as ordinary +// graph nodes and cuts this op into three stages executed by this layer: +// "full" : Q, K, V -> Y (no sub-graphs) +// "qk" : Q, K -> scores (feeds score_mod / Softmax) +// "av" : probs, V-> Y (consumes prob_mod / Softmax output) + +class FlexAttentionLayerImpl CV_FINAL : public FlexAttentionLayer +{ +public: + FlexAttentionLayerImpl(const LayerParams& params) + { + setParamsFrom(params); + scale = params.get("scale", 0.f); + has_scale = params.has("scale"); + stage = params.get("stage", "full"); + CV_Check(stage, stage == "full" || stage == "qk" || stage == "av", "FlexAttention: bad stage"); + opt.init(); + } + + bool supportBackend(int backendId) CV_OVERRIDE { return backendId == DNN_BACKEND_OPENCV; } + + void getTypes(const std::vector& inputs, const int requiredOutputs, const int requiredInternals, + std::vector& outputs, std::vector& internals) const CV_OVERRIDE + { + CV_CheckGE(inputs.size(), (size_t)2, "FlexAttention needs at least 2 inputs"); + CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_64F || inputs[0] == CV_16F, + "FlexAttention: only FP32/FP64/FP16 are supported"); + outputs.assign(requiredOutputs, inputs[0]); + internals.assign(requiredInternals, CV_32F); // scores buffer is always fp32 + } + + bool getMemoryShapes(const std::vector& inputs, const int /*ro*/, + std::vector& outputs, std::vector& internals) const CV_OVERRIDE + { + CV_CheckEQ(inputs[0].dims, 4, "FlexAttention: inputs must be 4D [batch, heads, seq, head_size]"); + const int B = inputs[0][0], Hq = inputs[0][1], L = inputs[0][2]; + internals.clear(); + if (stage == "qk") // Q[B,Hq,L,D], K[B,Hkv,S,D] -> scores[B,Hq,L,S] + outputs.assign(1, MatShape{B, Hq, L, inputs[1][2]}); + else if (stage == "av") // probs[B,Hq,L,S], V[B,Hkv,S,Dv] -> Y[B,Hq,L,Dv] + outputs.assign(1, MatShape{B, Hq, L, inputs[1][3]}); + else // Q,K,V -> Y[B,Hq,L,Dv] + { + outputs.assign(1, MatShape{B, Hq, L, inputs[2][3]}); + // scores buffer, pooled across forward(); the fp64 path uses its own scratch. + internals.assign(1, MatShape{B, Hq, L, inputs[1][2]}); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + internals_arr.getMatVector(internals); + + if (inputs[0].depth() == CV_64F) + run(inputs, outputs); // scalar path (fastGemmBatch is float-only) + else + runFloat(inputs, outputs, internals, inputs[0].depth() == CV_16F); + } + + int64 getFLOPS(const std::vector& inputs, const std::vector& /*outputs*/) const CV_OVERRIDE + { + const int64 B = inputs[0][0], Hq = inputs[0][1], L = inputs[0][2]; + if (stage == "qk") // Q[B,Hq,L,D] x K[B,Hkv,S,D]^T + return CV_BIG_INT(2) * B * Hq * L * inputs[1][2] * inputs[0][3]; + if (stage == "av") // P[B,Hq,L,S] x V[B,Hkv,S,Dv] + return CV_BIG_INT(2) * B * Hq * L * inputs[0][3] * inputs[1][3]; + const int64 S = inputs[1][2], D = inputs[0][3], Dv = inputs[2][3]; + return CV_BIG_INT(2) * B * Hq * L * S * (D + Dv) + 4 * B * Hq * L * S; // + softmax + } + +private: + // fp32/fp16 path: batched GEMM via fastGemmBatch (MLAS-accelerated when built with + // HAVE_MLAS). fp16 is computed in fp32. K/V are shared across each GQA group through the + // per-head offset arithmetic (n -> n/group), matching the ONNX Attention layer. + void runFloat(std::vector& rawIn, std::vector& rawOut, std::vector& internals, bool fp16) + { + std::vector in, out; + if (fp16) + { + in.resize(rawIn.size()); + for (size_t i = 0; i < rawIn.size(); ++i) + if (!rawIn[i].empty()) rawIn[i].convertTo(in[i], CV_32F); + out.resize(rawOut.size()); + out[0].create(rawOut[0].dims, rawOut[0].size.p, CV_32F); + } + std::vector& I = fp16 ? in : rawIn; + std::vector& O = fp16 ? out : rawOut; + + if (stage == "qk") + qkGemm(I[0], I[1], O[0]); + else if (stage == "av") + avGemm(I[0], I[1], O[0]); + else + { + CV_CheckEQ(internals.size(), (size_t)1, "FlexAttention: missing scores buffer"); + fullFloat(I[0], I[1], I[2], O[0], internals[0]); + } + + if (fp16) O[0].convertTo(rawOut[0], CV_16F); + } + + // scores[B,Hq,L,Skv] = scl * Q * K^T (K^T via ldb0=1,ldb1=D). + void qkGemm(const Mat& Q, const Mat& K, Mat& S) + { + const int B = Q.size[0], Hq = Q.size[1], L = Q.size[2], D = Q.size[3]; + const int Hkv = K.size[1], Skv = K.size[2], group = Hq / Hkv; + const float scl = has_scale ? scale : (float)(1.0 / std::sqrt((double)D)); + // offset tables assume fully packed tensors + CV_Assert(Q.isContinuous() && K.isContinuous() && S.isContinuous()); + + const size_t batch = (size_t)B * Hq; + std::vector qo(batch), ko(batch), so(batch); + for (int b = 0; b < B; ++b) + for (int n = 0; n < Hq; ++n) + { + const size_t bn = (size_t)b * Hq + n; + qo[bn] = ((size_t)b * Hq + n) * L * D; + ko[bn] = ((size_t)b * Hkv + n / group) * Skv * D; + so[bn] = ((size_t)b * Hq + n) * L * Skv; + } + fastGemmBatch(batch, qo.data(), ko.data(), so.data(), + L, Skv, D, scl, Q.ptr(), D, 1, K.ptr(), 1, D, + 0.f, S.ptr(), Skv, opt); + } + + // Y[B,Hq,L,Dv] = P * V. + void avGemm(const Mat& P, const Mat& V, Mat& Y) + { + const int B = P.size[0], Hq = P.size[1], L = P.size[2], Skv = P.size[3]; + const int Hkv = V.size[1], Dv = V.size[3], group = Hq / Hkv; + CV_Assert(P.isContinuous() && V.isContinuous() && Y.isContinuous()); + + const size_t batch = (size_t)B * Hq; + std::vector po(batch), vo(batch), yo(batch); + for (int b = 0; b < B; ++b) + for (int n = 0; n < Hq; ++n) + { + const size_t bn = (size_t)b * Hq + n; + po[bn] = ((size_t)b * Hq + n) * L * Skv; + vo[bn] = ((size_t)b * Hkv + n / group) * Skv * Dv; + yo[bn] = ((size_t)b * Hq + n) * L * Dv; + } + fastGemmBatch(batch, po.data(), vo.data(), yo.data(), + L, Dv, Skv, 1.f, P.ptr(), Skv, 1, V.ptr(), Dv, 1, + 0.f, Y.ptr(), Dv, opt); + } + + void fullFloat(const Mat& Q, const Mat& K, const Mat& V, Mat& Y, Mat& scores) + { + qkGemm(Q, K, scores); + softmax(scores, scores, 3); // in-place, last axis + avGemm(scores, V, Y); + } + + // ---- CV_64F reference path (scalar templates; fastGemmBatch is float-only) ---- + template + void run(std::vector& I, std::vector& O) + { + if (stage == "qk") + qk(I[0], I[1], O[0]); + else if (stage == "av") + av(I[0], I[1], O[0]); + else + full(I[0], I[1], I[2], O[0]); + } + + // scores[b,n,l,s] = scale * sum_d Q[b,n,l,d] * K[b, n/group, s, d] + template + void qk(const Mat& Q, const Mat& K, Mat& S) + { + const int B = Q.size[0], Hq = Q.size[1], L = Q.size[2], D = Q.size[3]; + const int Hkv = K.size[1], Skv = K.size[2], group = Hq / Hkv; + const T scl = static_cast(has_scale ? scale : 1.0 / std::sqrt((double)D)); + parallel_for_(Range(0, B * Hq), [&](const Range& r) { + for (int bn = r.start; bn < r.end; ++bn) { + const int b = bn / Hq, n = bn % Hq, h = n / group; + const T* q = Q.ptr() + ((size_t)b * Hq + n) * L * D; + const T* k = K.ptr() + ((size_t)b * Hkv + h) * Skv * D; + T* s = S.ptr() + ((size_t)b * Hq + n) * L * Skv; + for (int l = 0; l < L; ++l) + for (int j = 0; j < Skv; ++j) { + T acc = 0; + for (int d = 0; d < D; ++d) acc += q[l * D + d] * k[j * D + d]; + s[l * Skv + j] = acc * scl; + } + } + }); + } + + // Y[b,n,l,:] = sum_s P[b,n,l,s] * V[b, n/group, s, :] + template + void av(const Mat& P, const Mat& V, Mat& Y) + { + const int B = P.size[0], Hq = P.size[1], L = P.size[2], Skv = P.size[3]; + const int Hkv = V.size[1], Dv = V.size[3], group = Hq / Hkv; + parallel_for_(Range(0, B * Hq), [&](const Range& r) { + for (int bn = r.start; bn < r.end; ++bn) { + const int b = bn / Hq, n = bn % Hq, h = n / group; + const T* p = P.ptr() + ((size_t)b * Hq + n) * L * Skv; + const T* v = V.ptr() + ((size_t)b * Hkv + h) * Skv * Dv; + T* y = Y.ptr() + ((size_t)b * Hq + n) * L * Dv; + for (int l = 0; l < L; ++l) { + for (int c = 0; c < Dv; ++c) y[l * Dv + c] = 0; + for (int j = 0; j < Skv; ++j) { + const T pw = p[l * Skv + j]; + for (int c = 0; c < Dv; ++c) y[l * Dv + c] += pw * v[j * Dv + c]; + } + } + } + }); + } + + template + void full(const Mat& Q, const Mat& K, const Mat& V, Mat& Y) + { + const int B = Q.size[0], Hq = Q.size[1], L = Q.size[2], D = Q.size[3]; + const int Hkv = K.size[1], Skv = K.size[2], Dv = V.size[3], group = Hq / Hkv; + const T scl = static_cast(has_scale ? scale : 1.0 / std::sqrt((double)D)); + parallel_for_(Range(0, B * Hq), [&](const Range& r) { + std::vector s(Skv); + for (int bn = r.start; bn < r.end; ++bn) { + const int b = bn / Hq, n = bn % Hq, h = n / group; + const T* q = Q.ptr() + ((size_t)b * Hq + n) * L * D; + const T* k = K.ptr() + ((size_t)b * Hkv + h) * Skv * D; + const T* v = V.ptr() + ((size_t)b * Hkv + h) * Skv * Dv; + T* y = Y.ptr() + ((size_t)b * Hq + n) * L * Dv; + for (int l = 0; l < L; ++l) { + T mx = -std::numeric_limits::infinity(); + for (int j = 0; j < Skv; ++j) { + T acc = 0; + for (int d = 0; d < D; ++d) acc += q[l * D + d] * k[j * D + d]; + s[j] = acc * scl; + mx = std::max(mx, s[j]); + } + T sum = 0; + for (int j = 0; j < Skv; ++j) { s[j] = std::exp(s[j] - mx); sum += s[j]; } + const T inv = sum > 0 ? (T)1 / sum : 0; + for (int c = 0; c < Dv; ++c) y[l * Dv + c] = 0; + for (int j = 0; j < Skv; ++j) { + const T pw = s[j] * inv; + for (int c = 0; c < Dv; ++c) y[l * Dv + c] += pw * v[j * Dv + c]; + } + } + } + }); + } + + float scale = 0.f; + bool has_scale = false; + std::string stage; + FastGemmOpt opt; +}; + +Ptr FlexAttentionLayer::create(const LayerParams& params) +{ + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/layers/linear_attention_layer.cpp b/modules/dnn/src/layers/linear_attention_layer.cpp new file mode 100644 index 0000000000..b7f39d4d90 --- /dev/null +++ b/modules/dnn/src/layers/linear_attention_layer.cpp @@ -0,0 +1,288 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2026, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +#include + +namespace cv { +namespace dnn { + +/* + Implementation of LinearAttention, as defined in ONNX specification: + https://onnx.ai/onnx/operators/onnx__LinearAttention.html + + Opset 27 is covered. +*/ +// Recurrent (O(T)) attention over a fixed-size state S of shape [Dk, Dv], maintained +// per (batch, kv-head). For each time step t the state is updated and read out: +// linear : S += k_t (x) v_t +// delta : S += k_t (x) (beta_t * (v_t - S^T k_t)) +// gated : S = S * exp(decay_t); S += k_t (x) v_t +// gated_delta : S = S * exp(decay_t); S += k_t (x) (beta_t * (v_t - S^T k_t)) (default) +// out_t = scale * (q_t . S) +// GQA/MQA: q has q_num_heads, the state has kv_num_heads; the state is shared across +// the q_num_heads / kv_num_heads queries in each group. +// chunk_size is a prefill tuning hint only; ignored. + +class LinearAttentionLayerImpl CV_FINAL : public LinearAttentionLayer +{ +public: + LinearAttentionLayerImpl(const LayerParams& params) + { + setParamsFrom(params); + q_num_heads = params.get("q_num_heads", 0); + kv_num_heads = params.get("kv_num_heads", 0); + scale = params.get("scale", 0.f); + // Spec: scale 0.0 means "derive 1/sqrt(d_k)", so test the value, not presence. + has_scale = (scale != 0.f); + update_rule = params.get("update_rule", "gated_delta"); + CV_Check(update_rule, update_rule == "linear" || update_rule == "gated" || + update_rule == "delta" || update_rule == "gated_delta", + "LinearAttention: unknown update_rule"); + CV_CheckGT(q_num_heads, 0, "LinearAttention: q_num_heads is required"); + CV_CheckGT(kv_num_heads, 0, "LinearAttention: kv_num_heads is required"); + CV_CheckEQ(q_num_heads % kv_num_heads, 0, "LinearAttention: q_num_heads must be divisible by kv_num_heads"); + use_decay = (update_rule == "gated" || update_rule == "gated_delta"); + use_delta = (update_rule == "delta" || update_rule == "gated_delta"); + } + + bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + // Optional inputs arrive as empty Mats in fixed slots: 3=past_state, 4=decay, 5=beta. + static bool present(const std::vector& in, size_t i) { return in.size() > i && in[i].dims > 0; } + static bool present(const std::vector& in, size_t i) { return in.size() > i && !in[i].empty(); } + + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_CheckGE(inputs.size(), (size_t)3, "LinearAttention needs query, key, value"); + CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_16F, "LinearAttention: only FP32/FP16 are supported"); + outputs.assign(requiredOutputs, inputs[0]); + internals.assign(requiredInternals, CV_32F); // recurrence-state scratch is always fp32 + } + + // requiredOutputs can be 1 (trimmed graph); state then lives in internals. + bool getMemoryShapes(const std::vector& inputs, + const int requiredOutputs, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + CV_CheckGE(inputs.size(), (size_t)3, "LinearAttention needs query, key, value"); + CV_CheckEQ(inputs[0].dims, 3, "LinearAttention: query must be 3D [batch, seq, q_num_heads*head_size]"); + CV_Check(requiredOutputs, requiredOutputs == 1 || requiredOutputs == 2, + "LinearAttention: expects 1 (output) or 2 (output, present_state) outputs"); + + const int batch = inputs[0][0]; + const int seq = inputs[0][1]; + const int dk = inputs[0][2] / q_num_heads; + const int dv = inputs[2][2] / kv_num_heads; + CV_CheckEQ(inputs[1][2] / kv_num_heads, dk, "LinearAttention: key head_size must equal query head_size"); + + const MatShape stateShape{batch, kv_num_heads, dk, dv}; + outputs.assign(1, MatShape{batch, seq, q_num_heads * dv}); // output + internals.clear(); + if (requiredOutputs >= 2) + outputs.push_back(stateShape); // present_state + else + internals.assign(1, stateShape); // recurrence scratch, not exposed + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + std::vector rawInputs, rawOutputs, internals; + inputs_arr.getMatVector(rawInputs); + outputs_arr.getMatVector(rawOutputs); + internals_arr.getMatVector(internals); + const bool has_state_output = rawOutputs.size() >= 2; // else present_state lives in internals[0] + + // past_state/present_state use an independent ONNX dtype from Q/K/V; convert per-tensor. + std::vector in32(rawInputs.size()), out32(rawOutputs.size()); + for (size_t i = 0; i < rawInputs.size(); ++i) + { + if (rawInputs[i].empty()) continue; + if (rawInputs[i].depth() == CV_16F) rawInputs[i].convertTo(in32[i], CV_32F); + else in32[i] = rawInputs[i]; + } + for (size_t i = 0; i < rawOutputs.size(); ++i) + { + if (rawOutputs[i].depth() == CV_16F) out32[i].create(rawOutputs[i].dims, rawOutputs[i].size.p, CV_32F); + else out32[i] = rawOutputs[i]; + } + std::vector& inputs = in32; + std::vector& outputs = out32; + + const Mat& query = inputs[0]; + const Mat& key = inputs[1]; + const Mat& value = inputs[2]; + const bool has_past = present(inputs, 3); + const bool has_decay = present(inputs, 4) && use_decay; + const bool has_beta = present(inputs, 5) && use_delta; + + const int batch = query.size[0]; + const int seq = query.size[1]; + const int Hq = q_num_heads; + const int Hkv = kv_num_heads; + const int group = Hq / Hkv; + const int Dk = query.size[2] / Hq; + const int Dv = value.size[2] / Hkv; + + const float scl = has_scale ? scale : 1.0f / std::sqrt(static_cast(Dk)); + + // decay: per-kv-head vector of length Dk, or a single value broadcast over Dk (per_head_decay). + const int decayDim = has_decay ? inputs[4].size[2] / Hkv : 0; + // beta: one scalar per kv-head, or one scalar broadcast over all heads. + const bool betaPerHead = has_beta && inputs[5].size[2] >= Hkv; + + const float* Qp = query.ptr(); + const float* Kp = key.ptr(); + const float* Vp = value.ptr(); + const float* Dp = has_decay ? inputs[4].ptr() : nullptr; + const float* Bp = has_beta ? inputs[5].ptr() : nullptr; + const float* Pp = has_past ? inputs[3].ptr() : nullptr; + + float* Op = outputs[0].ptr(); + float* Sp = has_state_output ? outputs[1].ptr() : internals[0].ptr(); + + const size_t qStride = (size_t)seq * Hq * Dk; // per batch + const size_t kStride = (size_t)seq * Hkv * Dk; + const size_t vStride = (size_t)seq * Hkv * Dv; + const size_t oStride = (size_t)seq * Hq * Dv; + const size_t stateSz = (size_t)Dk * Dv; + + parallel_for_(Range(0, batch * Hkv), [&](const Range& r) + { + std::vector retrieved(Dv); + for (int bh = r.start; bh < r.end; ++bh) + { + const int b = bh / Hkv; + const int h = bh % Hkv; + + // state[i*Dv + j], initialised from past_state or zeros. + float* S = Sp + (size_t)bh * stateSz; + if (has_past) + std::memcpy(S, Pp + (size_t)bh * stateSz, stateSz * sizeof(float)); + else + std::memset(S, 0, stateSz * sizeof(float)); + + for (int t = 0; t < seq; ++t) + { + const float* k_t = Kp + b * kStride + (size_t)t * Hkv * Dk + (size_t)h * Dk; + const float* v_t = Vp + b * vStride + (size_t)t * Hkv * Dv + (size_t)h * Dv; + + // 1) forget gate: S[i,:] *= exp(decay_i). Resolve the per-head-scalar vs + // per-Dk-vector layout outside the i-loop so it stays branchless. + if (has_decay) + { + const float* d_t = Dp + b * (size_t)seq * Hkv * decayDim + (size_t)t * Hkv * decayDim + (size_t)h * decayDim; + if (decayDim == 1) + { + const float gate = std::exp(d_t[0]); + for (int i = 0; i < Dk; ++i) + { + float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) Si[j] *= gate; + } + } + else + { + for (int i = 0; i < Dk; ++i) + { + const float gate = std::exp(d_t[i]); + float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) Si[j] *= gate; + } + } + } + + // 2) write: outer-product update, optionally delta-corrected + if (has_beta) + { + // retrieved = S^T k_t ([Dv]) + for (int j = 0; j < Dv; ++j) retrieved[j] = 0.f; + for (int i = 0; i < Dk; ++i) + { + const float ki = k_t[i]; + const float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) retrieved[j] += Si[j] * ki; + } + const float beta = Bp[b * (size_t)seq * (betaPerHead ? Hkv : 1) + (size_t)t * (betaPerHead ? Hkv : 1) + (betaPerHead ? h : 0)]; + for (int i = 0; i < Dk; ++i) + { + const float ki = k_t[i]; + float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) + Si[j] += ki * (beta * (v_t[j] - retrieved[j])); + } + } + else + { + for (int i = 0; i < Dk; ++i) + { + const float ki = k_t[i]; + float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) Si[j] += ki * v_t[j]; + } + } + + // 3) read-out for every query head in this kv group: out = scale * (q_t . S) + for (int g = 0; g < group; ++g) + { + const int n = h * group + g; + const float* q_t = Qp + b * qStride + (size_t)t * Hq * Dk + (size_t)n * Dk; + float* o_t = Op + b * oStride + (size_t)t * Hq * Dv + (size_t)n * Dv; + for (int j = 0; j < Dv; ++j) o_t[j] = 0.f; + for (int i = 0; i < Dk; ++i) + { + const float qi = q_t[i]; + const float* Si = S + (size_t)i * Dv; + for (int j = 0; j < Dv; ++j) o_t[j] += qi * Si[j]; + } + for (int j = 0; j < Dv; ++j) o_t[j] *= scl; + } + } + } + }); + + for (size_t i = 0; i < rawOutputs.size(); ++i) + if (rawOutputs[i].depth() == CV_16F) + out32[i].convertTo(rawOutputs[i], CV_16F); + } + + int64 getFLOPS(const std::vector& inputs, const std::vector& /*outputs*/) const CV_OVERRIDE + { + const int64 batch = inputs[0][0], seq = inputs[0][1]; + const int64 Dk = inputs[0][2] / q_num_heads, Dv = inputs[2][2] / kv_num_heads; + // per step, per kv-head: ~2 outer products (write) + one read per q head + int64 perStep = kv_num_heads * CV_BIG_INT(4) * Dk * Dv + q_num_heads * CV_BIG_INT(2) * Dk * Dv; + return batch * seq * perStep; + } + +private: + int q_num_heads = 0; + int kv_num_heads = 0; + float scale = 0.f; + bool has_scale = false; + std::string update_rule; + bool use_decay = false; + bool use_delta = false; +}; + +Ptr LinearAttentionLayer::create(const LayerParams& params) +{ + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 4196869977..c2e386aa14 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -136,6 +136,13 @@ protected: void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto, int max_inputs = std::numeric_limits::max()); + // Append a layer to the current program from explicit input/output arg names. + void addComputedLayer(const std::string& type, LayerParams& lp, + const std::vector& inNames, + const std::vector& outNames); + // Inline a single-in/single-out sub-graph into the current program; returns the output arg name. + std::string inlineSubgraph(const opencv_onnx::GraphProto& g, + const std::string& srcArg, const std::string& prefix); void setParamsDtype(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); Arg resolveConstThroughIdentity(Arg arg); @@ -174,6 +181,7 @@ protected: std::string getLayerTypeDomain(const opencv_onnx::NodeProto& node_proto); const DispatchMap& getDispatchMap(const opencv_onnx::NodeProto& node_proto); void buildDispatchMap_ONNX_AI(); + void buildDispatchMap_ONNX_AI_PREVIEW(); void buildDispatchMap_COM_MICROSOFT(); // Domain: 'ai.onnx' (default) @@ -280,6 +288,8 @@ protected: void parseSimplifiedLayerNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSkipSimplifiedLayerNorm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseCausalConvWithState (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLinearAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlexAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSDPA (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -305,6 +315,7 @@ protected: void parseOperatorSet(); const std::string str_domain_ai_onnx = "ai.onnx"; + const std::string str_domain_ai_onnx_preview = "ai.onnx.preview"; const std::string str_domain_com_microsoft = "com.microsoft"; bool useLegacyNames; @@ -628,6 +639,7 @@ void ONNXImporter2::parseOperatorSet() } } buildDispatchMap_ONNX_AI(); + buildDispatchMap_ONNX_AI_PREVIEW(); buildDispatchMap_COM_MICROSOFT(); } @@ -2908,6 +2920,124 @@ void ONNXImporter2::parseCausalConvWithState(LayerParams& params, const opencv_o addLayer(params, node_proto); } +void ONNXImporter2::parseLinearAttention(LayerParams& params, const opencv_onnx::NodeProto& node_proto) { + params.type = "LinearAttention"; + addLayer(params, node_proto); +} + +void ONNXImporter2::parseFlexAttention(LayerParams& params, const opencv_onnx::NodeProto& node_proto) { + const opencv_onnx::GraphProto* scoreMod = nullptr; + const opencv_onnx::GraphProto* probMod = nullptr; + for (int i = 0; i < node_proto.attribute_size(); i++) { + const opencv_onnx::AttributeProto& a = node_proto.attribute(i); + // Only the spec-default softmax precision is implemented. + if (a.name() == "softmax_precision") + CV_Error(Error::StsNotImplemented, "ONNXImporter2/parseFlexAttention: explicit softmax_precision is not supported"); + if (!a.has_g()) continue; + if (a.name() == "score_mod") scoreMod = &a.g(); + else if (a.name() == "prob_mod") probMod = &a.g(); + } + + // No sub-graphs: a single fused FlexAttention layer computes Q,K,V -> Y. + if (!scoreMod && !probMod) { + params.type = "FlexAttention"; + addLayer(params, node_proto, 3); + return; + } + + // score_mod / prob_mod are pure data-flow, so decompose into stages and inline the + // sub-graph nodes as ordinary graph nodes (no runtime sub-graph execution): + // qk -> [score_mod] -> Softmax -> [prob_mod] -> av + const std::string nm = node_proto.name().empty() ? node_proto.output(0) : node_proto.name(); + const std::string qArg = node_proto.input(0); + const std::string kArg = node_proto.input(1); + const std::string vArg = node_proto.input(2); + const std::string yArg = node_proto.output(0); + const std::string scoresArg = nm + "/scores"; + const std::string probsArg = nm + "/probs"; + + LayerParams qkp; + qkp.name = nm + "/qk"; + if (params.has("scale")) qkp.set("scale", params.get("scale")); + qkp.set("stage", "qk"); + addComputedLayer("FlexAttention", qkp, {qArg, kArg}, {scoresArg}); + + std::string afterScore = scoreMod ? inlineSubgraph(*scoreMod, scoresArg, nm + "/smod#") : scoresArg; + + LayerParams smp; + smp.name = nm + "/softmax"; + smp.set("axis", -1); + addComputedLayer("Softmax", smp, {afterScore}, {probsArg}); + + std::string afterProb = probMod ? inlineSubgraph(*probMod, probsArg, nm + "/pmod#") : probsArg; + + LayerParams avp; + avp.name = nm + "/av"; + avp.set("stage", "av"); + addComputedLayer("FlexAttention", avp, {afterProb, vArg}, {yArg}); +} + +void ONNXImporter2::addComputedLayer(const std::string& type, LayerParams& lp, + const std::vector& inNames, + const std::vector& outNames) +{ + lp.type = type; + Ptr layer = LayerFactory::createLayerInstance(type, lp); + if (!layer) { + rememberMissingOp(type); + raiseError(); + return; + } + layer->inputs.clear(); + for (const std::string& n : inNames) { + if (!net.haveArg(n)) { + CV_LOG_ERROR(NULL, "DNN/ONNX: unknown input '" << n << "' of computed layer '" << lp.name << "'"); + raiseError(); + return; + } + layer->inputs.push_back(net.getArg(n)); + } + layer->outputs.clear(); + for (const std::string& n : outNames) + layer->outputs.push_back(net.getArg(n)); + layer->netimpl = netimpl; + curr_prog.push_back(layer); +} + +std::string ONNXImporter2::inlineSubgraph(const opencv_onnx::GraphProto& g, + const std::string& srcArg, const std::string& prefix) +{ + CV_CheckEQ(g.input_size(), 1, "ONNXImporter2/FlexAttention: sub-graph must have exactly one input"); + CV_CheckEQ(g.output_size(), 1, "ONNXImporter2/FlexAttention: sub-graph must have exactly one output"); + + std::vector undos; + // sub-graph input tensor resolves to the arg feeding this modifier + { + RenameUndo u; + u.key = g.input(0).name(); + auto it = rename_map.find(u.key); + u.had_prev = (it != rename_map.end()); + if (u.had_prev) u.prev_value = it->second; + undos.push_back(u); + rename_map[u.key] = srcArg; + } + // prefix every value the sub-graph defines to keep the parent namespace collision-free + for (int i = 0; i < g.initializer_size(); i++) + recordSubgraphRename(g.initializer(i).name(), prefix, undos); + for (int i = 0; i < g.node_size(); i++) + for (int j = 0; j < g.node(i).output_size(); j++) + recordSubgraphRename(g.node(i).output(j), prefix, undos); + + for (int i = 0; i < g.initializer_size(); i++) + netimpl->newConstArg(remap(g.initializer(i).name()), parseTensor(g.initializer(i))); + for (int i = 0; i < g.node_size(); i++) + parseNode(g.node(i)); + + std::string out = remap(g.output(0).name()); + popRenames(undos); + return out; +} + void ONNXImporter2::parseRoiAlign(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { layerParams.type = "RoiAlign"; @@ -3051,11 +3181,20 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI() dispatch["Attention"] = &ONNXImporter2::parseAttentionOnnxAi; dispatch["CausalConvWithState"] = &ONNXImporter2::parseCausalConvWithState; dispatch["SimplifiedLayerNormalization"] = &ONNXImporter2::parseSimplifiedLayerNormalization; + dispatch["LinearAttention"] = &ONNXImporter2::parseLinearAttention; dispatch["SkipSimplifiedLayerNormalization"] = &ONNXImporter2::parseSkipSimplifiedLayerNorm; domain_dispatch_map[str_domain_ai_onnx] = dispatch; } +// Domain: ai.onnx.preview +void ONNXImporter2::buildDispatchMap_ONNX_AI_PREVIEW() +{ + DispatchMap dispatch; + dispatch["FlexAttention"] = &ONNXImporter2::parseFlexAttention; + domain_dispatch_map[str_domain_ai_onnx_preview] = dispatch; +} + // Domain: com.microsoft // URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md void ONNXImporter2::buildDispatchMap_COM_MICROSOFT() diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index 37483d944a..9132fe55f4 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -2058,6 +2058,18 @@ TEST_P(Test_ONNX_conformance, Layer_Test) default_l1 = std::max(default_l1, 2e-4); default_lInf = std::max(default_lInf, 2e-3); } + // fp16 LinearAttention retains fp16 output precision (~4e-5 L1, ~5e-4 Inf) on fp32 targets. + if (name == "test_linear_attention_fp16" || + name == "test_linear_attention_fp16_expanded") { + default_l1 = std::max(default_l1, 2e-4); + default_lInf = std::max(default_lInf, 1e-3); + } + // fp16 FlexAttention likewise keeps fp16 output precision (~3e-4 Inf) on fp32 targets. + if (name == "test_flexattention_fp16" || + name == "test_flexattention_fp16_expanded_ver26") { + default_l1 = std::max(default_l1, 2e-4); + default_lInf = std::max(default_lInf, 1e-3); + } } #ifdef HAVE_HALIDE else if (backend == DNN_BACKEND_HALIDE) @@ -2135,6 +2147,20 @@ TEST_P(Test_ONNX_conformance, Layer_Test) default_l1 = std::max(default_l1, 2e-4); default_lInf = std::max(default_lInf, 2e-3); } + // fp16 LinearAttention retains fp16 output precision (~4e-5 L1, ~5e-4 Inf) on fp32 targets + // (the layer falls back to the CPU path). + if (name == "test_linear_attention_fp16" || + name == "test_linear_attention_fp16_expanded") { + default_l1 = std::max(default_l1, 2e-4); + default_lInf = std::max(default_lInf, 1e-3); + } + // fp16 FlexAttention likewise keeps fp16 output precision (~3e-4 Inf) on fp32 targets + // (the layer falls back to the CPU path). + if (name == "test_flexattention_fp16" || + name == "test_flexattention_fp16_expanded_ver26") { + default_l1 = std::max(default_l1, 2e-4); + default_lInf = std::max(default_lInf, 1e-3); + } } #endif else diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 7d1b4a2a9c..961a58add1 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -3426,6 +3426,108 @@ CASE(test_cumprod_2d_negative_axis) SKIP; CASE(test_flexattention_scaled_expanded_ver26) SKIP; +// FlexAttention op not supported by OpenVINO +CASE(test_flexattention) + SKIP; +CASE(test_flexattention_causal_mask) + SKIP; +CASE(test_flexattention_diff_head_sizes) + SKIP; +CASE(test_flexattention_double) + SKIP; +CASE(test_flexattention_fp16) + SKIP; +CASE(test_flexattention_gqa) + SKIP; +CASE(test_flexattention_prob_mod) + SKIP; +CASE(test_flexattention_relative_positional) + SKIP; +CASE(test_flexattention_scaled) + SKIP; +CASE(test_flexattention_score_mod) + SKIP; +CASE(test_flexattention_soft_cap) + SKIP; +// FlexAttention (expanded) not supported by OpenVINO +CASE(test_flexattention_causal_mask_expanded_ver26) + SKIP; +CASE(test_flexattention_diff_head_sizes_expanded_ver26) + SKIP; +CASE(test_flexattention_double_expanded_ver26) + SKIP; +CASE(test_flexattention_expanded_ver26) + SKIP; +CASE(test_flexattention_fp16_expanded_ver26) + SKIP; +CASE(test_flexattention_gqa_expanded_ver26) + SKIP; +CASE(test_flexattention_prob_mod_expanded_ver26) + SKIP; +CASE(test_flexattention_relative_positional_expanded_ver26) + SKIP; +CASE(test_flexattention_score_mod_expanded_ver26) + SKIP; +CASE(test_flexattention_soft_cap_expanded_ver26) + SKIP; +// LinearAttention op not supported by OpenVINO +CASE(test_linear_attention_decode_step) + SKIP; +CASE(test_linear_attention_delta) + SKIP; +CASE(test_linear_attention_explicit_scale) + SKIP; +CASE(test_linear_attention_fp16) + SKIP; +CASE(test_linear_attention_gated) + SKIP; +CASE(test_linear_attention_gated_delta) + SKIP; +CASE(test_linear_attention_gated_delta_beta_scalar) + SKIP; +CASE(test_linear_attention_gated_delta_gqa) + SKIP; +CASE(test_linear_attention_gated_delta_mqa) + SKIP; +CASE(test_linear_attention_gated_per_head_decay) + SKIP; +CASE(test_linear_attention_linear) + SKIP; +CASE(test_linear_attention_linear_t1_no_past) + SKIP; +CASE(test_linear_attention_no_past_explicit_zeros) + SKIP; +CASE(test_linear_attention_prefill_with_past) + SKIP; +// LinearAttention (expanded) decomposes to Scan, not supported by OpenVINO +CASE(test_linear_attention_decode_step_expanded) + SKIP; +CASE(test_linear_attention_delta_expanded) + SKIP; +CASE(test_linear_attention_explicit_scale_expanded) + SKIP; +CASE(test_linear_attention_fp16_expanded) + SKIP; +CASE(test_linear_attention_gated_delta_beta_scalar_expanded) + SKIP; +CASE(test_linear_attention_gated_delta_expanded) + SKIP; +CASE(test_linear_attention_gated_delta_gqa_expanded) + SKIP; +CASE(test_linear_attention_gated_delta_mqa_expanded) + SKIP; +CASE(test_linear_attention_gated_expanded) + SKIP; +CASE(test_linear_attention_gated_per_head_decay_expanded) + SKIP; +CASE(test_linear_attention_linear_expanded) + SKIP; +CASE(test_linear_attention_linear_t1_no_past_expanded) + SKIP; +CASE(test_linear_attention_no_past_explicit_zeros_expanded) + SKIP; +CASE(test_linear_attention_prefill_with_past_expanded) + SKIP; CASE(test_range_bfloat16_type_positive_delta) SKIP; CASE(test_range_float16_type_positive_delta) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp index d332dffafb..907cd628e8 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_all_denylist.inl.hpp @@ -12,3 +12,4 @@ "test_maxpool_with_argmax_2d_precomputed_strides", // wrong output "test_maxunpool_export_with_output_shape", // unexception during net.forward() call "test_upsample_nearest", // Dimension mismatch of input +"test_flexattention_double_expanded_ver26", // Softmax kernel is fp32-only; fp64 decomposition unsupported (fused test_flexattention_double passes) diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 29d32dbb60..82dc196637 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -370,47 +370,6 @@ "test_dequantizelinear_uint2", "test_quantizelinear_int2", "test_quantizelinear_uint2", -// FlexAttention op not supported -"test_flexattention", -"test_flexattention_causal_mask", -"test_flexattention_diff_head_sizes", -"test_flexattention_double", -"test_flexattention_fp16", -"test_flexattention_gqa", -"test_flexattention_prob_mod", -"test_flexattention_relative_positional", -"test_flexattention_scaled", -"test_flexattention_score_mod", -"test_flexattention_soft_cap", -// LinearAttention op not supported -"test_linear_attention_decode_step", -"test_linear_attention_decode_step_expanded", -"test_linear_attention_delta", -"test_linear_attention_delta_expanded", -"test_linear_attention_explicit_scale", -"test_linear_attention_explicit_scale_expanded", -"test_linear_attention_fp16", -"test_linear_attention_fp16_expanded", -"test_linear_attention_gated", -"test_linear_attention_gated_delta", -"test_linear_attention_gated_delta_beta_scalar", -"test_linear_attention_gated_delta_beta_scalar_expanded", -"test_linear_attention_gated_delta_expanded", -"test_linear_attention_gated_delta_gqa", -"test_linear_attention_gated_delta_gqa_expanded", -"test_linear_attention_gated_delta_mqa", -"test_linear_attention_gated_delta_mqa_expanded", -"test_linear_attention_gated_expanded", -"test_linear_attention_gated_per_head_decay", -"test_linear_attention_gated_per_head_decay_expanded", -"test_linear_attention_linear", -"test_linear_attention_linear_expanded", -"test_linear_attention_linear_t1_no_past", -"test_linear_attention_linear_t1_no_past_expanded", -"test_linear_attention_no_past_explicit_zeros", -"test_linear_attention_no_past_explicit_zeros_expanded", -"test_linear_attention_prefill_with_past", -"test_linear_attention_prefill_with_past_expanded", // misc unsupported (expanded subgraphs / new ops) "test_castlike_no_saturate_FLOAT_to_FLOAT8E4M3FNUZ_expanded", "test_castlike_no_saturate_FLOAT_to_FLOAT8E4M3FN_expanded", @@ -430,14 +389,3 @@ // CausalConvWithState fp16 (expanded) accuracy "test_causal_conv_with_state_fp16_expanded", "test_causal_conv_with_state_silu_fp16_expanded", -// FlexAttention (expanded) accuracy -"test_flexattention_causal_mask_expanded_ver26", -"test_flexattention_diff_head_sizes_expanded_ver26", -"test_flexattention_double_expanded_ver26", -"test_flexattention_expanded_ver26", -"test_flexattention_fp16_expanded_ver26", -"test_flexattention_gqa_expanded_ver26", -"test_flexattention_prob_mod_expanded_ver26", -"test_flexattention_relative_positional_expanded_ver26", -"test_flexattention_score_mod_expanded_ver26", -"test_flexattention_soft_cap_expanded_ver26",