Administrator/opencv-MIRROR
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-11-09 18:04:29 -06:00
Code Issues Packages Projects Releases Wiki Activity

dnn: check weight, bias and past_state in CausalConvWithState 🤖🤖🤖 (#30628)

Browse Source 
* dnn: check weight, bias and past_state in CausalConvWithState

Check the ONNX shapes of weight [C, 1, k], bias [C] and past_state [B, C, k - 1] against the input in getMemoryShapes, and require them to be float in forward, since the kernel indexes all three by input channel and reads them as float.

* dnn: check CausalConvWithState input types before fp16 conversion, require rank-1 bias
This commit is contained in:
uwezkhan
2026-11-09 12:23:45 +05:30
committed by GitHub
parent 566df44633
commit dfbf43df29
2 changed files with 108 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
modules/dnn/src/layers/causal_conv_with_state_layer.cpp
View File

@@ -32,7 +32,9 @@ public:
bool supportBackend(int backendId) CV_OVERRIDE { return backendId == DNN_BACKEND_OPENCV; }
static bool present(const std::vector<Mat>& in, size_t i) { return in.size() > i && !in[i].empty(); }
// optional inputs arrive as empty Mats / MatShapes
template<typename T>
static bool present(const std::vector<T>& in, size_t i) { return in.size() > i && !in[i].empty(); }
void getTypes(const std::vector<MatType>& inputs, const int requiredOutputs, const int,
std::vector<MatType>& outputs, std::vector<MatType>& internals) const CV_OVERRIDE
@@ -49,6 +51,21 @@ public:
CV_CheckEQ(inputs[0].dims, 3, "input must be [batch, channels, seq]");
CV_CheckEQ(inputs[1].dims, 3, "weight must be [channels, 1, kernel]");
const int B = inputs[0][0], C = inputs[0][1], T = inputs[0][2], K = inputs[1][2];
CV_CheckEQ(inputs[1][0], C, "weight must have one filter per input channel");
CV_CheckEQ(inputs[1][1], 1, "weight must be [channels, 1, kernel]");
CV_CheckGE(K, 1, "kernel size must be positive");
if (present(inputs, 2))
{
CV_CheckEQ(inputs[2].dims, 1, "bias must be [channels]");
CV_CheckEQ(inputs[2][0], C, "bias must be [channels]");
}
if (present(inputs, 3))
{
CV_CheckEQ(inputs[3].dims, 3, "past_state must be [batch, channels, kernel - 1]");
CV_CheckEQ(inputs[3][0], B, "past_state must be [batch, channels, kernel - 1]");
CV_CheckEQ(inputs[3][1], C, "past_state must be [batch, channels, kernel - 1]");
CV_CheckEQ(inputs[3][2], K - 1, "past_state must be [batch, channels, kernel - 1]");
}
outputs.assign(1, MatShape{B, C, T});
outputs.push_back(MatShape{B, C, K - 1});
return false;
@@ -60,6 +77,11 @@ public:
inputs_arr.getMatVector(rawIn);
outputs_arr.getMatVector(rawOut);
// weight, bias and past_state share the input type, checked before the fp16 conversion below
for (size_t i = 1; i < rawIn.size(); ++i)
if (!rawIn[i].empty())
CV_CheckTypeEQ(rawIn[i].type(), rawIn[0].type(), "weight, bias and past_state must have the input type");
const bool fp16 = rawIn[0].depth() == CV_16F;
std::vector<Mat> in32, out32;
if (fp16)

85
modules/dnn/test/test_layers.cpp
View File

@@ -4687,4 +4687,89 @@ TEST(Test_Compress, AllFalse_WithoutAxis)
ASSERT_EQ(shape(outputs[0]), MatShape({0}));
}
TEST(Layer_Test_CausalConvWithState, RejectMismatchedWeightBiasAndState)
{
const int B = 2, C = 64, T = 8, K = 4;
Mat x(std::vector<int>{B, C, T}, CV_32F);
randu(x, -1, 1);
Mat w(std::vector<int>{C, 1, K}, CV_32F);
randu(w, -1, 1);
Mat bias(std::vector<int>{C}, CV_32F);
randu(bias, -1, 1);
Mat past(std::vector<int>{B, C, K - 1}, CV_32F);
randu(past, -1, 1);
LayerParams lp;
lp.type = "CausalConvWithState";
lp.name = "causal_conv_with_state";
Ptr<Layer> layer = LayerFactory::createLayerInstance(lp.type, lp);
ASSERT_TRUE(layer);
// weight must be [C, 1, K]
std::vector<Mat> inputs{x, Mat(std::vector<int>{1, 1, K}, CV_32F, Scalar(1))};
std::vector<Mat> outputs;
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x, Mat(std::vector<int>{C, 2, K}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
// bias must be [C]
inputs = {x, w, Mat(std::vector<int>{1}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x, w, Mat(std::vector<int>{1, C}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
// past_state must be [B, C, K - 1]
inputs = {x, w, bias, Mat(std::vector<int>{B, 1, K - 1}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x, w, bias, Mat(std::vector<int>{1, C, K - 1}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x, w, bias, Mat(std::vector<int>{B, C, K}, CV_32F, Scalar(1))};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
// weight, bias and past_state must have the input type, on both the fp32 and the fp16 path
Mat x16, w16, bias16, past16;
x.convertTo(x16, CV_16F);
w.convertTo(w16, CV_16F);
bias.convertTo(bias16, CV_16F);
past.convertTo(past16, CV_16F);
inputs = {x, w16};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x16, w};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x16, w16, bias};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x16, w16, bias16, past};
EXPECT_THROW(runLayer(layer, inputs, outputs), cv::Exception);
inputs = {x16, w16, bias16, past16};
EXPECT_NO_THROW(runLayer(layer, inputs, outputs));
// valid inputs give the depthwise causal convolution, with past_state read per batch
Mat ref(std::vector<int>{B, C, T}, CV_32F), refState(std::vector<int>{B, C, K - 1}, CV_32F);
for (int b = 0; b < B; b++)
{
for (int c = 0; c < C; c++)
{
std::vector<float> pad(K - 1 + T);
for (int j = 0; j < K - 1; j++)
pad[j] = past.ptr<float>(b, c)[j];
for (int t = 0; t < T; t++)
pad[K - 1 + t] = x.ptr<float>(b, c)[t];
for (int t = 0; t < T; t++)
{
float acc = bias.at<float>(c);
for (int k = 0; k < K; k++)
acc += w.ptr<float>(c)[k] * pad[t + k];
ref.ptr<float>(b, c)[t] = acc;
}
for (int j = 0; j < K - 1; j++)
refState.ptr<float>(b, c)[j] = pad[T + j];
}
}
inputs = {x, w, bias, past};
ASSERT_NO_THROW(runLayer(layer, inputs, outputs));
ASSERT_EQ(outputs.size(), (size_t)2);
normAssert(ref, outputs[0], "output");
normAssert(refState, outputs[1], "present_state");
}
}} // namespace