diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 1f54d9563d..1c962dcae4 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -2012,6 +2012,15 @@ Rect_<_Tp>& operator &= ( Rect_<_Tp>& a, const Rect_<_Tp>& b ) a = Rect_<_Tp>(); return a; } + + // If the delta between x/y coordinates exceeds the corresponding width/height, + // the rectangles cannot overlap and `width - delta` would underflow for unsigned types. + if (Rx_min.width < (Rx_max.x - Rx_min.x) || + Ry_min.height < (Ry_max.y - Ry_min.y)) { + a = Rect_<_Tp>(); + return a; + } + // We now know that either Rx_min.x >= 0, or // Rx_min.x < 0 && Rx_min.x + Rx_min.width >= Rx_max.x and therefore // Rx_min.width >= (Rx_max.x - Rx_min.x) which means (Rx_max.x - Rx_min.x) diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 94b4d83781..1682ff27ea 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -996,6 +996,17 @@ REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows, OnTheEdge); typedef ::testing::Types RectTypes; INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes); +TEST(Core_Rect, test_unsigned_overflow_11988) +{ + typedef Rect_ R; + R r1(0, 0, 1u, 1u); + R r2(2u, 2u, 1u, 1u); + auto inter = r1 & r2; + EXPECT_EQ(R(), inter); + EXPECT_EQ(0u, inter.area()); + EXPECT_TRUE(inter.empty()); +} + // Expected that SkipTestException thrown in the constructor should skip test but not fail struct TestFixtureSkip: public ::testing::Test { TestFixtureSkip(bool throwEx = true) {