diff --git a/doc/opencv.bib b/doc/opencv.bib index 80e789eaf5..c96d65c919 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -1066,6 +1066,15 @@ publisher = {IEEE}, url = {https://static.aminer.org/pdf/PDF/000/128/789/non_parametric_similarity_measures_for_unsupervised_texture_segmentation_and_image.pdf} } +@inproceedings{potje2024cvpr, + author={Potje, Guilherme and Cadar, Felipe and Araujo, André and Martins, Renato and Nascimento, Erickson R.}, + booktitle={2024 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + title={XFeat: Accelerated Features for Lightweight Image Matching}, + year={2024}, + pages={2682-2691}, + keywords={Visualization;Accuracy;Image matching;Pose estimation;Feature extraction;Hardware;Real-time systems;Image matching;Local features;Lightweight;Fast}, + doi={10.1109/CVPR52733.2024.00259} +} @inproceedings{RB99, author = {Robertson, Mark A and Borman, Sean and Stevenson, Robert L}, title = {Dynamic range improvement through multiple exposures}, diff --git a/modules/features/include/opencv2/features.hpp b/modules/features/include/opencv2/features.hpp index db3b1e5a13..271ae88a72 100644 --- a/modules/features/include/opencv2/features.hpp +++ b/modules/features/include/opencv2/features.hpp @@ -753,6 +753,57 @@ public: CV_WRAP virtual String getDefaultName() const CV_OVERRIDE; }; +/** @brief XFeat feature detector and descriptor, based on a DNN model. + +XFeat is a compact learned local-feature extractor. This class wraps an ONNX export through +cv::dnn::Net and exposes score-map detections with 64-D float descriptors under the standard +cv::Feature2D interface. + +The class assumes the ONNX model has a single grayscale input tensor N×1×H×W in [0, 1] and +returns descriptor and score maps. The descriptor map must have 64 channels, and the keypoint +logit map must have either 64 channels (no extra class) or 65 channels, where the last channel +is treated as a dustbin/background class used only in softmax normalization. Images are resized +with preserved aspect ratio and padded to the configured network input size. + */ +class CV_EXPORTS_W XFeat : public Feature2D +{ +public: + /** @brief Creates an XFeat detector. + @param modelPath Path to the XFeat ONNX model. + @param maxKeypoints Maximum number of keypoints to return per image. The strongest + responses are kept; -1 keeps all detections. + @param scoreThreshold Discard keypoints with network score not greater than this value. + @param inputSize Input size fed to the network, default Size(640, 640). + @param backendId DNN backend identifier (see cv::dnn::Backend); 0 = DNN_BACKEND_DEFAULT. + @param targetId DNN target identifier (see cv::dnn::Target); 0 = DNN_TARGET_CPU. + */ + CV_WRAP static Ptr create(const String& modelPath, + int maxKeypoints = -1, + float scoreThreshold = 0.5f, + const Size& inputSize = Size(640, 640), + int backendId = 0, + int targetId = 0); + + /** @brief Creates an XFeat detector from an in-memory model buffer. */ + CV_WRAP_AS(createFromMemory) static Ptr create(const std::vector& bufferModel, + int maxKeypoints = -1, + float scoreThreshold = 0.5f, + const Size& inputSize = Size(640, 640), + int backendId = 0, + int targetId = 0); + + CV_WRAP virtual void setMaxKeypoints(int maxKeypoints) = 0; + CV_WRAP virtual int getMaxKeypoints() const = 0; + + CV_WRAP virtual void setScoreThreshold(float threshold) = 0; + CV_WRAP virtual float getScoreThreshold() const = 0; + + CV_WRAP virtual void setInputSize(const Size& inputSize) = 0; + CV_WRAP virtual Size getInputSize() const = 0; + + CV_WRAP virtual String getDefaultName() const CV_OVERRIDE; +}; + #endif // HAVE_OPENCV_DNN || CV_DOXYGEN /** @brief Class for extracting blobs from an image. : diff --git a/modules/features/src/feature2d_xfeat.cpp b/modules/features/src/feature2d_xfeat.cpp new file mode 100644 index 0000000000..d9b6bacc7e --- /dev/null +++ b/modules/features/src/feature2d_xfeat.cpp @@ -0,0 +1,453 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#ifdef HAVE_OPENCV_DNN + +#include +#include +#include + +namespace cv { + +using namespace dnn; + +namespace { + +static const int kXFeatDescriptorSize = 64; +static const std::vector kXFeatOutputNames = +{ + "output_feats", + "output_keypoints", + "output_heatmap" +}; + +struct XFeatCandidate +{ + Point2f ptPadded; + Point2f pt; + float score; +}; + +static Mat toGray(InputArray _image) +{ + Mat image = _image.getMat(); + if (image.channels() == 1) + return image; + + Mat gray; + if (image.channels() == 3) + cvtColor(image, gray, COLOR_BGR2GRAY); + else if (image.channels() == 4) + cvtColor(image, gray, COLOR_BGRA2GRAY); + else + CV_Error(Error::StsBadArg, "XFeat expects a grayscale, BGR, or BGRA image"); + return gray; +} + +static Mat toNCHW(const Mat& blob, int channelsHint) +{ + CV_Assert(blob.dims == 4); + if (blob.size[1] == channelsHint) + return blob; + + CV_Assert(blob.size[3] == channelsHint); + Mat out; + Mat src = blob.isContinuous() ? blob : blob.clone(); + transposeND(src, {0, 3, 1, 2}, out); + return out; +} + +static float sampleNearest(const Mat& map, float x, float y, int normW, int normH) +{ + CV_Assert(map.type() == CV_32F); + if (map.empty() || normW <= 1 || normH <= 1) + return 0.0f; + + const float fx = x * static_cast(map.cols - 1) / static_cast(normW - 1); + const float fy = y * static_cast(map.rows - 1) / static_cast(normH - 1); + const int ix = std::max(0, std::min(map.cols - 1, cvRound(fx))); + const int iy = std::max(0, std::min(map.rows - 1, cvRound(fy))); + return map.at(iy, ix); +} + +static float sampleBilinear(const Mat& map, float x, float y, int normW, int normH) +{ + CV_Assert(map.type() == CV_32F); + if (map.empty() || normW <= 1 || normH <= 1) + return 0.0f; + + float fx = x * static_cast(map.cols - 1) / static_cast(normW - 1); + float fy = y * static_cast(map.rows - 1) / static_cast(normH - 1); + fx = std::max(0.0f, std::min(fx, static_cast(map.cols - 1))); + fy = std::max(0.0f, std::min(fy, static_cast(map.rows - 1))); + + const int x0 = cvFloor(fx); + const int y0 = cvFloor(fy); + const int x1 = std::min(x0 + 1, map.cols - 1); + const int y1 = std::min(y0 + 1, map.rows - 1); + const float dx = fx - x0; + const float dy = fy - y0; + + const float v00 = map.at(y0, x0); + const float v01 = map.at(y0, x1); + const float v10 = map.at(y1, x0); + const float v11 = map.at(y1, x1); + return (1.f - dx) * (1.f - dy) * v00 + + dx * (1.f - dy) * v01 + + (1.f - dx) * dy * v10 + + dx * dy * v11; +} + +} // namespace + +class XFeat_Impl CV_FINAL : public XFeat +{ +public: + XFeat_Impl(const String& modelPath, int maxKeypoints, float scoreThreshold, + const Size& inputSize, int backendId, int targetId) + : maxKeypoints_(maxKeypoints), + scoreThreshold_(scoreThreshold), + inputSize_(inputSize) + { + CV_Assert(inputSize_.width > 0 && inputSize_.height > 0); + initNet(readNetFromONNX(modelPath), backendId, targetId); + } + + XFeat_Impl(const std::vector& bufferModel, int maxKeypoints, float scoreThreshold, + const Size& inputSize, int backendId, int targetId) + : maxKeypoints_(maxKeypoints), + scoreThreshold_(scoreThreshold), + inputSize_(inputSize) + { + CV_Assert(inputSize_.width > 0 && inputSize_.height > 0); + initNet(readNetFromONNX(bufferModel), backendId, targetId); + } + + void detectAndCompute(InputArray _image, InputArray _mask, + std::vector& keypoints, + OutputArray _descriptors, + bool useProvidedKeypoints) CV_OVERRIDE + { + CV_Assert(!useProvidedKeypoints && "XFeat does not support providing keypoints externally"); + + keypoints.clear(); + + Mat image = _image.getMat(); + if (image.empty()) + { + if (_descriptors.needed()) + _descriptors.release(); + return; + } + + Mat mask = _mask.getMat(); + if (!mask.empty()) + { + CV_Assert(mask.type() == CV_8UC1 || mask.type() == CV_BoolC1); + CV_Assert(mask.size() == image.size()); + } + + Mat gray = toGray(image); + const float scaleX = static_cast(inputSize_.width) / static_cast(gray.cols); + const float scaleY = static_cast(inputSize_.height) / static_cast(gray.rows); + const float scale = std::min(scaleX, scaleY); + const int resizedW = std::max(1, cvRound(static_cast(gray.cols) * scale)); + const int resizedH = std::max(1, cvRound(static_cast(gray.rows) * scale)); + const int padX = (inputSize_.width - resizedW) / 2; + const int padY = (inputSize_.height - resizedH) / 2; + + Mat blob; + Image2BlobParams blobParams; + blobParams.scalefactor = Scalar::all(1.0 / 255.0); + blobParams.size = inputSize_; + blobParams.mean = Scalar(); + blobParams.swapRB = false; + blobParams.ddepth = CV_32F; + blobParams.datalayout = DNN_LAYOUT_NCHW; + blobParams.paddingmode = DNN_PMODE_LETTERBOX; + blobParams.borderValue = Scalar(); + blobFromImageWithParams(gray, blob, blobParams); + net_.setInput(blob); + + std::vector outs; + net_.forward(outs, kXFeatOutputNames); + CV_Assert(outs.size() == 3); + + Mat featBlob = toNCHW(outs[0], kXFeatDescriptorSize); + Mat kptBlob = outs[1]; + CV_Assert(kptBlob.dims == 4); + if (!(kptBlob.size[1] == 64 || kptBlob.size[1] == 65)) + { + CV_Assert(kptBlob.size[3] == 64 || kptBlob.size[3] == 65); + Mat src = kptBlob.isContinuous() ? kptBlob : kptBlob.clone(); + transposeND(src, {0, 3, 1, 2}, kptBlob); + } + Mat relBlob = toNCHW(outs[2], 1); + CV_Assert(featBlob.dims == 4 && kptBlob.dims == 4 && relBlob.dims == 4); + if (!featBlob.isContinuous()) + featBlob = featBlob.clone(); + if (!kptBlob.isContinuous()) + kptBlob = kptBlob.clone(); + if (!relBlob.isContinuous()) + relBlob = relBlob.clone(); + + const int featH = featBlob.size[2]; + const int featW = featBlob.size[3]; + const int kptC = kptBlob.size[1]; + const int kptH = kptBlob.size[2]; + const int kptW = kptBlob.size[3]; + CV_Assert(featBlob.size[1] == kXFeatDescriptorSize && (kptC == 64 || kptC == 65)); + + Mat reliability(relBlob.size[2], relBlob.size[3], CV_32F, relBlob.ptr()); + Mat heatmap = Mat::zeros(kptH * 8, kptW * 8, CV_32F); + const float* kptPtr = kptBlob.ptr(); + const int kptHW = kptH * kptW; + + parallel_for_(Range(0, kptH), [&](const Range& range) + { + for (int y = range.start; y < range.end; ++y) + { + for (int x = 0; x < kptW; ++x) + { + const int offset = y * kptW + x; + float maxLogit = -FLT_MAX; + for (int ch = 0; ch < kptC; ++ch) + maxLogit = std::max(maxLogit, kptPtr[ch * kptHW + offset]); + + float sumExp = 0.f; + float logits[64]; + float probs[64]; + for (int ch = 0; ch < 64; ++ch) + logits[ch] = kptPtr[ch * kptHW + offset] - maxLogit; +#if (defined(CV_SIMD) && CV_SIMD) || (defined(CV_SIMD_SCALABLE) && CV_SIMD_SCALABLE) + const int vlanes = VTraits::vlanes(); + int ch = 0; + v_float32 vSum = vx_setzero_f32(); + for (; ch <= 64 - vlanes; ch += vlanes) + { + v_float32 v = vx_load(logits + ch); + v_float32 e = v_exp(v); + vx_store(probs + ch, e); + vSum = v_add(vSum, e); + } + sumExp = v_reduce_sum(vSum); + for (; ch < 64; ++ch) + { + probs[ch] = std::exp(logits[ch]); + sumExp += probs[ch]; + } +#else + for (int ch = 0; ch < 64; ++ch) + { + probs[ch] = std::exp(logits[ch]); + sumExp += probs[ch]; + } +#endif + if (kptC == 65) + sumExp += std::exp(kptPtr[64 * kptHW + offset] - maxLogit); + if (sumExp <= 0.f) + continue; + + for (int ch = 0; ch < 64; ++ch) + { + const int dy = ch / 8; + const int dx = ch % 8; + heatmap.at(y * 8 + dy, x * 8 + dx) = probs[ch] / sumExp; + } + } + } + }); + + Mat localMax; + dilate(heatmap, localMax, getStructuringElement(MORPH_RECT, Size(5, 5))); + + std::vector candidates; + candidates.reserve(4096); + + for (int y = 0; y < heatmap.rows; ++y) + { + const float* hm = heatmap.ptr(y); + const float* mx = localMax.ptr(y); + for (int x = 0; x < heatmap.cols; ++x) + { + const float h = hm[x]; + if (h <= scoreThreshold_ || h != mx[x]) + continue; + + const float xp = static_cast(x); + const float yp = static_cast(y); + const float score = sampleNearest(heatmap, xp, yp, inputSize_.width, inputSize_.height) * + sampleBilinear(reliability, xp, yp, inputSize_.width, inputSize_.height); + if (score <= 0.f) + continue; + + const float px = (xp - static_cast(padX)) / scale; + const float py = (yp - static_cast(padY)) / scale; + const int ix = cvFloor(px); + const int iy = cvFloor(py); + if (ix < 0 || iy < 0 || ix >= image.cols || iy >= image.rows) + continue; + if (!mask.empty() && mask.at(iy, ix) == 0) + continue; + + candidates.push_back({Point2f(xp, yp), Point2f(px, py), score}); + } + } + + if (maxKeypoints_ > 0 && static_cast(candidates.size()) > maxKeypoints_) + { + std::partial_sort(candidates.begin(), candidates.begin() + maxKeypoints_, candidates.end(), + [](const XFeatCandidate& a, const XFeatCandidate& b) + { + return a.score > b.score; + }); + candidates.resize(maxKeypoints_); + } + + keypoints.reserve(candidates.size()); + for (const XFeatCandidate& c : candidates) + keypoints.emplace_back(c.pt, 1.0f, -1.0f, c.score); + + if (_descriptors.needed()) + { + if (candidates.empty()) + { + _descriptors.release(); + return; + } + + _descriptors.create(static_cast(candidates.size()), kXFeatDescriptorSize, CV_32F); + Mat descriptors = _descriptors.getMat(); + const float* featPtr = featBlob.ptr(); + const int featHW = featH * featW; + + parallel_for_(Range(0, static_cast(candidates.size())), + [&](const Range& range) + { + for (int i = range.start; i < range.end; ++i) + { + float* dst = descriptors.ptr(i); + const XFeatCandidate& c = candidates[i]; + for (int ch = 0; ch < kXFeatDescriptorSize; ++ch) + { + Mat channel(featH, featW, CV_32F, + const_cast(featPtr + ch * featHW)); + dst[ch] = sampleBilinear(channel, c.ptPadded.x, c.ptPadded.y, + inputSize_.width, inputSize_.height); + } + normalize(descriptors.row(i), descriptors.row(i), 1.0, 0.0, NORM_L2); + } + }); + } + } + + int descriptorSize() const CV_OVERRIDE { return kXFeatDescriptorSize; } + int descriptorType() const CV_OVERRIDE { return CV_32F; } + int defaultNorm() const CV_OVERRIDE { return NORM_L2; } + + bool empty() const CV_OVERRIDE { return net_.empty(); } + + void setMaxKeypoints(int maxKeypoints) CV_OVERRIDE { maxKeypoints_ = maxKeypoints; } + int getMaxKeypoints() const CV_OVERRIDE { return maxKeypoints_; } + + void setScoreThreshold(float threshold) CV_OVERRIDE { scoreThreshold_ = threshold; } + float getScoreThreshold() const CV_OVERRIDE { return scoreThreshold_; } + + void setInputSize(const Size& inputSize) CV_OVERRIDE + { + CV_Assert(inputSize.width > 0 && inputSize.height > 0); + inputSize_ = inputSize; + } + Size getInputSize() const CV_OVERRIDE { return inputSize_; } + + String getDefaultName() const CV_OVERRIDE { return Feature2D::getDefaultName() + ".XFeat"; } + +private: + void initNet(const Net& net, int backendId, int targetId) + { + net_ = net; + net_.setPreferableBackend(backendId); + net_.setPreferableTarget(targetId); + + // Check output names once and fail early. + const std::vector modelOutNames = net_.getUnconnectedOutLayersNames(); + if (modelOutNames.size() != kXFeatOutputNames.size()) + { + String msg = "XFeat ONNX output count mismatch: expected " + + std::to_string(kXFeatOutputNames.size()) + " outputs ("; + for (size_t i = 0; i < kXFeatOutputNames.size(); ++i) + { + msg += kXFeatOutputNames[i]; + if (i + 1 < kXFeatOutputNames.size()) + msg += ", "; + } + msg += "), but model has " + std::to_string(modelOutNames.size()) + " outputs: "; + for (size_t i = 0; i < modelOutNames.size(); ++i) + { + msg += modelOutNames[i]; + if (i + 1 < modelOutNames.size()) + msg += ", "; + } + CV_Error(Error::StsError, msg); + } + + for (size_t i = 0; i < kXFeatOutputNames.size(); ++i) + { + bool found = false; + for (size_t j = 0; j < modelOutNames.size(); ++j) + { + if (modelOutNames[j] == kXFeatOutputNames[i]) + { + found = true; + break; + } + } + + if (!found) + { + String msg = "XFeat ONNX output name mismatch: expected output '" + kXFeatOutputNames[i] + + "'. Please check model outputs. Available outputs: "; + for (size_t j = 0; j < modelOutNames.size(); ++j) + { + msg += modelOutNames[j]; + if (j + 1 < modelOutNames.size()) + msg += ", "; + } + CV_Error(Error::StsError, msg); + } + } + } + + int maxKeypoints_; + float scoreThreshold_; + Size inputSize_; + Net net_; +}; + +Ptr XFeat::create(const String& modelPath, int maxKeypoints, float scoreThreshold, + const Size& inputSize, int backendId, int targetId) +{ + CV_TRACE_FUNCTION(); + return makePtr(modelPath, maxKeypoints, scoreThreshold, + inputSize, backendId, targetId); +} + +Ptr XFeat::create(const std::vector& bufferModel, int maxKeypoints, + float scoreThreshold, const Size& inputSize, int backendId, int targetId) +{ + CV_TRACE_FUNCTION(); + return makePtr(bufferModel, maxKeypoints, scoreThreshold, + inputSize, backendId, targetId); +} + +String XFeat::getDefaultName() const +{ + return Feature2D::getDefaultName() + ".XFeat"; +} + +} // namespace cv + +#endif // HAVE_OPENCV_DNN diff --git a/modules/features/test/test_xfeat.cpp b/modules/features/test/test_xfeat.cpp new file mode 100644 index 0000000000..a528cf0df9 --- /dev/null +++ b/modules/features/test/test_xfeat.cpp @@ -0,0 +1,200 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" + +#ifdef HAVE_OPENCV_DNN + +#include "opencv2/core/utils/configuration.private.hpp" +#include "opencv2/dnn.hpp" + +namespace opencv_test { namespace { + +static int countNearbyKeypoints(const std::vector& keypoints, const Mat& refKpts, float maxDistance) +{ + const float maxDistSq = maxDistance * maxDistance; + int matched = 0; + for (const KeyPoint& kp : keypoints) + { + float bestDistSq = maxDistSq; + for (int i = 0; i < refKpts.rows; ++i) + { + const float dx = kp.pt.x - refKpts.at(i, 0); + const float dy = kp.pt.y - refKpts.at(i, 1); + const float distSq = dx * dx + dy * dy; + if (distSq < bestDistSq) + bestDistSq = distSq; + } + if (bestDistSq < maxDistSq) + ++matched; + } + return matched; +} + +static int countDescriptorMatches(const std::vector& keypoints, const Mat& descriptors, + const Mat& refKpts, const Mat& refDesc, + float maxDistance, float maxL2Distance) +{ + const float maxDistSq = maxDistance * maxDistance; + int matched = 0; + for (int i = 0; i < descriptors.rows; ++i) + { + const KeyPoint& kp = keypoints[i]; + float bestDistSq = maxDistSq; + int bestIdx = -1; + for (int j = 0; j < refKpts.rows; ++j) + { + const float dx = kp.pt.x - refKpts.at(j, 0); + const float dy = kp.pt.y - refKpts.at(j, 1); + const float distSq = dx * dx + dy * dy; + if (distSq < bestDistSq) + { + bestDistSq = distSq; + bestIdx = j; + } + } + + if (bestIdx < 0) + continue; + + const double l2 = cvtest::norm(descriptors.row(i), refDesc.row(bestIdx), NORM_L2); + if (l2 <= maxL2Distance) + ++matched; + } + return matched; +} + +static void testXFeatRegression(const std::string& imageName, const std::string& tag) +{ + Mat refKpts = blobFromNPY(cvtest::findDataFile("dnn/xfeat_" + tag + "_640_kpts.npy")); + Mat refDesc = blobFromNPY(cvtest::findDataFile("dnn/xfeat_" + tag + "_640_desc.npy")); + if (refKpts.type() != CV_32F) + refKpts.convertTo(refKpts, CV_32F); + ASSERT_EQ(refKpts.cols, 3); + const int n = refKpts.rows; + ASSERT_GT(n, 0); + ASSERT_EQ(refDesc.rows, n); + + Ptr detector; + ASSERT_NO_THROW(detector = XFeat::create(cvtest::findDataFile("dnn/onnx/models/xfeat.onnx"), n, 0.5f, Size(640, 640))); + ASSERT_TRUE(detector); + EXPECT_FALSE(detector->empty()); + EXPECT_EQ(detector->descriptorSize(), 64); + EXPECT_EQ(detector->descriptorType(), CV_32F); + EXPECT_EQ(detector->defaultNorm(), NORM_L2); + + Mat img = imread(cvtest::findDataFile("shared/" + imageName)); + ASSERT_FALSE(img.empty()); + + std::vector keypoints; + Mat descriptors; + detector->detectAndCompute(img, noArray(), keypoints, descriptors); + + ASSERT_EQ(descriptors.rows, static_cast(keypoints.size())); + ASSERT_EQ(descriptors.cols, refDesc.cols); + ASSERT_EQ(descriptors.type(), CV_32F); + + const int matched = countNearbyKeypoints(keypoints, refKpts, 1.0f); + const double matchedRatio = static_cast(matched) / keypoints.size(); + EXPECT_GE(matchedRatio, 0.95) + << "only " << matched << " of " << keypoints.size() + << " keypoints matched reference within 1 px (" << tag << ")"; + + const int descMatched = countDescriptorMatches(keypoints, descriptors, refKpts, refDesc, 1.0f, 0.25f); + const double descMatchedRatio = static_cast(descMatched) / descriptors.rows; + EXPECT_GE(descMatchedRatio, 0.95) + << "only " << descMatched << " of " << descriptors.rows + << " descriptors matched reference (L2 <= 0.25 after 1 px keypoint association, " << tag << ")"; +} + +TEST(Features2d_XFeat, regression_box) +{ + testXFeatRegression("box.png", "box"); +} + +TEST(Features2d_XFeat, regression_box_in_scene) +{ + testXFeatRegression("box_in_scene.png", "box_in_scene"); +} + +TEST(Features2d_XFeat, Basic) +{ + Ptr detector = XFeat::create(cvtest::findDataFile("dnn/onnx/models/xfeat.onnx"), 200, 0.5f, Size(640, 640)); + ASSERT_TRUE(detector); + EXPECT_FALSE(detector->empty()); + EXPECT_EQ(detector->descriptorSize(), 64); + EXPECT_EQ(detector->descriptorType(), CV_32F); + EXPECT_EQ(detector->defaultNorm(), NORM_L2); + + Mat img = imread(cvtest::findDataFile("shared/box.png")); + ASSERT_FALSE(img.empty()); + + std::vector keypoints; + Mat descriptors; + detector->detectAndCompute(img, noArray(), keypoints, descriptors); + + ASSERT_FALSE(keypoints.empty()); + EXPECT_LE(keypoints.size(), 200u); + ASSERT_EQ(descriptors.rows, static_cast(keypoints.size())); + EXPECT_EQ(descriptors.cols, 64); + EXPECT_EQ(descriptors.type(), CV_32F); + + for (const KeyPoint& kp : keypoints) + { + EXPECT_GE(kp.pt.x, 0.f); + EXPECT_GE(kp.pt.y, 0.f); + EXPECT_LT(kp.pt.x, static_cast(img.cols)); + EXPECT_LT(kp.pt.y, static_cast(img.rows)); + EXPECT_GT(kp.response, 0.f); + } +} + +TEST(Features2d_XFeat, ParametersAndMask) +{ + Ptr detector = XFeat::create(cvtest::findDataFile("dnn/onnx/models/xfeat.onnx")); + ASSERT_TRUE(detector); + + detector->setMaxKeypoints(50); + detector->setScoreThreshold(0.25f); + detector->setInputSize(Size(640, 640)); + EXPECT_EQ(detector->getMaxKeypoints(), 50); + EXPECT_EQ(detector->getScoreThreshold(), 0.25f); + EXPECT_EQ(detector->getInputSize(), Size(640, 640)); + + Mat img = imread(cvtest::findDataFile("shared/box_in_scene.png")); + ASSERT_FALSE(img.empty()); + + Mat mask = Mat::zeros(img.size(), CV_8UC1); + const Rect roi(img.cols / 4, img.rows / 4, img.cols / 2, img.rows / 2); + mask(roi).setTo(255); + + std::vector keypoints; + Mat descriptors; + detector->detectAndCompute(img, mask, keypoints, descriptors); + + EXPECT_LE(keypoints.size(), 50u); + ASSERT_EQ(descriptors.rows, static_cast(keypoints.size())); + + for (const KeyPoint& kp : keypoints){ + EXPECT_TRUE(roi.contains(Point(cvFloor(kp.pt.x), cvFloor(kp.pt.y)))); + } + + Mat boolMask = Mat::zeros(img.size(), CV_BoolC1); + boolMask(roi).setTo(Scalar(1)); + EXPECT_NO_THROW(detector->detectAndCompute(img, boolMask, keypoints, descriptors)); +} + +TEST(Features2d_XFeat, InvalidInputSize) +{ + EXPECT_THROW(XFeat::create(cvtest::findDataFile("dnn/onnx/models/xfeat.onnx"), -1, 0.5f, Size(0, 640)), cv::Exception); + Ptr detector = XFeat::create(cvtest::findDataFile("dnn/onnx/models/xfeat.onnx")); + ASSERT_TRUE(detector); + EXPECT_THROW(detector->setInputSize(Size(0, 320)), cv::Exception); + EXPECT_NO_THROW(detector->setInputSize(Size(320, 320))); +} + +}} // namespace + +#endif // HAVE_OPENCV_DNN diff --git a/samples/cpp/example_features_aliked_lightglue.cpp b/samples/cpp/aliked_lightglue.cpp similarity index 80% rename from samples/cpp/example_features_aliked_lightglue.cpp rename to samples/cpp/aliked_lightglue.cpp index d562cbc110..110e97cd82 100644 --- a/samples/cpp/example_features_aliked_lightglue.cpp +++ b/samples/cpp/aliked_lightglue.cpp @@ -2,8 +2,8 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. -// ALIKED + LightGlueMatcher usage example -// Demonstrates feature detection, extraction, and matching using ALIKED and LightGlue. +// Learned feature usage examples. +// Demonstrates ALIKED + LightGlue matching and XFeat feature extraction. #include #include @@ -14,11 +14,47 @@ using namespace cv; using namespace std; +static int runXFeatExample(const String& imgPath, const String& xfeatModel, const String& outputPath) +{ + Mat img = imread(imgPath); + if (img.empty()) + { + cerr << "Error: cannot load image: " << imgPath << endl; + return -1; + } + + Ptr xfeat = XFeat::create(xfeatModel, 2000, 0.05f, Size(640, 640)); + vector keypoints; + Mat descriptors; + xfeat->detectAndCompute(img, Mat(), keypoints, descriptors); + + Mat canvas; + drawKeypoints(img, keypoints, canvas, Scalar(0, 255, 0), + DrawMatchesFlags::DRAW_RICH_KEYPOINTS); + + if (!outputPath.empty()) + { + imwrite(outputPath, canvas); + cout << "Saved XFeat keypoint visualization to: " << outputPath << endl; + } + + imshow("XFeat Keypoints", canvas); + cout << "Press any key to exit..." << endl; + waitKey(0); + return 0; +} + int main(int argc, char** argv) { // ---- Parse arguments ---- String alikedModel, lightglueModel, imgPath1, imgPath2; + if (argc >= 4 && String(argv[1]) == "--xfeat") + { + const String outputPath = argc >= 5 ? argv[4] : String(); + return runXFeatExample(argv[2], argv[3], outputPath); + } + if (argc >= 5) { imgPath1 = argv[1]; @@ -26,12 +62,15 @@ int main(int argc, char** argv) alikedModel = argv[3]; lightglueModel = argv[4]; } -else + else { - cout << "Usage: " << argv[0] << " " << endl; + cout << "Usage:" << endl; + cout << " " << argv[0] << " " << endl; + cout << " " << argv[0] << " --xfeat [output_image]" << endl; cout << endl; - cout << "Example:" << endl; + cout << "Examples:" << endl; cout << " " << argv[0] << " img1.jpg img2.jpg aliked-n16rot-top1k-640.onnx aliked_lightglue.onnx" << endl; + cout << " " << argv[0] << " --xfeat img.jpg xfeat.onnx xfeat_keypoints.jpg" << endl; return 0; } diff --git a/samples/cpp/stitching_detailed.cpp b/samples/cpp/stitching_detailed.cpp index b1779d2f6f..dce32ae727 100644 --- a/samples/cpp/stitching_detailed.cpp +++ b/samples/cpp/stitching_detailed.cpp @@ -47,11 +47,12 @@ static void printUsage(char** argv) "\nMotion Estimation Flags:\n" " --work_megapix \n" " Resolution for image registration step. The default is 0.6 Mpx.\n" - " --features (surf|orb|sift|akaze|aliked)\n" + " --features (surf|orb|sift|akaze|aliked|xfeat)\n" " Type of features used for images matching.\n" " The default is surf if available, orb otherwise.\n" " When using 'aliked', requires --matcher lightglue and DNN model paths.\n" - " --matcher (homography|affine)\n" + " When using 'xfeat', requires --xfeat_model and uses the standard matcher.\n" + " --matcher (homography|affine|lightglue)\n" " Matcher used for pairwise image matching.\n" " --estimator (homography|affine)\n" " Type of estimator used for transformation estimation.\n" @@ -107,13 +108,15 @@ static void printUsage(char** argv) " Output warped images separately as frames of a time lapse movie, with 'fixed_' prepended to input file names.\n" " --rangewidth \n" " uses range_width to limit number of images to match with.\n" - "\nDNN Feature Options (when --features aliked --matcher lightglue):\n" + "\nDNN Feature Options:\n" " --aliked_model \n" " Path to ALIKED ONNX model file.\n" " --lightglue_model \n" " Path to LightGlue ONNX model file (for ALIKED descriptors).\n" " --lg_score_thresh \n" - " LightGlue confidence threshold. The default is 0.0 (accept all).\n"; + " LightGlue confidence threshold. The default is 0.0 (accept all).\n" + " --xfeat_model \n" + " Path to XFeat ONNX model file.\n"; } @@ -154,6 +157,7 @@ bool timelapse = false; int range_width = -1; String aliked_model_path; String lightglue_model_path; +String xfeat_model_path; float lg_score_thresh = 0.0f; @@ -399,6 +403,11 @@ static int parseCmdArgs(int argc, char** argv) lightglue_model_path = argv[i + 1]; i++; } + else if (string(argv[i]) == "--xfeat_model") + { + xfeat_model_path = argv[i + 1]; + i++; + } else if (string(argv[i]) == "--lg_score_thresh") { lg_score_thresh = static_cast(atof(argv[i + 1])); @@ -423,6 +432,16 @@ static int parseCmdArgs(int argc, char** argv) cout << "Error: --features aliked requires --aliked_model and --lightglue_model\n"; return -1; } + if (features_type == "xfeat" && xfeat_model_path.empty()) + { + cout << "Error: --features xfeat requires --xfeat_model\n"; + return -1; + } + if (features_type == "xfeat" && matcher_type == "lightglue") + { + cout << "Error: --features xfeat does not support --matcher lightglue; use homography or affine\n"; + return -1; + } return 0; } @@ -444,7 +463,8 @@ int main(int argc, char* argv[]) // Disable OpenCL for DNN-based features to avoid backend sync issues bool use_aliked = (features_type == "aliked"); - if (use_aliked) + bool use_xfeat = (features_type == "xfeat"); + if (use_aliked || use_xfeat) cv::ocl::setUseOpenCL(false); // Check if have enough images @@ -464,9 +484,9 @@ int main(int argc, char* argv[]) #endif Ptr finder; - if (use_aliked) + if (features_type == "aliked") { - // ALIKED will be created per-image in the loop below + finder = ALIKED::create(aliked_model_path); } else if (features_type == "orb") { @@ -494,6 +514,15 @@ int main(int argc, char* argv[]) { finder = SIFT::create(); } + else if (features_type == "xfeat") + { +#ifdef HAVE_OPENCV_DNN + finder = XFeat::create(xfeat_model_path, 4096, 0.05f, Size(640, 640)); +#else + cout << "OpenCV is built without opencv_dnn module. XFeat algorithm is not available!" << std::endl; + return -1; +#endif + } else { cout << "Unknown 2D features type: '" << features_type << "'.\n"; @@ -538,15 +567,7 @@ int main(int argc, char* argv[]) is_seam_scale_set = true; } - if (use_aliked) - { - Ptr aliked = ALIKED::create(aliked_model_path); - computeImageFeatures(aliked, img, features[i]); - } - else - { - computeImageFeatures(finder, img, features[i]); - } + computeImageFeatures(finder, img, features[i]); features[i].img_idx = i; LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size());