Files
opencv-MIRROR/modules/features/src/feature2d_xfeat.cpp
Shelia J. fb90451e95 Merge pull request #29361 from SheliaJimenez:xfeat-feature
Xfeat feature - #29361

## PR Description

### Summary

Integrate XFeat into OpenCV's `features` module as a native `Feature2D` implementation, enabling lightweight neural feature detection and descriptor extraction through OpenCV's standard feature extraction API.

---

### What's included

#### New class

- **`cv::XFeat`** extends `Feature2D`
  - CNN-based keypoint detection
  - 64-D descriptor extraction via ONNX/DNN
  - Score-map based keypoint selection
  - Descriptor sampling from the dense feature map

---

### Files added

| File | Description |
|------|-------------|
| `src/feature2d_xfeat.cpp` | XFeat `Feature2D` implementation |
| `test/test_xfeat.cpp` | XFeat unit and regression tests |

---

### Files modified

- `features.hpp`
  - Add `cv::XFeat` declaration and public factory APIs

---

### Usage

```cpp
#include <opencv2/features.hpp>

using namespace cv;

// Feature extraction
Ptr<XFeat> xfeat =
    XFeat::create("xfeat.onnx", 2000, 0.5f, 640);

std::vector<KeyPoint> keypoints;
Mat descriptors;

xfeat->detectAndCompute(image, noArray(), keypoints, descriptors);
```

---


### Test dependency

Depends on the opencv_extra changes adding the XFeat ONNX model and reference outputs.

Required test data:
https://github.com/opencv/opencv_extra/pull/1383

- `xfeat.onnx`
- `xfeat_lena_640_kpts.npy`
- `xfeat_lena_640_desc.npy`

These files are required for the `Features2d_XFeat` tests in the main OpenCV repository to validate XFeat feature extraction and descriptor generation.


### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2026-08-14 15:07:52 +03:00

454 lines
16 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "precomp.hpp"
#ifdef HAVE_OPENCV_DNN
#include <cfloat>
#include <cmath>
#include <numeric>
namespace cv {
using namespace dnn;
namespace {
static const int kXFeatDescriptorSize = 64;
static const std::vector<String> kXFeatOutputNames =
{
"output_feats",
"output_keypoints",
"output_heatmap"
};
struct XFeatCandidate
{
Point2f ptPadded;
Point2f pt;
float score;
};
static Mat toGray(InputArray _image)
{
Mat image = _image.getMat();
if (image.channels() == 1)
return image;
Mat gray;
if (image.channels() == 3)
cvtColor(image, gray, COLOR_BGR2GRAY);
else if (image.channels() == 4)
cvtColor(image, gray, COLOR_BGRA2GRAY);
else
CV_Error(Error::StsBadArg, "XFeat expects a grayscale, BGR, or BGRA image");
return gray;
}
static Mat toNCHW(const Mat& blob, int channelsHint)
{
CV_Assert(blob.dims == 4);
if (blob.size[1] == channelsHint)
return blob;
CV_Assert(blob.size[3] == channelsHint);
Mat out;
Mat src = blob.isContinuous() ? blob : blob.clone();
transposeND(src, {0, 3, 1, 2}, out);
return out;
}
static float sampleNearest(const Mat& map, float x, float y, int normW, int normH)
{
CV_Assert(map.type() == CV_32F);
if (map.empty() || normW <= 1 || normH <= 1)
return 0.0f;
const float fx = x * static_cast<float>(map.cols - 1) / static_cast<float>(normW - 1);
const float fy = y * static_cast<float>(map.rows - 1) / static_cast<float>(normH - 1);
const int ix = std::max(0, std::min(map.cols - 1, cvRound(fx)));
const int iy = std::max(0, std::min(map.rows - 1, cvRound(fy)));
return map.at<float>(iy, ix);
}
static float sampleBilinear(const Mat& map, float x, float y, int normW, int normH)
{
CV_Assert(map.type() == CV_32F);
if (map.empty() || normW <= 1 || normH <= 1)
return 0.0f;
float fx = x * static_cast<float>(map.cols - 1) / static_cast<float>(normW - 1);
float fy = y * static_cast<float>(map.rows - 1) / static_cast<float>(normH - 1);
fx = std::max(0.0f, std::min(fx, static_cast<float>(map.cols - 1)));
fy = std::max(0.0f, std::min(fy, static_cast<float>(map.rows - 1)));
const int x0 = cvFloor(fx);
const int y0 = cvFloor(fy);
const int x1 = std::min(x0 + 1, map.cols - 1);
const int y1 = std::min(y0 + 1, map.rows - 1);
const float dx = fx - x0;
const float dy = fy - y0;
const float v00 = map.at<float>(y0, x0);
const float v01 = map.at<float>(y0, x1);
const float v10 = map.at<float>(y1, x0);
const float v11 = map.at<float>(y1, x1);
return (1.f - dx) * (1.f - dy) * v00 +
dx * (1.f - dy) * v01 +
(1.f - dx) * dy * v10 +
dx * dy * v11;
}
} // namespace
class XFeat_Impl CV_FINAL : public XFeat
{
public:
XFeat_Impl(const String& modelPath, int maxKeypoints, float scoreThreshold,
const Size& inputSize, int backendId, int targetId)
: maxKeypoints_(maxKeypoints),
scoreThreshold_(scoreThreshold),
inputSize_(inputSize)
{
CV_Assert(inputSize_.width > 0 && inputSize_.height > 0);
initNet(readNetFromONNX(modelPath), backendId, targetId);
}
XFeat_Impl(const std::vector<uchar>& bufferModel, int maxKeypoints, float scoreThreshold,
const Size& inputSize, int backendId, int targetId)
: maxKeypoints_(maxKeypoints),
scoreThreshold_(scoreThreshold),
inputSize_(inputSize)
{
CV_Assert(inputSize_.width > 0 && inputSize_.height > 0);
initNet(readNetFromONNX(bufferModel), backendId, targetId);
}
void detectAndCompute(InputArray _image, InputArray _mask,
std::vector<KeyPoint>& keypoints,
OutputArray _descriptors,
bool useProvidedKeypoints) CV_OVERRIDE
{
CV_Assert(!useProvidedKeypoints && "XFeat does not support providing keypoints externally");
keypoints.clear();
Mat image = _image.getMat();
if (image.empty())
{
if (_descriptors.needed())
_descriptors.release();
return;
}
Mat mask = _mask.getMat();
if (!mask.empty())
{
CV_Assert(mask.type() == CV_8UC1 || mask.type() == CV_BoolC1);
CV_Assert(mask.size() == image.size());
}
Mat gray = toGray(image);
const float scaleX = static_cast<float>(inputSize_.width) / static_cast<float>(gray.cols);
const float scaleY = static_cast<float>(inputSize_.height) / static_cast<float>(gray.rows);
const float scale = std::min(scaleX, scaleY);
const int resizedW = std::max(1, cvRound(static_cast<float>(gray.cols) * scale));
const int resizedH = std::max(1, cvRound(static_cast<float>(gray.rows) * scale));
const int padX = (inputSize_.width - resizedW) / 2;
const int padY = (inputSize_.height - resizedH) / 2;
Mat blob;
Image2BlobParams blobParams;
blobParams.scalefactor = Scalar::all(1.0 / 255.0);
blobParams.size = inputSize_;
blobParams.mean = Scalar();
blobParams.swapRB = false;
blobParams.ddepth = CV_32F;
blobParams.datalayout = DNN_LAYOUT_NCHW;
blobParams.paddingmode = DNN_PMODE_LETTERBOX;
blobParams.borderValue = Scalar();
blobFromImageWithParams(gray, blob, blobParams);
net_.setInput(blob);
std::vector<Mat> outs;
net_.forward(outs, kXFeatOutputNames);
CV_Assert(outs.size() == 3);
Mat featBlob = toNCHW(outs[0], kXFeatDescriptorSize);
Mat kptBlob = outs[1];
CV_Assert(kptBlob.dims == 4);
if (!(kptBlob.size[1] == 64 || kptBlob.size[1] == 65))
{
CV_Assert(kptBlob.size[3] == 64 || kptBlob.size[3] == 65);
Mat src = kptBlob.isContinuous() ? kptBlob : kptBlob.clone();
transposeND(src, {0, 3, 1, 2}, kptBlob);
}
Mat relBlob = toNCHW(outs[2], 1);
CV_Assert(featBlob.dims == 4 && kptBlob.dims == 4 && relBlob.dims == 4);
if (!featBlob.isContinuous())
featBlob = featBlob.clone();
if (!kptBlob.isContinuous())
kptBlob = kptBlob.clone();
if (!relBlob.isContinuous())
relBlob = relBlob.clone();
const int featH = featBlob.size[2];
const int featW = featBlob.size[3];
const int kptC = kptBlob.size[1];
const int kptH = kptBlob.size[2];
const int kptW = kptBlob.size[3];
CV_Assert(featBlob.size[1] == kXFeatDescriptorSize && (kptC == 64 || kptC == 65));
Mat reliability(relBlob.size[2], relBlob.size[3], CV_32F, relBlob.ptr<float>());
Mat heatmap = Mat::zeros(kptH * 8, kptW * 8, CV_32F);
const float* kptPtr = kptBlob.ptr<float>();
const int kptHW = kptH * kptW;
parallel_for_(Range(0, kptH), [&](const Range& range)
{
for (int y = range.start; y < range.end; ++y)
{
for (int x = 0; x < kptW; ++x)
{
const int offset = y * kptW + x;
float maxLogit = -FLT_MAX;
for (int ch = 0; ch < kptC; ++ch)
maxLogit = std::max(maxLogit, kptPtr[ch * kptHW + offset]);
float sumExp = 0.f;
float logits[64];
float probs[64];
for (int ch = 0; ch < 64; ++ch)
logits[ch] = kptPtr[ch * kptHW + offset] - maxLogit;
#if (defined(CV_SIMD) && CV_SIMD) || (defined(CV_SIMD_SCALABLE) && CV_SIMD_SCALABLE)
const int vlanes = VTraits<v_float32>::vlanes();
int ch = 0;
v_float32 vSum = vx_setzero_f32();
for (; ch <= 64 - vlanes; ch += vlanes)
{
v_float32 v = vx_load(logits + ch);
v_float32 e = v_exp(v);
vx_store(probs + ch, e);
vSum = v_add(vSum, e);
}
sumExp = v_reduce_sum(vSum);
for (; ch < 64; ++ch)
{
probs[ch] = std::exp(logits[ch]);
sumExp += probs[ch];
}
#else
for (int ch = 0; ch < 64; ++ch)
{
probs[ch] = std::exp(logits[ch]);
sumExp += probs[ch];
}
#endif
if (kptC == 65)
sumExp += std::exp(kptPtr[64 * kptHW + offset] - maxLogit);
if (sumExp <= 0.f)
continue;
for (int ch = 0; ch < 64; ++ch)
{
const int dy = ch / 8;
const int dx = ch % 8;
heatmap.at<float>(y * 8 + dy, x * 8 + dx) = probs[ch] / sumExp;
}
}
}
});
Mat localMax;
dilate(heatmap, localMax, getStructuringElement(MORPH_RECT, Size(5, 5)));
std::vector<XFeatCandidate> candidates;
candidates.reserve(4096);
for (int y = 0; y < heatmap.rows; ++y)
{
const float* hm = heatmap.ptr<float>(y);
const float* mx = localMax.ptr<float>(y);
for (int x = 0; x < heatmap.cols; ++x)
{
const float h = hm[x];
if (h <= scoreThreshold_ || h != mx[x])
continue;
const float xp = static_cast<float>(x);
const float yp = static_cast<float>(y);
const float score = sampleNearest(heatmap, xp, yp, inputSize_.width, inputSize_.height) *
sampleBilinear(reliability, xp, yp, inputSize_.width, inputSize_.height);
if (score <= 0.f)
continue;
const float px = (xp - static_cast<float>(padX)) / scale;
const float py = (yp - static_cast<float>(padY)) / scale;
const int ix = cvFloor(px);
const int iy = cvFloor(py);
if (ix < 0 || iy < 0 || ix >= image.cols || iy >= image.rows)
continue;
if (!mask.empty() && mask.at<uchar>(iy, ix) == 0)
continue;
candidates.push_back({Point2f(xp, yp), Point2f(px, py), score});
}
}
if (maxKeypoints_ > 0 && static_cast<int>(candidates.size()) > maxKeypoints_)
{
std::partial_sort(candidates.begin(), candidates.begin() + maxKeypoints_, candidates.end(),
[](const XFeatCandidate& a, const XFeatCandidate& b)
{
return a.score > b.score;
});
candidates.resize(maxKeypoints_);
}
keypoints.reserve(candidates.size());
for (const XFeatCandidate& c : candidates)
keypoints.emplace_back(c.pt, 1.0f, -1.0f, c.score);
if (_descriptors.needed())
{
if (candidates.empty())
{
_descriptors.release();
return;
}
_descriptors.create(static_cast<int>(candidates.size()), kXFeatDescriptorSize, CV_32F);
Mat descriptors = _descriptors.getMat();
const float* featPtr = featBlob.ptr<float>();
const int featHW = featH * featW;
parallel_for_(Range(0, static_cast<int>(candidates.size())),
[&](const Range& range)
{
for (int i = range.start; i < range.end; ++i)
{
float* dst = descriptors.ptr<float>(i);
const XFeatCandidate& c = candidates[i];
for (int ch = 0; ch < kXFeatDescriptorSize; ++ch)
{
Mat channel(featH, featW, CV_32F,
const_cast<float*>(featPtr + ch * featHW));
dst[ch] = sampleBilinear(channel, c.ptPadded.x, c.ptPadded.y,
inputSize_.width, inputSize_.height);
}
normalize(descriptors.row(i), descriptors.row(i), 1.0, 0.0, NORM_L2);
}
});
}
}
int descriptorSize() const CV_OVERRIDE { return kXFeatDescriptorSize; }
int descriptorType() const CV_OVERRIDE { return CV_32F; }
int defaultNorm() const CV_OVERRIDE { return NORM_L2; }
bool empty() const CV_OVERRIDE { return net_.empty(); }
void setMaxKeypoints(int maxKeypoints) CV_OVERRIDE { maxKeypoints_ = maxKeypoints; }
int getMaxKeypoints() const CV_OVERRIDE { return maxKeypoints_; }
void setScoreThreshold(float threshold) CV_OVERRIDE { scoreThreshold_ = threshold; }
float getScoreThreshold() const CV_OVERRIDE { return scoreThreshold_; }
void setInputSize(const Size& inputSize) CV_OVERRIDE
{
CV_Assert(inputSize.width > 0 && inputSize.height > 0);
inputSize_ = inputSize;
}
Size getInputSize() const CV_OVERRIDE { return inputSize_; }
String getDefaultName() const CV_OVERRIDE { return Feature2D::getDefaultName() + ".XFeat"; }
private:
void initNet(const Net& net, int backendId, int targetId)
{
net_ = net;
net_.setPreferableBackend(backendId);
net_.setPreferableTarget(targetId);
// Check output names once and fail early.
const std::vector<String> modelOutNames = net_.getUnconnectedOutLayersNames();
if (modelOutNames.size() != kXFeatOutputNames.size())
{
String msg = "XFeat ONNX output count mismatch: expected " +
std::to_string(kXFeatOutputNames.size()) + " outputs (";
for (size_t i = 0; i < kXFeatOutputNames.size(); ++i)
{
msg += kXFeatOutputNames[i];
if (i + 1 < kXFeatOutputNames.size())
msg += ", ";
}
msg += "), but model has " + std::to_string(modelOutNames.size()) + " outputs: ";
for (size_t i = 0; i < modelOutNames.size(); ++i)
{
msg += modelOutNames[i];
if (i + 1 < modelOutNames.size())
msg += ", ";
}
CV_Error(Error::StsError, msg);
}
for (size_t i = 0; i < kXFeatOutputNames.size(); ++i)
{
bool found = false;
for (size_t j = 0; j < modelOutNames.size(); ++j)
{
if (modelOutNames[j] == kXFeatOutputNames[i])
{
found = true;
break;
}
}
if (!found)
{
String msg = "XFeat ONNX output name mismatch: expected output '" + kXFeatOutputNames[i] +
"'. Please check model outputs. Available outputs: ";
for (size_t j = 0; j < modelOutNames.size(); ++j)
{
msg += modelOutNames[j];
if (j + 1 < modelOutNames.size())
msg += ", ";
}
CV_Error(Error::StsError, msg);
}
}
}
int maxKeypoints_;
float scoreThreshold_;
Size inputSize_;
Net net_;
};
Ptr<XFeat> XFeat::create(const String& modelPath, int maxKeypoints, float scoreThreshold,
const Size& inputSize, int backendId, int targetId)
{
CV_TRACE_FUNCTION();
return makePtr<XFeat_Impl>(modelPath, maxKeypoints, scoreThreshold,
inputSize, backendId, targetId);
}
Ptr<XFeat> XFeat::create(const std::vector<uchar>& bufferModel, int maxKeypoints,
float scoreThreshold, const Size& inputSize, int backendId, int targetId)
{
CV_TRACE_FUNCTION();
return makePtr<XFeat_Impl>(bufferModel, maxKeypoints, scoreThreshold,
inputSize, backendId, targetId);
}
String XFeat::getDefaultName() const
{
return Feature2D::getDefaultName() + ".XFeat";
}
} // namespace cv
#endif // HAVE_OPENCV_DNN