mirror of
https://github.com/opencv/opencv.git
synced 2026-11-22 01:13:24 -06:00
[GSOC] feat: Add ALIKED feature extractor and LightGlue matcher with DNN integration #28986 ## PR Description ### Summary Integrate ALIKED and LightGlue into OpenCV's `features` module as native`Feature2D` and `DescriptorMatcher` implementations, enabling end-to-end neural feature matching within OpenCV's ecosystem. --- ### What's included #### New classes - **`cv::ALIKED`** extends `Feature2D` - CNN-based keypoint detection - 128-D descriptor extraction via ONNX Runtime - **`cv::LightGlueMatcher`** extends `DescriptorMatcher` - Deep feature matching with spatial context - Uses keypoints and image sizes during matching --- #### API design - Standard OpenCV patterns: - `detectAndCompute()` - `match()` - `knnMatch()` - Multiple factory methods: - ONNX model path - In-memory model buffer - Pre-loaded `dnn::Net` - `Params` structs use `CV_EXPORTS_W_SIMPLE` for Python/Java bindings support - Optional DNN dependency: - `HAVE_OPENCV_DNN` guards - Stub implementations throw `StsNotImplemented` --- ### Files added | File | Description | |------|-------------| | `src/feature2d_aliked.cpp` | ALIKED implementation | | `src/matchers_lightglue.cpp` | LightGlueMatcher implementation | | `src/aliked_context.hpp` | Shared internal context struct | | `test/test_aliked_lightglue.cpp` | Unit tests (9 test cases) | | `samples/cpp/example_features_aliked_lightglue.cpp` | Demo application | --- ### Files modified - `CMakeLists.txt` - Add `opencv_dnn` as optional dependency - `features.hpp` - Add ALIKED and LightGlueMatcher declarations - `precomp.hpp` - Add DNN include guard --- ### Usage ```cpp // Feature extraction Ptr<ALIKED> aliked = ALIKED::create("aliked-n16rot-top1k-640.onnx"); vector<KeyPoint> kpts; Mat descs; aliked->detectAndCompute(image, Mat(), kpts, descs); // Feature matching Ptr<LightGlueMatcher> lg = LightGlueMatcher::create("aliked_lightglue.onnx"); lg->setPairInfo( kpts1Mat, kpts2Mat, img1.size(), img2.size() ); vector<DMatch> matches; lg->match(descs1, descs2, matches); ```` please refer to samples/cpp/example_features_aliked_lightglue.cpp --- ### Test plan * Build with `BUILD_LIST=features,dnn` * Build without DNN: * Verify stubs compile * Verify `StsNotImplemented` is thrown * Run: * `ctest -R Features2d_ALIKED` * `ctest -R Features2d_LightGlueMatcher` * Run sample application with: * Real images * Real ONNX models * Verify Python/Java bindings compile and work --- ### Related Phase 1 of the "End-to-End AI Feature Extraction and LightGlue Matching Pipeline" GSoC project. Designed to be extensible to: * XFeat * SuperPoint * Other neural feature extractors ### test dependency Depends on the opencv_extra PR adding ALIKED and LightGlue test models: - [opencv_extra PR](https://github.com/opencv/opencv_extra/pull/1366) This PR adds the following ONNX models to `download_models.py`: - `aliked-n16rot-top1k-640.onnx` - `aliked_lightglue.onnx` These models are required for the `features2d` tests in the main OpenCV repository to validate the ALIKED and LightGlue feature extraction and matching pipeline. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
187 lines
5.2 KiB
C++
187 lines
5.2 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
|
|
#include "precomp.hpp"
|
|
|
|
#ifdef HAVE_OPENCV_DNN
|
|
#include "opencv2/dnn.hpp"
|
|
#include "aliked_context.hpp"
|
|
#endif
|
|
|
|
namespace cv
|
|
{
|
|
|
|
ALIKED::ALIKED() {}
|
|
ALIKED::~ALIKED() {}
|
|
|
|
ALIKED::Params::Params()
|
|
{
|
|
inputSize = Size(640, 640);
|
|
normalizeDescriptors = true;
|
|
#ifdef HAVE_OPENCV_DNN
|
|
engine = dnn::ENGINE_NEW;
|
|
backend = dnn::DNN_BACKEND_DEFAULT;
|
|
target = dnn::DNN_TARGET_CPU;
|
|
#else
|
|
engine = -1;
|
|
backend = -1;
|
|
target = -1;
|
|
#endif
|
|
}
|
|
|
|
#ifdef HAVE_OPENCV_DNN
|
|
|
|
class ALIKEDImpl : public ALIKED
|
|
{
|
|
public:
|
|
ALIKEDImpl(const ALIKED::Params& _params, const String& modelPath)
|
|
: params(_params)
|
|
{
|
|
net = dnn::readNet(modelPath, "", "", static_cast<dnn::EngineType>(params.engine));
|
|
CV_Assert(!net.empty());
|
|
net.setPreferableBackend(params.backend);
|
|
net.setPreferableTarget(params.target);
|
|
}
|
|
|
|
ALIKEDImpl(const std::vector<uchar>& modelData, const ALIKED::Params& _params)
|
|
: params(_params)
|
|
{
|
|
net = dnn::readNetFromONNX(modelData);
|
|
CV_Assert(!net.empty());
|
|
net.setPreferableBackend(params.backend);
|
|
net.setPreferableTarget(params.target);
|
|
}
|
|
|
|
void detectAndCompute(InputArray image, InputArray mask,
|
|
std::vector<KeyPoint>& keypoints,
|
|
OutputArray descriptors,
|
|
bool useProvidedKeypoints) CV_OVERRIDE;
|
|
|
|
int descriptorSize() const CV_OVERRIDE;
|
|
int descriptorType() const CV_OVERRIDE;
|
|
int defaultNorm() const CV_OVERRIDE;
|
|
bool empty() const CV_OVERRIDE;
|
|
|
|
const ALIKEDContext& getLastContext() const { return lastContext; }
|
|
|
|
protected:
|
|
dnn::Net net;
|
|
ALIKED::Params params;
|
|
ALIKEDContext lastContext;
|
|
|
|
void runNetwork(InputArray image, std::vector<KeyPoint>& keypoints,
|
|
Mat& descriptors, Mat& scores);
|
|
};
|
|
|
|
void ALIKEDImpl::runNetwork(InputArray _image, std::vector<KeyPoint>& keypoints,
|
|
Mat& descriptors, Mat& scores)
|
|
{
|
|
Mat image = _image.getMat();
|
|
Size inputSz = params.inputSize;
|
|
Size origSize = image.size();
|
|
|
|
// BGR->RGB conversion via swapRB=true
|
|
Mat blob = dnn::blobFromImage(image, 1.0/255.0, inputSz, Scalar(), /*swapRB=*/true, /*crop=*/false);
|
|
|
|
net.setInput(blob, "image");
|
|
|
|
std::vector<String> outNames = {"keypoints", "descriptors", "scores"};
|
|
std::vector<Mat> outputs;
|
|
net.forward(outputs, outNames);
|
|
|
|
CV_Assert(outputs.size() == 3);
|
|
|
|
// ORT engine drops the batch dimension, so outputs are:
|
|
// keypoints: [N, 2] (not [1, N, 2])
|
|
// descriptors: [N, 128] (not [1, N, 128])
|
|
// scores: [N] (not [1, N, 1])
|
|
int N = outputs[0].rows;
|
|
|
|
Mat normKpts = outputs[0].reshape(0, N); // Nx2
|
|
Mat desc = outputs[1].reshape(0, N); // Nx128
|
|
Mat scr = outputs[2].reshape(0, N); // Nx1
|
|
|
|
// Store normalized keypoints for LightGlue context
|
|
lastContext.normalizedKeypoints = normKpts.clone();
|
|
lastContext.imageSize = origSize;
|
|
|
|
// Convert normalized [-1,1] coordinates to pixel coordinates
|
|
keypoints.resize(N);
|
|
for (int i = 0; i < N; i++)
|
|
{
|
|
float nx = normKpts.at<float>(i, 0);
|
|
float ny = normKpts.at<float>(i, 1);
|
|
float px = (nx + 1.0f) * 0.5f * (float)origSize.width;
|
|
float py = (ny + 1.0f) * 0.5f * (float)origSize.height;
|
|
float score = scr.at<float>(i, 0);
|
|
keypoints[i] = KeyPoint(px, py, 1.0f, -1.0f, score, 0, -1);
|
|
}
|
|
|
|
// Optionally L2-normalize descriptors
|
|
if (params.normalizeDescriptors)
|
|
{
|
|
for (int i = 0; i < N; i++)
|
|
{
|
|
Mat row = desc.row(i);
|
|
normalize(row, row);
|
|
}
|
|
}
|
|
|
|
descriptors = desc;
|
|
scores = scr;
|
|
}
|
|
|
|
void ALIKEDImpl::detectAndCompute(InputArray image, InputArray mask,
|
|
std::vector<KeyPoint>& keypoints,
|
|
OutputArray descriptors,
|
|
bool useProvidedKeypoints)
|
|
{
|
|
CV_INSTRUMENT_REGION();
|
|
CV_UNUSED(mask);
|
|
CV_UNUSED(useProvidedKeypoints);
|
|
|
|
if (image.empty())
|
|
{
|
|
keypoints.clear();
|
|
descriptors.release();
|
|
return;
|
|
}
|
|
|
|
Mat desc;
|
|
Mat sc;
|
|
runNetwork(image, keypoints, desc, sc);
|
|
|
|
if (descriptors.needed())
|
|
desc.copyTo(descriptors);
|
|
}
|
|
|
|
int ALIKEDImpl::descriptorSize() const { return 128; }
|
|
int ALIKEDImpl::descriptorType() const { return CV_32F; }
|
|
int ALIKEDImpl::defaultNorm() const { return NORM_L2; }
|
|
bool ALIKEDImpl::empty() const { return net.empty(); }
|
|
|
|
Ptr<ALIKED> ALIKED::create(const String& modelPath, const ALIKED::Params& params)
|
|
{
|
|
return makePtr<ALIKEDImpl>(params, modelPath);
|
|
}
|
|
|
|
Ptr<ALIKED> ALIKED::create(const std::vector<uchar>& modelData, const ALIKED::Params& params)
|
|
{
|
|
return makePtr<ALIKEDImpl>(modelData, params);
|
|
}
|
|
|
|
#else // !HAVE_OPENCV_DNN
|
|
|
|
Ptr<ALIKED> ALIKED::create(const String& modelPath, const ALIKED::Params& params)
|
|
{
|
|
CV_UNUSED(modelPath);
|
|
CV_UNUSED(params);
|
|
CV_Error(cv::Error::StsNotImplemented,
|
|
"ALIKED requires OpenCV built with opencv_dnn module!");
|
|
}
|
|
|
|
#endif // HAVE_OPENCV_DNN
|
|
|
|
} // namespace cv
|