Files
opencv-MIRROR/modules/ptcloud/src/load_point_cloud.cpp
Savya Sanchi Sharma 303f83ef92 Merge pull request #29729 from SavyaSanchi-Sharma:vizsplat
Added Gaussian Splat Renderer Support - #29729

Adds 3D Gaussian Splatting support to `ptcloud`: a loader for trained scenes and a `viz3d` renderer that draws them alpha-blended and depth-sorted alongside existing opaque geometry.

### Result

<img width="1842" height="811" alt="splat" src="https://github.com/user-attachments/assets/abfa5c01-6dbb-4367-b426-a618e3748649" />

[.splat source](https://huggingface.co/cakewalk/splat-data/resolve/main/train.splat)

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
2026-10-12 16:33:54 +03:00

390 lines
11 KiB
C++

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "precomp.hpp"
#include "io_base.hpp"
#include "io_obj.hpp"
#include "io_ply.hpp"
#include "io_splat.hpp"
#include "utils.hpp"
#include "opencv2/core/utils/filesystem.private.hpp"
#include <opencv2/core/utils/logger.hpp>
#include <memory>
namespace cv {
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
static PointCloudDecoder findDecoder(const String &filename)
{
auto file_ext = getExtension(filename);
if (file_ext == "obj" || file_ext == "OBJ")
{
return std::unique_ptr<ObjDecoder>(new ObjDecoder());
}
if (file_ext == "ply" || file_ext == "PLY")
{
return std::unique_ptr<PlyDecoder>(new PlyDecoder());
}
return nullptr;
}
static PointCloudEncoder findEncoder(const String &filename)
{
auto file_ext = getExtension(filename);
if (file_ext == "obj" || file_ext == "OBJ")
{
return std::unique_ptr<ObjEncoder>(new ObjEncoder());
}
if (file_ext == "ply" || file_ext == "PLY")
{
return std::unique_ptr<PlyEncoder>(new PlyEncoder());
}
return nullptr;
}
#endif
void loadPointCloud(const String &filename, OutputArray vertices, OutputArray normals, OutputArray rgb)
{
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
auto decoder = findDecoder(filename);
if (!decoder) {
String file_ext = getExtension(filename);
CV_LOG_ERROR(NULL, "File extension '" << file_ext << "' is not supported");
return;
}
decoder->setSource(filename);
std::vector<Point3f> vec_vertices, vec_normals, vec_rgb;
decoder->readData(vec_vertices, vec_normals, vec_rgb);
if (!vec_vertices.empty())
Mat(static_cast<int>(vec_vertices.size()), 1, CV_32FC3, vec_vertices.data()).copyTo(vertices);
if (!vec_normals.empty() && normals.needed())
Mat(static_cast<int>(vec_normals.size()), 1, CV_32FC3, vec_normals.data()).copyTo(normals);
if (!vec_rgb.empty() && rgb.needed())
Mat(static_cast<int>(vec_rgb.size()), 1, CV_32FC3, vec_rgb.data()).copyTo(rgb);
#else // OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_UNUSED(filename);
CV_UNUSED(vertices);
CV_UNUSED(normals);
CV_UNUSED(rgb);
CV_LOG_WARNING(NULL, "File system support is disabled in this OpenCV build!");
#endif
}
void loadGaussianSplats(const String &filename, OutputArray splats,
const std::vector<String> &ply_properties)
{
splats.release();
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
String file_ext = getExtension(filename);
Mat decoded;
if (file_ext == "splat" || file_ext == "SPLAT")
{
if (!readSplatFile(filename, decoded))
return;
}
else if (file_ext == "ply" || file_ext == "PLY")
{
PlyDecoder decoder;
decoder.setSource(filename);
const std::vector<String> names =
ply_properties.empty() ? getGaussianSplatPlyPropNames() : ply_properties;
Mat raw;
if (!decoder.readNamedProperties(names, raw))
{
CV_LOG_ERROR(NULL, "'" << filename << "' is not a 3D Gaussian Splatting PLY file");
return;
}
decodeGaussianSplats(raw, decoded);
}
else
{
CV_LOG_ERROR(NULL, "Gaussian splats can only be loaded from PLY or SPLAT files, got '" << file_ext << "'");
return;
}
splats.move(decoded);
#else // OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_UNUSED(filename);
CV_UNUSED(splats);
CV_UNUSED(ply_properties);
CV_LOG_WARNING(NULL, "File system support is disabled in this OpenCV build!");
#endif
}
void savePointCloud(const String &filename, InputArray vertices, InputArray normals, InputArray rgb)
{
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
if (vertices.empty()) {
CV_LOG_WARNING(NULL, "Have no vertices to save");
return;
};
auto encoder = findEncoder(filename);
if (!encoder) {
String file_ext = getExtension(filename);
CV_LOG_ERROR(NULL, "File extension '" << file_ext << "' is not supported");
return;
}
encoder->setDestination(filename);
std::vector<Point3f> vec_vertices(vertices.getMat()), vec_normals, vec_rgb;
if (!normals.empty())
{
vec_normals = normals.getMat();
}
if (!rgb.empty())
{
vec_rgb = rgb.getMat();
}
encoder->writeData(vec_vertices, vec_normals, vec_rgb);
#else // OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_UNUSED(filename);
CV_UNUSED(vertices);
CV_UNUSED(normals);
CV_UNUSED(rgb);
CV_LOG_WARNING(NULL, "File system support is disabled in this OpenCV build!");
#endif
}
void loadMesh(const String &filename, OutputArray vertices, OutputArrayOfArrays indices,
OutputArray normals, OutputArray colors, OutputArray texCoords)
{
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_Assert(vertices.needed());
CV_Assert(indices.needed());
PointCloudDecoder decoder = findDecoder(filename);
String file_ext = getExtension(filename);
if (!decoder) {
CV_LOG_ERROR(NULL, "File extension '" << file_ext << "' is not supported");
return;
}
decoder->setSource(filename);
std::vector<Point3f> vec_vertices, vec_normals, vec_rgb;
std::vector<std::vector<int32_t>> vec_indices;
std::vector<Point3f> vec_texCoords;
int nTexCoords = 0;
decoder->readData(vec_vertices, vec_normals, vec_rgb, vec_texCoords, nTexCoords, vec_indices, 0);
if (!vec_vertices.empty())
{
Mat(1, static_cast<int>(vec_vertices.size()), CV_32FC3, vec_vertices.data()).copyTo(vertices);
}
if (normals.needed() && !vec_normals.empty())
{
Mat(1, static_cast<int>(vec_normals.size()), CV_32FC3, vec_normals.data()).copyTo(normals);
}
if (colors.needed() && !vec_rgb.empty())
{
Mat(1, static_cast<int>(vec_rgb.size()), CV_32FC3, vec_rgb.data()).copyTo(colors);
}
if (!vec_indices.empty())
{
_InputArray::KindFlag kind = indices.kind();
int vecsz = (int)vec_indices.size();
if (kind == _InputArray::KindFlag::STD_VECTOR_VECTOR)
{
CV_Assert(indices.depth() == CV_32S);
std::vector<std::vector<int32_t>>& vec = *(std::vector<std::vector<int32_t>>*)indices.getObj();
vec.resize(vecsz);
for (int i = 0; i < vecsz; ++i)
{
Mat(1, static_cast<int>(vec_indices[i].size()), CV_32SC1, vec_indices[i].data()).copyTo(vec[i]);
}
}
// std::array<Mat> has fixed size, unsupported
else if (kind == _InputArray::KindFlag::STD_VECTOR_MAT)
{
indices.create(vecsz, 1, CV_32S);
for (int i = 0; i < vecsz; i++)
{
std::vector<int> vi = vec_indices[i];
indices.create(1, (int)vi.size(), CV_32S, i);
Mat(vi).copyTo(indices.getMat(i));
}
}
else
{
std::vector<Vec3i> vec(vec_indices.size());
for (int i = 0; i < vecsz; ++i)
{
Vec3i tri;
size_t sz = vec_indices[i].size();
if (sz != 3)
{
CV_Error(Error::StsBadArg, "Face contains " + std::to_string(sz) + " vertices, can not put it into 3-channel indices array");
}
else
{
for (int j = 0; j < 3; j++)
{
tri[j] = vec_indices[i][j];
}
}
vec[i] = tri;
}
indices.create(1, (int)vec_indices.size(), CV_32SC3);
Mat(1, static_cast<int>(vec_indices.size()), CV_32SC3, vec.data()).copyTo(indices);
}
}
if (texCoords.needed())
{
if (nTexCoords)
{
CV_Assert(!texCoords.fixedType() || (texCoords.type() == CV_MAKE_TYPE(CV_32F, nTexCoords)));
Mat tex3(vec_texCoords);
if (nTexCoords == 3)
{
tex3.copyTo(texCoords);
}
else if (nTexCoords == 2)
{
// if texCoords is empty then channels() can be any number
bool has3ch = texCoords.channels() == 3;
int ch = has3ch ? 3 : 2;
std::vector<int> permut = has3ch ? std::vector<int>{ 0, 0, 1, 1, -1, 2 } : std::vector<int>{ 0, 0, 1, 1 };
texCoords.createSameSize(vec_texCoords, CV_MAKE_TYPE(CV_32F, ch));
Mat out = texCoords.getMat();
cv::mixChannels(tex3, out, permut);
}
}
else
{
texCoords.clear();
}
}
#else // OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_UNUSED(filename);
CV_UNUSED(vertices);
CV_UNUSED(normals);
CV_UNUSED(colors);
CV_UNUSED(indices);
CV_UNUSED(texCoords);
CV_LOG_WARNING(NULL, "File system support is disabled in this OpenCV build!");
#endif
}
void saveMesh(const String &filename, InputArray vertices, InputArrayOfArrays indices,
InputArray normals, InputArray colors, InputArray texCoords)
{
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
if (vertices.empty()) {
CV_LOG_WARNING(NULL, "Have no vertices to save");
return;
}
auto encoder = findEncoder(filename);
String file_ext = getExtension(filename);
if (!encoder) {
CV_LOG_ERROR(NULL, "File extension '" << file_ext << "' is not supported");
return;
}
encoder->setDestination(filename);
std::vector<Point3f> vec_vertices(vertices.getMat()), vec_normals, vec_rgb;
if (!normals.empty())
{
vec_normals = normals.getMat();
}
if (!colors.empty())
{
vec_rgb = colors.getMat();
}
std::vector<std::vector<int32_t>> vec_indices;
CV_Assert(indices.depth() == CV_32S);
if (indices.kind() == _InputArray::KindFlag::STD_VECTOR_VECTOR ||
indices.kind() == _InputArray::KindFlag::STD_VECTOR_MAT)
{
std::vector<Mat> mat_indices;
indices.getMatVector(mat_indices);
vec_indices.resize(mat_indices.size());
for (size_t i = 0; i < mat_indices.size(); ++i)
{
mat_indices[i].copyTo(vec_indices[i]);
}
}
else
{
CV_Assert(indices.channels() == 3);
std::vector<Vec3i>& vec = *(std::vector<Vec3i>*)indices.getObj();
vec_indices.resize(vec.size());
for (size_t i = 0; i < vec.size(); ++i)
{
for (int j = 0; j < 3; j++)
{
vec_indices[i].push_back(vec[i][j]);
}
}
}
std::vector<Point3f> vec_texCoords;
int nTexCoords = 0;
if (!texCoords.empty())
{
nTexCoords = texCoords.channels();
}
if (nTexCoords == 2)
{
// extend by 3rd zero channel
vec_texCoords.resize(texCoords.total());
cv::mixChannels(texCoords, vec_texCoords, {0, 0, 1, 1, -1, 2});
}
if (nTexCoords == 3)
{
texCoords.copyTo(vec_texCoords);
}
encoder->writeData(vec_vertices, vec_normals, vec_rgb, vec_texCoords, nTexCoords, vec_indices);
#else // OPENCV_HAVE_FILESYSTEM_SUPPORT
CV_UNUSED(filename);
CV_UNUSED(vertices);
CV_UNUSED(colors);
CV_UNUSED(normals);
CV_UNUSED(indices);
CV_UNUSED(texCoords);
CV_LOG_WARNING(NULL, "File system support is disabled in this OpenCV build!");
#endif
}
}/* namespace cv */