mirror of
https://github.com/opencv/opencv.git
synced 2026-11-16 14:27:47 -06:00
2244 lines
82 KiB
C++
2244 lines
82 KiB
C++
// This file is part of OpenCV project.
|
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
|
// of this distribution and at http://opencv.org/license.html.
|
|
|
|
#include "precomp.hpp"
|
|
|
|
#include "net_impl.hpp"
|
|
|
|
#include <limits>
|
|
|
|
#ifdef HAVE_CUDA
|
|
#include <opencv2/core/cuda_stream_accessor.hpp>
|
|
#endif
|
|
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
#include <onnxruntime_cxx_api.h>
|
|
#endif
|
|
|
|
namespace cv {
|
|
namespace dnn {
|
|
CV__DNN_INLINE_NS_BEGIN
|
|
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
struct OrtNamesCache
|
|
{
|
|
std::vector<std::string> input_names;
|
|
std::vector<std::string> output_names;
|
|
std::unordered_map<std::string, int> input_name_to_index;
|
|
std::unordered_map<std::string, int> output_name_to_index;
|
|
|
|
explicit OrtNamesCache(Ort::Session& session)
|
|
{
|
|
Ort::AllocatorWithDefaultOptions allocator;
|
|
|
|
const size_t ninputs = session.GetInputCount();
|
|
input_names.reserve(ninputs);
|
|
for (size_t i = 0; i < ninputs; ++i)
|
|
{
|
|
Ort::AllocatedStringPtr in = session.GetInputNameAllocated(i, allocator);
|
|
std::string n = in ? std::string(in.get()) : std::string();
|
|
input_name_to_index[n] = (int)i;
|
|
input_names.push_back(std::move(n));
|
|
}
|
|
|
|
const size_t noutputs = session.GetOutputCount();
|
|
output_names.reserve(noutputs);
|
|
for (size_t i = 0; i < noutputs; ++i)
|
|
{
|
|
Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator);
|
|
std::string n = out ? std::string(out.get()) : std::string();
|
|
output_name_to_index[n] = (int)i;
|
|
output_names.push_back(std::move(n));
|
|
}
|
|
}
|
|
};
|
|
#endif
|
|
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
void Net::Impl::applyStagedOrtInputs()
|
|
{
|
|
if (!ort_session || ort_staged_inputs.empty())
|
|
return;
|
|
|
|
if (!ort_names_cache)
|
|
ort_names_cache = std::make_shared<OrtNamesCache>(*ort_session);
|
|
|
|
OrtNamesCache& names = *ort_names_cache;
|
|
const size_t ninputs = names.input_names.size();
|
|
if (ninputs == 0)
|
|
CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs");
|
|
|
|
if (!netInputLayer)
|
|
{
|
|
netInputLayer = Ptr<DataLayer>(new DataLayer());
|
|
netInputLayer->name = "ort_data_layer";
|
|
netInputLayer->type = "Data";
|
|
}
|
|
if (netInputLayer->blobs.size() != ninputs)
|
|
netInputLayer->blobs.resize(ninputs);
|
|
|
|
for (size_t k = 0; k < ort_staged_inputs.size(); ++k)
|
|
{
|
|
const std::string& inpname = ort_staged_inputs[k].first;
|
|
const Mat& inputMat = ort_staged_inputs[k].second;
|
|
if (inputMat.empty())
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty");
|
|
|
|
size_t inputIdx = 0;
|
|
if (inpname.empty())
|
|
{
|
|
if (ninputs != 1)
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: input name must be specified for models with multiple inputs");
|
|
inputIdx = 0;
|
|
}
|
|
else
|
|
{
|
|
auto it = names.input_name_to_index.find(inpname);
|
|
if (it == names.input_name_to_index.end())
|
|
CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: input '%s' is not found", inpname.c_str()));
|
|
inputIdx = (size_t)it->second;
|
|
}
|
|
inputMat.copyTo(netInputLayer->blobs[inputIdx]);
|
|
}
|
|
ort_staged_inputs.clear();
|
|
}
|
|
|
|
static int cvTypeFromONNXElemType(const ONNXTensorElementDataType t)
|
|
{
|
|
switch (t)
|
|
{
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return CV_32F;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return CV_8U;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return CV_8S;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return CV_16U;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return CV_16S;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return CV_32S;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return CV_64S;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return CV_8U;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return CV_64F;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return CV_16F;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:return CV_16BF;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return CV_32U;
|
|
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return CV_64U;
|
|
default:
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
static Ort::Value createOrtTensorFromMat(Ort::Session& session,
|
|
size_t inputIdx,
|
|
const Ort::MemoryInfo& memory_info,
|
|
Mat& inputBlob,
|
|
std::vector<int64_t>& inputDims,
|
|
ONNXTensorElementDataType& in_elem_type)
|
|
{
|
|
Ort::TypeInfo in_type_info = session.GetInputTypeInfo(inputIdx);
|
|
Ort::ConstTensorTypeAndShapeInfo in_tensor_info = in_type_info.GetTensorTypeAndShapeInfo();
|
|
in_elem_type = in_tensor_info.GetElementType();
|
|
|
|
const int cvInType = cvTypeFromONNXElemType(in_elem_type);
|
|
if (cvInType < 0)
|
|
CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT input element type: %d", (int)in_elem_type));
|
|
|
|
if (inputBlob.type() != cvInType)
|
|
inputBlob.convertTo(inputBlob, cvInType);
|
|
|
|
if (!inputBlob.isContinuous())
|
|
inputBlob = inputBlob.clone();
|
|
|
|
inputDims.clear();
|
|
inputDims.reserve((size_t)inputBlob.dims);
|
|
for (int i = 0; i < inputBlob.dims; i++)
|
|
inputDims.push_back((int64_t)inputBlob.size[i]);
|
|
|
|
const size_t nbytes = (size_t)inputBlob.total() * inputBlob.elemSize();
|
|
OrtValue* input_tensor_raw = nullptr;
|
|
Ort::ThrowOnError(Ort::GetApi().CreateTensorWithDataAsOrtValue(
|
|
memory_info,
|
|
inputBlob.data,
|
|
nbytes,
|
|
inputDims.data(),
|
|
inputDims.size(),
|
|
in_elem_type,
|
|
&input_tensor_raw));
|
|
return Ort::Value(input_tensor_raw);
|
|
}
|
|
|
|
std::vector<Mat> Net::Impl::runOrtSession(std::vector<Mat> inputBlobs, const std::vector<int>& outIdxs)
|
|
{
|
|
CV_Assert(this->ort_session);
|
|
Ort::Session& session = *this->ort_session;
|
|
|
|
if (!this->ort_names_cache)
|
|
this->ort_names_cache = std::make_shared<OrtNamesCache>(session);
|
|
|
|
OrtNamesCache& names = *this->ort_names_cache;
|
|
if (names.input_names.empty())
|
|
CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs");
|
|
if (names.output_names.empty())
|
|
CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs");
|
|
|
|
const size_t ninputs = names.input_names.size();
|
|
if (inputBlobs.size() != ninputs)
|
|
CV_Error_(Error::StsBadArg, ("DNN/ORT: expected %zu inputs, but got %zu", ninputs, inputBlobs.size()));
|
|
|
|
std::vector<const char*> in_names;
|
|
in_names.reserve(ninputs);
|
|
for (size_t i = 0; i < ninputs; ++i)
|
|
in_names.push_back(names.input_names[i].c_str());
|
|
|
|
std::vector<const char*> out_names;
|
|
if (outIdxs.empty())
|
|
{
|
|
out_names.reserve(names.output_names.size());
|
|
for (const std::string& n : names.output_names)
|
|
out_names.push_back(n.c_str());
|
|
}
|
|
else
|
|
{
|
|
out_names.reserve(outIdxs.size());
|
|
for (int idx : outIdxs)
|
|
{
|
|
CV_CheckGE(idx, 0, "DNN/ORT: output index must be non-negative");
|
|
CV_CheckLT((size_t)idx, names.output_names.size(), "DNN/ORT: output index is out of range");
|
|
out_names.push_back(names.output_names[(size_t)idx].c_str());
|
|
}
|
|
}
|
|
|
|
static const Ort::MemoryInfo memory_info =
|
|
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
|
|
|
|
std::vector<Ort::Value> input_tensors;
|
|
input_tensors.reserve(ninputs);
|
|
for (size_t i = 0; i < ninputs; ++i)
|
|
{
|
|
if (inputBlobs[i].empty())
|
|
CV_Error_(Error::StsError, ("DNN/ORT: input '%s' is empty", names.input_names[i].c_str()));
|
|
|
|
std::vector<int64_t> inputDims;
|
|
ONNXTensorElementDataType in_elem_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;
|
|
input_tensors.push_back(createOrtTensorFromMat(session, i, memory_info, inputBlobs[i], inputDims, in_elem_type));
|
|
}
|
|
|
|
std::vector<Ort::Value> output_tensors = session.Run(
|
|
Ort::RunOptions{nullptr},
|
|
in_names.data(), input_tensors.data(), input_tensors.size(),
|
|
out_names.data(), out_names.size());
|
|
if (profilingMode != DNN_PROFILE_NONE) ort_profile_runs++;
|
|
|
|
CV_CheckEQ(output_tensors.size(), out_names.size(), "DNN/ORT: ORT returned unexpected number of outputs");
|
|
|
|
std::vector<Mat> results;
|
|
results.reserve(output_tensors.size());
|
|
|
|
for (Ort::Value& outv : output_tensors)
|
|
{
|
|
Ort::TensorTypeAndShapeInfo shape_info = outv.GetTensorTypeAndShapeInfo();
|
|
std::vector<int64_t> out_shape = shape_info.GetShape();
|
|
|
|
std::vector<int> out_dims;
|
|
out_dims.reserve(out_shape.size());
|
|
for (int64_t d : out_shape)
|
|
{
|
|
if (d < 0)
|
|
CV_Error(Error::StsError, "DNN/ORT: dynamic output shapes are not supported at runtime");
|
|
if (d > (int64_t)std::numeric_limits<int>::max())
|
|
CV_Error(Error::StsError, "DNN/ORT: output shape dimension is too large");
|
|
out_dims.push_back((int)d);
|
|
}
|
|
|
|
const ONNXTensorElementDataType out_elem_type = shape_info.GetElementType();
|
|
const int cvOutType = cvTypeFromONNXElemType(out_elem_type);
|
|
if (cvOutType < 0)
|
|
CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT output element type: %d", (int)out_elem_type));
|
|
|
|
uint8_t* out_bytes = outv.GetTensorMutableData<uint8_t>();
|
|
Mat view(out_dims, cvOutType, out_bytes);
|
|
results.push_back(view.clone()); // detach from ORT-owned memory
|
|
}
|
|
|
|
return results;
|
|
}
|
|
#endif
|
|
|
|
std::string modelFormatToString(ModelFormat modelFormat)
|
|
{
|
|
return
|
|
modelFormat == DNN_MODEL_ONNX ? "ONNX" :
|
|
modelFormat == DNN_MODEL_TF ? "TF" :
|
|
modelFormat == DNN_MODEL_TFLITE ? "TFLite" : "Unknown/Generic";
|
|
}
|
|
|
|
std::string argKindToString(ArgKind kind)
|
|
{
|
|
return
|
|
kind == DNN_ARG_CONST ? "Const" :
|
|
kind == DNN_ARG_INPUT ? "Input" :
|
|
kind == DNN_ARG_OUTPUT ? "Output" :
|
|
kind == DNN_ARG_TEMP ? "Temp" :
|
|
kind == DNN_ARG_PATTERN ? "Pattern" : "???";
|
|
}
|
|
|
|
ArgData::ArgData()
|
|
{
|
|
kind = DNN_ARG_EMPTY;
|
|
type = -1;
|
|
}
|
|
|
|
class GraphImpl : public Graph
|
|
{
|
|
public:
|
|
GraphImpl(Net::Impl* netimpl, const std::string& name,
|
|
const std::vector<Arg>& inputs)
|
|
{
|
|
netimpl_ = netimpl;
|
|
name_ = name;
|
|
inputs_ = inputs;
|
|
}
|
|
|
|
virtual ~GraphImpl()
|
|
{
|
|
}
|
|
|
|
virtual std::string name() const override { return name_; }
|
|
virtual bool empty() const override { return prog_.empty(); }
|
|
virtual void clear() override
|
|
{
|
|
prog_.clear();
|
|
}
|
|
|
|
/*Ptr<Graph> clone(Net* newnet) const
|
|
{
|
|
Graph g = std::make_shared<GraphData>((newnet ? *newnet : *net_), name_, inputs_, ispattern_);
|
|
g->outputs_ = outputs_;
|
|
g->backend_ = backend_;
|
|
// don't copy optigraph_. It has to be re-created
|
|
for (auto n : prog_) {
|
|
g->prog_.push_back(n->clone(g->net_));
|
|
}
|
|
return g;
|
|
}*/
|
|
|
|
virtual const std::vector<Arg>& append(Ptr<LayerInfo>& op,
|
|
const std::vector<std::string>& outnames) override
|
|
{
|
|
CV_Assert(op);
|
|
int i, noutputs = (int)outnames.size();
|
|
//CV_Assert(op->minNumOutputs() <= noutputs && noutputs <= op->maxNumOutputs());
|
|
|
|
op->outputs.resize(noutputs);
|
|
for (i = 0; i < noutputs; i++) {
|
|
Arg outarg = netimpl_->getArg(outnames[i]);
|
|
ArgKind kind = netimpl_->argKind(outarg);
|
|
CV_Assert(kind == DNN_ARG_TEMP || kind == DNN_ARG_OUTPUT);
|
|
op->outputs[i] = outarg;
|
|
}
|
|
|
|
prog_.push_back(op);
|
|
return op->outputs;
|
|
}
|
|
|
|
virtual Arg append(Ptr<LayerInfo>& op,
|
|
const std::string& outname) override
|
|
{
|
|
std::vector<std::string> outnames = {outname};
|
|
const std::vector<Arg>& outputs = append(op, outnames);
|
|
CV_Assert(outputs.size() == 1);
|
|
return outputs[0];
|
|
}
|
|
|
|
virtual std::ostream& dump(std::ostream& strm, int indent, bool comma) override
|
|
{
|
|
CV_Assert(netimpl_);
|
|
size_t ninputs = inputs_.size(), noutputs = outputs_.size();
|
|
int delta_indent = netimpl_->dump_indent;
|
|
int subindent = indent + delta_indent;
|
|
int argindent = subindent + delta_indent;
|
|
strm << "{\n";
|
|
prindent(strm, subindent);
|
|
strm << "name: ";
|
|
if (name_.empty())
|
|
strm << "<noname>\n";
|
|
else
|
|
strm << '\"' << name_ << "\"\n";
|
|
prindent(strm, subindent);
|
|
strm << "inputs: [\n";
|
|
for (size_t i = 0; i < ninputs; i++) {
|
|
netimpl_->dumpArg(strm, inputs_[i], argindent, i+1 < ninputs, true);
|
|
}
|
|
prindent(strm, subindent);
|
|
strm << "],\n";
|
|
prindent(strm, subindent);
|
|
strm << "outputs: [\n";
|
|
for (size_t i = 0; i < noutputs; i++) {
|
|
netimpl_->dumpArg(strm, outputs_[i], argindent, i+1 < noutputs, true);
|
|
}
|
|
prindent(strm, subindent);
|
|
strm << "],\n";
|
|
prindent(strm, subindent);
|
|
strm << "layers: [\n";
|
|
size_t nlayers = prog_.size();
|
|
for (size_t i = 0; i < nlayers; i++) {
|
|
prindent(strm, argindent);
|
|
strm << "// op #" << i << "\n";
|
|
const Ptr<LayerInfo>& op = prog_[i];
|
|
op->dump(strm, argindent, i+1 < nlayers);
|
|
}
|
|
prindent(strm, subindent);
|
|
strm << "]\n";
|
|
prindent(strm, indent);
|
|
strm << '}';
|
|
if (comma)
|
|
strm << ',';
|
|
strm << '\n';
|
|
return strm;
|
|
}
|
|
|
|
//virtual Net* net() const override { return net_; }
|
|
virtual const std::vector<Arg>& inputs() const override { return inputs_; }
|
|
virtual const std::vector<Arg>& outputs() const override { return outputs_; }
|
|
|
|
virtual void setOutputs(const std::vector<Arg>& outputs) override {
|
|
CV_Assert(netimpl_);
|
|
netimpl_->checkArgs(outputs);
|
|
outputs_ = outputs;
|
|
}
|
|
virtual const std::vector<Ptr<LayerInfo> >& prog() const override { return prog_; }
|
|
virtual int opBackend(int opidx) const override
|
|
{
|
|
return (opidx >= 0 && opidx < (int)execBackend_.size()) ? execBackend_[opidx]
|
|
: DNN_BACKEND_OPENCV;
|
|
}
|
|
virtual void setProg(const std::vector<Ptr<LayerInfo> >& newprog) override
|
|
{
|
|
prog_ = newprog;
|
|
exec_.clear();
|
|
execBackend_.clear();
|
|
inH2D_.clear();
|
|
outD2H_.clear();
|
|
}
|
|
|
|
Net::Impl* netimpl_;
|
|
std::string name_;
|
|
std::vector<Arg> inputs_;
|
|
std::vector<Arg> outputs_;
|
|
std::vector<Ptr<LayerInfo> > prog_;
|
|
std::vector<Ptr<Layer> > exec_;
|
|
std::vector<int> execBackend_;
|
|
std::vector<std::vector<uchar> > inH2D_;
|
|
std::vector<std::vector<uchar> > outD2H_;
|
|
};
|
|
|
|
Ptr<Graph> Graph::create(void* netimpl, const std::string& name,
|
|
const std::vector<Arg>& inputs)
|
|
{
|
|
return Ptr<Graph>(new GraphImpl(reinterpret_cast<Net::Impl*>(netimpl), name, inputs));
|
|
}
|
|
|
|
Graph::~Graph() {}
|
|
|
|
bool Net::Impl::isConstArg(Arg arg) const
|
|
{
|
|
return argKind(arg) == DNN_ARG_CONST;
|
|
}
|
|
|
|
const ArgData& Net::Impl::argData(Arg arg) const
|
|
{
|
|
CV_Assert((size_t)arg.idx < args.size());
|
|
return args[arg.idx];
|
|
}
|
|
|
|
const std::string& Net::Impl::argName(Arg arg) const
|
|
{
|
|
return argData(arg).name;
|
|
}
|
|
|
|
ArgKind Net::Impl::argKind(Arg arg) const
|
|
{
|
|
return argData(arg).kind;
|
|
}
|
|
|
|
Mat& Net::Impl::argTensor(Arg arg) const
|
|
{
|
|
const ArgData& adata = argData(arg);
|
|
if (adata.kind == DNN_ARG_TEMP) {
|
|
CV_Assert(__tensors__.at(arg.idx).empty());
|
|
int bufidx = bufidxs.at(arg.idx);
|
|
CV_Assert(bufidx >= 0);
|
|
return const_cast<Mat&>(buffers.at(bufidx));
|
|
}
|
|
return const_cast<Mat&>(__tensors__.at(arg.idx));
|
|
}
|
|
|
|
Arg Net::Impl::getArg(const std::string& name)
|
|
{
|
|
auto it = argnames.find(name);
|
|
if (it != argnames.end()) {
|
|
return Arg((int)it->second);
|
|
}
|
|
return newArg(name, DNN_ARG_TEMP);
|
|
}
|
|
|
|
bool Net::Impl::haveArg(const std::string& name) const
|
|
{
|
|
return argnames.find(name) != argnames.end();
|
|
}
|
|
|
|
Arg Net::Impl::newConstArg(const std::string& name, const Mat& m)
|
|
{
|
|
if (name.empty()) {
|
|
CV_Assert(m.empty());
|
|
return Arg();
|
|
}
|
|
Arg arg = newArg(name, DNN_ARG_CONST, true);
|
|
__tensors__[arg.idx] = m;
|
|
ArgData& adata = args[arg.idx];
|
|
adata.type = m.type();
|
|
adata.shape = m.shape();
|
|
return arg;
|
|
}
|
|
|
|
Arg Net::Impl::newArg(const std::string& name, ArgKind kind, bool allowEmptyName)
|
|
{
|
|
CV_Assert(allowEmptyName || !name.empty());
|
|
int idx = (int)args.size();
|
|
|
|
if (!name.empty()) {
|
|
CV_Assert(argnames.find(name) == argnames.end());
|
|
argnames.insert(std::make_pair(name, (int64_t)idx));
|
|
}
|
|
|
|
ArgData adata;
|
|
adata.name = name;
|
|
adata.kind = kind;
|
|
args.push_back(adata);
|
|
__tensors__.push_back(Mat());
|
|
bufidxs.push_back(-1);
|
|
|
|
return Arg(idx);
|
|
}
|
|
|
|
int Net::Impl::findDim(const std::string& dimname, bool insert)
|
|
{
|
|
if (!dimname.empty()) {
|
|
auto it = dimnames.find(dimname);
|
|
if (it != dimnames.end()) {
|
|
return (int)it->second;
|
|
}
|
|
}
|
|
if (!insert) {
|
|
CV_Error_(Error::StsObjectNotFound, ("symbolic dimension '%s' is not found",
|
|
dimname.empty() ? "<some unique name>" : dimname.c_str()));
|
|
}
|
|
int value = -(int)dimnames_vec.size() - 1;
|
|
std::string inserted_dimname = dimname.empty() ? format("N!%d", -value) : dimname;
|
|
dimnames.insert(std::make_pair(inserted_dimname, (int64_t)value));
|
|
dimnames_vec.push_back(inserted_dimname);
|
|
return value;
|
|
}
|
|
|
|
Ptr<Graph> Net::Impl::newGraph(const std::string& name_, const std::vector<Arg>& inpargs, bool ismain)
|
|
{
|
|
if (ismain)
|
|
globGraphIdx = 0;
|
|
std::string name = name_;
|
|
if (name_.empty())
|
|
name = ismain ? std::string("main") : format("subgraph_%d", globGraphIdx);
|
|
globGraphIdx++;
|
|
Ptr<Graph> graph = Graph::create(this, name, inpargs);
|
|
if (ismain)
|
|
mainGraph = graph;
|
|
return graph;
|
|
}
|
|
|
|
void Net::Impl::prepareForInference()
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (this->ort_session)
|
|
{
|
|
prepared = true;
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
if (!prepared) {
|
|
fuseQDQ();
|
|
constFold();
|
|
fuseBN();
|
|
constArgs();
|
|
fuseAttention();
|
|
fuseMatMulConstBToGemm();
|
|
fuseSharedInputGemm();
|
|
fuseReshapeTranspose();
|
|
fuseTransposeMatMul();
|
|
fuseScaleSoftmax();
|
|
fuseBasic();
|
|
totalLayers = updateGraphOfs(mainGraph, 0, true);
|
|
prepared = true;
|
|
}
|
|
}
|
|
|
|
void Net::Impl::finalizeGraph(const Ptr<Graph>& graph, bool useCUDA)
|
|
{
|
|
GraphImpl* g = static_cast<GraphImpl*>(graph.get());
|
|
const std::vector<Ptr<LayerInfo> >& prog = g->prog_;
|
|
size_t i, nops = prog.size();
|
|
g->exec_.assign(nops, Ptr<Layer>());
|
|
g->execBackend_.assign(nops, DNN_BACKEND_OPENCV);
|
|
|
|
#ifdef HAVE_CUDA
|
|
// Whole-graph CUDA gate: run on CUDA only if *every* op has a CUDA executor; if any op is
|
|
// unsupported (or the graph has control-flow subgraphs) run the entire graph on CPU. This
|
|
// avoids partial CPU<->CUDA execution and its host/device coherence hazards for now.
|
|
std::vector<Ptr<Layer> > cudaExecs;
|
|
bool graphOnCuda = false;
|
|
if (useCUDA && cudaInfo) {
|
|
graphOnCuda = true;
|
|
cudaExecs.assign(nops, Ptr<Layer>());
|
|
for (i = 0; i < nops; i++) {
|
|
const Ptr<LayerInfo>& op = prog[i];
|
|
if (!op)
|
|
continue;
|
|
if (op->subgraphs()) { graphOnCuda = false; break; } // control-flow bodies stay on CPU
|
|
Ptr<Layer> e = LayerFactory::createExec(op->type, DNN_BACKEND_CUDA, op, &cudaInfo->context);
|
|
if (!e) { graphOnCuda = false; break; }
|
|
cudaExecs[i] = e;
|
|
}
|
|
if (!graphOnCuda) {
|
|
cudaExecs.clear();
|
|
CV_LOG_INFO(NULL, cv::format("DNN/NewEngine: graph '%s' has layer(s) without CUDA support; "
|
|
"running the whole graph on CPU", graph->name().c_str()));
|
|
}
|
|
}
|
|
#endif
|
|
|
|
for (i = 0; i < nops; i++) {
|
|
const Ptr<LayerInfo>& op = prog[i];
|
|
if (!op)
|
|
continue;
|
|
|
|
// recurse into subgraphs (If/Loop bodies) first
|
|
const std::vector<Ptr<Graph> >* subs = op->subgraphs();
|
|
if (subs) {
|
|
for (const Ptr<Graph>& sub : *subs)
|
|
finalizeGraph(sub, useCUDA);
|
|
}
|
|
|
|
Ptr<Layer> exec;
|
|
int backend = DNN_BACKEND_OPENCV;
|
|
#ifdef HAVE_CUDA
|
|
if (graphOnCuda && cudaExecs[i]) {
|
|
exec = cudaExecs[i];
|
|
exec->preferableTarget = preferableTarget;
|
|
backend = DNN_BACKEND_CUDA;
|
|
}
|
|
#endif
|
|
if (!exec) {
|
|
exec = LayerFactory::createExec(op->type, DNN_BACKEND_OPENCV, op, nullptr);
|
|
if (!exec)
|
|
exec = op.dynamicCast<Layer>();
|
|
backend = DNN_BACKEND_OPENCV;
|
|
}
|
|
CV_Assert(exec);
|
|
// Re-finalize can hand back the same object, so reset state for the new backend.
|
|
exec->packedWeightEpoch = 0;
|
|
exec->finalizedOnce = false;
|
|
exec->lastInpShapes.clear();
|
|
exec->lastInpTypes.clear();
|
|
g->exec_[i] = exec;
|
|
g->execBackend_[i] = backend;
|
|
CV_LOG_INFO(NULL, cv::format("DNN/NewEngine: finalize op #%zu '%s' (%s) -> %s",
|
|
i, op->name.c_str(), op->type.c_str(),
|
|
backend == DNN_BACKEND_CUDA ? "CUDA" : "CPU"));
|
|
}
|
|
}
|
|
|
|
void Net::Impl::saveFusedSnapshot()
|
|
{
|
|
fusedSnapshot.clear();
|
|
for (const Ptr<Graph>& g : allgraphs) {
|
|
FusedGraphSnapshot snap;
|
|
snap.graph = g;
|
|
const std::vector<Ptr<LayerInfo> >& prog = g->prog();
|
|
snap.prog = prog;
|
|
snap.inputs.reserve(prog.size());
|
|
snap.outputs.reserve(prog.size());
|
|
for (const Ptr<LayerInfo>& op : prog) {
|
|
snap.inputs.push_back(op ? op->inputs : std::vector<Arg>());
|
|
snap.outputs.push_back(op ? op->outputs : std::vector<Arg>());
|
|
}
|
|
fusedSnapshot.push_back(std::move(snap));
|
|
}
|
|
}
|
|
|
|
void Net::Impl::restoreFusedSnapshot()
|
|
{
|
|
// Roll the graph back to its post-fusion state: undo the layer-input rewiring
|
|
// and remove the TransformLayout ops inserted by a previous useBlockLayout().
|
|
for (const FusedGraphSnapshot& snap : fusedSnapshot) {
|
|
for (size_t i = 0; i < snap.prog.size(); i++) {
|
|
const Ptr<LayerInfo>& op = snap.prog[i];
|
|
if (!op)
|
|
continue;
|
|
op->inputs = snap.inputs[i];
|
|
op->outputs = snap.outputs[i];
|
|
}
|
|
snap.graph->setProg(snap.prog);
|
|
}
|
|
totalLayers = updateGraphOfs(mainGraph, 0, true);
|
|
}
|
|
|
|
void Net::Impl::finalize()
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (ort_session)
|
|
return; // ONNX Runtime manages its own execution session
|
|
#endif
|
|
if (!mainGraph)
|
|
return;
|
|
if (!prepared)
|
|
prepareForInference();
|
|
if (finalized)
|
|
return;
|
|
|
|
// Snapshot the fused graph once so finalize() can re-run cleanly on a
|
|
// backend/target change (block layout + buffer assignment are destructive).
|
|
if (!fusedSnapshotValid) {
|
|
saveFusedSnapshot();
|
|
fusedSnapshotValid = true;
|
|
} else {
|
|
restoreFusedSnapshot();
|
|
}
|
|
|
|
bool useCUDA = false;
|
|
#ifdef HAVE_CUDA
|
|
cudaArgBuffers.clear();
|
|
cudaArgHostDirty.clear();
|
|
cudaArgDeviceDirty.clear();
|
|
if (preferableBackend == DNN_BACKEND_CUDA && haveCUDA()) {
|
|
useCUDA = true;
|
|
if (!cudaInfo) {
|
|
cuda4dnn::csl::CSLContext context;
|
|
context.stream = cuda4dnn::csl::Stream(true);
|
|
context.cublas_handle = cuda4dnn::csl::cublas::Handle(context.stream);
|
|
context.cudnn_handle = cuda4dnn::csl::cudnn::Handle(context.stream);
|
|
auto d2h_stream = cuda4dnn::csl::Stream(true);
|
|
cudaInfo = std::unique_ptr<CudaInfo_t>(new CudaInfo_t(std::move(context), std::move(d2h_stream)));
|
|
}
|
|
}
|
|
#endif
|
|
|
|
CV_LOG_INFO(NULL, cv::format("DNN/NewEngine: finalize() backend=%d target=%d useCUDA=%d over %zu graph(s)",
|
|
preferableBackend, preferableTarget, (int)useCUDA, allgraphs.size()));
|
|
|
|
for (const Ptr<Graph>& g : allgraphs)
|
|
finalizeGraph(g, useCUDA);
|
|
useBlockLayout();
|
|
assignBuffers();
|
|
totalLayers = updateGraphOfs(mainGraph, 0, true);
|
|
|
|
for (const Ptr<Graph>& g : allgraphs)
|
|
finalizeGraph(g, useCUDA);
|
|
|
|
finalized = true;
|
|
}
|
|
|
|
void Net::Impl::allocateLayerOutputs(
|
|
const Ptr<LayerInfo>& layer,
|
|
const std::vector<int>& inpTypes,
|
|
const std::vector<MatShape>& inpShapes,
|
|
std::vector<int>& outTypes,
|
|
std::vector<MatShape>& outShapes,
|
|
std::vector<std::pair<uchar*, size_t> >& outOrigData,
|
|
std::vector<Mat>& outputs,
|
|
std::vector<int>& tempTypes,
|
|
std::vector<MatShape>& tempShapes,
|
|
std::vector<Mat>& temps,
|
|
std::vector<Mat>& globalTemps,
|
|
bool useBufferPool)
|
|
{
|
|
// In theory, when
|
|
// 1) useBufferPool==true,
|
|
// 2) the buffers in the pool are already big enough (e.g. when we already run inference a few times to let them grow)
|
|
// 3) getMemoryShapes() and getTypes() are implemented efficiently without any memory allocations
|
|
// the method allocateLayerOutputs() should not allocate any memory either.
|
|
//
|
|
// Well, currently it still may do so, because Mat::fit() may create e.g. 4D tensor on top of 1D buffer and then
|
|
// MatSize and MatStep will require dynamic memory allocation (those are very small buffers though).
|
|
// But we plan to make MatSize and MatStep lighter so that they don't use dynamic memory.
|
|
size_t noutputs = layer->outputs.size();
|
|
outShapes.clear();
|
|
outTypes.clear();
|
|
tempShapes.clear();
|
|
tempTypes.clear();
|
|
layer->getMemoryShapes(inpShapes, (int)noutputs, outShapes, tempShapes);
|
|
layer->getTypes(inpTypes, (int)noutputs, (int)tempShapes.size(), outTypes, tempTypes);
|
|
CV_Assert(tempShapes.size() == tempTypes.size());
|
|
CV_Assert(outShapes.size() == outTypes.size());
|
|
CV_Assert(outShapes.size() == noutputs);
|
|
|
|
for (int i = 0; i < (int)tempShapes.size(); i++)
|
|
CV_CheckGT(total(tempShapes[i]), (size_t)0, "");
|
|
outputs.assign(noutputs, Mat());
|
|
outOrigData.resize(noutputs);
|
|
for (size_t i = 0; i < noutputs; i++) {
|
|
Arg out = layer->outputs[i];
|
|
if (useBufferPool) {
|
|
Mat& out_t = argTensor(out);
|
|
out_t.fit(outShapes[i], outTypes[i]);
|
|
outputs[i] = out_t;
|
|
} else {
|
|
outputs[i].fit(outShapes[i], outTypes[i]);
|
|
}
|
|
outOrigData[i].first = outputs[i].u ? outputs[i].u->data : nullptr;
|
|
outOrigData[i].second = outputs[i].u ? outputs[i].u->size : 0;
|
|
}
|
|
// [TODO] probably there should be a smarter algorithm that e.g. sorts
|
|
// temp buffers by size in decreasing order and assigns global temps accordingly
|
|
// in order to minimize the total size of temp buffers
|
|
size_t ntemps = tempShapes.size();
|
|
temps.resize(ntemps);
|
|
globalTemps.resize(std::max(ntemps, globalTemps.size()));
|
|
for (size_t i = 0; i < ntemps; i++) {
|
|
globalTemps[i].fit(tempShapes[i], tempTypes[i]);
|
|
temps[i] = globalTemps[i];
|
|
}
|
|
}
|
|
|
|
void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs)
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (useOrtEngine && mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty())
|
|
finalizeOrt();
|
|
if (this->ort_session)
|
|
{
|
|
if (!netInputLayer || netInputLayer->blobs.empty())
|
|
CV_Error(Error::StsError, "DNN/ORT: No input data found");
|
|
|
|
std::vector<Mat> ortOuts = runOrtSession(netInputLayer->blobs, std::vector<int>());
|
|
|
|
std::vector<Mat>* outMats = nullptr;
|
|
std::vector<UMat>* outUMats = nullptr;
|
|
_InputArray::KindFlag outKind = outputs.kind();
|
|
if (outKind == _InputArray::STD_VECTOR_MAT)
|
|
{
|
|
outMats = &outputs.getMatVecRef();
|
|
*outMats = ortOuts;
|
|
}
|
|
else if (outKind == _InputArray::STD_VECTOR_UMAT)
|
|
{
|
|
outUMats = &outputs.getUMatVecRef();
|
|
outUMats->resize(ortOuts.size());
|
|
for (size_t i = 0; i < ortOuts.size(); ++i)
|
|
ortOuts[i].copyTo(outUMats->at(i));
|
|
}
|
|
else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT)
|
|
{
|
|
CV_CheckEQ((int)ortOuts.size(), 1, "DNN/ORT: single Mat/UMat output requires exactly one ORT output");
|
|
ortOuts[0].copyTo(outputs);
|
|
}
|
|
else
|
|
{
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's");
|
|
}
|
|
return;
|
|
}
|
|
|
|
#endif
|
|
if (!mainGraph) {
|
|
CV_Error(Error::StsNullPtr, "the model was not loaded");
|
|
}
|
|
finalize(); // select per-op executors for the chosen backend/target (idempotent)
|
|
// ************ uncomment one of the lines below for debugging **********
|
|
//tracingMode = DNN_TRACE_OP;
|
|
//tracingMode = DNN_TRACE_ALL;
|
|
// [TODO] initialize profile, tracer, symbolic shapes etc.
|
|
size_t nsymdims = dimnames_vec.size();
|
|
dimvalues.assign(nsymdims, -1);
|
|
layersTimings.assign(totalLayers + 1, 0.);
|
|
|
|
forwardGraph(mainGraph, inputs, outputs, true);
|
|
|
|
// Feed present.* outputs back as past_key_values.* inputs for the next step (causal-lm-with-past).
|
|
if (useKVCache && kvCacheManager.hasRoutes)
|
|
kvCacheManager.applyRoutes();
|
|
}
|
|
|
|
// Assign a single result to an output array, including a (pre-allocated) vector
|
|
// of Mat/UMat, which _OutputArray::assign(Mat) does not handle.
|
|
static void assignSingleOutput(OutputArrayOfArrays outputBlobs, const Mat& result)
|
|
{
|
|
_InputArray::KindFlag k = outputBlobs.kind();
|
|
if (k == _InputArray::STD_VECTOR_MAT) {
|
|
std::vector<Mat>& v = outputBlobs.getMatVecRef();
|
|
v.resize(1);
|
|
result.copyTo(v[0]);
|
|
} else if (k == _InputArray::STD_VECTOR_UMAT) {
|
|
std::vector<UMat>& v = outputBlobs.getUMatVecRef();
|
|
v.resize(1);
|
|
result.copyTo(v[0]);
|
|
} else {
|
|
outputBlobs.assign(result);
|
|
}
|
|
}
|
|
|
|
void Net::Impl::forwardWithSingleOutput(const std::string& outname, OutputArrayOfArrays outputBlobs)
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (useOrtEngine && mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty())
|
|
finalizeOrt();
|
|
if (this->ort_session)
|
|
{
|
|
if (!netInputLayer || netInputLayer->blobs.empty())
|
|
CV_Error(Error::StsError, "DNN/ORT: No input data found");
|
|
|
|
if (!this->ort_names_cache)
|
|
this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
|
|
|
|
int outIdx = 0;
|
|
if (!outname.empty())
|
|
{
|
|
OrtNamesCache& names = *this->ort_names_cache;
|
|
auto it = names.output_name_to_index.find(outname);
|
|
if (it == names.output_name_to_index.end())
|
|
CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", outname.c_str()));
|
|
outIdx = it->second;
|
|
}
|
|
|
|
std::vector<int> outIdxs(1, outIdx);
|
|
std::vector<Mat> outs = runOrtSession(netInputLayer->blobs, outIdxs);
|
|
CV_Assert(outs.size() == 1);
|
|
assignSingleOutput(outputBlobs, outs[0]);
|
|
return;
|
|
}
|
|
#endif
|
|
{
|
|
if (!mainGraph) {
|
|
CV_Error(Error::StsNullPtr, "the model was not loaded");
|
|
}
|
|
if (!outname.empty()) {
|
|
auto it = argnames.find(outname);
|
|
if (it == argnames.end()) {
|
|
size_t excl = outname.rfind('!');
|
|
if (excl != std::string::npos) {
|
|
it = argnames.find(outname.substr(excl + 1));
|
|
}
|
|
}
|
|
if (it == argnames.end())
|
|
CV_Error_(Error::StsObjectNotFound,
|
|
("DNN: tensor '%s' is not found in the graph", outname.c_str()));
|
|
|
|
Arg targetArg((int)it->second);
|
|
|
|
std::vector<Mat> inps, outs;
|
|
forwardMainGraph(inps, outs);
|
|
|
|
const std::vector<Arg>& gr_outputs = mainGraph->outputs();
|
|
for (size_t i = 0; i < gr_outputs.size(); i++) {
|
|
if (gr_outputs[i].idx == targetArg.idx) {
|
|
assignSingleOutput(outputBlobs, outs[i]);
|
|
return;
|
|
}
|
|
}
|
|
|
|
const ArgData& adata = args.at(targetArg.idx);
|
|
Mat result;
|
|
if (adata.kind == DNN_ARG_TEMP) {
|
|
int bufidx = bufidxs.at(targetArg.idx);
|
|
CV_Assert(bufidx >= 0 && bufidx < (int)buffers.size());
|
|
result = buffers[bufidx];
|
|
} else {
|
|
result = __tensors__.at(targetArg.idx);
|
|
}
|
|
if (result.shape().layout == DATA_LAYOUT_BLOCK) {
|
|
Mat converted;
|
|
transformLayout(result, converted, originalLayout, originalLayout, defaultC0);
|
|
assignSingleOutput(outputBlobs, converted);
|
|
} else {
|
|
assignSingleOutput(outputBlobs, result.clone());
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
std::vector<Mat> inps, outs;
|
|
forwardMainGraph(inps, outs);
|
|
CV_Assert(!outs.empty());
|
|
assignSingleOutput(outputBlobs, outs[0]);
|
|
}
|
|
|
|
void Net::Impl::forwardWithMultipleOutputs(OutputArrayOfArrays outblobs, const std::vector<std::string>& outnames)
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (useOrtEngine && mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty())
|
|
finalizeOrt();
|
|
if (this->ort_session)
|
|
{
|
|
if (!netInputLayer || netInputLayer->blobs.empty())
|
|
CV_Error(Error::StsError, "DNN/ORT: No input data found");
|
|
|
|
if (!this->ort_names_cache)
|
|
this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
|
|
|
|
OrtNamesCache& names = *this->ort_names_cache;
|
|
const int totalOutputs = (int)names.output_names.size();
|
|
if (totalOutputs <= 0)
|
|
CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs");
|
|
|
|
std::vector<int> outIdxs;
|
|
if (outnames.empty())
|
|
{
|
|
outIdxs.resize((size_t)totalOutputs);
|
|
for (int i = 0; i < totalOutputs; ++i)
|
|
outIdxs[(size_t)i] = i;
|
|
}
|
|
else
|
|
{
|
|
outIdxs.reserve(outnames.size());
|
|
for (const std::string& n : outnames)
|
|
{
|
|
auto it = names.output_name_to_index.find(n);
|
|
if (it == names.output_name_to_index.end())
|
|
CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", n.c_str()));
|
|
outIdxs.push_back(it->second);
|
|
}
|
|
}
|
|
|
|
std::vector<Mat> outs = runOrtSession(netInputLayer->blobs, outIdxs);
|
|
|
|
std::vector<Mat>* outMats = nullptr;
|
|
std::vector<UMat>* outUMats = nullptr;
|
|
_InputArray::KindFlag outKind = outblobs.kind();
|
|
if (outKind == _InputArray::STD_VECTOR_MAT) {
|
|
outMats = &outblobs.getMatVecRef();
|
|
outMats->resize(outs.size());
|
|
} else if (outKind == _InputArray::STD_VECTOR_UMAT) {
|
|
outUMats = &outblobs.getUMatVecRef();
|
|
outUMats->resize(outs.size());
|
|
} else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT) {
|
|
CV_CheckEQ((int)outs.size(), 1, "DNN/ORT: Mat/UMat output requires exactly one output");
|
|
} else {
|
|
CV_Error(Error::StsBadArg, "outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's");
|
|
}
|
|
|
|
for (size_t i = 0; i < outs.size(); ++i) {
|
|
Mat src = outs[i];
|
|
if (outMats) {
|
|
src.copyTo(outMats->at(i));
|
|
} else if (outUMats) {
|
|
src.copyTo(outUMats->at(i));
|
|
} else {
|
|
src.copyTo(outblobs);
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
if (!mainGraph) {
|
|
CV_Error(Error::StsNullPtr, "the model was not loaded");
|
|
}
|
|
const std::vector<Arg>& outargs = mainGraph->outputs();
|
|
std::vector<int> outidxs;
|
|
int i, j, noutputs = (int)outargs.size();
|
|
if (outnames.empty() || (noutputs == 1 && outnames.size() == 1 && outnames[0].empty())) {
|
|
for (i = 0; i < noutputs; i++)
|
|
outidxs.push_back(i);
|
|
} else {
|
|
for (i = 0; i < (int)outnames.size(); i++) {
|
|
const std::string& outname = outnames[i];
|
|
for (j = 0; j < noutputs; j++) {
|
|
const ArgData& adata = args.at(outargs[j].idx);
|
|
if (adata.name == outname) {
|
|
outidxs.push_back((int)j);
|
|
break;
|
|
}
|
|
}
|
|
if (j == noutputs) {
|
|
CV_Error_(Error::StsObjectNotFound, ("the required output '%s' is not found", outname.c_str()));
|
|
}
|
|
}
|
|
}
|
|
std::vector<Mat> inps={}, outs;
|
|
forwardMainGraph(inps, outs);
|
|
CV_Assert(outs.size() == noutputs);
|
|
int nout = (int)outidxs.size();
|
|
std::vector<Mat>* outMats = nullptr;
|
|
std::vector<UMat>* outUMats = nullptr;
|
|
_InputArray::KindFlag outKind = outblobs.kind();
|
|
if (outKind == _InputArray::STD_VECTOR_MAT) {
|
|
outMats = &outblobs.getMatVecRef();
|
|
outMats->resize(nout);
|
|
} else if (outKind == _InputArray::STD_VECTOR_UMAT) {
|
|
outUMats = &outblobs.getUMatVecRef();
|
|
outUMats->resize(nout);
|
|
} else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT) {
|
|
CV_Assert(nout == 1);
|
|
} else {
|
|
CV_Error(Error::StsBadArg, "outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's");
|
|
}
|
|
for (i = 0; i < nout; i++) {
|
|
int j = outidxs[i];
|
|
Mat src = outs[j];
|
|
if (outMats) {
|
|
src.copyTo(outMats->at(i));
|
|
} else if (outUMats) {
|
|
src.copyTo(outUMats->at(i));
|
|
} else {
|
|
src.copyTo(outblobs);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*void Net::Impl::checkAndUpdateDim(const Ptr<Graph>& g, const Ptr<Layer>& layer, Arg inp, int j, int value)
|
|
{
|
|
const ArgData& adata = args[inp.idx];
|
|
int64_t value0 = adata.size.size[j];
|
|
if (value0 >= 0) {
|
|
if (value0 != value) {
|
|
CV_Error_(Error::StsBadArg, ("graph '%s': node '%s': %d-th dimension of argument '%s' is wrong: %lld given, %lld expected",
|
|
g->name().data(), node ? node->name().data() : "none (graph input)", j, adata.name.c_str(), value, value0));
|
|
}
|
|
} else {
|
|
int64_t idx = -value0-1;
|
|
CV_Assert(0 <= idx && idx < (int64_t)dimvalues.size());
|
|
value0 = dimvalues[idx];
|
|
if (value0 < 0) {
|
|
dimvalues[idx] = value;
|
|
} else if (value0 != value) {
|
|
CV_Error_(Error::StsBadArg,
|
|
("graph '%s': node '%s': %d-th dimension '%s' of argument '%s' is wrong: %lld given, but '%s' is already set to %lld",
|
|
g->name().data(), node ? node->name().data() : "none (graph input)",
|
|
j, dimnames_[idx].c_str(), adata.name.c_str(),
|
|
value, dimnames_[idx].c_str(), value0));
|
|
}
|
|
}
|
|
}*/
|
|
|
|
void Net::Impl::traceArg(std::ostream& strm_, const char* prefix, size_t i, Arg arg, bool dumpdata)
|
|
{
|
|
const int PPRINT_CONTEXT = 3;
|
|
const int PPRINT_CONST_THRESHOLD = 16;
|
|
const int PPRINT_ALL_THRESHOLD = 100;
|
|
const Mat& m = argTensor(arg);
|
|
const ArgData& adata = args.at(arg.idx);
|
|
bool constArg = adata.kind == DNN_ARG_CONST;
|
|
// [TODO] replace with type compatibility check
|
|
// CV_Assert(m.type() == adata.type);
|
|
strm_ << prefix << " " << i << ". Name: " << (arg.idx > 0 ? adata.name.c_str() : "<empty>") << "\n";
|
|
if (arg.idx == 0)
|
|
return;
|
|
strm_ << " Buf: " << bufidxs.at(arg.idx) << "\n";
|
|
strm_ << " Type: " << typeToString(adata.type) << " \n";
|
|
MatShape shape = m.shape();
|
|
strm_ << " Shape: " << shape;
|
|
if (constArg && m.total() <= PPRINT_CONST_THRESHOLD) {
|
|
strm_ << " /* ";
|
|
pprint(strm_, m, 0, PPRINT_CONTEXT, PPRINT_CONST_THRESHOLD, '{');
|
|
strm_ << " */";
|
|
}
|
|
strm_ << "\n Layout: " << layoutToString(shape.layout) << "\n";
|
|
if (dumpdata && !constArg) {
|
|
Mat temp;
|
|
if (m.size.layout == DATA_LAYOUT_BLOCK) {
|
|
transformLayout(m, temp, originalLayout, originalLayout, m.size.C);
|
|
} else {
|
|
temp = m;
|
|
}
|
|
pprint(strm_, temp, 0, PPRINT_CONTEXT, PPRINT_ALL_THRESHOLD, '[');
|
|
strm_ << "\n";
|
|
}
|
|
}
|
|
|
|
void Net::Impl::setMainGraphInput(InputArray m, const std::string& inpname)
|
|
{
|
|
#ifdef HAVE_ONNXRUNTIME
|
|
if (useOrtEngine && ortNeedsReinit && mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty())
|
|
{
|
|
Mat inputMat = m.getMat();
|
|
if (inputMat.empty())
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty");
|
|
|
|
bool updated = false;
|
|
for (size_t i = 0; i < ort_staged_inputs.size(); ++i)
|
|
{
|
|
if (ort_staged_inputs[i].first == inpname)
|
|
{
|
|
inputMat.copyTo(ort_staged_inputs[i].second);
|
|
updated = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!updated)
|
|
ort_staged_inputs.push_back(std::make_pair(inpname, inputMat.clone()));
|
|
return;
|
|
}
|
|
|
|
if (this->ort_session)
|
|
{
|
|
if (!this->ort_names_cache)
|
|
this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
|
|
|
|
OrtNamesCache& names = *this->ort_names_cache;
|
|
const size_t ninputs = names.input_names.size();
|
|
if (ninputs == 0)
|
|
CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs");
|
|
|
|
if (!netInputLayer) {
|
|
netInputLayer = Ptr<DataLayer>(new DataLayer());
|
|
netInputLayer->name = "ort_data_layer";
|
|
netInputLayer->type = "Data";
|
|
}
|
|
|
|
Mat inputMat = m.getMat();
|
|
if (inputMat.empty())
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty");
|
|
|
|
if (netInputLayer->blobs.size() != ninputs)
|
|
netInputLayer->blobs.resize(ninputs);
|
|
|
|
size_t inputIdx = 0;
|
|
if (inpname.empty())
|
|
{
|
|
if (ninputs != 1)
|
|
CV_Error(Error::StsBadArg, "DNN/ORT: input name must be specified for models with multiple inputs");
|
|
inputIdx = 0;
|
|
}
|
|
else
|
|
{
|
|
auto it = names.input_name_to_index.find(inpname);
|
|
if (it == names.input_name_to_index.end())
|
|
CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: input '%s' is not found", inpname.c_str()));
|
|
inputIdx = (size_t)it->second;
|
|
}
|
|
|
|
inputMat.copyTo(netInputLayer->blobs[inputIdx]);
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
CV_Assert(mainGraph);
|
|
const std::vector<Arg>& gr_inputs = mainGraph->inputs();
|
|
size_t i, ninputs = gr_inputs.size();
|
|
if (inpname.empty()) {
|
|
CV_Assert(ninputs == 1 && "empty name can only be used to set input if there is just one input");
|
|
i = 0;
|
|
} else {
|
|
for (i = 0; i < ninputs; i++) {
|
|
const ArgData& adata = args.at(gr_inputs[i].idx);
|
|
CV_Assert(adata.kind == DNN_ARG_INPUT);
|
|
if (adata.name == inpname)
|
|
break;
|
|
}
|
|
if ((i == ninputs) && (!isdigit(inpname[0]) || !sscanf(inpname.c_str(), "%zu", &i))) {
|
|
CV_Error_(Error::StsObjectNotFound, ("input '%s' is not found", inpname.c_str()));
|
|
}
|
|
}
|
|
setGraphInput(mainGraph, i, m.getMat());
|
|
}
|
|
|
|
void Net::Impl::setGraphInput(Ptr<Graph>& graph, size_t idx, const Mat& m)
|
|
{
|
|
int mtype = m.type();
|
|
MatShape mshape = m.shape();
|
|
const std::vector<Arg>& gr_inputs = graph->inputs();
|
|
if (idx >= gr_inputs.size())
|
|
{
|
|
return;
|
|
}
|
|
Arg inp = gr_inputs[idx];
|
|
const ArgData& adata = args.at(inp.idx);
|
|
/*
|
|
[TODO] add more detailed shape check
|
|
if (adata.shape.dims != mshape.dims) {
|
|
CV_Error_(Error::StsBadArg, ("wrong dimensionality of argument '%s': %d given, %d expected",
|
|
adata.name.c_str(), tsize.ndims, adata.size.ndims));
|
|
}
|
|
|
|
for (int k = 0; k < mshape.dims; k++) {
|
|
checkAndUpdateDim(graph, Node(), inp, k, tsize.size[k]);
|
|
}
|
|
*/
|
|
|
|
if (adata.kind == DNN_ARG_INPUT) {
|
|
int adata_type = adata.type;
|
|
if ((adata_type == CV_16F || adata_type == CV_16BF) && !enableFP16)
|
|
adata_type = CV_32F;
|
|
|
|
if (adata_type != mtype &&
|
|
!((adata_type == CV_64F || adata_type == CV_32F || adata_type == CV_16F || adata_type == CV_16BF) &&
|
|
(mtype == CV_64F || mtype == CV_32F || mtype == CV_16F || mtype == CV_16BF)) &&
|
|
!((adata_type == CV_8U || adata_type == CV_8S || adata_type == CV_16U || adata_type == CV_16S || adata_type == CV_32S || adata_type == CV_32U || adata_type == CV_64S || adata_type == CV_64U) &&
|
|
(mtype == CV_8U || mtype == CV_8S || mtype == CV_16U || mtype == CV_16S || mtype == CV_32S || mtype == CV_32U || mtype == CV_64S || mtype == CV_64U)) &&
|
|
!(adata.type == CV_16BF && mtype == CV_16U) && !(adata.type == CV_16F && mtype == CV_16U) &&
|
|
!m.empty())
|
|
{
|
|
CV_Error_(Error::StsBadArg, ("incompatible type of input tensor #%zu '%s': %s given, %s expected",
|
|
idx, adata.name.c_str(), typeToString(mtype).c_str(),
|
|
typeToString(adata.type).c_str()));
|
|
}
|
|
Mat& inp_t = argTensor(inp);
|
|
// The op loop detects signature changes per layer; no global flag needed.
|
|
inp_t.fit(mshape, adata_type);
|
|
|
|
if (adata.type == CV_16BF && mtype == CV_16U)
|
|
{
|
|
Mat tmp(mshape, CV_16BF, (void*)m.data);
|
|
tmp.convertTo(inp_t, adata_type);
|
|
}
|
|
else if (adata.type == CV_16F && mtype == CV_16U)
|
|
{
|
|
Mat tmp(mshape, CV_16F, (void*)m.data);
|
|
tmp.convertTo(inp_t, adata_type);
|
|
}
|
|
else
|
|
{
|
|
m.convertTo(inp_t, adata_type);
|
|
}
|
|
} else if (adata.kind == DNN_ARG_TEMP) {
|
|
int bufidx = bufidxs.at(inp.idx);
|
|
Mat& buf = buffers.at(bufidx);
|
|
buf.fit(mshape, mtype); // minimize reallocations
|
|
m.copyTo(buf);
|
|
} else {
|
|
CV_Error_(Error::StsBadArg, ("graph %s: argument '%s' must be 'INPUT' or 'TEMP', not '%s'",
|
|
graph->name().data(), adata.name.c_str(),
|
|
argKindToString(adata.kind).c_str()));
|
|
}
|
|
}
|
|
|
|
#ifdef HAVE_CUDA
|
|
// cv::cuda::Stream view over the (non-owning) cuda4dnn inference stream, so GpuMatND transfers
|
|
// are ordered against the op compute that runs on the same cudaStream_t.
|
|
static inline cuda::Stream wrapCudaStream(cuda4dnn::csl::Stream& s)
|
|
{
|
|
return cuda::StreamAccessor::wrapStream(s.get());
|
|
}
|
|
|
|
// GpuMatND upload/download require at least one dimension (rank-0 shapes trip GpuMatND::setFields).
|
|
// View a rank-0 scalar as a 1-element 1D header over the same data; higher-rank Mats pass through.
|
|
static inline Mat asAtLeast1D(const Mat& m)
|
|
{
|
|
if (m.shape().dims != 0)
|
|
return m;
|
|
int one = 1;
|
|
return Mat(1, &one, m.type(), const_cast<uchar*>(m.data));
|
|
}
|
|
|
|
// Device element type for a host tensor: float tensors are stored as half under the FP16 target,
|
|
// everything else mirrors the host type. fit() reuses the existing allocation when large enough,
|
|
// so buffers persist across forwards.
|
|
int Net::Impl::cudaDeviceType(const Mat& hostMat) const
|
|
{
|
|
if (preferableTarget == DNN_TARGET_CUDA_FP16 && CV_MAT_DEPTH(hostMat.type()) == CV_32F)
|
|
return CV_MAKETYPE(CV_16F, CV_MAT_CN(hostMat.type()));
|
|
return hostMat.type();
|
|
}
|
|
|
|
cuda::GpuMatND& Net::Impl::getCudaArgBuffer(Arg arg, const Mat& hostMat)
|
|
{
|
|
int idx = arg.idx;
|
|
if ((int)cudaArgBuffers.size() != (int)args.size()) {
|
|
cudaArgBuffers.assign(args.size(), cuda::GpuMatND());
|
|
cudaArgHostDirty.assign(args.size(), 1);
|
|
cudaArgDeviceDirty.assign(args.size(), 0);
|
|
}
|
|
// GpuMatND (and cuda4dnn tensors) require at least one dimension; represent a rank-0 scalar
|
|
// tensor as a 1-element 1D tensor so element-wise ops still see the single value.
|
|
MatShape shape = hostMat.shape();
|
|
if (shape.dims == 0)
|
|
shape = MatShape({1});
|
|
cudaArgBuffers[idx].fit(shape, cudaDeviceType(hostMat));
|
|
return cudaArgBuffers[idx];
|
|
}
|
|
|
|
void Net::Impl::cudaSetHostDirty(Arg arg)
|
|
{
|
|
int idx = arg.idx;
|
|
if (idx >= 0 && idx < (int)cudaArgHostDirty.size()) {
|
|
cudaArgHostDirty[idx] = 1;
|
|
cudaArgDeviceDirty[idx] = 0;
|
|
}
|
|
}
|
|
|
|
void Net::Impl::cudaUploadArg(Arg arg, const Mat& hostMat)
|
|
{
|
|
int idx = arg.idx;
|
|
Mat src = asAtLeast1D(hostMat);
|
|
cuda::GpuMatND& g = getCudaArgBuffer(arg, src);
|
|
if (cudaArgHostDirty[idx]) {
|
|
cuda::Stream s = wrapCudaStream(cudaInfo->context.stream);
|
|
if (g.type() == src.type()) {
|
|
g.upload(src, s);
|
|
} else {
|
|
// FP32 -> FP16 (device stores half): convert on host, then copy up.
|
|
Mat tmp;
|
|
src.convertTo(tmp, CV_MAT_DEPTH(g.type()));
|
|
g.upload(tmp, s);
|
|
}
|
|
cudaArgHostDirty[idx] = 0;
|
|
cudaArgDeviceDirty[idx] = 0;
|
|
}
|
|
}
|
|
|
|
void Net::Impl::cudaDownloadArg(Arg arg, Mat& hostMat)
|
|
{
|
|
int idx = arg.idx;
|
|
if (idx < 0 || idx >= (int)cudaArgDeviceDirty.size())
|
|
return;
|
|
if (cudaArgDeviceDirty[idx]) {
|
|
cuda::GpuMatND& g = cudaArgBuffers[idx];
|
|
cuda::Stream s = wrapCudaStream(cudaInfo->context.stream);
|
|
Mat dst = asAtLeast1D(hostMat); // fill the scalar's storage through a 1D header
|
|
if (g.type() == dst.type()) {
|
|
g.download(dst, s);
|
|
cudaInfo->context.stream.synchronize(); // host read follows immediately
|
|
} else {
|
|
// device stores half: copy down, then convert up to the host FP32 tensor.
|
|
Mat tmp;
|
|
g.download(tmp, s);
|
|
cudaInfo->context.stream.synchronize();
|
|
tmp.convertTo(dst, dst.type());
|
|
}
|
|
cudaArgDeviceDirty[idx] = 0;
|
|
cudaArgHostDirty[idx] = 0;
|
|
}
|
|
}
|
|
|
|
static void forwardOpCUDA(Net::Impl* netimpl, GraphImpl* gimpl, size_t opidx,
|
|
const std::vector<Arg>& inputs, const std::vector<Arg>& outputs,
|
|
std::vector<Mat>& inpMats, std::vector<Mat>& outMats)
|
|
{
|
|
Ptr<Layer> exec = gimpl->exec_[opidx];
|
|
CV_Assert(exec && netimpl->cudaInfo);
|
|
std::vector<cuda::GpuMatND> inpG(inputs.size()), outG(outputs.size());
|
|
for (size_t i = 0; i < inputs.size(); i++) {
|
|
netimpl->cudaUploadArg(inputs[i], inpMats[i]); // H2D only if host-authoritative
|
|
inpG[i] = netimpl->cudaArgBuffers[inputs[i].idx];
|
|
}
|
|
for (size_t i = 0; i < outputs.size(); i++) {
|
|
outG[i] = netimpl->getCudaArgBuffer(outputs[i], outMats[i]);
|
|
int oidx = outputs[i].idx; // op writes the device buffer
|
|
netimpl->cudaArgDeviceDirty[oidx] = 1;
|
|
netimpl->cudaArgHostDirty[oidx] = 0;
|
|
}
|
|
exec->forwardCUDA(inpG, outG, &netimpl->cudaInfo->workspace);
|
|
}
|
|
#endif
|
|
|
|
// Slice a Scan input at index `idx` along `axis`, removing that axis (contiguous result).
|
|
static Mat sliceScanAxis(const Mat& m, int axis, int idx)
|
|
{
|
|
std::vector<Range> r(m.dims, Range::all());
|
|
r[axis] = Range(idx, idx + 1);
|
|
Mat sub = m(r).clone();
|
|
std::vector<int> ns;
|
|
for (int d = 0; d < m.dims; d++)
|
|
if (d != axis) ns.push_back(m.size[d]);
|
|
if (ns.empty()) ns.push_back(1);
|
|
return sub.reshape(0, (int)ns.size(), &ns[0]);
|
|
}
|
|
|
|
// Stack per-iteration Scan outputs into one tensor with a new axis at `axis`.
|
|
static Mat stackScanAxis(const std::vector<Mat>& perIter, int axis, bool reverse)
|
|
{
|
|
if (perIter.empty()) return Mat();
|
|
const Mat& first = perIter[0];
|
|
const int e = first.dims, T = (int)perIter.size();
|
|
if (axis < 0) axis += e + 1;
|
|
CV_Assert(axis >= 0 && axis <= e);
|
|
|
|
std::vector<int> os, ss;
|
|
for (int d = 0; d < e; d++) {
|
|
if (d == axis) { os.push_back(T); ss.push_back(1); }
|
|
os.push_back(first.size[d]);
|
|
ss.push_back(first.size[d]);
|
|
}
|
|
if (axis == e) { os.push_back(T); ss.push_back(1); }
|
|
|
|
Mat stacked;
|
|
stacked.create((int)os.size(), &os[0], first.type());
|
|
for (int k = 0; k < T; k++) {
|
|
int idx = reverse ? (T - 1 - k) : k;
|
|
std::vector<Range> r(os.size(), Range::all());
|
|
r[axis] = Range(k, k + 1);
|
|
Mat dst = stacked(r);
|
|
Mat src = perIter[idx].reshape(0, (int)ss.size(), &ss[0]);
|
|
src.copyTo(dst);
|
|
}
|
|
return stacked;
|
|
}
|
|
|
|
void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
|
|
OutputArrayOfArrays outputs_, bool isMainGraph)
|
|
{
|
|
auto graphofs_it = graphofs.find(graph->name());
|
|
if (graphofs_it == graphofs.end()) {
|
|
CV_Error_(Error::StsObjectNotFound, ("graph '%s' does not belong to the model", graph->name().c_str()));
|
|
}
|
|
std::ostream& strm_ = dump_strm ? *dump_strm : std::cout;
|
|
GraphImpl* gimpl = static_cast<GraphImpl*>(graph.get());
|
|
const std::vector<Ptr<LayerInfo> >& prog = graph->prog();
|
|
size_t i, nops = prog.size();
|
|
const std::vector<Arg>& gr_inputs = graph->inputs();
|
|
const std::vector<Arg>& gr_outputs = graph->outputs();
|
|
size_t n_gr_inputs = gr_inputs.size(), n_gr_outputs = gr_outputs.size();
|
|
std::vector<Mat> inpMats, outMats, tempMats;
|
|
std::vector<int> inpTypes, outTypes, tempTypes;
|
|
std::vector<std::pair<uchar*, size_t> > outOrigData;
|
|
std::vector<MatShape> inpShapes, outShapes, tempShapes;
|
|
double tickfreq = getTickFrequency();
|
|
int64_t timestamp = 0;
|
|
|
|
size_t graph_ofs = (size_t)graphofs_it->second;
|
|
CV_Assert(graph_ofs + nops <= totalLayers);
|
|
if (!inputs_.empty()) {
|
|
if (inputs_.total() != n_gr_inputs) {
|
|
CV_Error_(Error::StsBadArg, ("wrong number of inputs in graph '%s': %zu given, %zu expected",
|
|
graph->name().data(), inputs_.total(), n_gr_inputs));
|
|
}
|
|
for (i = 0; i < n_gr_inputs; i++) {
|
|
Mat m = inputs_.getMat((int)i);
|
|
setGraphInput(graph, i, m);
|
|
}
|
|
}
|
|
#ifdef HAVE_CUDA
|
|
// Graph inputs are host-authoritative and may be updated between forward() calls
|
|
// (setInput writes them outside this function, so the loop above is skipped when
|
|
// forward() is called with no explicit inputs). Mark their device copies stale each
|
|
// forward so the current input is re-uploaded; otherwise a second forward with a
|
|
// changed input would read the previous forward's stale device data.
|
|
if (cudaInfo) {
|
|
for (i = 0; i < n_gr_inputs; i++)
|
|
cudaSetHostDirty(gr_inputs[i]);
|
|
}
|
|
#endif
|
|
|
|
for (size_t opidx = 0; opidx < nops; opidx++) {
|
|
const Ptr<LayerInfo>& op = prog.at(opidx);
|
|
if (!op) // in theory we shouldn't have any 'nops' at this stage, but just in case we skip them.
|
|
continue;
|
|
Ptr<Layer> layer = (opidx < gimpl->exec_.size()) ? gimpl->exec_[opidx] : Ptr<Layer>();
|
|
if (!layer)
|
|
layer = op.dynamicCast<Layer>();
|
|
CV_Assert(layer);
|
|
int opBackend = (opidx < gimpl->execBackend_.size()) ? gimpl->execBackend_[opidx]
|
|
: DNN_BACKEND_OPENCV;
|
|
CV_UNUSED(opBackend); // only consumed by the HAVE_CUDA dispatch below
|
|
const std::vector<Arg>& inputs = op->inputs;
|
|
const std::vector<Arg>& outputs = op->outputs;
|
|
size_t ninputs = inputs.size(), noutputs = outputs.size();
|
|
|
|
inpMats.resize(ninputs);
|
|
inpTypes.resize(ninputs);
|
|
inpShapes.resize(ninputs);
|
|
outMats.clear();
|
|
outOrigData.clear();
|
|
|
|
for (i = 0; i < ninputs; i++) {
|
|
Arg inp = inputs[i];
|
|
#ifdef HAVE_CUDA
|
|
// CPU op: bring any device-resident input back to host before reading its
|
|
// shape/data. This must happen before allocateLayerOutputs(), which may fit()
|
|
// an in-place op's output onto this very buffer and rewrite its header.
|
|
if (opBackend != DNN_BACKEND_CUDA)
|
|
cudaDownloadArg(inp, argTensor(inp));
|
|
#endif
|
|
const Mat& m = argTensor(inp);
|
|
inpMats[i] = m;
|
|
inpTypes[i] = m.type();
|
|
inpShapes[i] = m.shape();
|
|
}
|
|
|
|
if (tracingMode != DNN_TRACE_NONE) {
|
|
strm_ << "-----------\n";
|
|
strm_ << "'" << graph->name() << "' [" << opidx << "/" << nops << "]. " << op->type << " node: " << op->name << "\n";
|
|
for (i = 0; i < ninputs; i++) {
|
|
Arg inp = inputs[i];
|
|
traceArg(strm_, "Input", i, inp, false);
|
|
}
|
|
}
|
|
bool dynamicOutShapes = op->dynamicOutputShapes();
|
|
if (!dynamicOutShapes) {
|
|
allocateLayerOutputs(op, inpTypes, inpShapes, outTypes, outShapes, outOrigData, outMats,
|
|
tempTypes, tempShapes, tempMats, scratchBufs, true);
|
|
} else {
|
|
outMats.resize(noutputs);
|
|
for (i = 0; i < noutputs; i++) {
|
|
Arg out = outputs[i];
|
|
outMats[i] = argTensor(out);
|
|
}
|
|
tempMats = scratchBufs;
|
|
}
|
|
|
|
timestamp = getTickCount();
|
|
|
|
std::vector<Ptr<Graph> >* subgraphs = op->subgraphs();
|
|
if (!subgraphs) {
|
|
// Blobs live on 'op', packed buffers on the executor 'layer'.
|
|
if (layer->packedWeightEpoch != op->weightEpoch) {
|
|
layer->prepackWeights();
|
|
layer->packedWeightEpoch = op->weightEpoch;
|
|
// New weights: re-finalize too, for layers that pack inside finalize().
|
|
layer->finalizedOnce = false;
|
|
}
|
|
// Re-finalize only when this layer's own input signature changed.
|
|
if (!layer->finalizedOnce || layer->lastInpShapes != inpShapes || layer->lastInpTypes != inpTypes) {
|
|
layer->finalize((InputArrayOfArrays)inpMats, (OutputArrayOfArrays)outMats);
|
|
layer->lastInpShapes = inpShapes;
|
|
layer->lastInpTypes = inpTypes;
|
|
layer->finalizedOnce = true;
|
|
}
|
|
#ifdef HAVE_CUDA
|
|
if (opBackend == DNN_BACKEND_CUDA) {
|
|
forwardOpCUDA(this, gimpl, opidx, inputs, outputs, inpMats, outMats);
|
|
} else
|
|
#endif
|
|
{
|
|
// Device-resident inputs were already synced to host in the capture loop above.
|
|
layer->forward(inpMats, outMats, tempMats);
|
|
#ifdef HAVE_CUDA
|
|
// CPU produced fresh host data; invalidate any stale device copy of its outputs.
|
|
for (size_t k = 0; k < noutputs; k++)
|
|
cudaSetHostDirty(outputs[k]);
|
|
#endif
|
|
}
|
|
}
|
|
else {
|
|
Ptr<IfLayer> iflayer = layer.dynamicCast<IfLayer>();
|
|
Ptr<LoopLayer> loopLayer = layer.dynamicCast<LoopLayer>();
|
|
Ptr<ScanLayer> scanLayer = layer.dynamicCast<ScanLayer>();
|
|
if (iflayer) {
|
|
int branch = iflayer->branch(inpMats[0]);
|
|
Ptr<Graph> subgraph = subgraphs->at(branch);
|
|
std::vector<Mat> branchInputs;
|
|
if (inpMats.size() > 1)
|
|
branchInputs.assign(inpMats.begin() + 1, inpMats.end());
|
|
forwardGraph(subgraph, branchInputs, outMats, false);
|
|
}
|
|
else if (loopLayer) {
|
|
CV_Assert(subgraphs->size() == 1);
|
|
Ptr<Graph> body = subgraphs->at(0);
|
|
|
|
int n_in = (int)inpMats.size();
|
|
int n_state = n_in - 2;
|
|
int n_accum = (int)body->outputs().size() - n_state - 1;
|
|
CV_Assert(n_in >= 2 && n_state >= 0 && n_accum >= 0);
|
|
|
|
Mat mIter, mCond, tmp;
|
|
mIter.create(1, 1, CV_64S);
|
|
if (inpMats[1].empty())
|
|
{
|
|
mCond.create(1, 1, CV_8U);
|
|
mCond.at<uchar>(0) = 1;
|
|
}
|
|
else
|
|
{
|
|
mCond = inpMats[1];
|
|
}
|
|
int64 iter = 0;
|
|
int64 max_iter = inpMats[0].empty() ? -1 :
|
|
(inpMats[0].convertTo(tmp, CV_64S), *tmp.ptr<int64>());
|
|
bool active = loopLayer->cond(mCond);
|
|
|
|
std::vector<Mat> state(inpMats.begin() + 2, inpMats.end());
|
|
std::vector<std::vector<Mat> > history(n_accum);
|
|
std::vector<Mat> inputs(n_in), outputs;
|
|
|
|
while (active && (max_iter < 0 || iter < max_iter)) {
|
|
mIter.at<int64>(0) = iter++;
|
|
inputs[0] = mIter;
|
|
inputs[1] = mCond;
|
|
std::copy(state.begin(), state.end(), inputs.begin() + 2);
|
|
|
|
forwardGraph(body, inputs, outputs, false);
|
|
|
|
mCond = outputs[0];
|
|
active = loopLayer->cond(mCond);
|
|
|
|
// Deep-copy: body buffers (and their UMats via Mat::fit reuse) are recycled across iterations.
|
|
for (int i = 0; i < n_state; i++)
|
|
state[i] = outputs[1 + i].clone();
|
|
for (int i = 0; i < n_accum; i++)
|
|
history[i].push_back(outputs[1 + n_state + i].clone());
|
|
}
|
|
|
|
outMats.assign(state.begin(), state.end());
|
|
outMats.resize(n_state + n_accum);
|
|
|
|
for (int i = 0; i < n_accum; ++i) {
|
|
const std::vector<Mat>& perIter = history[i];
|
|
if (perIter.empty()) {
|
|
outMats[n_state + i] = Mat();
|
|
continue;
|
|
}
|
|
|
|
const Mat& first = perIter[0];
|
|
MatShape elemShape = first.shape();
|
|
int ndims = elemShape.dims;
|
|
CV_Assert(ndims >= 0);
|
|
|
|
MatShape outShape(ndims + 1);
|
|
outShape[0] = (int)perIter.size();
|
|
for (int d = 0; d < ndims; ++d)
|
|
outShape[d + 1] = elemShape[d];
|
|
|
|
Mat stacked;
|
|
stacked.create(outShape, first.type());
|
|
|
|
for (size_t k = 0; k < perIter.size(); ++k)
|
|
{
|
|
const Mat& src = perIter[k];
|
|
CV_Assert(src.type() == first.type());
|
|
CV_Assert(src.total() == first.total());
|
|
|
|
Mat dst = stacked.reshape(0, (int)perIter.size())
|
|
.row((int)k)
|
|
.reshape(first.channels(), elemShape.dims, &elemShape[0]);
|
|
src.copyTo(dst);
|
|
}
|
|
outMats[n_state + i] = stacked;
|
|
}
|
|
}
|
|
else if (scanLayer) {
|
|
CV_Assert(subgraphs->size() == 1);
|
|
Ptr<Graph> body = subgraphs->at(0);
|
|
|
|
const int M = scanLayer->numScanInputs();
|
|
const int bodyNIn = (int)body->inputs().size();
|
|
const int S = bodyNIn - M; // loop-carried state count
|
|
const int K = (int)body->outputs().size() - S; // scan output count
|
|
CV_Assert(M > 0 && S >= 0 && K >= 0);
|
|
|
|
// Reject opset-8 (batch dim + sequence_lens); those semantics are not implemented.
|
|
CV_CheckEQ((int)inpMats.size(), S + M,
|
|
"Scan: opset-8 form (batch dim + sequence_lens) is not supported; re-export with opset>=9");
|
|
|
|
const std::vector<int>& iax = scanLayer->scanInputAxes();
|
|
const std::vector<int>& oax = scanLayer->scanOutputAxes();
|
|
const std::vector<int>& idir = scanLayer->scanInputDirections();
|
|
const std::vector<int>& odir = scanLayer->scanOutputDirections();
|
|
const std::vector<int>& orank = scanLayer->scanOutputRanks();
|
|
|
|
std::vector<Mat> scanIn(M);
|
|
std::vector<int> inAxis(M);
|
|
std::vector<char> inRev(M);
|
|
int T = -1;
|
|
for (int j = 0; j < M; j++) {
|
|
scanIn[j] = inpMats[S + j];
|
|
int ax = iax.empty() ? 0 : iax[j];
|
|
if (ax < 0) ax += scanIn[j].dims;
|
|
inAxis[j] = ax;
|
|
inRev[j] = (char)(!idir.empty() && idir[j] != 0);
|
|
int len = scanIn[j].size[ax];
|
|
if (T < 0) T = len; else CV_Assert(T == len);
|
|
}
|
|
CV_Assert(T >= 0);
|
|
|
|
std::vector<Mat> state(S);
|
|
for (int i = 0; i < S; i++) state[i] = inpMats[i];
|
|
|
|
std::vector<std::vector<Mat> > history(K);
|
|
std::vector<Mat> inputs(bodyNIn), outputs;
|
|
|
|
for (int t = 0; t < T; t++) {
|
|
for (int i = 0; i < S; i++) inputs[i] = state[i];
|
|
for (int j = 0; j < M; j++) {
|
|
int idx = inRev[j] ? (T - 1 - t) : t;
|
|
inputs[S + j] = sliceScanAxis(scanIn[j], inAxis[j], idx);
|
|
}
|
|
forwardGraph(body, inputs, outputs, false);
|
|
// Deep-copy: body buffers are recycled across iterations.
|
|
// TODO: alias state in/out buffers like Loop to drop this copy.
|
|
for (int i = 0; i < S; i++) state[i] = outputs[i].clone();
|
|
for (int k = 0; k < K; k++) history[k].push_back(outputs[S + k].clone());
|
|
}
|
|
|
|
outMats.assign(state.begin(), state.end());
|
|
outMats.resize(S + K);
|
|
for (int k = 0; k < K; k++) {
|
|
int ax = oax.empty() ? 0 : oax[k];
|
|
bool rev = !odir.empty() && odir[k] != 0;
|
|
// 0-D scalar output is stored as [1]; drop it so T scalars stack to [T], not [T,1].
|
|
if (k < (int)orank.size() && orank[k] == 0) {
|
|
for (Mat& e : history[k])
|
|
e = e.reshape(0, 0, nullptr);
|
|
}
|
|
outMats[S + k] = stackScanAxis(history[k], ax, rev);
|
|
}
|
|
}
|
|
else {
|
|
CV_Error_(Error::StsNotImplemented,
|
|
("unknown layer type '%s' with subgraphs", op->type.c_str()));
|
|
}
|
|
}
|
|
CV_Assert(outMats.size() == noutputs);
|
|
|
|
for (i = 0; i < noutputs; i++) {
|
|
Arg out = outputs[i];
|
|
ArgData& adata = args[out.idx];
|
|
const Mat& m = outMats[i];
|
|
//checkRange(m, false);
|
|
adata.type = m.type();
|
|
adata.shape = m.shape();
|
|
if (adata.kind == DNN_ARG_TEMP) {
|
|
int bufidx = bufidxs.at(out.idx);
|
|
Mat& buf = buffers.at(bufidx);
|
|
|
|
if (!dynamicOutShapes) {
|
|
// a sanity check: make sure that the data was not reallocated during Layer::forward()
|
|
// if the layer claims it does not produce dynamic-shape outputs.
|
|
CV_Assert_N(buf.u == m.u,
|
|
buf.shape() == m.shape(),
|
|
buf.type() == m.type(),
|
|
(!m.u || m.u->data == outOrigData[i].first),
|
|
(!m.u || m.u->size == outOrigData[i].second));
|
|
} else if (!buf.u || (m.u && m.u->size > buf.u->size)) {
|
|
buf = m;
|
|
} else {
|
|
// this branch means that the layer still calls
|
|
// 'create()' rather than 'fit()'; that needs to be fixed, but
|
|
// we provide workaround here at the expense of extra copy.
|
|
buf.fit(m.shape(), m.type());
|
|
m.copyTo(buf);
|
|
}
|
|
} else {
|
|
if (!dynamicOutShapes) {
|
|
// the same sanity check for non-temp (graph output/state) tensors: the layer must
|
|
// write into the preallocated tensor of the inferred shape/type. A mismatch here
|
|
// means some op inside Layer::forward() reallocated it (e.g. a broadcast produced
|
|
// an unexpected shape) and the result would silently detach from the graph.
|
|
if (m.shape() != outShapes[i] || m.type() != outTypes[i] ||
|
|
(m.u && (m.u->data != outOrigData[i].first || m.u->size != outOrigData[i].second)))
|
|
{
|
|
std::ostringstream oss;
|
|
oss << "layer '" << layer->name << "' (" << layer->type << "): output #" << i
|
|
<< " changed during forward(): inferred shape " << outShapes[i]
|
|
<< " / type " << typeToString(outTypes[i])
|
|
<< ", actual " << m.shape() << " / " << typeToString(m.type())
|
|
<< (m.u && m.u->data != outOrigData[i].first
|
|
? "; the tensor was reallocated (the layer must write in place)" : "");
|
|
CV_Error(Error::StsInternal, oss.str());
|
|
}
|
|
}
|
|
__tensors__.at(out.idx) = m;
|
|
}
|
|
}
|
|
|
|
size_t ntemps = tempMats.size();
|
|
scratchBufs.resize(std::max(ntemps, scratchBufs.size()));
|
|
for (size_t i = 0; i < ntemps; i++) {
|
|
size_t newtotal_i = tempMats[i].total()*tempMats[i].elemSize();
|
|
size_t total_i = scratchBufs[i].total()*scratchBufs[i].elemSize();
|
|
if (newtotal_i > total_i) {
|
|
scratchBufs[i] = tempMats[i];
|
|
}
|
|
}
|
|
|
|
timestamp = getTickCount() - timestamp;
|
|
layersTimings[opidx + graph_ofs + 1] += timestamp;
|
|
|
|
if (tracingMode != DNN_TRACE_NONE) {
|
|
strm_ << "TIME (\"" << layer->name << "\", \"" << layer->type << "\"): " <<
|
|
format("%.2fms", (double)timestamp*1000./tickfreq) << "\n";
|
|
for (i = 0; i < noutputs; i++) {
|
|
Arg out = outputs[i];
|
|
traceArg(strm_, "Output", i, out, tracingMode == DNN_TRACE_ALL);
|
|
}
|
|
}
|
|
}
|
|
|
|
std::vector<Mat>& outputsVec = outputs_.getMatVecRef();
|
|
outputsVec.resize(n_gr_outputs);
|
|
for (i = 0; i < n_gr_outputs; i++) {
|
|
Arg out = gr_outputs[i];
|
|
#ifdef HAVE_CUDA
|
|
// A graph output produced on the device must be brought back to host before it is read.
|
|
if (out.idx >= 0) {
|
|
Mat& t = argTensor(out);
|
|
cudaDownloadArg(out, t);
|
|
}
|
|
#endif
|
|
const Mat& outm = argTensor(out);
|
|
if (isMainGraph) {
|
|
if (outm.size.layout == DATA_LAYOUT_BLOCK) {
|
|
transformLayout(outm, outputsVec[i], originalLayout, originalLayout, outm.size.C);
|
|
} else {
|
|
outputsVec[i].fit(outm.shape(), outm.type());
|
|
outm.copyTo(outputsVec[i]);
|
|
}
|
|
} else {
|
|
outputsVec[i] = outm;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void Net::Impl::updateUseCounts(const Ptr<Graph>& graph, std::vector<int>& usecounts) const
|
|
{
|
|
if (!graph)
|
|
return;
|
|
const std::vector<Arg>& gr_outputs = graph->outputs();
|
|
for (const Arg& output: gr_outputs) {
|
|
CV_Assert(output.idx < (int)usecounts.size());
|
|
usecounts[output.idx]++;
|
|
}
|
|
const std::vector<Ptr<LayerInfo> >& prog = graph->prog();
|
|
for (const Ptr<LayerInfo>& layer: prog) {
|
|
const std::vector<Arg>& inputs = layer->inputs;
|
|
for (const Arg& input: inputs) {
|
|
CV_Assert(input.idx < (int)usecounts.size());
|
|
usecounts[input.idx]++;
|
|
}
|
|
const std::vector<Ptr<Graph> >* subgraphs = layer->subgraphs();
|
|
if (subgraphs) {
|
|
for (const Ptr<Graph>& subgraph: *subgraphs) {
|
|
updateUseCounts(subgraph, usecounts);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Net::Impl::useCounts(std::vector<int>& usecounts) const
|
|
{
|
|
size_t nargs = args.size();
|
|
usecounts.assign(nargs, 0);
|
|
usecounts[0] = 1; // empty Arg() is always useful
|
|
updateUseCounts(mainGraph, usecounts);
|
|
}
|
|
|
|
int Net::Impl::updateGraphOfs(const Ptr<Graph>& graph, int currofs, bool ismain)
|
|
{
|
|
CV_Assert(currofs >= 0);
|
|
if (ismain) {
|
|
graphofs.clear();
|
|
allgraphs.clear();
|
|
layerNameToId.clear();
|
|
}
|
|
const std::vector<Ptr<LayerInfo> >& prog = graph->prog();
|
|
size_t i, nops = prog.size();
|
|
int subgraph_ofs = currofs + (int)nops;
|
|
std::string name = graph->name();
|
|
graphofs.insert(std::make_pair(name, currofs));
|
|
allgraphs.push_back(graph);
|
|
for (i = 0; i < nops; i++) {
|
|
const Ptr<LayerInfo>& layer = prog[i];
|
|
layerNameToId.insert(std::make_pair(layer->name, currofs + (int)i));
|
|
const std::vector<Ptr<Graph> >* subgraphs = layer->subgraphs();
|
|
if (subgraphs) {
|
|
for (const Ptr<Graph>& subgraph : *subgraphs) {
|
|
subgraph_ofs = updateGraphOfs(subgraph, subgraph_ofs, false);
|
|
}
|
|
}
|
|
}
|
|
return subgraph_ofs;
|
|
}
|
|
|
|
bool Net::Impl::tryInferShapes(const std::vector<MatShape>& suggestedInpShapes,
|
|
const std::vector<MatType>& suggestedInpTypes,
|
|
LayerShapes& result,
|
|
std::vector<MatShape>& shapeCache,
|
|
std::vector<MatType>& typeCache) const
|
|
{
|
|
result.in.clear();
|
|
result.out.clear();
|
|
result.inTypes.clear();
|
|
result.outTypes.clear();
|
|
|
|
CV_Assert(mainGraph);
|
|
size_t nargs = args.size();
|
|
shapeCache.assign(nargs, MatShape());
|
|
typeCache.assign(nargs, -1);
|
|
|
|
const std::vector<Arg>& inputs = mainGraph->inputs();
|
|
const std::vector<Arg>& outputs = mainGraph->outputs();
|
|
|
|
size_t ninputs = inputs.size();
|
|
size_t noutputs = outputs.size();
|
|
|
|
size_t nsuggestedShapes = suggestedInpShapes.size();
|
|
size_t nsuggestedTypes = suggestedInpTypes.size();
|
|
CV_Assert(nsuggestedShapes == 0 || nsuggestedShapes == ninputs ||
|
|
|
|
// workaround, but this is not quite correct usage of the function
|
|
(nsuggestedShapes == 1 && suggestedInpShapes[0].empty())
|
|
);
|
|
CV_Assert(nsuggestedTypes <= 1 || nsuggestedTypes == ninputs);
|
|
bool dynamicInputShapes = false;
|
|
|
|
result.in.resize(ninputs);
|
|
result.inTypes.resize(ninputs);
|
|
|
|
for (size_t i = 0; i < ninputs; i++) {
|
|
Arg inp = inputs[i];
|
|
const ArgData& adata = args.at(inp.idx);
|
|
CV_Assert(adata.kind == DNN_ARG_INPUT);
|
|
|
|
int type;
|
|
MatShape shape;
|
|
const Mat& tensor = argTensor(inp);
|
|
if (!tensor.empty()) {
|
|
type = tensor.type();
|
|
shape = tensor.shape();
|
|
} else {
|
|
type = adata.type;
|
|
shape = adata.shape;
|
|
}
|
|
|
|
if (nsuggestedTypes) {
|
|
int suggestedType = suggestedInpTypes[i < nsuggestedTypes ? i : 0];
|
|
if (suggestedType == -1)
|
|
suggestedType = type;
|
|
if (adata.type == type ||
|
|
((adata.type == CV_32F || adata.type == CV_16F || adata.type == CV_16BF) &&
|
|
(suggestedType == CV_32F || suggestedType == CV_16F || suggestedType == CV_16BF)))
|
|
;
|
|
else {
|
|
CV_Error_(Error::StsBadArg, ("mismatched type for model input '%s': %s provided, %s expected",
|
|
adata.name.c_str(), typeToString(suggestedType).c_str(),
|
|
typeToString(adata.type).c_str()));
|
|
}
|
|
type = suggestedType;
|
|
}
|
|
|
|
if (nsuggestedShapes) {
|
|
MatShape suggestedShape = suggestedInpShapes[i < nsuggestedShapes ? i : 0];
|
|
if (suggestedShape.empty()) {
|
|
suggestedShape = shape;
|
|
}
|
|
// [TODO] shut up it for now;
|
|
// too many ONNX conformance tests
|
|
// depend on this "liberal" behaviour
|
|
//
|
|
// CV_Assert(suggestedShape.dims == adata.shape.dims);
|
|
shape = suggestedShape;
|
|
}
|
|
|
|
typeCache[inp.idx] = type;
|
|
shapeCache[inp.idx] = shape;
|
|
|
|
if (shape.hasSymbols()) {
|
|
CV_LOG_WARNING(NULL, format("the shape of model input '%s' includes symbols. Shape inference is impossible without prior calls to setInput()",
|
|
adata.name.c_str()));
|
|
dynamicInputShapes = true;
|
|
shape = MatShape();
|
|
}
|
|
|
|
result.inTypes[i] = type;
|
|
result.in[i] = shape;
|
|
}
|
|
|
|
bool inferenced = false;
|
|
if (!dynamicInputShapes)
|
|
inferenced = tryInferGraphShapes(mainGraph, shapeCache, typeCache);
|
|
bool missingOutputs = false;
|
|
|
|
result.outTypes.resize(noutputs, -1);
|
|
result.out.resize(noutputs);
|
|
|
|
for (size_t i = 0; i < noutputs; i++) {
|
|
Arg out = outputs[i];
|
|
const ArgData adata = args.at(out.idx);
|
|
int type = typeCache.at(out.idx);
|
|
MatShape shape = shapeCache.at(out.idx);
|
|
if (type < 0) {
|
|
if (!inferenced)
|
|
type = adata.type;
|
|
if (type < 0) {
|
|
CV_LOG_WARNING(NULL, format("type for output '%s' was not inferred", adata.name.c_str()));
|
|
missingOutputs = true;
|
|
}
|
|
}
|
|
|
|
result.outTypes[i] = type;
|
|
result.out[i] = shape;
|
|
}
|
|
|
|
return inferenced && !missingOutputs;
|
|
}
|
|
|
|
// [TODO]
|
|
// The current 'pure' shape inference is quite fragile, it does not handle any dynamic cases
|
|
// or even some seemingly dynamic cases.
|
|
// It would be nice maybe to some optional speculative forward() with some dummy inputs when
|
|
// straight-forward shape inference mechanism failed.
|
|
bool Net::Impl::tryInferGraphShapes(const Ptr<Graph>& graph,
|
|
std::vector<MatShape>& shapeCache,
|
|
std::vector<MatType>& typeCache) const
|
|
{
|
|
if (!graph)
|
|
return true;
|
|
|
|
const std::vector<Ptr<LayerInfo> >& prog = graph->prog();
|
|
|
|
std::vector<MatShape> inpShapes, outShapes, tempShapes;
|
|
std::vector<int> inpTypes, outTypes, tempTypes;
|
|
|
|
for (const Ptr<LayerInfo>& layer: prog) {
|
|
if (!layer)
|
|
continue;
|
|
|
|
const std::vector<Ptr<Graph> >* subgraphs = layer->subgraphs();
|
|
if (subgraphs) {
|
|
CV_LOG_WARNING(NULL, format("shape inference for the model with subgraphs (node %s (%s)) is not supported yet", layer->name.c_str(), layer->type.c_str()));
|
|
}
|
|
|
|
if (layer->dynamicOutputShapes()) {
|
|
CV_LOG_WARNING(NULL, format("DNN/InferShape: Layer '%s' (%s) output shapes cannot be inferenced without running forward()", layer->name.c_str(), layer->type.c_str()));
|
|
return false;
|
|
}
|
|
|
|
const std::vector<Arg>& inputs = layer->inputs;
|
|
const std::vector<Arg>& outputs = layer->outputs;
|
|
|
|
int ninputs = (int)inputs.size();
|
|
int noutputs = (int)outputs.size();
|
|
|
|
inpShapes.resize(ninputs);
|
|
inpTypes.resize(ninputs);
|
|
outShapes.clear();
|
|
outTypes.clear();
|
|
tempShapes.clear();
|
|
tempTypes.clear();
|
|
|
|
for (int i = 0; i < ninputs; i++) {
|
|
Arg inp = inputs[i];
|
|
const ArgData& adata = args.at(inp.idx);
|
|
MatShape shape;
|
|
int type;
|
|
|
|
if (adata.kind == DNN_ARG_CONST || adata.kind == DNN_ARG_EMPTY) {
|
|
shape = adata.shape;
|
|
type = adata.type;
|
|
|
|
// unnecessary, but nice to have for consistency
|
|
shapeCache[inp.idx] = shape;
|
|
typeCache[inp.idx] = type;
|
|
} else {
|
|
shape = shapeCache[inp.idx];
|
|
type = typeCache[inp.idx];
|
|
if (type < 0) {
|
|
CV_Error_(Error::StsInternal, ("input '%s' of operation '%s' (%s) does not have a proper type",
|
|
adata.name.c_str(), layer->name.c_str(), layer->type.c_str()));
|
|
}
|
|
}
|
|
inpShapes[i] = shape;
|
|
inpTypes[i] = type;
|
|
}
|
|
|
|
layer->getMemoryShapes(inpShapes, noutputs, outShapes, tempShapes);
|
|
CV_Assert((int)outShapes.size() == noutputs);
|
|
layer->getTypes(inpTypes, noutputs, (int)tempShapes.size(), outTypes, tempTypes);
|
|
CV_Assert((int)outTypes.size() == noutputs);
|
|
|
|
for (int i = 0; i < (int)tempShapes.size(); i++)
|
|
CV_CheckGT(total(tempShapes[i]), (size_t)0, "");
|
|
|
|
for (int i = 0; i < noutputs; i++) {
|
|
Arg out = outputs[i];
|
|
if (out.idx == 0)
|
|
continue;
|
|
shapeCache[out.idx] = outShapes[i];
|
|
typeCache[out.idx] = outTypes[i];
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void Net::Impl::checkArgs(const std::vector<Arg>& args_) const
|
|
{
|
|
for (const Arg& a: args_) {
|
|
checkArg(a);
|
|
}
|
|
}
|
|
|
|
void Net::Impl::checkArg(Arg a) const
|
|
{
|
|
CV_Assert(a.idx >= 0);
|
|
CV_Assert(a.idx < (int)args.size());
|
|
}
|
|
|
|
std::ostream& Net::Impl::dumpDim(std::ostream& strm, int value) const
|
|
{
|
|
if (value >= 0) {
|
|
strm << value;
|
|
} else {
|
|
size_t idx = -value;
|
|
if (idx < dimnames_vec.size())
|
|
strm << dimnames_vec[idx];
|
|
else
|
|
strm << "sym(" << idx << ")";
|
|
}
|
|
return strm;
|
|
}
|
|
|
|
std::ostream& Net::Impl::dumpTypeShape(std::ostream& strm, int type, const MatShape& shape) const
|
|
{
|
|
if (shape.empty()) {
|
|
strm << "<empty>";
|
|
} else {
|
|
strm << typeToString(type);
|
|
if (shape.dims > 0 && shape.layout != DATA_LAYOUT_UNKNOWN) {
|
|
strm << " " << layoutToString(shape.layout);
|
|
}
|
|
strm << " [";
|
|
for (int i = 0; i < shape.dims; i++) {
|
|
strm << (i > 0 ? " x " : "");
|
|
dumpDim(strm, shape[i]);
|
|
}
|
|
strm << "]";
|
|
}
|
|
return strm;
|
|
}
|
|
|
|
std::ostream& Net::Impl::dumpArg(std::ostream& strm, Arg arg, int indent,
|
|
bool comma, bool dump_details) const
|
|
{
|
|
checkArg(arg);
|
|
const ArgData& adata = args.at(arg.idx);
|
|
prindent(strm, indent);
|
|
if (arg.empty()) {
|
|
strm << "<empty>" << (comma ? "," : "");
|
|
} else {
|
|
strm << '\"' << adata.name << (comma ? "\"," : "\"");
|
|
if (dump_details && arg.idx > 0) {
|
|
strm << " // ";
|
|
strm << (adata.kind == DNN_ARG_INPUT ? "<Input>" :
|
|
adata.kind == DNN_ARG_OUTPUT ? "<Output>" :
|
|
adata.kind == DNN_ARG_CONST ? "<Const>" :
|
|
adata.kind == DNN_ARG_TEMP ? "<Temp>" :
|
|
"<Uknown kind>");
|
|
if (adata.type >= 0) {
|
|
strm << " ";
|
|
dumpTypeShape(strm, adata.type, adata.shape);
|
|
}
|
|
if (adata.kind == DNN_ARG_TEMP && ((size_t)arg.idx < bufidxs.size()))
|
|
strm << " (buf #" << bufidxs[arg.idx] << ")";
|
|
}
|
|
}
|
|
strm << "\n";
|
|
return strm;
|
|
}
|
|
|
|
std::ostream& Net::Impl::dump(std::ostream& strm)
|
|
{
|
|
int indent = dump_indent;
|
|
strm << "{\n";
|
|
prindent(strm, indent);
|
|
strm << "model_format: \"" << modelFormatToString(modelFormat) << "\",\n";
|
|
prindent(strm, indent);
|
|
strm << "layout: \"" << layoutToString(originalLayout) << "\",\n";
|
|
if (mainGraph) {
|
|
prindent(strm, indent);
|
|
strm << "main_graph: ";
|
|
mainGraph->dump(strm, indent, false);
|
|
}
|
|
strm << "}\n";
|
|
return strm;
|
|
}
|
|
|
|
CV__DNN_INLINE_NS_END
|
|
}} // namespace cv::dnn
|