Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-09-12 22:59:44 -05:00
Code Issues Packages Projects Releases 1 Wiki Activity

add face detector support

Browse Source
This commit is contained in:
Neucrack
2024-05-16 19:08:08 +08:00
parent 115dab3c89
commit f9f82e2a70
15 changed files with 1732 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
components/basic/include/maix_tensor.hpp
View File

@@ -16,6 +16,7 @@
#include <algorithm>
#include <tuple>
#include <map>
#include <valarray>
#include "maix_log.hpp"
#include "maix_err.hpp"
@@ -247,6 +248,16 @@ namespace maix
void *data() { return _data; }
/**
* get tensor data and return a list
* @return list type data
* @maixpy maix.tensor.Tensor.to_float_list
*/
std::valarray<float>* to_float_list()
{
return new std::valarray<float>((float*)_data, size_int());
}
void operator=(Tensor &t)
{
printf("copy tensor %d, %d\n", _is_alloc, size_int());

BIN
components/maixcam_lib/lib/libmaixcam_lib.so
View File

Binary file not shown.

45
components/nn/include/libmaix_nn_decoder_retinaface.hpp Normal file
View File

@@ -0,0 +1,45 @@
/*
retinaface decoder
@author neucrack@sipeed
@date 2021-5-15 create for libmaix by neucrack
2021-8-18 update for libmaix by neucrack
2024-5-15 copy and edit for MaixCDK by neucrack
@license MIT
*/
#ifndef __DECODER_RETINAFACE_H
#define __DECODER_RETINAFACE_H
#include "maix_nn_object.hpp"
#include <stdint.h>
#include <stdbool.h>
using namespace maix;
#define ANCHOR_SIZE_NUM 3
#define MIN_SIZE_LEN 6
typedef struct
{
float variance[2];
int steps[ANCHOR_SIZE_NUM];
int min_sizes[ANCHOR_SIZE_NUM * 2];
float nms;
float score_thresh;
int input_w;
int input_h;
// set by init func
int channel_num;
}libmaix_nn_decoder_retinaface_config_t;
/************ direct API ***********/
extern nn::ObjectFloat* retinaface_get_priorboxes(libmaix_nn_decoder_retinaface_config_t* config, int* boxes_num);
extern int retinaface_decode(float* net_out_loc, float* net_out_conf, float* net_out_landmark, nn::ObjectFloat* prior_boxes, std::vector<nn::Object> *faces, int* boxes_num, bool chw, libmaix_nn_decoder_retinaface_config_t* config);
extern int retinaface_get_channel_num(libmaix_nn_decoder_retinaface_config_t* config);
#endif

505
components/nn/include/maix_nn_face_detector.hpp Normal file
View File

@@ -0,0 +1,505 @@
/**
* @author neucrack@sipeed
* @copyright Sipeed Ltd 2023-
* @license Apache 2.0
* @update 2024.5.15: Create this file.
*/
#pragma once
#include "maix_basic.hpp"
#include "maix_nn.hpp"
#include "maix_image.hpp"
#include "maix_nn_F.hpp"
#include "maix_nn_object.hpp"
#include <tuple>
namespace maix::nn
{
/**
* FaceDetector class
* @maixpy maix.nn.FaceDetector
*/
class FaceDetector
{
public:
public:
/**
* Constructor of FaceDetector class
* @param model model path, default empty, you can load model later by load function.
* @throw If model arg is not empty and load failed, will throw err::Exception.
* @maixpy maix.nn.FaceDetector.__init__
* @maixcdk maix.nn.FaceDetector.FaceDetector
*/
FaceDetector(const string &model = "")
{
_model = nullptr;
if (!model.empty())
{
err::Err e = load(model);
if (e != err::ERR_NONE)
{
throw err::Exception(e, "load model failed");
}
}
}
~FaceDetector()
{
if (_model)
{
delete _model;
_model = nullptr;
}
}
/**
* Load model from file
* @param model Model path want to load
* @return err::Err
* @maixpy maix.nn.FaceDetector.load
*/
err::Err load(const string &model)
{
if (_model)
{
delete _model;
_model = nullptr;
}
_model = new nn::NN(model);
if (!_model)
{
return err::ERR_NO_MEM;
}
_extra_info = _model->extra_info();
if (_extra_info.find("model_type") != _extra_info.end())
{
if (_extra_info["model_type"] != "face_detector")
{
log::error("model_type not match, expect 'face_detector', but got '%s'", _extra_info["model_type"].c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("model_type key not found");
return err::ERR_ARGS;
}
log::info("model info:\n\ttype: face_detector");
if (_extra_info.find("input_type") != _extra_info.end())
{
std::string input_type = _extra_info["input_type"];
if (input_type == "rgb")
{
_input_img_fmt = maix::image::FMT_RGB888;
log::print("\tinput type: rgb\n");
}
else if (input_type == "bgr")
{
_input_img_fmt = maix::image::FMT_BGR888;
log::print("\tinput type: bgr\n");
}
else
{
log::error("unknown input type: %s", input_type.c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("input_type key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("mean") != _extra_info.end())
{
std::string mean_str = _extra_info["mean"];
std::vector<std::string> mean_strs = split(mean_str, ",");
log::print("\tmean:");
for (auto &it : mean_strs)
{
try
{
this->mean.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("mean value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->mean.back());
}
log::print("\n");
}
else
{
log::error("mean key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("scale") != _extra_info.end())
{
std::string scale_str = _extra_info["scale"];
std::vector<std::string> scale_strs = split(scale_str, ",");
log::print("\tscale:");
for (auto &it : scale_strs)
{
try
{
this->scale.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("scale value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->scale.back());
}
log::print("\n");
}
else
{
log::error("scale key not found");
return err::ERR_ARGS;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
// decoder params
// priorbox
std::vector<std::tuple<int, int>> feature_maps;
int steps[] = {8, 16, 32, 64};
_variance.push_back(0.1);
_variance.push_back(0.2);
std::vector<std::vector<int>> min_sizes = {
{10, 16, 24},
{32, 48},
{64, 96},
{128, 192, 256}
};
for(size_t i=0; i<sizeof(steps) / sizeof(int); ++i)
{
feature_maps.push_back(std::tuple<int, int>(ceil(_input_size.height() / steps[i]), ceil(_input_size.width() / steps[i])));
}
for(size_t i=0; i<feature_maps.size(); ++i)
{
for(int j=0; j < std::get<0>(feature_maps[i]); ++j)
{
for(int k=0; k < std::get<1>(feature_maps[i]); ++k)
{
for(size_t m=0; m < min_sizes[i].size(); ++m)
{
float s_kx = min_sizes[i][m] * 1.0 / _input_size.width();
float s_ky = min_sizes[i][m] * 1.0 / _input_size.height();
float dense_cx = (k + 0.5) * steps[i] / _input_size.width();
float dense_cy = (j + 0.5) * steps[i] / _input_size.height();
_anchor.push_back({dense_cx, dense_cy, s_kx, s_ky});
}
}
}
}
return err::ERR_NONE;
}
/**
* Detect objects from image
* @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
* @param conf_th Confidence threshold, default 0.5.
* @param iou_th IoU threshold, default 0.45.
* @param fit Resize method, default image.Fit.FIT_CONTAIN.
* @throw If image format not match model input format, will throw err::Exception.
* @return Object list. In C++, you should delete it after use.
* @maixpy maix.nn.FaceDetector.detect
*/
std::vector<nn::Object> *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN)
{
this->_conf_th = conf_th;
this->_iou_th = iou_th;
if (img.format() != _input_img_fmt)
{
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
if (!outputs)
{
throw err::Exception("forward image failed");
}
std::vector<nn::Object> *res = _post_process(outputs, img.width(), img.height(), fit);
delete outputs;
return res;
}
/**
* Get model input size
* @return model input size
* @maixpy maix.nn.FaceDetector.input_size
*/
image::Size input_size()
{
return _input_size;
}
/**
* Get model input width
* @return model input size of width
* @maixpy maix.nn.FaceDetector.input_width
*/
int input_width()
{
return _input_size.width();
}
/**
* Get model input height
* @return model input size of height
* @maixpy maix.nn.FaceDetector.input_height
*/
int input_height()
{
return _input_size.height();
}
/**
* Get input image format
* @return input image format, image::Format type.
* @maixpy maix.nn.FaceDetector.input_format
*/
image::Format input_format()
{
return _input_img_fmt;
}
public:
/**
* Get mean value, list type
* @maixpy maix.nn.FaceDetector.mean
*/
std::vector<float> mean;
/**
* Get scale value, list type
* @maixpy maix.nn.FaceDetector.scale
*/
std::vector<float> scale;
private:
image::Size _input_size;
image::Format _input_img_fmt;
nn::NN *_model;
std::map<string, string> _extra_info;
float _conf_th = 0.5;
float _iou_th = 0.45;
std::vector<std::vector<float>> _anchor; // [[dense_cx, dense_cy, s_kx, s_ky],]
std::vector<float> _variance;
private:
std::vector<nn::Object> *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit)
{
std::vector<nn::Object> *objects = new std::vector<nn::Object>();
tensor::Tensor *conf = nullptr;
tensor::Tensor *loc = nullptr;
tensor::Tensor *landms = nullptr;
for(auto i : outputs->tensors)
{
if(i.second->shape()[2] == 2)
conf = i.second;
else if(i.second->shape()[2] == 4)
loc = i.second;
else if(i.second->shape()[2] == 10)
landms = i.second;
}
if(!conf || !loc || !landms)
return nullptr;
float *conf_data = (float *)conf->data();
float *loc_data = (float *)loc->data();
float *landms_data = (float *)landms->data();
std::vector<int> valid_idx;
for (int i = 0; i < conf->size_int() / 2; ++i)
{
if (conf_data[i * 2 + 1] >= this->_conf_th)
{
valid_idx.push_back(i);
}
}
for (int idx : valid_idx)
{
float x = (_anchor[idx][0] + loc_data[idx * 4] * _variance[0] * _anchor[idx][2]) * _input_size.width();
float y = (_anchor[idx][1] + loc_data[idx * 4 + 1] * _variance[0] * _anchor[idx][3]) * _input_size.height();
float w = _anchor[idx][2] * exp(loc_data[idx * 4 + 2] * _variance[1]) * _input_size.width();
float h = _anchor[idx][3] * exp(loc_data[idx * 4 + 3] * _variance[1]) * _input_size.height();
std::vector<int> points;
points.push_back(_input_size.width() * (_anchor[idx][0] + landms_data[idx * 10] * _variance[0] * _anchor[idx][2]));
points.push_back(_input_size.height() * (_anchor[idx][1] + landms_data[idx * 10 + 1] * _variance[0] * _anchor[idx][3]));
points.push_back(_input_size.width() * (_anchor[idx][0] + landms_data[idx * 10 + 2] * _variance[0] * _anchor[idx][2]));
points.push_back(_input_size.height() * (_anchor[idx][1] + landms_data[idx * 10 + 3] * _variance[0] * _anchor[idx][3]));
points.push_back(_input_size.width() * (_anchor[idx][0] + landms_data[idx * 10 + 4] * _variance[0] * _anchor[idx][2]));
points.push_back(_input_size.height() * (_anchor[idx][1] + landms_data[idx * 10 + 5] * _variance[0] * _anchor[idx][3]));
points.push_back(_input_size.width() * (_anchor[idx][0] + landms_data[idx * 10 + 6] * _variance[0] * _anchor[idx][2]));
points.push_back(_input_size.height() * (_anchor[idx][1] + landms_data[idx * 10 + 7] * _variance[0] * _anchor[idx][3]));
points.push_back(_input_size.width() * (_anchor[idx][0] + landms_data[idx * 10 + 8] * _variance[0] * _anchor[idx][2]));
points.push_back(_input_size.height() * (_anchor[idx][1] + landms_data[idx * 10 + 9] * _variance[0] * _anchor[idx][3]));
nn::Object object((int)(x - w/2), (int)(y - h/2), w, h, 0, conf_data[idx * 2 + 1], points);
objects->push_back(object);
}
if (objects->size() > 0)
{
std::vector<nn::Object> *objects_total = objects;
objects = _nms(*objects);
delete objects_total;
}
if (objects->size() > 0)
_correct_bbox(*objects, img_w, img_h, fit);
return objects;
}
std::vector<nn::Object> *_nms(std::vector<nn::Object> &objs)
{
std::vector<nn::Object> *result = new std::vector<nn::Object>();
std::sort(objs.begin(), objs.end(), [](const nn::Object &a, const nn::Object &b)
{ return a.score < b.score; });
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &a = objs.at(i);
if (a.score == 0)
continue;
for (size_t j = i + 1; j < objs.size(); ++j)
{
nn::Object &b = objs.at(j);
if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
{
b.score = 0;
}
}
}
for (nn::Object &a : objs)
{
if (a.score != 0)
result->push_back(a);
}
return result;
}
void _correct_bbox(std::vector<nn::Object> &objs, int img_w, int img_h, maix::image::Fit fit)
{
if (img_w == _input_size.width() && img_h == _input_size.height())
return;
if (fit == maix::image::FIT_FILL)
{
float scale_x = (float)img_w / _input_size.width();
float scale_y = (float)img_h / _input_size.height();
for (nn::Object &obj : objs)
{
obj.x *= scale_x;
obj.y *= scale_y;
obj.w *= scale_x;
obj.h *= scale_y;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) *= scale_x;
obj.points.at(i * 2 + 1) *= scale_y;
}
}
}
else if (fit == maix::image::FIT_CONTAIN)
{
float scale_x = ((float)_input_size.width()) / img_w;
float scale_y = ((float)_input_size.height()) / img_h;
float scale = std::min(scale_x, scale_y);
float scale_reverse = 1.0 / scale;
float pad_w = (_input_size.width() - img_w * scale) / 2.0;
float pad_h = (_input_size.height() - img_h * scale) / 2.0;
for (nn::Object &obj : objs)
{
obj.x = (obj.x - pad_w) * scale_reverse;
obj.y = (obj.y - pad_h) * scale_reverse;
obj.w *= scale_reverse;
obj.h *= scale_reverse;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) = (obj.points.at(i * 2) - pad_w) * scale_reverse;
obj.points.at(i * 2 + 1) = (obj.points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
}
}
else if (fit == maix::image::FIT_COVER)
{
float scale_x = ((float)_input_size.width()) / img_w;
float scale_y = ((float)_input_size.height()) / img_h;
float scale = std::max(scale_x, scale_y);
float scale_reverse = 1.0 / scale;
float pad_w = (img_w * scale - _input_size.width()) / 2.0;
float pad_h = (img_h * scale - _input_size.height()) / 2.0;
for (nn::Object &obj : objs)
{
obj.x = (obj.x + pad_w) * scale_reverse;
obj.y = (obj.y + pad_h) * scale_reverse;
obj.w *= scale_reverse;
obj.h *= scale_reverse;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) = (obj.points.at(i * 2) - pad_w) * scale_reverse;
obj.points.at(i * 2 + 1) = (obj.points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
}
}
else
{
throw err::Exception(err::ERR_ARGS, "fit type not support");
}
}
inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); }
inline static float _calc_iou(Object &a, Object &b)
{
float area1 = a.w * a.h;
float area2 = b.w * b.h;
float wi = std::min((a.x + a.w), (b.x + b.w)) -
std::max(a.x, b.x);
float hi = std::min((a.y + a.h), (b.y + b.h)) -
std::max(a.y, b.y);
float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
return area_i / (area1 + area2 - area_i);
}
template <typename T>
static int _argmax(const T *data, size_t len, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = 1; i < len; i++)
{
int idx = i * stride;
if (data[maxIndex * stride] < data[idx])
{
maxIndex = i;
}
}
return maxIndex;
}
static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
{
items.clear();
size_t pos_start = 0, pos_end, delim_len = delimiter.length();
std::string token;
while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
{
token = s.substr(pos_start, pos_end - pos_start);
pos_start = pos_end + delim_len;
items.push_back(token);
}
items.push_back(s.substr(pos_start));
}
static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
{
std::vector<std::string> tokens;
split0(tokens, s, delimiter);
return tokens;
}
};
} // namespace maix::nn

92
components/nn/include/maix_nn_object.hpp
View File

@@ -8,6 +8,7 @@
#pragma once
#include <string>
#include <vector>
namespace maix::nn
{
@@ -29,8 +30,8 @@ namespace maix::nn
* @maixpy maix.nn.Object.__init__
* @maixcdk maix.nn.Object.Object
*/
Object(int x = 0, int y = 0, int w = 0, int h = 0, int class_id = 0, float score = 0)
: x(x), y(y), w(w), h(h), class_id(class_id), score(score)
Object(int x = 0, int y = 0, int w = 0, int h = 0, int class_id = 0, float score = 0, std::vector<int> points = std::vector<int>())
: x(x), y(y), w(w), h(h), class_id(class_id), score(score), points(points)
{
}
@@ -84,6 +85,93 @@ namespace maix::nn
* @maixpy maix.nn.Object.score
*/
float score;
/**
* keypoints
* @maixpy maix.nn.Object.points
*/
std::vector<int> points;
};
/**
* Object for detect result
* @maixpy maix.nn.ObjectFloat
*/
class ObjectFloat
{
public:
/**
* Constructor of Object for detect result
* @param x left top x
* @param y left top y
* @param w width
* @param h height
* @param class_id class id
* @param score score
* @maixpy maix.nn.ObjectFloat.__init__
* @maixcdk maix.nn.ObjectFloat.ObjectFloat
*/
ObjectFloat(float x = 0, float y = 0, float w = 0, float h = 0, float class_id = 0, float score = 0, std::vector<float> points = std::vector<float>())
: x(x), y(y), w(w), h(h), class_id(class_id), score(score), points(points)
{
}
~ObjectFloat()
{
}
/**
* Object info to string
* @return Object info string
* @maixpy maix.nn.ObjectFloat.__str__
* @maixcdk maix.nn.ObjectFloat.to_str
*/
std::string to_str()
{
return "x: " + std::to_string(x) + ", y: " + std::to_string(y) + ", w: " + std::to_string(w) + ", h: " + std::to_string(h) + ", class_id: " + std::to_string(class_id) + ", score: " + std::to_string(score);
}
/**
* Object left top coordinate x
* @maixpy maix.nn.ObjectFloat.x
*/
float x;
/**
* Object left top coordinate y
* @maixpy maix.nn.ObjectFloat.y
*/
float y;
/**
* Object width
* @maixpy maix.nn.ObjectFloat.w
*/
float w;
/**
* Object height
* @maixpy maix.nn.ObjectFloat.h
*/
float h;
/**
* Object class id
* @maixpy maix.nn.ObjectFloat.class_id
*/
float class_id;
/**
* Object score
* @maixpy maix.nn.ObjectFloat.score
*/
float score;
/**
* keypoints
* @maixpy maix.nn.ObjectFloat.points
*/
std::vector<float> points;
};
}

478
components/nn/include/maix_nn_retinaface.hpp Normal file
View File

@@ -0,0 +1,478 @@
/**
* @author neucrack@sipeed
* @copyright Sipeed Ltd 2023-
* @license Apache 2.0
* @update 2024.5.15: Create this file.
*/
#pragma once
#include "maix_basic.hpp"
#include "maix_nn.hpp"
#include "maix_image.hpp"
#include "maix_nn_F.hpp"
#include "maix_nn_object.hpp"
#include <tuple>
#include "libmaix_nn_decoder_retinaface.hpp"
namespace maix::nn
{
/**
* Retinaface class
* @maixpy maix.nn.Retinaface
*/
class Retinaface
{
public:
public:
/**
* Constructor of Retinaface class
* @param model model path, default empty, you can load model later by load function.
* @throw If model arg is not empty and load failed, will throw err::Exception.
* @maixpy maix.nn.Retinaface.__init__
* @maixcdk maix.nn.Retinaface.Retinaface
*/
Retinaface(const string &model = "")
{
_model = nullptr;
_priorboxes = nullptr;
if (!model.empty())
{
err::Err e = load(model);
if (e != err::ERR_NONE)
{
throw err::Exception(e, "load model failed");
}
}
}
~Retinaface()
{
if (_model)
{
delete _model;
_model = nullptr;
}
if(_priorboxes)
{
free(_priorboxes);
}
}
/**
* Load model from file
* @param model Model path want to load
* @return err::Err
* @maixpy maix.nn.Retinaface.load
*/
err::Err load(const string &model)
{
if (_model)
{
delete _model;
_model = nullptr;
}
_model = new nn::NN(model);
if (!_model)
{
return err::ERR_NO_MEM;
}
_extra_info = _model->extra_info();
if (_extra_info.find("model_type") != _extra_info.end())
{
if (_extra_info["model_type"] != "retinaface")
{
log::error("model_type not match, expect 'retinaface', but got '%s'", _extra_info["model_type"].c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("model_type key not found");
return err::ERR_ARGS;
}
log::info("model info:\n\ttype: retinaface");
if (_extra_info.find("input_type") != _extra_info.end())
{
std::string input_type = _extra_info["input_type"];
if (input_type == "rgb")
{
_input_img_fmt = maix::image::FMT_RGB888;
log::print("\tinput type: rgb\n");
}
else if (input_type == "bgr")
{
_input_img_fmt = maix::image::FMT_BGR888;
log::print("\tinput type: bgr\n");
}
else
{
log::error("unknown input type: %s", input_type.c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("input_type key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("mean") != _extra_info.end())
{
std::string mean_str = _extra_info["mean"];
std::vector<std::string> mean_strs = split(mean_str, ",");
log::print("\tmean:");
for (auto &it : mean_strs)
{
try
{
this->mean.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("mean value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->mean.back());
}
log::print("\n");
}
else
{
log::error("mean key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("scale") != _extra_info.end())
{
std::string scale_str = _extra_info["scale"];
std::vector<std::string> scale_strs = split(scale_str, ",");
log::print("\tscale:");
for (auto &it : scale_strs)
{
try
{
this->scale.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("scale value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->scale.back());
}
log::print("\n");
}
else
{
log::error("scale key not found");
return err::ERR_ARGS;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
// decoder params
_config.variance[0] = 0.1;
_config.variance[1] = 0.2;
_config.nms = 0.2;
_config.score_thresh = 0.5;
_config.input_w = _input_size.width();
_config.input_h = _input_size.height();
_config.steps[0] = 8;
_config.steps[1] = 16;
_config.steps[2] = 32;
_config.min_sizes[0] = 16;
_config.min_sizes[1] = 32;
_config.min_sizes[2] = 64;
_config.min_sizes[3] = 128;
_config.min_sizes[4] = 256;
_config.min_sizes[5] = 512;
_channel_num = retinaface_get_channel_num(&_config);
_priorboxes = retinaface_get_priorboxes(&_config, &_channel_num);
return err::ERR_NONE;
}
/**
* Detect objects from image
* @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
* @param conf_th Confidence threshold, default 0.4.
* @param iou_th IoU threshold, default 0.45.
* @param fit Resize method, default image.Fit.FIT_CONTAIN.
* @throw If image format not match model input format, will throw err::Exception.
* @return Object list. In C++, you should delete it after use.
* @maixpy maix.nn.Retinaface.detect
*/
std::vector<nn::Object> *detect(image::Image &img, float conf_th = 0.4, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN)
{
this->_conf_th = conf_th;
this->_iou_th = iou_th;
if (img.format() != _input_img_fmt)
{
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
if (!outputs)
{
throw err::Exception("forward image failed");
}
std::vector<nn::Object> *res = _post_process(outputs, img.width(), img.height(), fit);
delete outputs;
return res;
}
/**
* Get model input size
* @return model input size
* @maixpy maix.nn.Retinaface.input_size
*/
image::Size input_size()
{
return _input_size;
}
/**
* Get model input width
* @return model input size of width
* @maixpy maix.nn.Retinaface.input_width
*/
int input_width()
{
return _input_size.width();
}
/**
* Get model input height
* @return model input size of height
* @maixpy maix.nn.Retinaface.input_height
*/
int input_height()
{
return _input_size.height();
}
/**
* Get input image format
* @return input image format, image::Format type.
* @maixpy maix.nn.Retinaface.input_format
*/
image::Format input_format()
{
return _input_img_fmt;
}
public:
/**
* Get mean value, list type
* @maixpy maix.nn.Retinaface.mean
*/
std::vector<float> mean;
/**
* Get scale value, list type
* @maixpy maix.nn.Retinaface.scale
*/
std::vector<float> scale;
private:
image::Size _input_size;
image::Format _input_img_fmt;
nn::NN *_model;
std::map<string, string> _extra_info;
float _conf_th = 0.5;
float _iou_th = 0.45;
libmaix_nn_decoder_retinaface_config_t _config;
nn::ObjectFloat *_priorboxes;
int _channel_num;
private:
std::vector<nn::Object> *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit)
{
std::vector<nn::Object> *objects = new std::vector<nn::Object>(_channel_num);
tensor::Tensor *conf = nullptr;
tensor::Tensor *loc = nullptr;
tensor::Tensor *landms = nullptr;
for(auto i : outputs->tensors)
{
if(i.second->shape()[2] == 2)
conf = i.second;
else if(i.second->shape()[2] == 4)
loc = i.second;
else if(i.second->shape()[2] == 10)
landms = i.second;
}
if(!conf || !loc || !landms)
return nullptr;
float *conf_data = (float *)conf->data();
float *loc_data = (float *)loc->data();
float *landms_data = (float *)landms->data();
int valid_num = _channel_num;
_config.nms = _iou_th;
_config.score_thresh = _conf_th;
retinaface_decode(loc_data, conf_data, landms_data, _priorboxes, objects, &valid_num, true, &_config);
if (valid_num > 0)
{
std::vector<nn::Object> *objects_total = objects;
objects = _nms(*objects, valid_num);
delete objects_total;
}
else
{
delete objects;
return new std::vector<nn::Object>();
}
_correct_bbox(*objects, img_w, img_h, fit);
return objects;
}
std::vector<nn::Object> *_nms(std::vector<nn::Object> &objs, int num)
{
std::vector<nn::Object> *result = new std::vector<nn::Object>();
std::sort(objs.begin(), objs.begin() + num, [](const nn::Object &a, const nn::Object &b)
{ return a.score < b.score; });
for (int i = 0; i < num; ++i)
{
nn::Object &a = objs.at(i);
if (a.score == 0)
continue;
for (int j = i + 1; j < num; ++j)
{
nn::Object &b = objs.at(j);
if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
{
b.score = 0;
}
}
}
for (int i=0; i<num; ++i)
{
nn::Object &a = objs.at(i);
if (a.score != 0)
result->push_back(a);
}
return result;
}
void _correct_bbox(std::vector<nn::Object> &objs, int img_w, int img_h, maix::image::Fit fit)
{
if (img_w == _input_size.width() && img_h == _input_size.height())
return;
if (fit == maix::image::FIT_FILL)
{
float scale_x = (float)img_w / _input_size.width();
float scale_y = (float)img_h / _input_size.height();
for (nn::Object &obj : objs)
{
obj.x *= scale_x;
obj.y *= scale_y;
obj.w *= scale_x;
obj.h *= scale_y;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) *= scale_x;
obj.points.at(i * 2 + 1) *= scale_y;
}
}
}
else if (fit == maix::image::FIT_CONTAIN)
{
float scale_x = ((float)_input_size.width()) / img_w;
float scale_y = ((float)_input_size.height()) / img_h;
float scale = std::min(scale_x, scale_y);
float scale_reverse = 1.0 / scale;
float pad_w = (_input_size.width() - img_w * scale) / 2.0;
float pad_h = (_input_size.height() - img_h * scale) / 2.0;
for (nn::Object &obj : objs)
{
obj.x = (obj.x - pad_w) * scale_reverse;
obj.y = (obj.y - pad_h) * scale_reverse;
obj.w *= scale_reverse;
obj.h *= scale_reverse;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) = (obj.points.at(i * 2) - pad_w) * scale_reverse;
obj.points.at(i * 2 + 1) = (obj.points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
}
}
else if (fit == maix::image::FIT_COVER)
{
float scale_x = ((float)_input_size.width()) / img_w;
float scale_y = ((float)_input_size.height()) / img_h;
float scale = std::max(scale_x, scale_y);
float scale_reverse = 1.0 / scale;
float pad_w = (img_w * scale - _input_size.width()) / 2.0;
float pad_h = (img_h * scale - _input_size.height()) / 2.0;
for (nn::Object &obj : objs)
{
obj.x = (obj.x + pad_w) * scale_reverse;
obj.y = (obj.y + pad_h) * scale_reverse;
obj.w *= scale_reverse;
obj.h *= scale_reverse;
for(size_t i=0; i<obj.points.size() / 2; ++i)
{
obj.points.at(i * 2) = (obj.points.at(i * 2) - pad_w) * scale_reverse;
obj.points.at(i * 2 + 1) = (obj.points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
}
}
else
{
throw err::Exception(err::ERR_ARGS, "fit type not support");
}
}
inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); }
inline static float _calc_iou(Object &a, Object &b)
{
float area1 = a.w * a.h;
float area2 = b.w * b.h;
float wi = std::min((a.x + a.w), (b.x + b.w)) -
std::max(a.x, b.x);
float hi = std::min((a.y + a.h), (b.y + b.h)) -
std::max(a.y, b.y);
float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
return area_i / (area1 + area2 - area_i);
}
template <typename T>
static int _argmax(const T *data, size_t len, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = 1; i < len; i++)
{
int idx = i * stride;
if (data[maxIndex * stride] < data[idx])
{
maxIndex = i;
}
}
return maxIndex;
}
static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
{
items.clear();
size_t pos_start = 0, pos_end, delim_len = delimiter.length();
std::string token;
while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
{
token = s.substr(pos_start, pos_end - pos_start);
pos_start = pos_end + delim_len;
items.push_back(token);
}
items.push_back(s.substr(pos_start));
}
static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
{
std::vector<std::string> tokens;
split0(tokens, s, delimiter);
return tokens;
}
};
} // namespace maix::nn

371
components/nn/src/libmaix_nn_decoder_retinaface.cpp Normal file
View File

@@ -0,0 +1,371 @@
#include <math.h>
#include "libmaix_nn_decoder_retinaface.hpp"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#define debug_line //printf("%s:%d %s %s %s \r\n", __FILE__, __LINE__, __FUNCTION__, __DATE__, __TIME__)
// int *steps = NULL; // config->steps;
// int *min_sizes = NULL; // config->min_sizes;
int min_size_len = MIN_SIZE_LEN;
int anchor_size_len = ANCHOR_SIZE_NUM;
static float overlap(float x1, float w1, float x2, float w2)
{
float l1 = x1 - w1 / 2;
float l2 = x2 - w2 / 2;
float left = l1 > l2 ? l1 : l2;
float r1 = x1 + w1 / 2;
float r2 = x2 + w2 / 2;
float right = r1 < r2 ? r1 : r2;
return right - left;
}
static float box_intersection(nn::Object* a, nn::Object* b)
{
float w = overlap(a->x, a->w, b->x, b->w);
float h = overlap(a->y, a->h, b->y, b->h);
if (w < 0 || h < 0)
return 0;
return w * h;
}
static float box_union(nn::Object* a, nn::Object* b)
{
float i = box_intersection(a, b);
float u = a->w * a->h + b->w * b->h - i;
return u;
}
static float box_iou(nn::Object* a, nn::Object* b)
{
return box_intersection(a, b) / box_union(a, b);
}
typedef struct
{
int index;
int class_id;
std::vector<nn::Object>* faces;
}sortable_box_t;
static int nms_comparator(const void *pa, const void *pb)
{
sortable_box_t* a = (sortable_box_t *)pa;
sortable_box_t* b = (sortable_box_t *)pb;
float diff = a->faces->at(a->index).score - b->faces->at(b->index).score;
// if (diff < 0)
// return 1;
// else if (diff > 0)
// return -1;
// return 0;
return (int)(-*(int32_t*)(&diff));
}
static void do_nms_sort(uint32_t boxes_number, float nms_value, float score_thresh, std::vector<nn::Object>* faces)
{
uint32_t i = 0, j = 0, k = 0;
sortable_box_t s[boxes_number];
for (i = 0; i < boxes_number; ++i)
{
s[i].index = i;
s[i].class_id = 0;
s[i].faces = faces;
}
// for (k = 0; k < classes; ++k) // only one(face) class
{
for (i = 0; i < boxes_number; ++i)
s[i].class_id = k;
qsort(s, boxes_number, sizeof(sortable_box_t), nms_comparator);
for (i = 0; i < boxes_number; ++i)
{
if (faces->at(s[i].index).score < score_thresh)
continue;
nn::Object* a = &faces->at(s[i].index);
for (j = i + 1; j < boxes_number; ++j)
{
nn::Object* b = &faces->at(s[j].index);
if (box_iou(a, b) > nms_value)
faces->at(s[j].index).score = 0;
}
}
}
}
int retinaface_get_channel_num(libmaix_nn_decoder_retinaface_config_t* config)
{
int anchors_size[anchor_size_len * 2];
int anchor_num = 0;
if(anchor_size_len * 2 != min_size_len)
{
int step_of_min_sizes [] = {3,2,2,3};
for(int i = 0 ; i<anchor_size_len; i++)
{
anchors_size[i * 2] = ceil(config->input_h * 1.0 / config->steps[i]);
anchors_size[i * 2 + 1] = ceil(config->input_w * 1.0 / config->steps[i]);
anchor_num += anchors_size[i * 2] * anchors_size[i * 2 + 1] * step_of_min_sizes[i];
}
}
else{
// int step_of_min_sizes [] = {2,2,2,2};
for(int i=0; i < anchor_size_len; ++i)
{
anchor_num += config->input_w / config->steps[i] * (config->input_h / config->steps[i]) * 2;
}
}
debug_line;
return anchor_num;
}
nn::ObjectFloat* retinaface_get_priorboxes(libmaix_nn_decoder_retinaface_config_t* config, int* boxes_num)
{
int anchors_size[anchor_size_len * 2];
int anchor_num = 0;
int count = 0;
if(anchor_size_len * 2 != min_size_len)
{
int step_of_min_sizes [] = {3,2,2,3};
for(int i = 0 ; i<anchor_size_len; i++)
{
anchors_size[i * 2] = ceil(config->input_h * 1.0 / config->steps[i]);
anchors_size[i * 2 + 1] = ceil(config->input_w * 1.0 / config->steps[i]);
anchor_num += anchors_size[i * 2] * anchors_size[i * 2 + 1] * step_of_min_sizes[i];
}
}
else{
for(int i=0; i < anchor_size_len; ++i)
{
anchors_size[i * 2] = ceil(config->input_h * 1.0 / config->steps[i]);
anchors_size[i * 2 + 1] = ceil(config->input_w * 1.0 / config->steps[i]);
anchor_num += anchors_size[i * 2] * anchors_size[i * 2 + 1] * 2;
}
}
*boxes_num = anchor_num;
nn::ObjectFloat* boxes = (nn::ObjectFloat*)malloc(sizeof(nn::ObjectFloat) * anchor_num);
if(!boxes)
{
printf("malloc fail\n");
return NULL;
}
if(anchor_size_len *2 != min_size_len)
{
int start = 0;
int step_of_min_sizes [] = {3,2,2,3};
for (int i=0 ; i < anchor_size_len;i++ )
{
for (int j=0 ; j < anchors_size[i*2];j++)
{
for(int k=0 ; k< anchors_size[i*2+1];k++)
{
int end = start + step_of_min_sizes[i];
for(int l = start; l < end ; l++)
{
int min_size = config->min_sizes[l];
boxes[count].x = (k + 0.5) * config->steps[i] / config->input_w;
boxes[count].y = (j + 0.5) * config->steps[i] / config->input_h;
boxes[count].w = min_size * 1.0 / config->input_w;
boxes[count].h = min_size * 1.0 / config->input_h;
count++;
}
}
}
start += step_of_min_sizes[i];
}
debug_line;
}
else
{
for(int i=0; i < anchor_size_len; ++i)
{
for(int j=0; j < anchors_size[i * 2]; ++j)
{
for(int k=0; k < anchors_size[i * 2 + 1]; ++k)
{
for(int m=0; m < 2; ++m)
{
int min_size = config->min_sizes[i * 2 + m];
boxes[count].x = (k + 0.5) * config->steps[i] / config->input_w;
boxes[count].y = (j + 0.5) * config->steps[i] / config->input_h;
boxes[count].w = min_size * 1.0 / config->input_w;
boxes[count].h = min_size * 1.0 / config->input_h;
++count;
}
}
}
}
}
debug_line;
return boxes;
}
static void softmax(float *data, int stride, int n )
{
int i;
// int diff;
// float e;
float sum = 0;
float largest_i = data[0];
for (i = 0; i < n; ++i)
{
if (data[i + stride] > largest_i)
largest_i = data[i + stride];
}
for (i = 0; i < n; ++i)
{
float value = expf(data[i + stride] - largest_i);
sum += value;
data[i + stride] = value;
}
for (i = 0; i < n; ++i)
{
data[i + stride] /= sum;
}
}
int retinaface_decode(float* net_out_loc, float* net_out_conf, float* net_out_landmark, nn::ObjectFloat* prior_boxes, std::vector<nn::Object> *faces, int* boxes_num, bool chw, libmaix_nn_decoder_retinaface_config_t* config)
{
int valid_boxes_count = 0;
int all_boxes_num = *boxes_num;
int idx = 0;
debug_line;
if(!chw) // hwc: [[[x, x,x,x,....], [y,y,y,y..][w....], [h...]]]
{
debug_line;
/* 1 remove boxes which score < threshhold */
for(int i=0; i < *boxes_num; ++i)
{
/* 1.1 softmax */
// softmax(net_out_conf + i, all_boxes_num, 2);
/* 1.2. decode conf score */
faces->at(i).score = net_out_conf[all_boxes_num + i];
/* 1.3 tag only copy valid faces info*/
if(faces->at(i).score > config->score_thresh)
{
faces->at(valid_boxes_count).score = faces->at(i).score;
faces->at(valid_boxes_count).class_id = i;
++valid_boxes_count;
}
}
*boxes_num = valid_boxes_count;
for(int i=0; i < *boxes_num; ++i)
{
idx = faces->at(i).class_id;
/* 2. decode boxes*/
faces->at(i).x = config->input_w * (prior_boxes[idx].x + net_out_loc[idx] * config->variance[0] * prior_boxes[idx].w);
faces->at(i).y = config->input_h * (prior_boxes[idx].y + net_out_loc[idx + all_boxes_num] * config->variance[0] * prior_boxes[idx].h);
faces->at(i).w = config->input_w * (prior_boxes[idx].w * exp(net_out_loc[idx + all_boxes_num * 2] * config->variance[1]));
faces->at(i).h = config->input_h * (prior_boxes[idx].h * exp(net_out_loc[idx + all_boxes_num * 3] * config->variance[1]));
faces->at(i).x = faces->at(i).x - faces->at(i).w / 2.0;
faces->at(i).y = faces->at(i).y - faces->at(i).h / 2.0;
/* 3. decode landmarks*/
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx + all_boxes_num] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx + all_boxes_num * 2] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx + all_boxes_num * 3] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx + all_boxes_num * 4] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx + all_boxes_num * 5] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx + all_boxes_num * 6] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx + all_boxes_num * 7] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx + all_boxes_num * 8] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx + all_boxes_num * 9] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).class_id = 0;
}
}
else // chw: x,y,w,h......x,y,w,h
{
debug_line;
/* 1 remove boxes which score < threshhold */
// CALC_TIME_START();
for(int i=0; i < *boxes_num; i++)
{
/* 1.1 softmax */
// debug_line("%f, %f ==> ", net_out_conf[i * 2 ], net_out_conf[i * 2 + 1]);
// softmax(net_out_conf + i * 2, 0, 2);
// debug_line("%f, %f\n", net_out_conf[i * 2 ], net_out_conf[i * 2 + 1]);
/* 1.2. decode conf score */
faces->at(i).score = net_out_conf[i * 2 +1 ];
/* 1.3 tag only copy valid faces info*/
if(faces->at(i).score > config->score_thresh)
{
faces->at(valid_boxes_count).score = faces->at(i).score;
faces->at(valid_boxes_count).class_id = i;
++valid_boxes_count;
}
}
*boxes_num = valid_boxes_count;
// debug_line("[libmaix_nn decoder ] valid_boxes_count is %d\n",valid_boxes_count);
// CALC_TIME_END("find valid boxes");
// CALC_TIME_START();
for(int i=0; i < *boxes_num; ++i)
{
idx = faces->at(i).class_id;
/* 2. decode boxes*/
faces->at(i).x = config->input_w * (prior_boxes[idx].x + net_out_loc[idx * 4] * config->variance[0] * prior_boxes[idx].w);
faces->at(i).y = config->input_h * (prior_boxes[idx].y + net_out_loc[idx * 4 + 1] * config->variance[0] * prior_boxes[idx].h);
faces->at(i).w = config->input_w * (prior_boxes[idx].w * exp(net_out_loc[idx * 4 + 2] * config->variance[1]));
faces->at(i).h = config->input_h * (prior_boxes[idx].h * exp(net_out_loc[idx * 4 + 3] * config->variance[1]));
faces->at(i).x = faces->at(i).x - faces->at(i).w / 2.0;
faces->at(i).y = faces->at(i).y - faces->at(i).h / 2.0;
// debug_line("%f %f %f %f, %f %f, %f %f\n", faces->at(i).box.x, faces->at(i).box.y, faces->at(i).box.w, faces->at(i).box.h, prior_boxes[i].w , prior_boxes[i].h, net_out_loc[i * 4 + 2], net_out_loc[i * 4 + 3]);
/* 3. decode landmarks*/
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx * 10] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx * 10 + 1] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx * 10 + 2] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx * 10 + 3] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx * 10 + 4] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx * 10 + 5] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx * 10 + 6] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx * 10 + 7] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).points.push_back(config->input_w * (prior_boxes[idx].x + net_out_landmark[idx * 10 + 8] * config->variance[0] * prior_boxes[idx].w));
faces->at(i).points.push_back(config->input_h * (prior_boxes[idx].y + net_out_landmark[idx * 10 + 9] * config->variance[0] * prior_boxes[idx].h));
faces->at(i).class_id = 0;
}
// CALC_TIME_END("decode valid boxes");
}
/* 4. nms, remove boxes */
// CALC_TIME_START();
debug_line;
do_nms_sort(*boxes_num, config->nms, config->score_thresh, faces);
debug_line;
// CALC_TIME_END("do nms");
return 0;
}

7
components/vision/include/maix_image.hpp
View File

@@ -438,15 +438,14 @@ namespace maix::image
/**
* Draw keypoints on image
* @param keypoints keypoints, [x, y, rotation_andle_in_degrees], TODO: rotation_andle_in_degrees support in the feature
* @param keypoints keypoints, [x1, y1, x2, y2...] or [x, y, rotation_andle_in_degrees, x2, y2, rotation_andle_in_degrees2](TODO: rotation_andle_in_degrees support in the feature)
* @param color keypoints color @see image::Color
* @param size size of keypoints
* @param thickness keypoints thickness(line width), by default(value is 1)
* @param fill if true, will fill keypoints, by default(value is false)
* @param thickness keypoints thickness(line width), by default(value is -1 means fill circle)
* @return this image object self
* @maixpy maix.image.Image.draw_keypoints
*/
image::Image *draw_keypoints(std::vector<int> keypoints, const image::Color &color, int size = 10, int thickness = 1, bool fill = false);
image::Image *draw_keypoints(std::vector<int> keypoints, const image::Color &color, int size = 10, int thickness = -1);
//************************** image operations **************************//

16
components/vision/src/maix_image.cpp
View File

@@ -1019,23 +1019,23 @@ namespace maix::image
return this;
}
image::Image *Image::draw_keypoints(std::vector<int> keypoints, const image::Color &color, int size, int thickness, bool fill)
image::Image *Image::draw_keypoints(std::vector<int> keypoints, const image::Color &color, int size, int thickness)
{
int ch_format = 0;
cv::Scalar cv_color;
_get_cv_format_color(_format, color, &ch_format, cv_color);
cv::Mat img(_height, _width, ch_format, _data);
if (keypoints.size() < 2) {
throw std::runtime_error("keypoints size must >= 2");
if (keypoints.size() < 2 || keypoints.size() % 2 != 0) {
throw std::runtime_error("keypoints size must >= 2 and multiple of 2");
return nullptr;
}
cv::Point center(keypoints[0], keypoints[1]);
int radius = size;
if (fill) {
thickness = -1;
for(size_t i=0; i<keypoints.size() / 2; ++i)
{
cv::Point center(keypoints[i * 2], keypoints[i * 2 + 1]);
int radius = size;
cv::circle(img, center, radius, cv_color, thickness);
}
cv::circle(img, center, radius, cv_color, thickness);
return this;
}

9
examples/nn_face_detector/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

12
examples/nn_face_detector/README.md Normal file
View File

@@ -0,0 +1,12 @@
face detect based on MaixCDK
====
Model download from:
https://maixhub.com/model/zoo/377 (face_detector https://github.com/biubug6/Face-Detector-1MB-with-landmark)
https://maixhub.com/model/zoo/378 (retinafate https://github.com/biubug6/Pytorch_Retinaface)
Build method please visit [MaixCDK](https://github.com/sipeed/MaixCDK).

74
examples/nn_face_detector/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components denpend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_face_detector/main/Kconfig Normal file
View File

3
examples/nn_face_detector/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

123
examples/nn_face_detector/main/src/main.cpp Normal file
View File

@@ -0,0 +1,123 @@
#include "maix_basic.hpp"
#include "maix_vision.hpp"
#include "main.h"
#define USE_RETINAFACE 0
#if USE_RETINAFACE
#include "maix_nn_retinaface.hpp"
#else
#include "maix_nn_face_detector.hpp"
#endif
using namespace maix;
int _main(int argc, char *argv[])
{
log::info("Program start");
std::string model_type = "unknown";
int ret = 0;
err::Err e;
maix::image::Format img_fmt = maix::image::FMT_RGB888;
char tmp_chars[64] = {0};
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
if (argc < 2)
{
log::info(help.c_str());
return -1;
}
const char *model_path = argv[1];
float conf_threshold = 0.4;
float iou_threshold = 0.45;
#if USE_RETINAFACE
nn::Retinaface detector;
#else
nn::FaceDetector detector;
#endif
e = detector.load(model_path);
err::check_raise(e, "load model failed");
log::info("load yolov5 model %s success", model_path);
if (argc >= 3)
{
const char *img_path = argv[2];
log::info("load image now");
maix::image::Image *img = maix::image::load(img_path, img_fmt);
err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
log::info("load image %s success: %s", img_path, img->to_str().c_str());
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
}
log::info("detect now");
uint64_t t = time::time_ms();
std::vector<nn::Object> *result = detector.detect(*img, conf_threshold, iou_threshold);
log::info("time: %lldms", time::time_ms() - t);
if(result->size() == 0)
{
log::info("no object detected !");
}
for (auto &r : *result)
{
log::info("result: %s", r.to_str().c_str());
img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0));
snprintf(tmp_chars, sizeof(tmp_chars), "%.2f", r.score);
img->draw_string(r.x, r.y, tmp_chars, maix::image::Color::from_rgb(255, 0, 0));
int radius = ceil(r.w / 10);
img->draw_keypoints(r.points, image::COLOR_RED, radius > 4 ? 4 : radius);
}
img->save("result.jpg");
delete result;
delete img;
}
else
{
log::info("open camera now");
maix::image::Size input_size = detector.input_size();
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
while (!app::need_exit())
{
uint64_t t = time::time_ms();
maix::image::Image *img = cam.read();
err::check_null_raise(img, "read camera failed");
std::vector<nn::Object> *result = detector.detect(*img);
for (auto &r : *result)
{
img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0));
// snprintf(tmp_chars, sizeof(tmp_chars), "%.2f", r.score);
// img->draw_string(r.x, r.y, tmp_chars, maix::image::Color::from_rgb(255, 0, 0));
int radius = ceil(r.w / 10);
img->draw_keypoints(r.points, image::COLOR_RED, radius > 4 ? 4 : radius);
}
disp.show(*img);
delete result;
delete img;
log::info("time: %d ms", time::time_ms() - t);
}
}
log::info("Program exit");
return ret;
}
int main(int argc, char *argv[])
{
// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
signal(SIGINT, [](int sig)
{ app::set_exit_flag(true); });
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}