Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-11-09 17:04:38 -06:00
Code Issues Packages Projects Releases 1 Wiki Activity

add yolo11 support

Browse Source
This commit is contained in:
Neucrack
2024-10-10 18:35:10 +08:00
parent 330a6db00f
commit e80fd15e30
21 changed files with 1568 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

925
components/nn/include/maix_nn_yolo11.hpp Normal file
View File

@@ -0,0 +1,925 @@
/**
* @author neucrack@sipeed
* @copyright Sipeed Ltd 2024-
* @license Apache 2.0
* @update 2024.10.10: Add yolo11 support.
*/
#pragma once
#include "maix_basic.hpp"
#include "maix_nn.hpp"
#include "maix_image.hpp"
#include "maix_nn_F.hpp"
#include "maix_nn_object.hpp"
namespace maix::nn
{
enum class YOLO11_Type
{
DETECT = 0,
POSE = 1,
SEG = 2
};
class _KpInfoYolo11
{
public:
_KpInfoYolo11(int idx, int ax, int ay, float stride)
: idx(idx), anchor_x(ax), anchor_y(ay), stride(stride)
{
}
int idx;
int anchor_x;
int anchor_y;
float stride;
};
/**
* YOLO11 class
* @maixpy maix.nn.YOLO11
*/
class YOLO11
{
public:
/**
* Constructor of YOLO11 class
* @param model model path, default empty, you can load model later by load function.
* @param[in] dual_buff prepare dual input output buffer to accelarate forward, that is, when NPU is forwarding we not wait and prepare the next input buff.
* If you want to ensure every time forward output the input's result, set this arg to false please.
* Default true to ensure speed.
* @throw If model arg is not empty and load failed, will throw err::Exception.
* @maixpy maix.nn.YOLO11.__init__
* @maixcdk maix.nn.YOLO11.YOLO11
*/
YOLO11(const string &model = "", bool dual_buff = true)
{
_model = nullptr;
_type = YOLO11_Type::DETECT;
_dual_buff = dual_buff;
if (!model.empty())
{
err::Err e = load(model);
if (e != err::ERR_NONE)
{
throw err::Exception(e, "load model failed");
}
}
}
~YOLO11()
{
if (_model)
{
delete _model;
_model = nullptr;
}
}
std::string type()
{
return _extra_info["type"];
}
/**
* Load model from file
* @param model Model path want to load
* @return err::Err
* @maixpy maix.nn.YOLO11.load
*/
err::Err load(const string &model)
{
if (_model)
{
delete _model;
_model = nullptr;
}
_model = new nn::NN(model, _dual_buff);
if (!_model)
{
return err::ERR_NO_MEM;
}
_extra_info = _model->extra_info();
if (_extra_info.find("model_type") != _extra_info.end())
{
if (_extra_info["model_type"] != this->type_str)
{
log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("model_type key not found");
return err::ERR_ARGS;
}
log::info("model info:\n\ttype: %d", this->type_str.c_str());
if (_extra_info.find("input_type") != _extra_info.end())
{
std::string input_type = _extra_info["input_type"];
if (input_type == "rgb")
{
_input_img_fmt = maix::image::FMT_RGB888;
log::print("\tinput type: rgb\n");
}
else if (input_type == "bgr")
{
_input_img_fmt = maix::image::FMT_BGR888;
log::print("\tinput type: bgr\n");
}
else
{
log::error("unknown input type: %s", input_type.c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("input_type key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("mean") != _extra_info.end())
{
std::string mean_str = _extra_info["mean"];
std::vector<std::string> mean_strs = split(mean_str, ",");
log::print("\tmean:");
for (auto &it : mean_strs)
{
try
{
this->mean.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("mean value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->mean.back());
}
log::print("\n");
}
else
{
log::error("mean key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("scale") != _extra_info.end())
{
std::string scale_str = _extra_info["scale"];
std::vector<std::string> scale_strs = split(scale_str, ",");
log::print("\tscale:");
for (auto &it : scale_strs)
{
try
{
this->scale.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("scale value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->scale.back());
}
log::print("\n");
}
else
{
log::error("scale key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("labels") != _extra_info.end())
{
// if "," in labels, will treat as label list, else will treat as label file path
std::string &labels_str = _extra_info["labels"];
if (labels_str.find(",") != std::string::npos)
{
split0(labels, labels_str, ",");
}
else if (labels_str.find(".") != std::string::npos)
{
label_path = fs::dirname(model) + "/" + _extra_info["labels"];
err::Err e = _load_labels_from_file(labels, label_path);
if (e != err::ERR_NONE)
{
log::error("Load labels file %s failed", label_path.c_str());
return e;
}
}
else
{
labels.clear();
labels.push_back(labels_str);
}
log::print("\tlabels num: %ld\n", labels.size());
}
else
{
log::error("labels key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("type") != _extra_info.end())
{
if (_extra_info["type"] == "pose")
{
_type = YOLO11_Type::POSE;
}
else if (_extra_info["type"] == "seg")
{
_type = YOLO11_Type::SEG;
}
else if (_extra_info["type"] != "detector")
{
log::error("type [%s] not support, suport detector and pose", _extra_info["type"].c_str());
return err::ERR_ARGS;
}
}
else
{
_type = YOLO11_Type::DETECT;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
return err::ERR_NONE;
}
/**
* Detect objects from image
* @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
* @param conf_th Confidence threshold, default 0.5.
* @param iou_th IoU threshold, default 0.45.
* @param fit Resize method, default image.Fit.FIT_CONTAIN.
* @param keypoint_th keypoint threshold, default 0.5, only for yolo11-pose model.
* @throw If image format not match model input format, will throw err::Exception.
* @return Object list. In C++, you should delete it after use.
* If model is yolo11-pose, object's points have value, and if points' value < 0 means that point is invalid(conf < keypoint_th).
* @maixpy maix.nn.YOLO11.detect
*/
nn::Objects *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, float keypoint_th = 0.5)
{
this->_conf_th = conf_th;
this->_iou_th = iou_th;
this->_keypoint_th = keypoint_th;
if (img.format() != _input_img_fmt)
{
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
if (!outputs) // not ready, return empty result.
{
return new nn::Objects();
}
nn::Objects *res = _post_process(outputs, img.width(), img.height(), fit);
delete outputs;
if(!res)
{
throw err::Exception("post process failed, please see log before");
}
return res;
}
/**
* Get model input size
* @return model input size
* @maixpy maix.nn.YOLO11.input_size
*/
image::Size input_size()
{
return _input_size;
}
/**
* Get model input width
* @return model input size of width
* @maixpy maix.nn.YOLO11.input_width
*/
int input_width()
{
return _input_size.width();
}
/**
* Get model input height
* @return model input size of height
* @maixpy maix.nn.YOLO11.input_height
*/
int input_height()
{
return _input_size.height();
}
/**
* Get input image format
* @return input image format, image::Format type.
* @maixpy maix.nn.YOLO11.input_format
*/
image::Format input_format()
{
return _input_img_fmt;
}
/**
* Draw pose keypoints on image
* @param img image object, maix.image.Image type.
* @param points keypoits, int list type, [x, y, x, y ...]
* @param radius radius of points.
* @param color color of points.
* @param body true, if points' length is 17*2 and body is ture, will draw lines as human body, if set to false won't draw lines, default true.
* @maixpy maix.nn.YOLO11.draw_pose
*/
void draw_pose(image::Image &img, std::vector<int> points, int radius = 4, image::Color color = image::COLOR_RED, bool body = true)
{
if (points.size() < 2 || points.size() % 2 != 0)
{
throw std::runtime_error("keypoints size must >= 2 and multiple of 2");
return;
}
if (points.size() == 17 * 2 && body)
{
int pos[] = {9, 7, 7, 5, 6, 8, 8, 10, 5, 11, 6, 12, 5, 6, 11, 12, 11, 13, 15, 13, 14, 12, 14, 16};
for (int i = 0; i < 12; ++i)
{
int x1 = points[pos[i * 2] * 2];
int y1 = points[pos[i * 2] * 2 + 1];
int x2 = points[pos[i * 2 + 1] * 2];
int y2 = points[pos[i * 2 + 1] * 2 + 1];
if (x1 < 0 || y1 < 0 || x2 < 0 || y2 < 0)
continue;
img.draw_line(x1, y1, x2, y2, image::COLOR_RED, 2);
}
int x = (points[5 * 2] + points[6 * 2]) / 2;
int y = (points[5 * 2 + 1] + points[6 * 2 + 1]) / 2;
if (!(points[5 * 2] < 0 || points[5 * 2 + 1] < 0 || points[6 * 2] < 0 || points[6 * 2 + 1] < 0 || x < 0 || y < 0 || points[0] < 0 || points[1] < 0))
img.draw_line(points[0], points[1], x, y, image::COLOR_RED, 2);
}
for (size_t i = 0; i < points.size() / 2; ++i)
{
int x = points[i * 2];
int y = points[i * 2 + 1];
if (x < 0 || y < 0)
continue;
img.draw_circle(x, y, radius, color, -1);
}
}
/**
* Draw segmentation on image
* @param img image object, maix.image.Image type.
* @param seg_mask segmentation mask image by detect method, a grayscale image
* @param threshold only mask's value > threshold will be draw on image, value from 0 to 255.
* @maixpy maix.nn.YOLO11.draw_seg_mask
*/
void draw_seg_mask(image::Image &img, int x, int y, image::Image &seg_mask, int threshold = 127)
{
if (seg_mask.format() != image::FMT_GRAYSCALE)
{
throw err::Exception(err::ERR_ARGS, "seg_mask only support grascale");
}
if (img.format() != image::FMT_RGB888 &&
img.format() != image::FMT_BGR888 &&
img.format() != image::FMT_RGBA8888 &&
img.format() != image::FMT_BGRA8888 &&
img.format() != image::FMT_GRAYSCALE)
{
throw err::Exception(err::ERR_ARGS, "img not support");
}
int fmt_size = image::fmt_size[img.format()];
uint8_t *to = (uint8_t*)img.data();
uint8_t *from = (uint8_t*)seg_mask.data();
for (int i = 0; i < seg_mask.height(); ++i)
{
for (int j = 0; j < seg_mask.width(); ++j)
{
uint8_t data = from[i * seg_mask.width() + j];
if(data > threshold)
{
to[((y + i) * img.width() + x + j) * fmt_size] = data;
}
}
}
}
public:
/**
* Labels list
* @maixpy maix.nn.YOLO11.labels
*/
std::vector<string> labels;
/**
* Label file path
* @maixpy maix.nn.YOLO11.label_path
*/
std::string label_path;
/**
* Get mean value, list type
* @maixpy maix.nn.YOLO11.mean
*/
std::vector<float> mean;
/**
* Get scale value, list type
* @maixpy maix.nn.YOLO11.scale
*/
std::vector<float> scale;
protected:
std::string type_str = "yolo11";
private:
image::Size _input_size;
image::Format _input_img_fmt;
nn::NN *_model;
std::map<string, string> _extra_info;
float _conf_th = 0.5;
float _iou_th = 0.45;
float _keypoint_th = 0.5;
YOLO11_Type _type;
bool _dual_buff;
private:
err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
{
// load labels from labels file
labels.clear();
fs::File *f = fs::open(label_path, "r");
if (!f)
{
log::error("open label file %s failed", label_path.c_str());
return err::ERR_ARGS;
}
std::string line;
while (f->readline(line) > 0)
{
// strip line
line.erase(0, line.find_first_not_of(" \t\r\n"));
line.erase(line.find_last_not_of(" \t\r\n") + 1);
labels.push_back(line);
}
f->close();
delete f;
return err::ERR_NONE;
}
nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit)
{
nn::Objects *objects = new nn::Objects();
tensor::Tensor *kp_out = NULL;
tensor::Tensor *mask_out = NULL;
float scale_w = 1;
float scale_h = 1;
if(!_decode_objs(*objects, outputs, _conf_th, _input_size.width(), _input_size.height(), &kp_out, &mask_out))
{
delete objects;
return NULL;
}
if (objects->size() > 0)
{
nn::Objects *objects_total = objects;
objects = _nms(*objects);
delete objects_total;
}
// decode keypoints
if (_type == YOLO11_Type::POSE)
{
_decode_keypoints(*objects, kp_out);
}
else if (_type == YOLO11_Type::SEG)
{
_decode_seg_points(*objects, kp_out, mask_out);
}
if (objects->size() > 0)
{
_correct_bbox(*objects, img_w, img_h, fit, &scale_w, &scale_h);
}
return objects;
}
bool _decode_objs(nn::Objects &objs, tensor::Tensors *outputs, float conf_thresh, int w, int h, tensor::Tensor **kp_out, tensor::Tensor **mask_out)
{
float stride[3] = {8, 16, 32};
tensor::Tensor *score_out = NULL; // shape 1, 80, 8400, 1
tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400
for (auto i : *outputs)
{
if (i.second->shape()[2] == 4)
{
box_out = i.second;
}
else if (strstr(i.first.c_str(), "Sigmoid") != NULL)
{
score_out = i.second;
if((size_t)score_out->shape()[1] != labels.size())
{
log::error("MUD labels(%d) must equal model's(%d)", score_out->shape()[1], labels.size());
return false;
}
}
else if (strstr(i.first.c_str(), "output1") != NULL)
{
*mask_out = i.second;
}
else
{
*kp_out = i.second;
}
}
if (!score_out || !kp_out)
{
throw err::Exception(err::ERR_ARGS, "model output not valid");
}
int total_box_num = box_out->shape()[3];
// int class_num = this->labels.size();
int class_num = score_out->shape()[1];
float *scores_ptr = (float *)score_out->data();
float *dets_ptr = (float *)box_out->data();
int idx_start[3] = {
0,
(int)(h / stride[0] * w / stride[0]),
(int)(h / stride[0] * w / stride[0] + h / stride[1] * w / stride[1])};
for (int i = 0; i < 3; i++)
{
int nh = h / stride[i];
int nw = w / stride[i];
for (int ay = 0; ay < nh; ++ay)
{
for (int ax = 0; ax < nw; ++ax)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, total_box_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num);
// int class_id = (offset + max_idx) / total_box_num;
float obj_score = scores_ptr[offset + class_id * total_box_num];
if (obj_score <= conf_thresh)
{
continue;
}
float bbox_x = (ax + 0.5 - dets_ptr[offset]) * stride[i];
float bbox_y = (ay + 0.5 - dets_ptr[offset + total_box_num]) * stride[i];
float bbox_w = (ax + 0.5 + dets_ptr[offset + total_box_num * 2]) * stride[i] - bbox_x;
float bbox_h = (ay + 0.5 + dets_ptr[offset + total_box_num * 3]) * stride[i] - bbox_y;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score);
obj.temp = (void *)kp_info;
}
}
}
return true;
}
nn::Objects *_nms(nn::Objects &objs)
{
nn::Objects *result = new nn::Objects();
std::sort(objs.begin(), objs.end(), [](const nn::Object *a, const nn::Object *b)
{ return a->score > b->score; });
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &a = objs.at(i);
if (a.score == 0)
continue;
for (size_t j = i + 1; j < objs.size(); ++j)
{
nn::Object &b = objs.at(j);
{
if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
{
b.score = 0;
}
}
}
}
for (nn::Object *a : objs)
{
if (a->score != 0)
{
Object &obj = result->add(a->x, a->y, a->w, a->h, a->class_id, a->score, a->points);
if (obj.x < 0)
{
obj.w += obj.x;
obj.x = 0;
}
if (obj.y < 0)
{
obj.h += obj.y;
obj.y = 0;
}
if (obj.x + obj.w > _input_size.width())
{
obj.w = _input_size.width() - obj.x;
}
if (obj.y + obj.h > _input_size.height())
{
obj.h = _input_size.height() - obj.y;
}
obj.temp = a->temp;
}
else
{
delete (_KpInfoYolo11 *)a->temp;
a->temp = NULL;
}
}
return result;
}
void _decode_keypoints(nn::Objects &objs, tensor::Tensor *kp_out)
{
float *data = (float *)kp_out->data();
int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 51, 8400, 1
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < keypoint_num; ++k)
{
float score = _sigmoid(p[(k * 3 + 2) * total_box_num]);
int x = -1;
int y = -1;
if (score > _keypoint_th)
{
x = (p[(k * 3) * total_box_num] * 2.0 + kp_info->anchor_x) * kp_info->stride;
y = (p[(k * 3 + 1) * total_box_num] * 2.0 + kp_info->anchor_y) * kp_info->stride;
}
o.points.push_back(x);
o.points.push_back(y);
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
void _decode_seg_points(nn::Objects &objs, tensor::Tensor *kp_out, tensor::Tensor *mask_out)
{
float *data = (float *)kp_out->data();
float *mask_data = (float *)mask_out->data(); // 1, 32, 160, 160
int mask_h = mask_out->shape()[2];
int mask_w = mask_out->shape()[3];
int mask_squre = mask_h * mask_w;
int mask_num = kp_out->shape()[1]; // 1, 32, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 32, 8400, 1
float mask_weights[mask_num];
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
int mask_x = o.x * mask_w / _input_size.width();
int mask_y = o.y * mask_h / _input_size.height();
int mask_x2 = (o.x + o.w) * mask_w / _input_size.width();
int mask_y2 = (o.y + o.h) * mask_h / _input_size.height();
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < mask_num; ++k)
{
mask_weights[k] = p[k * total_box_num];
}
o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE);
uint8_t *p_img_data = (uint8_t *)o.seg_mask->data();
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] *= mask_weights[0];
}
}
for (int n = 1; n < mask_num; ++n)
{
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255);
}
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
void _correct_bbox(nn::Objects &objs, int img_w, int img_h, maix::image::Fit fit, float *scale_w, float *scale_h)
{
#define CORRECT_BBOX_RANGE_YOLO11(obj) \
do \
{ \
if (obj->x < 0) \
{ \
obj->w += obj->x; \
obj->x = 0; \
} \
if (obj->y < 0) \
{ \
obj->h += obj->y; \
obj->y = 0; \
} \
if (obj->x + obj->w > img_w) \
{ \
obj->w = img_w - obj->x; \
} \
if (obj->y + obj->h > img_h) \
{ \
obj->h = img_h - obj->y; \
} \
} while (0)
for (nn::Object *obj : objs)
{
if (obj->temp)
{
delete (_KpInfoYolo11 *)obj->temp;
obj->temp = NULL;
}
}
if (img_w == _input_size.width() && img_h == _input_size.height())
{
if (_type == YOLO11_Type::SEG)
{
for (nn::Object *obj : objs)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
return;
}
if (fit == maix::image::FIT_FILL)
{
*scale_w = (float)img_w / _input_size.width();
*scale_h = (float)img_h / _input_size.height();
for (nn::Object *obj : objs)
{
obj->x *= *scale_w;
obj->y *= *scale_h;
obj->w *= *scale_w;
obj->h *= *scale_h;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) *= *scale_w;
obj->points.at(i * 2 + 1) *= *scale_h;
}
CORRECT_BBOX_RANGE_YOLO11(obj);
if (_type == YOLO11_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else if (fit == maix::image::FIT_CONTAIN)
{
*scale_w = ((float)_input_size.width()) / img_w;
*scale_h = ((float)_input_size.height()) / img_h;
float scale = std::min(*scale_w, *scale_h);
float scale_reverse = 1.0 / scale;
*scale_w = scale_reverse;
*scale_h = scale_reverse;
float pad_w = (_input_size.width() - img_w * scale) / 2.0;
float pad_h = (_input_size.height() - img_h * scale) / 2.0;
for (nn::Object *obj : objs)
{
obj->x = (obj->x - pad_w) * scale_reverse;
obj->y = (obj->y - pad_h) * scale_reverse;
obj->w *= scale_reverse;
obj->h *= scale_reverse;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse;
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLO11(obj);
if (_type == YOLO11_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else if (fit == maix::image::FIT_COVER)
{
*scale_w = ((float)_input_size.width()) / img_w;
*scale_h = ((float)_input_size.height()) / img_h;
float scale = std::max(*scale_w, *scale_h);
float scale_reverse = 1.0 / scale;
*scale_w = scale_reverse;
*scale_h = scale_reverse;
float pad_w = (img_w * scale - _input_size.width()) / 2.0;
float pad_h = (img_h * scale - _input_size.height()) / 2.0;
for (nn::Object *obj : objs)
{
obj->x = (obj->x + pad_w) * scale_reverse;
obj->y = (obj->y + pad_h) * scale_reverse;
obj->w *= scale_reverse;
obj->h *= scale_reverse;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse;
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLO11(obj);
if (_type == YOLO11_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else
{
throw err::Exception(err::ERR_ARGS, "fit type not support");
}
}
inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); }
inline static float _calc_iou(Object &a, Object &b)
{
float area1 = a.w * a.h;
float area2 = b.w * b.h;
float wi = std::min((a.x + a.w), (b.x + b.w)) -
std::max(a.x, b.x);
float hi = std::min((a.y + a.h), (b.y + b.h)) -
std::max(a.y, b.y);
float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
return area_i / (area1 + area2 - area_i);
}
template <typename T>
static int _argmax(const T *data, size_t len, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = 1; i < len; i++)
{
int idx = i * stride;
if (data[maxIndex * stride] < data[idx])
{
maxIndex = i;
}
}
return maxIndex;
}
template <typename T>
static int _argmax2(const T *data, size_t end, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = stride; i < end; i += stride)
{
if (data[maxIndex] < data[i])
{
maxIndex = i;
}
}
return maxIndex;
}
static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
{
items.clear();
size_t pos_start = 0, pos_end, delim_len = delimiter.length();
std::string token;
while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
{
token = s.substr(pos_start, pos_end - pos_start);
pos_start = pos_end + delim_len;
items.push_back(token);
}
items.push_back(s.substr(pos_start));
}
static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
{
std::vector<std::string> tokens;
split0(tokens, s, delimiter);
return tokens;
}
};
} // namespace maix::nn

37
components/nn/include/maix_nn_yolov8.hpp
View File

@@ -14,11 +14,11 @@
namespace maix::nn
{
enum YOLOv8_Type
enum class YOLOv8_Type
{
TYPE_DETECT = 0,
TYPE_POSE = 1,
TYPE_SEG = 2
DETECT = 0,
POSE = 1,
SEG = 2
};
class _KpInfo
@@ -54,7 +54,7 @@ namespace maix::nn
YOLOv8(const string &model = "", bool dual_buff = true)
{
_model = nullptr;
_type = TYPE_DETECT;
_type = YOLOv8_Type::DETECT;
_dual_buff = dual_buff;
if (!model.empty())
{
@@ -101,9 +101,9 @@ namespace maix::nn
_extra_info = _model->extra_info();
if (_extra_info.find("model_type") != _extra_info.end())
{
if (_extra_info["model_type"] != "yolov8")
if (_extra_info["model_type"] != this->type_str)
{
log::error("model_type not match, expect 'yolov8', but got '%s'", _extra_info["model_type"].c_str());
log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str());
return err::ERR_ARGS;
}
}
@@ -112,7 +112,7 @@ namespace maix::nn
log::error("model_type key not found");
return err::ERR_ARGS;
}
log::info("model info:\n\ttype: yolov8");
log::info("model info:\n\ttype: %d", this->type_str.c_str());
if (_extra_info.find("input_type") != _extra_info.end())
{
std::string input_type = _extra_info["input_type"];
@@ -221,11 +221,11 @@ namespace maix::nn
{
if (_extra_info["type"] == "pose")
{
_type = TYPE_POSE;
_type = YOLOv8_Type::POSE;
}
else if (_extra_info["type"] == "seg")
{
_type = TYPE_SEG;
_type = YOLOv8_Type::SEG;
}
else if (_extra_info["type"] != "detector")
{
@@ -235,7 +235,7 @@ namespace maix::nn
}
else
{
_type = TYPE_DETECT;
_type = YOLOv8_Type::DETECT;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
@@ -425,6 +425,9 @@ namespace maix::nn
*/
std::vector<float> scale;
protected:
std::string type_str = "yolov8";
private:
image::Size _input_size;
image::Format _input_img_fmt;
@@ -480,11 +483,11 @@ namespace maix::nn
delete objects_total;
}
// decode keypoints
if (_type == TYPE_POSE)
if (_type == YOLOv8_Type::POSE)
{
_decode_keypoints(*objects, kp_out);
}
else if (_type == TYPE_SEG)
else if (_type == YOLOv8_Type::SEG)
{
_decode_seg_points(*objects, kp_out, mask_out);
}
@@ -739,7 +742,7 @@ namespace maix::nn
}
if (img_w == _input_size.width() && img_h == _input_size.height())
{
if (_type == TYPE_SEG)
if (_type == YOLOv8_Type::SEG)
{
for (nn::Object *obj : objs)
{
@@ -769,7 +772,7 @@ namespace maix::nn
obj->points.at(i * 2 + 1) *= *scale_h;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == TYPE_SEG)
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
@@ -802,7 +805,7 @@ namespace maix::nn
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == TYPE_SEG)
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
@@ -835,7 +838,7 @@ namespace maix::nn
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == TYPE_SEG)
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{

9
examples/nn_yolo11_detect/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

9
examples/nn_yolo11_detect/README.md Normal file
View File

@@ -0,0 +1,9 @@
nn_yolov8 examples based on MaixCDK
====
## only run yolov8 model
- Use `maixcdk build` to compile binary files.
- Move files and runtime libraries to device.
- Use SSH command with parameters to run programs.
` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

74
examples/nn_yolo11_detect/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_yolo11_detect/main/Kconfig Normal file
View File

3
examples/nn_yolo11_detect/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

111
examples/nn_yolo11_detect/main/src/main.cpp Normal file
View File

@@ -0,0 +1,111 @@
#include "maix_basic.hpp"
#include "maix_vision.hpp"
#include "maix_nn_yolo11.hpp"
#include "main.h"
using namespace maix;
int _main(int argc, char *argv[])
{
log::info("Program start");
std::string model_type = "unknown";
int ret = 0;
err::Err e;
maix::image::Format img_fmt = maix::image::FMT_RGB888;
std::vector<float> mean = {};
std::vector<float> scale = {};
char tmp_chars[64] = {0};
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
if (argc < 2)
{
log::info(help.c_str());
return -1;
}
const char *model_path = argv[1];
float conf_threshold = 0.5;
float iou_threshold = 0.45;
bool dual_buff = true;
nn::YOLO11 detector("", dual_buff);
e = detector.load(model_path);
err::check_raise(e, "load model failed");
log::info("load yolo11 model %s success", model_path);
if (argc >= 3)
{
const char *img_path = argv[2];
log::info("load image now");
maix::image::Image *img = maix::image::load(img_path, img_fmt);
err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
log::info("load image %s success: %s", img_path, img->to_str().c_str());
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
}
log::info("detect now");
nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold);
if(result->size() == 0)
{
log::info("no object detected !");
}
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", detector.labels[r->class_id].c_str(), r->score);
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
}
img->save("result.jpg");
delete result;
delete img;
}
else
{
log::info("open camera now");
maix::image::Size input_size = detector.input_size();
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
while (!app::need_exit())
{
uint64_t t = time::ticks_ms();
maix::image::Image *img = cam.read();
err::check_null_raise(img, "read camera failed");
uint64_t t2 = time::ticks_ms();
nn::Objects *result = detector.detect(*img);
uint64_t t3 = time::ticks_ms();
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
}
disp.show(*img);
delete result;
delete img;
log::info("time: all %d ms, detect %d ms, fps: %.1f", time::ticks_ms() - t, t3 - t2, time::fps());
}
}
log::info("Program exit");
return ret;
}
int main(int argc, char *argv[])
{
// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
signal(SIGINT, [](int sig)
{ app::set_exit_flag(true); });
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}

9
examples/nn_yolo11_pose/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

9
examples/nn_yolo11_pose/README.md Normal file
View File

@@ -0,0 +1,9 @@
nn_yolov8_pose examples based on MaixCDK
====
## only run yolov8_pose model
- Use `maixcdk build` to compile binary files.
- Move files and runtime libraries to device.
- Use SSH command with parameters to run programs.
` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

74
examples/nn_yolo11_pose/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_yolo11_pose/main/Kconfig Normal file
View File

3
examples/nn_yolo11_pose/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

113
examples/nn_yolo11_pose/main/src/main.cpp Normal file
View File

@@ -0,0 +1,113 @@
#include "maix_basic.hpp"
#include "maix_vision.hpp"
#include "maix_nn_yolo11.hpp"
#include "main.h"
using namespace maix;
int _main(int argc, char *argv[])
{
log::info("Program start");
std::string model_type = "unknown";
int ret = 0;
err::Err e;
maix::image::Format img_fmt = maix::image::FMT_RGB888;
std::vector<float> mean = {};
std::vector<float> scale = {};
char tmp_chars[64] = {0};
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
if (argc < 2)
{
log::info(help.c_str());
return -1;
}
const char *model_path = argv[1];
float conf_threshold = 0.5;
float iou_threshold = 0.45;
bool dual_buff = true;
nn::YOLO11 detector("", dual_buff);
e = detector.load(model_path);
err::check_raise(e, "load model failed");
log::info("load yolo11 model %s success", model_path);
if (argc >= 3)
{
const char *img_path = argv[2];
log::info("load image now");
maix::image::Image *img = maix::image::load(img_path, img_fmt);
err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
log::info("load image %s success: %s", img_path, img->to_str().c_str());
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
}
log::info("detect now");
nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold);
if(result->size() == 0)
{
log::info("no object detected !");
}
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", detector.labels[r->class_id].c_str(), r->score);
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
detector.draw_pose(*img, r->points, 4, image::COLOR_RED);
}
img->save("result.jpg");
delete result;
delete img;
}
else
{
log::info("open camera now");
maix::image::Size input_size = detector.input_size();
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
while (!app::need_exit())
{
uint64_t t = time::ticks_ms();
maix::image::Image *img = cam.read();
err::check_null_raise(img, "read camera failed");
uint64_t t2 = time::ticks_ms();
nn::Objects *result = detector.detect(*img);
uint64_t t3 = time::ticks_ms();
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
detector.draw_pose(*img, r->points, 4, image::COLOR_RED);
}
disp.show(*img);
delete result;
delete img;
log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2);
}
}
log::info("Program exit");
return ret;
}
int main(int argc, char *argv[])
{
// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
signal(SIGINT, [](int sig)
{ app::set_exit_flag(true); });
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}

9
examples/nn_yolo11_seg/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

9
examples/nn_yolo11_seg/README.md Normal file
View File

@@ -0,0 +1,9 @@
nn_yolov8_seg examples based on MaixCDK
====
## only run yolov8_seg model
- Use `maixcdk build` to compile binary files.
- Move files and runtime libraries to device.
- Use SSH command with parameters to run programs.
` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

74
examples/nn_yolo11_seg/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_yolo11_seg/main/Kconfig Normal file
View File

3
examples/nn_yolo11_seg/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

113
examples/nn_yolo11_seg/main/src/main.cpp Normal file
View File

@@ -0,0 +1,113 @@
#include "maix_basic.hpp"
#include "maix_vision.hpp"
#include "maix_nn_yolo11.hpp"
#include "main.h"
using namespace maix;
int _main(int argc, char *argv[])
{
log::info("Program start");
std::string model_type = "unknown";
int ret = 0;
err::Err e;
maix::image::Format img_fmt = maix::image::FMT_RGB888;
std::vector<float> mean = {};
std::vector<float> scale = {};
char tmp_chars[64] = {0};
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
if (argc < 2)
{
log::info(help.c_str());
return -1;
}
const char *model_path = argv[1];
float conf_threshold = 0.5;
float iou_threshold = 0.45;
bool dual_buff = true;
nn::YOLO11 detector("", dual_buff);
e = detector.load(model_path);
err::check_raise(e, "load model failed");
log::info("load yolo11 model %s success", model_path);
if (argc >= 3)
{
const char *img_path = argv[2];
log::info("load image now");
maix::image::Image *img = maix::image::load(img_path, img_fmt);
err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
log::info("load image %s success: %s", img_path, img->to_str().c_str());
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
}
log::info("detect now");
nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold);
if(result->size() == 0)
{
log::info("no object detected !");
}
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", detector.labels[r->class_id].c_str(), r->score);
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
detector.draw_seg_mask(*img, r->x, r->y, *r->seg_mask);
}
img->save("result.jpg");
delete result;
delete img;
}
else
{
log::info("open camera now");
maix::image::Size input_size = detector.input_size();
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
while (!app::need_exit())
{
uint64_t t = time::ticks_ms();
maix::image::Image *img = cam.read();
err::check_null_raise(img, "read camera failed");
uint64_t t2 = time::ticks_ms();
nn::Objects *result = detector.detect(*img);
uint64_t t3 = time::ticks_ms();
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
detector.draw_seg_mask(*img, r->x, r->y, *r->seg_mask);
}
disp.show(*img);
delete result;
delete img;
log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2);
}
}
log::info("Program exit");
return ret;
}
int main(int argc, char *argv[])
{
// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
signal(SIGINT, [](int sig)
{ app::set_exit_flag(true); });
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}

2
examples/nn_yolov8/main/src/main.cpp
View File

@@ -89,7 +89,7 @@ int _main(int argc, char *argv[])
disp.show(*img);
delete result;
delete img;
log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2);
log::info("time: all %d ms, detect %d ms, fps: %.1f", time::ticks_ms() - t, t3 - t2, time::fps());
}
}