Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-11-09 17:04:38 -06:00
Code Issues Packages Projects Releases 1 Wiki Activity

maixcam2 support nn and yolo11 yolov8 yolov5 classifier

Browse Source
This commit is contained in:
Neucrack
2025-04-03 17:17:46 +08:00
parent 02308ae79f
commit b0eb756227
21 changed files with 1169 additions and 1010 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
components/basic/include/maix_tensor.hpp
View File

@@ -519,6 +519,7 @@ namespace maix
tensors[key] = tensor;
_auto_delete[key] = auto_delete;
}
_keys.push_back(key);
}
/**
@@ -527,6 +528,22 @@ namespace maix
*/
void rm_tensor(const std::string &key)
{
size_t i = 0;
for(; i < _keys.size(); ++i)
{
auto &_key = _keys[i];
if(_key == key)
{
break;
}
}
if(i == _keys.size())
{
log::warn("rm_tensor: key %s not in tensor", key.c_str());
return;
}
_keys.erase(_keys.begin() + i);
if(_auto_delete[key])
{
delete tensors[key];
@@ -540,11 +557,11 @@ namespace maix
*/
void clear()
{
auto _keys = keys();
for(const auto &k : _keys)
{
rm_tensor(k);
}
_keys.clear();
}
/**
@@ -594,6 +611,25 @@ namespace maix
return *tensors[key];
}
/**
* Operator []
* @maixcdk maix.tensor.Tensors.operator[]
*/
tensor::Tensor &operator[](int idx)
{
int i = 0;
for(auto &key : _keys)
{
if(i == idx)
{
return *tensors[key];
}
++i;
}
log::error("%s: %d", "get tensor idx error", idx);
throw err::Exception(err::ERR_ARGS, "Tensors get tensor idx error");
}
/**
* Size
* @maixpy maix.tensor.Tensors.__len__
@@ -610,12 +646,7 @@ namespace maix
*/
std::vector<std::string> keys()
{
std::vector<std::string> names;
for(auto &item : tensors)
{
names.push_back(item.first);
}
return names;
return _keys;
}
public:
@@ -626,6 +657,7 @@ namespace maix
std::map<std::string, tensor::Tensor*> tensors;
private:
std::map<std::string, bool> _auto_delete;
std::vector<std::string> _keys;
};

6
components/maixcam_lib/CMakeLists.txt
View File

@@ -36,11 +36,11 @@ elseif(PLATFORM_MAIXCAM2)
list(APPEND ADD_INCLUDE "../vision/include")
list(APPEND ADD_INCLUDE "../nn/include")
set(source_dir "maixcdk_maixcam_lib")
append_srcs_dir(ADD_SRCS "${source_dir}/maixcam2")
append_srcs_dir(ADD_SRCS "${source_dir}/common")
if(CONFIG_MAIXCAM_LIB_COMPILE_FROM_SOURCE)
set(source_dir "maixcdk_maixcam_lib")
append_srcs_dir(ADD_SRCS "${source_dir}/maixcam2")
append_srcs_dir(ADD_SRCS "${source_dir}/common")
register_component(DYNAMIC)
else()
list(APPEND ADD_DYNAMIC_LIB "lib/libmaixcam2_lib.so")

1
components/nn/CMakeLists.txt
View File

@@ -11,6 +11,7 @@ if(PLATFORM_MAIXCAM)
list(APPEND ADD_DEFINITIONS -DDR_WAV_IMPLEMENTATION)
else()
append_srcs_dir(ADD_SRCS "port/linux")
list(APPEND ADD_DEFINITIONS -fopenmp)
endif()
register_component()

3
components/nn/include/maix_nn.hpp
View File

@@ -375,7 +375,8 @@ namespace maix::nn
* @param copy_result If set true, will copy result to a new variable; else will use a internal memory, you can only use it until to the next forward.
* Default true to avoid problems, you can set it to false manually to make speed faster.
* @param dual_buff_wait bool type, only for dual_buff mode, if true, will inference this image and wait for result, default false.
* @param chw chw channel format, forward model with hwc format image input if set to false, default true(chw).
* @param chw !!depracated!! This arg will be ignored!!! Please set extra.input_layout in mud file instead.
* chw channel format, forward model with hwc format image input if set to false, default true(chw).
* @return output tensor. In C++, you should manually delete tensors in return value and return value.
* If dual_buff mode, it can be NULL(None in MaixPy) means not ready.
* @throw If error occurs, like arg error or alloc memory failed, will raise err.Exception.

16
components/nn/include/maix_nn_classifier.hpp
View File

@@ -34,7 +34,6 @@ namespace maix::nn
{
_model = nullptr;
_dual_buff = dual_buff;
_chw = true;
if (!model.empty())
{
err::Err e = load(model);
@@ -163,12 +162,6 @@ namespace maix::nn
log::error("scale key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("input_channel") != _extra_info.end())
{
std::string channel_str = _extra_info["input_channel"];
if(channel_str == "hwc")
_chw = false;
}
err::Err e = _model->extra_info_labels(labels);
if(e == err::Err::ERR_NONE)
{
@@ -180,10 +173,10 @@ namespace maix::nn
return err::ERR_ARGS;
}
_inputs = _model->inputs_info();
if(_chw)
_input_size = image::Size(_inputs[0].shape[3], _inputs[0].shape[2]);
if(inputs[0].shape[3] <= 4) // nhwc
_input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]);
else
_input_size = image::Size(_inputs[0].shape[2], _inputs[0].shape[1]);
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
return err::ERR_NONE;
}
@@ -226,7 +219,7 @@ namespace maix::nn
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
outputs = _model->forward_image(img, this->mean, this->scale, fit, false, false, _chw);
outputs = _model->forward_image(img, this->mean, this->scale, fit, false, false);
if (!outputs)
{
std::vector<std::pair<int, float>> *res = new std::vector<std::pair<int, float>>(1);
@@ -348,7 +341,6 @@ namespace maix::nn
image::Format _input_img_fmt;
bool _input_is_img;
bool _dual_buff;
bool _chw;
nn::NN *_model;
std::map<string, string> _extra_info;
image::Size _input_size;

752
components/nn/include/maix_nn_yolo11.hpp
View File

@@ -12,6 +12,7 @@
#include "maix_nn_F.hpp"
#include "maix_nn_object.hpp"
#include <math.h>
#include <omp.h>
namespace maix::nn
{
@@ -36,6 +37,30 @@ namespace maix::nn
float stride;
};
class _OutIdxes
{
public:
// mode 1
int det0; // 1x144x80x80
int det1; // 1x144x40x40
int det2; // 1x144x20x20
// mode 2
int dfl; // 1x1x4x8400
int sigmoid; // 1x80x8400
// optional node
// seg mask 1x32x160x160
// seg mask weight 1x32x8400
int seg_mask;
int seg_mask_weight;
// obb 1x1x8400
int obb_angle;
// pose 1x51x8400
int pose;
};
/**
* YOLO11 class
* @maixpy maix.nn.YOLO11
@@ -215,7 +240,7 @@ namespace maix::nn
}
else if (_extra_info["type"] != "detector")
{
log::error("type [%s] not support, suport detector and pose", _extra_info["type"].c_str());
log::error("type [%s] not support, suport [detector, pose, seg, obb]", _extra_info["type"].c_str());
return err::ERR_ARGS;
}
}
@@ -224,8 +249,167 @@ namespace maix::nn
_type = YOLO11_Type::DETECT;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
if(inputs[0].shape[3] <= 4) // nhwc
_input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]);
else
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
// output info
_out_chw = true;
int _anchor_num = 0;
for(size_t i=0; i < _stride.size(); ++i)
{
_anchor_num += _input_size.width()/_stride[i] * _input_size.height() / _stride[i];
}
std::vector<nn::LayerInfo> outputs = _model->outputs_info();
auto print_outputs = [outputs](){
log::info("Outputs:");
for(auto item : outputs)
{
log::print(" %s\n", item.to_str().c_str());
}
};
// mode 1
if((outputs.size() == 5 && _type == YOLO11_Type::SEG) ||
(outputs.size() == 4 && (_type == YOLO11_Type::OBB || _type == YOLO11_Type::POSE)) ||
(outputs.size() == 3 && _type == YOLO11_Type::DETECT))
{
_out_node_mode = 1;
int ch_num = labels.size() + _reg_max * 4;
for(size_t i = 0; i < outputs.size(); ++i)
{
nn::LayerInfo &item = outputs[i];
if(item.shape.size() == 4)
{
if(item.shape[3] == ch_num && item.shape[1] == _input_size.height() / _stride[0]
&& item.shape[2] == _input_size.width() / _stride[0]) // hwc
{
_out_chw = false;
_out_idxes.det0 = i;
break;
}
}
}
for(size_t i = 0; i < outputs.size(); ++i)
{
nn::LayerInfo &item = outputs[i];
if(item.shape.size() == 4)
{
if(_type == YOLO11_Type::SEG &&
((item.shape[1] == 32 && // 1 32 160 160
item.shape[2] == _input_size.height() / 4 &&
item.shape[3] == _input_size.width() / 4) || // 1 160 160 32
(item.shape[3] == 32 &&
item.shape[1] == _input_size.height() / 4 &&
item.shape[2] == _input_size.width() / 4)
))
{
_out_idxes.seg_mask = i;
}
else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[0])
{
_out_idxes.det0 = i;
}
else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[1])
{
_out_idxes.det1 = i;
}
else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[2])
{
_out_idxes.det2 = i;
}
else
{
log::error("output node shape error, please check %d, chw: %d, %d", 1, _out_chw, item.shape[_out_chw ? 2 : 1]);
print_outputs();
return err::ERR_ARGS;
}
}
else if(item.shape.size() == 3)
{
if (_type == YOLO11_Type::SEG)
_out_idxes.seg_mask_weight = i;
else if (_type == YOLO11_Type::OBB)
{
if(item.name.find("Sigmoid") == std::string::npos)
_obb_need_sigmoid = true;
else
_obb_need_sigmoid = false;
_out_idxes.obb_angle = i;
}
else if (_type == YOLO11_Type::POSE)
_out_idxes.pose = i;
else
{
log::error("output node shape error, please check %d", 2);
print_outputs();
return err::ERR_ARGS;
}
}
else
{
log::error("output node shape error, please check %d", 3);
print_outputs();
return err::ERR_ARGS;
}
}
}
else // mode 2
{
_out_node_mode = 2;
for(int i = 0; i < (int)outputs.size(); ++i)
{
nn::LayerInfo &item = outputs[i];
if(item.shape.size() == 4)
{
if(item.shape[3] == 4 && item.shape[2] == _anchor_num && item.shape[1] == 1) // hwc
{
_out_chw = false;
_out_idxes.dfl = i;
break;
}
}
}
for(int i = 0; i < (int)outputs.size(); ++i)
{
nn::LayerInfo &item = outputs[i];
if(i == _out_idxes.dfl)
continue;
if(item.shape.size() == 4)
{
_out_idxes.seg_mask = i;
continue;
}
if(_type == YOLO11_Type::DETECT && item.shape[_out_chw ? 1 : 2] == (int)labels.size() && item.shape[_out_chw ? 2 : 1] == _anchor_num)
{
_out_idxes.sigmoid = i;
}
else if(_type == YOLO11_Type::SEG && item.shape[_out_chw ? 1 : 2] == 32)
{
_out_idxes.seg_mask_weight = i;
}
else if(_type == YOLO11_Type::OBB)
{
if(item.name.find("Sigmoid") == std::string::npos)
_obb_need_sigmoid = true;
else
_obb_need_sigmoid = false;
_out_idxes.obb_angle = i;
}
else if(_type == YOLO11_Type::POSE && (item.shape[_out_chw ? 1 : 2] % 3 == 0)&& item.shape[_out_chw ? 2 : 1] == _anchor_num)
{
_out_idxes.pose = i;
}
else
{
log::error("output node shape error, please check %d", 4);
print_outputs();
return err::ERR_ARGS;
}
}
}
return err::ERR_NONE;
}
@@ -244,6 +428,7 @@ namespace maix::nn
*/
nn::Objects *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, float keypoint_th = 0.5, int sort = 0)
{
#define SHOW_DETECT_TIME 1
this->_conf_th = conf_th;
this->_iou_th = iou_th;
this->_keypoint_th = keypoint_th;
@@ -252,12 +437,22 @@ namespace maix::nn
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
#if SHOW_DETECT_TIME
uint64_t start = time::ticks_ms();
#endif
outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
#if SHOW_DETECT_TIME
log::info("forward time: %ld", time::ticks_ms() - start);
start = time::ticks_ms();
#endif
if (!outputs) // not ready, return empty result.
{
return new nn::Objects();
}
nn::Objects *res = _post_process(outputs, img.width(), img.height(), fit, sort);
#if SHOW_DETECT_TIME
log::info("postprocess time: %ld", time::ticks_ms() - start);
#endif
delete outputs;
if(!res)
{
@@ -447,6 +642,13 @@ namespace maix::nn
float _keypoint_th = 0.5;
YOLO11_Type _type;
bool _dual_buff;
_OutIdxes _out_idxes;
int _out_node_mode;
bool _out_chw;
int _reg_max = 16;
std::vector<float> _stride = {8, 16, 32};
int _anchor_num;
bool _obb_need_sigmoid;
private:
err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
@@ -513,113 +715,255 @@ namespace maix::nn
bool _decode_objs(nn::Objects &objs, tensor::Tensors *outputs, float conf_thresh, int w, int h, tensor::Tensor **kp_out, tensor::Tensor **mask_out)
{
float stride[3] = {8, 16, 32};
tensor::Tensor *score_out = NULL; // shape 1, 80, 8400, 1
tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400
for (auto i : *outputs)
if(_type == YOLO11_Type::SEG)
{
if (i.second->shape()[2] == 4 && !box_out)
*mask_out = &(*outputs)[_out_idxes.seg_mask];
*kp_out = &(*outputs)[_out_idxes.seg_mask_weight];
}
else if (_type == YOLO11_Type::OBB)
{
*kp_out = &(*outputs)[_out_idxes.obb_angle];
}
else if (_type == YOLO11_Type::POSE)
{
*kp_out = &(*outputs)[_out_idxes.pose];
}
int idx_start[3] = {
0,
(int)(h / _stride[0] * w / _stride[0]),
(int)(h / _stride[0] * w / _stride[0] + h / _stride[1] * w / _stride[1])};
// detect
if(_out_node_mode == 1)
{
tensor::Tensor *dets[3] = {&(*outputs)[_out_idxes.det0], &(*outputs)[_out_idxes.det1], &(*outputs)[_out_idxes.det2]};
if (_type == YOLO11_Type::OBB)
{
box_out = i.second;
}
else if (strstr(i.first.c_str(), "Sigmoid") != NULL && !score_out)
{
score_out = i.second;
if((size_t)score_out->shape()[1] != labels.size())
if(_out_chw)
{
log::error("MUD labels(%d) must equal model's(%d)", score_out->shape()[1], labels.size());
return false;
throw err::Exception(err::ERR_NOT_IMPL, "not support output chw layout");
}
else
{
int class_num = (int)labels.size();
int prob_offset = _reg_max * 4;
int batch_data_size = prob_offset + class_num;
float *angle_ptr = (float *)(*kp_out)->data();
for(int i = 0; i < 3; ++i) // 1 x nh x nw x (_reg_max * 4 + class_num)
{
int nh = dets[i]->shape()[1];
int nw = dets[i]->shape()[2];
int anchor_num = nh * nw;
float *feature = (float*)dets[i]->data();
for (int anchor_idx = 0; anchor_idx < anchor_num; ++anchor_idx)
{
int ax = anchor_idx % nw;
int ay = anchor_idx / nw;
int offset = idx_start[i] + anchor_idx;
float *p = feature + batch_data_size * anchor_idx;
int class_id = _argmax(p + prob_offset, class_num, 1);
float score = _sigmoid(p[prob_offset + class_id]);
if (score <= conf_thresh)
{
continue;
}
float dis[4];
for(int k = 0; k < 4; ++k)
{
// std::vector<float> softmax_res(_reg_max, 0.f);
float softmax_res[_reg_max];
dis[k] = _softmax(p + k * _reg_max, softmax_res, _reg_max);
}
float lt_x = dis[0];
float lt_y = dis[1];
float rb_x = dis[2];
float rb_y = dis[3];
float angle = ((_obb_need_sigmoid ? _sigmoid(angle_ptr[offset]) : angle_ptr[offset]) - 0.25);
float angle_rad = angle * M_PI;
float cos_angle = cosf(angle_rad);
float sin_angle = sinf(angle_rad);
float xf = (rb_x - lt_x) / 2.0;
float yf = (rb_y - lt_y) / 2.0;
float bbox_w = (lt_x + rb_x) * _stride[i];
float bbox_h = (lt_y + rb_y) * _stride[i];
float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * _stride[i] - bbox_w * 0.5;
float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * _stride[i] - bbox_h * 0.5;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, score, {}, angle);
obj.temp = (void *)kp_info;
}
}
}
}
else if (strstr(i.first.c_str(), "output1") != NULL)
{
*mask_out = i.second;
}
else
{
*kp_out = i.second;
}
}
if (!score_out || !box_out)
{
throw err::Exception(err::ERR_ARGS, "model output not valid");
}
int total_box_num = box_out->shape()[3];
// int class_num = this->labels.size();
int class_num = score_out->shape()[1];
float *scores_ptr = (float *)score_out->data();
float *dets_ptr = (float *)box_out->data();
int idx_start[3] = {
0,
(int)(h / stride[0] * w / stride[0]),
(int)(h / stride[0] * w / stride[0] + h / stride[1] * w / stride[1])};
if (_type == YOLO11_Type::OBB)
{
float *angle_ptr = (float *)(*kp_out)->data();
for (int i = 0; i < 3; i++)
{
int nh = h / stride[i];
int nw = w / stride[i];
for (int ay = 0; ay < nh; ++ay)
if(_out_chw)
{
for (int ax = 0; ax < nw; ++ax)
throw err::Exception(err::ERR_NOT_IMPL, "not support output chw layout");
}
else
{
int class_num = (int)labels.size();
int prob_offset = _reg_max * 4;
int batch_data_size = prob_offset + class_num;
for(int i = 0; i < 3; ++i) // 1 x nh x nw x (_reg_max * 4 + class_num)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, total_box_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num);
// int class_id = (offset + max_idx) / total_box_num;
float obj_score = scores_ptr[offset + class_id * total_box_num];
if (obj_score <= conf_thresh)
int nh = dets[i]->shape()[1];
int nw = dets[i]->shape()[2];
int anchor_num = nh * nw;
float *feature = (float*)dets[i]->data();
for (int anchor_idx = 0; anchor_idx < anchor_num; ++anchor_idx)
{
continue;
int ax = anchor_idx % nw;
int ay = anchor_idx / nw;
float *p = feature + batch_data_size * anchor_idx;
int class_id = _argmax(p + prob_offset, class_num, 1);
float score = _sigmoid(p[prob_offset + class_id]);
if (score <= conf_thresh)
{
continue;
}
float dis[4];
for(int k = 0; k < 4; ++k)
{
// std::vector<float> softmax_res(_reg_max, 0.f);
float softmax_res[_reg_max];
dis[k] = _softmax(p + k * _reg_max, softmax_res, _reg_max);
}
float bbox_x = (ax + 0.5 - dis[0]) * _stride[i];
float bbox_y = (ay + 0.5 - dis[1]) * _stride[i];
float bbox_w = (ax + 0.5 + dis[2]) * _stride[i] - bbox_x;
float bbox_h = (ay + 0.5 + dis[3]) * _stride[i] - bbox_y;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(idx_start[i] + anchor_idx, ax, ay, _stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, score);
obj.temp = (void *)kp_info;
}
float angle = (angle_ptr[offset] - 0.25);
float angle_rad = angle * M_PI;
float cos_angle = cosf(angle_rad);
float sin_angle = sinf(angle_rad);
float lt_x = dets_ptr[offset];
float lt_y = dets_ptr[offset + total_box_num];
float rb_x = dets_ptr[offset + total_box_num * 2];
float rb_y = dets_ptr[offset + total_box_num * 3];
float xf = (rb_x - lt_x) / 2.0;
float yf = (rb_y - lt_y) / 2.0;
float bbox_w = (lt_x + rb_x) * stride[i];
float bbox_h = (lt_y + rb_y) * stride[i];
float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * stride[i] - bbox_w * 0.5;
float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * stride[i] - bbox_h * 0.5;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle);
obj.temp = (void *)kp_info;
}
}
}
}
else
else // mode 2
{
for (int i = 0; i < 3; i++)
tensor::Tensor *score_out = NULL; // shape 1, 80, 8400 hwc: 1, 8400, 80
tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400 hwc: 1, 1, 8400, 4
score_out = &(*outputs)[_out_idxes.sigmoid];
box_out = &(*outputs)[_out_idxes.dfl];
int class_num = (int)labels.size();
float *scores_ptr = (float *)score_out->data();
float *dets_ptr = (float *)box_out->data();
if (_type == YOLO11_Type::OBB)
{
int nh = h / stride[i];
int nw = w / stride[i];
for (int ay = 0; ay < nh; ++ay)
if(!_out_chw)
{
for (int ax = 0; ax < nw; ++ax)
throw err::Exception(err::ERR_NOT_IMPL, "not support output hwc layout");
}
float *angle_ptr = (float *)(*kp_out)->data();
for (int i = 0; i < 3; i++)
{
int nh = h / _stride[i];
int nw = w / _stride[i];
for (int ay = 0; ay < nh; ++ay)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, total_box_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num);
// int class_id = (offset + max_idx) / total_box_num;
float obj_score = scores_ptr[offset + class_id * total_box_num];
if (obj_score <= conf_thresh)
for (int ax = 0; ax < nw; ++ax)
{
continue;
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, _anchor_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * _anchor_num - offset, _anchor_num);
// int class_id = (offset + max_idx) / _anchor_num;
float obj_score = scores_ptr[offset + class_id * _anchor_num];
if (obj_score <= conf_thresh)
{
continue;
}
float angle = ((_obb_need_sigmoid ? _sigmoid(angle_ptr[offset]) : angle_ptr[offset]) - 0.25);
float angle_rad = angle * M_PI;
float cos_angle = cosf(angle_rad);
float sin_angle = sinf(angle_rad);
float lt_x = dets_ptr[offset];
float lt_y = dets_ptr[offset + _anchor_num];
float rb_x = dets_ptr[offset + _anchor_num * 2];
float rb_y = dets_ptr[offset + _anchor_num * 3];
float xf = (rb_x - lt_x) / 2.0;
float yf = (rb_y - lt_y) / 2.0;
float bbox_w = (lt_x + rb_x) * _stride[i];
float bbox_h = (lt_y + rb_y) * _stride[i];
float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * _stride[i] - bbox_w * 0.5;
float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * _stride[i] - bbox_h * 0.5;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle);
obj.temp = (void *)kp_info;
}
}
}
}
else
{
if(_out_chw)
{
for (int i = 0; i < 3; i++)
{
int nh = h / _stride[i];
int nw = w / _stride[i];
for (int ay = 0; ay < nh; ++ay)
{
for (int ax = 0; ax < nw; ++ax)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, _anchor_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * _anchor_num - offset, _anchor_num);
// int class_id = (offset + max_idx) / _anchor_num;
float obj_score = scores_ptr[offset + class_id * _anchor_num];
if (obj_score <= conf_thresh)
{
continue;
}
float bbox_x = (ax + 0.5 - dets_ptr[offset]) * _stride[i];
float bbox_y = (ay + 0.5 - dets_ptr[offset + _anchor_num]) * _stride[i];
float bbox_w = (ax + 0.5 + dets_ptr[offset + _anchor_num * 2]) * _stride[i] - bbox_x;
float bbox_h = (ay + 0.5 + dets_ptr[offset + _anchor_num * 3]) * _stride[i] - bbox_y;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score);
obj.temp = (void *)kp_info;
}
}
}
}
else
{
typedef struct
{
float bbox_x;
float bbox_y;
float bbox_w;
float bbox_h;
int class_id;
float score;
} valid_item_t;
for (int i = 0; i < 3; i++)
{
int nh = h / _stride[i];
int nw = w / _stride[i];
for (int ay = 0; ay < nh; ++ay)
{
for (int ax = 0; ax < nw; ++ax)
{
valid_item_t item;
int offset = idx_start[i] + ay * nw + ax;
float *p = scores_ptr + offset * class_num;
item.class_id = _argmax(p, class_num, 1);
item.score = p[item.class_id];
if (item.score <= conf_thresh)
{
continue;
}
item.bbox_x = (ax + 0.5 - dets_ptr[offset * 4]) * _stride[i];
item.bbox_y = (ay + 0.5 - dets_ptr[offset * 4 + 1]) * _stride[i];
item.bbox_w = (ax + 0.5 + dets_ptr[offset * 4 + 2]) * _stride[i] - item.bbox_x;
item.bbox_h = (ay + 0.5 + dets_ptr[offset * 4 + 3]) * _stride[i] - item.bbox_y;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]);
Object &obj = objs.add(item.bbox_x, item.bbox_y, item.bbox_w, item.bbox_h, item.class_id, item.score);
obj.temp = (void *)kp_info;
}
}
float bbox_x = (ax + 0.5 - dets_ptr[offset]) * stride[i];
float bbox_y = (ay + 0.5 - dets_ptr[offset + total_box_num]) * stride[i];
float bbox_w = (ax + 0.5 + dets_ptr[offset + total_box_num * 2]) * stride[i] - bbox_x;
float bbox_h = (ay + 0.5 + dets_ptr[offset + total_box_num * 3]) * stride[i] - bbox_y;
_KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score);
obj.temp = (void *)kp_info;
}
}
}
@@ -695,82 +1039,193 @@ namespace maix::nn
void _decode_keypoints(nn::Objects &objs, tensor::Tensor *kp_out)
{
float *data = (float *)kp_out->data();
int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 51, 8400, 1
for (size_t i = 0; i < objs.size(); ++i)
if(_out_chw)
{
nn::Object &o = objs.at(i);
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < keypoint_num; ++k)
int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400
for (size_t i = 0; i < objs.size(); ++i)
{
float score = _sigmoid(p[(k * 3 + 2) * total_box_num]);
int x = -1;
int y = -1;
if (score > _keypoint_th)
nn::Object &o = objs.at(i);
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < keypoint_num; ++k)
{
x = (p[(k * 3) * total_box_num] * 2.0 + kp_info->anchor_x) * kp_info->stride;
y = (p[(k * 3 + 1) * total_box_num] * 2.0 + kp_info->anchor_y) * kp_info->stride;
float score = _sigmoid(p[(k * 3 + 2) * _anchor_num]);
int x = -1;
int y = -1;
if (score > _keypoint_th)
{
x = (p[(k * 3) * _anchor_num] * 2.0 + kp_info->anchor_x) * kp_info->stride;
y = (p[(k * 3 + 1) * _anchor_num] * 2.0 + kp_info->anchor_y) * kp_info->stride;
}
o.points.push_back(x);
o.points.push_back(y);
}
o.points.push_back(x);
o.points.push_back(y);
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
else
{
int keypoint_num = kp_out->shape()[2] / 3; // 1, 8400, 51
log::info("%d", keypoint_num);
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx * kp_out->shape()[2];
for (int k = 0; k < keypoint_num; ++k)
{
float score = _sigmoid(p[k * 3 + 2]);
int x = -1;
int y = -1;
if (score > _keypoint_th)
{
x = (p[k * 3] * 2.0 + kp_info->anchor_x) * kp_info->stride;
y = (p[k * 3 + 1] * 2.0 + kp_info->anchor_y) * kp_info->stride;
}
o.points.push_back(x);
o.points.push_back(y);
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
void _decode_seg_points(nn::Objects &objs, tensor::Tensor *kp_out, tensor::Tensor *mask_out)
{
float *data = (float *)kp_out->data();
float *mask_data = (float *)mask_out->data(); // 1, 32, 160, 160
int mask_h = mask_out->shape()[2];
int mask_w = mask_out->shape()[3];
int mask_squre = mask_h * mask_w;
int mask_num = kp_out->shape()[1]; // 1, 32, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 32, 8400, 1
float mask_weights[mask_num];
for (size_t i = 0; i < objs.size(); ++i)
float *mask_data = (float *)mask_out->data();
if(_out_chw)
{
nn::Object &o = objs.at(i);
int mask_x = o.x * mask_w / _input_size.width();
int mask_y = o.y * mask_h / _input_size.height();
int mask_x2 = (o.x + o.w) * mask_w / _input_size.width();
int mask_y2 = (o.y + o.h) * mask_h / _input_size.height();
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < mask_num; ++k)
int mask_h = mask_out->shape()[2]; // 1, 32, 160, 160
int mask_w = mask_out->shape()[3];
int mask_squre = mask_h * mask_w;
int mask_num = kp_out->shape()[1]; // 1, 32, 8400
float mask_weights[mask_num];
for (size_t i = 0; i < objs.size(); ++i)
{
mask_weights[k] = p[k * total_box_num];
}
o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE);
uint8_t *p_img_data = (uint8_t *)o.seg_mask->data();
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
nn::Object &o = objs.at(i);
int mask_x = o.x * mask_w / _input_size.width();
int mask_y = o.y * mask_h / _input_size.height();
int mask_x2 = (o.x + o.w) * mask_w / _input_size.width();
int mask_y2 = (o.y + o.h) * mask_h / _input_size.height();
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < mask_num; ++k)
{
mask_data[j * mask_w + k] *= mask_weights[0];
mask_weights[k] = p[k * _anchor_num];
}
}
for (int n = 1; n < mask_num; ++n)
{
o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE);
uint8_t *p_img_data = (uint8_t *)o.seg_mask->data();
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k];
mask_data[j * mask_w + k] *= mask_weights[0];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
for (int n = 1; n < mask_num; ++n)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255);
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255);
}
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
else
{
bool mask_chw = false;
int mask_h = _input_size.height() / 4;
int mask_w = _input_size.width() / 4;
int mask_num = 32;
int mask_squre = mask_h * mask_w;
// 1, 32, 160, 160 or 1 160 160 32
if(mask_out->shape()[1] == mask_num && mask_out->shape()[2] == mask_h && mask_out->shape()[3] == mask_w)
{
mask_chw = true;
}
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
int mask_x = o.x * mask_w / _input_size.width();
int mask_y = o.y * mask_h / _input_size.height();
int mask_x2 = (o.x + o.w) * mask_w / _input_size.width();
int mask_y2 = (o.y + o.h) * mask_h / _input_size.height();
_KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp;
float *mask_weights = data + kp_info->idx * mask_num; //1 8400 32
o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE);
uint8_t *p_img_data = (uint8_t *)o.seg_mask->data();
if(mask_chw) // 1 32 160 160
{
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] *= mask_weights[0];
}
}
for (int n = 1; n < mask_num; ++n)
{
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255);
}
}
}
else // 1 160 160 32
{
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[(j * mask_w + k) * mask_num] *= mask_weights[0];
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
for (int n = 1; n < mask_num; ++n)
{
mask_data[(j * mask_w + k) * mask_num] += mask_weights[n] * mask_data[(j * mask_w + k) * mask_num + n];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[(j * mask_w + k) * mask_num]) * 255);
}
}
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
delete (_KpInfoYolo11 *)o.temp;
o.temp = NULL;
}
}
@@ -989,6 +1444,23 @@ namespace maix::nn
split0(tokens, s, delimiter);
return tokens;
}
static float _softmax(const float* src, float* dst, int length)
{
const float alpha = *std::max_element(src, src + length);
float denominator = 0;
float dis_sum = 0;
for (int i = 0; i < length; ++i)
{
dst[i] = exp(src[i] - alpha);
denominator += dst[i];
}
for (int i = 0; i < length; ++i)
{
dst[i] /= denominator;
dis_sum += i * dst[i];
}
return dis_sum;
}
};
} // namespace maix::nn

5
components/nn/include/maix_nn_yolov5.hpp
View File

@@ -203,7 +203,10 @@ namespace maix::nn
return err::ERR_ARGS;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
if(inputs[0].shape[3] <= 4) // nhwc
_input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]);
else
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
return err::ERR_NONE;
}

861
components/nn/include/maix_nn_yolov8.hpp
View File

@@ -12,35 +12,15 @@
#include "maix_nn_F.hpp"
#include "maix_nn_object.hpp"
#include <math.h>
#include "maix_nn_yolo11.hpp"
namespace maix::nn
{
enum class YOLOv8_Type
{
DETECT = 0,
POSE = 1,
SEG = 2,
OBB = 3
};
class _KpInfo
{
public:
_KpInfo(int idx, int ax, int ay, float stride)
: idx(idx), anchor_x(ax), anchor_y(ay), stride(stride)
{
}
int idx;
int anchor_x;
int anchor_y;
float stride;
};
/**
* YOLOv8 class
* @maixpy maix.nn.YOLOv8
*/
class YOLOv8
class YOLOv8 : public YOLO11
{
public:
/**
@@ -54,32 +34,9 @@ namespace maix::nn
* @maixcdk maix.nn.YOLOv8.YOLOv8
*/
YOLOv8(const string &model = "", bool dual_buff = true)
: YOLO11(model, dual_buff)
{
_model = nullptr;
_type = YOLOv8_Type::DETECT;
_dual_buff = dual_buff;
if (!model.empty())
{
err::Err e = load(model);
if (e != err::ERR_NONE)
{
throw err::Exception(e, "load model failed");
}
}
}
~YOLOv8()
{
if (_model)
{
delete _model;
_model = nullptr;
}
}
std::string type()
{
return _extra_info["type"];
type_str = "yolov8"
}
/**
@@ -90,143 +47,7 @@ namespace maix::nn
*/
err::Err load(const string &model)
{
if (_model)
{
delete _model;
_model = nullptr;
}
_model = new nn::NN(model, _dual_buff);
if (!_model)
{
return err::ERR_NO_MEM;
}
_extra_info = _model->extra_info();
if (_extra_info.find("model_type") != _extra_info.end())
{
if (_extra_info["model_type"] != this->type_str)
{
log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("model_type key not found");
return err::ERR_ARGS;
}
log::info("model info:\n\ttype: %s", this->type_str.c_str());
if (_extra_info.find("input_type") != _extra_info.end())
{
std::string input_type = _extra_info["input_type"];
if (input_type == "rgb")
{
_input_img_fmt = maix::image::FMT_RGB888;
log::print("\tinput type: rgb\n");
}
else if (input_type == "bgr")
{
_input_img_fmt = maix::image::FMT_BGR888;
log::print("\tinput type: bgr\n");
}
else
{
log::error("unknown input type: %s", input_type.c_str());
return err::ERR_ARGS;
}
}
else
{
log::error("input_type key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("mean") != _extra_info.end())
{
std::string mean_str = _extra_info["mean"];
std::vector<std::string> mean_strs = split(mean_str, ",");
log::print("\tmean:");
for (auto &it : mean_strs)
{
try
{
this->mean.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("mean value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->mean.back());
}
log::print("\n");
}
else
{
log::error("mean key not found");
return err::ERR_ARGS;
}
if (_extra_info.find("scale") != _extra_info.end())
{
std::string scale_str = _extra_info["scale"];
std::vector<std::string> scale_strs = split(scale_str, ",");
log::print("\tscale:");
for (auto &it : scale_strs)
{
try
{
this->scale.push_back(std::stof(it));
}
catch (std::exception &e)
{
log::error("scale value error, should float");
return err::ERR_ARGS;
}
log::print("%f ", this->scale.back());
}
log::print("\n");
}
else
{
log::error("scale key not found");
return err::ERR_ARGS;
}
err::Err e = _model->extra_info_labels(labels);
if(e == err::Err::ERR_NONE)
{
log::print("\tlabels num: %ld\n", labels.size());
}
else
{
log::error("labels key not found: %s", err::to_str(e).c_str());
return err::ERR_ARGS;
}
if (_extra_info.find("type") != _extra_info.end())
{
if (_extra_info["type"] == "pose")
{
_type = YOLOv8_Type::POSE;
}
else if (_extra_info["type"] == "seg")
{
_type = YOLOv8_Type::SEG;
}
else if (_extra_info["type"] == "obb")
{
_type = YOLOv8_Type::OBB;
}
else if (_extra_info["type"] != "detector")
{
log::error("type [%s] not support, suport detector and pose", _extra_info["type"].c_str());
return err::ERR_ARGS;
}
}
else
{
_type = YOLOv8_Type::DETECT;
}
std::vector<nn::LayerInfo> inputs = _model->inputs_info();
_input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
return err::ERR_NONE;
return YOLO11::load(model);
}
/**
@@ -244,26 +65,7 @@ namespace maix::nn
*/
nn::Objects *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, float keypoint_th = 0.5, int sort = 0)
{
this->_conf_th = conf_th;
this->_iou_th = iou_th;
this->_keypoint_th = keypoint_th;
if (img.format() != _input_img_fmt)
{
throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
}
tensor::Tensors *outputs;
outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
if (!outputs) // not ready, return empty result.
{
return new nn::Objects();
}
nn::Objects *res = _post_process(outputs, img.width(), img.height(), fit, sort);
delete outputs;
if(!res)
{
throw err::Exception("post process failed, please see log before");
}
return res;
return YOLO11::detect(img, conf_th, iou_th, fit, keypoint_th, sort);
}
/**
@@ -273,7 +75,7 @@ namespace maix::nn
*/
image::Size input_size()
{
return _input_size;
return YOLO11::input_size();
}
/**
@@ -283,7 +85,7 @@ namespace maix::nn
*/
int input_width()
{
return _input_size.width();
return YOLO11::input_width();
}
/**
@@ -293,7 +95,7 @@ namespace maix::nn
*/
int input_height()
{
return _input_size.height();
return YOLO11::input_height();
}
/**
@@ -303,7 +105,7 @@ namespace maix::nn
*/
image::Format input_format()
{
return _input_img_fmt;
return YOLO11::input_format();
}
/**
@@ -319,57 +121,7 @@ namespace maix::nn
*/
void draw_pose(image::Image &img, std::vector<int> points, int radius = 4, image::Color color = image::COLOR_RED, const std::vector<image::Color> &colors = std::vector<image::Color>(), bool body = true, bool close = false)
{
bool line_drawed = false;
if (points.size() < 2 || points.size() % 2 != 0)
{
throw std::runtime_error("keypoints size must >= 2 and multiple of 2");
return;
}
if (points.size() == 17 * 2 && body)
{
int pos[] = {9, 7, 7, 5, 6, 8, 8, 10, 5, 11, 6, 12, 5, 6, 11, 12, 11, 13, 15, 13, 14, 12, 14, 16};
for (int i = 0; i < 12; ++i)
{
int x1 = points[pos[i * 2] * 2];
int y1 = points[pos[i * 2] * 2 + 1];
int x2 = points[pos[i * 2 + 1] * 2];
int y2 = points[pos[i * 2 + 1] * 2 + 1];
if (x1 < 0 || y1 < 0 || x2 < 0 || y2 < 0)
continue;
img.draw_line(x1, y1, x2, y2, color, 2);
}
int x = (points[5 * 2] + points[6 * 2]) / 2;
int y = (points[5 * 2 + 1] + points[6 * 2 + 1]) / 2;
if (!(points[5 * 2] < 0 || points[5 * 2 + 1] < 0 || points[6 * 2] < 0 || points[6 * 2 + 1] < 0 || x < 0 || y < 0 || points[0] < 0 || points[1] < 0))
img.draw_line(points[0], points[1], x, y, color, 2);
line_drawed = true;
}
for (size_t i = 0; i < points.size() / 2; ++i)
{
int x = points[i * 2];
int y = points[i * 2 + 1];
if (x < 0 || y < 0)
continue;
auto &_color = color;
if (colors.size() > i)
{
_color = colors[i];
}
img.draw_circle(x, y, radius, _color, -1);
}
if(close && !line_drawed)
{
for (size_t i = 0; i < points.size() / 2; ++i)
{
int x1 = points[i * 2];
int y1 = points[i * 2 + 1];
int x2 = points[(i + 1) % (points.size() / 2) * 2];
int y2 = points[(i + 1) % (points.size() / 2) * 2 + 1];
if (x1 < 0 || y1 < 0 || x2 < 0 || y2 < 0)
continue;
img.draw_line(x1, y1, x2, y2, color, 2);
}
}
YOLO11::draw_pose(img, points, radius, color, colors, body,close);
}
/**
@@ -381,32 +133,7 @@ namespace maix::nn
*/
void draw_seg_mask(image::Image &img, int x, int y, image::Image &seg_mask, int threshold = 127)
{
if (seg_mask.format() != image::FMT_GRAYSCALE)
{
throw err::Exception(err::ERR_ARGS, "seg_mask only support grascale");
}
if (img.format() != image::FMT_RGB888 &&
img.format() != image::FMT_BGR888 &&
img.format() != image::FMT_RGBA8888 &&
img.format() != image::FMT_BGRA8888 &&
img.format() != image::FMT_GRAYSCALE)
{
throw err::Exception(err::ERR_ARGS, "img not support");
}
int fmt_size = image::fmt_size[img.format()];
uint8_t *to = (uint8_t*)img.data();
uint8_t *from = (uint8_t*)seg_mask.data();
for (int i = 0; i < seg_mask.height(); ++i)
{
for (int j = 0; j < seg_mask.width(); ++j)
{
uint8_t data = from[i * seg_mask.width() + j];
if(data > threshold)
{
to[((y + i) * img.width() + x + j) * fmt_size] = data;
}
}
}
YOLO11::draw_seg_mask(img, x, y, seg_mask, threshold);
}
public:
@@ -414,581 +141,25 @@ namespace maix::nn
* Labels list
* @maixpy maix.nn.YOLOv8.labels
*/
std::vector<string> labels;
//std::vector<string> labels;
/**
* Label file path
* @maixpy maix.nn.YOLOv8.label_path
*/
std::string label_path;
//std::string label_path;
/**
* Get mean value, list type
* @maixpy maix.nn.YOLOv8.mean
*/
std::vector<float> mean;
//std::vector<float> mean;
/**
* Get scale value, list type
* @maixpy maix.nn.YOLOv8.scale
*/
std::vector<float> scale;
protected:
std::string type_str = "yolov8";
private:
image::Size _input_size;
image::Format _input_img_fmt;
nn::NN *_model;
std::map<string, string> _extra_info;
float _conf_th = 0.5;
float _iou_th = 0.45;
float _keypoint_th = 0.5;
YOLOv8_Type _type;
bool _dual_buff;
private:
err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
{
// load labels from labels file
labels.clear();
fs::File *f = fs::open(label_path, "r");
if (!f)
{
log::error("open label file %s failed", label_path.c_str());
return err::ERR_ARGS;
}
std::string line;
while (f->readline(line) > 0)
{
// strip line
line.erase(0, line.find_first_not_of(" \t\r\n"));
line.erase(line.find_last_not_of(" \t\r\n") + 1);
labels.push_back(line);
}
f->close();
delete f;
return err::ERR_NONE;
}
nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit, int sort)
{
nn::Objects *objects = new nn::Objects();
tensor::Tensor *kp_out = NULL;
tensor::Tensor *mask_out = NULL;
float scale_w = 1;
float scale_h = 1;
if(!_decode_objs(*objects, outputs, _conf_th, _input_size.width(), _input_size.height(), &kp_out, &mask_out))
{
delete objects;
return NULL;
}
if (objects->size() > 0)
{
nn::Objects *objects_total = objects;
objects = _nms(*objects);
delete objects_total;
if(sort != 0)
{
_sort_objects(*objects, sort);
}
}
// decode keypoints
if (_type == YOLOv8_Type::POSE)
{
_decode_keypoints(*objects, kp_out);
}
else if (_type == YOLOv8_Type::SEG)
{
_decode_seg_points(*objects, kp_out, mask_out);
}
if (objects->size() > 0)
{
_correct_bbox(*objects, img_w, img_h, fit, &scale_w, &scale_h);
}
return objects;
}
bool _decode_objs(nn::Objects &objs, tensor::Tensors *outputs, float conf_thresh, int w, int h, tensor::Tensor **kp_out, tensor::Tensor **mask_out)
{
float stride[3] = {8, 16, 32};
tensor::Tensor *score_out = NULL; // shape 1, 80, 8400, 1
tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400
for (auto i : *outputs)
{
if (i.second->shape()[2] == 4 && !box_out)
{
box_out = i.second;
}
else if (strstr(i.first.c_str(), "Sigmoid") != NULL && !score_out)
{
score_out = i.second;
if((size_t)score_out->shape()[1] != labels.size())
{
log::error("MUD labels(%d) must equal model's(%d)", score_out->shape()[1], labels.size());
return false;
}
}
else if (strstr(i.first.c_str(), "output1") != NULL)
{
*mask_out = i.second;
}
else
{
*kp_out = i.second;
}
}
if (!score_out || !box_out)
{
throw err::Exception(err::ERR_ARGS, "model output not valid");
}
int total_box_num = box_out->shape()[3];
// int class_num = this->labels.size();
int class_num = score_out->shape()[1];
float *scores_ptr = (float *)score_out->data();
float *dets_ptr = (float *)box_out->data();
int idx_start[3] = {
0,
(int)(h / stride[0] * w / stride[0]),
(int)(h / stride[0] * w / stride[0] + h / stride[1] * w / stride[1])};
if (_type == YOLOv8_Type::OBB)
{
float *angle_ptr = (float *)(*kp_out)->data();
for (int i = 0; i < 3; i++)
{
int nh = h / stride[i];
int nw = w / stride[i];
for (int ay = 0; ay < nh; ++ay)
{
for (int ax = 0; ax < nw; ++ax)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, total_box_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num);
// int class_id = (offset + max_idx) / total_box_num;
float obj_score = scores_ptr[offset + class_id * total_box_num];
if (obj_score <= conf_thresh)
{
continue;
}
float angle = (angle_ptr[offset] - 0.25);
float angle_rad = angle * M_PI;
float cos_angle = cosf(angle_rad);
float sin_angle = sinf(angle_rad);
float lt_x = dets_ptr[offset];
float lt_y = dets_ptr[offset + total_box_num];
float rb_x = dets_ptr[offset + total_box_num * 2];
float rb_y = dets_ptr[offset + total_box_num * 3];
float xf = (rb_x - lt_x) / 2.0;
float yf = (rb_y - lt_y) / 2.0;
float bbox_w = (lt_x + rb_x) * stride[i];
float bbox_h = (lt_y + rb_y) * stride[i];
float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * stride[i] - bbox_w * 0.5;
float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * stride[i] - bbox_h * 0.5;
_KpInfo *kp_info = new _KpInfo(offset, ax, ay, stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle);
obj.temp = (void *)kp_info;
}
}
}
}
else
{
for (int i = 0; i < 3; i++)
{
int nh = h / stride[i];
int nw = w / stride[i];
for (int ay = 0; ay < nh; ++ay)
{
for (int ax = 0; ax < nw; ++ax)
{
int offset = idx_start[i] + ay * nw + ax;
int class_id = _argmax(scores_ptr + offset, class_num, total_box_num);
// int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num);
// int class_id = (offset + max_idx) / total_box_num;
float obj_score = scores_ptr[offset + class_id * total_box_num];
if (obj_score <= conf_thresh)
{
continue;
}
float bbox_x = (ax + 0.5 - dets_ptr[offset]) * stride[i];
float bbox_y = (ay + 0.5 - dets_ptr[offset + total_box_num]) * stride[i];
float bbox_w = (ax + 0.5 + dets_ptr[offset + total_box_num * 2]) * stride[i] - bbox_x;
float bbox_h = (ay + 0.5 + dets_ptr[offset + total_box_num * 3]) * stride[i] - bbox_y;
_KpInfo *kp_info = new _KpInfo(offset, ax, ay, stride[i]);
Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score);
obj.temp = (void *)kp_info;
}
}
}
}
return true;
}
nn::Objects *_nms(nn::Objects &objs)
{
nn::Objects *result = new nn::Objects();
std::sort(objs.begin(), objs.end(), [](const nn::Object *a, const nn::Object *b)
{ return a->score > b->score; });
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &a = objs.at(i);
if (a.score == 0)
continue;
for (size_t j = i + 1; j < objs.size(); ++j)
{
nn::Object &b = objs.at(j);
{
if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
{
b.score = 0;
}
}
}
}
for (nn::Object *a : objs)
{
if (a->score != 0)
{
Object &obj = result->add(a->x, a->y, a->w, a->h, a->class_id, a->score, a->points);
if (obj.x < 0)
{
obj.w += obj.x;
obj.x = 0;
}
if (obj.y < 0)
{
obj.h += obj.y;
obj.y = 0;
}
if (obj.x + obj.w > _input_size.width())
{
obj.w = _input_size.width() - obj.x;
}
if (obj.y + obj.h > _input_size.height())
{
obj.h = _input_size.height() - obj.y;
}
obj.temp = a->temp;
}
else
{
delete (_KpInfo *)a->temp;
a->temp = NULL;
}
}
return result;
}
void _sort_objects(nn::Objects &objects, int sort)
{
if (sort > 0)
std::sort(objects.begin(), objects.end(), [](const nn::Object *a, const nn::Object *b)
{ return (a->w * a->h) > (b->w * b->h); });
else
std::sort(objects.begin(), objects.end(), [](const nn::Object *a, const nn::Object *b)
{ return (a->w * a->h) < (b->w * b->h); });
}
void _decode_keypoints(nn::Objects &objs, tensor::Tensor *kp_out)
{
float *data = (float *)kp_out->data();
int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 51, 8400, 1
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
_KpInfo *kp_info = (_KpInfo *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < keypoint_num; ++k)
{
float score = _sigmoid(p[(k * 3 + 2) * total_box_num]);
int x = -1;
int y = -1;
if (score > _keypoint_th)
{
x = (p[(k * 3) * total_box_num] * 2.0 + kp_info->anchor_x) * kp_info->stride;
y = (p[(k * 3 + 1) * total_box_num] * 2.0 + kp_info->anchor_y) * kp_info->stride;
}
o.points.push_back(x);
o.points.push_back(y);
}
delete (_KpInfo *)o.temp;
o.temp = NULL;
}
}
void _decode_seg_points(nn::Objects &objs, tensor::Tensor *kp_out, tensor::Tensor *mask_out)
{
float *data = (float *)kp_out->data();
float *mask_data = (float *)mask_out->data(); // 1, 32, 160, 160
int mask_h = mask_out->shape()[2];
int mask_w = mask_out->shape()[3];
int mask_squre = mask_h * mask_w;
int mask_num = kp_out->shape()[1]; // 1, 32, 8400, 1
int total_box_num = kp_out->shape()[2]; // 1, 32, 8400, 1
float mask_weights[mask_num];
for (size_t i = 0; i < objs.size(); ++i)
{
nn::Object &o = objs.at(i);
int mask_x = o.x * mask_w / _input_size.width();
int mask_y = o.y * mask_h / _input_size.height();
int mask_x2 = (o.x + o.w) * mask_w / _input_size.width();
int mask_y2 = (o.y + o.h) * mask_h / _input_size.height();
_KpInfo *kp_info = (_KpInfo *)o.temp;
float *p = data + kp_info->idx;
for (int k = 0; k < mask_num; ++k)
{
mask_weights[k] = p[k * total_box_num];
}
o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE);
uint8_t *p_img_data = (uint8_t *)o.seg_mask->data();
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] *= mask_weights[0];
}
}
for (int n = 1; n < mask_num; ++n)
{
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k];
}
}
}
for (int j = mask_y; j < mask_y2; ++j)
{
for (int k = mask_x; k < mask_x2; ++k)
{
*p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255);
}
}
delete (_KpInfo *)o.temp;
o.temp = NULL;
}
}
void _correct_bbox(nn::Objects &objs, int img_w, int img_h, maix::image::Fit fit, float *scale_w, float *scale_h)
{
#define CORRECT_BBOX_RANGE_YOLOV8(obj) \
do \
{ \
if (obj->x < 0) \
{ \
obj->w += obj->x; \
obj->x = 0; \
} \
if (obj->y < 0) \
{ \
obj->h += obj->y; \
obj->y = 0; \
} \
if (obj->x + obj->w > img_w) \
{ \
obj->w = img_w - obj->x; \
} \
if (obj->y + obj->h > img_h) \
{ \
obj->h = img_h - obj->y; \
} \
} while (0)
for (nn::Object *obj : objs)
{
if (obj->temp)
{
delete (_KpInfo *)obj->temp;
obj->temp = NULL;
}
}
if (img_w == _input_size.width() && img_h == _input_size.height())
{
if (_type == YOLOv8_Type::SEG)
{
for (nn::Object *obj : objs)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
return;
}
if (fit == maix::image::FIT_FILL)
{
*scale_w = (float)img_w / _input_size.width();
*scale_h = (float)img_h / _input_size.height();
for (nn::Object *obj : objs)
{
obj->x *= *scale_w;
obj->y *= *scale_h;
obj->w *= *scale_w;
obj->h *= *scale_h;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) *= *scale_w;
obj->points.at(i * 2 + 1) *= *scale_h;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else if (fit == maix::image::FIT_CONTAIN)
{
*scale_w = ((float)_input_size.width()) / img_w;
*scale_h = ((float)_input_size.height()) / img_h;
float scale = std::min(*scale_w, *scale_h);
float scale_reverse = 1.0 / scale;
*scale_w = scale_reverse;
*scale_h = scale_reverse;
float pad_w = (_input_size.width() - img_w * scale) / 2.0;
float pad_h = (_input_size.height() - img_h * scale) / 2.0;
for (nn::Object *obj : objs)
{
obj->x = (obj->x - pad_w) * scale_reverse;
obj->y = (obj->y - pad_h) * scale_reverse;
obj->w *= scale_reverse;
obj->h *= scale_reverse;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse;
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else if (fit == maix::image::FIT_COVER)
{
*scale_w = ((float)_input_size.width()) / img_w;
*scale_h = ((float)_input_size.height()) / img_h;
float scale = std::max(*scale_w, *scale_h);
float scale_reverse = 1.0 / scale;
*scale_w = scale_reverse;
*scale_h = scale_reverse;
float pad_w = (img_w * scale - _input_size.width()) / 2.0;
float pad_h = (img_h * scale - _input_size.height()) / 2.0;
for (nn::Object *obj : objs)
{
obj->x = (obj->x + pad_w) * scale_reverse;
obj->y = (obj->y + pad_h) * scale_reverse;
obj->w *= scale_reverse;
obj->h *= scale_reverse;
for (size_t i = 0; i < obj->points.size() / 2; ++i)
{
obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse;
obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse;
}
CORRECT_BBOX_RANGE_YOLOV8(obj);
if (_type == YOLOv8_Type::SEG)
{
if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height())
{
image::Image *old = obj->seg_mask;
obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL);
delete old;
}
}
}
}
else
{
throw err::Exception(err::ERR_ARGS, "fit type not support");
}
}
inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); }
inline static float _calc_iou(Object &a, Object &b)
{
float area1 = a.w * a.h;
float area2 = b.w * b.h;
float wi = std::min((a.x + a.w), (b.x + b.w)) -
std::max(a.x, b.x);
float hi = std::min((a.y + a.h), (b.y + b.h)) -
std::max(a.y, b.y);
float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
return area_i / (area1 + area2 - area_i);
}
template <typename T>
static int _argmax(const T *data, size_t len, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = 1; i < len; i++)
{
int idx = i * stride;
if (data[maxIndex * stride] < data[idx])
{
maxIndex = i;
}
}
return maxIndex;
}
template <typename T>
static int _argmax2(const T *data, size_t end, size_t stride = 1)
{
int maxIndex = 0;
for (size_t i = stride; i < end; i += stride)
{
if (data[maxIndex] < data[i])
{
maxIndex = i;
}
}
return maxIndex;
}
static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
{
items.clear();
size_t pos_start = 0, pos_end, delim_len = delimiter.length();
std::string token;
while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
{
token = s.substr(pos_start, pos_end - pos_start);
pos_start = pos_end + delim_len;
token.erase(0, token.find_first_not_of(" \t\r\n"));
token.erase(token.find_last_not_of(" \t\r\n") + 1);
items.push_back(token);
}
items.push_back(s.substr(pos_start));
}
static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
{
std::vector<std::string> tokens;
split0(tokens, s, delimiter);
return tokens;
}
//std::vector<float> scale;
};
} // namespace maix::nn

9
examples/nn_runner_simple/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

5
examples/nn_runner_simple/README.md Normal file
View File

@@ -0,0 +1,5 @@
Hello World Project based on MaixCDK
====
Hello world example code for MaixCDK of Sipeed, build method please visit [MaixCDK](https://github.com/sipeed/MaixCDK).

74
examples/nn_runner_simple/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_runner_simple/main/Kconfig Normal file
View File

3
examples/nn_runner_simple/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

132
examples/nn_runner_simple/main/src/main.cpp Normal file
View File

@@ -0,0 +1,132 @@
#include "maix_basic.hpp"
#include "maix_nn.hpp"
#include "maix_image.hpp"
#include "main.h"
using namespace maix;
image::Image a = image::Image();
static std::vector<std::string> split(const std::string &s, const std::string &delimiter) {
std::vector<std::string> tokens;
size_t pos_start = 0, pos_end, delim_len = delimiter.length();
std::string token;
while ((pos_end = s.find (delimiter, pos_start)) != std::string::npos) {
token = s.substr (pos_start, pos_end - pos_start);
pos_start = pos_end + delim_len;
tokens.push_back (token);
}
tokens.push_back (s.substr (pos_start));
return tokens;
}
int _main(int argc, char* argv[])
{
log::info("Program start");
int ret = 0;
int repeat_times = 1;
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path repeate_times";
// model path from argv[1], image from argv[2]
if(argc < 3)
{
log::error("args error, %s", help.c_str());
return -1;
}
const char *model_path = argv[1];
repeat_times = std::stoi(argv[2]);
uint64_t ts = time::ticks_ms();
nn::NN model(model_path);
log::info("load model %s success, time: %d ms", model_path, time::ticks_ms() - ts);
std::vector<nn::LayerInfo> inputs_info = model.inputs_info();
std::vector<nn::LayerInfo> outputs_info = model.outputs_info();
log::info("Inputs:");
for(auto item : inputs_info)
{
log::print(" %s\n", item.to_str().c_str());
}
log::info("Outputs:");
for(auto item : outputs_info)
{
log::print(" %s\n", item.to_str().c_str());
}
log::info("generate inputs now");
tensor::Tensors input_tensors;
for(auto item : inputs_info)
{
tensor::Tensor *t = new tensor::Tensor(item.shape, item.dtype);
input_tensors.add_tensor(item.name, t, false, true);
}
tensor::Tensors outputs;
int count = 3;
while (repeat_times > 1 && count-- > 0) // warn up
{
err::Err e = model.forward(input_tensors, outputs, false);
if(e != err::Err::ERR_NONE)
{
log::error("forward failed");
goto end;
}
outputs.clear();
}
ts = time::ticks_ms();
while (1)
{
err::Err e = model.forward(input_tensors, outputs, false);
if(e != err::Err::ERR_NONE)
{
log::error("forward failed");
goto end;
}
++ count;
if(count >= repeat_times)
break;
outputs.clear();
}
log::info("forward success, average time: %d ms", (int)((time::ticks_ms() - ts) / repeat_times));
for(auto &it : outputs)
{
tensor::Tensor *tensor = it.second;
log::info("======= output '%s':", it.first.c_str());
log::print(" info: %s\n",(*tensor).to_str().c_str());
auto names = split(it.first, "/");
std::string filename = names[names.size() - 1] + ".bin";
log::print(" save as bin file to '%s'\n", filename.c_str());
fs::File *f = fs::open(filename, "wb");
if(!f)
{
log::error("open file faild");
goto end;
}
f->write(tensor->data(), tensor->size_int() * sizeof(float));
delete f;
}
end:
log::info("Program exit");
return ret;
}
int main(int argc, char* argv[])
{
// Catch signal and process
sys::register_default_signal_handle();
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}

31
examples/nn_yolo11_detect/main/src/main.cpp
View File

@@ -29,6 +29,7 @@ int _main(int argc, char *argv[])
float conf_threshold = 0.5;
float iou_threshold = 0.45;
bool dual_buff = true;
int repeat_times = 20;
nn::YOLO11 detector("", dual_buff);
@@ -46,9 +47,37 @@ int _main(int argc, char *argv[])
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
// we resize here manually, if not, the detect function will auto resize.
maix::image::Image *img2 = img->resize(detector.input_size().width(), detector.input_size().height(), image::Fit::FIT_CONTAIN);
delete img;
img = img2;
}
log::info("detect now");
nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold);
nn::Objects *result = nullptr;
if(repeat_times > 1)
{
for(int i = 0; i < 3; ++i) // warmup
{
result = detector.detect(*img, conf_threshold, iou_threshold);
delete result;
}
}
std::vector<int> intervals(repeat_times);
for(int i = 0; i < repeat_times; ++i) // average time
{
uint64_t start = time::ticks_ms();
result = detector.detect(*img, conf_threshold, iou_threshold);
intervals[i] = time::ticks_ms() - start;
if(i < repeat_times - 1)
delete result;
}
float sum = 0;
for(auto i : intervals)
{
sum += i;
}
log::info("run %d times, average time: %dms", repeat_times, (int)(sum / repeat_times));
if(result->size() == 0)
{
log::info("no object detected !");

9
examples/nn_yolo11_obb/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

9
examples/nn_yolo11_obb/README.md Normal file
View File

@@ -0,0 +1,9 @@
nn_yolov8 examples based on MaixCDK
====
## only run yolov8 model
- Use `maixcdk build` to compile binary files.
- Move files and runtime libraries to device.
- Use SSH command with parameters to run programs.
` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

74
examples/nn_yolo11_obb/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic nn vision)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_yolo11_obb/main/Kconfig Normal file
View File

3
examples/nn_yolo11_obb/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

140
examples/nn_yolo11_obb/main/src/main.cpp Normal file
View File

@@ -0,0 +1,140 @@
#include "maix_basic.hpp"
#include "maix_vision.hpp"
#include "maix_nn_yolo11.hpp"
#include "main.h"
using namespace maix;
int _main(int argc, char *argv[])
{
log::info("Program start");
std::string model_type = "unknown";
int ret = 0;
err::Err e;
maix::image::Format img_fmt = maix::image::FMT_RGB888;
std::vector<float> mean = {};
std::vector<float> scale = {};
char tmp_chars[128] = {0};
std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
if (argc < 2)
{
log::info(help.c_str());
return -1;
}
const char *model_path = argv[1];
float conf_threshold = 0.5;
float iou_threshold = 0.45;
bool dual_buff = true;
int repeat_times = 20;
nn::YOLO11 detector("", dual_buff);
e = detector.load(model_path);
err::check_raise(e, "load model failed");
log::info("load yolo11 model %s success", model_path);
if (argc >= 3)
{
const char *img_path = argv[2];
log::info("load image now");
maix::image::Image *img = maix::image::load(img_path, img_fmt);
err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
log::info("load image %s success: %s", img_path, img->to_str().c_str());
if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
{
log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
// we resize here manually, if not, the detect function will auto resize.
maix::image::Image *img2 = img->resize(detector.input_size().width(), detector.input_size().height(), image::Fit::FIT_CONTAIN);
delete img;
img = img2;
}
log::info("detect now");
nn::Objects *result = nullptr;
if(repeat_times > 1)
{
for(int i = 0; i < 3; ++i) // warmup
{
result = detector.detect(*img, conf_threshold, iou_threshold);
delete result;
}
}
std::vector<int> intervals(repeat_times);
for(int i = 0; i < repeat_times; ++i) // average time
{
uint64_t start = time::ticks_ms();
result = detector.detect(*img, conf_threshold, iou_threshold);
intervals[i] = time::ticks_ms() - start;
if(i < repeat_times - 1)
delete result;
}
float sum = 0;
for(auto i : intervals)
{
sum += i;
}
log::info("run %d times, average time: %dms", repeat_times, (int)(sum / repeat_times));
if(result->size() == 0)
{
log::info("no object detected !");
}
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
auto points = r->get_obb_points();
snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f, %.1f", detector.labels[r->class_id].c_str(), r->score, r->angle * 180.0);
img->draw_string(points[0], points[1] - 4, tmp_chars, maix::image::Color::from_rgb(255, 0, 0));
detector.draw_pose(*img, points, detector.input_width() > 480 ? 3 : 1, image::COLOR_RED, std::vector<image::Color>(), false, true);
}
img->save("result.jpg");
delete result;
delete img;
}
else
{
log::info("open camera now");
maix::image::Size input_size = detector.input_size();
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
while (!app::need_exit())
{
uint64_t t = time::ticks_ms();
maix::image::Image *img = cam.read();
err::check_null_raise(img, "read camera failed");
uint64_t t2 = time::ticks_ms();
nn::Objects *result = detector.detect(*img);
uint64_t t3 = time::ticks_ms();
for (auto &r : *result)
{
log::info("result: %s", r->to_str().c_str());
img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
}
disp.show(*img);
delete result;
delete img;
log::info("time: all %d ms, detect %d ms, fps: %.1f", time::ticks_ms() - t, t3 - t2, time::fps());
}
}
log::info("Program exit");
return ret;
}
int main(int argc, char *argv[])
{
// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
signal(SIGINT, [](int sig)
{ app::set_exit_flag(true); });
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}