diff --git a/components/basic/include/maix_tensor.hpp b/components/basic/include/maix_tensor.hpp index 67782673..8ae9643a 100644 --- a/components/basic/include/maix_tensor.hpp +++ b/components/basic/include/maix_tensor.hpp @@ -519,6 +519,7 @@ namespace maix tensors[key] = tensor; _auto_delete[key] = auto_delete; } + _keys.push_back(key); } /** @@ -527,6 +528,22 @@ namespace maix */ void rm_tensor(const std::string &key) { + size_t i = 0; + for(; i < _keys.size(); ++i) + { + auto &_key = _keys[i]; + if(_key == key) + { + break; + } + } + if(i == _keys.size()) + { + log::warn("rm_tensor: key %s not in tensor", key.c_str()); + return; + } + _keys.erase(_keys.begin() + i); + if(_auto_delete[key]) { delete tensors[key]; @@ -540,11 +557,11 @@ namespace maix */ void clear() { - auto _keys = keys(); for(const auto &k : _keys) { rm_tensor(k); } + _keys.clear(); } /** @@ -594,6 +611,25 @@ namespace maix return *tensors[key]; } + /** + * Operator [] + * @maixcdk maix.tensor.Tensors.operator[] + */ + tensor::Tensor &operator[](int idx) + { + int i = 0; + for(auto &key : _keys) + { + if(i == idx) + { + return *tensors[key]; + } + ++i; + } + log::error("%s: %d", "get tensor idx error", idx); + throw err::Exception(err::ERR_ARGS, "Tensors get tensor idx error"); + } + /** * Size * @maixpy maix.tensor.Tensors.__len__ @@ -610,12 +646,7 @@ namespace maix */ std::vector keys() { - std::vector names; - for(auto &item : tensors) - { - names.push_back(item.first); - } - return names; + return _keys; } public: @@ -626,6 +657,7 @@ namespace maix std::map tensors; private: std::map _auto_delete; + std::vector _keys; }; diff --git a/components/maixcam_lib/CMakeLists.txt b/components/maixcam_lib/CMakeLists.txt index 26a1d95c..dbdc3d3e 100644 --- a/components/maixcam_lib/CMakeLists.txt +++ b/components/maixcam_lib/CMakeLists.txt @@ -36,11 +36,11 @@ elseif(PLATFORM_MAIXCAM2) list(APPEND ADD_INCLUDE "../vision/include") list(APPEND ADD_INCLUDE "../nn/include") - set(source_dir "maixcdk_maixcam_lib") - append_srcs_dir(ADD_SRCS "${source_dir}/maixcam2") - append_srcs_dir(ADD_SRCS "${source_dir}/common") if(CONFIG_MAIXCAM_LIB_COMPILE_FROM_SOURCE) + set(source_dir "maixcdk_maixcam_lib") + append_srcs_dir(ADD_SRCS "${source_dir}/maixcam2") + append_srcs_dir(ADD_SRCS "${source_dir}/common") register_component(DYNAMIC) else() list(APPEND ADD_DYNAMIC_LIB "lib/libmaixcam2_lib.so") diff --git a/components/nn/CMakeLists.txt b/components/nn/CMakeLists.txt index 864ba2a9..e3b1f72b 100644 --- a/components/nn/CMakeLists.txt +++ b/components/nn/CMakeLists.txt @@ -11,6 +11,7 @@ if(PLATFORM_MAIXCAM) list(APPEND ADD_DEFINITIONS -DDR_WAV_IMPLEMENTATION) else() append_srcs_dir(ADD_SRCS "port/linux") + list(APPEND ADD_DEFINITIONS -fopenmp) endif() register_component() diff --git a/components/nn/include/maix_nn.hpp b/components/nn/include/maix_nn.hpp index f71e7bb8..96b1fdb8 100644 --- a/components/nn/include/maix_nn.hpp +++ b/components/nn/include/maix_nn.hpp @@ -375,7 +375,8 @@ namespace maix::nn * @param copy_result If set true, will copy result to a new variable; else will use a internal memory, you can only use it until to the next forward. * Default true to avoid problems, you can set it to false manually to make speed faster. * @param dual_buff_wait bool type, only for dual_buff mode, if true, will inference this image and wait for result, default false. - * @param chw chw channel format, forward model with hwc format image input if set to false, default true(chw). + * @param chw !!depracated!! This arg will be ignored!!! Please set extra.input_layout in mud file instead. + * chw channel format, forward model with hwc format image input if set to false, default true(chw). * @return output tensor. In C++, you should manually delete tensors in return value and return value. * If dual_buff mode, it can be NULL(None in MaixPy) means not ready. * @throw If error occurs, like arg error or alloc memory failed, will raise err.Exception. diff --git a/components/nn/include/maix_nn_classifier.hpp b/components/nn/include/maix_nn_classifier.hpp index caae30df..59536f77 100644 --- a/components/nn/include/maix_nn_classifier.hpp +++ b/components/nn/include/maix_nn_classifier.hpp @@ -34,7 +34,6 @@ namespace maix::nn { _model = nullptr; _dual_buff = dual_buff; - _chw = true; if (!model.empty()) { err::Err e = load(model); @@ -163,12 +162,6 @@ namespace maix::nn log::error("scale key not found"); return err::ERR_ARGS; } - if (_extra_info.find("input_channel") != _extra_info.end()) - { - std::string channel_str = _extra_info["input_channel"]; - if(channel_str == "hwc") - _chw = false; - } err::Err e = _model->extra_info_labels(labels); if(e == err::Err::ERR_NONE) { @@ -180,10 +173,10 @@ namespace maix::nn return err::ERR_ARGS; } _inputs = _model->inputs_info(); - if(_chw) - _input_size = image::Size(_inputs[0].shape[3], _inputs[0].shape[2]); + if(inputs[0].shape[3] <= 4) // nhwc + _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]); else - _input_size = image::Size(_inputs[0].shape[2], _inputs[0].shape[1]); + _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); return err::ERR_NONE; } @@ -226,7 +219,7 @@ namespace maix::nn throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); } tensor::Tensors *outputs; - outputs = _model->forward_image(img, this->mean, this->scale, fit, false, false, _chw); + outputs = _model->forward_image(img, this->mean, this->scale, fit, false, false); if (!outputs) { std::vector> *res = new std::vector>(1); @@ -348,7 +341,6 @@ namespace maix::nn image::Format _input_img_fmt; bool _input_is_img; bool _dual_buff; - bool _chw; nn::NN *_model; std::map _extra_info; image::Size _input_size; diff --git a/components/nn/include/maix_nn_yolo11.hpp b/components/nn/include/maix_nn_yolo11.hpp index 38e7a394..075694d7 100644 --- a/components/nn/include/maix_nn_yolo11.hpp +++ b/components/nn/include/maix_nn_yolo11.hpp @@ -12,6 +12,7 @@ #include "maix_nn_F.hpp" #include "maix_nn_object.hpp" #include +#include namespace maix::nn { @@ -36,6 +37,30 @@ namespace maix::nn float stride; }; + class _OutIdxes + { + public: + // mode 1 + int det0; // 1x144x80x80 + int det1; // 1x144x40x40 + int det2; // 1x144x20x20 + + // mode 2 + int dfl; // 1x1x4x8400 + int sigmoid; // 1x80x8400 + + // optional node + // seg mask 1x32x160x160 + // seg mask weight 1x32x8400 + int seg_mask; + int seg_mask_weight; + + // obb 1x1x8400 + int obb_angle; + // pose 1x51x8400 + int pose; + }; + /** * YOLO11 class * @maixpy maix.nn.YOLO11 @@ -215,7 +240,7 @@ namespace maix::nn } else if (_extra_info["type"] != "detector") { - log::error("type [%s] not support, suport detector and pose", _extra_info["type"].c_str()); + log::error("type [%s] not support, suport [detector, pose, seg, obb]", _extra_info["type"].c_str()); return err::ERR_ARGS; } } @@ -224,8 +249,167 @@ namespace maix::nn _type = YOLO11_Type::DETECT; } std::vector inputs = _model->inputs_info(); - _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); + if(inputs[0].shape[3] <= 4) // nhwc + _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]); + else + _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height()); + + // output info + _out_chw = true; + int _anchor_num = 0; + for(size_t i=0; i < _stride.size(); ++i) + { + _anchor_num += _input_size.width()/_stride[i] * _input_size.height() / _stride[i]; + } + std::vector outputs = _model->outputs_info(); + auto print_outputs = [outputs](){ + log::info("Outputs:"); + for(auto item : outputs) + { + log::print(" %s\n", item.to_str().c_str()); + } + }; + // mode 1 + if((outputs.size() == 5 && _type == YOLO11_Type::SEG) || + (outputs.size() == 4 && (_type == YOLO11_Type::OBB || _type == YOLO11_Type::POSE)) || + (outputs.size() == 3 && _type == YOLO11_Type::DETECT)) + { + _out_node_mode = 1; + int ch_num = labels.size() + _reg_max * 4; + for(size_t i = 0; i < outputs.size(); ++i) + { + nn::LayerInfo &item = outputs[i]; + if(item.shape.size() == 4) + { + if(item.shape[3] == ch_num && item.shape[1] == _input_size.height() / _stride[0] + && item.shape[2] == _input_size.width() / _stride[0]) // hwc + { + _out_chw = false; + _out_idxes.det0 = i; + break; + } + } + } + for(size_t i = 0; i < outputs.size(); ++i) + { + nn::LayerInfo &item = outputs[i]; + if(item.shape.size() == 4) + { + if(_type == YOLO11_Type::SEG && + ((item.shape[1] == 32 && // 1 32 160 160 + item.shape[2] == _input_size.height() / 4 && + item.shape[3] == _input_size.width() / 4) || // 1 160 160 32 + (item.shape[3] == 32 && + item.shape[1] == _input_size.height() / 4 && + item.shape[2] == _input_size.width() / 4) + )) + { + _out_idxes.seg_mask = i; + } + else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[0]) + { + _out_idxes.det0 = i; + } + else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[1]) + { + _out_idxes.det1 = i; + } + else if(item.shape[_out_chw ? 2 : 1] == _input_size.height() / _stride[2]) + { + _out_idxes.det2 = i; + } + else + { + log::error("output node shape error, please check %d, chw: %d, %d", 1, _out_chw, item.shape[_out_chw ? 2 : 1]); + print_outputs(); + return err::ERR_ARGS; + } + } + else if(item.shape.size() == 3) + { + if (_type == YOLO11_Type::SEG) + _out_idxes.seg_mask_weight = i; + else if (_type == YOLO11_Type::OBB) + { + if(item.name.find("Sigmoid") == std::string::npos) + _obb_need_sigmoid = true; + else + _obb_need_sigmoid = false; + _out_idxes.obb_angle = i; + } + else if (_type == YOLO11_Type::POSE) + _out_idxes.pose = i; + else + { + log::error("output node shape error, please check %d", 2); + print_outputs(); + return err::ERR_ARGS; + } + } + else + { + log::error("output node shape error, please check %d", 3); + print_outputs(); + return err::ERR_ARGS; + } + } + } + else // mode 2 + { + _out_node_mode = 2; + for(int i = 0; i < (int)outputs.size(); ++i) + { + nn::LayerInfo &item = outputs[i]; + if(item.shape.size() == 4) + { + if(item.shape[3] == 4 && item.shape[2] == _anchor_num && item.shape[1] == 1) // hwc + { + _out_chw = false; + _out_idxes.dfl = i; + break; + } + + } + } + for(int i = 0; i < (int)outputs.size(); ++i) + { + nn::LayerInfo &item = outputs[i]; + if(i == _out_idxes.dfl) + continue; + if(item.shape.size() == 4) + { + _out_idxes.seg_mask = i; + continue; + } + if(_type == YOLO11_Type::DETECT && item.shape[_out_chw ? 1 : 2] == (int)labels.size() && item.shape[_out_chw ? 2 : 1] == _anchor_num) + { + _out_idxes.sigmoid = i; + } + else if(_type == YOLO11_Type::SEG && item.shape[_out_chw ? 1 : 2] == 32) + { + _out_idxes.seg_mask_weight = i; + } + else if(_type == YOLO11_Type::OBB) + { + if(item.name.find("Sigmoid") == std::string::npos) + _obb_need_sigmoid = true; + else + _obb_need_sigmoid = false; + _out_idxes.obb_angle = i; + } + else if(_type == YOLO11_Type::POSE && (item.shape[_out_chw ? 1 : 2] % 3 == 0)&& item.shape[_out_chw ? 2 : 1] == _anchor_num) + { + _out_idxes.pose = i; + } + else + { + log::error("output node shape error, please check %d", 4); + print_outputs(); + return err::ERR_ARGS; + } + } + } return err::ERR_NONE; } @@ -244,6 +428,7 @@ namespace maix::nn */ nn::Objects *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, float keypoint_th = 0.5, int sort = 0) { + #define SHOW_DETECT_TIME 1 this->_conf_th = conf_th; this->_iou_th = iou_th; this->_keypoint_th = keypoint_th; @@ -252,12 +437,22 @@ namespace maix::nn throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); } tensor::Tensors *outputs; +#if SHOW_DETECT_TIME + uint64_t start = time::ticks_ms(); +#endif outputs = _model->forward_image(img, this->mean, this->scale, fit, false); +#if SHOW_DETECT_TIME + log::info("forward time: %ld", time::ticks_ms() - start); + start = time::ticks_ms(); +#endif if (!outputs) // not ready, return empty result. { return new nn::Objects(); } nn::Objects *res = _post_process(outputs, img.width(), img.height(), fit, sort); +#if SHOW_DETECT_TIME + log::info("postprocess time: %ld", time::ticks_ms() - start); +#endif delete outputs; if(!res) { @@ -447,6 +642,13 @@ namespace maix::nn float _keypoint_th = 0.5; YOLO11_Type _type; bool _dual_buff; + _OutIdxes _out_idxes; + int _out_node_mode; + bool _out_chw; + int _reg_max = 16; + std::vector _stride = {8, 16, 32}; + int _anchor_num; + bool _obb_need_sigmoid; private: err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) @@ -513,113 +715,255 @@ namespace maix::nn bool _decode_objs(nn::Objects &objs, tensor::Tensors *outputs, float conf_thresh, int w, int h, tensor::Tensor **kp_out, tensor::Tensor **mask_out) { - float stride[3] = {8, 16, 32}; - tensor::Tensor *score_out = NULL; // shape 1, 80, 8400, 1 - tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400 - for (auto i : *outputs) + if(_type == YOLO11_Type::SEG) { - if (i.second->shape()[2] == 4 && !box_out) + *mask_out = &(*outputs)[_out_idxes.seg_mask]; + *kp_out = &(*outputs)[_out_idxes.seg_mask_weight]; + } + else if (_type == YOLO11_Type::OBB) + { + *kp_out = &(*outputs)[_out_idxes.obb_angle]; + } + else if (_type == YOLO11_Type::POSE) + { + *kp_out = &(*outputs)[_out_idxes.pose]; + } + int idx_start[3] = { + 0, + (int)(h / _stride[0] * w / _stride[0]), + (int)(h / _stride[0] * w / _stride[0] + h / _stride[1] * w / _stride[1])}; + // detect + if(_out_node_mode == 1) + { + tensor::Tensor *dets[3] = {&(*outputs)[_out_idxes.det0], &(*outputs)[_out_idxes.det1], &(*outputs)[_out_idxes.det2]}; + if (_type == YOLO11_Type::OBB) { - box_out = i.second; - } - else if (strstr(i.first.c_str(), "Sigmoid") != NULL && !score_out) - { - score_out = i.second; - if((size_t)score_out->shape()[1] != labels.size()) + if(_out_chw) { - log::error("MUD labels(%d) must equal model's(%d)", score_out->shape()[1], labels.size()); - return false; + throw err::Exception(err::ERR_NOT_IMPL, "not support output chw layout"); + } + else + { + int class_num = (int)labels.size(); + int prob_offset = _reg_max * 4; + int batch_data_size = prob_offset + class_num; + float *angle_ptr = (float *)(*kp_out)->data(); + for(int i = 0; i < 3; ++i) // 1 x nh x nw x (_reg_max * 4 + class_num) + { + int nh = dets[i]->shape()[1]; + int nw = dets[i]->shape()[2]; + int anchor_num = nh * nw; + float *feature = (float*)dets[i]->data(); + for (int anchor_idx = 0; anchor_idx < anchor_num; ++anchor_idx) + { + int ax = anchor_idx % nw; + int ay = anchor_idx / nw; + int offset = idx_start[i] + anchor_idx; + float *p = feature + batch_data_size * anchor_idx; + int class_id = _argmax(p + prob_offset, class_num, 1); + float score = _sigmoid(p[prob_offset + class_id]); + if (score <= conf_thresh) + { + continue; + } + float dis[4]; + for(int k = 0; k < 4; ++k) + { + // std::vector softmax_res(_reg_max, 0.f); + float softmax_res[_reg_max]; + dis[k] = _softmax(p + k * _reg_max, softmax_res, _reg_max); + } + float lt_x = dis[0]; + float lt_y = dis[1]; + float rb_x = dis[2]; + float rb_y = dis[3]; + float angle = ((_obb_need_sigmoid ? _sigmoid(angle_ptr[offset]) : angle_ptr[offset]) - 0.25); + float angle_rad = angle * M_PI; + float cos_angle = cosf(angle_rad); + float sin_angle = sinf(angle_rad); + float xf = (rb_x - lt_x) / 2.0; + float yf = (rb_y - lt_y) / 2.0; + float bbox_w = (lt_x + rb_x) * _stride[i]; + float bbox_h = (lt_y + rb_y) * _stride[i]; + float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * _stride[i] - bbox_w * 0.5; + float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * _stride[i] - bbox_h * 0.5; + _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]); + Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, score, {}, angle); + obj.temp = (void *)kp_info; + } + } } - } - else if (strstr(i.first.c_str(), "output1") != NULL) - { - *mask_out = i.second; } else { - *kp_out = i.second; - } - } - if (!score_out || !box_out) - { - throw err::Exception(err::ERR_ARGS, "model output not valid"); - } - int total_box_num = box_out->shape()[3]; - // int class_num = this->labels.size(); - int class_num = score_out->shape()[1]; - float *scores_ptr = (float *)score_out->data(); - float *dets_ptr = (float *)box_out->data(); - int idx_start[3] = { - 0, - (int)(h / stride[0] * w / stride[0]), - (int)(h / stride[0] * w / stride[0] + h / stride[1] * w / stride[1])}; - if (_type == YOLO11_Type::OBB) - { - float *angle_ptr = (float *)(*kp_out)->data(); - for (int i = 0; i < 3; i++) - { - int nh = h / stride[i]; - int nw = w / stride[i]; - for (int ay = 0; ay < nh; ++ay) + if(_out_chw) { - for (int ax = 0; ax < nw; ++ax) + throw err::Exception(err::ERR_NOT_IMPL, "not support output chw layout"); + } + else + { + int class_num = (int)labels.size(); + int prob_offset = _reg_max * 4; + int batch_data_size = prob_offset + class_num; + for(int i = 0; i < 3; ++i) // 1 x nh x nw x (_reg_max * 4 + class_num) { - int offset = idx_start[i] + ay * nw + ax; - int class_id = _argmax(scores_ptr + offset, class_num, total_box_num); - // int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num); - // int class_id = (offset + max_idx) / total_box_num; - float obj_score = scores_ptr[offset + class_id * total_box_num]; - if (obj_score <= conf_thresh) + int nh = dets[i]->shape()[1]; + int nw = dets[i]->shape()[2]; + int anchor_num = nh * nw; + float *feature = (float*)dets[i]->data(); + for (int anchor_idx = 0; anchor_idx < anchor_num; ++anchor_idx) { - continue; + int ax = anchor_idx % nw; + int ay = anchor_idx / nw; + float *p = feature + batch_data_size * anchor_idx; + int class_id = _argmax(p + prob_offset, class_num, 1); + float score = _sigmoid(p[prob_offset + class_id]); + if (score <= conf_thresh) + { + continue; + } + float dis[4]; + for(int k = 0; k < 4; ++k) + { + // std::vector softmax_res(_reg_max, 0.f); + float softmax_res[_reg_max]; + dis[k] = _softmax(p + k * _reg_max, softmax_res, _reg_max); + } + float bbox_x = (ax + 0.5 - dis[0]) * _stride[i]; + float bbox_y = (ay + 0.5 - dis[1]) * _stride[i]; + float bbox_w = (ax + 0.5 + dis[2]) * _stride[i] - bbox_x; + float bbox_h = (ay + 0.5 + dis[3]) * _stride[i] - bbox_y; + _KpInfoYolo11 *kp_info = new _KpInfoYolo11(idx_start[i] + anchor_idx, ax, ay, _stride[i]); + Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, score); + obj.temp = (void *)kp_info; } - float angle = (angle_ptr[offset] - 0.25); - float angle_rad = angle * M_PI; - float cos_angle = cosf(angle_rad); - float sin_angle = sinf(angle_rad); - float lt_x = dets_ptr[offset]; - float lt_y = dets_ptr[offset + total_box_num]; - float rb_x = dets_ptr[offset + total_box_num * 2]; - float rb_y = dets_ptr[offset + total_box_num * 3]; - float xf = (rb_x - lt_x) / 2.0; - float yf = (rb_y - lt_y) / 2.0; - float bbox_w = (lt_x + rb_x) * stride[i]; - float bbox_h = (lt_y + rb_y) * stride[i]; - float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * stride[i] - bbox_w * 0.5; - float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * stride[i] - bbox_h * 0.5; - _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, stride[i]); - Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle); - obj.temp = (void *)kp_info; } } } } - else + else // mode 2 { - for (int i = 0; i < 3; i++) + tensor::Tensor *score_out = NULL; // shape 1, 80, 8400 hwc: 1, 8400, 80 + tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400 hwc: 1, 1, 8400, 4 + score_out = &(*outputs)[_out_idxes.sigmoid]; + box_out = &(*outputs)[_out_idxes.dfl]; + int class_num = (int)labels.size(); + float *scores_ptr = (float *)score_out->data(); + float *dets_ptr = (float *)box_out->data(); + if (_type == YOLO11_Type::OBB) { - int nh = h / stride[i]; - int nw = w / stride[i]; - for (int ay = 0; ay < nh; ++ay) + if(!_out_chw) { - for (int ax = 0; ax < nw; ++ax) + throw err::Exception(err::ERR_NOT_IMPL, "not support output hwc layout"); + } + float *angle_ptr = (float *)(*kp_out)->data(); + for (int i = 0; i < 3; i++) + { + int nh = h / _stride[i]; + int nw = w / _stride[i]; + for (int ay = 0; ay < nh; ++ay) { - int offset = idx_start[i] + ay * nw + ax; - int class_id = _argmax(scores_ptr + offset, class_num, total_box_num); - // int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num); - // int class_id = (offset + max_idx) / total_box_num; - float obj_score = scores_ptr[offset + class_id * total_box_num]; - if (obj_score <= conf_thresh) + for (int ax = 0; ax < nw; ++ax) { - continue; + int offset = idx_start[i] + ay * nw + ax; + int class_id = _argmax(scores_ptr + offset, class_num, _anchor_num); + // int max_idx = _argmax2(scores_ptr + offset, class_num * _anchor_num - offset, _anchor_num); + // int class_id = (offset + max_idx) / _anchor_num; + float obj_score = scores_ptr[offset + class_id * _anchor_num]; + if (obj_score <= conf_thresh) + { + continue; + } + float angle = ((_obb_need_sigmoid ? _sigmoid(angle_ptr[offset]) : angle_ptr[offset]) - 0.25); + float angle_rad = angle * M_PI; + float cos_angle = cosf(angle_rad); + float sin_angle = sinf(angle_rad); + float lt_x = dets_ptr[offset]; + float lt_y = dets_ptr[offset + _anchor_num]; + float rb_x = dets_ptr[offset + _anchor_num * 2]; + float rb_y = dets_ptr[offset + _anchor_num * 3]; + float xf = (rb_x - lt_x) / 2.0; + float yf = (rb_y - lt_y) / 2.0; + float bbox_w = (lt_x + rb_x) * _stride[i]; + float bbox_h = (lt_y + rb_y) * _stride[i]; + float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * _stride[i] - bbox_w * 0.5; + float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * _stride[i] - bbox_h * 0.5; + _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]); + Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle); + obj.temp = (void *)kp_info; + } + } + } + } + else + { + if(_out_chw) + { + for (int i = 0; i < 3; i++) + { + int nh = h / _stride[i]; + int nw = w / _stride[i]; + for (int ay = 0; ay < nh; ++ay) + { + for (int ax = 0; ax < nw; ++ax) + { + int offset = idx_start[i] + ay * nw + ax; + int class_id = _argmax(scores_ptr + offset, class_num, _anchor_num); + // int max_idx = _argmax2(scores_ptr + offset, class_num * _anchor_num - offset, _anchor_num); + // int class_id = (offset + max_idx) / _anchor_num; + float obj_score = scores_ptr[offset + class_id * _anchor_num]; + if (obj_score <= conf_thresh) + { + continue; + } + float bbox_x = (ax + 0.5 - dets_ptr[offset]) * _stride[i]; + float bbox_y = (ay + 0.5 - dets_ptr[offset + _anchor_num]) * _stride[i]; + float bbox_w = (ax + 0.5 + dets_ptr[offset + _anchor_num * 2]) * _stride[i] - bbox_x; + float bbox_h = (ay + 0.5 + dets_ptr[offset + _anchor_num * 3]) * _stride[i] - bbox_y; + _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]); + Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score); + obj.temp = (void *)kp_info; + } + } + } + } + else + { + typedef struct + { + float bbox_x; + float bbox_y; + float bbox_w; + float bbox_h; + int class_id; + float score; + } valid_item_t; + for (int i = 0; i < 3; i++) + { + int nh = h / _stride[i]; + int nw = w / _stride[i]; + for (int ay = 0; ay < nh; ++ay) + { + for (int ax = 0; ax < nw; ++ax) + { + valid_item_t item; + int offset = idx_start[i] + ay * nw + ax; + float *p = scores_ptr + offset * class_num; + item.class_id = _argmax(p, class_num, 1); + item.score = p[item.class_id]; + if (item.score <= conf_thresh) + { + continue; + } + item.bbox_x = (ax + 0.5 - dets_ptr[offset * 4]) * _stride[i]; + item.bbox_y = (ay + 0.5 - dets_ptr[offset * 4 + 1]) * _stride[i]; + item.bbox_w = (ax + 0.5 + dets_ptr[offset * 4 + 2]) * _stride[i] - item.bbox_x; + item.bbox_h = (ay + 0.5 + dets_ptr[offset * 4 + 3]) * _stride[i] - item.bbox_y; + _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, _stride[i]); + Object &obj = objs.add(item.bbox_x, item.bbox_y, item.bbox_w, item.bbox_h, item.class_id, item.score); + obj.temp = (void *)kp_info; + } } - float bbox_x = (ax + 0.5 - dets_ptr[offset]) * stride[i]; - float bbox_y = (ay + 0.5 - dets_ptr[offset + total_box_num]) * stride[i]; - float bbox_w = (ax + 0.5 + dets_ptr[offset + total_box_num * 2]) * stride[i] - bbox_x; - float bbox_h = (ay + 0.5 + dets_ptr[offset + total_box_num * 3]) * stride[i] - bbox_y; - _KpInfoYolo11 *kp_info = new _KpInfoYolo11(offset, ax, ay, stride[i]); - Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score); - obj.temp = (void *)kp_info; } } } @@ -695,82 +1039,193 @@ namespace maix::nn void _decode_keypoints(nn::Objects &objs, tensor::Tensor *kp_out) { float *data = (float *)kp_out->data(); - int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400, 1 - int total_box_num = kp_out->shape()[2]; // 1, 51, 8400, 1 - for (size_t i = 0; i < objs.size(); ++i) + if(_out_chw) { - nn::Object &o = objs.at(i); - _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; - float *p = data + kp_info->idx; - for (int k = 0; k < keypoint_num; ++k) + int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400 + for (size_t i = 0; i < objs.size(); ++i) { - float score = _sigmoid(p[(k * 3 + 2) * total_box_num]); - int x = -1; - int y = -1; - if (score > _keypoint_th) + nn::Object &o = objs.at(i); + _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; + float *p = data + kp_info->idx; + for (int k = 0; k < keypoint_num; ++k) { - x = (p[(k * 3) * total_box_num] * 2.0 + kp_info->anchor_x) * kp_info->stride; - y = (p[(k * 3 + 1) * total_box_num] * 2.0 + kp_info->anchor_y) * kp_info->stride; + float score = _sigmoid(p[(k * 3 + 2) * _anchor_num]); + int x = -1; + int y = -1; + if (score > _keypoint_th) + { + x = (p[(k * 3) * _anchor_num] * 2.0 + kp_info->anchor_x) * kp_info->stride; + y = (p[(k * 3 + 1) * _anchor_num] * 2.0 + kp_info->anchor_y) * kp_info->stride; + } + o.points.push_back(x); + o.points.push_back(y); } - o.points.push_back(x); - o.points.push_back(y); + delete (_KpInfoYolo11 *)o.temp; + o.temp = NULL; + } + } + else + { + int keypoint_num = kp_out->shape()[2] / 3; // 1, 8400, 51 + log::info("%d", keypoint_num); + for (size_t i = 0; i < objs.size(); ++i) + { + nn::Object &o = objs.at(i); + _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; + float *p = data + kp_info->idx * kp_out->shape()[2]; + for (int k = 0; k < keypoint_num; ++k) + { + float score = _sigmoid(p[k * 3 + 2]); + int x = -1; + int y = -1; + if (score > _keypoint_th) + { + x = (p[k * 3] * 2.0 + kp_info->anchor_x) * kp_info->stride; + y = (p[k * 3 + 1] * 2.0 + kp_info->anchor_y) * kp_info->stride; + } + o.points.push_back(x); + o.points.push_back(y); + } + delete (_KpInfoYolo11 *)o.temp; + o.temp = NULL; } - delete (_KpInfoYolo11 *)o.temp; - o.temp = NULL; } } void _decode_seg_points(nn::Objects &objs, tensor::Tensor *kp_out, tensor::Tensor *mask_out) { float *data = (float *)kp_out->data(); - float *mask_data = (float *)mask_out->data(); // 1, 32, 160, 160 - int mask_h = mask_out->shape()[2]; - int mask_w = mask_out->shape()[3]; - int mask_squre = mask_h * mask_w; - int mask_num = kp_out->shape()[1]; // 1, 32, 8400, 1 - int total_box_num = kp_out->shape()[2]; // 1, 32, 8400, 1 - float mask_weights[mask_num]; - for (size_t i = 0; i < objs.size(); ++i) + float *mask_data = (float *)mask_out->data(); + if(_out_chw) { - nn::Object &o = objs.at(i); - int mask_x = o.x * mask_w / _input_size.width(); - int mask_y = o.y * mask_h / _input_size.height(); - int mask_x2 = (o.x + o.w) * mask_w / _input_size.width(); - int mask_y2 = (o.y + o.h) * mask_h / _input_size.height(); - _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; - float *p = data + kp_info->idx; - for (int k = 0; k < mask_num; ++k) + int mask_h = mask_out->shape()[2]; // 1, 32, 160, 160 + int mask_w = mask_out->shape()[3]; + int mask_squre = mask_h * mask_w; + int mask_num = kp_out->shape()[1]; // 1, 32, 8400 + float mask_weights[mask_num]; + for (size_t i = 0; i < objs.size(); ++i) { - mask_weights[k] = p[k * total_box_num]; - } - o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE); - uint8_t *p_img_data = (uint8_t *)o.seg_mask->data(); - for (int j = mask_y; j < mask_y2; ++j) - { - for (int k = mask_x; k < mask_x2; ++k) + nn::Object &o = objs.at(i); + int mask_x = o.x * mask_w / _input_size.width(); + int mask_y = o.y * mask_h / _input_size.height(); + int mask_x2 = (o.x + o.w) * mask_w / _input_size.width(); + int mask_y2 = (o.y + o.h) * mask_h / _input_size.height(); + _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; + float *p = data + kp_info->idx; + for (int k = 0; k < mask_num; ++k) { - mask_data[j * mask_w + k] *= mask_weights[0]; + mask_weights[k] = p[k * _anchor_num]; } - } - for (int n = 1; n < mask_num; ++n) - { + o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE); + uint8_t *p_img_data = (uint8_t *)o.seg_mask->data(); for (int j = mask_y; j < mask_y2; ++j) { for (int k = mask_x; k < mask_x2; ++k) { - mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k]; + mask_data[j * mask_w + k] *= mask_weights[0]; } } - } - for (int j = mask_y; j < mask_y2; ++j) - { - for (int k = mask_x; k < mask_x2; ++k) + for (int n = 1; n < mask_num; ++n) { - *p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255); + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k]; + } + } } + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + *p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255); + } + } + delete (_KpInfoYolo11 *)o.temp; + o.temp = NULL; + } + } + else + { + bool mask_chw = false; + int mask_h = _input_size.height() / 4; + int mask_w = _input_size.width() / 4; + int mask_num = 32; + int mask_squre = mask_h * mask_w; + // 1, 32, 160, 160 or 1 160 160 32 + if(mask_out->shape()[1] == mask_num && mask_out->shape()[2] == mask_h && mask_out->shape()[3] == mask_w) + { + mask_chw = true; + } + for (size_t i = 0; i < objs.size(); ++i) + { + nn::Object &o = objs.at(i); + int mask_x = o.x * mask_w / _input_size.width(); + int mask_y = o.y * mask_h / _input_size.height(); + int mask_x2 = (o.x + o.w) * mask_w / _input_size.width(); + int mask_y2 = (o.y + o.h) * mask_h / _input_size.height(); + _KpInfoYolo11 *kp_info = (_KpInfoYolo11 *)o.temp; + float *mask_weights = data + kp_info->idx * mask_num; //1 8400 32 + o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE); + uint8_t *p_img_data = (uint8_t *)o.seg_mask->data(); + if(mask_chw) // 1 32 160 160 + { + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + mask_data[j * mask_w + k] *= mask_weights[0]; + } + } + for (int n = 1; n < mask_num; ++n) + { + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k]; + } + } + } + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + *p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255); + } + } + } + else // 1 160 160 32 + { + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + mask_data[(j * mask_w + k) * mask_num] *= mask_weights[0]; + } + } + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + for (int n = 1; n < mask_num; ++n) + { + mask_data[(j * mask_w + k) * mask_num] += mask_weights[n] * mask_data[(j * mask_w + k) * mask_num + n]; + } + } + } + for (int j = mask_y; j < mask_y2; ++j) + { + for (int k = mask_x; k < mask_x2; ++k) + { + *p_img_data++ = (uint8_t)(_sigmoid(mask_data[(j * mask_w + k) * mask_num]) * 255); + } + } + } + delete (_KpInfoYolo11 *)o.temp; + o.temp = NULL; } - delete (_KpInfoYolo11 *)o.temp; - o.temp = NULL; } } @@ -989,6 +1444,23 @@ namespace maix::nn split0(tokens, s, delimiter); return tokens; } + static float _softmax(const float* src, float* dst, int length) + { + const float alpha = *std::max_element(src, src + length); + float denominator = 0; + float dis_sum = 0; + for (int i = 0; i < length; ++i) + { + dst[i] = exp(src[i] - alpha); + denominator += dst[i]; + } + for (int i = 0; i < length; ++i) + { + dst[i] /= denominator; + dis_sum += i * dst[i]; + } + return dis_sum; + } }; } // namespace maix::nn diff --git a/components/nn/include/maix_nn_yolov5.hpp b/components/nn/include/maix_nn_yolov5.hpp index 27f88c0c..2c8bfe1e 100644 --- a/components/nn/include/maix_nn_yolov5.hpp +++ b/components/nn/include/maix_nn_yolov5.hpp @@ -203,7 +203,10 @@ namespace maix::nn return err::ERR_ARGS; } std::vector inputs = _model->inputs_info(); - _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); + if(inputs[0].shape[3] <= 4) // nhwc + _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]); + else + _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height()); return err::ERR_NONE; } diff --git a/components/nn/include/maix_nn_yolov8.hpp b/components/nn/include/maix_nn_yolov8.hpp index 03000da1..94c27192 100644 --- a/components/nn/include/maix_nn_yolov8.hpp +++ b/components/nn/include/maix_nn_yolov8.hpp @@ -12,35 +12,15 @@ #include "maix_nn_F.hpp" #include "maix_nn_object.hpp" #include +#include "maix_nn_yolo11.hpp" namespace maix::nn { - enum class YOLOv8_Type - { - DETECT = 0, - POSE = 1, - SEG = 2, - OBB = 3 - }; - - class _KpInfo - { - public: - _KpInfo(int idx, int ax, int ay, float stride) - : idx(idx), anchor_x(ax), anchor_y(ay), stride(stride) - { - } - int idx; - int anchor_x; - int anchor_y; - float stride; - }; - /** * YOLOv8 class * @maixpy maix.nn.YOLOv8 */ - class YOLOv8 + class YOLOv8 : public YOLO11 { public: /** @@ -54,32 +34,9 @@ namespace maix::nn * @maixcdk maix.nn.YOLOv8.YOLOv8 */ YOLOv8(const string &model = "", bool dual_buff = true) + : YOLO11(model, dual_buff) { - _model = nullptr; - _type = YOLOv8_Type::DETECT; - _dual_buff = dual_buff; - if (!model.empty()) - { - err::Err e = load(model); - if (e != err::ERR_NONE) - { - throw err::Exception(e, "load model failed"); - } - } - } - - ~YOLOv8() - { - if (_model) - { - delete _model; - _model = nullptr; - } - } - - std::string type() - { - return _extra_info["type"]; + type_str = "yolov8" } /** @@ -90,143 +47,7 @@ namespace maix::nn */ err::Err load(const string &model) { - if (_model) - { - delete _model; - _model = nullptr; - } - _model = new nn::NN(model, _dual_buff); - if (!_model) - { - return err::ERR_NO_MEM; - } - _extra_info = _model->extra_info(); - if (_extra_info.find("model_type") != _extra_info.end()) - { - if (_extra_info["model_type"] != this->type_str) - { - log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str()); - return err::ERR_ARGS; - } - } - else - { - log::error("model_type key not found"); - return err::ERR_ARGS; - } - log::info("model info:\n\ttype: %s", this->type_str.c_str()); - if (_extra_info.find("input_type") != _extra_info.end()) - { - std::string input_type = _extra_info["input_type"]; - if (input_type == "rgb") - { - _input_img_fmt = maix::image::FMT_RGB888; - log::print("\tinput type: rgb\n"); - } - else if (input_type == "bgr") - { - _input_img_fmt = maix::image::FMT_BGR888; - log::print("\tinput type: bgr\n"); - } - else - { - log::error("unknown input type: %s", input_type.c_str()); - return err::ERR_ARGS; - } - } - else - { - log::error("input_type key not found"); - return err::ERR_ARGS; - } - if (_extra_info.find("mean") != _extra_info.end()) - { - std::string mean_str = _extra_info["mean"]; - std::vector mean_strs = split(mean_str, ","); - log::print("\tmean:"); - for (auto &it : mean_strs) - { - try - { - this->mean.push_back(std::stof(it)); - } - catch (std::exception &e) - { - log::error("mean value error, should float"); - return err::ERR_ARGS; - } - log::print("%f ", this->mean.back()); - } - log::print("\n"); - } - else - { - log::error("mean key not found"); - return err::ERR_ARGS; - } - if (_extra_info.find("scale") != _extra_info.end()) - { - std::string scale_str = _extra_info["scale"]; - std::vector scale_strs = split(scale_str, ","); - log::print("\tscale:"); - for (auto &it : scale_strs) - { - try - { - this->scale.push_back(std::stof(it)); - } - catch (std::exception &e) - { - log::error("scale value error, should float"); - return err::ERR_ARGS; - } - log::print("%f ", this->scale.back()); - } - log::print("\n"); - } - else - { - log::error("scale key not found"); - return err::ERR_ARGS; - } - err::Err e = _model->extra_info_labels(labels); - if(e == err::Err::ERR_NONE) - { - log::print("\tlabels num: %ld\n", labels.size()); - } - else - { - log::error("labels key not found: %s", err::to_str(e).c_str()); - return err::ERR_ARGS; - } - if (_extra_info.find("type") != _extra_info.end()) - { - if (_extra_info["type"] == "pose") - { - _type = YOLOv8_Type::POSE; - } - else if (_extra_info["type"] == "seg") - { - _type = YOLOv8_Type::SEG; - } - else if (_extra_info["type"] == "obb") - { - _type = YOLOv8_Type::OBB; - } - else if (_extra_info["type"] != "detector") - { - log::error("type [%s] not support, suport detector and pose", _extra_info["type"].c_str()); - return err::ERR_ARGS; - } - } - else - { - _type = YOLOv8_Type::DETECT; - } - std::vector inputs = _model->inputs_info(); - _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); - log::print("\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height()); - return err::ERR_NONE; + return YOLO11::load(model); } /** @@ -244,26 +65,7 @@ namespace maix::nn */ nn::Objects *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, float keypoint_th = 0.5, int sort = 0) { - this->_conf_th = conf_th; - this->_iou_th = iou_th; - this->_keypoint_th = keypoint_th; - if (img.format() != _input_img_fmt) - { - throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); - } - tensor::Tensors *outputs; - outputs = _model->forward_image(img, this->mean, this->scale, fit, false); - if (!outputs) // not ready, return empty result. - { - return new nn::Objects(); - } - nn::Objects *res = _post_process(outputs, img.width(), img.height(), fit, sort); - delete outputs; - if(!res) - { - throw err::Exception("post process failed, please see log before"); - } - return res; + return YOLO11::detect(img, conf_th, iou_th, fit, keypoint_th, sort); } /** @@ -273,7 +75,7 @@ namespace maix::nn */ image::Size input_size() { - return _input_size; + return YOLO11::input_size(); } /** @@ -283,7 +85,7 @@ namespace maix::nn */ int input_width() { - return _input_size.width(); + return YOLO11::input_width(); } /** @@ -293,7 +95,7 @@ namespace maix::nn */ int input_height() { - return _input_size.height(); + return YOLO11::input_height(); } /** @@ -303,7 +105,7 @@ namespace maix::nn */ image::Format input_format() { - return _input_img_fmt; + return YOLO11::input_format(); } /** @@ -319,57 +121,7 @@ namespace maix::nn */ void draw_pose(image::Image &img, std::vector points, int radius = 4, image::Color color = image::COLOR_RED, const std::vector &colors = std::vector(), bool body = true, bool close = false) { - bool line_drawed = false; - if (points.size() < 2 || points.size() % 2 != 0) - { - throw std::runtime_error("keypoints size must >= 2 and multiple of 2"); - return; - } - if (points.size() == 17 * 2 && body) - { - int pos[] = {9, 7, 7, 5, 6, 8, 8, 10, 5, 11, 6, 12, 5, 6, 11, 12, 11, 13, 15, 13, 14, 12, 14, 16}; - for (int i = 0; i < 12; ++i) - { - int x1 = points[pos[i * 2] * 2]; - int y1 = points[pos[i * 2] * 2 + 1]; - int x2 = points[pos[i * 2 + 1] * 2]; - int y2 = points[pos[i * 2 + 1] * 2 + 1]; - if (x1 < 0 || y1 < 0 || x2 < 0 || y2 < 0) - continue; - img.draw_line(x1, y1, x2, y2, color, 2); - } - int x = (points[5 * 2] + points[6 * 2]) / 2; - int y = (points[5 * 2 + 1] + points[6 * 2 + 1]) / 2; - if (!(points[5 * 2] < 0 || points[5 * 2 + 1] < 0 || points[6 * 2] < 0 || points[6 * 2 + 1] < 0 || x < 0 || y < 0 || points[0] < 0 || points[1] < 0)) - img.draw_line(points[0], points[1], x, y, color, 2); - line_drawed = true; - } - for (size_t i = 0; i < points.size() / 2; ++i) - { - int x = points[i * 2]; - int y = points[i * 2 + 1]; - if (x < 0 || y < 0) - continue; - auto &_color = color; - if (colors.size() > i) - { - _color = colors[i]; - } - img.draw_circle(x, y, radius, _color, -1); - } - if(close && !line_drawed) - { - for (size_t i = 0; i < points.size() / 2; ++i) - { - int x1 = points[i * 2]; - int y1 = points[i * 2 + 1]; - int x2 = points[(i + 1) % (points.size() / 2) * 2]; - int y2 = points[(i + 1) % (points.size() / 2) * 2 + 1]; - if (x1 < 0 || y1 < 0 || x2 < 0 || y2 < 0) - continue; - img.draw_line(x1, y1, x2, y2, color, 2); - } - } + YOLO11::draw_pose(img, points, radius, color, colors, body,close); } /** @@ -381,32 +133,7 @@ namespace maix::nn */ void draw_seg_mask(image::Image &img, int x, int y, image::Image &seg_mask, int threshold = 127) { - if (seg_mask.format() != image::FMT_GRAYSCALE) - { - throw err::Exception(err::ERR_ARGS, "seg_mask only support grascale"); - } - if (img.format() != image::FMT_RGB888 && - img.format() != image::FMT_BGR888 && - img.format() != image::FMT_RGBA8888 && - img.format() != image::FMT_BGRA8888 && - img.format() != image::FMT_GRAYSCALE) - { - throw err::Exception(err::ERR_ARGS, "img not support"); - } - int fmt_size = image::fmt_size[img.format()]; - uint8_t *to = (uint8_t*)img.data(); - uint8_t *from = (uint8_t*)seg_mask.data(); - for (int i = 0; i < seg_mask.height(); ++i) - { - for (int j = 0; j < seg_mask.width(); ++j) - { - uint8_t data = from[i * seg_mask.width() + j]; - if(data > threshold) - { - to[((y + i) * img.width() + x + j) * fmt_size] = data; - } - } - } + YOLO11::draw_seg_mask(img, x, y, seg_mask, threshold); } public: @@ -414,581 +141,25 @@ namespace maix::nn * Labels list * @maixpy maix.nn.YOLOv8.labels */ - std::vector labels; + //std::vector labels; /** * Label file path * @maixpy maix.nn.YOLOv8.label_path */ - std::string label_path; + //std::string label_path; /** * Get mean value, list type * @maixpy maix.nn.YOLOv8.mean */ - std::vector mean; + //std::vector mean; /** * Get scale value, list type * @maixpy maix.nn.YOLOv8.scale */ - std::vector scale; - - protected: - std::string type_str = "yolov8"; - - private: - image::Size _input_size; - image::Format _input_img_fmt; - nn::NN *_model; - std::map _extra_info; - float _conf_th = 0.5; - float _iou_th = 0.45; - float _keypoint_th = 0.5; - YOLOv8_Type _type; - bool _dual_buff; - - private: - err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) - { - // load labels from labels file - labels.clear(); - fs::File *f = fs::open(label_path, "r"); - if (!f) - { - log::error("open label file %s failed", label_path.c_str()); - return err::ERR_ARGS; - } - std::string line; - while (f->readline(line) > 0) - { - // strip line - line.erase(0, line.find_first_not_of(" \t\r\n")); - line.erase(line.find_last_not_of(" \t\r\n") + 1); - labels.push_back(line); - } - f->close(); - delete f; - return err::ERR_NONE; - } - - nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit, int sort) - { - nn::Objects *objects = new nn::Objects(); - tensor::Tensor *kp_out = NULL; - tensor::Tensor *mask_out = NULL; - float scale_w = 1; - float scale_h = 1; - - if(!_decode_objs(*objects, outputs, _conf_th, _input_size.width(), _input_size.height(), &kp_out, &mask_out)) - { - delete objects; - return NULL; - } - if (objects->size() > 0) - { - nn::Objects *objects_total = objects; - objects = _nms(*objects); - delete objects_total; - if(sort != 0) - { - _sort_objects(*objects, sort); - } - } - // decode keypoints - if (_type == YOLOv8_Type::POSE) - { - _decode_keypoints(*objects, kp_out); - } - else if (_type == YOLOv8_Type::SEG) - { - _decode_seg_points(*objects, kp_out, mask_out); - } - if (objects->size() > 0) - { - _correct_bbox(*objects, img_w, img_h, fit, &scale_w, &scale_h); - } - return objects; - } - - bool _decode_objs(nn::Objects &objs, tensor::Tensors *outputs, float conf_thresh, int w, int h, tensor::Tensor **kp_out, tensor::Tensor **mask_out) - { - float stride[3] = {8, 16, 32}; - tensor::Tensor *score_out = NULL; // shape 1, 80, 8400, 1 - tensor::Tensor *box_out = NULL; // shape 1, 1, 4, 8400 - for (auto i : *outputs) - { - if (i.second->shape()[2] == 4 && !box_out) - { - box_out = i.second; - } - else if (strstr(i.first.c_str(), "Sigmoid") != NULL && !score_out) - { - score_out = i.second; - if((size_t)score_out->shape()[1] != labels.size()) - { - log::error("MUD labels(%d) must equal model's(%d)", score_out->shape()[1], labels.size()); - return false; - } - } - else if (strstr(i.first.c_str(), "output1") != NULL) - { - *mask_out = i.second; - } - else - { - *kp_out = i.second; - } - } - if (!score_out || !box_out) - { - throw err::Exception(err::ERR_ARGS, "model output not valid"); - } - int total_box_num = box_out->shape()[3]; - // int class_num = this->labels.size(); - int class_num = score_out->shape()[1]; - float *scores_ptr = (float *)score_out->data(); - float *dets_ptr = (float *)box_out->data(); - int idx_start[3] = { - 0, - (int)(h / stride[0] * w / stride[0]), - (int)(h / stride[0] * w / stride[0] + h / stride[1] * w / stride[1])}; - if (_type == YOLOv8_Type::OBB) - { - float *angle_ptr = (float *)(*kp_out)->data(); - for (int i = 0; i < 3; i++) - { - int nh = h / stride[i]; - int nw = w / stride[i]; - for (int ay = 0; ay < nh; ++ay) - { - for (int ax = 0; ax < nw; ++ax) - { - int offset = idx_start[i] + ay * nw + ax; - int class_id = _argmax(scores_ptr + offset, class_num, total_box_num); - // int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num); - // int class_id = (offset + max_idx) / total_box_num; - float obj_score = scores_ptr[offset + class_id * total_box_num]; - if (obj_score <= conf_thresh) - { - continue; - } - float angle = (angle_ptr[offset] - 0.25); - float angle_rad = angle * M_PI; - float cos_angle = cosf(angle_rad); - float sin_angle = sinf(angle_rad); - float lt_x = dets_ptr[offset]; - float lt_y = dets_ptr[offset + total_box_num]; - float rb_x = dets_ptr[offset + total_box_num * 2]; - float rb_y = dets_ptr[offset + total_box_num * 3]; - float xf = (rb_x - lt_x) / 2.0; - float yf = (rb_y - lt_y) / 2.0; - float bbox_w = (lt_x + rb_x) * stride[i]; - float bbox_h = (lt_y + rb_y) * stride[i]; - float bbox_x = ((xf * cos_angle - yf * sin_angle) + ax + 0.5) * stride[i] - bbox_w * 0.5; - float bbox_y = ((xf * sin_angle + yf * cos_angle) + ay + 0.5) * stride[i] - bbox_h * 0.5; - _KpInfo *kp_info = new _KpInfo(offset, ax, ay, stride[i]); - Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score, {}, angle); - obj.temp = (void *)kp_info; - } - } - } - } - else - { - for (int i = 0; i < 3; i++) - { - int nh = h / stride[i]; - int nw = w / stride[i]; - for (int ay = 0; ay < nh; ++ay) - { - for (int ax = 0; ax < nw; ++ax) - { - int offset = idx_start[i] + ay * nw + ax; - int class_id = _argmax(scores_ptr + offset, class_num, total_box_num); - // int max_idx = _argmax2(scores_ptr + offset, class_num * total_box_num - offset, total_box_num); - // int class_id = (offset + max_idx) / total_box_num; - float obj_score = scores_ptr[offset + class_id * total_box_num]; - if (obj_score <= conf_thresh) - { - continue; - } - float bbox_x = (ax + 0.5 - dets_ptr[offset]) * stride[i]; - float bbox_y = (ay + 0.5 - dets_ptr[offset + total_box_num]) * stride[i]; - float bbox_w = (ax + 0.5 + dets_ptr[offset + total_box_num * 2]) * stride[i] - bbox_x; - float bbox_h = (ay + 0.5 + dets_ptr[offset + total_box_num * 3]) * stride[i] - bbox_y; - _KpInfo *kp_info = new _KpInfo(offset, ax, ay, stride[i]); - Object &obj = objs.add(bbox_x, bbox_y, bbox_w, bbox_h, class_id, obj_score); - obj.temp = (void *)kp_info; - } - } - } - } - return true; - } - - nn::Objects *_nms(nn::Objects &objs) - { - nn::Objects *result = new nn::Objects(); - std::sort(objs.begin(), objs.end(), [](const nn::Object *a, const nn::Object *b) - { return a->score > b->score; }); - for (size_t i = 0; i < objs.size(); ++i) - { - nn::Object &a = objs.at(i); - if (a.score == 0) - continue; - for (size_t j = i + 1; j < objs.size(); ++j) - { - nn::Object &b = objs.at(j); - { - if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th) - { - b.score = 0; - } - } - } - } - for (nn::Object *a : objs) - { - if (a->score != 0) - { - Object &obj = result->add(a->x, a->y, a->w, a->h, a->class_id, a->score, a->points); - if (obj.x < 0) - { - obj.w += obj.x; - obj.x = 0; - } - if (obj.y < 0) - { - obj.h += obj.y; - obj.y = 0; - } - if (obj.x + obj.w > _input_size.width()) - { - obj.w = _input_size.width() - obj.x; - } - if (obj.y + obj.h > _input_size.height()) - { - obj.h = _input_size.height() - obj.y; - } - obj.temp = a->temp; - } - else - { - delete (_KpInfo *)a->temp; - a->temp = NULL; - } - } - return result; - } - - void _sort_objects(nn::Objects &objects, int sort) - { - if (sort > 0) - std::sort(objects.begin(), objects.end(), [](const nn::Object *a, const nn::Object *b) - { return (a->w * a->h) > (b->w * b->h); }); - else - std::sort(objects.begin(), objects.end(), [](const nn::Object *a, const nn::Object *b) - { return (a->w * a->h) < (b->w * b->h); }); - } - - void _decode_keypoints(nn::Objects &objs, tensor::Tensor *kp_out) - { - float *data = (float *)kp_out->data(); - int keypoint_num = kp_out->shape()[1] / 3; // 1, 51, 8400, 1 - int total_box_num = kp_out->shape()[2]; // 1, 51, 8400, 1 - for (size_t i = 0; i < objs.size(); ++i) - { - nn::Object &o = objs.at(i); - _KpInfo *kp_info = (_KpInfo *)o.temp; - float *p = data + kp_info->idx; - for (int k = 0; k < keypoint_num; ++k) - { - float score = _sigmoid(p[(k * 3 + 2) * total_box_num]); - int x = -1; - int y = -1; - if (score > _keypoint_th) - { - x = (p[(k * 3) * total_box_num] * 2.0 + kp_info->anchor_x) * kp_info->stride; - y = (p[(k * 3 + 1) * total_box_num] * 2.0 + kp_info->anchor_y) * kp_info->stride; - } - o.points.push_back(x); - o.points.push_back(y); - } - delete (_KpInfo *)o.temp; - o.temp = NULL; - } - } - - void _decode_seg_points(nn::Objects &objs, tensor::Tensor *kp_out, tensor::Tensor *mask_out) - { - float *data = (float *)kp_out->data(); - float *mask_data = (float *)mask_out->data(); // 1, 32, 160, 160 - int mask_h = mask_out->shape()[2]; - int mask_w = mask_out->shape()[3]; - int mask_squre = mask_h * mask_w; - int mask_num = kp_out->shape()[1]; // 1, 32, 8400, 1 - int total_box_num = kp_out->shape()[2]; // 1, 32, 8400, 1 - float mask_weights[mask_num]; - for (size_t i = 0; i < objs.size(); ++i) - { - nn::Object &o = objs.at(i); - int mask_x = o.x * mask_w / _input_size.width(); - int mask_y = o.y * mask_h / _input_size.height(); - int mask_x2 = (o.x + o.w) * mask_w / _input_size.width(); - int mask_y2 = (o.y + o.h) * mask_h / _input_size.height(); - _KpInfo *kp_info = (_KpInfo *)o.temp; - float *p = data + kp_info->idx; - for (int k = 0; k < mask_num; ++k) - { - mask_weights[k] = p[k * total_box_num]; - } - o.seg_mask = new image::Image(mask_x2 - mask_x, mask_y2 - mask_y, image::Format::FMT_GRAYSCALE); - uint8_t *p_img_data = (uint8_t *)o.seg_mask->data(); - for (int j = mask_y; j < mask_y2; ++j) - { - for (int k = mask_x; k < mask_x2; ++k) - { - mask_data[j * mask_w + k] *= mask_weights[0]; - } - } - for (int n = 1; n < mask_num; ++n) - { - for (int j = mask_y; j < mask_y2; ++j) - { - for (int k = mask_x; k < mask_x2; ++k) - { - mask_data[j * mask_w + k] += mask_weights[n] * mask_data[n * mask_squre + j * mask_w + k]; - } - } - } - for (int j = mask_y; j < mask_y2; ++j) - { - for (int k = mask_x; k < mask_x2; ++k) - { - *p_img_data++ = (uint8_t)(_sigmoid(mask_data[j * mask_w + k]) * 255); - } - } - delete (_KpInfo *)o.temp; - o.temp = NULL; - } - } - - void _correct_bbox(nn::Objects &objs, int img_w, int img_h, maix::image::Fit fit, float *scale_w, float *scale_h) - { -#define CORRECT_BBOX_RANGE_YOLOV8(obj) \ - do \ - { \ - if (obj->x < 0) \ - { \ - obj->w += obj->x; \ - obj->x = 0; \ - } \ - if (obj->y < 0) \ - { \ - obj->h += obj->y; \ - obj->y = 0; \ - } \ - if (obj->x + obj->w > img_w) \ - { \ - obj->w = img_w - obj->x; \ - } \ - if (obj->y + obj->h > img_h) \ - { \ - obj->h = img_h - obj->y; \ - } \ - } while (0) - - for (nn::Object *obj : objs) - { - if (obj->temp) - { - delete (_KpInfo *)obj->temp; - obj->temp = NULL; - } - } - if (img_w == _input_size.width() && img_h == _input_size.height()) - { - if (_type == YOLOv8_Type::SEG) - { - for (nn::Object *obj : objs) - { - if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height()) - { - image::Image *old = obj->seg_mask; - obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL); - delete old; - } - } - } - return; - } - if (fit == maix::image::FIT_FILL) - { - *scale_w = (float)img_w / _input_size.width(); - *scale_h = (float)img_h / _input_size.height(); - for (nn::Object *obj : objs) - { - obj->x *= *scale_w; - obj->y *= *scale_h; - obj->w *= *scale_w; - obj->h *= *scale_h; - for (size_t i = 0; i < obj->points.size() / 2; ++i) - { - obj->points.at(i * 2) *= *scale_w; - obj->points.at(i * 2 + 1) *= *scale_h; - } - CORRECT_BBOX_RANGE_YOLOV8(obj); - if (_type == YOLOv8_Type::SEG) - { - if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height()) - { - image::Image *old = obj->seg_mask; - obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL); - delete old; - } - } - } - } - else if (fit == maix::image::FIT_CONTAIN) - { - *scale_w = ((float)_input_size.width()) / img_w; - *scale_h = ((float)_input_size.height()) / img_h; - float scale = std::min(*scale_w, *scale_h); - float scale_reverse = 1.0 / scale; - *scale_w = scale_reverse; - *scale_h = scale_reverse; - float pad_w = (_input_size.width() - img_w * scale) / 2.0; - float pad_h = (_input_size.height() - img_h * scale) / 2.0; - for (nn::Object *obj : objs) - { - obj->x = (obj->x - pad_w) * scale_reverse; - obj->y = (obj->y - pad_h) * scale_reverse; - obj->w *= scale_reverse; - obj->h *= scale_reverse; - for (size_t i = 0; i < obj->points.size() / 2; ++i) - { - obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse; - obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse; - } - CORRECT_BBOX_RANGE_YOLOV8(obj); - if (_type == YOLOv8_Type::SEG) - { - if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height()) - { - image::Image *old = obj->seg_mask; - obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL); - delete old; - } - } - } - } - else if (fit == maix::image::FIT_COVER) - { - *scale_w = ((float)_input_size.width()) / img_w; - *scale_h = ((float)_input_size.height()) / img_h; - float scale = std::max(*scale_w, *scale_h); - float scale_reverse = 1.0 / scale; - *scale_w = scale_reverse; - *scale_h = scale_reverse; - float pad_w = (img_w * scale - _input_size.width()) / 2.0; - float pad_h = (img_h * scale - _input_size.height()) / 2.0; - for (nn::Object *obj : objs) - { - obj->x = (obj->x + pad_w) * scale_reverse; - obj->y = (obj->y + pad_h) * scale_reverse; - obj->w *= scale_reverse; - obj->h *= scale_reverse; - for (size_t i = 0; i < obj->points.size() / 2; ++i) - { - obj->points.at(i * 2) = (obj->points.at(i * 2) - pad_w) * scale_reverse; - obj->points.at(i * 2 + 1) = (obj->points.at(i * 2 + 1) - pad_h) * scale_reverse; - } - CORRECT_BBOX_RANGE_YOLOV8(obj); - if (_type == YOLOv8_Type::SEG) - { - if (obj->w != obj->seg_mask->width() || obj->h != obj->seg_mask->height()) - { - image::Image *old = obj->seg_mask; - obj->seg_mask = old->resize(obj->w, obj->h, image::FIT_FILL); - delete old; - } - } - } - } - else - { - throw err::Exception(err::ERR_ARGS, "fit type not support"); - } - } - - inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); } - - inline static float _calc_iou(Object &a, Object &b) - { - float area1 = a.w * a.h; - float area2 = b.w * b.h; - float wi = std::min((a.x + a.w), (b.x + b.w)) - - std::max(a.x, b.x); - float hi = std::min((a.y + a.h), (b.y + b.h)) - - std::max(a.y, b.y); - float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f); - return area_i / (area1 + area2 - area_i); - } - - template - static int _argmax(const T *data, size_t len, size_t stride = 1) - { - int maxIndex = 0; - for (size_t i = 1; i < len; i++) - { - int idx = i * stride; - if (data[maxIndex * stride] < data[idx]) - { - maxIndex = i; - } - } - return maxIndex; - } - - template - static int _argmax2(const T *data, size_t end, size_t stride = 1) - { - int maxIndex = 0; - for (size_t i = stride; i < end; i += stride) - { - if (data[maxIndex] < data[i]) - { - maxIndex = i; - } - } - return maxIndex; - } - - static void split0(std::vector &items, const std::string &s, const std::string &delimiter) - { - items.clear(); - size_t pos_start = 0, pos_end, delim_len = delimiter.length(); - std::string token; - - while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) - { - token = s.substr(pos_start, pos_end - pos_start); - pos_start = pos_end + delim_len; - token.erase(0, token.find_first_not_of(" \t\r\n")); - token.erase(token.find_last_not_of(" \t\r\n") + 1); - items.push_back(token); - } - - items.push_back(s.substr(pos_start)); - } - - static std::vector split(const std::string &s, const std::string &delimiter) - { - std::vector tokens; - split0(tokens, s, delimiter); - return tokens; - } + //std::vector scale; }; } // namespace maix::nn diff --git a/examples/nn_runner_simple/.gitignore b/examples/nn_runner_simple/.gitignore new file mode 100644 index 00000000..7171eaac --- /dev/null +++ b/examples/nn_runner_simple/.gitignore @@ -0,0 +1,9 @@ +build +dist +.config.mk +.flash.conf.json +data + +/CMakeLists.txt + +__pycache__ diff --git a/examples/nn_runner_simple/README.md b/examples/nn_runner_simple/README.md new file mode 100644 index 00000000..2ce5c828 --- /dev/null +++ b/examples/nn_runner_simple/README.md @@ -0,0 +1,5 @@ +Hello World Project based on MaixCDK +==== + +Hello world example code for MaixCDK of Sipeed, build method please visit [MaixCDK](https://github.com/sipeed/MaixCDK). + diff --git a/examples/nn_runner_simple/main/CMakeLists.txt b/examples/nn_runner_simple/main/CMakeLists.txt new file mode 100644 index 00000000..452f67bd --- /dev/null +++ b/examples/nn_runner_simple/main/CMakeLists.txt @@ -0,0 +1,74 @@ +############### Add include ################### +list(APPEND ADD_INCLUDE "include" + ) +list(APPEND ADD_PRIVATE_INCLUDE "") +############################################### + +############ Add source files ################# +# list(APPEND ADD_SRCS "src/main.c" +# "src/test.c" +# ) +append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c") +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + +###### Add required/dependent components ###### +list(APPEND ADD_REQUIREMENTS basic nn vision) +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (OpenCV_DIR opencv/lib/cmake/opencv4) +# find_package(OpenCV REQUIRED) +############################################### + +############ Add static libs ################## +# list(APPEND ADD_STATIC_LIB "lib/libtest.a") +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components depend on this component +# list(APPEND ADD_DEFINITIONS -DAAAAA222=1 +# -DAAAAA333=1) +############################################### + +############ Add static libs ################## +#### Update parent's variables like CMAKE_C_LINK_FLAGS +# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE) +############################################### + +######### Add files need to download ######### +# list(APPEND ADD_FILE_DOWNLOADS "{ +# 'url': 'https://*****/abcde.tar.xz', +# 'urls': [], # backup urls, if url failed, will try urls +# 'sites': [], # download site, user can manually download file and put it into dl_path +# 'sha256sum': '', +# 'filename': 'abcde.tar.xz', +# 'path': 'toolchains/xxxxx', +# 'check_files': [] +# }" +# ) +# +# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx, +# you can directly use then, for example use it in add_custom_command +############################################## + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() diff --git a/examples/nn_runner_simple/main/Kconfig b/examples/nn_runner_simple/main/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/examples/nn_runner_simple/main/include/main.h b/examples/nn_runner_simple/main/include/main.h new file mode 100644 index 00000000..45dcbb04 --- /dev/null +++ b/examples/nn_runner_simple/main/include/main.h @@ -0,0 +1,3 @@ +#pragma once + + diff --git a/examples/nn_runner_simple/main/src/main.cpp b/examples/nn_runner_simple/main/src/main.cpp new file mode 100644 index 00000000..fcc0f791 --- /dev/null +++ b/examples/nn_runner_simple/main/src/main.cpp @@ -0,0 +1,132 @@ + +#include "maix_basic.hpp" +#include "maix_nn.hpp" +#include "maix_image.hpp" +#include "main.h" + + +using namespace maix; + +image::Image a = image::Image(); +static std::vector split(const std::string &s, const std::string &delimiter) { + std::vector tokens; + size_t pos_start = 0, pos_end, delim_len = delimiter.length(); + std::string token; + + while ((pos_end = s.find (delimiter, pos_start)) != std::string::npos) { + token = s.substr (pos_start, pos_end - pos_start); + pos_start = pos_end + delim_len; + tokens.push_back (token); + } + + tokens.push_back (s.substr (pos_start)); + return tokens; +} + +int _main(int argc, char* argv[]) +{ + log::info("Program start"); + int ret = 0; + int repeat_times = 1; + + std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path repeate_times"; + + // model path from argv[1], image from argv[2] + if(argc < 3) + { + log::error("args error, %s", help.c_str()); + return -1; + } + const char *model_path = argv[1]; + repeat_times = std::stoi(argv[2]); + + uint64_t ts = time::ticks_ms(); + nn::NN model(model_path); + log::info("load model %s success, time: %d ms", model_path, time::ticks_ms() - ts); + + std::vector inputs_info = model.inputs_info(); + std::vector outputs_info = model.outputs_info(); + log::info("Inputs:"); + for(auto item : inputs_info) + { + log::print(" %s\n", item.to_str().c_str()); + } + log::info("Outputs:"); + for(auto item : outputs_info) + { + log::print(" %s\n", item.to_str().c_str()); + } + + + log::info("generate inputs now"); + tensor::Tensors input_tensors; + for(auto item : inputs_info) + { + tensor::Tensor *t = new tensor::Tensor(item.shape, item.dtype); + input_tensors.add_tensor(item.name, t, false, true); + } + + tensor::Tensors outputs; + int count = 3; + while (repeat_times > 1 && count-- > 0) // warn up + { + err::Err e = model.forward(input_tensors, outputs, false); + if(e != err::Err::ERR_NONE) + { + log::error("forward failed"); + goto end; + } + outputs.clear(); + } + ts = time::ticks_ms(); + while (1) + { + err::Err e = model.forward(input_tensors, outputs, false); + if(e != err::Err::ERR_NONE) + { + log::error("forward failed"); + goto end; + } + ++ count; + if(count >= repeat_times) + break; + outputs.clear(); + } + log::info("forward success, average time: %d ms", (int)((time::ticks_ms() - ts) / repeat_times)); + for(auto &it : outputs) + { + tensor::Tensor *tensor = it.second; + log::info("======= output '%s':", it.first.c_str()); + log::print(" info: %s\n",(*tensor).to_str().c_str()); + + auto names = split(it.first, "/"); + std::string filename = names[names.size() - 1] + ".bin"; + log::print(" save as bin file to '%s'\n", filename.c_str()); + fs::File *f = fs::open(filename, "wb"); + if(!f) + { + log::error("open file faild"); + goto end; + } + f->write(tensor->data(), tensor->size_int() * sizeof(float)); + delete f; + } +end: + + log::info("Program exit"); + + return ret; +} + +int main(int argc, char* argv[]) +{ + // Catch signal and process + sys::register_default_signal_handle(); + + // Use CATCH_EXCEPTION_RUN_RETURN to catch exception, + // if we don't catch exception, when program throw exception, the objects will not be destructed. + // So we catch exception here to let resources be released(call objects' destructor) before exit. + CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); +} + + diff --git a/examples/nn_yolo11_detect/main/src/main.cpp b/examples/nn_yolo11_detect/main/src/main.cpp index 22d03af0..9ac8ddfd 100644 --- a/examples/nn_yolo11_detect/main/src/main.cpp +++ b/examples/nn_yolo11_detect/main/src/main.cpp @@ -29,6 +29,7 @@ int _main(int argc, char *argv[]) float conf_threshold = 0.5; float iou_threshold = 0.45; bool dual_buff = true; + int repeat_times = 20; nn::YOLO11 detector("", dual_buff); @@ -46,9 +47,37 @@ int _main(int argc, char *argv[]) if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height()) { log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height()); + // we resize here manually, if not, the detect function will auto resize. + maix::image::Image *img2 = img->resize(detector.input_size().width(), detector.input_size().height(), image::Fit::FIT_CONTAIN); + delete img; + img = img2; } log::info("detect now"); - nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold); + nn::Objects *result = nullptr; + if(repeat_times > 1) + { + for(int i = 0; i < 3; ++i) // warmup + { + result = detector.detect(*img, conf_threshold, iou_threshold); + delete result; + } + } + std::vector intervals(repeat_times); + for(int i = 0; i < repeat_times; ++i) // average time + { + uint64_t start = time::ticks_ms(); + result = detector.detect(*img, conf_threshold, iou_threshold); + intervals[i] = time::ticks_ms() - start; + if(i < repeat_times - 1) + delete result; + } + float sum = 0; + for(auto i : intervals) + { + sum += i; + } + log::info("run %d times, average time: %dms", repeat_times, (int)(sum / repeat_times)); + if(result->size() == 0) { log::info("no object detected !"); diff --git a/examples/nn_yolo11_obb/.gitignore b/examples/nn_yolo11_obb/.gitignore new file mode 100644 index 00000000..7171eaac --- /dev/null +++ b/examples/nn_yolo11_obb/.gitignore @@ -0,0 +1,9 @@ +build +dist +.config.mk +.flash.conf.json +data + +/CMakeLists.txt + +__pycache__ diff --git a/examples/nn_yolo11_obb/README.md b/examples/nn_yolo11_obb/README.md new file mode 100644 index 00000000..2c1cf569 --- /dev/null +++ b/examples/nn_yolo11_obb/README.md @@ -0,0 +1,9 @@ +nn_yolov8 examples based on MaixCDK +==== +## only run yolov8 model + +- Use `maixcdk build` to compile binary files. +- Move files and runtime libraries to device. +- Use SSH command with parameters to run programs. + +` "Usage: " + std::string(argv[0]) + " mud_model_path "; ` diff --git a/examples/nn_yolo11_obb/main/CMakeLists.txt b/examples/nn_yolo11_obb/main/CMakeLists.txt new file mode 100644 index 00000000..452f67bd --- /dev/null +++ b/examples/nn_yolo11_obb/main/CMakeLists.txt @@ -0,0 +1,74 @@ +############### Add include ################### +list(APPEND ADD_INCLUDE "include" + ) +list(APPEND ADD_PRIVATE_INCLUDE "") +############################################### + +############ Add source files ################# +# list(APPEND ADD_SRCS "src/main.c" +# "src/test.c" +# ) +append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c") +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + +###### Add required/dependent components ###### +list(APPEND ADD_REQUIREMENTS basic nn vision) +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (OpenCV_DIR opencv/lib/cmake/opencv4) +# find_package(OpenCV REQUIRED) +############################################### + +############ Add static libs ################## +# list(APPEND ADD_STATIC_LIB "lib/libtest.a") +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components depend on this component +# list(APPEND ADD_DEFINITIONS -DAAAAA222=1 +# -DAAAAA333=1) +############################################### + +############ Add static libs ################## +#### Update parent's variables like CMAKE_C_LINK_FLAGS +# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE) +############################################### + +######### Add files need to download ######### +# list(APPEND ADD_FILE_DOWNLOADS "{ +# 'url': 'https://*****/abcde.tar.xz', +# 'urls': [], # backup urls, if url failed, will try urls +# 'sites': [], # download site, user can manually download file and put it into dl_path +# 'sha256sum': '', +# 'filename': 'abcde.tar.xz', +# 'path': 'toolchains/xxxxx', +# 'check_files': [] +# }" +# ) +# +# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx, +# you can directly use then, for example use it in add_custom_command +############################################## + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() diff --git a/examples/nn_yolo11_obb/main/Kconfig b/examples/nn_yolo11_obb/main/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/examples/nn_yolo11_obb/main/include/main.h b/examples/nn_yolo11_obb/main/include/main.h new file mode 100644 index 00000000..45dcbb04 --- /dev/null +++ b/examples/nn_yolo11_obb/main/include/main.h @@ -0,0 +1,3 @@ +#pragma once + + diff --git a/examples/nn_yolo11_obb/main/src/main.cpp b/examples/nn_yolo11_obb/main/src/main.cpp new file mode 100644 index 00000000..ec0f4d37 --- /dev/null +++ b/examples/nn_yolo11_obb/main/src/main.cpp @@ -0,0 +1,140 @@ + +#include "maix_basic.hpp" +#include "maix_vision.hpp" +#include "maix_nn_yolo11.hpp" +#include "main.h" + +using namespace maix; + +int _main(int argc, char *argv[]) +{ + log::info("Program start"); + std::string model_type = "unknown"; + int ret = 0; + err::Err e; + maix::image::Format img_fmt = maix::image::FMT_RGB888; + std::vector mean = {}; + std::vector scale = {}; + char tmp_chars[128] = {0}; + + std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path "; + + if (argc < 2) + { + log::info(help.c_str()); + return -1; + } + + const char *model_path = argv[1]; + float conf_threshold = 0.5; + float iou_threshold = 0.45; + bool dual_buff = true; + int repeat_times = 20; + + + nn::YOLO11 detector("", dual_buff); + e = detector.load(model_path); + err::check_raise(e, "load model failed"); + log::info("load yolo11 model %s success", model_path); + + if (argc >= 3) + { + const char *img_path = argv[2]; + log::info("load image now"); + maix::image::Image *img = maix::image::load(img_path, img_fmt); + err::check_null_raise(img, "load image " + std::string(img_path) + " failed"); + log::info("load image %s success: %s", img_path, img->to_str().c_str()); + if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height()) + { + log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height()); + // we resize here manually, if not, the detect function will auto resize. + maix::image::Image *img2 = img->resize(detector.input_size().width(), detector.input_size().height(), image::Fit::FIT_CONTAIN); + delete img; + img = img2; + } + log::info("detect now"); + nn::Objects *result = nullptr; + if(repeat_times > 1) + { + for(int i = 0; i < 3; ++i) // warmup + { + result = detector.detect(*img, conf_threshold, iou_threshold); + delete result; + } + } + std::vector intervals(repeat_times); + for(int i = 0; i < repeat_times; ++i) // average time + { + uint64_t start = time::ticks_ms(); + result = detector.detect(*img, conf_threshold, iou_threshold); + intervals[i] = time::ticks_ms() - start; + if(i < repeat_times - 1) + delete result; + } + float sum = 0; + for(auto i : intervals) + { + sum += i; + } + log::info("run %d times, average time: %dms", repeat_times, (int)(sum / repeat_times)); + + if(result->size() == 0) + { + log::info("no object detected !"); + } + for (auto &r : *result) + { + log::info("result: %s", r->to_str().c_str()); + auto points = r->get_obb_points(); + snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f, %.1f", detector.labels[r->class_id].c_str(), r->score, r->angle * 180.0); + img->draw_string(points[0], points[1] - 4, tmp_chars, maix::image::Color::from_rgb(255, 0, 0)); + detector.draw_pose(*img, points, detector.input_width() > 480 ? 3 : 1, image::COLOR_RED, std::vector(), false, true); + } + img->save("result.jpg"); + delete result; + delete img; + } + else + { + log::info("open camera now"); + maix::image::Size input_size = detector.input_size(); + camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format()); + log::info("open camera success"); + display::Display disp = display::Display(); + while (!app::need_exit()) + { + uint64_t t = time::ticks_ms(); + maix::image::Image *img = cam.read(); + err::check_null_raise(img, "read camera failed"); + uint64_t t2 = time::ticks_ms(); + nn::Objects *result = detector.detect(*img); + uint64_t t3 = time::ticks_ms(); + for (auto &r : *result) + { + log::info("result: %s", r->to_str().c_str()); + img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0)); + img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0)); + } + disp.show(*img); + delete result; + delete img; + log::info("time: all %d ms, detect %d ms, fps: %.1f", time::ticks_ms() - t, t3 - t2, time::fps()); + } + } + + log::info("Program exit"); + + return ret; +} + +int main(int argc, char *argv[]) +{ + // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true. + signal(SIGINT, [](int sig) + { app::set_exit_flag(true); }); + + // Use CATCH_EXCEPTION_RUN_RETURN to catch exception, + // if we don't catch exception, when program throw exception, the objects will not be destructed. + // So we catch exception here to let resources be released(call objects' destructor) before exit. + CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); +}