From ab358bad64fca11523bc047e1f68b0552224dffe Mon Sep 17 00:00:00 2001 From: Neucrack Date: Thu, 5 Sep 2024 13:42:34 +0800 Subject: [PATCH] add self learn detector/tracker support --- components/basic/include/maix_tensor.hpp | 22 +- components/nn/include/maix_nn_nanotrack.hpp | 681 ++++++++++++++++++ components/nn/include/maix_nn_yolov5.hpp | 48 +- components/nn/include/maix_nn_yolov8.hpp | 47 +- examples/nn_nanotrack/.gitignore | 9 + examples/nn_nanotrack/README.md | 9 + examples/nn_nanotrack/main/CMakeLists.txt | 74 ++ examples/nn_nanotrack/main/Kconfig | 0 examples/nn_nanotrack/main/include/main.h | 3 + examples/nn_nanotrack/main/src/main.cpp | 148 ++++ projects/app_self_learn_tracker/.gitignore | 9 + projects/app_self_learn_tracker/README.md | 4 + projects/app_self_learn_tracker/app.yaml | 13 + .../app_self_learn_tracker/assets/icon.png | Bin 0 -> 887 bytes .../main/CMakeLists.txt | 74 ++ projects/app_self_learn_tracker/main/Kconfig | 0 .../main/include/main.h | 3 + .../app_self_learn_tracker/main/src/main.cpp | 141 ++++ 18 files changed, 1234 insertions(+), 51 deletions(-) create mode 100644 components/nn/include/maix_nn_nanotrack.hpp create mode 100644 examples/nn_nanotrack/.gitignore create mode 100644 examples/nn_nanotrack/README.md create mode 100644 examples/nn_nanotrack/main/CMakeLists.txt create mode 100644 examples/nn_nanotrack/main/Kconfig create mode 100644 examples/nn_nanotrack/main/include/main.h create mode 100644 examples/nn_nanotrack/main/src/main.cpp create mode 100644 projects/app_self_learn_tracker/.gitignore create mode 100644 projects/app_self_learn_tracker/README.md create mode 100644 projects/app_self_learn_tracker/app.yaml create mode 100644 projects/app_self_learn_tracker/assets/icon.png create mode 100644 projects/app_self_learn_tracker/main/CMakeLists.txt create mode 100644 projects/app_self_learn_tracker/main/Kconfig create mode 100644 projects/app_self_learn_tracker/main/include/main.h create mode 100644 projects/app_self_learn_tracker/main/src/main.cpp diff --git a/components/basic/include/maix_tensor.hpp b/components/basic/include/maix_tensor.hpp index 7f7fd981..87a25d87 100644 --- a/components/basic/include/maix_tensor.hpp +++ b/components/basic/include/maix_tensor.hpp @@ -134,9 +134,10 @@ namespace maix * @param dtype tensor element data type, see DType of this module * @param data pointer to data content, can be nullptr, it will automatically alloc memory * and detroy it when this object is destroyed + * @param copy defalt true to alloc new memory and copy from data. * @maixcdk maix.tensor.Tensor.Tensor */ - Tensor(std::vector shape, tensor::DType dtype, void *data) + Tensor(std::vector shape, tensor::DType dtype, void *data, bool copy = true) { _shape = shape; _dtype = dtype; @@ -147,11 +148,15 @@ namespace maix { size *= shape[i]; } - if(!_data) + if((!_data) || (data && copy)) { _data = malloc(size * dtype_size[dtype]); _is_alloc = true; log::debug("malloc tensor data\n"); + if(data) + { + memcpy(_data, data, size * dtype_size[dtype]); + } } // log::info("new tensor: %p", this); } @@ -528,6 +533,19 @@ namespace maix tensors.erase(key); } + /** + * Clear tensors + * @maixpy maix.tensor.Tensors.clear + */ + void clear() + { + auto _keys = keys(); + for(const auto &k : _keys) + { + rm_tensor(k); + } + } + /** * Begin of tensors * @maixcdk maix.tensor.Tensors.begin diff --git a/components/nn/include/maix_nn_nanotrack.hpp b/components/nn/include/maix_nn_nanotrack.hpp new file mode 100644 index 00000000..a3d3f2ae --- /dev/null +++ b/components/nn/include/maix_nn_nanotrack.hpp @@ -0,0 +1,681 @@ +/** + * @author neucrack@sipeed + * @copyright Sipeed Ltd 2023- + * @license Apache 2.0 + * @update 2024.9.2: Create this file. + */ + +#pragma once +#include "maix_basic.hpp" +#include "maix_nn.hpp" +#include "maix_image.hpp" +#include "maix_nn_F.hpp" +#include "maix_nn_object.hpp" +#include + +namespace maix::nn +{ + /** + * NanoTrack class + * @maixpy maix.nn.NanoTrack + */ + class NanoTrack + { + public: + /** + * Constructor of NanoTrack class + * @param model model path, default empty, you can load model later by load function. + * @throw If model arg is not empty and load failed, will throw err::Exception. + * @maixpy maix.nn.NanoTrack.__init__ + * @maixcdk maix.nn.NanoTrack.NanoTrack + */ + NanoTrack(const string &model = "") + { + _model = nullptr; + _model_target = nullptr; + _model_head = nullptr; + _dual_buff = false; + if (!model.empty()) + { + err::Err e = load(model); + if (e != err::ERR_NONE) + { + throw err::Exception(e, "load model failed"); + } + } + + // nanotrack params + _context_amount = 0.5; + _stride = 16; + _head_out_size = 15; + _penalty_k = 0.138; + _window_influence = 0.455; + _lr = 0.348; + _generate_window(_head_out_size, _window); + _points = _generate_points(_stride, _head_out_size); + _bbox.resize(_head_out_size * _head_out_size); + _penalty.resize(_head_out_size * _head_out_size); + } + + ~NanoTrack() + { + _free_models(); + } + + /** + * Load model from file + * @param model Model path want to load + * @return err::Err + * @maixpy maix.nn.NanoTrack.load + */ + err::Err load(const string &model) + { + _free_models(); + _model = new nn::NN(model, _dual_buff); + if (!_model) + { + return err::ERR_NO_MEM; + } + _extra_info = _model->extra_info(); + if (_extra_info.find("model_type") != _extra_info.end()) + { + if (_extra_info["model_type"] != "nanotrack") + { + log::error("model_type not match, expect 'nanotrack', but got '%s'", _extra_info["model_type"].c_str()); + return err::ERR_ARGS; + } + } + else + { + log::error("model_type key not found"); + return err::ERR_ARGS; + } + log::info("model info:\n\ttype: nanotrack"); + if (_extra_info.find("input_type") != _extra_info.end()) + { + std::string input_type = _extra_info["input_type"]; + if (input_type == "rgb") + { + _input_img_fmt = maix::image::FMT_RGB888; + log::print("\tinput type: rgb\n"); + } + else if (input_type == "bgr") + { + _input_img_fmt = maix::image::FMT_BGR888; + log::print("\tinput type: bgr\n"); + } + else + { + log::error("unknown input type: %s", input_type.c_str()); + return err::ERR_ARGS; + } + } + else + { + log::error("input_type key not found"); + return err::ERR_ARGS; + } + if (_extra_info.find("mean") != _extra_info.end()) + { + std::string mean_str = _extra_info["mean"]; + std::vector mean_strs = split(mean_str, ","); + log::print("\tmean:"); + for (auto &it : mean_strs) + { + try + { + this->mean.push_back(std::stof(it)); + } + catch (std::exception &e) + { + log::error("mean value error, should float"); + return err::ERR_ARGS; + } + log::print("%f ", this->mean.back()); + } + log::print("\n"); + } + else + { + log::error("mean use 0"); + } + if (_extra_info.find("scale") != _extra_info.end()) + { + std::string scale_str = _extra_info["scale"]; + std::vector scale_strs = split(scale_str, ","); + log::print("\tscale:"); + for (auto &it : scale_strs) + { + try + { + this->scale.push_back(std::stof(it)); + } + catch (std::exception &e) + { + log::error("scale value error, should float"); + return err::ERR_ARGS; + } + log::print("%f ", this->scale.back()); + } + log::print("\n"); + } + else + { + log::error("scale use 1"); + } + std::string model_dir = fs::abspath(fs::dirname(model)); + if (_extra_info.find("target_model") != _extra_info.end()) + { + std::string model_target = _extra_info["target_model"]; + _model_target = new nn::NN(model_dir + "/" + model_target, _dual_buff); + if (!_model_target) + { + delete _model; + _model = nullptr; + return err::ERR_NO_MEM; + } + } + else + { + log::error("target_model key not found"); + return err::ERR_ARGS; + } + if (_extra_info.find("head_model") != _extra_info.end()) + { + std::string model_head = _extra_info["head_model"]; + _model_head = new nn::NN(model_dir + "/" + model_head, _dual_buff); + if (!_model_head) + { + delete _model_target; + _model_target = nullptr; + delete _model; + _model = nullptr; + return err::ERR_NO_MEM; + } + } + else + { + log::error("target_model key not found"); + return err::ERR_ARGS; + } + std::vector inputs = _model->inputs_info(); + _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]); + inputs = _model_target->inputs_info(); + _input_size_target = image::Size(inputs[0].shape[3], inputs[0].shape[2]); + log::print("\tinput size: %dx%d, target size: %dx%d\n\n", _input_size.width(), _input_size.height(), _input_size_target.width(), _input_size_target.height()); + return err::ERR_NONE; + } + + /** + * Init tracker, give tacker first target image and target position. + * @param img Image want to detect, target should be in this image. + * @param x the target position left top coordinate x. + * @param y the target position left top coordinate y. + * @param w the target width. + * @param h the target height. + * @throw If image format not match model input format, will throw err::Exception. + * @maixpy maix.nn.NanoTrack.init + */ + void init(image::Image &img, int x, int y, int w, int h) + { + if (img.format() != _input_img_fmt) + { + throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); + } + // crop image + _get_avg_color(img, _input_img_fmt); + int c_x = x + w / 2; + int c_y = y + h / 2; + float size_target = std::sqrt((w + _context_amount * (w + h)) * (h + _context_amount * (w + h))); + int size_target_int = std::round(size_target); + int crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2; + image::Image *img_target = _padding_crop(img, c_x, c_y, size_target_int, crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2); + + // forward target feature + tensor::Tensors *outputs; + outputs = _model_target->forward_image(*img_target, this->mean, this->scale, image::FIT_FILL, false, true); + delete img_target; + _head_inputs.clear(); + for (auto it = outputs->begin(); it != outputs->end(); it++) + { + _head_inputs.add_tensor("input1", it->second, true, true); + break; + } + _last_target.x = c_x; // center, not left-top + _last_target.y = c_y; // center, not left-top + _last_target.w = w; + _last_target.h = h; + delete outputs; + } + + /** + * Track object acoording to last object position and the init function learned target feature. + * @param img image to detect object and track, can be any resolution, before detect it will crop a area according to last time target's position. + * @param threshold If score < threshold, will see this new detection is invalid, but remain return this new detecion, default 0.9. + * @return object, position and score, and detect area in points's first 4 element(x, y, w, h, center_x, center_y, input_size, target_size) + * @maixpy maix.nn.NanoTrack.track + */ + nn::Object track(image::Image &img, float threshold = 0.9) + { + if (img.format() != _input_img_fmt) + { + throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); + } + // crop image + float size_target = std::sqrt((_last_target.w + _context_amount * (_last_target.w + _last_target.h)) * (_last_target.h + _context_amount * (_last_target.w + _last_target.h))); + int size_x = std::round(size_target * _input_size.width() / _input_size_target.width()); + float scale_z = _input_size_target.width() / size_target; + int crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2; + image::Image *img_target = _padding_crop(img, _last_target.x, _last_target.y, size_x, crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2); + + // forward target feature + tensor::Tensors *outputs; + outputs = _model->forward_image(*img_target, this->mean, this->scale, image::FIT_FILL, false, true); + delete img_target; + for (auto it = outputs->begin(); it != outputs->end(); it++) + { + _head_inputs.add_tensor("input2", it->second, false, false); + break; + } + + // forward head + tensor::Tensors *outputs2 = NULL; + tensor::Tensor *score_out = NULL; // 1x2x15x15 + tensor::Tensor *bbox_out = NULL; // 1x4x15x15 + outputs2 = _model_head->forward(_head_inputs, false, true); + delete outputs; + for (auto it = outputs2->begin(); it != outputs2->end(); it++) + { + if (it->second->shape()[1] == 2) + score_out = it->second; + else if (it->second->shape()[1] == 4) + bbox_out = it->second; + } + if(!bbox_out || ! score_out) + { + delete outputs2; + throw err::Exception(err::ERR_ARGS, "wrong model"); + } + + // decode score and bbox + float *score = (float *)score_out->data(); + _softmax_2xn(score, _points.size()); + score += _points.size(); // we only use the target prob + float *bbox = (float *)bbox_out->data(); + int stride1 = _points.size(); + int stride2 = stride1 * 2; + int stride3 = stride1 * 3; + float max_score = 0; + int best_idx = 0; + for (size_t i = 0; i < _points.size(); ++i) + { + nn::ObjectFloat &box = _bbox.at(i); + box.x = _points[i][0] - bbox[i]; + box.y = _points[i][1] - bbox[i + stride1]; + box.w = _points[i][0] + bbox[i + stride2]; + box.h = _points[i][1] + bbox[i + stride3]; + // center bbox + box.w = box.w - box.x; + box.h = box.h - box.y; + box.x += box.w / 2.0; + box.y += box.h / 2.0; + // penalty + float s_c = _change(_sz(box.w, box.h) / _sz(_last_target.w * scale_z, _last_target.h * scale_z)); + float r_c = _change(((float)_last_target.w / _last_target.h) / (box.w / box.h)); + _penalty[i] = std::exp(-(r_c * s_c - 1) * _penalty_k); + score[i] *= _penalty[i]; + score[i] = score[i] * (1 - _window_influence) + _window[i] * _window_influence; + if (score[i] > max_score) + { + max_score = score[i]; + best_idx = i; + } + } + nn::ObjectFloat &box = _bbox.at(best_idx); + float temp = 1.0 / scale_z; + box.x *= temp; + box.y *= temp; + box.w *= temp; + box.h *= temp; + int cx = box.x + _last_target.x; + int cy = box.y + _last_target.y; + // smooth + float lr = _penalty[best_idx] * max_score * _lr; + int w = _last_target.w * (1 - lr) + box.w * lr; + int h = _last_target.h * (1 - lr) + box.h * lr; + _bbox_clip(cx, cy, w, h, img.width(), img.height()); + + // update status + if(max_score >= threshold) + { + _last_target.x = cx; + _last_target.y = cy; + _last_target.w = w; + _last_target.h = h; + } + + // clear data + _head_inputs.rm_tensor("input2"); + delete outputs2; + nn::Object res(cx - w / 2, cy - h / 2, w, h, 0, max_score); + res.points.resize(8); + res.points.at(0) = crop_x_x1; + res.points.at(1) = crop_x_y1; + res.points.at(2) = crop_x_x2 - crop_x_x1; + res.points.at(3) = crop_x_y2 - crop_x_y1; + res.points.at(4) = cx; + res.points.at(5) = cy; + res.points.at(6) = size_x; + res.points.at(7) = size_target; + return res; + } + + /** + * Get model input size + * @return model input size + * @maixpy maix.nn.NanoTrack.input_size + */ + image::Size input_size() + { + return _input_size; + } + + /** + * Get model input width + * @return model input size of width + * @maixpy maix.nn.NanoTrack.input_width + */ + int input_width() + { + return _input_size.width(); + } + + /** + * Get model input height + * @return model input size of height + * @maixpy maix.nn.NanoTrack.input_height + */ + int input_height() + { + return _input_size.height(); + } + + /** + * Get input image format + * @return input image format, image::Format type. + * @maixpy maix.nn.NanoTrack.input_format + */ + image::Format input_format() + { + return _input_img_fmt; + } + + public: + /** + * Get mean value, list type + * @maixpy maix.nn.NanoTrack.mean + */ + std::vector mean; + + /** + * Get scale value, list type + * @maixpy maix.nn.NanoTrack.scale + */ + std::vector scale; + + private: + image::Size _input_size; + image::Size _input_size_target; + image::Format _input_img_fmt; + nn::NN *_model; + nn::NN *_model_target; + nn::NN *_model_head; + std::map _extra_info; + bool _dual_buff; + float _context_amount; + int _stride; + int _head_out_size; + float _penalty_k; + float _window_influence; + float _lr; + uint8_t _avg_color[3]; + std::vector> _points; + std::vector _bbox; + tensor::Tensors _head_inputs; + nn::Object _last_target; + std::vector _penalty; + std::vector _window; + + private: + inline float _sz(float w, float h) + { + float pad = (w + h) * 0.5; + return std::sqrt((w + pad) * (h + pad)); + } + + inline float _change(float r) + { + float t = 1.0 / r; + return r > t ? r : t; + } + + inline void _bbox_clip(int &cx, int &cy, int &w, int &h, int max_w, int max_h) + { + cx = max(0, min(cx, max_w - 1)); + cy = max(0, min(cy, max_h - 1)); + w = max(10, min(w, max_w)); + h = max(10, min(h, max_h)); + } + + void _free_models() + { + if (_model) + { + delete _model; + _model = nullptr; + } + if (_model_target) + { + delete _model_target; + _model_target = nullptr; + } + if (_model_head) + { + delete _model_head; + _model_head = nullptr; + } + } + + void _get_avg_color(image::Image &img, const image::Format &fmt) + { + uint8_t max_v_ch1 = 0, max_v_ch2 = 0, max_v_ch3 = 0; + uint8_t min_v_ch1 = 255, min_v_ch2 = 255, min_v_ch3 = 255; + uint8_t *p = (uint8_t *)img.data(); + for (int i = 0; i < img.data_size() / 3; ++i) + { + if (p[i * 3] > max_v_ch1) + max_v_ch1 = p[i * 3]; + else if (p[i * 3] < min_v_ch1) + min_v_ch1 = p[i * 3]; + if (p[i * 3 + 1] > max_v_ch2) + max_v_ch2 = p[i * 3 + 1]; + else if (p[i * 3 + 1] < min_v_ch2) + min_v_ch2 = p[i * 3 + 1]; + if (p[i * 3 + 2] > max_v_ch3) + max_v_ch3 = p[i * 3 + 2]; + else if (p[i * 3 + 2] < min_v_ch3) + min_v_ch3 = p[i * 3 + 2]; + } + _avg_color[0] = (max_v_ch1 - min_v_ch1) / 2; + _avg_color[1] = (max_v_ch2 - min_v_ch2) / 2; + _avg_color[2] = (max_v_ch3 - min_v_ch3) / 2; + } + + image::Image *_padding_crop(image::Image &img, int c_x, int c_y, int size_target, int ©_x1, int ©_y1, int ©_x2, int ©_y2) + { + int x1 = c_x - size_target / 2; + int y1 = c_y - size_target / 2; + int x2 = c_x + size_target / 2; + int y2 = c_y + size_target / 2; + // not pad + if (x1 >= 0 && y1 >= 0 && x2 < img.width() && y2 < img.height()) + { + copy_x1 = x1; + copy_y1 = y1; + copy_x2 = x2; + copy_y2 = y2; + return img.crop(x1, y1, size_target, size_target); + } + image::Image *res = new image::Image(size_target, size_target, img.format()); + if (!res) + throw err::Exception(err::ERR_NO_MEM); + copy_x1 = max(0, x1); + copy_y1 = max(0, y1); + copy_x2 = min(img.width(), x2); + copy_y2 = min(img.height(), y2); + uint8_t *data = (uint8_t *)res->data(); + uint8_t *src = (uint8_t *)img.data(); + // pad + for (int y = 0; y < copy_y1 - y1; ++y) + { + for (int x = 0; x < x2 - x1; ++x) + { + uint8_t *p = data + (y * res->width() + x) * 3; + *p = _avg_color[0]; + *(p + 1) = _avg_color[1]; + *(p + 2) = _avg_color[2]; + } + } + for (int y = copy_y2; y < y2; ++y) + { + for (int x = 0; x < x2 - x1; ++x) + { + uint8_t *p = data + ((y + - y1) * res->width() + x) * 3; + *p = _avg_color[0]; + *(p + 1) = _avg_color[1]; + *(p + 2) = _avg_color[2]; + } + } + for (int y = copy_y1; y < copy_y2; ++y) + { + for (int x = 0; x < copy_x1 - x1; ++x) + { + uint8_t *p = data + ((y - y1) * res->width() + x) * 3; + *p = _avg_color[0]; + *(p + 1) = _avg_color[1]; + *(p + 2) = _avg_color[2]; + } + } + for (int y = copy_y1; y < copy_y2; ++y) + { + for (int x = copy_x2; x < x2; ++x) + { + uint8_t *p = data + ((y - y1) * res->width() + x - x1) * 3; + *p = _avg_color[0]; + *(p + 1) = _avg_color[1]; + *(p + 2) = _avg_color[2]; + } + } + for (int y = copy_y1; y < copy_y2; ++y) + { + uint8_t *p = data + ((y - y1) * res->width() + (copy_x1 - x1)) * 3; + uint8_t *s = src + (y * img.width() + copy_x1) * 3; + int length = (copy_x2 - copy_x1) * 3; + memcpy(p, s, length); + } + // res->save("/root/test.png"); + return res; + } + + std::vector> _generate_points(int stride = 16, int size = 15) + { + // 计算原点位置 + float ori = -(size / 2) * stride; + + // 创建points向量 + std::vector> points(size * size, std::vector(2, 0.0f)); + + int index = 0; + for (int dy = 0; dy < size; ++dy) + { + for (int dx = 0; dx < size; ++dx) + { + float x = ori + stride * dx; + float y = ori + stride * dy; + points[index][0] = x; + points[index][1] = y; + ++index; + } + } + + return points; + } + + std::vector _generate_hanning_window(int size) + { + std::vector hanning(size); + for (int i = 0; i < size; ++i) + { + hanning[i] = 0.5f * (1.0f - std::cos(2.0f * M_PI * i / (size - 1))); + } + return hanning; + } + + void _generate_window(int size, std::vector &window) + { + std::vector hanning = _generate_hanning_window(size); + + window.resize(size * size); + for (int i = 0; i < size; ++i) + { + for (int j = 0; j < size; ++j) + { + window[i * size + j] = hanning[i] * hanning[j]; + } + } + } + + static void split0(std::vector &items, const std::string &s, const std::string &delimiter) + { + items.clear(); + size_t pos_start = 0, pos_end, delim_len = delimiter.length(); + std::string token; + + while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) + { + token = s.substr(pos_start, pos_end - pos_start); + pos_start = pos_end + delim_len; + items.push_back(token); + } + + items.push_back(s.substr(pos_start)); + } + + static std::vector split(const std::string &s, const std::string &delimiter) + { + std::vector tokens; + split0(tokens, s, delimiter); + return tokens; + } + + static void _softmax_2xn(float *data, int size) + { + float *p1 = data; + float *p2 = data + size; + float large; + float sum; + for(int i=0; i< size; ++i) + { + large = *p1 > *p2 ? *p1 : *p2; + *p1 = expf(*p1 - large); + *p2 = expf(*p2 - large); + sum = *p1 + *p2; + *p1 /= sum; + *p2 /= sum; + ++p1; + ++p2; + } + } + }; + +} // namespace maix::nn diff --git a/components/nn/include/maix_nn_yolov5.hpp b/components/nn/include/maix_nn_yolov5.hpp index 8bfa31ee..6d4730a8 100644 --- a/components/nn/include/maix_nn_yolov5.hpp +++ b/components/nn/include/maix_nn_yolov5.hpp @@ -20,8 +20,6 @@ namespace maix::nn */ class YOLOv5 { - public: - public: /** * Constructor of YOLOv5 class @@ -231,29 +229,6 @@ namespace maix::nn return err::ERR_NONE; } - err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) - { - // load labels from labels file - labels.clear(); - fs::File *f = fs::open(label_path, "r"); - if (!f) - { - log::error("open label file %s failed", label_path.c_str()); - return err::ERR_ARGS; - } - std::string line; - while (f->readline(line) > 0) - { - // strip line - line.erase(0, line.find_first_not_of(" \t\r\n")); - line.erase(line.find_last_not_of(" \t\r\n") + 1); - labels.push_back(line); - } - f->close(); - delete f; - return err::ERR_NONE; - } - /** * Detect objects from image * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method. @@ -368,6 +343,29 @@ namespace maix::nn bool _dual_buff; private: + err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) + { + // load labels from labels file + labels.clear(); + fs::File *f = fs::open(label_path, "r"); + if (!f) + { + log::error("open label file %s failed", label_path.c_str()); + return err::ERR_ARGS; + } + std::string line; + while (f->readline(line) > 0) + { + // strip line + line.erase(0, line.find_first_not_of(" \t\r\n")); + line.erase(line.find_last_not_of(" \t\r\n") + 1); + labels.push_back(line); + } + f->close(); + delete f; + return err::ERR_NONE; + } + std::vector *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit) { std::vector *objects = new std::vector(); diff --git a/components/nn/include/maix_nn_yolov8.hpp b/components/nn/include/maix_nn_yolov8.hpp index cfeac659..d0974a18 100644 --- a/components/nn/include/maix_nn_yolov8.hpp +++ b/components/nn/include/maix_nn_yolov8.hpp @@ -40,7 +40,6 @@ namespace maix::nn */ class YOLOv8 { - public: public: /** * Constructor of YOLOv8 class @@ -244,29 +243,6 @@ namespace maix::nn return err::ERR_NONE; } - err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) - { - // load labels from labels file - labels.clear(); - fs::File *f = fs::open(label_path, "r"); - if (!f) - { - log::error("open label file %s failed", label_path.c_str()); - return err::ERR_ARGS; - } - std::string line; - while (f->readline(line) > 0) - { - // strip line - line.erase(0, line.find_first_not_of(" \t\r\n")); - line.erase(line.find_last_not_of(" \t\r\n") + 1); - labels.push_back(line); - } - f->close(); - delete f; - return err::ERR_NONE; - } - /** * Detect objects from image * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method. @@ -461,6 +437,29 @@ namespace maix::nn bool _dual_buff; private: + err::Err _load_labels_from_file(std::vector &labels, const std::string &label_path) + { + // load labels from labels file + labels.clear(); + fs::File *f = fs::open(label_path, "r"); + if (!f) + { + log::error("open label file %s failed", label_path.c_str()); + return err::ERR_ARGS; + } + std::string line; + while (f->readline(line) > 0) + { + // strip line + line.erase(0, line.find_first_not_of(" \t\r\n")); + line.erase(line.find_last_not_of(" \t\r\n") + 1); + labels.push_back(line); + } + f->close(); + delete f; + return err::ERR_NONE; + } + nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit) { nn::Objects *objects = new nn::Objects(); diff --git a/examples/nn_nanotrack/.gitignore b/examples/nn_nanotrack/.gitignore new file mode 100644 index 00000000..7171eaac --- /dev/null +++ b/examples/nn_nanotrack/.gitignore @@ -0,0 +1,9 @@ +build +dist +.config.mk +.flash.conf.json +data + +/CMakeLists.txt + +__pycache__ diff --git a/examples/nn_nanotrack/README.md b/examples/nn_nanotrack/README.md new file mode 100644 index 00000000..2c1cf569 --- /dev/null +++ b/examples/nn_nanotrack/README.md @@ -0,0 +1,9 @@ +nn_yolov8 examples based on MaixCDK +==== +## only run yolov8 model + +- Use `maixcdk build` to compile binary files. +- Move files and runtime libraries to device. +- Use SSH command with parameters to run programs. + +` "Usage: " + std::string(argv[0]) + " mud_model_path "; ` diff --git a/examples/nn_nanotrack/main/CMakeLists.txt b/examples/nn_nanotrack/main/CMakeLists.txt new file mode 100644 index 00000000..452f67bd --- /dev/null +++ b/examples/nn_nanotrack/main/CMakeLists.txt @@ -0,0 +1,74 @@ +############### Add include ################### +list(APPEND ADD_INCLUDE "include" + ) +list(APPEND ADD_PRIVATE_INCLUDE "") +############################################### + +############ Add source files ################# +# list(APPEND ADD_SRCS "src/main.c" +# "src/test.c" +# ) +append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c") +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + +###### Add required/dependent components ###### +list(APPEND ADD_REQUIREMENTS basic nn vision) +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (OpenCV_DIR opencv/lib/cmake/opencv4) +# find_package(OpenCV REQUIRED) +############################################### + +############ Add static libs ################## +# list(APPEND ADD_STATIC_LIB "lib/libtest.a") +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components depend on this component +# list(APPEND ADD_DEFINITIONS -DAAAAA222=1 +# -DAAAAA333=1) +############################################### + +############ Add static libs ################## +#### Update parent's variables like CMAKE_C_LINK_FLAGS +# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE) +############################################### + +######### Add files need to download ######### +# list(APPEND ADD_FILE_DOWNLOADS "{ +# 'url': 'https://*****/abcde.tar.xz', +# 'urls': [], # backup urls, if url failed, will try urls +# 'sites': [], # download site, user can manually download file and put it into dl_path +# 'sha256sum': '', +# 'filename': 'abcde.tar.xz', +# 'path': 'toolchains/xxxxx', +# 'check_files': [] +# }" +# ) +# +# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx, +# you can directly use then, for example use it in add_custom_command +############################################## + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() diff --git a/examples/nn_nanotrack/main/Kconfig b/examples/nn_nanotrack/main/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/examples/nn_nanotrack/main/include/main.h b/examples/nn_nanotrack/main/include/main.h new file mode 100644 index 00000000..45dcbb04 --- /dev/null +++ b/examples/nn_nanotrack/main/include/main.h @@ -0,0 +1,3 @@ +#pragma once + + diff --git a/examples/nn_nanotrack/main/src/main.cpp b/examples/nn_nanotrack/main/src/main.cpp new file mode 100644 index 00000000..1107e74d --- /dev/null +++ b/examples/nn_nanotrack/main/src/main.cpp @@ -0,0 +1,148 @@ + +#include "maix_basic.hpp" +#include "maix_vision.hpp" +#include "maix_touchscreen.hpp" +#include "maix_nn_nanotrack.hpp" +#include "main.h" + +using namespace maix; + +int _main(int argc, char *argv[]) +{ + log::info("Program start"); + std::string model_type = "unknown"; + int ret = 0; + char tmp_char[64]; + + std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path"; + + if (argc < 2) + { + log::info(help.c_str()); + return -1; + } + + const char *model_path = argv[1]; + + nn::NanoTrack tracker(model_path); + log::info("load NanoTrack model %s success", model_path); + + log::info("open camera now"); + display::Display disp = display::Display(); + touchscreen::TouchScreen touch = touchscreen::TouchScreen(); + camera::Camera cam = camera::Camera(disp.width(), disp.height(), tracker.input_format()); + log::info("open camera success"); + int status = 0; // 0: select target box, 1: tracking + bool pressing = false; + nn::Object target; + const char *btn_str = "Select"; + auto font_size = image::string_size(btn_str); + image::Image *ret_img = image::load("/maixapp/share/icon/ret.png", image::Format::FMT_RGB888); + if (!ret_img) + { + log::error("load ret image failed"); + return -12345; + } + + uint64_t t3; + while (!app::need_exit()) + { + uint64_t t = time::ticks_ms(); + maix::image::Image *img = cam.read(); + err::check_null_raise(img, "read camera failed"); + uint64_t t2 = time::ticks_ms(); + auto touch_status = touch.read(); + if (status == 0) + { + if(touch_status[2]) + { + if(!pressing) + { + target.x = touch_status[0]; + target.y = touch_status[1]; + log::info("start select"); + } + pressing = true; + } + else + { + if(pressing) // click + { + target.w = touch_status[0] - target.x; + target.h = touch_status[1] - target.y; + if(target.w > 0 && target.h > 0) + { + log::info("init tracker with rectangle x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h); + tracker.init(*img, target.x, target.y, target.w, target.h); + log::info("init tracker ok"); + status = 1; + } + else + log::info("rectangle invalid, x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h); + } + pressing = false; + } + if(pressing) + { + img->draw_string(2, img->height() - font_size.height() * 2, "Select and release to complete", maix::image::Color::from_rgb(255, 0, 0), 1.5); + img->draw_rect(target.x, target.y, touch_status[0] - target.x, touch_status[1] - target.y, maix::image::Color::from_rgb(255, 0, 0), 3); + } + else + img->draw_string(2, img->height() - font_size.height() * 2, "Select target on screen", maix::image::Color::from_rgb(255, 0, 0), 1.5); + t3 = time::ticks_ms(); + } + else + { + if(touch_status[2]) + { + pressing = true; + } + else + { + if(pressing) // click + { + if(touch_status[0] >= disp.width() - 80 && touch_status[1] >= disp.height() - 60) + { + status = 0; + } + } + pressing = false; + } + nn::Object r = tracker.track(*img); + t3 = time::ticks_ms(); + // log::info("result: %s", r.to_str().c_str()); + img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0), 4); + img->draw_rect(r.points[0], r.points[1], r.points[2], r.points[3], maix::image::Color::from_rgb(158, 158, 158), 1); // input area + img->draw_rect(r.points[4] - r.points[7] / 2, r.points[5] - r.points[7] / 2, r.points[7], r.points[7], maix::image::Color::from_rgb(158, 158, 158), 1); // target size + snprintf(tmp_char, sizeof(tmp_char), "%.2f", r.score); + img->draw_string(r.x, r.y - font_size.height() - 2, tmp_char, maix::image::Color::from_rgb(255, 0, 0), 1.5); + img->draw_rect(disp.width() - 100, disp.height() - 60, 100, 60, maix::image::Color::from_rgb(255, 255, 255), 4); + img->draw_string(disp.width() - 100 + (100 - font_size.width()) / 2, disp.height() - 60 + (60 - font_size.height()) / 2, btn_str, maix::image::Color::from_rgb(255, 255, 255), 1); + } + if(touch_status[2] && touch_status[0] < ret_img->width() && touch_status[1] < ret_img->height()) + { + app::set_exit_flag(true); + } + img->draw_image(0, 0, *ret_img); + disp.show(*img); + delete img; + // log::info("time: all %d ms, track %d ms", time::ticks_ms() - t, t3 - t2); + } + + delete ret_img; + log::info("Program exit"); + + return ret; +} + +int main(int argc, char *argv[]) +{ + // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true. + signal(SIGINT, [](int sig) + { app::set_exit_flag(true); }); + + // Use CATCH_EXCEPTION_RUN_RETURN to catch exception, + // if we don't catch exception, when program throw exception, the objects will not be destructed. + // So we catch exception here to let resources be released(call objects' destructor) before exit. + CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); +} diff --git a/projects/app_self_learn_tracker/.gitignore b/projects/app_self_learn_tracker/.gitignore new file mode 100644 index 00000000..7171eaac --- /dev/null +++ b/projects/app_self_learn_tracker/.gitignore @@ -0,0 +1,9 @@ +build +dist +.config.mk +.flash.conf.json +data + +/CMakeLists.txt + +__pycache__ diff --git a/projects/app_self_learn_tracker/README.md b/projects/app_self_learn_tracker/README.md new file mode 100644 index 00000000..f5fb336e --- /dev/null +++ b/projects/app_self_learn_tracker/README.md @@ -0,0 +1,4 @@ +Self learn tracker +=== + + diff --git a/projects/app_self_learn_tracker/app.yaml b/projects/app_self_learn_tracker/app.yaml new file mode 100644 index 00000000..52ee702e --- /dev/null +++ b/projects/app_self_learn_tracker/app.yaml @@ -0,0 +1,13 @@ +id: self_learn_tracker +name: Self Learn Tracker +name[zh]: 自学习检测器 +version: 1.0.0 +icon: assets/icon.png +author: Sipeed Ltd +desc: Learn and detect and track any object +desc[zh]: 学习、检测、追踪任意物体 +files: + app.yaml: app.yaml + assets: assets + README.md: README.md + diff --git a/projects/app_self_learn_tracker/assets/icon.png b/projects/app_self_learn_tracker/assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..1104aa7bed16862b66149924cb470d7bd9276df9 GIT binary patch literal 887 zcmV--1Bm>IP)Px&F-b&0RCr$Pn#*bxK@f)j|2b6fvQQKynuQA9@&G=7pv1UT6vfNdWUCQH(WOQr z_y9hDS1?(qK~Z#J3>crIoPs%UJe)aQJ>7j`Gd;_5x~r?duCAWHr-+U}?D+gq0F(&m zIDxVSIvxNe37WTni0lS%4!{ur2d$Aq(mcj6n<66Cu>dRscn9Dy$#0f^Cg&pA7_{!*3?}XO!<20Cxdg&J@5sl2<|jd;&0$ zDS#D{`$GW?OF~mtAS7Q|+cqS@6CV9v`sN_Z78nX3bYdQpGBp(u1yul)2pAyXI)%y> zaBZ-LH?gUb0|K+V3%u|DB?;zq3W&&F0F%|Rb-Z5b*8o0{{NlUvz4lo|fQa0$CVCRU z)`p3$RbXC_yyOewzXl*8N2^IM0N8Gv*eZatB;Oj_BN+gz15NEJIj@pb<#e0JBx&0i z?)7oL2ZR_Gk(~g(hu8qHi{y_$8;#vl16cC8J!oV1J9~lTqsTUs0ayfZ zIaepp#b!F`pTd`ZgK$NpcF*V&`*IPl0_QVyVfO zBmi_U5&1Pj^!Tz_Be~7Dlcjx10IK{hTAVPndC*278tkUY0lk4m=KGW84bYviPZoGSu(<(>^jd*@Z|90DU<@ z72>H8B1u2PKX0S_vH;KrLdBA5{KNHc?*S}@h@Gqe_$De#`vM@R?*f(;^=(rkphQ3^ zkSG-*V=a-fCaKUmf;zcbhf7i;6!dY!RifgWnP;i&pgHG96{i>yk$C`DU5+;1xQw?} zc+ibIB(II~-2y<1q`zhgV2`V4+6f?3(r(}un61X1b1j?+Kn+2QrZt@OxG|T`jv2s8 z<)2oj&bx}Fdtl4}TqJY-zW~T} 0 && target.h > 0) + { + log::info("init tracker with rectangle x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h); + tracker.init(*img, target.x, target.y, target.w, target.h); + log::info("init tracker ok"); + status = 1; + } + else + log::info("rectangle invalid, x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h); + } + pressing = false; + } + if(pressing) + { + img->draw_string(2, img->height() - font_size.height() * 2, "Select and release to complete", maix::image::Color::from_rgb(255, 0, 0), 1.5); + img->draw_rect(target.x, target.y, touch_status[0] - target.x, touch_status[1] - target.y, maix::image::Color::from_rgb(255, 0, 0), 3); + } + else + img->draw_string(2, img->height() - font_size.height() * 2, "Select target on screen", maix::image::Color::from_rgb(255, 0, 0), 1.5); + t3 = time::ticks_ms(); + } + else + { + if(touch_status[2]) + { + pressing = true; + } + else + { + if(pressing) // click + { + if(touch_status[0] >= disp.width() - 80 && touch_status[1] >= disp.height() - 60) + { + status = 0; + } + } + pressing = false; + } + nn::Object r = tracker.track(*img); + t3 = time::ticks_ms(); + // log::info("result: %s", r.to_str().c_str()); + img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0), 4); + img->draw_rect(r.points[0], r.points[1], r.points[2], r.points[3], maix::image::Color::from_rgb(158, 158, 158), 1); // input area + img->draw_rect(r.points[4] - r.points[7] / 2, r.points[5] - r.points[7] / 2, r.points[7], r.points[7], maix::image::Color::from_rgb(158, 158, 158), 1); // target size + snprintf(tmp_char, sizeof(tmp_char), "%.2f", r.score); + img->draw_string(r.x, r.y - font_size.height() - 2, tmp_char, maix::image::Color::from_rgb(255, 0, 0), 1.5); + img->draw_rect(disp.width() - 100, disp.height() - 60, 100, 60, maix::image::Color::from_rgb(255, 255, 255), 4); + img->draw_string(disp.width() - 100 + (100 - font_size.width()) / 2, disp.height() - 60 + (60 - font_size.height()) / 2, btn_str, maix::image::Color::from_rgb(255, 255, 255), 1); + } + if(touch_status[2] && touch_status[0] < ret_img->width() && touch_status[1] < ret_img->height()) + { + app::set_exit_flag(true); + } + img->draw_image(0, 0, *ret_img); + disp.show(*img); + delete img; + // log::info("time: all %d ms, track %d ms", time::ticks_ms() - t, t3 - t2); + } + + delete ret_img; + log::info("Program exit"); + + return ret; +} + +int main(int argc, char *argv[]) +{ + // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true. + signal(SIGINT, [](int sig) + { app::set_exit_flag(true); }); + + // Use CATCH_EXCEPTION_RUN_RETURN to catch exception, + // if we don't catch exception, when program throw exception, the objects will not be destructed. + // So we catch exception here to let resources be released(call objects' destructor) before exit. + CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); +}