add self learn detector/tracker support

components/basic/include/maix_tensor.hpp

@@ -134,9 +134,10 @@ namespace maix
              * @param dtype tensor element data type, see DType of this module
              * @param data pointer to data content, can be nullptr, it will automatically alloc memory
              *             and detroy it when this object is destroyed
+             * @param copy defalt true to alloc new memory and copy from data.
              * @maixcdk maix.tensor.Tensor.Tensor
              */
-            Tensor(std::vector<int> shape, tensor::DType dtype, void *data)
+            Tensor(std::vector<int> shape, tensor::DType dtype, void *data, bool copy = true)
             {
                 _shape = shape;
                 _dtype = dtype;
@@ -147,11 +148,15 @@ namespace maix
                 {
                     size *= shape[i];
                 }
-                if(!_data)
+                if((!_data) || (data && copy))
                 {
                     _data = malloc(size * dtype_size[dtype]);
                     _is_alloc = true;
                     log::debug("malloc tensor data\n");
+                    if(data)
+                    {
+                        memcpy(_data, data, size * dtype_size[dtype]);
+                    }
                 }
                 // log::info("new tensor: %p", this);
             }
@@ -528,6 +533,19 @@ namespace maix
                 tensors.erase(key);
             }
 
+            /**
+             * Clear tensors
+             * @maixpy maix.tensor.Tensors.clear
+            */
+            void clear()
+            {
+                auto _keys = keys();
+                for(const auto &k : _keys)
+                {
+                    rm_tensor(k);
+                }
+            }
+
             /**
              * Begin of tensors
              * @maixcdk maix.tensor.Tensors.begin

components/nn/include/maix_nn_nanotrack.hpp

@@ -0,0 +1,681 @@
+/**
+ * @author neucrack@sipeed
+ * @copyright Sipeed Ltd 2023-
+ * @license Apache 2.0
+ * @update 2024.9.2: Create this file.
+ */
+
+#pragma once
+#include "maix_basic.hpp"
+#include "maix_nn.hpp"
+#include "maix_image.hpp"
+#include "maix_nn_F.hpp"
+#include "maix_nn_object.hpp"
+#include <cmath>
+
+namespace maix::nn
+{
+    /**
+     * NanoTrack class
+     * @maixpy maix.nn.NanoTrack
+     */
+    class NanoTrack
+    {
+    public:
+        /**
+         * Constructor of NanoTrack class
+         * @param model model path, default empty, you can load model later by load function.
+         * @throw If model arg is not empty and load failed, will throw err::Exception.
+         * @maixpy maix.nn.NanoTrack.__init__
+         * @maixcdk maix.nn.NanoTrack.NanoTrack
+         */
+        NanoTrack(const string &model = "")
+        {
+            _model = nullptr;
+            _model_target = nullptr;
+            _model_head = nullptr;
+            _dual_buff = false;
+            if (!model.empty())
+            {
+                err::Err e = load(model);
+                if (e != err::ERR_NONE)
+                {
+                    throw err::Exception(e, "load model failed");
+                }
+            }
+
+            // nanotrack params
+            _context_amount = 0.5;
+            _stride = 16;
+            _head_out_size = 15;
+            _penalty_k = 0.138;
+            _window_influence = 0.455;
+            _lr = 0.348;
+            _generate_window(_head_out_size, _window);
+            _points = _generate_points(_stride, _head_out_size);
+            _bbox.resize(_head_out_size * _head_out_size);
+            _penalty.resize(_head_out_size * _head_out_size);
+        }
+
+        ~NanoTrack()
+        {
+            _free_models();
+        }
+
+        /**
+         * Load model from file
+         * @param model Model path want to load
+         * @return err::Err
+         * @maixpy maix.nn.NanoTrack.load
+         */
+        err::Err load(const string &model)
+        {
+            _free_models();
+            _model = new nn::NN(model, _dual_buff);
+            if (!_model)
+            {
+                return err::ERR_NO_MEM;
+            }
+            _extra_info = _model->extra_info();
+            if (_extra_info.find("model_type") != _extra_info.end())
+            {
+                if (_extra_info["model_type"] != "nanotrack")
+                {
+                    log::error("model_type not match, expect 'nanotrack', but got '%s'", _extra_info["model_type"].c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("model_type key not found");
+                return err::ERR_ARGS;
+            }
+            log::info("model info:\n\ttype: nanotrack");
+            if (_extra_info.find("input_type") != _extra_info.end())
+            {
+                std::string input_type = _extra_info["input_type"];
+                if (input_type == "rgb")
+                {
+                    _input_img_fmt = maix::image::FMT_RGB888;
+                    log::print("\tinput type: rgb\n");
+                }
+                else if (input_type == "bgr")
+                {
+                    _input_img_fmt = maix::image::FMT_BGR888;
+                    log::print("\tinput type: bgr\n");
+                }
+                else
+                {
+                    log::error("unknown input type: %s", input_type.c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("input_type key not found");
+                return err::ERR_ARGS;
+            }
+            if (_extra_info.find("mean") != _extra_info.end())
+            {
+                std::string mean_str = _extra_info["mean"];
+                std::vector<std::string> mean_strs = split(mean_str, ",");
+                log::print("\tmean:");
+                for (auto &it : mean_strs)
+                {
+                    try
+                    {
+                        this->mean.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("mean value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print("%f ", this->mean.back());
+                }
+                log::print("\n");
+            }
+            else
+            {
+                log::error("mean use 0");
+            }
+            if (_extra_info.find("scale") != _extra_info.end())
+            {
+                std::string scale_str = _extra_info["scale"];
+                std::vector<std::string> scale_strs = split(scale_str, ",");
+                log::print("\tscale:");
+                for (auto &it : scale_strs)
+                {
+                    try
+                    {
+                        this->scale.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("scale value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print("%f ", this->scale.back());
+                }
+                log::print("\n");
+            }
+            else
+            {
+                log::error("scale use 1");
+            }
+            std::string model_dir = fs::abspath(fs::dirname(model));
+            if (_extra_info.find("target_model") != _extra_info.end())
+            {
+                std::string model_target = _extra_info["target_model"];
+                _model_target = new nn::NN(model_dir + "/" + model_target, _dual_buff);
+                if (!_model_target)
+                {
+                    delete _model;
+                    _model = nullptr;
+                    return err::ERR_NO_MEM;
+                }
+            }
+            else
+            {
+                log::error("target_model key not found");
+                return err::ERR_ARGS;
+            }
+            if (_extra_info.find("head_model") != _extra_info.end())
+            {
+                std::string model_head = _extra_info["head_model"];
+                _model_head = new nn::NN(model_dir + "/" + model_head, _dual_buff);
+                if (!_model_head)
+                {
+                    delete _model_target;
+                    _model_target = nullptr;
+                    delete _model;
+                    _model = nullptr;
+                    return err::ERR_NO_MEM;
+                }
+            }
+            else
+            {
+                log::error("target_model key not found");
+                return err::ERR_ARGS;
+            }
+            std::vector<nn::LayerInfo> inputs = _model->inputs_info();
+            _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
+            inputs = _model_target->inputs_info();
+            _input_size_target = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
+            log::print("\tinput size: %dx%d, target size: %dx%d\n\n", _input_size.width(), _input_size.height(), _input_size_target.width(), _input_size_target.height());
+            return err::ERR_NONE;
+        }
+
+        /**
+         * Init tracker, give tacker first target image and target position.
+         * @param img Image want to detect, target should be in this image.
+         * @param x the target position left top coordinate x.
+         * @param y the target position left top coordinate y.
+         * @param w the target width.
+         * @param h the target height.
+         * @throw If image format not match model input format, will throw err::Exception.
+         * @maixpy maix.nn.NanoTrack.init
+         */
+        void init(image::Image &img, int x, int y, int w, int h)
+        {
+            if (img.format() != _input_img_fmt)
+            {
+                throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
+            }
+            // crop image
+            _get_avg_color(img, _input_img_fmt);
+            int c_x = x + w / 2;
+            int c_y = y + h / 2;
+            float size_target = std::sqrt((w + _context_amount * (w + h)) * (h + _context_amount * (w + h)));
+            int size_target_int = std::round(size_target);
+            int crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2;
+            image::Image *img_target = _padding_crop(img, c_x, c_y, size_target_int, crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2);
+
+            // forward target feature
+            tensor::Tensors *outputs;
+            outputs = _model_target->forward_image(*img_target, this->mean, this->scale, image::FIT_FILL, false, true);
+            delete img_target;
+            _head_inputs.clear();
+            for (auto it = outputs->begin(); it != outputs->end(); it++)
+            {
+                _head_inputs.add_tensor("input1", it->second, true, true);
+                break;
+            }
+            _last_target.x = c_x; // center, not left-top
+            _last_target.y = c_y; // center, not left-top
+            _last_target.w = w;
+            _last_target.h = h;
+            delete outputs;
+        }
+
+        /**
+         * Track object acoording to last object position and the init function learned target feature.
+         * @param img image to detect object and track, can be any resolution, before detect it will crop a area according to last time target's position.
+         * @param threshold If score < threshold, will see this new detection is invalid, but remain return this new detecion,  default 0.9.
+         * @return object, position and score, and detect area in points's first 4 element(x, y, w, h, center_x, center_y, input_size, target_size)
+         * @maixpy maix.nn.NanoTrack.track
+        */
+        nn::Object track(image::Image &img, float threshold = 0.9)
+        {
+            if (img.format() != _input_img_fmt)
+            {
+                throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
+            }
+            // crop image
+            float size_target = std::sqrt((_last_target.w + _context_amount * (_last_target.w + _last_target.h)) * (_last_target.h + _context_amount * (_last_target.w + _last_target.h)));
+            int size_x = std::round(size_target * _input_size.width() / _input_size_target.width());
+            float scale_z = _input_size_target.width() / size_target;
+            int crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2;
+            image::Image *img_target = _padding_crop(img, _last_target.x, _last_target.y, size_x, crop_x_x1, crop_x_y1, crop_x_x2, crop_x_y2);
+
+            // forward target feature
+            tensor::Tensors *outputs;
+            outputs = _model->forward_image(*img_target, this->mean, this->scale, image::FIT_FILL, false, true);
+            delete img_target;
+            for (auto it = outputs->begin(); it != outputs->end(); it++)
+            {
+                _head_inputs.add_tensor("input2", it->second, false, false);
+                break;
+            }
+
+            // forward head
+            tensor::Tensors *outputs2  = NULL;
+            tensor::Tensor *score_out  = NULL; // 1x2x15x15
+            tensor::Tensor *bbox_out = NULL;  // 1x4x15x15
+            outputs2 = _model_head->forward(_head_inputs, false, true);
+            delete outputs;
+            for (auto it = outputs2->begin(); it != outputs2->end(); it++)
+            {
+                if (it->second->shape()[1] == 2)
+                    score_out = it->second;
+                else if (it->second->shape()[1] == 4)
+                    bbox_out = it->second;
+            }
+            if(!bbox_out || ! score_out)
+            {
+                delete outputs2;
+                throw err::Exception(err::ERR_ARGS, "wrong model");
+            }
+
+            // decode score and bbox
+            float *score = (float *)score_out->data();
+            _softmax_2xn(score, _points.size());
+            score += _points.size(); // we only use the target prob
+            float *bbox = (float *)bbox_out->data();
+            int stride1 = _points.size();
+            int stride2 = stride1 * 2;
+            int stride3 = stride1 * 3;
+            float max_score = 0;
+            int best_idx = 0;
+            for (size_t i = 0; i < _points.size(); ++i)
+            {
+                nn::ObjectFloat &box = _bbox.at(i);
+                box.x = _points[i][0] - bbox[i];
+                box.y = _points[i][1] - bbox[i + stride1];
+                box.w = _points[i][0] + bbox[i + stride2];
+                box.h = _points[i][1] + bbox[i + stride3];
+                // center bbox
+                box.w = box.w - box.x;
+                box.h = box.h - box.y;
+                box.x += box.w / 2.0;
+                box.y += box.h / 2.0;
+                // penalty
+                float s_c = _change(_sz(box.w, box.h) / _sz(_last_target.w * scale_z, _last_target.h * scale_z));
+                float r_c = _change(((float)_last_target.w / _last_target.h) / (box.w / box.h));
+                _penalty[i] = std::exp(-(r_c * s_c - 1) * _penalty_k);
+                score[i] *= _penalty[i];
+                score[i] = score[i] * (1 - _window_influence) + _window[i] * _window_influence;
+                if (score[i] > max_score)
+                {
+                    max_score = score[i];
+                    best_idx = i;
+                }
+            }
+            nn::ObjectFloat &box = _bbox.at(best_idx);
+            float temp = 1.0 / scale_z;
+            box.x *= temp;
+            box.y *= temp;
+            box.w *= temp;
+            box.h *= temp;
+            int cx = box.x + _last_target.x;
+            int cy = box.y + _last_target.y;
+            // smooth
+            float lr = _penalty[best_idx] * max_score * _lr;
+            int w = _last_target.w * (1 - lr) + box.w * lr;
+            int h = _last_target.h * (1 - lr) + box.h * lr;
+            _bbox_clip(cx, cy, w, h, img.width(), img.height());
+
+            // update status
+            if(max_score >= threshold)
+            {
+                _last_target.x = cx;
+                _last_target.y = cy;
+                _last_target.w = w;
+                _last_target.h = h;
+            }
+
+            // clear data
+            _head_inputs.rm_tensor("input2");
+            delete outputs2;
+            nn::Object res(cx - w / 2, cy - h / 2, w, h, 0, max_score);
+            res.points.resize(8);
+            res.points.at(0) = crop_x_x1;
+            res.points.at(1) = crop_x_y1;
+            res.points.at(2) = crop_x_x2 - crop_x_x1;
+            res.points.at(3) = crop_x_y2 - crop_x_y1;
+            res.points.at(4) = cx;
+            res.points.at(5) = cy;
+            res.points.at(6) = size_x;
+            res.points.at(7) = size_target;
+            return res;
+        }
+
+        /**
+         * Get model input size
+         * @return model input size
+         * @maixpy maix.nn.NanoTrack.input_size
+         */
+        image::Size input_size()
+        {
+            return _input_size;
+        }
+
+        /**
+         * Get model input width
+         * @return model input size of width
+         * @maixpy maix.nn.NanoTrack.input_width
+         */
+        int input_width()
+        {
+            return _input_size.width();
+        }
+
+        /**
+         * Get model input height
+         * @return model input size of height
+         * @maixpy maix.nn.NanoTrack.input_height
+         */
+        int input_height()
+        {
+            return _input_size.height();
+        }
+
+        /**
+         * Get input image format
+         * @return input image format, image::Format type.
+         * @maixpy maix.nn.NanoTrack.input_format
+         */
+        image::Format input_format()
+        {
+            return _input_img_fmt;
+        }
+
+    public:
+        /**
+         * Get mean value, list type
+         * @maixpy maix.nn.NanoTrack.mean
+         */
+        std::vector<float> mean;
+
+        /**
+         * Get scale value, list type
+         * @maixpy maix.nn.NanoTrack.scale
+         */
+        std::vector<float> scale;
+
+    private:
+        image::Size _input_size;
+        image::Size _input_size_target;
+        image::Format _input_img_fmt;
+        nn::NN *_model;
+        nn::NN *_model_target;
+        nn::NN *_model_head;
+        std::map<string, string> _extra_info;
+        bool _dual_buff;
+        float _context_amount;
+        int _stride;
+        int _head_out_size;
+        float _penalty_k;
+        float _window_influence;
+        float _lr;
+        uint8_t _avg_color[3];
+        std::vector<std::vector<float>> _points;
+        std::vector<nn::ObjectFloat> _bbox;
+        tensor::Tensors _head_inputs;
+        nn::Object _last_target;
+        std::vector<float> _penalty;
+        std::vector<float> _window;
+
+    private:
+        inline float _sz(float w, float h)
+        {
+            float pad = (w + h) * 0.5;
+            return std::sqrt((w + pad) * (h + pad));
+        }
+
+        inline float _change(float r)
+        {
+            float t = 1.0 / r;
+            return r > t ? r : t;
+        }
+
+        inline void _bbox_clip(int &cx, int &cy, int &w, int &h, int max_w, int max_h)
+        {
+            cx = max(0, min(cx, max_w - 1));
+            cy = max(0, min(cy, max_h - 1));
+            w = max(10, min(w, max_w));
+            h = max(10, min(h, max_h));
+        }
+
+        void _free_models()
+        {
+            if (_model)
+            {
+                delete _model;
+                _model = nullptr;
+            }
+            if (_model_target)
+            {
+                delete _model_target;
+                _model_target = nullptr;
+            }
+            if (_model_head)
+            {
+                delete _model_head;
+                _model_head = nullptr;
+            }
+        }
+
+        void _get_avg_color(image::Image &img, const image::Format &fmt)
+        {
+            uint8_t max_v_ch1 = 0, max_v_ch2 = 0, max_v_ch3 = 0;
+            uint8_t min_v_ch1 = 255, min_v_ch2 = 255, min_v_ch3 = 255;
+            uint8_t *p = (uint8_t *)img.data();
+            for (int i = 0; i < img.data_size() / 3; ++i)
+            {
+                if (p[i * 3] > max_v_ch1)
+                    max_v_ch1 = p[i * 3];
+                else if (p[i * 3] < min_v_ch1)
+                    min_v_ch1 = p[i * 3];
+                if (p[i * 3 + 1] > max_v_ch2)
+                    max_v_ch2 = p[i * 3 + 1];
+                else if (p[i * 3 + 1] < min_v_ch2)
+                    min_v_ch2 = p[i * 3 + 1];
+                if (p[i * 3 + 2] > max_v_ch3)
+                    max_v_ch3 = p[i * 3 + 2];
+                else if (p[i * 3 + 2] < min_v_ch3)
+                    min_v_ch3 = p[i * 3 + 2];
+            }
+            _avg_color[0] = (max_v_ch1 - min_v_ch1) / 2;
+            _avg_color[1] = (max_v_ch2 - min_v_ch2) / 2;
+            _avg_color[2] = (max_v_ch3 - min_v_ch3) / 2;
+        }
+
+        image::Image *_padding_crop(image::Image &img, int c_x, int c_y, int size_target, int &copy_x1, int &copy_y1, int &copy_x2, int &copy_y2)
+        {
+            int x1 = c_x - size_target / 2;
+            int y1 = c_y - size_target / 2;
+            int x2 = c_x + size_target / 2;
+            int y2 = c_y + size_target / 2;
+            // not pad
+            if (x1 >= 0 && y1 >= 0 && x2 < img.width() && y2 < img.height())
+            {
+                copy_x1 = x1;
+                copy_y1 = y1;
+                copy_x2 = x2;
+                copy_y2 = y2;
+                return img.crop(x1, y1, size_target, size_target);
+            }
+            image::Image *res = new image::Image(size_target, size_target, img.format());
+            if (!res)
+                throw err::Exception(err::ERR_NO_MEM);
+            copy_x1 = max(0, x1);
+            copy_y1 = max(0, y1);
+            copy_x2 = min(img.width(), x2);
+            copy_y2 = min(img.height(), y2);
+            uint8_t *data = (uint8_t *)res->data();
+            uint8_t *src = (uint8_t *)img.data();
+            // pad
+            for (int y = 0; y < copy_y1 - y1; ++y)
+            {
+                for (int x = 0; x < x2 - x1; ++x)
+                {
+                    uint8_t *p = data + (y * res->width() + x) * 3;
+                    *p = _avg_color[0];
+                    *(p + 1) = _avg_color[1];
+                    *(p + 2) = _avg_color[2];
+                }
+            }
+            for (int y = copy_y2; y < y2; ++y)
+            {
+                for (int x = 0; x < x2 - x1; ++x)
+                {
+                    uint8_t *p = data + ((y +  - y1) * res->width() + x) * 3;
+                    *p = _avg_color[0];
+                    *(p + 1) = _avg_color[1];
+                    *(p + 2) = _avg_color[2];
+                }
+            }
+            for (int y = copy_y1; y < copy_y2; ++y)
+            {
+                for (int x = 0; x < copy_x1 - x1; ++x)
+                {
+                    uint8_t *p = data + ((y - y1) * res->width() + x) * 3;
+                    *p = _avg_color[0];
+                    *(p + 1) = _avg_color[1];
+                    *(p + 2) = _avg_color[2];
+                }
+            }
+            for (int y = copy_y1; y < copy_y2; ++y)
+            {
+                for (int x = copy_x2; x < x2; ++x)
+                {
+                    uint8_t *p = data + ((y - y1) * res->width() + x - x1) * 3;
+                    *p = _avg_color[0];
+                    *(p + 1) = _avg_color[1];
+                    *(p + 2) = _avg_color[2];
+                }
+            }
+            for (int y = copy_y1; y < copy_y2; ++y)
+            {
+                uint8_t *p = data + ((y - y1) * res->width() + (copy_x1 - x1)) * 3;
+                uint8_t *s = src + (y * img.width() + copy_x1) * 3;
+                int length = (copy_x2 - copy_x1) * 3;
+                memcpy(p, s, length);
+            }
+            // res->save("/root/test.png");
+            return res;
+        }
+
+        std::vector<std::vector<float>> _generate_points(int stride = 16, int size = 15)
+        {
+            // 计算原点位置
+            float ori = -(size / 2) * stride;
+
+            // 创建points向量
+            std::vector<std::vector<float>> points(size * size, std::vector<float>(2, 0.0f));
+
+            int index = 0;
+            for (int dy = 0; dy < size; ++dy)
+            {
+                for (int dx = 0; dx < size; ++dx)
+                {
+                    float x = ori + stride * dx;
+                    float y = ori + stride * dy;
+                    points[index][0] = x;
+                    points[index][1] = y;
+                    ++index;
+                }
+            }
+
+            return points;
+        }
+
+        std::vector<float> _generate_hanning_window(int size)
+        {
+            std::vector<float> hanning(size);
+            for (int i = 0; i < size; ++i)
+            {
+                hanning[i] = 0.5f * (1.0f - std::cos(2.0f * M_PI * i / (size - 1)));
+            }
+            return hanning;
+        }
+
+        void _generate_window(int size, std::vector<float> &window)
+        {
+            std::vector<float> hanning = _generate_hanning_window(size);
+
+            window.resize(size * size);
+            for (int i = 0; i < size; ++i)
+            {
+                for (int j = 0; j < size; ++j)
+                {
+                    window[i * size + j] = hanning[i] * hanning[j];
+                }
+            }
+        }
+
+        static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
+        {
+            items.clear();
+            size_t pos_start = 0, pos_end, delim_len = delimiter.length();
+            std::string token;
+
+            while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
+            {
+                token = s.substr(pos_start, pos_end - pos_start);
+                pos_start = pos_end + delim_len;
+                items.push_back(token);
+            }
+
+            items.push_back(s.substr(pos_start));
+        }
+
+        static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
+        {
+            std::vector<std::string> tokens;
+            split0(tokens, s, delimiter);
+            return tokens;
+        }
+
+        static void _softmax_2xn(float *data, int size)
+        {
+            float *p1 = data;
+            float *p2 = data + size;
+            float large;
+            float sum;
+            for(int i=0; i< size; ++i)
+            {
+                large = *p1 > *p2 ? *p1 : *p2;
+                *p1 = expf(*p1 - large);
+                *p2 = expf(*p2 - large);
+                sum = *p1 + *p2;
+                *p1 /= sum;
+                *p2 /= sum;
+                ++p1;
+                ++p2;
+            }
+        }
+    };
+
+} // namespace maix::nn

components/nn/include/maix_nn_yolov5.hpp

@@ -20,8 +20,6 @@ namespace maix::nn
     */
     class YOLOv5
     {
-    public:
-
     public:
         /**
          * Constructor of YOLOv5 class
@@ -231,29 +229,6 @@ namespace maix::nn
             return err::ERR_NONE;
         }
 
-        err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
-        {
-            // load labels from labels file
-            labels.clear();
-            fs::File *f = fs::open(label_path, "r");
-            if (!f)
-            {
-                log::error("open label file %s failed", label_path.c_str());
-                return err::ERR_ARGS;
-            }
-            std::string line;
-            while (f->readline(line) > 0)
-            {
-                // strip line
-                line.erase(0, line.find_first_not_of(" \t\r\n"));
-                line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
-            }
-            f->close();
-            delete f;
-            return err::ERR_NONE;
-        }
-
         /**
          * Detect objects from image
          * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
@@ -368,6 +343,29 @@ namespace maix::nn
         bool _dual_buff;
 
     private:
+        err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
+        {
+            // load labels from labels file
+            labels.clear();
+            fs::File *f = fs::open(label_path, "r");
+            if (!f)
+            {
+                log::error("open label file %s failed", label_path.c_str());
+                return err::ERR_ARGS;
+            }
+            std::string line;
+            while (f->readline(line) > 0)
+            {
+                // strip line
+                line.erase(0, line.find_first_not_of(" \t\r\n"));
+                line.erase(line.find_last_not_of(" \t\r\n") + 1);
+                labels.push_back(line);
+            }
+            f->close();
+            delete f;
+            return err::ERR_NONE;
+        }
+
         std::vector<nn::Object> *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit)
         {
             std::vector<nn::Object> *objects = new std::vector<nn::Object>();

components/nn/include/maix_nn_yolov8.hpp

@@ -40,7 +40,6 @@ namespace maix::nn
      */
     class YOLOv8
     {
-    public:
     public:
         /**
          * Constructor of YOLOv8 class
@@ -244,29 +243,6 @@ namespace maix::nn
             return err::ERR_NONE;
         }
 
-        err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
-        {
-            // load labels from labels file
-            labels.clear();
-            fs::File *f = fs::open(label_path, "r");
-            if (!f)
-            {
-                log::error("open label file %s failed", label_path.c_str());
-                return err::ERR_ARGS;
-            }
-            std::string line;
-            while (f->readline(line) > 0)
-            {
-                // strip line
-                line.erase(0, line.find_first_not_of(" \t\r\n"));
-                line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
-            }
-            f->close();
-            delete f;
-            return err::ERR_NONE;
-        }
-
         /**
          * Detect objects from image
          * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
@@ -461,6 +437,29 @@ namespace maix::nn
         bool _dual_buff;
 
     private:
+        err::Err _load_labels_from_file(std::vector<std::string> &labels, const std::string &label_path)
+        {
+            // load labels from labels file
+            labels.clear();
+            fs::File *f = fs::open(label_path, "r");
+            if (!f)
+            {
+                log::error("open label file %s failed", label_path.c_str());
+                return err::ERR_ARGS;
+            }
+            std::string line;
+            while (f->readline(line) > 0)
+            {
+                // strip line
+                line.erase(0, line.find_first_not_of(" \t\r\n"));
+                line.erase(line.find_last_not_of(" \t\r\n") + 1);
+                labels.push_back(line);
+            }
+            f->close();
+            delete f;
+            return err::ERR_NONE;
+        }
+
         nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit)
         {
             nn::Objects *objects = new nn::Objects();

examples/nn_nanotrack/.gitignore

@@ -0,0 +1,9 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+
+/CMakeLists.txt
+
+__pycache__

examples/nn_nanotrack/README.md

@@ -0,0 +1,9 @@
+nn_yolov8 examples based on MaixCDK
+====
+## only run yolov8 model
+
+- Use `maixcdk build` to compile binary files.
+- Move files and runtime libraries to device.
+- Use SSH command with parameters to run programs.
+  
+` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

examples/nn_nanotrack/main/CMakeLists.txt

@@ -0,0 +1,74 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic nn vision)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+# 'url': 'https://*****/abcde.tar.xz',
+# 'urls': [],  # backup urls, if url failed, will try urls
+# 'sites': [], # download site, user can manually download file and put it into dl_path
+# 'sha256sum': '',
+# 'filename': 'abcde.tar.xz',
+# 'path': 'toolchains/xxxxx',
+# 'check_files': []
+# }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

examples/nn_nanotrack/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

examples/nn_nanotrack/main/src/main.cpp

@@ -0,0 +1,148 @@
+
+#include "maix_basic.hpp"
+#include "maix_vision.hpp"
+#include "maix_touchscreen.hpp"
+#include "maix_nn_nanotrack.hpp"
+#include "main.h"
+
+using namespace maix;
+
+int _main(int argc, char *argv[])
+{
+    log::info("Program start");
+    std::string model_type = "unknown";
+    int ret = 0;
+    char tmp_char[64];
+
+    std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path";
+
+    if (argc < 2)
+    {
+        log::info(help.c_str());
+        return -1;
+    }
+
+    const char *model_path = argv[1];
+
+    nn::NanoTrack tracker(model_path);
+    log::info("load NanoTrack model %s success", model_path);
+
+    log::info("open camera now");
+    display::Display disp = display::Display();
+    touchscreen::TouchScreen touch = touchscreen::TouchScreen();
+    camera::Camera cam = camera::Camera(disp.width(), disp.height(), tracker.input_format());
+    log::info("open camera success");
+    int status = 0; // 0: select target box, 1: tracking
+    bool pressing = false;
+    nn::Object target;
+    const char *btn_str = "Select";
+    auto font_size = image::string_size(btn_str);
+    image::Image *ret_img = image::load("/maixapp/share/icon/ret.png", image::Format::FMT_RGB888);
+    if (!ret_img)
+    {
+        log::error("load ret image failed");
+        return -12345;
+    }
+
+    uint64_t t3;
+    while (!app::need_exit())
+    {
+        uint64_t t = time::ticks_ms();
+        maix::image::Image *img = cam.read();
+        err::check_null_raise(img, "read camera failed");
+        uint64_t t2 = time::ticks_ms();
+        auto touch_status = touch.read();
+        if (status == 0)
+        {
+            if(touch_status[2])
+            {
+                if(!pressing)
+                {
+                    target.x = touch_status[0];
+                    target.y = touch_status[1];
+                    log::info("start select");
+                }
+                pressing = true;
+            }
+            else
+            {
+                if(pressing) // click
+                {
+                    target.w = touch_status[0] - target.x;
+                    target.h = touch_status[1] - target.y;
+                    if(target.w > 0 && target.h > 0)
+                    {
+                        log::info("init tracker with rectangle x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h);
+                        tracker.init(*img, target.x, target.y, target.w, target.h);
+                        log::info("init tracker ok");
+                        status = 1;
+                    }
+                    else
+                        log::info("rectangle invalid, x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h);
+                }
+                pressing = false;
+            }
+            if(pressing)
+            {
+                img->draw_string(2, img->height() - font_size.height() * 2, "Select and release to complete", maix::image::Color::from_rgb(255, 0, 0), 1.5);
+                img->draw_rect(target.x, target.y, touch_status[0] - target.x, touch_status[1] - target.y, maix::image::Color::from_rgb(255, 0, 0), 3);
+            }
+            else
+                img->draw_string(2, img->height() - font_size.height() * 2, "Select target on screen", maix::image::Color::from_rgb(255, 0, 0), 1.5);
+            t3 = time::ticks_ms();
+        }
+        else
+        {
+            if(touch_status[2])
+            {
+                pressing = true;
+            }
+            else
+            {
+                if(pressing) // click
+                {
+                    if(touch_status[0] >= disp.width() - 80 && touch_status[1] >= disp.height() - 60)
+                    {
+                        status = 0;
+                    }
+                }
+                pressing = false;
+            }
+            nn::Object r = tracker.track(*img);
+            t3 = time::ticks_ms();
+            // log::info("result: %s", r.to_str().c_str());
+            img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0), 4);
+            img->draw_rect(r.points[0], r.points[1], r.points[2], r.points[3], maix::image::Color::from_rgb(158, 158, 158), 1); // input area
+            img->draw_rect(r.points[4] - r.points[7] / 2, r.points[5] - r.points[7] / 2, r.points[7], r.points[7], maix::image::Color::from_rgb(158, 158, 158), 1); // target size
+            snprintf(tmp_char, sizeof(tmp_char), "%.2f", r.score);
+            img->draw_string(r.x, r.y - font_size.height() - 2, tmp_char, maix::image::Color::from_rgb(255, 0, 0), 1.5);
+            img->draw_rect(disp.width() - 100, disp.height() - 60, 100, 60, maix::image::Color::from_rgb(255, 255, 255), 4);
+            img->draw_string(disp.width() - 100 + (100 - font_size.width()) / 2, disp.height() - 60 + (60 - font_size.height()) / 2, btn_str, maix::image::Color::from_rgb(255, 255, 255), 1);
+        }
+        if(touch_status[2] && touch_status[0] < ret_img->width() && touch_status[1] < ret_img->height())
+        {
+            app::set_exit_flag(true);
+        }
+        img->draw_image(0, 0, *ret_img);
+        disp.show(*img);
+        delete img;
+        // log::info("time: all %d ms, track %d ms", time::ticks_ms() - t, t3 - t2);
+    }
+
+    delete ret_img;
+    log::info("Program exit");
+
+    return ret;
+}
+
+int main(int argc, char *argv[])
+{
+    // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+    signal(SIGINT, [](int sig)
+           { app::set_exit_flag(true); });
+
+    // Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
+    // if we don't catch exception, when program throw exception, the objects will not be destructed.
+    // So we catch exception here to let resources be released(call objects' destructor) before exit.
+    CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
+}

projects/app_self_learn_tracker/.gitignore

@@ -0,0 +1,9 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+
+/CMakeLists.txt
+
+__pycache__

projects/app_self_learn_tracker/README.md

@@ -0,0 +1,4 @@
+Self learn tracker
+===
+
+

projects/app_self_learn_tracker/app.yaml

@@ -0,0 +1,13 @@
+id: self_learn_tracker
+name: Self Learn Tracker
+name[zh]: 自学习检测器
+version: 1.0.0
+icon: assets/icon.png
+author: Sipeed Ltd
+desc: Learn and detect and track any object
+desc[zh]: 学习、检测、追踪任意物体
+files:
+  app.yaml: app.yaml
+  assets: assets
+  README.md: README.md
+

projects/app_self_learn_tracker/main/CMakeLists.txt

@@ -0,0 +1,74 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic nn vision)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+# 'url': 'https://*****/abcde.tar.xz',
+# 'urls': [],  # backup urls, if url failed, will try urls
+# 'sites': [], # download site, user can manually download file and put it into dl_path
+# 'sha256sum': '',
+# 'filename': 'abcde.tar.xz',
+# 'path': 'toolchains/xxxxx',
+# 'check_files': []
+# }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

projects/app_self_learn_tracker/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

projects/app_self_learn_tracker/main/src/main.cpp

@@ -0,0 +1,141 @@
+
+#include "maix_basic.hpp"
+#include "maix_vision.hpp"
+#include "maix_touchscreen.hpp"
+#include "maix_nn_nanotrack.hpp"
+#include "main.h"
+
+using namespace maix;
+
+int _main(int argc, char *argv[])
+{
+    log::info("Program start");
+    std::string model_type = "unknown";
+    int ret = 0;
+    char tmp_char[64];
+
+
+    const char *model_path = "/root/models/nanotrack.mud";
+
+    nn::NanoTrack tracker(model_path);
+    log::info("load NanoTrack model %s success", model_path);
+
+    log::info("open camera now");
+    display::Display disp = display::Display();
+    touchscreen::TouchScreen touch = touchscreen::TouchScreen();
+    camera::Camera cam = camera::Camera(disp.width(), disp.height(), tracker.input_format());
+    log::info("open camera success");
+    int status = 0; // 0: select target box, 1: tracking
+    bool pressing = false;
+    nn::Object target;
+    const char *btn_str = "Select";
+    auto font_size = image::string_size(btn_str);
+    image::Image *ret_img = image::load("/maixapp/share/icon/ret.png", image::Format::FMT_RGB888);
+    if (!ret_img)
+    {
+        log::error("load ret image failed");
+        return -12345;
+    }
+
+    uint64_t t3;
+    while (!app::need_exit())
+    {
+        uint64_t t = time::ticks_ms();
+        maix::image::Image *img = cam.read();
+        err::check_null_raise(img, "read camera failed");
+        uint64_t t2 = time::ticks_ms();
+        auto touch_status = touch.read();
+        if (status == 0)
+        {
+            if(touch_status[2])
+            {
+                if(!pressing)
+                {
+                    target.x = touch_status[0];
+                    target.y = touch_status[1];
+                    log::info("start select");
+                }
+                pressing = true;
+            }
+            else
+            {
+                if(pressing) // click
+                {
+                    target.w = touch_status[0] - target.x;
+                    target.h = touch_status[1] - target.y;
+                    if(target.w > 0 && target.h > 0)
+                    {
+                        log::info("init tracker with rectangle x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h);
+                        tracker.init(*img, target.x, target.y, target.w, target.h);
+                        log::info("init tracker ok");
+                        status = 1;
+                    }
+                    else
+                        log::info("rectangle invalid, x: %d, y: %d, w: %d, h: %d", target.x, target.y, target.w, target.h);
+                }
+                pressing = false;
+            }
+            if(pressing)
+            {
+                img->draw_string(2, img->height() - font_size.height() * 2, "Select and release to complete", maix::image::Color::from_rgb(255, 0, 0), 1.5);
+                img->draw_rect(target.x, target.y, touch_status[0] - target.x, touch_status[1] - target.y, maix::image::Color::from_rgb(255, 0, 0), 3);
+            }
+            else
+                img->draw_string(2, img->height() - font_size.height() * 2, "Select target on screen", maix::image::Color::from_rgb(255, 0, 0), 1.5);
+            t3 = time::ticks_ms();
+        }
+        else
+        {
+            if(touch_status[2])
+            {
+                pressing = true;
+            }
+            else
+            {
+                if(pressing) // click
+                {
+                    if(touch_status[0] >= disp.width() - 80 && touch_status[1] >= disp.height() - 60)
+                    {
+                        status = 0;
+                    }
+                }
+                pressing = false;
+            }
+            nn::Object r = tracker.track(*img);
+            t3 = time::ticks_ms();
+            // log::info("result: %s", r.to_str().c_str());
+            img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0), 4);
+            img->draw_rect(r.points[0], r.points[1], r.points[2], r.points[3], maix::image::Color::from_rgb(158, 158, 158), 1); // input area
+            img->draw_rect(r.points[4] - r.points[7] / 2, r.points[5] - r.points[7] / 2, r.points[7], r.points[7], maix::image::Color::from_rgb(158, 158, 158), 1); // target size
+            snprintf(tmp_char, sizeof(tmp_char), "%.2f", r.score);
+            img->draw_string(r.x, r.y - font_size.height() - 2, tmp_char, maix::image::Color::from_rgb(255, 0, 0), 1.5);
+            img->draw_rect(disp.width() - 100, disp.height() - 60, 100, 60, maix::image::Color::from_rgb(255, 255, 255), 4);
+            img->draw_string(disp.width() - 100 + (100 - font_size.width()) / 2, disp.height() - 60 + (60 - font_size.height()) / 2, btn_str, maix::image::Color::from_rgb(255, 255, 255), 1);
+        }
+        if(touch_status[2] && touch_status[0] < ret_img->width() && touch_status[1] < ret_img->height())
+        {
+            app::set_exit_flag(true);
+        }
+        img->draw_image(0, 0, *ret_img);
+        disp.show(*img);
+        delete img;
+        // log::info("time: all %d ms, track %d ms", time::ticks_ms() - t, t3 - t2);
+    }
+
+    delete ret_img;
+    log::info("Program exit");
+
+    return ret;
+}
+
+int main(int argc, char *argv[])
+{
+    // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+    signal(SIGINT, [](int sig)
+           { app::set_exit_flag(true); });
+
+    // Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
+    // if we don't catch exception, when program throw exception, the objects will not be destructed.
+    // So we catch exception here to let resources be released(call objects' destructor) before exit.
+    CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
+}