add hand landmarks support, nn support hwc input, cdk support image <-> CV obj convert

components/maixcam_lib/include/maix_nn_maixcam.hpp

@@ -71,7 +71,7 @@ namespace maix::nn
          * @param[in] img input image
          * @return output tensor
          */
-        virtual tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = std::vector<float>(), std::vector<float> scale = std::vector<float>(), image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool clear_buff = false) final;
+        virtual tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = std::vector<float>(), std::vector<float> scale = std::vector<float>(), image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool clear_buff = false, bool chw = true) final;
 
     private:
         bool _loaded;

components/nn/include/maix_nn.hpp

@@ -245,7 +245,7 @@ namespace maix::nn
          *           default is image.Fit.FIT_FILL for easy coordinate calculation, but for more accurate result, use image.Fit.FIT_CONTAIN is better.
          * @return output tensor
          */
-        virtual tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = {}, std::vector<float> scale = {}, image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool dual_buff_wait = false) = 0;
+        virtual tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = {}, std::vector<float> scale = {}, image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool dual_buff_wait = false, bool chw = true) = 0;
     };
 
     /**
@@ -375,12 +375,13 @@ namespace maix::nn
          * @param copy_result If set true, will copy result to a new variable; else will use a internal memory, you can only use it until to the next forward.
          *                    Default true to avoid problems, you can set it to false manually to make speed faster.
          * @param dual_buff_wait bool type, only for dual_buff mode, if true, will inference this image and wait for result, default false.
+         * @param chw chw channel format, forward model with hwc format image input if set to false, default true(chw).
          * @return output tensor. In C++, you should manually delete tensors in return value and return value.
          *         If dual_buff mode, it can be NULL(None in MaixPy) means not ready.
          * @throw If error occurs, like arg error or alloc memory failed, will raise err.Exception.
          * @maixpy maix.nn.NN.forward_image
          */
-        tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = std::vector<float>(), std::vector<float> scale = std::vector<float>(), image::Fit fit = image::Fit::FIT_FILL, bool copy_result = true, bool dual_buff_wait = false);
+        tensor::Tensors *forward_image(image::Image &img, std::vector<float> mean = std::vector<float>(), std::vector<float> scale = std::vector<float>(), image::Fit fit = image::Fit::FIT_FILL, bool copy_result = true, bool dual_buff_wait = false, bool chw = true);
 
     private:
         MUD _mud;

components/nn/include/maix_nn_hand_landmarks.hpp

@@ -0,0 +1,778 @@
+/**
+ * @author neucrack@sipeed
+ * @copyright Sipeed Ltd 2024-
+ * @license Apache 2.0
+ * @update 2024.12.27: Add hand keypoints support.
+ */
+
+#pragma once
+#include "maix_basic.hpp"
+#include "maix_nn.hpp"
+#include "maix_image.hpp"
+#include "maix_image_cv.hpp"
+#include "maix_nn_F.hpp"
+#include "maix_nn_object.hpp"
+#include <math.h>
+
+#define DRAW_STD_IMG 0
+
+namespace maix::nn
+{
+    /**
+     * HandLandmarks class
+     * @maixpy maix.nn.HandLandmarks
+     */
+    class HandLandmarks
+    {
+    public:
+        /**
+         * Constructor of HandLandmarks class
+         * @param model model path, default empty, you can load model later by load function.
+         * @throw If model arg is not empty and load failed, will throw err::Exception.
+         * @maixpy maix.nn.HandLandmarks.__init__
+         * @maixcdk maix.nn.HandLandmarks.HandLandmarks
+         */
+        HandLandmarks(const string &model = "")
+        {
+            _model = nullptr;
+            _model_detect = nullptr;
+            if (!model.empty())
+            {
+                err::Err e = load(model);
+                if (e != err::ERR_NONE)
+                {
+                    throw err::Exception(e, "load model failed");
+                }
+            }
+        }
+
+        ~HandLandmarks()
+        {
+            if (_model_detect)
+            {
+                delete _model_detect;
+                _model_detect = nullptr;
+            }
+            if (_model)
+            {
+                delete _model;
+                _model = nullptr;
+            }
+        }
+
+        std::string type()
+        {
+            return _extra_info["type"];
+        }
+
+        /**
+         * Load model from file
+         * @param model Model path want to load
+         * @return err::Err
+         * @maixpy maix.nn.HandLandmarks.load
+         */
+        err::Err load(const string &model)
+        {
+            if (_model)
+            {
+                delete _model;
+                _model = nullptr;
+            }
+            _model = new nn::NN(model, false);
+            if (!_model)
+            {
+                return err::ERR_NO_MEM;
+            }
+            _extra_info = _model->extra_info();
+            if (_extra_info.find("model_type") != _extra_info.end())
+            {
+                if (_extra_info["model_type"] != this->type_str)
+                {
+                    log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("model_type key not found");
+                return err::ERR_ARGS;
+            }
+            log::info("model info:\n\ttype: %s", this->type_str.c_str());
+            if (_extra_info.find("input_type") != _extra_info.end())
+            {
+                std::string input_type = _extra_info["input_type"];
+                if (input_type == "rgb")
+                {
+                    _input_img_fmt = maix::image::FMT_RGB888;
+                    log::print("\tinput type: rgb\n");
+                }
+                else if (input_type == "bgr")
+                {
+                    _input_img_fmt = maix::image::FMT_BGR888;
+                    log::print("\tinput type: bgr\n");
+                }
+                else
+                {
+                    log::error("unknown input type: %s", input_type.c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("input_type key not found");
+                return err::ERR_ARGS;
+            }
+            if (_extra_info.find("mean") != _extra_info.end())
+            {
+                std::string mean_str = _extra_info["mean"];
+                std::vector<std::string> mean_strs = split(mean_str, ",");
+                log::print("\tmean:");
+                for (auto &it : mean_strs)
+                {
+                    try
+                    {
+                        this->mean.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("mean value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print("%f ", this->mean.back());
+                }
+                log::print("\n");
+            }
+            else
+            {
+                log::error("mean key not found");
+                return err::ERR_ARGS;
+            }
+            if (_extra_info.find("scale") != _extra_info.end())
+            {
+                std::string scale_str = _extra_info["scale"];
+                std::vector<std::string> scale_strs = split(scale_str, ",");
+                log::print("\tscale:");
+                for (auto &it : scale_strs)
+                {
+                    try
+                    {
+                        this->scale.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("scale value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print("%f ", this->scale.back());
+                }
+                log::print("\n");
+            }
+            else
+            {
+                log::error("scale key not found");
+                return err::ERR_ARGS;
+            }
+            err::Err e = _model->extra_info_labels(labels);
+            if (e == err::Err::ERR_NONE)
+            {
+                log::print("\tlabels num: %ld\n", labels.size());
+            }
+            else
+            {
+                log::error("labels key not found: %s", err::to_str(e).c_str());
+                return err::ERR_ARGS;
+            }
+            if (_extra_info.find("detect_model") != _extra_info.end())
+            {
+                std::string model_file = fs::dirname(model) + "/" + _extra_info["detect_model"];
+                _model_detect = new nn::NN(model_file, false);
+                if (!_model_detect)
+                {
+                    log::error("load detect model failed");
+                    return err::ERR_NO_MEM;
+                }
+            }
+            else
+            {
+                log::error("detect_model key not found");
+                return err::ERR_ARGS;
+            }
+            std::vector<nn::LayerInfo> inputs = _model->inputs_info();
+            std::vector<nn::LayerInfo> inputs_detect = _model_detect->inputs_info();
+            _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]);
+            _input_size_detect = image::Size(inputs_detect[0].shape[2], inputs_detect[0].shape[1]);
+            log::print("\tinput size: %dx%d\n\n", _input_size_detect.width(), _input_size_detect.height());
+
+            // anchors
+            if (_extra_info.find("anchors") != _extra_info.end())
+            {
+                std::string anchors_file = fs::dirname(model) + "/" + _extra_info["anchors"];
+                fs::File *f = fs::open(anchors_file, "r");
+                if(!f)
+                {
+                    log::error("open anchor file %s failed", anchors_file.c_str());
+                    return err::ERR_ARGS;
+                }
+
+                while(1)
+                {
+                    std::string *line = f->readline();
+                    if(!line)
+                        break;
+                    if(!_parse_anchor_line(*line, _anchors))
+                    {
+                        log::error("parse_anchor_line %s failed", line->c_str());
+                        delete line;
+                        delete f;
+                        return err::ERR_ARGS;
+                    }
+                    delete line;
+                }
+                delete f;
+            }
+            else
+            {
+                log::error("detect_model key not found");
+                return err::ERR_ARGS;
+            }
+
+            return err::ERR_NONE;
+        }
+
+        /**
+         * Detect objects from image
+         * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
+         * @param conf_th Hand detect confidence threshold, default 0.7.
+         * @param iou_th IoU threshold, default 0.45.
+         * @param conf_th2 Hand detect confidence second time check threshold, default 0.8.
+         * @param landmarks_rel outputs the relative coordinates of 21 points with respect to the top-left vertex of the hand.
+         *                      In obj.points, the last 21x2 values are arranged as x0y0x1y1...x20y20.
+         *                      Value from 0 to obj.w.
+         * @throw If image format not match model input format, will throw err::Exception.
+         * @return Object list. In C++, you should delete it after use.
+         *         Object's points value format: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y,
+         *                          x0, y0, z1, x1, y1, z2, ..., x20, y20, z20.
+         *         If landmarks_rel is True, will be box_topleft_x, box_topleft_y...,x20,y20,z20,x0_rel,y0_rel,...,x20_rel,y20_rel.
+         *         Z is depth, the larger the value, the farther away from the palm, and the positive value means closer to the camera.
+         * @maixpy maix.nn.HandLandmarks.detect
+         */
+        nn::Objects *detect(image::Image &img, float conf_th = 0.7, float iou_th = 0.45, float conf_th2 = 0.8, bool landmarks_rel = false)
+        {
+            maix::image::Fit fit = maix::image::FIT_CONTAIN;
+            this->_conf_th = conf_th;
+            this->_iou_th = iou_th;
+            this->_conf_th2 = conf_th2;
+            if (img.format() != _input_img_fmt)
+            {
+                throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
+            }
+            image::Image *detect_img_input = &img;
+            bool resized = false;
+            if (img.width() != _input_size_detect.width() || img.height() != _input_size_detect.height())
+            {
+                detect_img_input = img.resize(_input_size_detect.width(), _input_size_detect.height(), fit);
+                resized = true;
+            }
+            nn::Objects *objs = new nn::Objects();
+            tensor::Tensors *outputs;
+            outputs = _model_detect->forward_image(*detect_img_input, this->mean, this->scale, fit, false, true, false);
+            if (resized)
+                delete detect_img_input;
+            if (!outputs) // not ready, return empty result.
+            {
+                return objs;
+            }
+            _decode_objs(*objs, outputs, conf_th, _input_size_detect.width(), _input_size_detect.height(), resized, img.width(), img.height());
+            delete outputs;
+            if (objs->size() > 0)
+            {
+                nn::Objects *objects_total = objs;
+                objs = _nms(*objs);
+                delete objects_total;
+            }
+            std::vector<cv::Mat> M_inverse;
+            std::vector<image::Image *> landmarks_input = _crop_reize_hand(*objs, img, _input_size.width(), _input_size.height(), M_inverse, landmarks_rel);
+            #if DRAW_STD_IMG
+                int y=0;
+            #endif
+            bool have_invalid = false;
+            for(size_t i=0; i<landmarks_input.size(); ++i)
+            {
+                #if DRAW_STD_IMG
+                    img.draw_image(0, y, *landmarks_input[i]);
+                    y += landmarks_input[i]->height();
+                #endif
+                outputs = _model->forward_image(*landmarks_input[i], this->mean, this->scale, fit, false, true, false);
+                delete landmarks_input[i];
+                if (!outputs) // not ready, return empty result.
+                {
+                    return objs;
+                }
+                have_invalid |= _decode_landmarks(*objs, i, outputs, conf_th2, M_inverse, _input_size.width(), _input_size.height(), img.width(), img.height(), landmarks_rel);
+                delete outputs;
+            }
+            if(have_invalid)
+            {
+                nn::Objects *res = new nn::Objects();
+                for(size_t i = 0; i < objs->size(); ++i)
+                {
+                    nn::Object &obj = objs->at(i);
+                    if(obj.score > 0)
+                        res->add(obj);
+                }
+                delete objs;
+                return res;
+            }
+            return objs;
+        }
+
+        /**
+         * Get model input size
+         * @param detect detect or landmarks model, default true.
+         * @return model input size
+         * @maixpy maix.nn.HandLandmarks.input_size
+         */
+        image::Size input_size(bool detect = true)
+        {
+            if (detect)
+                return _input_size_detect;
+            return _input_size;
+        }
+
+        /**
+         * Get model input width
+         * @param detect detect or landmarks model, default true.
+         * @return model input size of width
+         * @maixpy maix.nn.HandLandmarks.input_width
+         */
+        int input_width(bool detect = true)
+        {
+            if (detect)
+                return _input_size_detect.width();
+            return _input_size.width();
+        }
+
+        /**
+         * Get model input height
+         * @param detect detect or landmarks model, default true.
+         * @return model input size of height
+         * @maixpy maix.nn.HandLandmarks.input_height
+         */
+        int input_height(bool detect = true)
+        {
+            if (detect)
+                return _input_size_detect.height();
+            return _input_size.height();
+        }
+
+        /**
+         * Get input image format
+         * @return input image format, image::Format type.
+         * @maixpy maix.nn.HandLandmarks.input_format
+         */
+        image::Format input_format()
+        {
+            return _input_img_fmt;
+        }
+
+        /**
+         * Draw hand and landmarks on image
+         * @param img image object, maix.image.Image type.
+         * @param leftright, 0 means left, 1 means right
+         * @param points points result from detect method: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y,
+         *                          x0, y0, z1, x1, y1, z2, ..., x20, y20, z20
+         * @param r_min min radius of points.
+         * @param r_max min radius of points.
+         * @param box draw box or not, default true.
+         * @param box_color color of box.
+         * @maixpy maix.nn.HandLandmarks.draw_hand
+         */
+        void draw_hand(image::Image &img, int leftright, const std::vector<int> &points, int r_min = 4, int r_max = 10, bool box = true, int box_thickness = 1, image::Color box_color_l = image::COLOR_RED, image::Color box_color_r = image::COLOR_GREEN)
+        {
+            if (points.size() < 71)
+            {
+                throw std::runtime_error("keypoints size must 72");
+            }
+            if (box)
+            {
+                img.draw_line(points[0], points[1], points[2], points[3], leftright == 0 ? box_color_l : box_color_r, box_thickness);
+                img.draw_line(points[2], points[3], points[4], points[5], leftright == 0 ? box_color_l : box_color_r, box_thickness);
+                img.draw_line(points[4], points[5], points[6], points[7], leftright == 0 ? box_color_l : box_color_r, box_thickness);
+                img.draw_line(points[6], points[7], points[0], points[1], leftright == 0 ? box_color_l : box_color_r, box_thickness);
+            }
+            int max_z = INT_MIN, min_z = INT_MAX;
+            for (int i = 0; i < 21; ++i)
+            {
+                if (points[10 + i * 3] > max_z)
+                    max_z = points[10 + i * 3];
+                else if (points[10 + i * 3] < min_z)
+                    min_z = points[10 + i * 3];
+            }
+            float range = 1.0 / (max_z - min_z);
+            for (int i = 0; i < 5; ++i)
+            {
+                float radius = (points[10 + i * 3] - min_z) * range;
+                uint8_t c = (uint8_t)(radius * 255);
+                auto color = image::Color::from_rgb(c, c, c);
+                int r = (int)(radius * r_max);
+                if (r < r_min)
+                    r = r_min;
+                img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1);
+            }
+            for (int i = 5; i < 9; ++i)
+            {
+                float radius = (points[10 + i * 3] - min_z) * range;
+                auto color = image::Color::from_rgb(radius * 255, 0, 0);
+                int r = (int)(radius * r_max);
+                if (r < r_min)
+                    r = r_min;
+                img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1);
+            }
+            for (int i = 9; i < 13; ++i)
+            {
+                float radius = (points[10 + i * 3] - min_z) * range;
+                auto color = image::Color::from_rgb(0, radius * 255, 0);
+                int r = (int)(radius * r_max);
+                if (r < r_min)
+                    r = r_min;
+                img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1);
+            }
+            for (int i = 13; i < 17; ++i)
+            {
+                float radius = (points[10 + i * 3] - min_z) * range;
+                auto color = image::Color::from_rgb(0, 0, radius * 255);
+                int r = (int)(radius * r_max);
+                if (r < r_min)
+                    r = r_min;
+                img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1);
+            }
+            for (int i = 17; i < 21; ++i)
+            {
+                float radius = (points[10 + i * 3] - min_z) * range;
+                uint8_t c = (uint8_t)(radius * 255);
+                auto color = image::Color::from_rgb(0, c, c);
+                int r = (int)(radius * r_max);
+                if (r < r_min)
+                    r = r_min;
+                img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1);
+            }
+        }
+
+    public:
+        /**
+         * Labels list
+         * @maixpy maix.nn.HandLandmarks.labels
+         */
+        std::vector<string> labels;
+
+        /**
+         * Label file path
+         * @maixpy maix.nn.HandLandmarks.label_path
+         */
+        std::string label_path;
+
+        /**
+         * Get mean value, list type
+         * @maixpy maix.nn.HandLandmarks.mean
+         */
+        std::vector<float> mean;
+
+        /**
+         * Get scale value, list type
+         * @maixpy maix.nn.HandLandmarks.scale
+         */
+        std::vector<float> scale;
+
+    protected:
+        std::string type_str = "hand_landmarks";
+
+    private:
+        image::Size _input_size;
+        image::Size _input_size_detect;
+        image::Format _input_img_fmt;
+        nn::NN *_model;
+        nn::NN *_model_detect;
+        std::map<string, string> _extra_info;
+        float _conf_th = 0.7;
+        float _iou_th = 0.45;
+        float _conf_th2 = 0.8;
+        std::vector<std::vector<float>> _anchors;
+
+    private:
+        bool _parse_anchor_line(std::string &line, std::vector<std::vector<float>> &anchors)
+        {
+            std::vector<std::string> tokens;
+            split0(tokens, line, ",");
+            if(tokens.size() != 4)
+                return false;
+            std::vector<float> anchor = {std::stof(tokens[0]) * _input_size_detect.width(), std::stof(tokens[1]) * _input_size_detect.height(), std::stof(tokens[2]), std::stof(tokens[3])};
+            anchors.push_back(anchor);
+            return true;
+        }
+
+        void _decode_objs(nn::Objects &objects, tensor::Tensors *outputs, const float &conf_th, int input_w, int input_h, bool resized, int img_w, int img_h)
+        {
+            tensor::Tensor *score_out = NULL; // shape 1, 2016, 18, 1
+            tensor::Tensor *box_out = NULL;   // shape 1,  2016, 1, 1
+            int bbox_size = 2016;
+            for (auto i : *outputs)
+            {
+                if (i.second->shape()[2] == 18)
+                {
+                    box_out = i.second;
+                    bbox_size = i.second->shape()[1];
+                }
+                else
+                {
+                    score_out = i.second;
+                }
+            }
+            if (!box_out)
+            {
+                throw err::Exception(err::ERR_ARGS, "wrong model");
+            }
+            float *scores = (float*)score_out->data();
+            float *bboxes = (float*)box_out->data();
+            for (int i = 0; i < bbox_size; ++i)
+            {
+                scores[i] = _sigmoid(scores[i]);
+                if(scores[i] >= conf_th)
+                {
+                    float *p = bboxes + i*18;
+                    int w = p[2] * _anchors[i][2];
+                    int h = p[3] * _anchors[i][3];
+                    int x = p[0] + _anchors[i][0] - w * 0.5;
+                    int y = p[1] + _anchors[i][1] - h * 0.5;
+                    int p0_x = _anchors[i][0] + p[4];
+                    int p0_y = _anchors[i][1] + p[5];
+                    int p2_x = _anchors[i][0] + p[8];
+                    int p2_y = _anchors[i][1] + p[9];
+                    float angle = atan2(p2_y - p0_y, p2_x - p0_x) + M_PI;
+                    if(resized)
+                    {
+                        auto bbox = image::resize_map_pos_reverse(img_w, img_h, input_w, input_h, image::FIT_CONTAIN, x, y, w, h);
+                        auto p2 = image::resize_map_pos_reverse(img_w, img_h, input_w, input_h, image::FIT_CONTAIN, p2_x, p2_y);
+                        objects.add(bbox[0], bbox[1], bbox[2], bbox[3], 0, scores[i], std::vector<int>({p2[0], p2[1]}), angle);
+                    }
+                    else
+                        objects.add(x, y, w, h, 0, scores[i], std::vector<int>({p2_x, p2_y}), angle);
+                }
+            }
+        }
+
+        std::vector<image::Image *> _crop_reize_hand(nn::Objects &objs, image::Image &img, int input_w, int input_h, std::vector<cv::Mat> &M_inverse, bool landmarks_rel)
+        {
+            std::vector<image::Image *> res;
+            float dscale = 2.6;
+            for (size_t i = 0; i < objs.size(); ++i)
+            {
+                nn::Object &obj = objs.at(i);
+                float dy = -0.5 * sin(obj.angle);
+                int cx = (int)((obj.x + obj.w * 0.5 + obj.points[0]) * 0.5);
+                int cy = (int)((obj.y + obj.h * 0.5 + obj.w * dy + obj.points[1]) * 0.5);
+                // img.draw_circle(cx, cy, 5, image::COLOR_RED, -1);
+                // img.draw_circle(obj.points[0], obj.points[1], 5, image::COLOR_YELLOW, -1);
+                obj.points.resize(71 + (landmarks_rel ? 21 * 2 : 0), 0);
+                int hand_size = (int)(obj.w * dscale);
+                float theta = obj.angle - M_PI * 0.5;
+                cv::Mat A = (cv::Mat_<float>(4, 2) << -1, -1, -1, 1, 1, 1, 1, -1);
+                cv::Mat R = (cv::Mat_<float>(2, 2) << cos(theta), sin(theta), -sin(theta), cos(theta));
+                cv::Mat C;
+                float half_w = hand_size * 0.5;
+                cv::gemm(A, R, half_w, cv::Mat(), 0.0, C);
+                for (int i = 0; i < C.rows; ++i) {
+                    C.at<float>(i, 0) += cx;
+                    C.at<float>(i, 1) += cy;
+                    obj.points[i*2] = C.at<float>(i, 0);
+                    obj.points[1 + i*2] = C.at<float>(i, 1);
+                }
+                obj.x = cx - half_w;
+                obj.y = cy - half_w;
+                obj.w = hand_size;
+                obj.h = hand_size;
+                cv::Mat element = C(cv::Range(0, 3), cv::Range(0, 2));
+                cv::Mat dst = (cv::Mat_<float>(3, 2) << 0, 0, 0, input_h, input_w, input_h);
+                auto M = cv::getAffineTransform(element, dst);
+                image::Image *img_dst = new image::Image(input_w, input_h, image::FMT_RGB888);
+                if(!img_dst)
+                {
+                    log::error("no memory");
+                    break;
+                }
+                cv::Mat img_dst2(input_h, input_w, CV_8UC3, img_dst->data());
+                cv::Mat img_src;
+                image::image2cv(img, img_src, false, false);
+                cv::warpAffine(img_src, img_dst2, M, cv::Size(input_w, input_h), cv::INTER_NEAREST);
+                assert(img_dst2.data() == img_dst->data());
+                res.push_back(img_dst);
+                cv::Mat iM;
+                cv::invertAffineTransform(M, iM);
+                M_inverse.push_back(iM);
+            }
+            return res;
+        }
+
+        bool _decode_landmarks(nn::Objects &objs, int idx, tensor::Tensors *outputs, float conf_th2, std::vector<cv::Mat> &M_inverse, int input_w, int input_h, int img_w, int img_h, bool landmarks_rel)
+        {
+            tensor::Tensor *leftright_out = NULL;   // shape 1,  1, 1, 1
+            tensor::Tensor *score_out = NULL; // shape 1, 1, 1, 1
+            tensor::Tensor *points_out = NULL;   // shape 1,  63, 1, 1
+            for (auto i : *outputs)
+            {
+                if (i.second->shape()[1] == 63)
+                {
+                    points_out = i.second;
+                }
+                else if(i.first.find("score") != std::string::npos)
+                {
+                    score_out = i.second;
+                }
+                else
+                {
+                    leftright_out = i.second;
+                }
+            }
+            if (!leftright_out || !score_out || !points_out)
+            {
+                throw err::Exception(err::ERR_ARGS, "wrong model");
+            }
+            auto &obj = objs.at(idx);
+            float score = _sigmoid(*(float*)score_out->data());
+            float leftright = _sigmoid(*(float*)leftright_out->data());
+            float *points = (float*)points_out->data();
+            if(score < conf_th2)
+            {
+                obj.score = 0;
+                return true; // invalid
+            }
+            obj.score = score;
+            obj.class_id = leftright > 0.5 ? 1 : 0;
+            cv::Mat iM;
+            cv::transpose(M_inverse[idx], iM);
+            cv::Mat A = cv::Mat_<double>(21, 3);
+            for (int i = 0; i < A.rows; ++i) {
+                A.at<double>(i, 0) = points[i*3];
+                A.at<double>(i, 1) = points[i*3 + 1];
+                A.at<double>(i, 2) = 1;
+                obj.points[8 + i*3 + 2] = (int)(-points[i*3 + 2] * obj.w);
+            }
+            cv::Mat C;
+            cv::gemm(A, iM, 1.0, cv::Mat(), 0.0, C);
+            for (int i = 0; i < C.rows; ++i) {
+                obj.points[8 + 3*i] = C.at<double>(i, 0);
+                obj.points[9 + 3*i] = C.at<double>(i, 1);
+            }
+            if(landmarks_rel)
+            {
+                for (int i = 0; i < 21; ++i)
+                {
+                    obj.points[71 + i * 2] = points[i*3];
+                    obj.points[72 + i * 2] = points[i*3 + 1];
+                }
+            }
+            // update angle
+            obj.angle = atan2(obj.points[36] - obj.points[9], obj.points[35] - obj.points[8]) + M_PI;
+            return false; // valid
+        }
+
+        nn::Objects *_nms(nn::Objects &objs)
+        {
+            nn::Objects *result = new nn::Objects();
+            std::sort(objs.begin(), objs.end(), [](const nn::Object *a, const nn::Object *b)
+                      { return a->score > b->score; });
+            for (size_t i = 0; i < objs.size(); ++i)
+            {
+                nn::Object &a = objs.at(i);
+                if (a.score == 0)
+                    continue;
+                for (size_t j = i + 1; j < objs.size(); ++j)
+                {
+                    nn::Object &b = objs.at(j);
+                    {
+                        if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
+                        {
+                            b.score = 0;
+                        }
+                    }
+                }
+            }
+            for (nn::Object *a : objs)
+            {
+                if (a->score != 0)
+                {
+                    result->add(a->x, a->y, a->w, a->h, a->class_id, a->score, a->points, a->angle);
+                }
+            }
+            return result;
+        }
+
+        inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); }
+
+        inline static float _calc_iou(Object &a, Object &b)
+        {
+            float area1 = a.w * a.h;
+            float area2 = b.w * b.h;
+            float wi = std::min((a.x + a.w), (b.x + b.w)) -
+                       std::max(a.x, b.x);
+            float hi = std::min((a.y + a.h), (b.y + b.h)) -
+                       std::max(a.y, b.y);
+            float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
+            return area_i / (area1 + area2 - area_i);
+        }
+
+        template <typename T>
+        static int _argmax(const T *data, size_t len, size_t stride = 1)
+        {
+            int maxIndex = 0;
+            for (size_t i = 1; i < len; i++)
+            {
+                int idx = i * stride;
+                if (data[maxIndex * stride] < data[idx])
+                {
+                    maxIndex = i;
+                }
+            }
+            return maxIndex;
+        }
+
+        template <typename T>
+        static int _argmax2(const T *data, size_t end, size_t stride = 1)
+        {
+            int maxIndex = 0;
+            for (size_t i = stride; i < end; i += stride)
+            {
+                if (data[maxIndex] < data[i])
+                {
+                    maxIndex = i;
+                }
+            }
+            return maxIndex;
+        }
+
+        static void split0(std::vector<std::string> &items, const std::string &s, const std::string &delimiter)
+        {
+            items.clear();
+            size_t pos_start = 0, pos_end, delim_len = delimiter.length();
+            std::string token;
+
+            while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
+            {
+                token = s.substr(pos_start, pos_end - pos_start);
+                pos_start = pos_end + delim_len;
+                token.erase(0, token.find_first_not_of(" \t\r\n"));
+                token.erase(token.find_last_not_of(" \t\r\n") + 1);
+                items.push_back(token);
+            }
+
+            items.push_back(s.substr(pos_start));
+        }
+
+        static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
+        {
+            std::vector<std::string> tokens;
+            split0(tokens, s, delimiter);
+            return tokens;
+        }
+    };
+
+} // namespace maix::nn

components/nn/include/maix_nn_object.hpp

@@ -300,6 +300,21 @@ namespace maix::nn
             return *obj;
         }
 
+        /**
+         * Add object to objects
+         * @throw Throw exception if no memory
+         * @maixcdk maix.nn.Objects.add
+         */
+        nn::Object &add(const nn::Object &obj)
+        {
+            Object *obj_new = new Object(obj.x, obj.y, obj.w, obj.h, obj.class_id, obj.score, obj.points, obj.angle);
+            if(!obj_new)
+                throw err::Exception(err::ERR_NO_MEM);
+            obj_new->seg_mask = NULL;
+            objs.push_back(obj_new);
+            return *obj_new;
+        }
+
         /**
          * Remove object form objects
          * @maixpy maix.nn.Objects.remove

components/nn/src/maix_nn.cpp

@@ -304,9 +304,9 @@ namespace maix::nn
         return _impl->forward(inputs, copy_result, dual_buff_wait);
     }
 
-    tensor::Tensors *NN::forward_image(image::Image &img, std::vector<float> mean, std::vector<float> scale, image::Fit fit, bool copy_result, bool dual_buff_wait)
+    tensor::Tensors *NN::forward_image(image::Image &img, std::vector<float> mean, std::vector<float> scale, image::Fit fit, bool copy_result, bool dual_buff_wait, bool chw)
     {
-        return _impl->forward_image(img, mean, scale, fit, copy_result, dual_buff_wait);
+        return _impl->forward_image(img, mean, scale, fit, copy_result, dual_buff_wait, chw);
     }
 
     int SelfLearnClassifier::learn()

components/vision/include/maix_image_cv.hpp

@@ -0,0 +1,31 @@
+#include "maix_image.hpp"
+#include "opencv2/opencv.hpp"
+
+namespace maix::image
+{
+
+    /**
+     * Convert image::Image object to opencv Mat object
+     * @param[in] img image::Image object
+     * @param[out] mat cv::Mat object, output image, will be allocated by this function.
+     * @param[in] ensure_bgr auto convert to BGR888 or BGRA8888 if img format is not BGR or BGRA, if set to false, will not auto convert and directly use img's data, default false.
+     *                       If copy is false, ensure_bgr always be false.
+     * @param[in] copy Whether alloc new image and copy data or not, if ensure_bgr and img is not bgr or bgra format, always copy,
+     *        if not copy, array object will directly use img's data buffer, will faster but change array will affect img's data, default true.
+     * @maixcdk maix.image.image2cv
+     */
+    err::Err image2cv(image::Image &img, cv::Mat &mat, bool ensure_bgr = false, bool copy = true);
+
+
+    /**
+     * OpenCV Mat object to Image object
+     * @param mat cv::Mat image object.
+     * @param bgr if set bgr, the return image will be marked as BGR888 or BGRA8888 format, grayscale will ignore this arg.
+     * @param copy if true, will alloc new buffer and copy data, else will directly use array's data buffer, default true.
+     *        Use this arg carefully, when set to false, ther array MUST keep alive until we don't use the return img of this func, or will cause program crash.
+     * @return Image object
+     * @maixcdk maix.image.cv2image
+     */
+    image::Image *cv2image(cv::Mat &mat, bool bgr = true, bool copy = true);
+
+}

examples/nn_hand_keypoints/.gitignore

@@ -0,0 +1,9 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+
+/CMakeLists.txt
+
+__pycache__

examples/nn_hand_keypoints/README.md

@@ -0,0 +1,13 @@
+Detect hand 21 keypoints
+====
+
+
+- Use `maixcdk build` to compile binary files.
+- Move files and runtime libraries to device.
+- Use SSH command with parameters to run programs.
+
+` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `
+
+more documentations refer to [MaixPy](https://wiki.sipeed.com/maixpy/doc/zh/vision/hand_landmarks.html)
+
+

examples/nn_hand_keypoints/main/CMakeLists.txt

@@ -0,0 +1,74 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic nn vision)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+# 'url': 'https://*****/abcde.tar.xz',
+# 'urls': [],  # backup urls, if url failed, will try urls
+# 'sites': [], # download site, user can manually download file and put it into dl_path
+# 'sha256sum': '',
+# 'filename': 'abcde.tar.xz',
+# 'path': 'toolchains/xxxxx',
+# 'check_files': []
+# }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

examples/nn_hand_keypoints/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

examples/nn_hand_keypoints/main/src/main.cpp

@@ -0,0 +1,123 @@
+
+#include "maix_basic.hpp"
+#include "maix_vision.hpp"
+#include "maix_nn_hand_landmarks.hpp"
+#include "main.h"
+
+using namespace maix;
+
+int _main(int argc, char *argv[])
+{
+    log::info("Program start");
+    std::string model_type = "unknown";
+    int ret = 0;
+    err::Err e;
+    maix::image::Format img_fmt = maix::image::FMT_RGB888;
+    std::vector<float> mean = {};
+    std::vector<float> scale = {};
+    char tmp_chars[64] = {0};
+
+    std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>";
+
+    if (argc < 2)
+    {
+        log::info(help.c_str());
+        return -1;
+    }
+
+    const char *model_path = argv[1];
+    float conf_threshold = 0.7;
+    float iou_threshold = 0.45;
+    float conf_threshold2 = 0.8;
+    bool landmarks_rel = true; // draw relative landmarks on image
+
+    nn::HandLandmarks detector("");
+    e = detector.load(model_path);
+    err::check_raise(e, "load model failed");
+    log::info("load yolo11 model %s success", model_path);
+
+    if (argc >= 3)
+    {
+        const char *img_path = argv[2];
+        log::info("load image now");
+        maix::image::Image *img = maix::image::load(img_path, img_fmt);
+        err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
+        log::info("load image %s success: %s", img_path, img->to_str().c_str());
+        log::info("detect now");
+        nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold, conf_threshold2, landmarks_rel);
+        if (result->size() == 0)
+        {
+            log::info("no object detected !");
+        }
+        for (auto &r : *result)
+        {
+            // points:  box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y,
+            //                       x0, y0, z1, x1, y1, z2, ..., x20, y20, z20,[x0_rel, y0_rel,...,x20_rel, y20_rel]
+            log::info("result: %s", r->to_str().c_str());
+            snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", r->points[0] == 0 ? "left" : "right", r->score);
+            img->draw_string(r->points[0], r->points[1], detector.labels[r->class_id],
+                                 r->class_id == 0 ? maix::image::Color::from_rgb(255, 0, 0) : maix::image::Color::from_rgb(0, 255, 0),
+                                 1.2, 2);
+            detector.draw_hand(*img, r->class_id, r->points, 4, 10, true);
+        }
+        img->save("result.jpg");
+        delete result;
+        delete img;
+    }
+    else
+    {
+        log::info("open camera now");
+        camera::Camera cam = camera::Camera(320, 320, detector.input_format());
+        log::info("open camera success");
+        display::Display disp = display::Display();
+        while (!app::need_exit())
+        {
+            uint64_t t = time::ticks_ms();
+            maix::image::Image *img = cam.read();
+            err::check_null_raise(img, "read camera failed");
+            uint64_t t2 = time::ticks_ms();
+            nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold, conf_threshold2, landmarks_rel);
+            uint64_t t3 = time::ticks_ms();
+            for (auto &r : *result)
+            {
+                // points:  box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y,
+                //                       x0, y0, z1, x1, y1, z2, ..., x20, y20, z20,[x0_rel, y0_rel,...,x20_rel, y20_rel]
+                log::info("result: %s", r->to_str().c_str());
+                snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", r->points[0] == 0 ? "left" : "right", r->score);
+                img->draw_string(r->points[0], r->points[1], detector.labels[r->class_id],
+                                 r->class_id == 0 ? maix::image::Color::from_rgb(255, 0, 0) : maix::image::Color::from_rgb(0, 255, 0),
+                                 1.4, 2);
+                // img->draw_rect(r->x, r->y, r->w, r->h, image::COLOR_RED, 1);
+                detector.draw_hand(*img, r->class_id, r->points, 4, 10, true);
+                if(landmarks_rel)
+                {
+                    img->draw_rect(0, 0, detector.input_width(false), detector.input_height(false), image::COLOR_YELLOW, 1);
+                    for(int i=0; i<21; i++)
+                    {
+                        img->draw_circle(r->points[8+21*3 + i*2], r->points[9+21*3 + i*2], 2, image::COLOR_YELLOW, -1);
+                    }
+                }
+            }
+            disp.show(*img);
+            delete result;
+            delete img;
+            log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2);
+        }
+    }
+
+    log::info("Program exit");
+
+    return ret;
+}
+
+int main(int argc, char *argv[])
+{
+    // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+    signal(SIGINT, [](int sig)
+           { app::set_exit_flag(true); });
+
+    // Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
+    // if we don't catch exception, when program throw exception, the objects will not be destructed.
+    // So we catch exception here to let resources be released(call objects' destructor) before exit.
+    CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
+}