diff --git a/components/maixcam_lib/include/maix_nn_maixcam.hpp b/components/maixcam_lib/include/maix_nn_maixcam.hpp index 1b1d55f2..1486935f 100644 --- a/components/maixcam_lib/include/maix_nn_maixcam.hpp +++ b/components/maixcam_lib/include/maix_nn_maixcam.hpp @@ -71,7 +71,7 @@ namespace maix::nn * @param[in] img input image * @return output tensor */ - virtual tensor::Tensors *forward_image(image::Image &img, std::vector mean = std::vector(), std::vector scale = std::vector(), image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool clear_buff = false) final; + virtual tensor::Tensors *forward_image(image::Image &img, std::vector mean = std::vector(), std::vector scale = std::vector(), image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool clear_buff = false, bool chw = true) final; private: bool _loaded; diff --git a/components/maixcam_lib/lib/libmaixcam_lib.so b/components/maixcam_lib/lib/libmaixcam_lib.so index 878103fa..10004242 100755 Binary files a/components/maixcam_lib/lib/libmaixcam_lib.so and b/components/maixcam_lib/lib/libmaixcam_lib.so differ diff --git a/components/nn/include/maix_nn.hpp b/components/nn/include/maix_nn.hpp index f3c5187e..f71e7bb8 100644 --- a/components/nn/include/maix_nn.hpp +++ b/components/nn/include/maix_nn.hpp @@ -245,7 +245,7 @@ namespace maix::nn * default is image.Fit.FIT_FILL for easy coordinate calculation, but for more accurate result, use image.Fit.FIT_CONTAIN is better. * @return output tensor */ - virtual tensor::Tensors *forward_image(image::Image &img, std::vector mean = {}, std::vector scale = {}, image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool dual_buff_wait = false) = 0; + virtual tensor::Tensors *forward_image(image::Image &img, std::vector mean = {}, std::vector scale = {}, image::Fit fit = image::Fit::FIT_CONTAIN, bool copy_result = true, bool dual_buff_wait = false, bool chw = true) = 0; }; /** @@ -375,12 +375,13 @@ namespace maix::nn * @param copy_result If set true, will copy result to a new variable; else will use a internal memory, you can only use it until to the next forward. * Default true to avoid problems, you can set it to false manually to make speed faster. * @param dual_buff_wait bool type, only for dual_buff mode, if true, will inference this image and wait for result, default false. + * @param chw chw channel format, forward model with hwc format image input if set to false, default true(chw). * @return output tensor. In C++, you should manually delete tensors in return value and return value. * If dual_buff mode, it can be NULL(None in MaixPy) means not ready. * @throw If error occurs, like arg error or alloc memory failed, will raise err.Exception. * @maixpy maix.nn.NN.forward_image */ - tensor::Tensors *forward_image(image::Image &img, std::vector mean = std::vector(), std::vector scale = std::vector(), image::Fit fit = image::Fit::FIT_FILL, bool copy_result = true, bool dual_buff_wait = false); + tensor::Tensors *forward_image(image::Image &img, std::vector mean = std::vector(), std::vector scale = std::vector(), image::Fit fit = image::Fit::FIT_FILL, bool copy_result = true, bool dual_buff_wait = false, bool chw = true); private: MUD _mud; diff --git a/components/nn/include/maix_nn_hand_landmarks.hpp b/components/nn/include/maix_nn_hand_landmarks.hpp new file mode 100644 index 00000000..8ef6d8fa --- /dev/null +++ b/components/nn/include/maix_nn_hand_landmarks.hpp @@ -0,0 +1,778 @@ +/** + * @author neucrack@sipeed + * @copyright Sipeed Ltd 2024- + * @license Apache 2.0 + * @update 2024.12.27: Add hand keypoints support. + */ + +#pragma once +#include "maix_basic.hpp" +#include "maix_nn.hpp" +#include "maix_image.hpp" +#include "maix_image_cv.hpp" +#include "maix_nn_F.hpp" +#include "maix_nn_object.hpp" +#include + +#define DRAW_STD_IMG 0 + +namespace maix::nn +{ + /** + * HandLandmarks class + * @maixpy maix.nn.HandLandmarks + */ + class HandLandmarks + { + public: + /** + * Constructor of HandLandmarks class + * @param model model path, default empty, you can load model later by load function. + * @throw If model arg is not empty and load failed, will throw err::Exception. + * @maixpy maix.nn.HandLandmarks.__init__ + * @maixcdk maix.nn.HandLandmarks.HandLandmarks + */ + HandLandmarks(const string &model = "") + { + _model = nullptr; + _model_detect = nullptr; + if (!model.empty()) + { + err::Err e = load(model); + if (e != err::ERR_NONE) + { + throw err::Exception(e, "load model failed"); + } + } + } + + ~HandLandmarks() + { + if (_model_detect) + { + delete _model_detect; + _model_detect = nullptr; + } + if (_model) + { + delete _model; + _model = nullptr; + } + } + + std::string type() + { + return _extra_info["type"]; + } + + /** + * Load model from file + * @param model Model path want to load + * @return err::Err + * @maixpy maix.nn.HandLandmarks.load + */ + err::Err load(const string &model) + { + if (_model) + { + delete _model; + _model = nullptr; + } + _model = new nn::NN(model, false); + if (!_model) + { + return err::ERR_NO_MEM; + } + _extra_info = _model->extra_info(); + if (_extra_info.find("model_type") != _extra_info.end()) + { + if (_extra_info["model_type"] != this->type_str) + { + log::error("model_type not match, expect '%s', but got '%s'", this->type_str.c_str(), _extra_info["model_type"].c_str()); + return err::ERR_ARGS; + } + } + else + { + log::error("model_type key not found"); + return err::ERR_ARGS; + } + log::info("model info:\n\ttype: %s", this->type_str.c_str()); + if (_extra_info.find("input_type") != _extra_info.end()) + { + std::string input_type = _extra_info["input_type"]; + if (input_type == "rgb") + { + _input_img_fmt = maix::image::FMT_RGB888; + log::print("\tinput type: rgb\n"); + } + else if (input_type == "bgr") + { + _input_img_fmt = maix::image::FMT_BGR888; + log::print("\tinput type: bgr\n"); + } + else + { + log::error("unknown input type: %s", input_type.c_str()); + return err::ERR_ARGS; + } + } + else + { + log::error("input_type key not found"); + return err::ERR_ARGS; + } + if (_extra_info.find("mean") != _extra_info.end()) + { + std::string mean_str = _extra_info["mean"]; + std::vector mean_strs = split(mean_str, ","); + log::print("\tmean:"); + for (auto &it : mean_strs) + { + try + { + this->mean.push_back(std::stof(it)); + } + catch (std::exception &e) + { + log::error("mean value error, should float"); + return err::ERR_ARGS; + } + log::print("%f ", this->mean.back()); + } + log::print("\n"); + } + else + { + log::error("mean key not found"); + return err::ERR_ARGS; + } + if (_extra_info.find("scale") != _extra_info.end()) + { + std::string scale_str = _extra_info["scale"]; + std::vector scale_strs = split(scale_str, ","); + log::print("\tscale:"); + for (auto &it : scale_strs) + { + try + { + this->scale.push_back(std::stof(it)); + } + catch (std::exception &e) + { + log::error("scale value error, should float"); + return err::ERR_ARGS; + } + log::print("%f ", this->scale.back()); + } + log::print("\n"); + } + else + { + log::error("scale key not found"); + return err::ERR_ARGS; + } + err::Err e = _model->extra_info_labels(labels); + if (e == err::Err::ERR_NONE) + { + log::print("\tlabels num: %ld\n", labels.size()); + } + else + { + log::error("labels key not found: %s", err::to_str(e).c_str()); + return err::ERR_ARGS; + } + if (_extra_info.find("detect_model") != _extra_info.end()) + { + std::string model_file = fs::dirname(model) + "/" + _extra_info["detect_model"]; + _model_detect = new nn::NN(model_file, false); + if (!_model_detect) + { + log::error("load detect model failed"); + return err::ERR_NO_MEM; + } + } + else + { + log::error("detect_model key not found"); + return err::ERR_ARGS; + } + std::vector inputs = _model->inputs_info(); + std::vector inputs_detect = _model_detect->inputs_info(); + _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]); + _input_size_detect = image::Size(inputs_detect[0].shape[2], inputs_detect[0].shape[1]); + log::print("\tinput size: %dx%d\n\n", _input_size_detect.width(), _input_size_detect.height()); + + // anchors + if (_extra_info.find("anchors") != _extra_info.end()) + { + std::string anchors_file = fs::dirname(model) + "/" + _extra_info["anchors"]; + fs::File *f = fs::open(anchors_file, "r"); + if(!f) + { + log::error("open anchor file %s failed", anchors_file.c_str()); + return err::ERR_ARGS; + } + + while(1) + { + std::string *line = f->readline(); + if(!line) + break; + if(!_parse_anchor_line(*line, _anchors)) + { + log::error("parse_anchor_line %s failed", line->c_str()); + delete line; + delete f; + return err::ERR_ARGS; + } + delete line; + } + delete f; + } + else + { + log::error("detect_model key not found"); + return err::ERR_ARGS; + } + + return err::ERR_NONE; + } + + /** + * Detect objects from image + * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method. + * @param conf_th Hand detect confidence threshold, default 0.7. + * @param iou_th IoU threshold, default 0.45. + * @param conf_th2 Hand detect confidence second time check threshold, default 0.8. + * @param landmarks_rel outputs the relative coordinates of 21 points with respect to the top-left vertex of the hand. + * In obj.points, the last 21x2 values are arranged as x0y0x1y1...x20y20. + * Value from 0 to obj.w. + * @throw If image format not match model input format, will throw err::Exception. + * @return Object list. In C++, you should delete it after use. + * Object's points value format: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y, + * x0, y0, z1, x1, y1, z2, ..., x20, y20, z20. + * If landmarks_rel is True, will be box_topleft_x, box_topleft_y...,x20,y20,z20,x0_rel,y0_rel,...,x20_rel,y20_rel. + * Z is depth, the larger the value, the farther away from the palm, and the positive value means closer to the camera. + * @maixpy maix.nn.HandLandmarks.detect + */ + nn::Objects *detect(image::Image &img, float conf_th = 0.7, float iou_th = 0.45, float conf_th2 = 0.8, bool landmarks_rel = false) + { + maix::image::Fit fit = maix::image::FIT_CONTAIN; + this->_conf_th = conf_th; + this->_iou_th = iou_th; + this->_conf_th2 = conf_th2; + if (img.format() != _input_img_fmt) + { + throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]); + } + image::Image *detect_img_input = &img; + bool resized = false; + if (img.width() != _input_size_detect.width() || img.height() != _input_size_detect.height()) + { + detect_img_input = img.resize(_input_size_detect.width(), _input_size_detect.height(), fit); + resized = true; + } + nn::Objects *objs = new nn::Objects(); + tensor::Tensors *outputs; + outputs = _model_detect->forward_image(*detect_img_input, this->mean, this->scale, fit, false, true, false); + if (resized) + delete detect_img_input; + if (!outputs) // not ready, return empty result. + { + return objs; + } + _decode_objs(*objs, outputs, conf_th, _input_size_detect.width(), _input_size_detect.height(), resized, img.width(), img.height()); + delete outputs; + if (objs->size() > 0) + { + nn::Objects *objects_total = objs; + objs = _nms(*objs); + delete objects_total; + } + std::vector M_inverse; + std::vector landmarks_input = _crop_reize_hand(*objs, img, _input_size.width(), _input_size.height(), M_inverse, landmarks_rel); + #if DRAW_STD_IMG + int y=0; + #endif + bool have_invalid = false; + for(size_t i=0; iheight(); + #endif + outputs = _model->forward_image(*landmarks_input[i], this->mean, this->scale, fit, false, true, false); + delete landmarks_input[i]; + if (!outputs) // not ready, return empty result. + { + return objs; + } + have_invalid |= _decode_landmarks(*objs, i, outputs, conf_th2, M_inverse, _input_size.width(), _input_size.height(), img.width(), img.height(), landmarks_rel); + delete outputs; + } + if(have_invalid) + { + nn::Objects *res = new nn::Objects(); + for(size_t i = 0; i < objs->size(); ++i) + { + nn::Object &obj = objs->at(i); + if(obj.score > 0) + res->add(obj); + } + delete objs; + return res; + } + return objs; + } + + /** + * Get model input size + * @param detect detect or landmarks model, default true. + * @return model input size + * @maixpy maix.nn.HandLandmarks.input_size + */ + image::Size input_size(bool detect = true) + { + if (detect) + return _input_size_detect; + return _input_size; + } + + /** + * Get model input width + * @param detect detect or landmarks model, default true. + * @return model input size of width + * @maixpy maix.nn.HandLandmarks.input_width + */ + int input_width(bool detect = true) + { + if (detect) + return _input_size_detect.width(); + return _input_size.width(); + } + + /** + * Get model input height + * @param detect detect or landmarks model, default true. + * @return model input size of height + * @maixpy maix.nn.HandLandmarks.input_height + */ + int input_height(bool detect = true) + { + if (detect) + return _input_size_detect.height(); + return _input_size.height(); + } + + /** + * Get input image format + * @return input image format, image::Format type. + * @maixpy maix.nn.HandLandmarks.input_format + */ + image::Format input_format() + { + return _input_img_fmt; + } + + /** + * Draw hand and landmarks on image + * @param img image object, maix.image.Image type. + * @param leftright, 0 means left, 1 means right + * @param points points result from detect method: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y, + * x0, y0, z1, x1, y1, z2, ..., x20, y20, z20 + * @param r_min min radius of points. + * @param r_max min radius of points. + * @param box draw box or not, default true. + * @param box_color color of box. + * @maixpy maix.nn.HandLandmarks.draw_hand + */ + void draw_hand(image::Image &img, int leftright, const std::vector &points, int r_min = 4, int r_max = 10, bool box = true, int box_thickness = 1, image::Color box_color_l = image::COLOR_RED, image::Color box_color_r = image::COLOR_GREEN) + { + if (points.size() < 71) + { + throw std::runtime_error("keypoints size must 72"); + } + if (box) + { + img.draw_line(points[0], points[1], points[2], points[3], leftright == 0 ? box_color_l : box_color_r, box_thickness); + img.draw_line(points[2], points[3], points[4], points[5], leftright == 0 ? box_color_l : box_color_r, box_thickness); + img.draw_line(points[4], points[5], points[6], points[7], leftright == 0 ? box_color_l : box_color_r, box_thickness); + img.draw_line(points[6], points[7], points[0], points[1], leftright == 0 ? box_color_l : box_color_r, box_thickness); + } + int max_z = INT_MIN, min_z = INT_MAX; + for (int i = 0; i < 21; ++i) + { + if (points[10 + i * 3] > max_z) + max_z = points[10 + i * 3]; + else if (points[10 + i * 3] < min_z) + min_z = points[10 + i * 3]; + } + float range = 1.0 / (max_z - min_z); + for (int i = 0; i < 5; ++i) + { + float radius = (points[10 + i * 3] - min_z) * range; + uint8_t c = (uint8_t)(radius * 255); + auto color = image::Color::from_rgb(c, c, c); + int r = (int)(radius * r_max); + if (r < r_min) + r = r_min; + img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1); + } + for (int i = 5; i < 9; ++i) + { + float radius = (points[10 + i * 3] - min_z) * range; + auto color = image::Color::from_rgb(radius * 255, 0, 0); + int r = (int)(radius * r_max); + if (r < r_min) + r = r_min; + img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1); + } + for (int i = 9; i < 13; ++i) + { + float radius = (points[10 + i * 3] - min_z) * range; + auto color = image::Color::from_rgb(0, radius * 255, 0); + int r = (int)(radius * r_max); + if (r < r_min) + r = r_min; + img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1); + } + for (int i = 13; i < 17; ++i) + { + float radius = (points[10 + i * 3] - min_z) * range; + auto color = image::Color::from_rgb(0, 0, radius * 255); + int r = (int)(radius * r_max); + if (r < r_min) + r = r_min; + img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1); + } + for (int i = 17; i < 21; ++i) + { + float radius = (points[10 + i * 3] - min_z) * range; + uint8_t c = (uint8_t)(radius * 255); + auto color = image::Color::from_rgb(0, c, c); + int r = (int)(radius * r_max); + if (r < r_min) + r = r_min; + img.draw_circle(points[8 + i * 3], points[9 + i * 3], r, color, -1); + } + } + + public: + /** + * Labels list + * @maixpy maix.nn.HandLandmarks.labels + */ + std::vector labels; + + /** + * Label file path + * @maixpy maix.nn.HandLandmarks.label_path + */ + std::string label_path; + + /** + * Get mean value, list type + * @maixpy maix.nn.HandLandmarks.mean + */ + std::vector mean; + + /** + * Get scale value, list type + * @maixpy maix.nn.HandLandmarks.scale + */ + std::vector scale; + + protected: + std::string type_str = "hand_landmarks"; + + private: + image::Size _input_size; + image::Size _input_size_detect; + image::Format _input_img_fmt; + nn::NN *_model; + nn::NN *_model_detect; + std::map _extra_info; + float _conf_th = 0.7; + float _iou_th = 0.45; + float _conf_th2 = 0.8; + std::vector> _anchors; + + private: + bool _parse_anchor_line(std::string &line, std::vector> &anchors) + { + std::vector tokens; + split0(tokens, line, ","); + if(tokens.size() != 4) + return false; + std::vector anchor = {std::stof(tokens[0]) * _input_size_detect.width(), std::stof(tokens[1]) * _input_size_detect.height(), std::stof(tokens[2]), std::stof(tokens[3])}; + anchors.push_back(anchor); + return true; + } + + void _decode_objs(nn::Objects &objects, tensor::Tensors *outputs, const float &conf_th, int input_w, int input_h, bool resized, int img_w, int img_h) + { + tensor::Tensor *score_out = NULL; // shape 1, 2016, 18, 1 + tensor::Tensor *box_out = NULL; // shape 1, 2016, 1, 1 + int bbox_size = 2016; + for (auto i : *outputs) + { + if (i.second->shape()[2] == 18) + { + box_out = i.second; + bbox_size = i.second->shape()[1]; + } + else + { + score_out = i.second; + } + } + if (!box_out) + { + throw err::Exception(err::ERR_ARGS, "wrong model"); + } + float *scores = (float*)score_out->data(); + float *bboxes = (float*)box_out->data(); + for (int i = 0; i < bbox_size; ++i) + { + scores[i] = _sigmoid(scores[i]); + if(scores[i] >= conf_th) + { + float *p = bboxes + i*18; + int w = p[2] * _anchors[i][2]; + int h = p[3] * _anchors[i][3]; + int x = p[0] + _anchors[i][0] - w * 0.5; + int y = p[1] + _anchors[i][1] - h * 0.5; + int p0_x = _anchors[i][0] + p[4]; + int p0_y = _anchors[i][1] + p[5]; + int p2_x = _anchors[i][0] + p[8]; + int p2_y = _anchors[i][1] + p[9]; + float angle = atan2(p2_y - p0_y, p2_x - p0_x) + M_PI; + if(resized) + { + auto bbox = image::resize_map_pos_reverse(img_w, img_h, input_w, input_h, image::FIT_CONTAIN, x, y, w, h); + auto p2 = image::resize_map_pos_reverse(img_w, img_h, input_w, input_h, image::FIT_CONTAIN, p2_x, p2_y); + objects.add(bbox[0], bbox[1], bbox[2], bbox[3], 0, scores[i], std::vector({p2[0], p2[1]}), angle); + } + else + objects.add(x, y, w, h, 0, scores[i], std::vector({p2_x, p2_y}), angle); + } + } + } + + std::vector _crop_reize_hand(nn::Objects &objs, image::Image &img, int input_w, int input_h, std::vector &M_inverse, bool landmarks_rel) + { + std::vector res; + float dscale = 2.6; + for (size_t i = 0; i < objs.size(); ++i) + { + nn::Object &obj = objs.at(i); + float dy = -0.5 * sin(obj.angle); + int cx = (int)((obj.x + obj.w * 0.5 + obj.points[0]) * 0.5); + int cy = (int)((obj.y + obj.h * 0.5 + obj.w * dy + obj.points[1]) * 0.5); + // img.draw_circle(cx, cy, 5, image::COLOR_RED, -1); + // img.draw_circle(obj.points[0], obj.points[1], 5, image::COLOR_YELLOW, -1); + obj.points.resize(71 + (landmarks_rel ? 21 * 2 : 0), 0); + int hand_size = (int)(obj.w * dscale); + float theta = obj.angle - M_PI * 0.5; + cv::Mat A = (cv::Mat_(4, 2) << -1, -1, -1, 1, 1, 1, 1, -1); + cv::Mat R = (cv::Mat_(2, 2) << cos(theta), sin(theta), -sin(theta), cos(theta)); + cv::Mat C; + float half_w = hand_size * 0.5; + cv::gemm(A, R, half_w, cv::Mat(), 0.0, C); + for (int i = 0; i < C.rows; ++i) { + C.at(i, 0) += cx; + C.at(i, 1) += cy; + obj.points[i*2] = C.at(i, 0); + obj.points[1 + i*2] = C.at(i, 1); + } + obj.x = cx - half_w; + obj.y = cy - half_w; + obj.w = hand_size; + obj.h = hand_size; + cv::Mat element = C(cv::Range(0, 3), cv::Range(0, 2)); + cv::Mat dst = (cv::Mat_(3, 2) << 0, 0, 0, input_h, input_w, input_h); + auto M = cv::getAffineTransform(element, dst); + image::Image *img_dst = new image::Image(input_w, input_h, image::FMT_RGB888); + if(!img_dst) + { + log::error("no memory"); + break; + } + cv::Mat img_dst2(input_h, input_w, CV_8UC3, img_dst->data()); + cv::Mat img_src; + image::image2cv(img, img_src, false, false); + cv::warpAffine(img_src, img_dst2, M, cv::Size(input_w, input_h), cv::INTER_NEAREST); + assert(img_dst2.data() == img_dst->data()); + res.push_back(img_dst); + cv::Mat iM; + cv::invertAffineTransform(M, iM); + M_inverse.push_back(iM); + } + return res; + } + + bool _decode_landmarks(nn::Objects &objs, int idx, tensor::Tensors *outputs, float conf_th2, std::vector &M_inverse, int input_w, int input_h, int img_w, int img_h, bool landmarks_rel) + { + tensor::Tensor *leftright_out = NULL; // shape 1, 1, 1, 1 + tensor::Tensor *score_out = NULL; // shape 1, 1, 1, 1 + tensor::Tensor *points_out = NULL; // shape 1, 63, 1, 1 + for (auto i : *outputs) + { + if (i.second->shape()[1] == 63) + { + points_out = i.second; + } + else if(i.first.find("score") != std::string::npos) + { + score_out = i.second; + } + else + { + leftright_out = i.second; + } + } + if (!leftright_out || !score_out || !points_out) + { + throw err::Exception(err::ERR_ARGS, "wrong model"); + } + auto &obj = objs.at(idx); + float score = _sigmoid(*(float*)score_out->data()); + float leftright = _sigmoid(*(float*)leftright_out->data()); + float *points = (float*)points_out->data(); + if(score < conf_th2) + { + obj.score = 0; + return true; // invalid + } + obj.score = score; + obj.class_id = leftright > 0.5 ? 1 : 0; + cv::Mat iM; + cv::transpose(M_inverse[idx], iM); + cv::Mat A = cv::Mat_(21, 3); + for (int i = 0; i < A.rows; ++i) { + A.at(i, 0) = points[i*3]; + A.at(i, 1) = points[i*3 + 1]; + A.at(i, 2) = 1; + obj.points[8 + i*3 + 2] = (int)(-points[i*3 + 2] * obj.w); + } + cv::Mat C; + cv::gemm(A, iM, 1.0, cv::Mat(), 0.0, C); + for (int i = 0; i < C.rows; ++i) { + obj.points[8 + 3*i] = C.at(i, 0); + obj.points[9 + 3*i] = C.at(i, 1); + } + if(landmarks_rel) + { + for (int i = 0; i < 21; ++i) + { + obj.points[71 + i * 2] = points[i*3]; + obj.points[72 + i * 2] = points[i*3 + 1]; + } + } + // update angle + obj.angle = atan2(obj.points[36] - obj.points[9], obj.points[35] - obj.points[8]) + M_PI; + return false; // valid + } + + nn::Objects *_nms(nn::Objects &objs) + { + nn::Objects *result = new nn::Objects(); + std::sort(objs.begin(), objs.end(), [](const nn::Object *a, const nn::Object *b) + { return a->score > b->score; }); + for (size_t i = 0; i < objs.size(); ++i) + { + nn::Object &a = objs.at(i); + if (a.score == 0) + continue; + for (size_t j = i + 1; j < objs.size(); ++j) + { + nn::Object &b = objs.at(j); + { + if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th) + { + b.score = 0; + } + } + } + } + for (nn::Object *a : objs) + { + if (a->score != 0) + { + result->add(a->x, a->y, a->w, a->h, a->class_id, a->score, a->points, a->angle); + } + } + return result; + } + + inline static float _sigmoid(float x) { return 1.0 / (1 + expf(-x)); } + + inline static float _calc_iou(Object &a, Object &b) + { + float area1 = a.w * a.h; + float area2 = b.w * b.h; + float wi = std::min((a.x + a.w), (b.x + b.w)) - + std::max(a.x, b.x); + float hi = std::min((a.y + a.h), (b.y + b.h)) - + std::max(a.y, b.y); + float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f); + return area_i / (area1 + area2 - area_i); + } + + template + static int _argmax(const T *data, size_t len, size_t stride = 1) + { + int maxIndex = 0; + for (size_t i = 1; i < len; i++) + { + int idx = i * stride; + if (data[maxIndex * stride] < data[idx]) + { + maxIndex = i; + } + } + return maxIndex; + } + + template + static int _argmax2(const T *data, size_t end, size_t stride = 1) + { + int maxIndex = 0; + for (size_t i = stride; i < end; i += stride) + { + if (data[maxIndex] < data[i]) + { + maxIndex = i; + } + } + return maxIndex; + } + + static void split0(std::vector &items, const std::string &s, const std::string &delimiter) + { + items.clear(); + size_t pos_start = 0, pos_end, delim_len = delimiter.length(); + std::string token; + + while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos) + { + token = s.substr(pos_start, pos_end - pos_start); + pos_start = pos_end + delim_len; + token.erase(0, token.find_first_not_of(" \t\r\n")); + token.erase(token.find_last_not_of(" \t\r\n") + 1); + items.push_back(token); + } + + items.push_back(s.substr(pos_start)); + } + + static std::vector split(const std::string &s, const std::string &delimiter) + { + std::vector tokens; + split0(tokens, s, delimiter); + return tokens; + } + }; + +} // namespace maix::nn diff --git a/components/nn/include/maix_nn_object.hpp b/components/nn/include/maix_nn_object.hpp index 0303c37f..26451e41 100644 --- a/components/nn/include/maix_nn_object.hpp +++ b/components/nn/include/maix_nn_object.hpp @@ -300,6 +300,21 @@ namespace maix::nn return *obj; } + /** + * Add object to objects + * @throw Throw exception if no memory + * @maixcdk maix.nn.Objects.add + */ + nn::Object &add(const nn::Object &obj) + { + Object *obj_new = new Object(obj.x, obj.y, obj.w, obj.h, obj.class_id, obj.score, obj.points, obj.angle); + if(!obj_new) + throw err::Exception(err::ERR_NO_MEM); + obj_new->seg_mask = NULL; + objs.push_back(obj_new); + return *obj_new; + } + /** * Remove object form objects * @maixpy maix.nn.Objects.remove diff --git a/components/nn/src/maix_nn.cpp b/components/nn/src/maix_nn.cpp index de572d09..e5650f63 100644 --- a/components/nn/src/maix_nn.cpp +++ b/components/nn/src/maix_nn.cpp @@ -304,9 +304,9 @@ namespace maix::nn return _impl->forward(inputs, copy_result, dual_buff_wait); } - tensor::Tensors *NN::forward_image(image::Image &img, std::vector mean, std::vector scale, image::Fit fit, bool copy_result, bool dual_buff_wait) + tensor::Tensors *NN::forward_image(image::Image &img, std::vector mean, std::vector scale, image::Fit fit, bool copy_result, bool dual_buff_wait, bool chw) { - return _impl->forward_image(img, mean, scale, fit, copy_result, dual_buff_wait); + return _impl->forward_image(img, mean, scale, fit, copy_result, dual_buff_wait, chw); } int SelfLearnClassifier::learn() diff --git a/components/vision/include/maix_image_cv.hpp b/components/vision/include/maix_image_cv.hpp new file mode 100644 index 00000000..64d8e254 --- /dev/null +++ b/components/vision/include/maix_image_cv.hpp @@ -0,0 +1,31 @@ +#include "maix_image.hpp" +#include "opencv2/opencv.hpp" + +namespace maix::image +{ + + /** + * Convert image::Image object to opencv Mat object + * @param[in] img image::Image object + * @param[out] mat cv::Mat object, output image, will be allocated by this function. + * @param[in] ensure_bgr auto convert to BGR888 or BGRA8888 if img format is not BGR or BGRA, if set to false, will not auto convert and directly use img's data, default false. + * If copy is false, ensure_bgr always be false. + * @param[in] copy Whether alloc new image and copy data or not, if ensure_bgr and img is not bgr or bgra format, always copy, + * if not copy, array object will directly use img's data buffer, will faster but change array will affect img's data, default true. + * @maixcdk maix.image.image2cv + */ + err::Err image2cv(image::Image &img, cv::Mat &mat, bool ensure_bgr = false, bool copy = true); + + + /** + * OpenCV Mat object to Image object + * @param mat cv::Mat image object. + * @param bgr if set bgr, the return image will be marked as BGR888 or BGRA8888 format, grayscale will ignore this arg. + * @param copy if true, will alloc new buffer and copy data, else will directly use array's data buffer, default true. + * Use this arg carefully, when set to false, ther array MUST keep alive until we don't use the return img of this func, or will cause program crash. + * @return Image object + * @maixcdk maix.image.cv2image + */ + image::Image *cv2image(cv::Mat &mat, bool bgr = true, bool copy = true); + +} diff --git a/components/vision/src/maix_image.cpp b/components/vision/src/maix_image.cpp index bef0fc23..1b28e271 100644 --- a/components/vision/src/maix_image.cpp +++ b/components/vision/src/maix_image.cpp @@ -25,10 +25,12 @@ namespace maix::image static bool path_is_format(const std::string &str, const std::string ext) { std::string lower_path(str); - std::transform(lower_path.begin(), lower_path.end(), lower_path.begin(), - [](unsigned char c){ return std::tolower(c); }); + std::transform(lower_path.begin(), lower_path.end(), lower_path.begin(), + [](unsigned char c) + { return std::tolower(c); }); - if (lower_path.size() >= ext.size() && lower_path.compare(lower_path.size() - ext.size(), ext.size(), ext) == 0) { + if (lower_path.size() >= ext.size() && lower_path.compare(lower_path.size() - ext.size(), ext.size(), ext) == 0) + { return true; } return false; @@ -36,19 +38,28 @@ namespace maix::image static uint8_t _get_char_size(const uint8_t c) { - if ((c & 0x80) == 0x00) { + if ((c & 0x80) == 0x00) + { // 0xxxxxxx: 1 byte character return 1; - } else if ((c & 0xE0) == 0xC0) { + } + else if ((c & 0xE0) == 0xC0) + { // 110xxxxx: 2 byte character return 2; - } else if ((c & 0xF0) == 0xE0) { + } + else if ((c & 0xF0) == 0xE0) + { // 1110xxxx: 3 byte character return 3; - } else if ((c & 0xF8) == 0xF0) { + } + else if ((c & 0xF8) == 0xF0) + { // 11110xxx: 4 byte character return 4; - } else { + } + else + { // Invalid UTF-8 start byte, default to 1 return 1; } @@ -57,14 +68,14 @@ namespace maix::image static image::Image *_mat_to_image(const cv::Mat &mat, image::Format format, void *buff, int buff_size, bool copy_data = true) { image::Image *img = nullptr; - if(buff) + if (buff) { - if(buff_size < image::fmt_size[format]) + if (buff_size < image::fmt_size[format]) { log::error("convert format failed, buffer size not enough, need %d, but %d\n", image::fmt_size[format], buff_size); throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } - img = new image::Image(mat.cols, mat.rows, format, (uint8_t*)buff, mat.cols * mat.rows * image::fmt_size[format], false); + img = new image::Image(mat.cols, mat.rows, format, (uint8_t *)buff, mat.cols * mat.rows * image::fmt_size[format], false); } else img = new image::Image(mat.cols, mat.rows, format); @@ -76,14 +87,14 @@ namespace maix::image static image::Image *_new_image(int w, int h, image::Format format, void *buff, int buff_size) { image::Image *img = nullptr; - if(buff) + if (buff) { - if(buff_size < image::fmt_size[format]) + if (buff_size < image::fmt_size[format]) { log::error("convert format failed, buffer size not enough, need %d, but %d\n", image::fmt_size[format], buff_size); throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } - img = new image::Image(w, h, format, (uint8_t*)buff, w * h * image::fmt_size[format], false); + img = new image::Image(w, h, format, (uint8_t *)buff, w * h * image::fmt_size[format], false); } else img = new image::Image(w, h, format); @@ -100,9 +111,10 @@ namespace maix::image uint8_t *uv_temp = (uint8_t *)malloc(nv_len); err::check_null_raise(uv_temp, "malloc uv_temp failed"); memcpy(uv_temp, nv21_data + offset, nv_len); - for (int i = 0; i < half_len; i ++) { - nv21_data[offset + i * 2] = uv_temp[i]; // V - nv21_data[offset + i * 2 + 1] = uv_temp[half_len + i]; // U + for (int i = 0; i < half_len; i++) + { + nv21_data[offset + i * 2] = uv_temp[i]; // V + nv21_data[offset + i * 2 + 1] = uv_temp[half_len + i]; // U } if (uv_temp) free(uv_temp); @@ -347,14 +359,17 @@ namespace maix::image if (width <= 0 || height <= 0) throw err::Exception(err::ERR_ARGS, "image width and height should > 0"); - if (_format > Format::FMT_COMPRESSED_MIN) { + if (_format > Format::FMT_COMPRESSED_MIN) + { if (!data || data_size < 0) - throw err::Exception(err::ERR_ARGS,"image data and data_size are incorrect"); + throw err::Exception(err::ERR_ARGS, "image data and data_size are incorrect"); _data_size = data_size; - } else { + } + else + { // not use data_size, uncompressed image only use fiexed size. int size_calc = width * height * image::fmt_size[format]; - if(data_size > 0 && data_size != size_calc) + if (data_size > 0 && data_size != size_calc) { log::error("data_size not match image content size, data_size: %d, image content size: %d\n", data_size, size_calc); throw err::Exception(err::ERR_ARGS, "data_size not match image content size"); @@ -373,7 +388,7 @@ namespace maix::image } else { - if(!copy) + if (!copy) { _data = data; _actual_data = _data; @@ -402,8 +417,8 @@ namespace maix::image { // if(!data) // { - _create_image(width, height, format, nullptr, 0, false); - // return; + _create_image(width, height, format, nullptr, 0, false); + // return; // } // _create_image(width, height, format, data->data, data->size(), copy); } @@ -434,9 +449,9 @@ namespace maix::image void Image::operator=(const image::Image &img) { - if(_data) + if (_data) { - if(_is_malloc) + if (_is_malloc) { log::info("free _actual_data"); free(_actual_data); @@ -468,7 +483,7 @@ namespace maix::image Bytes *Image::to_bytes(bool copy) { - if(copy) + if (copy) return new Bytes((uint8_t *)_data, _data_size, true, true); return new Bytes((uint8_t *)_data, _data_size, false, false); } @@ -507,91 +522,100 @@ namespace maix::image log::error("convert format failed, already the format %d\n", format); throw err::Exception(err::ERR_ARGS, "convert format failed, already the format"); } - cv::Mat src(_format > FMT_COMPRESSED_MIN ? 1 : _height, _format > FMT_COMPRESSED_MIN ? _data_size : _width, CV_8UC((int)image::fmt_size[_format]), _data); + cv::Mat src(_format > FMT_COMPRESSED_MIN ? 1 : _height, _format > FMT_COMPRESSED_MIN ? _data_size : _width, CV_8UC((int)image::fmt_size[_format]), _data); cv::ColorConversionCodes cvt_code; // special for convert to jpeg and png - if(format == image::FMT_JPEG) // compress + if (format == image::FMT_JPEG) // compress { - switch (_format) { + switch (_format) + { #ifdef PLATFORM_MAIXCAM case image::FMT_GRAYSCALE: - { + { image::Image *img = nullptr; - if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_GRAYSCALE), 95)) { + if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_GRAYSCALE), 95)) + { uint8_t *data; int data_size; - if (!mmf_enc_jpg_pop(0, &data, &data_size)) { - if(buff) + if (!mmf_enc_jpg_pop(0, &data, &data_size)) + { + if (buff) { - if(buff_size < (size_t)data_size) + if (buff_size < (size_t)data_size) throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); memcpy(buff, data, data_size); - img = new image::Image(_width, _height, format, (uint8_t*)buff, data_size, false); + img = new image::Image(_width, _height, format, (uint8_t *)buff, data_size, false); } else img = new image::Image(_width, _height, format, data, data_size, true); mmf_enc_jpg_free(0); } } - if(!img) + if (!img) throw err::Exception(err::ERR_RUNTIME, "convert format failed, see log"); return img; break; - } + } case image::FMT_YVU420SP: - { + { image::Image *img = nullptr; - if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_YVU420SP), 95)) { + if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_YVU420SP), 95)) + { uint8_t *data; int data_size; - if (!mmf_enc_jpg_pop(0, &data, &data_size)) { - if(buff) + if (!mmf_enc_jpg_pop(0, &data, &data_size)) + { + if (buff) { - if(buff_size < (size_t)data_size) + if (buff_size < (size_t)data_size) throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); memcpy(buff, data, data_size); - img = new image::Image(_width, _height, format, (uint8_t*)buff, data_size, false); + img = new image::Image(_width, _height, format, (uint8_t *)buff, data_size, false); } else img = new image::Image(_width, _height, format, data, data_size, true); mmf_enc_jpg_free(0); } } - if(!img) + if (!img) throw err::Exception(err::ERR_RUNTIME, "convert format failed, see log"); return img; break; - } + } #endif default: - { + { #ifdef PLATFORM_MAIXCAM image::Image *p_img = nullptr; image::Image *img = nullptr; bool src_alloc = false; - if(_format != Format::FMT_RGB888) + if (_format != Format::FMT_RGB888) { p_img = to_format(image::FMT_RGB888); src_alloc = true; - } else { + } + else + { p_img = this; } - if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)p_img->data(), _width, _height, mmf_invert_format_to_mmf(image::FMT_RGB888), 80)) { + if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)p_img->data(), _width, _height, mmf_invert_format_to_mmf(image::FMT_RGB888), 80)) + { uint8_t *data; int data_size; - if (!mmf_enc_jpg_pop(0, &data, &data_size)) { - if(buff) + if (!mmf_enc_jpg_pop(0, &data, &data_size)) + { + if (buff) { - if(buff_size < (size_t)data_size) + if (buff_size < (size_t)data_size) { if (src_alloc) delete p_img; throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } memcpy(buff, data, data_size); - img = new image::Image(_width, _height, format, (uint8_t*)buff, data_size, false); + img = new image::Image(_width, _height, format, (uint8_t *)buff, data_size, false); } else img = new image::Image(_width, _height, format, data, data_size, true); @@ -601,7 +625,7 @@ namespace maix::image if (src_alloc) delete p_img; - if(!img) + if (!img) throw err::Exception(err::ERR_RUNTIME, "convert format failed, see log"); return img; #else @@ -609,7 +633,7 @@ namespace maix::image image::Image *p_img = nullptr; cv::Mat *p_src = &src; bool src_alloc = false; - if(_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) + if (_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) { p_img = to_format(image::FMT_BGR888); p_src = new cv::Mat(p_img->_height, p_img->_width, CV_8UC((int)image::fmt_size[image::FMT_BGR888]), p_img->_data); @@ -622,11 +646,11 @@ namespace maix::image params.push_back(95); cv::imencode(".jpg", input, jpeg_buff, params); image::Image *img; - if(buff) + if (buff) { - if(buff_size < jpeg_buff.size()) + if (buff_size < jpeg_buff.size()) { - if(src_alloc) + if (src_alloc) { delete p_img; delete p_src; @@ -634,11 +658,11 @@ namespace maix::image throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } memcpy(buff, jpeg_buff.data(), jpeg_buff.size()); - img = new image::Image(src.cols, src.rows, format, (uint8_t*)buff, jpeg_buff.size(), false); + img = new image::Image(src.cols, src.rows, format, (uint8_t *)buff, jpeg_buff.size(), false); } else - img = new image::Image(src.cols, src.rows, format, (uint8_t*)jpeg_buff.data(), jpeg_buff.size(), true); - if(src_alloc) + img = new image::Image(src.cols, src.rows, format, (uint8_t *)jpeg_buff.data(), jpeg_buff.size(), true); + if (src_alloc) { delete p_img; delete p_src; @@ -646,16 +670,15 @@ namespace maix::image return img; #endif break; - } } - + } } - else if(format == image::FMT_PNG) + else if (format == image::FMT_PNG) { image::Image *p_img = nullptr; cv::Mat *p_src = &src; bool src_alloc = false; - if(_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) + if (_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) { p_img = to_format(image::FMT_BGRA8888); p_src = new cv::Mat(p_img->_height, p_img->_width, CV_8UC((int)image::fmt_size[image::FMT_BGRA8888]), p_img->_data); @@ -668,11 +691,11 @@ namespace maix::image params.push_back(3); cv::imencode(".png", input, png_buff, params); image::Image *img; - if(buff) + if (buff) { - if(buff_size < png_buff.size()) + if (buff_size < png_buff.size()) { - if(src_alloc) + if (src_alloc) { delete p_img; delete p_src; @@ -680,11 +703,11 @@ namespace maix::image throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } memcpy(buff, png_buff.data(), png_buff.size()); - img = new image::Image(src.cols, src.rows, format, (uint8_t*)buff, png_buff.size(), false); + img = new image::Image(src.cols, src.rows, format, (uint8_t *)buff, png_buff.size(), false); } else img = new image::Image(src.cols, src.rows, format, png_buff.data(), png_buff.size(), true); - if(src_alloc) + if (src_alloc) { delete p_img; delete p_src; @@ -693,40 +716,45 @@ namespace maix::image } // jpeg/png to other format - if(_format == image::FMT_JPEG || _format == image::FMT_PNG) + if (_format == image::FMT_JPEG || _format == image::FMT_PNG) { switch (format) { case image::FMT_GRAYSCALE: { cv::Mat dst = cv::imdecode(src, cv::IMREAD_GRAYSCALE); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); return _mat_to_image(dst, format, buff, buff_size); } case image::FMT_BGR888: { cv::Mat dst = cv::imdecode(src, cv::IMREAD_COLOR); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); return _mat_to_image(dst, format, buff, buff_size); } case image::FMT_RGB888: { cv::Mat dst = cv::imdecode(src, cv::IMREAD_COLOR); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); cv::cvtColor(dst, dst, cv::COLOR_BGR2RGB); return _mat_to_image(dst, format, buff, buff_size); } case image::FMT_RGBA8888: { - if(_format == FMT_PNG) + if (_format == FMT_PNG) { cv::Mat dst = cv::imdecode(src, cv::IMREAD_UNCHANGED); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); cv::cvtColor(dst, dst, cv::COLOR_BGRA2RGBA); return _mat_to_image(dst, format, buff, buff_size); } cv::Mat bgr = cv::imdecode(src, cv::IMREAD_COLOR); - if(bgr.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (bgr.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); image::Image *img = _new_image(bgr.cols, bgr.rows, format, buff, buff_size); cv::Mat dst(bgr.rows, bgr.cols, CV_8UC4, img->data()); cv::cvtColor(bgr, dst, cv::COLOR_BGR2RGBA); @@ -734,14 +762,16 @@ namespace maix::image } case image::FMT_BGRA8888: { - if(_format == FMT_PNG) + if (_format == FMT_PNG) { cv::Mat dst = cv::imdecode(src, cv::IMREAD_UNCHANGED); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); return _mat_to_image(dst, format, buff, buff_size); } cv::Mat bgr = cv::imdecode(src, cv::IMREAD_COLOR); - if(bgr.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (bgr.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); image::Image *img = _new_image(bgr.cols, bgr.rows, format, buff, buff_size); cv::Mat dst(bgr.rows, bgr.cols, CV_8UC4, img->data()); cv::cvtColor(bgr, dst, cv::COLOR_BGR2BGRA); @@ -750,7 +780,8 @@ namespace maix::image case image::FMT_YVU420SP: { cv::Mat dst = cv::imdecode(src, cv::IMREAD_COLOR); - if(dst.empty()) throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); + if (dst.empty()) + throw err::Exception(err::ERR_ARGS, "decode jpeg failed"); image::Image *img = _mat_to_image(dst, format, buff, buff_size, false); cv::Mat dst_nv21(img->height() + img->height() / 2, img->width(), CV_8UC1, img->data()); _cv_rgb_nv21(dst, dst_nv21, img->width(), img->height(), true); @@ -894,25 +925,28 @@ namespace maix::image throw err::Exception(err::ERR_NOT_IMPL, "not support format"); } image::Image *img; - if(buff) + if (buff) { - if(buff_size < image::fmt_size[format]) + if (buff_size < image::fmt_size[format]) { log::error("convert format failed, buffer size not enough, need %d, but %d\n", image::fmt_size[format], buff_size); throw err::Exception(err::ERR_ARGS, "convert format failed, buffer size not enough"); } - img = new image::Image(src.cols, src.rows, format, (uint8_t*)buff, src.cols * src.rows * image::fmt_size[format], false); + img = new image::Image(src.cols, src.rows, format, (uint8_t *)buff, src.cols * src.rows * image::fmt_size[format], false); } else img = new image::Image(src.cols, src.rows, format); - if (_format == image::FMT_GRAYSCALE && format == image::FMT_YVU420SP) { + if (_format == image::FMT_GRAYSCALE && format == image::FMT_YVU420SP) + { cv::Mat dst(src.rows * 3 / 2, src.cols, CV_8UC((int)image::fmt_size[format]), img->data()); int nv_len = src.cols * src.rows / 2; int offset = src.cols * src.rows; memcpy(dst.data, src.data, src.cols * src.rows); memset(dst.data + offset, 128, nv_len); - } else if (format == image::FMT_YVU420SP) { + } + else if (format == image::FMT_YVU420SP) + { cv::Mat dst(src.rows * 3 / 2, src.cols, CV_8UC((int)image::fmt_size[format]), img->data()); cv::cvtColor(src, dst, cvt_code); int nv_len = src.cols * src.rows / 2; @@ -922,23 +956,30 @@ namespace maix::image uint8_t *uv_temp = (uint8_t *)malloc(nv_len); err::check_null_raise(uv_temp, "malloc uv_temp failed"); memcpy(uv_temp, nv21 + offset, nv_len); - for (int i = 0; i < half_len; i ++) { - nv21[offset + i * 2] = uv_temp[i]; // V - nv21[offset + i * 2 + 1] = uv_temp[half_len + i]; // U + for (int i = 0; i < half_len; i++) + { + nv21[offset + i * 2] = uv_temp[i]; // V + nv21[offset + i * 2 + 1] = uv_temp[half_len + i]; // U } if (uv_temp) free(uv_temp); - } else if (_format == image::FMT_RGB888 && format == image::FMT_GRAYSCALE) { + } + else if (_format == image::FMT_RGB888 && format == image::FMT_GRAYSCALE) + { int height = _width; int width = _height; uint8_t *src = (uint8_t *)_data; uint8_t *dst = (uint8_t *)img->data(); - for (int i = 0; i < height; i ++) { - for (int j = 0; j < width; j ++) { + for (int i = 0; i < height; i++) + { + for (int j = 0; j < width; j++) + { dst[i * width + j] = (src[(i * width + j) * 3 + 0] * 38 + src[(i * width + j) * 3 + 1] * 75 + src[(i * width + j) * 3 + 2] * 15) >> 7; } } - } else { + } + else + { cv::Mat dst(src.rows, src.cols, CV_8UC((int)image::fmt_size[format]), img->data()); cv::cvtColor(src, dst, cvt_code); } @@ -946,7 +987,6 @@ namespace maix::image return img; } - image::Image *Image::to_format(const image::Format &format) { return to_format(format, nullptr, 0); @@ -955,20 +995,23 @@ namespace maix::image image::Image *Image::to_jpeg(int quality) { image::Format format = image::Format::FMT_JPEG; - if(quality <= 50) + if (quality <= 50) { quality = 51; } cv::Mat src(_height, _width, CV_8UC((int)image::fmt_size[_format]), _data); - switch (_format) { + switch (_format) + { case image::FMT_YVU420SP: - { + { #ifdef PLATFORM_MAIXCAM image::Image *img = nullptr; - if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_YVU420SP), quality)) { + if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)_data, _width, _height, mmf_invert_format_to_mmf(image::FMT_YVU420SP), quality)) + { uint8_t *data; int data_size; - if (!mmf_enc_jpg_pop(0, &data, &data_size)) { + if (!mmf_enc_jpg_pop(0, &data, &data_size)) + { img = new image::Image(_width, _height, format, data, data_size, true); mmf_enc_jpg_free(0); } @@ -976,25 +1019,29 @@ namespace maix::image return img; #endif break; - } + } default: - { + { #ifdef PLATFORM_MAIXCAM image::Image *p_img = nullptr; image::Image *img = nullptr; bool src_alloc = false; - if(_format != Format::FMT_RGB888) + if (_format != Format::FMT_RGB888) { p_img = to_format(image::FMT_RGB888); src_alloc = true; - } else { + } + else + { p_img = this; } - if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)p_img->data(), _width, _height, mmf_invert_format_to_mmf(image::FMT_RGB888), quality)) { + if (!mmf_enc_jpg_push_with_quality(0, (uint8_t *)p_img->data(), _width, _height, mmf_invert_format_to_mmf(image::FMT_RGB888), quality)) + { uint8_t *data; int data_size; - if (!mmf_enc_jpg_pop(0, &data, &data_size)) { + if (!mmf_enc_jpg_pop(0, &data, &data_size)) + { img = new image::Image(_width, _height, format, data, data_size, true); mmf_enc_jpg_free(0); } @@ -1009,7 +1056,7 @@ namespace maix::image image::Image *p_img = nullptr; cv::Mat *p_src = &src; bool src_alloc = false; - if(_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) + if (_format != Format::FMT_BGR888 && _format != Format::FMT_BGRA8888) { p_img = to_format(image::FMT_BGR888); p_src = new cv::Mat(p_img->_height, p_img->_width, CV_8UC((int)image::fmt_size[image::FMT_BGR888]), p_img->_data); @@ -1022,8 +1069,8 @@ namespace maix::image params.push_back(95); cv::imencode(".jpg", input, jpeg_buff, params); image::Image *img; - img = new image::Image(src.cols, src.rows, format, (uint8_t*)jpeg_buff.data(), jpeg_buff.size(), true); - if(src_alloc) + img = new image::Image(src.cols, src.rows, format, (uint8_t *)jpeg_buff.data(), jpeg_buff.size(), true); + if (src_alloc) { delete p_img; delete p_src; @@ -1031,13 +1078,14 @@ namespace maix::image return img; #endif break; - } + } } return nullptr; } - static cv::Rect _adjustRectToFit(const cv::Rect& rect, const cv::Size& dstSize) { + static cv::Rect _adjustRectToFit(const cv::Rect &rect, const cv::Size &dstSize) + { int newX = std::max(rect.x, 0); int newY = std::max(rect.y, 0); int newWidth = std::min(rect.width - (newX - rect.x), dstSize.width - newX); @@ -1061,7 +1109,8 @@ namespace maix::image int srcX = std::max(0, -x); int srcY = std::max(0, -y); cv::Rect srcRect(srcX, srcY, adjustedRect.width, adjustedRect.height); - if (adjustedRect.width <= 0 || adjustedRect.height <= 0) { + if (adjustedRect.width <= 0 || adjustedRect.height <= 0) + { throw err::Exception(err::ERR_ARGS, "range error"); } if (_format == fmt && image::fmt_size[fmt] < 4) @@ -1091,7 +1140,7 @@ namespace maix::image } else // merge two BGRA images { - uint8_t *data = (uint8_t*)_data; + uint8_t *data = (uint8_t *)_data; for (int y = 0; y < adjustedRect.height; y++) { for (int x = 0; x < adjustedRect.width; x++) @@ -1135,7 +1184,7 @@ namespace maix::image _get_cv_format_color(_format, color, &ch_format, cv_color); cv::Mat img(_height, _width, ch_format, _data); cv::Rect rect(x, y, w, h); - if( color.alpha != 1 && (_format == Format::FMT_RGBA8888 || _format == Format::FMT_BGRA8888)) + if (color.alpha != 1 && (_format == Format::FMT_RGBA8888 || _format == Format::FMT_BGRA8888)) { cv::Mat roi = img(rect); cv::Mat color_mat(roi.size(), roi.type(), cv_color); @@ -1177,7 +1226,7 @@ namespace maix::image } image::Image *image::Image::draw_string(int x, int y, const std::string &text, const image::Color &color, float scale, int thickness, - bool wrap, int wrap_space, const std::string &font) + bool wrap, int wrap_space, const std::string &font) { int ch_format = 0; cv::Scalar cv_color; @@ -1212,7 +1261,7 @@ namespace maix::image int text_height = size.height; int text_max_width = _width - x; bool wrap_now = false; - if(strstr(text.c_str(), "\n") != NULL || strstr(text.c_str(), "\r") != NULL) + if (strstr(text.c_str(), "\n") != NULL || strstr(text.c_str(), "\r") != NULL) wrap_now = true; // auto wrap @@ -1229,14 +1278,14 @@ namespace maix::image { char c = text[idx]; wrap_now = false; - if(c == '\r') + if (c == '\r') { last_is_r = true; wrap_now = true; } - else if(c == '\n') + else if (c == '\n') { - if(last_is_r) + if (last_is_r) { last_is_r = false; ++idx; @@ -1249,7 +1298,7 @@ namespace maix::image { last_is_r = false; } - if(wrap_now) + if (wrap_now) { _put_text(img, text_tmp, point, cv_color, scale, thickness, *final_font, final_font_id); point.x = x; @@ -1331,7 +1380,8 @@ namespace maix::image return nullptr; } - if (fill) { + if (fill) + { thickness = -1; } @@ -1348,32 +1398,36 @@ namespace maix::image _get_cv_format_color(_format, color, &ch_format, cv_color); cv::Mat img(_height, _width, ch_format, _data); - if (keypoints.size() < 2 || keypoints.size() % 2 != 0) { + if (keypoints.size() < 2 || keypoints.size() % 2 != 0) + { throw std::runtime_error("keypoints size must >= 2 and multiple of 2"); return nullptr; } - for(size_t i=0; i 0) { - for (size_t i = 1; i < keypoints.size() / 2; ++i) { + if (line_thickness > 0) + { + for (size_t i = 1; i < keypoints.size() / 2; ++i) + { cv::Point pt1(keypoints[(i - 1) * 2], keypoints[(i - 1) * 2 + 1]); cv::Point pt2(keypoints[i * 2], keypoints[i * 2 + 1]); if (pt1.x < 0 || pt1.y < 0 || pt2.x < 0 || pt2.y < 0) - continue; // Skip invalid lines with negative coordinates + continue; // Skip invalid lines with negative coordinates cv::line(img, pt1, pt2, cv_color, line_thickness); } // Draw a line from the last point to the first point to close the loop cv::Point pt_first(keypoints[0], keypoints[1]); cv::Point pt_last(keypoints[keypoints.size() - 2], keypoints[keypoints.size() - 1]); - if (pt_first.x >= 0 && pt_first.y >= 0 && pt_last.x >= 0 && pt_last.y >= 0) { + if (pt_first.x >= 0 && pt_first.y >= 0 && pt_last.x >= 0 && pt_last.y >= 0) + { cv::line(img, pt_last, pt_first, cv_color, line_thickness); } } @@ -1436,13 +1490,16 @@ namespace maix::image cv::InterpolationFlags inter_method = (cv::InterpolationFlags)method; if (object_fit == image::Fit::FIT_FILL) { - if (_format == image::FMT_YVU420SP) { + if (_format == image::FMT_YVU420SP) + { cv::Mat rgb, resize_rgb; cv::cvtColor(img, rgb, cv::COLOR_YUV2RGB_NV21); cv::resize(rgb, resize_rgb, cv::Size(width, height), 0, 0, inter_method); dst = cv::Mat(cv_dst_h, width, pixel_num, ret->data()); _cv_rgb_nv21(resize_rgb, dst, width, height); - } else { + } + else + { dst = cv::Mat(height, width, pixel_num, ret->data()); cv::resize(img, dst, cv::Size(width, height), 0, 0, inter_method); } @@ -1541,7 +1598,7 @@ namespace maix::image return ret; } - image::Image* Image::perspective(std::vector src_points, std::vector dst_points, int width, int height, image::ResizeMethod method) + image::Image *Image::perspective(std::vector src_points, std::vector dst_points, int width, int height, image::ResizeMethod method) { if (width < 0 && height < 0) { @@ -1565,7 +1622,7 @@ namespace maix::image } // Create output image - image::Image* ret = new image::Image(width, height, _format); + image::Image *ret = new image::Image(width, height, _format); // Convert source and destination points to cv::Point2f std::vector srcPts, dstPts; @@ -1643,26 +1700,27 @@ namespace maix::image return ret; } - - Image* Image::flip(const FlipDir dir) { + Image *Image::flip(const FlipDir dir) + { int pixel_num = _get_cv_pixel_num(_format); - Image* ret = new Image(_width, _height, _format); + Image *ret = new Image(_width, _height, _format); cv::Mat img(_height, _width, pixel_num, _data); cv::Mat dst(_height, _width, pixel_num, ret->data()); int flipCode; - switch (dir) { - case FlipDir::X: - flipCode = 0; // 垂直翻转 - break; - case FlipDir::Y: - flipCode = 1; // 水平翻转 - break; - case FlipDir::XY: - flipCode = -1; // 同时进行垂直和水平翻转 - break; - default: - throw err::Exception(err::ERR_ARGS); + switch (dir) + { + case FlipDir::X: + flipCode = 0; // 垂直翻转 + break; + case FlipDir::Y: + flipCode = 1; // 水平翻转 + break; + case FlipDir::XY: + flipCode = -1; // 同时进行垂直和水平翻转 + break; + default: + throw err::Exception(err::ERR_ARGS); } cv::flip(img, dst, flipCode); return ret; @@ -1713,32 +1771,33 @@ namespace maix::image case FMT_BGR888: case FMT_BGRA8888: ok = cv::imwrite(path, img, params); - if(!ok) + if (!ok) return err::ERR_RUNTIME; break; case FMT_RGB888: cv::cvtColor(img, img, cv::COLOR_RGB2BGR); ok = cv::imwrite(path, img, params); cv::cvtColor(img, img, cv::COLOR_RGB2BGR); - if(!ok) + if (!ok) return err::ERR_RUNTIME; break; case FMT_RGBA8888: cv::cvtColor(img, img, cv::COLOR_RGBA2BGRA); ok = cv::imwrite(path, img, params); cv::cvtColor(img, img, cv::COLOR_RGBA2BGRA); - if(!ok) + if (!ok) return err::ERR_RUNTIME; break; case FMT_JPEG: { std::string lower_path(path); std::transform(lower_path.begin(), lower_path.end(), lower_path.begin(), - [](unsigned char c){ return std::tolower(c); }); - if(path_is_format(path, ".jpeg") || path_is_format(path, ".jpg")) + [](unsigned char c) + { return std::tolower(c); }); + if (path_is_format(path, ".jpeg") || path_is_format(path, ".jpg")) { fs::File *f = fs::open(path, "wb"); - if(!f) + if (!f) return err::ERR_IO; f->write(_data, _data_size); f->close(); @@ -1747,22 +1806,22 @@ namespace maix::image else { Image *tmp = to_format(FMT_BGR888); - if(!tmp) + if (!tmp) return err::ERR_ARGS; cv::Mat img2(_height, _width, CV_8UC3, tmp->data()); ok = cv::imwrite(path, img2, params); delete tmp; - if(!ok) + if (!ok) return err::ERR_RUNTIME; } break; } case FMT_PNG: { - if(path_is_format(path, ".png")) + if (path_is_format(path, ".png")) { fs::File *f = fs::open(path, "wb"); - if(!f) + if (!f) return err::ERR_IO; f->write(_data, _data_size); f->close(); @@ -1771,12 +1830,12 @@ namespace maix::image else { Image *tmp = to_format(FMT_BGR888); - if(!tmp) + if (!tmp) return err::ERR_ARGS; cv::Mat img2(_height, _width, CV_8UC3, tmp->data()); ok = cv::imwrite(path, img2, params); delete tmp; - if(!ok) + if (!ok) return err::ERR_RUNTIME; } break; @@ -1810,108 +1869,244 @@ namespace maix::image return Size(size.width, size.height); } - std::vector resize_map_pos(int w_in, int h_in, int w_out, int h_out, image::Fit fit, int x, int y, int w, int h) { + err::Err image2cv(image::Image &img, cv::Mat &mat, bool ensure_bgr, bool copy) + { + int width = img.width(); + int height = img.height(); + int channels = img.shape()[2]; + uint8_t *img_data = (uint8_t *)img.data(); + + if (width <= 0 || height <= 0 || channels <= 0 || img_data == nullptr) + { + log::error("image2cv arg error"); + return err::Err::ERR_ARGS; + } + + if (channels == 1) + { + if (copy) + { + mat = cv::Mat(height, width, CV_8UC1); + if(ensure_bgr) + { + cv::cvtColor(mat, mat, cv::COLOR_GRAY2BGR); + } + else + { + memcpy(mat.data, img_data, width * height); + } + } + else + { + mat = cv::Mat(height, width, CV_8UC1, (void *)img_data); + } + } + else if (channels == 3) + { + if (copy) + { + mat = cv::Mat(height, width, CV_8UC3); + memcpy(mat.data, img_data, width * height * 3); + if(ensure_bgr && img.format() == image::FMT_RGB888) + { + cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); + } + } + else + { + mat = cv::Mat(height, width, CV_8UC3, (void *)img_data); + } + } + else if (channels == 4) + { + if (copy) + { + mat = cv::Mat(height, width, CV_8UC4); + memcpy(mat.data, img_data, width * height * 4); + if(ensure_bgr && img.format() == image::FMT_RGBA8888) + { + cv::cvtColor(mat, mat, cv::COLOR_RGBA2BGRA); + } + } + else + { + mat = cv::Mat(height, width, CV_8UC4, (void *)img_data); + } + } + else + { + log::error("not support channel num %d", channels); + return err::Err::ERR_ARGS; + } + return err::ERR_NONE; + } + + image::Image *cv2image(cv::Mat &mat, bool bgr = true, bool copy = true) + { + int width = mat.cols; + int height = mat.rows; + int channels = mat.channels(); + image::Format format = image::FMT_RGB888; + if (channels == 1) + { + format = image::FMT_GRAYSCALE; + } + else if (channels == 3) + { + format = bgr ? image::FMT_BGR888 : image::FMT_RGB888; + } + else if (channels == 4) + { + format = bgr ? image::FMT_BGRA8888 : image::FMT_RGBA8888; + } + else + { + log::error("not support channel num %d", channels); + return nullptr; + } + return new image::Image(width, height, format, mat.data, width * height * channels, copy); + } + + std::vector resize_map_pos(int w_in, int h_in, int w_out, int h_out, image::Fit fit, int x, int y, int w, int h) + { float scale_x = static_cast(w_out) / w_in; float scale_y = static_cast(h_out) / h_in; std::vector result; - if (fit == image::Fit::FIT_FILL) { + if (fit == image::Fit::FIT_FILL) + { x = static_cast(x * scale_x); y = static_cast(y * scale_y); - if (w > 0) { + if (w > 0) + { w = static_cast(w * scale_x); } - if (h > 0) { + if (h > 0) + { h = static_cast(h * scale_y); } - } else if (fit == image::Fit::FIT_CONTAIN) { + } + else if (fit == image::Fit::FIT_CONTAIN) + { float scale = std::min(scale_x, scale_y); - if (w_out > h_out) { + if (w_out > h_out) + { x = static_cast((w_out - w_in * scale) / 2 + x * scale); y = static_cast(y * scale); - } else { + } + else + { x = static_cast(x * scale); y = static_cast((h_out - h_in * scale) / 2 + y * scale); } - if (w > 0) { + if (w > 0) + { w = static_cast(w * scale); h = static_cast(h * scale); } - } else if (fit == image::Fit::FIT_COVER) { + } + else if (fit == image::Fit::FIT_COVER) + { float scale = std::max(scale_x, scale_y); - if (w_out > h_out) { + if (w_out > h_out) + { x = static_cast(x * scale); y = static_cast((h_out - h_in * scale) / 2 + y * scale); - } else { + } + else + { x = static_cast((w_out - w_in * scale) / 2 + x * scale); y = static_cast(y * scale); } - if (w > 0) { + if (w > 0) + { w = static_cast(w * scale); h = static_cast(h * scale); } - } else { + } + else + { throw err::Exception(err::ERR_ARGS, "Unsupported fit mode"); } // Populate the result vector result.push_back(x); result.push_back(y); - if (w > 0 && h > 0) { + if (w > 0 && h > 0) + { result.push_back(w); result.push_back(h); } return result; } - std::vector resize_map_pos_reverse(int w_in, int h_in, int w_out, int h_out, image::Fit fit, int x, int y, int w, int h) { + std::vector resize_map_pos_reverse(int w_in, int h_in, int w_out, int h_out, image::Fit fit, int x, int y, int w, int h) + { float scale_x = static_cast(w_out) / w_in; float scale_y = static_cast(h_out) / h_in; std::vector result; - if (fit == image::Fit::FIT_FILL) { + if (fit == image::Fit::FIT_FILL) + { x = static_cast(x / scale_x); y = static_cast(y / scale_y); - if (w > 0) { + if (w > 0) + { w = static_cast(w / scale_x); } - if (h > 0) { + if (h > 0) + { h = static_cast(h / scale_y); } - } else if (fit == image::Fit::FIT_CONTAIN) { + } + else if (fit == image::Fit::FIT_CONTAIN) + { float scale = std::min(scale_x, scale_y); - if (w_out > h_out) { + if (w_out > h_out) + { x = static_cast((x - (w_out - w_in * scale) / 2) / scale); y = static_cast(y / scale); - } else { + } + else + { x = static_cast(x / scale); y = static_cast((y - (h_out - h_in * scale) / 2) / scale); } - if (w > 0) { + if (w > 0) + { w = static_cast(w / scale); h = static_cast(h / scale); } - } else if (fit == image::Fit::FIT_COVER) { + } + else if (fit == image::Fit::FIT_COVER) + { float scale = std::max(scale_x, scale_y); - if (w_out > h_out) { + if (w_out > h_out) + { x = static_cast(x / scale); y = static_cast((y - (h_out - h_in * scale) / 2) / scale); - } else { + } + else + { x = static_cast((x - (w_out - w_in * scale) / 2) / scale); y = static_cast(y * scale); } - if (w > 0) { + if (w > 0) + { w = static_cast(w / scale); h = static_cast(h / scale); } - } else { + } + else + { throw err::Exception(err::ERR_ARGS, "Unsupported fit mode"); } // Populate the result vector result.push_back(x); result.push_back(y); - if (w > 0 && h > 0) { + if (w > 0 && h > 0) + { result.push_back(w); result.push_back(h); } @@ -1960,7 +2155,7 @@ namespace maix::image int Image::_get_cv_pixel_num(image::Format &format) { int pixel_num = 0; - switch (_format) + switch (format) { case image::FMT_RGB888: case image::FMT_BGR888: @@ -1984,34 +2179,45 @@ namespace maix::image return pixel_num; } - std::vector Image::_get_available_roi(std::vector roi, std::vector other_roi) { + std::vector Image::_get_available_roi(std::vector roi, std::vector other_roi) + { int src_x, src_y, src_w, src_h; - if (other_roi.size() == 0) { + if (other_roi.size() == 0) + { src_x = 0; src_y = 0; src_w = _width; src_h = _height; - } else if (other_roi.size() == 4) { + } + else if (other_roi.size() == 4) + { src_x = other_roi[0]; src_y = other_roi[1]; src_w = other_roi[2]; src_h = other_roi[3]; - } else { + } + else + { throw err::Exception("other_roi size must be 4"); } - if (roi.size() == 0) { + if (roi.size() == 0) + { return std::vector{0, 0, _width, _height}; - } else if (roi.size() != 4) { + } + else if (roi.size() != 4) + { throw err::Exception("roi size must be 4"); } - if (roi[2] <= 0 || roi[3] <= 0) { + if (roi[2] <= 0 || roi[3] <= 0) + { throw err::Exception("roi width and height must > 0"); } - if ((roi[0] >= (src_x + src_w)) || (roi[1] >= (src_y + src_h)) || (roi[0] + roi[2] <= src_x) || (roi[1] + roi[3] <= src_y)) { + if ((roi[0] >= (src_x + src_w)) || (roi[1] >= (src_y + src_h)) || (roi[0] + roi[2] <= src_x) || (roi[1] + roi[3] <= src_y)) + { throw err::Exception("roi does not overlap on the image!"); } diff --git a/examples/nn_hand_keypoints/.gitignore b/examples/nn_hand_keypoints/.gitignore new file mode 100644 index 00000000..7171eaac --- /dev/null +++ b/examples/nn_hand_keypoints/.gitignore @@ -0,0 +1,9 @@ +build +dist +.config.mk +.flash.conf.json +data + +/CMakeLists.txt + +__pycache__ diff --git a/examples/nn_hand_keypoints/README.md b/examples/nn_hand_keypoints/README.md new file mode 100644 index 00000000..7ca28940 --- /dev/null +++ b/examples/nn_hand_keypoints/README.md @@ -0,0 +1,13 @@ +Detect hand 21 keypoints +==== + + +- Use `maixcdk build` to compile binary files. +- Move files and runtime libraries to device. +- Use SSH command with parameters to run programs. + +` "Usage: " + std::string(argv[0]) + " mud_model_path "; ` + +more documentations refer to [MaixPy](https://wiki.sipeed.com/maixpy/doc/zh/vision/hand_landmarks.html) + + diff --git a/examples/nn_hand_keypoints/main/CMakeLists.txt b/examples/nn_hand_keypoints/main/CMakeLists.txt new file mode 100644 index 00000000..452f67bd --- /dev/null +++ b/examples/nn_hand_keypoints/main/CMakeLists.txt @@ -0,0 +1,74 @@ +############### Add include ################### +list(APPEND ADD_INCLUDE "include" + ) +list(APPEND ADD_PRIVATE_INCLUDE "") +############################################### + +############ Add source files ################# +# list(APPEND ADD_SRCS "src/main.c" +# "src/test.c" +# ) +append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c") +# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c") +# FILE(GLOB EXTRA_SRC "src/*.c") +# list(APPEND ADD_SRCS ${EXTRA_SRC}) +# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS +# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS +# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c") +# set(ADD_ASM_SRCS "src/asm.S") +# list(APPEND ADD_SRCS ${ADD_ASM_SRCS}) +# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language +# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB") +############################################### + +###### Add required/dependent components ###### +list(APPEND ADD_REQUIREMENTS basic nn vision) +############################################### + +###### Add link search path for requirements/libs ###### +# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib") +# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here +# set (OpenCV_DIR opencv/lib/cmake/opencv4) +# find_package(OpenCV REQUIRED) +############################################### + +############ Add static libs ################## +# list(APPEND ADD_STATIC_LIB "lib/libtest.a") +############################################### + +#### Add compile option for this component #### +#### Just for this component, won't affect other +#### modules, including component that depend +#### on this component +# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1) + +#### Add compile option for this component +#### and components depend on this component +# list(APPEND ADD_DEFINITIONS -DAAAAA222=1 +# -DAAAAA333=1) +############################################### + +############ Add static libs ################## +#### Update parent's variables like CMAKE_C_LINK_FLAGS +# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE) +############################################### + +######### Add files need to download ######### +# list(APPEND ADD_FILE_DOWNLOADS "{ +# 'url': 'https://*****/abcde.tar.xz', +# 'urls': [], # backup urls, if url failed, will try urls +# 'sites': [], # download site, user can manually download file and put it into dl_path +# 'sha256sum': '', +# 'filename': 'abcde.tar.xz', +# 'path': 'toolchains/xxxxx', +# 'check_files': [] +# }" +# ) +# +# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx, +# you can directly use then, for example use it in add_custom_command +############################################## + +# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib +register_component() diff --git a/examples/nn_hand_keypoints/main/Kconfig b/examples/nn_hand_keypoints/main/Kconfig new file mode 100644 index 00000000..e69de29b diff --git a/examples/nn_hand_keypoints/main/include/main.h b/examples/nn_hand_keypoints/main/include/main.h new file mode 100644 index 00000000..45dcbb04 --- /dev/null +++ b/examples/nn_hand_keypoints/main/include/main.h @@ -0,0 +1,3 @@ +#pragma once + + diff --git a/examples/nn_hand_keypoints/main/src/main.cpp b/examples/nn_hand_keypoints/main/src/main.cpp new file mode 100644 index 00000000..19791877 --- /dev/null +++ b/examples/nn_hand_keypoints/main/src/main.cpp @@ -0,0 +1,123 @@ + +#include "maix_basic.hpp" +#include "maix_vision.hpp" +#include "maix_nn_hand_landmarks.hpp" +#include "main.h" + +using namespace maix; + +int _main(int argc, char *argv[]) +{ + log::info("Program start"); + std::string model_type = "unknown"; + int ret = 0; + err::Err e; + maix::image::Format img_fmt = maix::image::FMT_RGB888; + std::vector mean = {}; + std::vector scale = {}; + char tmp_chars[64] = {0}; + + std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path "; + + if (argc < 2) + { + log::info(help.c_str()); + return -1; + } + + const char *model_path = argv[1]; + float conf_threshold = 0.7; + float iou_threshold = 0.45; + float conf_threshold2 = 0.8; + bool landmarks_rel = true; // draw relative landmarks on image + + nn::HandLandmarks detector(""); + e = detector.load(model_path); + err::check_raise(e, "load model failed"); + log::info("load yolo11 model %s success", model_path); + + if (argc >= 3) + { + const char *img_path = argv[2]; + log::info("load image now"); + maix::image::Image *img = maix::image::load(img_path, img_fmt); + err::check_null_raise(img, "load image " + std::string(img_path) + " failed"); + log::info("load image %s success: %s", img_path, img->to_str().c_str()); + log::info("detect now"); + nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold, conf_threshold2, landmarks_rel); + if (result->size() == 0) + { + log::info("no object detected !"); + } + for (auto &r : *result) + { + // points: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y, + // x0, y0, z1, x1, y1, z2, ..., x20, y20, z20,[x0_rel, y0_rel,...,x20_rel, y20_rel] + log::info("result: %s", r->to_str().c_str()); + snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", r->points[0] == 0 ? "left" : "right", r->score); + img->draw_string(r->points[0], r->points[1], detector.labels[r->class_id], + r->class_id == 0 ? maix::image::Color::from_rgb(255, 0, 0) : maix::image::Color::from_rgb(0, 255, 0), + 1.2, 2); + detector.draw_hand(*img, r->class_id, r->points, 4, 10, true); + } + img->save("result.jpg"); + delete result; + delete img; + } + else + { + log::info("open camera now"); + camera::Camera cam = camera::Camera(320, 320, detector.input_format()); + log::info("open camera success"); + display::Display disp = display::Display(); + while (!app::need_exit()) + { + uint64_t t = time::ticks_ms(); + maix::image::Image *img = cam.read(); + err::check_null_raise(img, "read camera failed"); + uint64_t t2 = time::ticks_ms(); + nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold, conf_threshold2, landmarks_rel); + uint64_t t3 = time::ticks_ms(); + for (auto &r : *result) + { + // points: box_topleft_x, box_topleft_y, box_topright_x, box_topright_y, box_bottomright_x, box_bottomright_y， box_bottomleft_x, box_bottomleft_y, + // x0, y0, z1, x1, y1, z2, ..., x20, y20, z20,[x0_rel, y0_rel,...,x20_rel, y20_rel] + log::info("result: %s", r->to_str().c_str()); + snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", r->points[0] == 0 ? "left" : "right", r->score); + img->draw_string(r->points[0], r->points[1], detector.labels[r->class_id], + r->class_id == 0 ? maix::image::Color::from_rgb(255, 0, 0) : maix::image::Color::from_rgb(0, 255, 0), + 1.4, 2); + // img->draw_rect(r->x, r->y, r->w, r->h, image::COLOR_RED, 1); + detector.draw_hand(*img, r->class_id, r->points, 4, 10, true); + if(landmarks_rel) + { + img->draw_rect(0, 0, detector.input_width(false), detector.input_height(false), image::COLOR_YELLOW, 1); + for(int i=0; i<21; i++) + { + img->draw_circle(r->points[8+21*3 + i*2], r->points[9+21*3 + i*2], 2, image::COLOR_YELLOW, -1); + } + } + } + disp.show(*img); + delete result; + delete img; + log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2); + } + } + + log::info("Program exit"); + + return ret; +} + +int main(int argc, char *argv[]) +{ + // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true. + signal(SIGINT, [](int sig) + { app::set_exit_flag(true); }); + + // Use CATCH_EXCEPTION_RUN_RETURN to catch exception, + // if we don't catch exception, when program throw exception, the objects will not be destructed. + // So we catch exception here to let resources be released(call objects' destructor) before exit. + CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); +}