diff --git a/components/maixcam_lib/include/maix_nn_maixcam.hpp b/components/maixcam_lib/include/maix_nn_maixcam.hpp index 91e2aacf..dd97f460 100644 --- a/components/maixcam_lib/include/maix_nn_maixcam.hpp +++ b/components/maixcam_lib/include/maix_nn_maixcam.hpp @@ -9,7 +9,7 @@ namespace maix::nn class NN_MaixCam : public NNBase { public: - NN_MaixCam(); + NN_MaixCam(bool dual_buff); ~NN_MaixCam(); /** @@ -31,6 +31,12 @@ namespace maix::nn */ virtual bool loaded() final; + /** + * Enable dual buff or disable dual buff + * @param enable true to enable, false to disable + */ + virtual void set_dual_buff(bool enable); + /** * Get model input layer info * @return input layer info @@ -69,6 +75,7 @@ namespace maix::nn private: bool _loaded; void *_data; + bool _enable_dual_buff; }; } // namespace maix::nn diff --git a/components/nn/include/maix_nn.hpp b/components/nn/include/maix_nn.hpp index efd4e82a..3d4cdc27 100644 --- a/components/nn/include/maix_nn.hpp +++ b/components/nn/include/maix_nn.hpp @@ -171,6 +171,13 @@ namespace maix::nn */ virtual bool loaded() = 0; + /** + * Enable dual buff or disable dual buff + * @param enable true to enable, false to disable + * @maixpy maix.nn.NN.set_dual_buff + */ + virtual void set_dual_buff(bool enable) = 0; + /** * Get model input layer info * @return input layer info @@ -223,9 +230,12 @@ namespace maix::nn * @param[in] model model file path, model format can be MUD(model universal describe file) file. * If model_path set, will load model from file, load failed will raise err.Exception. * If model_path not set, you can load model later by load function. + * @param[in] dual_buff prepare dual input output buffer to accelarate forward, that is, when NPU is forwarding we not wait and prepare the next input buff. + * If you want to ensure every time forward output the input's result, set this arg to false please. + * Default true to ensure speed. * @maixpy maix.nn.NN.__init__ */ - NN(const std::string &model = ""); + NN(const std::string &model = "", bool dual_buff = true); ~NN(); /** @@ -249,6 +259,13 @@ namespace maix::nn */ bool loaded(); + /** + * Enable dual buff or disable dual buff + * @param enable true to enable, false to disable + * @maixpy maix.nn.NN.set_dual_buff + */ + void set_dual_buff(bool enable); + /** * Get model input layer info * @return input layer info diff --git a/components/nn/include/maix_nn_classifier.hpp b/components/nn/include/maix_nn_classifier.hpp index 55d60552..89ce3cb1 100644 --- a/components/nn/include/maix_nn_classifier.hpp +++ b/components/nn/include/maix_nn_classifier.hpp @@ -24,12 +24,16 @@ namespace maix::nn * Construct a new Classifier object * @param model MUD model path, if empty, will not load model, you can call load() later. * if not empty, will load model and will raise err::Exception if load failed. + * @param[in] dual_buff prepare dual input output buffer to accelarate forward, that is, when NPU is forwarding we not wait and prepare the next input buff. + * If you want to ensure every time forward output the input's result, set this arg to false please. + * Default true to ensure speed. * @maixpy maix.nn.Classifier.__init__ * @maixcdk maix.nn.Classifier.Classifier */ - Classifier(const string &model = "") + Classifier(const string &model = "", bool dual_buff = true) { _model = nullptr; + _dual_buff = dual_buff; if (!model.empty()) { err::Err e = load(model); @@ -68,7 +72,7 @@ namespace maix::nn delete _model; _model = nullptr; } - _model = new nn::NN(model); + _model = new nn::NN(model, _dual_buff); if (!_model) { return err::ERR_NO_MEM; @@ -345,6 +349,7 @@ namespace maix::nn private: image::Format _input_img_fmt; bool _input_is_img; + bool _dual_buff; nn::NN *_model; std::map _extra_info; image::Size _input_size; diff --git a/components/nn/include/maix_nn_yolov5.hpp b/components/nn/include/maix_nn_yolov5.hpp index 5d15efae..abad77dc 100644 --- a/components/nn/include/maix_nn_yolov5.hpp +++ b/components/nn/include/maix_nn_yolov5.hpp @@ -26,13 +26,17 @@ namespace maix::nn /** * Constructor of YOLOv5 class * @param model model path, default empty, you can load model later by load function. + * @param[in] dual_buff prepare dual input output buffer to accelarate forward, that is, when NPU is forwarding we not wait and prepare the next input buff. + * If you want to ensure every time forward output the input's result, set this arg to false please. + * Default true to ensure speed. * @throw If model arg is not empty and load failed, will throw err::Exception. * @maixpy maix.nn.YOLOv5.__init__ * @maixcdk maix.nn.YOLOv5.YOLOv5 */ - YOLOv5(const string &model = "") + YOLOv5(const string &model = "", bool dual_buff = true) { _model = nullptr; + _dual_buff = dual_buff; if (!model.empty()) { err::Err e = load(model); @@ -65,7 +69,7 @@ namespace maix::nn delete _model; _model = nullptr; } - _model = new nn::NN(model); + _model = new nn::NN(model, _dual_buff); if (!_model) { return err::ERR_NO_MEM; @@ -361,6 +365,7 @@ namespace maix::nn std::map _extra_info; float _conf_th = 0.5; float _iou_th = 0.45; + bool _dual_buff; private: std::vector *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit) diff --git a/components/nn/include/maix_nn_yolov8.hpp b/components/nn/include/maix_nn_yolov8.hpp index 4d99d092..ccca067f 100644 --- a/components/nn/include/maix_nn_yolov8.hpp +++ b/components/nn/include/maix_nn_yolov8.hpp @@ -45,14 +45,18 @@ namespace maix::nn /** * Constructor of YOLOv8 class * @param model model path, default empty, you can load model later by load function. + * @param[in] dual_buff prepare dual input output buffer to accelarate forward, that is, when NPU is forwarding we not wait and prepare the next input buff. + * If you want to ensure every time forward output the input's result, set this arg to false please. + * Default true to ensure speed. * @throw If model arg is not empty and load failed, will throw err::Exception. * @maixpy maix.nn.YOLOv8.__init__ * @maixcdk maix.nn.YOLOv8.YOLOv8 */ - YOLOv8(const string &model = "") + YOLOv8(const string &model = "", bool dual_buff = true) { _model = nullptr; _type = TYPE_DETECT; + _dual_buff = dual_buff; if (!model.empty()) { err::Err e = load(model); @@ -90,7 +94,7 @@ namespace maix::nn delete _model; _model = nullptr; } - _model = new nn::NN(model); + _model = new nn::NN(model, _dual_buff); if (!_model) { return err::ERR_NO_MEM; @@ -454,6 +458,7 @@ namespace maix::nn float _iou_th = 0.45; float _keypoint_th = 0.5; YOLOv8_Type _type; + bool _dual_buff; private: nn::Objects *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit) diff --git a/components/nn/src/maix_nn.cpp b/components/nn/src/maix_nn.cpp index 14c0adcf..d67d10e6 100644 --- a/components/nn/src/maix_nn.cpp +++ b/components/nn/src/maix_nn.cpp @@ -104,11 +104,11 @@ namespace maix::nn return err::ERR_NONE; } - NN::NN(const std::string &model_path) + NN::NN(const std::string &model_path, bool dual_buff) { _impl = nullptr; #if PLATFORM_MAIXCAM - _impl = new NN_MaixCam(); + _impl = new NN_MaixCam(dual_buff); #endif if(!_impl) { @@ -168,6 +168,11 @@ namespace maix::nn return _impl->loaded(); } + void NN::set_dual_buff(bool enable) + { + _impl->set_dual_buff(enable); + } + std::vector NN::inputs_info() { return _impl->inputs_info(); diff --git a/examples/nn_yolov5/main/src/main.cpp b/examples/nn_yolov5/main/src/main.cpp index a1e8b3d7..3537e629 100644 --- a/examples/nn_yolov5/main/src/main.cpp +++ b/examples/nn_yolov5/main/src/main.cpp @@ -86,9 +86,10 @@ int _main(int argc, char *argv[]) img->draw_string(r.x, r.y, detector.labels[r.class_id], maix::image::Color::from_rgb(255, 0, 0)); } disp.show(*img); + uint64_t t4 = time::ticks_ms(); delete result; delete img; - log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2); + log::info("time: all %d ms, read %d ms, detect %d ms, show %d ms", time::ticks_ms() - t, t2 - t, t3 - t2, t4 - t3); } } diff --git a/projects/app_classifier/app.yaml b/projects/app_classifier/app.yaml index 4033fa14..1306d16a 100644 --- a/projects/app_classifier/app.yaml +++ b/projects/app_classifier/app.yaml @@ -1,7 +1,7 @@ id: classifier name: AI Classifier name[zh]: AI 分类器 -version: 1.0.0 +version: 1.0.1 icon: assets/classifier.png author: Sipeed Ltd desc: AI object classification diff --git a/projects/app_classifier/main/src/main.cpp b/projects/app_classifier/main/src/main.cpp index ac48ee8a..7e85a9cd 100644 --- a/projects/app_classifier/main/src/main.cpp +++ b/projects/app_classifier/main/src/main.cpp @@ -10,6 +10,7 @@ using namespace maix; int _main(int argc, char *argv[]) { int ret = 0; + char tmp_chars[128] = {0}; log::info("Program start"); const char *model_path = "/root/models/mobilenetv2.mud"; @@ -33,33 +34,41 @@ int _main(int argc, char *argv[]) int w = disp.width(); int h = disp.height(); int min_len = w > h ? h : w; - int max_len = w > h ? w : h; + // int max_len = w > h ? w : h; log::info("open camera now"); // image::Size input_size = classifier.input_size(); - camera::Camera cam = camera::Camera(min_len, min_len, classifier.input_format()); + camera::Camera cam = camera::Camera(w, h, classifier.input_format()); log::info("open camera success"); + uint64_t t, t2, t3, t_show, t_all = 0; while (!app::need_exit()) { ts.read(ts_x, ts_y, ts_pressed); - int x = (max_len - min_len) / 2; - if (ts_pressed && ts_x >= x && ts_x < x + 60 && ts_y < 40) + if (ts_pressed && ts_x < 40 + 60 && ts_y < 40) { break; } - uint64_t t = time::ticks_us(); + t = time::ticks_ms(); image::Image *img = cam.read(); err::check_null_raise(img, "read camera failed"); - std::vector> *result = classifier.classify(*img); + image::Image *img_final = img->resize(classifier.input_width(), classifier.input_height(), image::Fit::FIT_COVER); + err::check_null_raise(img_final, "resize failed"); + t2 = time::ticks_ms(); + std::vector> *result = classifier.classify(*img_final); + t3 = time::ticks_ms(); int max_idx = result->at(0).first; float max_score = result->at(0).second; - snprintf(msg, sizeof(msg), "%5.2f: %s", max_score, classifier.labels[max_idx].c_str()); - img->draw_string(2, 36, msg, image::COLOR_RED, 1.5); - + img->draw_rect((w - min_len) / 2, (h - min_len) / 2, min_len, min_len, image::COLOR_WHITE, 2); + snprintf(msg, sizeof(msg), "%4.1f %%:\n%s", max_score * 100, classifier.labels[max_idx].c_str()); + img->draw_string((w - min_len) / 2, disp.height() - 80, msg, image::COLOR_RED, 2, 2); img->draw_image(0, 0, *ret_img); + snprintf(tmp_chars, sizeof(tmp_chars), "All: %ldms, cam: %ldms\ndetect: %ldms, show: %ldms", t_all, t2 - t, t3 - t2, t_show); + img->draw_string((w - min_len) / 2, 4, tmp_chars, image::COLOR_RED, 1.5, 2); disp.show(*img); + t_show = time::ticks_ms() - t3; + t_all = time::ticks_ms() - t; delete result; delete img; - log::info("time: %d ms", time::ticks_us() - t); + delete img_final; } log::info("Program exit"); diff --git a/projects/app_detector/app.yaml b/projects/app_detector/app.yaml index da35c043..a49725da 100644 --- a/projects/app_detector/app.yaml +++ b/projects/app_detector/app.yaml @@ -1,7 +1,7 @@ id: detector name: AI Detector name[zh]: AI 检测器 -version: 1.0.0 +version: 1.0.1 icon: assets/detector.png author: Sipeed Ltd desc: AI object detection diff --git a/projects/app_detector/main/src/main.cpp b/projects/app_detector/main/src/main.cpp index 0946aeed..2a09fddb 100644 --- a/projects/app_detector/main/src/main.cpp +++ b/projects/app_detector/main/src/main.cpp @@ -13,10 +13,9 @@ int _main(int argc, char *argv[]) std::string model_type = "unknown"; int ret = 0; err::Err e; - maix::image::Format img_fmt = maix::image::FMT_RGB888; std::vector mean = {}; std::vector scale = {}; - char tmp_chars[64] = {0}; + char tmp_chars[128] = {0}; touchscreen::TouchScreen ts; int ts_x = 0, ts_y = 0; @@ -41,6 +40,7 @@ int _main(int argc, char *argv[]) camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format()); log::info("open camera success"); display::Display disp = display::Display(); + uint64_t t, t2, t3, t_show, t_all = 0; while (!app::need_exit()) { ts.read(ts_x, ts_y, ts_pressed); @@ -48,21 +48,25 @@ int _main(int argc, char *argv[]) { break; } - uint64_t t = time::ticks_us(); + t = time::ticks_ms(); maix::image::Image *img = cam.read(); err::check_null_raise(img, "read camera failed"); - std::vector *result = detector.detect(*img); + t2 = time::ticks_ms(); + std::vector *result = detector.detect(*img, conf_threshold, iou_threshold); + t3 = time::ticks_ms(); for (auto &r : *result) { - log::info("result: %s", r.to_str().c_str()); img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0)); img->draw_string(r.x, r.y, detector.labels[r.class_id], maix::image::Color::from_rgb(255, 0, 0)); } img->draw_image(0, 0, *ret_img); + snprintf(tmp_chars, sizeof(tmp_chars), "All: %ldms, cam: %ldms\ndetect: %ldms, show: %ldms", t_all, t2 - t, t3 - t2, t_show); + img->draw_string(ret_img->width() + 2, 2, tmp_chars, image::COLOR_RED); disp.show(*img); + t_show = time::ticks_ms() - t3; + t_all = time::ticks_ms() - t; delete result; delete img; - log::info("time: %d ms", time::ticks_us() - t); } log::info("Program exit"); diff --git a/tools/cmake/copy_assets.py b/tools/cmake/copy_assets.py index 30193588..4f53368c 100644 --- a/tools/cmake/copy_assets.py +++ b/tools/cmake/copy_assets.py @@ -35,8 +35,12 @@ def copy_assets(project_path, dist_path): for src_key, dest_value in assets.items(): src = os.path.join(project_path, src_key) dest = os.path.join(dist_path, dest_value) + os.makedirs(os.path.dirname(dest), exist_ok=True) if os.path.exists(src): - shutil.copytree(src, dest, dirs_exist_ok=True) + if os.path.isdir(src): + shutil.copytree(src, dest, dirs_exist_ok=True) + else: + shutil.copyfile(src, dest) print(f"Copied {src} to {dest}") else: raise Exception(f"{src} does not exist.")