add yolo-world support for maixcam2

components/basic/include/maix_tensor.hpp

@@ -557,11 +557,11 @@ namespace maix
             */
             void clear()
             {
-                for(const auto &k : _keys)
+                std::vector<std::string> keys = _keys;
+                for(const auto &k : keys)
                 {
                     rm_tensor(k);
                 }
-                _keys.clear();
             }
 
             /**

components/nn/include/maix_nn.hpp

@@ -223,7 +223,7 @@ namespace maix::nn
          * @param[in] input input tensor
          * @param[out] output output tensor, a key-value dict, key is layer name, value is tensor.
          *                    In C++, value can be allocated by caller, if outputs size is zero, forward will allocate memory, so you should delete it after use.
-         * @return error code, if forward success, return err::ERR_NONE
+         * @return error code, if forward success, return err::ERR_NONE, if dual_buff mode and not ready, return err::ERR_NOT_READY.
          */
         virtual err::Err forward(tensor::Tensors &inputs, tensor::Tensors &outputs, bool copy_result = true, bool dual_buff_wait = false) = 0;
 

components/nn/include/maix_nn_classifier.hpp

@@ -196,7 +196,8 @@ namespace maix::nn
                 // strip line
                 line.erase(0, line.find_first_not_of(" \t\r\n"));
                 line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
+                if(!line.empty())
+                    labels.push_back(line);
             }
             f->close();
             delete f;

components/nn/include/maix_nn_pp_ocr.hpp

@@ -551,7 +551,8 @@ namespace maix::nn
                 // strip line
                 line.erase(0, line.find_first_not_of(" \t\r\n"));
                 line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
+                if(!line.empty())
+                    labels.push_back(line);
             }
             f->close();
             delete f;

components/nn/include/maix_nn_yolo11.hpp

@@ -104,7 +104,9 @@ namespace maix::nn
                 err::Err e = load(model);
                 if (e != err::ERR_NONE)
                 {
-                    throw err::Exception(e, "load model failed");
+                    char tmp[128] = {0};
+                    snprintf(tmp, sizeof(tmp), "load model %s failed", model.c_str());
+                    throw err::Exception(e, tmp);
                 }
             }
         }
@@ -683,7 +685,8 @@ namespace maix::nn
                 // strip line
                 line.erase(0, line.find_first_not_of(" \t\r\n"));
                 line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
+                if(!line.empty())
+                    labels.push_back(line);
             }
             f->close();
             delete f;

components/nn/include/maix_nn_yolov5.hpp

@@ -40,7 +40,9 @@ namespace maix::nn
                 err::Err e = load(model);
                 if (e != err::ERR_NONE)
                 {
-                    throw err::Exception(e, "load model failed");
+                    char tmp[128] = {0};
+                    snprintf(tmp, sizeof(tmp), "load model %s failed", model.c_str());
+                    throw err::Exception(e, tmp);
                 }
             }
         }
@@ -342,7 +344,8 @@ namespace maix::nn
                 // strip line
                 line.erase(0, line.find_first_not_of(" \t\r\n"));
                 line.erase(line.find_last_not_of(" \t\r\n") + 1);
-                labels.push_back(line);
+                if(!line.empty())
+                    labels.push_back(line);
             }
             f->close();
             delete f;

examples/nn_yolo_world/.gitignore

@@ -0,0 +1,9 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+
+/CMakeLists.txt
+
+__pycache__

examples/nn_yolo_world/README.md

@@ -0,0 +1,9 @@
+nn_yolov8 examples based on MaixCDK
+====
+## only run yolov8 model
+
+- Use `maixcdk build` to compile binary files.
+- Move files and runtime libraries to device.
+- Use SSH command with parameters to run programs.
+  
+` "Usage: " + std::string(argv[0]) + " mud_model_path <image_path>"; `

examples/nn_yolo_world/main/CMakeLists.txt

@@ -0,0 +1,74 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic nn vision)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+# 'url': 'https://*****/abcde.tar.xz',
+# 'urls': [],  # backup urls, if url failed, will try urls
+# 'sites': [], # download site, user can manually download file and put it into dl_path
+# 'sha256sum': '',
+# 'filename': 'abcde.tar.xz',
+# 'path': 'toolchains/xxxxx',
+# 'check_files': []
+# }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

examples/nn_yolo_world/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

examples/nn_yolo_world/main/src/main.cpp

@@ -0,0 +1,139 @@
+
+#include "maix_basic.hpp"
+#include "maix_vision.hpp"
+#include "maix_nn_yolo_world.hpp"
+#include "main.h"
+
+using namespace maix;
+
+int _main(int argc, char *argv[])
+{
+    log::info("Program start");
+    std::string model_type = "unknown";
+    int ret = 0;
+    maix::image::Format img_fmt = maix::image::FMT_RGB888;
+    std::vector<float> mean = {};
+    std::vector<float> scale = {};
+    char tmp_chars[64] = {0};
+
+    std::string help = "Usage: " + std::string(argv[0]) + " mud_model_path text_feature_path labels_path [image_path]";
+
+    if (argc < 2)
+    {
+        log::info(help.c_str());
+        return -1;
+    }
+
+    const char *model_path = argv[1];
+    const char *text_feature_path = argv[2];
+    const char *labels_path = argv[3];
+    float conf_threshold = 0.5;
+    float iou_threshold = 0.45;
+    bool dual_buff = true;
+    int repeat_times = 20;
+
+
+    nn::YOLOWorld detector(model_path, text_feature_path, labels_path, dual_buff);
+    log::info("load yolo-world model %s success, class text feature path: %s, labels path: %s", model_path, text_feature_path, labels_path);
+
+    if (argc >= 5)
+    {
+        const char *img_path = argv[4];
+        log::info("load image now");
+        maix::image::Image *img = maix::image::load(img_path, img_fmt);
+        err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
+        log::info("load image %s success: %s", img_path, img->to_str().c_str());
+        if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
+        {
+            log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
+            // we resize here manually, if not, the detect function will auto resize.
+            maix::image::Image *img2 = img->resize(detector.input_size().width(), detector.input_size().height(), image::Fit::FIT_CONTAIN);
+            delete img;
+            img = img2;
+        }
+        log::info("detect now");
+        nn::Objects *result = nullptr;
+        if(repeat_times > 1)
+        {
+            for(int i = 0; i < 3; ++i) // warmup
+            {
+                result = detector.detect(*img, conf_threshold, iou_threshold);
+                delete result;
+            }
+        }
+        std::vector<int> intervals(repeat_times);
+        for(int i = 0; i < repeat_times; ++i) // average time
+        {
+            uint64_t start = time::ticks_ms();
+            result = detector.detect(*img, conf_threshold, iou_threshold);
+            intervals[i] = time::ticks_ms() - start;
+            if(i < repeat_times - 1)
+                delete result;
+        }
+        float sum = 0;
+        for(auto i : intervals)
+        {
+            sum += i;
+        }
+        log::info("run %d times, average time: %dms", repeat_times, (int)(sum / repeat_times));
+
+        if(result->size() == 0)
+        {
+            log::info("no object detected !");
+        }
+        for (auto &r : *result)
+        {
+
+            log::info("result: %s", r->to_str().c_str());
+            img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
+            snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.2f", detector.labels[r->class_id].c_str(), r->score);
+            img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
+        }
+        img->save("result.jpg");
+        delete result;
+        delete img;
+    }
+    else
+    {
+        log::info("open camera now");
+        maix::image::Size input_size = detector.input_size();
+        camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
+        log::info("open camera success");
+        display::Display disp = display::Display();
+        while (!app::need_exit())
+        {
+            uint64_t t = time::ticks_ms();
+            maix::image::Image *img = cam.read();
+            err::check_null_raise(img, "read camera failed");
+            uint64_t t2 = time::ticks_ms();
+            nn::Objects *result = detector.detect(*img);
+            uint64_t t3 = time::ticks_ms();
+            for (auto &r : *result)
+            {
+                log::info("result: %s", r->to_str().c_str());
+                img->draw_rect(r->x, r->y, r->w, r->h, maix::image::Color::from_rgb(255, 0, 0));
+                img->draw_string(r->x, r->y, detector.labels[r->class_id], maix::image::Color::from_rgb(255, 0, 0));
+            }
+            disp.show(*img);
+            delete result;
+            delete img;
+            log::info("time: all %d ms, detect %d ms, fps: %.1f", time::ticks_ms() - t, t3 - t2, time::fps());
+        }
+    }
+
+    log::info("Program exit");
+
+    return ret;
+}
+
+int main(int argc, char *argv[])
+{
+    // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+    signal(SIGINT, [](int sig)
+           { app::set_exit_flag(true); });
+
+    // Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
+    // if we don't catch exception, when program throw exception, the objects will not be destructed.
+    // So we catch exception here to let resources be released(call objects' destructor) before exit.
+    CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
+}