* [maixcam2] Supported yolo26

* [maixcam2] Supported maix-speech

components/3rd_party/WavFileReader/CMakeLists.txt

@@ -24,7 +24,7 @@ list(APPEND ADD_INCLUDE         "."
 ###############################################
 
 ############ Add dynamic libs ##################
-# list(APPEND ADD_DYNAMIC_LIB "lib/arch/v831/libmaix_nn.so"
+# list(APPEND ADD_DYNAMIC_LIB "lib/arch/v831/libmaix_nn.so"s
 #                             "lib/arch/v831/libmaix_cam.so"
 # )
 ###############################################

components/3rd_party/alsa_lib/CMakeLists.txt

@@ -30,9 +30,15 @@ list(APPEND ADD_INCLUDE ${alsa_lib_include_dir})
 set_property(SOURCE ${alsa_lib_include_dir} PROPERTY GENERATED 1)
 
 # lib
-set(alsa_lib_dynamic_lib_file   ${alsa_lib_dir}/libasound.so
-                                ${alsa_lib_dir}/libatopology.so)
-list(APPEND ADD_DYNAMIC_LIB ${alsa_lib_dynamic_lib_file})
-set_property(SOURCE ${alsa_lib_dynamic_lib_file} PROPERTY GENERATED 1)
-
+if(PLATFORM_MAIXCAM)
+    set(alsa_lib_dynamic_lib_file   ${alsa_lib_dir}/maixcam/libasound.so
+                                    ${alsa_lib_dir}/maixcam/libatopology.so)
+    list(APPEND ADD_DYNAMIC_LIB ${alsa_lib_dynamic_lib_file})
+    set_property(SOURCE ${alsa_lib_dynamic_lib_file} PROPERTY GENERATED 1)
+elseif(PLATFORM_MAIXCAM2)
+    set(alsa_lib_dynamic_lib_file   ${alsa_lib_dir}/maixcam2/libasound.so
+                                    ${alsa_lib_dir}/maixcam2/libatopology.so)
+    list(APPEND ADD_DYNAMIC_LIB ${alsa_lib_dynamic_lib_file})
+    set_property(SOURCE ${alsa_lib_dynamic_lib_file} PROPERTY GENERATED 1)
+endif()
 register_component()

components/3rd_party/alsa_lib/lib/maixcam2/libasound.so

@@ -0,0 +1 @@
+libasound.so.2.0.0

components/3rd_party/alsa_lib/lib/maixcam2/libasound.so.2

@@ -0,0 +1 @@
+libasound.so.2.0.0

components/3rd_party/alsa_lib/lib/maixcam2/libatopology.so

@@ -0,0 +1 @@
+libatopology.so.2.0.0

components/3rd_party/alsa_lib/lib/maixcam2/libatopology.so.2

@@ -0,0 +1 @@
+libatopology.so.2.0.0

components/nn/.gitignore

@@ -0,0 +1 @@
+!*

components/nn/CMakeLists.txt

@@ -12,6 +12,8 @@ if(PLATFORM_MAIXCAM)
 elseif(PLATFORM_MAIXCAM2)
     append_srcs_dir(ADD_SRCS "port/maixcam2")
     list(APPEND ADD_REQUIREMENTS eigen librosa_simple OpenCC onnxruntime uchardet)
+    list(APPEND ADD_DYNAMIC_LIB "lib/libms_asr_ax630c.so")
+    list(APPEND ADD_DEFINITIONS -DDR_WAV_IMPLEMENTATION)
 else()
     list(APPEND ADD_PRIVATE_INCLUDE "port/linux")
     append_srcs_dir(ADD_SRCS "port/linux")

components/nn/include/maix_nn_yolo26.hpp

@@ -0,0 +1,679 @@
+/**
+ * @author Tao@sipeed, modified for YOLO26
+ * @copyright Sipeed Ltd 2026-
+ * @license Apache 2.0
+ * @update 2026: Add YOLO26 support with NEON optimization.
+ */
+
+#pragma once
+#include "maix_basic.hpp"
+#include "maix_nn.hpp"
+#include "maix_image.hpp"
+#include "maix_nn_F.hpp"
+#include "maix_nn_object.hpp"
+
+#include <arm_neon.h>
+
+namespace maix::nn
+{
+    /**
+     * YOLO26 class for object detection
+     * @maixpy maix.nn.YOLO26
+     */
+    class YOLO26
+    {
+    public:
+        /**
+         * Constructor of YOLO26 class
+         * @param model model path, default empty, you can load model later by load function.
+         * @param[in] dual_buff prepare dual input output buffer to accelerate forward, that is, when NPU is forwarding we not wait and prepare the next input buff.
+         *                      If you want to ensure every time forward output the input's result, set this arg to false please.
+         *                      Default true to ensure speed.
+         * @throw If model arg is not empty and load failed, will throw err::Exception.
+         * @maixpy maix.nn.YOLO26.__init__
+         * @maixcdk maix.nn.YOLO26.YOLO26
+         */
+        YOLO26(const string &model = "", bool dual_buff = true)
+        {
+            _model = nullptr;
+            _dual_buff = dual_buff;
+            if (!model.empty())
+            {
+                err::Err e = load(model);
+                if (e != err::ERR_NONE)
+                {
+                    char tmp[128] = {0};
+                    snprintf(tmp, sizeof(tmp), "load model %s failed", model.c_str());
+                    throw err::Exception(e, tmp);
+                }
+            }
+        }
+
+        /**
+         * Destructor of YOLO26 class
+         * @maixcdk maix.nn.YOLO26.~YOLO26
+         */
+        ~YOLO26()
+        {
+            if (_model)
+            {
+                delete _model;
+                _model = nullptr;
+            }
+        }
+
+        /**
+         * Load model from file
+         * @param model Model path want to load
+         * @return err::Err
+         * @maixpy maix.nn.YOLO26.load
+         */
+        err::Err load(const string &model)
+        {
+            if (_model)
+            {
+                delete _model;
+                _model = nullptr;
+            }
+            _model = new nn::NN(model, _dual_buff);
+            if (!_model)
+            {
+                return err::ERR_NO_MEM;
+            }
+            _extra_info = _model->extra_info();
+            
+            // Check model type
+            if (_extra_info.find("model_type") != _extra_info.end())
+            {
+                if (_extra_info["model_type"] != "yolo26")
+                {
+                    log::error("model_type not match, expect 'yolo26', but got '%s'", _extra_info["model_type"].c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("model_type key not found");
+                return err::ERR_ARGS;
+            }
+            log::info("model info:\n\ttype: yolo26");
+            
+            // Parse input type
+            if (_extra_info.find("input_type") != _extra_info.end())
+            {
+                std::string input_type = _extra_info["input_type"];
+                if (input_type == "rgb")
+                {
+                    _input_img_fmt = maix::image::FMT_RGB888;
+                    log::print(log::LogLevel::LEVEL_INFO, "\tinput type: rgb\n");
+                }
+                else if (input_type == "bgr")
+                {
+                    _input_img_fmt = maix::image::FMT_BGR888;
+                    log::print(log::LogLevel::LEVEL_INFO, "\tinput type: bgr\n");
+                }
+                else
+                {
+                    log::error("unknown input type: %s", input_type.c_str());
+                    return err::ERR_ARGS;
+                }
+            }
+            else
+            {
+                log::error("input_type key not found");
+                return err::ERR_ARGS;
+            }
+            
+            // Parse mean
+            if (_extra_info.find("mean") != _extra_info.end())
+            {
+                std::string mean_str = _extra_info["mean"];
+                std::vector<std::string> mean_strs = split(mean_str, ",");
+                log::print(log::LogLevel::LEVEL_INFO, "\tmean:");
+                for (auto &it : mean_strs)
+                {
+                    try
+                    {
+                        this->mean.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("mean value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print(log::LogLevel::LEVEL_INFO, "%f ", this->mean.back());
+                }
+                log::print(log::LogLevel::LEVEL_INFO, "\n");
+            }
+            else
+            {
+                log::error("mean key not found");
+                return err::ERR_ARGS;
+            }
+            
+            // Parse scale
+            if (_extra_info.find("scale") != _extra_info.end())
+            {
+                std::string scale_str = _extra_info["scale"];
+                std::vector<std::string> scale_strs = split(scale_str, ",");
+                log::print(log::LogLevel::LEVEL_INFO, "\tscale:");
+                for (auto &it : scale_strs)
+                {
+                    try
+                    {
+                        this->scale.push_back(std::stof(it));
+                    }
+                    catch (std::exception &e)
+                    {
+                        log::error("scale value error, should float");
+                        return err::ERR_ARGS;
+                    }
+                    log::print(log::LogLevel::LEVEL_INFO, "%f ", this->scale.back());
+                }
+                log::print(log::LogLevel::LEVEL_INFO, "\n");
+            }
+            else
+            {
+                log::error("scale key not found");
+                return err::ERR_ARGS;
+            }
+            
+            // Parse labels
+            err::Err e = _model->extra_info_labels(labels);
+            if (e == err::Err::ERR_NONE)
+            {
+                log::print(log::LogLevel::LEVEL_INFO, "\tlabels num: %ld\n", labels.size());
+            }
+            else
+            {
+                log::error("labels key not found: %s", err::to_str(e).c_str());
+                return err::ERR_ARGS;
+            }
+            
+            // Get input size from model
+            std::vector<nn::LayerInfo> inputs = _model->inputs_info();
+            if (inputs[0].shape[3] <= 4) // nhwc
+                _input_size = image::Size(inputs[0].shape[2], inputs[0].shape[1]);
+            else
+                _input_size = image::Size(inputs[0].shape[3], inputs[0].shape[2]);
+            log::print(log::LogLevel::LEVEL_INFO, "\tinput size: %dx%d\n\n", _input_size.width(), _input_size.height());
+            
+            return err::ERR_NONE;
+        }
+
+        /**
+         * Detect objects from image
+         * @param img Image want to detect, if image's size not match model input's, will auto resize with fit method.
+         * @param conf_th Confidence threshold, default 0.5.
+         * @param iou_th IoU threshold, default 0.45.
+         * @param fit Resize method, default image.Fit.FIT_CONTAIN.
+         * @param sort Sort result according to object size, default 0 means not sort, 1 means bigger in front, -1 means smaller in front.
+         * @throw If image format not match model input format, will throw err::Exception.
+         * @return Object list. In C++, you should delete it after use.
+         * @maixpy maix.nn.YOLO26.detect
+         */
+        std::vector<nn::Object> *detect(image::Image &img, float conf_th = 0.5, float iou_th = 0.45, maix::image::Fit fit = maix::image::FIT_CONTAIN, int sort = 0)
+        {
+            this->_conf_th = conf_th;
+            this->_iou_th = iou_th;
+            
+            if (img.format() != _input_img_fmt)
+            {
+                throw err::Exception("image format not match, input_type: " + image::fmt_names[_input_img_fmt] + ", image format: " + image::fmt_names[img.format()]);
+            }
+            
+            tensor::Tensors *outputs;
+            outputs = _model->forward_image(img, this->mean, this->scale, fit, false);
+            if (!outputs) // not ready, return empty result
+            {
+                return new std::vector<nn::Object>();
+            }
+            
+            std::vector<nn::Object> *res = _post_process(outputs, img.width(), img.height(), fit, sort);
+            delete outputs;
+            
+            if (res == NULL)
+            {
+                throw err::Exception("post process failed, please see log before");
+            }
+            return res;
+        }
+
+        /**
+         * Get model input size
+         * @return model input size
+         * @maixpy maix.nn.YOLO26.input_size
+         */
+        image::Size input_size()
+        {
+            return _input_size;
+        }
+
+        /**
+         * Get model input width
+         * @return model input size of width
+         * @maixpy maix.nn.YOLO26.input_width
+         */
+        int input_width()
+        {
+            return _input_size.width();
+        }
+
+        /**
+         * Get model input height
+         * @return model input size of height
+         * @maixpy maix.nn.YOLO26.input_height
+         */
+        int input_height()
+        {
+            return _input_size.height();
+        }
+
+        /**
+         * Get input image format
+         * @return input image format, image::Format type.
+         * @maixpy maix.nn.YOLO26.input_format
+         */
+        image::Format input_format()
+        {
+            return _input_img_fmt;
+        }
+
+    public:
+        /**
+         * Labels list
+         * @maixpy maix.nn.YOLO26.labels
+         */
+        std::vector<string> labels;
+
+        /**
+         * Label file path
+         * @maixpy maix.nn.YOLO26.label_path
+         */
+        std::string label_path;
+
+        /**
+         * Get mean value, list type
+         * @maixpy maix.nn.YOLO26.mean
+         */
+        std::vector<float> mean;
+
+        /**
+         * Get scale value, list type
+         * @maixpy maix.nn.YOLO26.scale
+         */
+        std::vector<float> scale;
+
+    private:
+        image::Size _input_size;
+        image::Format _input_img_fmt;
+        nn::NN *_model;
+        std::map<string, string> _extra_info;
+        float _conf_th = 0.5;
+        float _iou_th = 0.45;
+        bool _dual_buff;
+        
+        // YOLO26 specific constants
+        static constexpr float LOGIT_THRESHOLD = -0.2f;
+
+    private:
+        /**
+         * Post process for YOLO26 output
+         */
+        std::vector<nn::Object> *_post_process(tensor::Tensors *outputs, int img_w, int img_h, maix::image::Fit fit, int sort)
+        {
+            std::vector<nn::Object> *objects = new std::vector<nn::Object>();
+            int num_class = labels.size();
+            
+            // Get output tensors - YOLO26 has 6 outputs: 3 bbox + 3 cls
+            float *bbox[3] = {
+                (float *)(*outputs)["output0"].data(),  // 1*80*80*4
+                (float *)(*outputs)["588"].data(),      // 1*40*40*4
+                (float *)(*outputs)["610"].data()       // 1*20*20*4
+            };
+            float *cls[3] = {
+                (float *)(*outputs)["580"].data(),      // 1*80*80*num_class
+                (float *)(*outputs)["602"].data(),      // 1*40*40*num_class
+                (float *)(*outputs)["624"].data()       // 1*20*20*num_class
+            };
+            
+            // Generate proposals for each scale
+            _generate_proposals(8, 80, 80, bbox[0], cls[0], num_class, *objects);
+            _generate_proposals(16, 40, 40, bbox[1], cls[1], num_class, *objects);
+            _generate_proposals(32, 20, 20, bbox[2], cls[2], num_class, *objects);
+            
+            // NMS
+            if (objects->size() > 0)
+            {
+                std::vector<nn::Object> *objects_total = objects;
+                objects = _nms(*objects);
+                delete objects_total;
+                
+                if (sort != 0)
+                {
+                    _sort_objects(*objects, sort);
+                }
+            }
+            
+            // Correct bbox to original image size
+            if (objects->size() > 0)
+            {
+                _correct_bbox(*objects, img_w, img_h, fit);
+            }
+            
+            return objects;
+        }
+
+        /**
+         * Generate proposals using NEON SIMD optimization
+         */
+        void _generate_proposals(int stride, int fw, int fh,
+                                  const float *__restrict__ bbox,
+                                  const float *__restrict__ cls,
+                                  int num_class,
+                                  std::vector<nn::Object> &objs)
+        {
+            const int total = fw * fh;
+            const float stride_f = (float)stride;
+            
+            for (int i = 0; i < total; i++)
+            {
+                const float *c = cls + i * num_class;
+                
+                // Prefetch next data
+                if (i + 4 < total)
+                {
+                    __builtin_prefetch(cls + (i + 4) * num_class, 0, 1);
+                }
+                
+                // Find max logit using NEON
+                float max_logit = _find_max_neon(c, num_class);
+                
+                // Early exit
+                if (max_logit < LOGIT_THRESHOLD)
+                    continue;
+                
+                // Find class id
+                int class_id = 0;
+                float max_val = c[0];
+                for (int j = 1; j < num_class; j++)
+                {
+                    if (c[j] > max_val)
+                    {
+                        max_val = c[j];
+                        class_id = j;
+                    }
+                }
+                
+                // Sigmoid and threshold
+                float score = _sigmoid(max_logit);
+                if (score <= _conf_th)
+                    continue;
+                
+                // Calculate bbox
+                int ax = i % fw;
+                int ay = i / fw;
+                const float *b = bbox + i * 4;
+                
+                float cx = (ax + 0.5f) * stride_f;
+                float cy = (ay + 0.5f) * stride_f;
+                
+                float x = cx - b[0] * stride_f;
+                float y = cy - b[1] * stride_f;
+                float w = (b[0] + b[2]) * stride_f;
+                float h = (b[1] + b[3]) * stride_f;
+                
+                // Clamp to input size
+                if (x < 0)
+                {
+                    w += x;
+                    x = 0;
+                }
+                if (y < 0)
+                {
+                    h += y;
+                    y = 0;
+                }
+                if (x + w > _input_size.width())
+                {
+                    w = _input_size.width() - x;
+                }
+                if (y + h > _input_size.height())
+                {
+                    h = _input_size.height() - y;
+                }
+                
+                if (w > 0 && h > 0)
+                {
+                    Object obj(x, y, w, h, class_id, score);
+                    objs.push_back(obj);
+                }
+            }
+        }
+
+        /**
+         * Find max value using NEON SIMD
+         */
+        inline float _find_max_neon(const float *data, int count)
+        {
+            if (count <= 0)
+                return -1e9f;
+            
+            int vec_count = count / 16 * 16;
+            float32x4_t vmax0 = vdupq_n_f32(-1e9f);
+            float32x4_t vmax1 = vdupq_n_f32(-1e9f);
+            float32x4_t vmax2 = vdupq_n_f32(-1e9f);
+            float32x4_t vmax3 = vdupq_n_f32(-1e9f);
+            
+            for (int j = 0; j < vec_count; j += 16)
+            {
+                vmax0 = vmaxq_f32(vmax0, vld1q_f32(data + j));
+                vmax1 = vmaxq_f32(vmax1, vld1q_f32(data + j + 4));
+                vmax2 = vmaxq_f32(vmax2, vld1q_f32(data + j + 8));
+                vmax3 = vmaxq_f32(vmax3, vld1q_f32(data + j + 12));
+            }
+            
+            // Merge
+            vmax0 = vmaxq_f32(vmax0, vmax1);
+            vmax2 = vmaxq_f32(vmax2, vmax3);
+            vmax0 = vmaxq_f32(vmax0, vmax2);
+            
+            // Horizontal max
+            float32x2_t vmax_pair = vpmax_f32(vget_low_f32(vmax0), vget_high_f32(vmax0));
+            vmax_pair = vpmax_f32(vmax_pair, vmax_pair);
+            float max_val = vget_lane_f32(vmax_pair, 0);
+            
+            // Handle remaining elements
+            for (int j = vec_count; j < count; j++)
+            {
+                if (data[j] > max_val)
+                    max_val = data[j];
+            }
+            
+            return max_val;
+        }
+
+        /**
+         * NMS (Non-Maximum Suppression)
+         */
+        std::vector<nn::Object> *_nms(std::vector<nn::Object> &objs)
+        {
+            std::vector<nn::Object> *result = new std::vector<nn::Object>();
+            
+            // Sort by score
+            std::sort(objs.begin(), objs.end(), [](const nn::Object &a, const nn::Object &b)
+                      { return a.score > b.score; });
+            
+            for (size_t i = 0; i < objs.size(); ++i)
+            {
+                nn::Object &a = objs.at(i);
+                if (a.score == 0)
+                    continue;
+                    
+                for (size_t j = i + 1; j < objs.size(); ++j)
+                {
+                    nn::Object &b = objs.at(j);
+                    if (b.score != 0 && a.class_id == b.class_id && _calc_iou(a, b) > this->_iou_th)
+                    {
+                        b.score = 0;
+                    }
+                }
+            }
+            
+            for (nn::Object &a : objs)
+            {
+                if (a.score != 0)
+                {
+                    result->push_back(a);
+                }
+            }
+            
+            return result;
+        }
+
+        /**
+         * Sort objects by size
+         */
+        void _sort_objects(std::vector<nn::Object> &objects, int sort)
+        {
+            if (sort > 0)
+                std::sort(objects.begin(), objects.end(), [](const nn::Object a, const nn::Object b)
+                          { return (a.w * a.h) > (b.w * b.h); });
+            else
+                std::sort(objects.begin(), objects.end(), [](const nn::Object a, const nn::Object b)
+                          { return (a.w * a.h) < (b.w * b.h); });
+        }
+
+        /**
+         * Correct bbox to original image size
+         */
+        void _correct_bbox(std::vector<nn::Object> &objs, int img_w, int img_h, maix::image::Fit fit)
+        {
+#define CORRECT_BBOX_RANGE(obj)        \
+    do                                 \
+    {                                  \
+        if (obj.x < 0)                 \
+        {                              \
+            obj.w += obj.x;            \
+            obj.x = 0;                 \
+        }                              \
+        if (obj.y < 0)                 \
+        {                              \
+            obj.h += obj.y;            \
+            obj.y = 0;                 \
+        }                              \
+        if (obj.x + obj.w > img_w)     \
+        {                              \
+            obj.w = img_w - obj.x;     \
+        }                              \
+        if (obj.y + obj.h > img_h)     \
+        {                              \
+            obj.h = img_h - obj.y;     \
+        }                              \
+    } while (0)
+
+            if (img_w == _input_size.width() && img_h == _input_size.height())
+                return;
+                
+            if (fit == maix::image::FIT_FILL)
+            {
+                float scale_x = (float)img_w / _input_size.width();
+                float scale_y = (float)img_h / _input_size.height();
+                for (nn::Object &obj : objs)
+                {
+                    obj.x *= scale_x;
+                    obj.y *= scale_y;
+                    obj.w *= scale_x;
+                    obj.h *= scale_y;
+                    CORRECT_BBOX_RANGE(obj);
+                }
+            }
+            else if (fit == maix::image::FIT_CONTAIN)
+            {
+                float scale_x = ((float)_input_size.width()) / img_w;
+                float scale_y = ((float)_input_size.height()) / img_h;
+                float scale = std::min(scale_x, scale_y);
+                float scale_reverse = 1.0 / scale;
+                float pad_w = (_input_size.width() - img_w * scale) / 2.0;
+                float pad_h = (_input_size.height() - img_h * scale) / 2.0;
+                for (nn::Object &obj : objs)
+                {
+                    obj.x = (obj.x - pad_w) * scale_reverse;
+                    obj.y = (obj.y - pad_h) * scale_reverse;
+                    obj.w *= scale_reverse;
+                    obj.h *= scale_reverse;
+                    CORRECT_BBOX_RANGE(obj);
+                }
+            }
+            else if (fit == maix::image::FIT_COVER)
+            {
+                float scale_x = ((float)_input_size.width()) / img_w;
+                float scale_y = ((float)_input_size.height()) / img_h;
+                float scale = std::max(scale_x, scale_y);
+                float scale_reverse = 1.0 / scale;
+                float pad_w = (img_w * scale - _input_size.width()) / 2.0;
+                float pad_h = (img_h * scale - _input_size.height()) / 2.0;
+                for (nn::Object &obj : objs)
+                {
+                    obj.x = (obj.x + pad_w) * scale_reverse;
+                    obj.y = (obj.y + pad_h) * scale_reverse;
+                    obj.w *= scale_reverse;
+                    obj.h *= scale_reverse;
+                    CORRECT_BBOX_RANGE(obj);
+                }
+            }
+            else
+            {
+                throw err::Exception(err::ERR_ARGS, "fit type not support");
+            }
+#undef CORRECT_BBOX_RANGE
+        }
+
+        /**
+         * Sigmoid function
+         */
+        inline static float _sigmoid(float x)
+        {
+            return 1.0f / (1.0f + expf(-x));
+        }
+
+        /**
+         * Calculate IoU (Intersection over Union)
+         */
+        inline static float _calc_iou(Object &a, Object &b)
+        {
+            float area1 = a.w * a.h;
+            float area2 = b.w * b.h;
+            float wi = std::min((a.x + a.w), (b.x + b.w)) - std::max(a.x, b.x);
+            float hi = std::min((a.y + a.h), (b.y + b.h)) - std::max(a.y, b.y);
+            float area_i = std::max(wi, 0.0f) * std::max(hi, 0.0f);
+            return area_i / (area1 + area2 - area_i);
+        }
+
+        /**
+         * Split string by delimiter
+         */
+        static std::vector<std::string> split(const std::string &s, const std::string &delimiter)
+        {
+            std::vector<std::string> tokens;
+            size_t pos_start = 0, pos_end, delim_len = delimiter.length();
+            std::string token;
+
+            while ((pos_end = s.find(delimiter, pos_start)) != std::string::npos)
+            {
+                token = s.substr(pos_start, pos_end - pos_start);
+                pos_start = pos_end + delim_len;
+                token.erase(0, token.find_first_not_of(" \t\r\n"));
+                token.erase(token.find_last_not_of(" \t\r\n") + 1);
+                tokens.push_back(token);
+            }
+            token = s.substr(pos_start);
+            token.erase(0, token.find_first_not_of(" \t\r\n"));
+            token.erase(token.find_last_not_of(" \t\r\n") + 1);
+            tokens.push_back(token);
+            
+            return tokens;
+        }
+    };
+
+} // namespace maix::nn

components/nn/port/maixcam2/maix_nn_speech_maixcam2.cpp

@@ -9,73 +9,259 @@
 
 namespace maix::nn
 {
+    static bool _is_skip_frames {false};
 
+    static void _signal_handle(int signal)
+    {
+        switch (signal) {
+        case SIGINT:
+            maix::app::set_exit_flag(true);
+        break;
+        default: break;
+        }
+    }
+
+    static void digit_callback_wrapper(void* data, int cnt) {
+        if (digit_callback && _is_skip_frames == false) {
+            digit_callback(static_cast<char*>(data), cnt);
+        }
+    }
+
+    static void kws_callback_wrapper(void* data, int cnt) {
+        if (kws_callback && _is_skip_frames == false) {
+            std::vector<float> kws_data;
+            float* p = (float*) data;
+            for(int i=0; i<cnt; i++){
+                kws_data.push_back(p[i]);
+            }
+            kws_callback(kws_data, cnt);
+        }
+    }
+
+    static void raw_callback_wrapper(void* data, int cnt) {
+        if (raw_callback && _is_skip_frames == false) {
+            std::vector<std::pair<int, float>> raw_data;
+            pnyp_t* res = (pnyp_t*)data;
+            for(int t=0; t<cnt; t++) {
+                pnyp_t* pp = res+BEAM_CNT*t;
+                raw_data.push_back({pp->idx, pp->p});
+            }
+            raw_callback(raw_data, cnt);
+        }
+    }
+
+    static void lvcsr_callback_wrapper(void* data, int cnt) {
+        if (lvcsr_callback && _is_skip_frames == false) {
+            char* words = ((char**)data)[0];
+            char* pnys = ((char**)data)[1];
+            lvcsr_callback({words, pnys}, cnt);
+        }
+    }
 
     Speech::Speech(const string &model)
     {
-        (void)model;
-		err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        if (!model.empty())
+        {
+            err::Err e = load(model);
+            if (e != err::ERR_NONE)
+            {
+                throw err::Exception(e, "load model failed");
+            }
+        }
+        signal(SIGINT, _signal_handle);
 	}
 
     Speech::~Speech()
     {
-        err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        if (_dev_type != SpeechDevice::DEVICE_NONE) {
+            this->deinit();
+        }
     }
 
     err::Err Speech::load(const string &model)
     {
-        (void)model;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        _model_path = model;
+        return err::ERR_NONE;
     }
 
     err::Err Speech::init(nn::SpeechDevice dev_type, const string &device_name)
     {
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        string _device_name = device_name;
+
+        if (_model_path == "") {
+            log::error("please load am model first.");
+            throw err::Exception(err::ERR_NOT_IMPL);
+        }
+
+        am_args_t am_args = {(char*)_model_path.c_str(), 192, 6, 6, CN_PNYTONE, 1};
+
+        if (this->dev_type() != SpeechDevice::DEVICE_NONE) {
+            log::error("device has been initialized, please use maix.nn.Speech.devive to reset devive.");
+            return err::ERR_RUNTIME;
+        }
+
+        if((int)dev_type > 2) {
+            log::error("not support device %d.", dev_type);
+            throw err::Exception(err::ERR_NOT_IMPL);
+        } else if (dev_type == SpeechDevice::DEVICE_MIC && device_name == "") {
+            _dev_type = dev_type;
+            _device_name = "hw:0,0";
+        } else if (dev_type == SpeechDevice::DEVICE_PCM && device_name == "") {
+            log::error("please enter the correct path to the PCM file.");
+            return err::ERR_ARGS;
+        } else if (dev_type == SpeechDevice::DEVICE_WAV && device_name == "") {
+            log::error("please enter the correct path to the WAV file.");
+            return err::ERR_ARGS;
+        } else {
+            _dev_type = dev_type;
+            _device_name = device_name;
+        }
+
+        int ret = ms_asr_init((int)_dev_type, (char*)_device_name.c_str(), &am_args, 0);
+        if(ret) {
+            log::error("asr init error!");
+            _dev_type = SpeechDevice::DEVICE_NONE;
+            return err::ERR_NOT_IMPL;
+        }
+
+        return err::ERR_NONE;
     }
 
     err::Err Speech::devive(nn::SpeechDevice dev_type, const string &device_name)
     {
-        (void)dev_type;
-        (void)device_name;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if((int)dev_type > 2) {
+            log::error("not support device %d.", dev_type);
+            throw err::Exception(err::ERR_NOT_IMPL);
+        } else {
+            _dev_type = dev_type;
+        }
+
+        int ret = ms_asr_set_dev((int)_dev_type, (char*)device_name.c_str());
+        if(ret) {
+            log::error("set devive error!\n");
+            _dev_type = SpeechDevice::DEVICE_NONE;
+            return err::ERR_NOT_IMPL;
+        }
+
+        return err::ERR_NONE;
     }
 
     void Speech::dec_deinit(nn::SpeechDecoder decoder)
     {
-        (void)decoder;
-        err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        ms_asr_decoder_cfg((int)decoder, NULL , NULL, 0);
+        switch (decoder)
+        {
+            case nn::SpeechDecoder::DECODER_RAW:
+                _decoder_raw = false;
+                break;
+            case nn::SpeechDecoder::DECODER_DIG:
+                _decoder_dig = false;
+                break;
+            case nn::SpeechDecoder::DECODER_LVCSR:
+                _decoder_lvcsr = false;
+                break;
+            case nn::SpeechDecoder::DECODER_KWS:
+                _decoder_kws = false;
+                break;
+            case nn::SpeechDecoder::DECODER_ALL:
+                _decoder_raw = false;
+                _decoder_dig = false;
+                _decoder_lvcsr = false;
+                _decoder_kws = false;
+                break;
+            default:
+                log::error("not support decoder %d.", decoder);
+                throw err::Exception(err::ERR_NOT_IMPL);
+        }
     }
 
     err::Err Speech::raw(std::function<void(std::vector<std::pair<int, float>>, int)> callback)
     {
-        (void)callback;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if (this->dev_type() == SpeechDevice::DEVICE_NONE) {
+            log::error("please init a type of audio device first.");
+            return err::ERR_NOT_INIT;
+        }
+
+        _raw_callback = callback;
+        int ret = ms_asr_decoder_cfg((int)nn::SpeechDecoder::DECODER_RAW, raw_callback_wrapper , NULL, 0);
+        if (ret != 0) {
+            log::error("raw decoder init error.");
+            return err::ERR_RUNTIME;
+        } else {
+            _decoder_raw = true;
+            return err::ERR_NONE;
+        }
     }
 
     bool Speech::raw() { return _decoder_raw; }
 
     err::Err Speech::digit(int blank, std::function<void(char*, int)> callback) 
     {
-        (void)blank;
-        (void)callback;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if (this->dev_type() == SpeechDevice::DEVICE_NONE) {
+            log::error("please init a type of audio device first.");
+            return err::ERR_NOT_INIT;
+        }
+
+        size_t decoder_args[10];
+        decoder_args[0] = blank;
+
+        _digit_callback = callback;
+
+        int ret = ms_asr_decoder_cfg((int)nn::SpeechDecoder::DECODER_DIG, digit_callback_wrapper, &decoder_args, 1);
+        if (ret != 0) {
+            log::error("digit decoder init error.");
+            return err::ERR_RUNTIME;
+        } else {
+            _decoder_dig = true;
+            return err::ERR_NONE;
+        }
     }
 
     bool Speech::digit() { return _decoder_dig; }
 
     err::Err Speech::kws(std::vector<string> kw_tbl, std::vector<float> kw_gate, std::function<void(std::vector<float>, int)> callback, bool auto_similar) 
     {
-        (void)kw_tbl;
-        (void)kw_gate;
-        (void)callback;
-        (void)auto_similar;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if (this->dev_type() == SpeechDevice::DEVICE_NONE) {
+            log::error("please init a type of audio device first.");
+            return err::ERR_NOT_INIT;
+        }
+
+        if (kw_tbl.size() != kw_gate.size()) {
+            log::error("kw_tbl num must equal to kw_gate num.");
+            return err::ERR_ARGS;
+        }
+
+        size_t decoder_args[10];
+        char** _kw_tbl = new char*[kw_tbl.size()];
+        for (size_t i = 0; i < kw_tbl.size(); ++i) {
+            _kw_tbl[i] = new char[kw_tbl[i].size() + 1];
+            std::strcpy(_kw_tbl[i], kw_tbl[i].c_str());
+        }
+        float* _kw_gate = new float[kw_gate.size()];
+        for (size_t i = 0; i < kw_gate.size(); ++i) {
+            _kw_gate[i] = kw_gate[i];
+        }
+
+        decoder_args[0] = (size_t)_kw_tbl;
+        decoder_args[1] = (size_t)_kw_gate;
+        decoder_args[2] = kw_tbl.size();
+        decoder_args[3] = auto_similar;
+        _kws_callback = callback;
+        int ret = ms_asr_decoder_cfg((int)nn::SpeechDecoder::DECODER_KWS, kws_callback_wrapper, &decoder_args, 3);
+
+        delete[] _kw_gate;
+        for (size_t i = 0; i < kw_tbl.size(); ++i) {
+            delete[] _kw_tbl[i];
+        }
+        delete[] _kw_tbl;
+
+        if (ret != 0) {
+            log::error("kws decoder init error.");
+            return err::ERR_RUNTIME;
+        } else {
+            _decoder_kws = true;
+            return err::ERR_NONE;
+        }
     }
 
     bool Speech::kws() { return _decoder_kws; }
@@ -85,52 +271,113 @@ namespace maix::nn
                     std::function<void(std::pair<char*, char*>, int)> callback,
                     float beam, float bg_prob, float scale, bool mmap) 
     {
-        (void)sfst_name;
-        (void)sym_name;
-        (void)phones_txt;
-        (void)words_txt;
-        (void)callback;
-        (void)beam;
-        (void)bg_prob;
-        (void)scale;
-        (void)mmap;
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if (this->dev_type() == SpeechDevice::DEVICE_NONE) {
+            log::error("please init a type of audio device first.");
+            return err::ERR_NOT_INIT;
+        }
+
+        size_t decoder_args[10];
+        decoder_args[0] = (size_t)sfst_name.c_str();
+        decoder_args[1] = (size_t)sym_name.c_str();
+        decoder_args[2] = (size_t)phones_txt.c_str();
+        decoder_args[3] = (size_t)words_txt.c_str();
+        memcpy(&decoder_args[4], &(beam), sizeof(float));
+        memcpy(&decoder_args[5], &(bg_prob), sizeof(float));
+        memcpy(&decoder_args[6], &(scale), sizeof(float));
+        decoder_args[7] = mmap;
+        _lvcsr_callback = callback;
+
+        int ret = ms_asr_decoder_cfg((int)nn::SpeechDecoder::DECODER_LVCSR, lvcsr_callback_wrapper, &decoder_args, 8);
+        if (ret != 0) {
+            log::error("lvcsr decoder init error.");
+            return err::ERR_RUNTIME;
+        } else {
+            _decoder_lvcsr = true;
+            return err::ERR_NONE;
+        }
     }
 
     bool Speech::lvcsr() { return _decoder_lvcsr; }
 
     int Speech::run(int frame)
     {
-        (void)frame;
-        log::error("This function is not implemented.");
-        return 0;
+        if (!(raw() || digit() || kws() || lvcsr())) {
+            log::error("please init at least one decoder before running.");
+            return 0;
+        }
+
+        raw_callback = this->_raw_callback;
+        digit_callback = this->_digit_callback;
+        kws_callback = this->_kws_callback;
+        lvcsr_callback = this->_lvcsr_callback;
+
+        int frames = ms_asr_run(frame);
+
+        // Set it to nullptr, otherwise MaixPy cannot exit properly.
+        raw_callback = nullptr;
+        digit_callback = nullptr;
+        kws_callback = nullptr;
+        lvcsr_callback = nullptr;
+
+        return frames;
     }
 
     void Speech::clear()
     {
-        err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        ms_asr_clear();
     }
 
     int Speech::frame_time()
     {
-        log::error("This function is not implemented.");
-        return 0;
+        return ms_asr_get_frame_time();
     }
 
     err::Err Speech::similar(const string &pny, std::vector<std::string> similar_pnys)
     {
-        log::error("This function is not implemented.");
-        return err::ERR_NOT_IMPL;
+        if (this->kws() != true) {
+            log::error("please init kws decoder first.");
+            return err::ERR_RUNTIME;
+        }
+
+        char** _similar_pnys = new char*[similar_pnys.size()];
+        for (size_t i = 0; i < similar_pnys.size(); ++i) {
+            _similar_pnys[i] = new char[similar_pnys[i].size() + 1];
+            std::strcpy(_similar_pnys[i], similar_pnys[i].c_str());
+        }
+
+        int ret = ms_asr_kws_reg_similar((char*)pny.c_str(), _similar_pnys, similar_pnys.size());
+
+        for (size_t i = 0; i < similar_pnys.size(); ++i) {
+            delete[] _similar_pnys[i];
+        }
+        delete[] _similar_pnys;
+
+        if (ret != 0) {
+            log::error("set similar pny error.");
+            return err::ERR_RUNTIME;
+        } else {
+            return err::ERR_NONE;
+        }
     }
 
     void Speech::skip_frames(int num) {
-        (void)num;
-        err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        _is_skip_frames = true;
+        this->run(num);
+        _is_skip_frames = false;
     }
 
     void Speech::deinit()
     {
-        err::check_raise(err::ERR_NOT_IMPL, "This function is not implemented.");
+        _dev_type = SpeechDevice::DEVICE_NONE;
+        _decoder_raw = false;
+        _decoder_dig = false;
+        _decoder_lvcsr = false;
+        _decoder_kws = false;
+        _raw_callback = nullptr;
+        _digit_callback = nullptr;
+        _kws_callback = nullptr;
+        _lvcsr_callback = nullptr;
+        ms_asr_deinit();
+        sys::register_default_signal_handle();
     }
-}
+}

examples/nn_yolo26_detect/.gitignore

@@ -0,0 +1,7 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+/CMakeLists.txt
+__pycache__

examples/nn_yolo26_detect/README.md

@@ -0,0 +1,9 @@
+ Project based on MaixCDK
+====
+
+
+
+
+
+This is a project based on MaixCDK, build method please visit [MaixCDK](https://github.com/sipeed/MaixCDK)
+

examples/nn_yolo26_detect/app.yaml

@@ -0,0 +1,10 @@
+id: yolo26
+name: 
+name[zh]:
+version: 1.0.0
+#icon: assets/hello.png
+author: 
+desc: 
+desc[zh]:
+files:
+  # assets: assets

examples/nn_yolo26_detect/main/CMakeLists.txt

@@ -0,0 +1,73 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic vision ext_dev nn)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+#     'url': 'https://*****/abcde.tar.xz',
+#     'urls': [],  # backup urls, if url failed, will try urls
+#     'sites': [], # download site, user can manually download file and put it into dl_path
+#     'sha256sum': '',
+#     'filename': 'abcde.tar.xz',
+#     'path': 'toolchains/xxxxx',
+#     }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

examples/nn_yolo26_detect/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

examples/nn_yolo26_detect/main/src/main.cpp

@@ -0,0 +1,88 @@
+/**
+ * YOLO26 Detection Example
+ */
+
+#include "maix_nn_yolo26.hpp"
+#include "maix_camera.hpp"
+#include "maix_display.hpp"
+#include "maix_app.hpp"
+
+using namespace maix;
+
+int main(int argc, char *argv[])
+{
+    std::string model_path = "/root/models/yolo26n.mud";
+    if (argc > 1)
+        model_path = argv[1];
+
+    // Initialize detector
+    // @maixpy detector = nn.YOLO26(model="/root/models/yolo26n.mud", dual_buff=True)
+    nn::YOLO26 detector(model_path, true);
+
+    // Initialize camera with model input size
+    // @maixpy cam = camera.Camera(detector.input_width(), detector.input_height(), detector.input_format())
+    camera::Camera cam(detector.input_width(), detector.input_height(), detector.input_format());
+    
+    // Initialize display
+    // @maixpy disp = display.Display()
+    display::Display disp;
+
+    // FPS statistics
+    int detect_count = 0;
+    uint64_t total_detect_time = 0;
+
+    // @maixpy while not app.need_exit():
+    while (!app::need_exit())
+    {
+        // Read image
+        // @maixpy img = cam.read()
+        image::Image *img = cam.read();
+        if (!img)
+            continue;
+
+        // Detect objects
+        // @maixpy objs = detector.detect(img, conf_th=0.5, iou_th=0.45)
+        uint64_t t0 = time::ticks_ms();
+        std::vector<nn::Object> *objs = detector.detect(*img, 0.5, 0.45);
+        uint64_t t1 = time::ticks_ms();
+
+        // Calculate FPS
+        uint64_t detect_time = t1 - t0;
+        total_detect_time += detect_time;
+        detect_count++;
+        float detect_fps = (total_detect_time > 0) ? (float)detect_count * 1000.0f / total_detect_time : 0.0f;
+
+        // Draw results
+        // @maixpy for obj in objs:
+        for (const auto &obj : *objs)
+        {
+            // @maixpy img.draw_rect(obj.x, obj.y, obj.w, obj.h, color=image.COLOR_RED)
+            img->draw_rect(obj.x, obj.y, obj.w, obj.h, image::COLOR_RED, 2);
+
+            // @maixpy msg = f'{detector.labels[obj.class_id]}: {obj.score:.2f}'
+            // @maixpy img.draw_string(obj.x, obj.y, msg, color=image.COLOR_RED)
+            char msg[64];
+            snprintf(msg, sizeof(msg), "%s: %.2f", detector.labels[obj.class_id].c_str(), obj.score);
+            img->draw_string(obj.x, std::max(0, obj.y - 15), msg, image::COLOR_RED);
+        }
+
+        // Draw FPS
+        char fps_str[64];
+        snprintf(fps_str, sizeof(fps_str), "Detect: %.1f FPS", detect_fps);
+        img->draw_string(10, 10, fps_str, image::COLOR_GREEN, 1.5);
+
+        // Show image
+        // @maixpy disp.show(img)
+        disp.show(*img);
+
+        // Log
+        log::info("检测耗时: %lu ms | 检测FPS: %.1f | 检测数: %zu",
+                  detect_time, detect_fps, objs->size());
+
+        // Clean up
+        delete objs;
+        delete img;
+    }
+
+    return 0;
+}

projects/app_speech/main/CMakeLists.txt

@@ -26,6 +26,10 @@ append_srcs_dir(ADD_SRCS "src" "asr_lib/src" "utils/src")       # append source
 
 ###### Add required/dependent components ######
 list(APPEND ADD_REQUIREMENTS basic comm nn vision)
+
+if(PLATFORM_MAIXCAM2)
+    list(APPEND ADD_REQUIREMENTS alsa_lib voice)
+endif()
 ###############################################
 
 ###### Add link search path for requirements/libs ######