add face emotion recognition support

components/nn/include/maix_nn_face_landmarks.hpp

@@ -272,14 +272,19 @@ namespace maix::nn
          * @param x,y,w,h face rectangle, x,y is left-top point.
          * @param img source image
          * @param points  2 points, eye_left_x, eye_left_y, eye_right_x, eye_right_y
+         * @param scale crop size scale relative to rectangle's max side length(w or h), final value is `scale *max(w, h)`,default 1.2.
          * @maixpy maix.nn.FaceLandmarks.crop_image
          */
-        maix::image::Image *crop_image(maix::image::Image &img, int x, int y, int w, int h, std::vector<int> points)
+        maix::image::Image *crop_image(maix::image::Image &img, int x, int y, int w, int h, std::vector<int> points, int new_width = -1, int new_height = -1, float scale = 1.2)
         {
+            if(new_width == -1)
+                new_width = _input_size.width();
+            if(new_height == -1)
+                new_height = _input_size.height();
             int cx = x + w * 0.5;
             int cy = y + h * 0.5;
             int new_size = w > h ? w : h;
-            new_size += new_size * 0.2;
+            new_size *= scale;
             float theta = atan2(points[3] - points[1], points[2] - points[0]);
             cv::Mat A = (cv::Mat_<float>(4, 2) << -1, -1, -1, 1, 1, 1, 1, -1);
             cv::Mat R = (cv::Mat_<float>(2, 2) << cos(theta), sin(theta), -sin(theta), cos(theta));
@@ -291,18 +296,18 @@ namespace maix::nn
                 C.at<float>(i, 1) += cy;
             }
             cv::Mat element = C(cv::Range(0, 3), cv::Range(0, 2));
-            cv::Mat dst = (cv::Mat_<float>(3, 2) << 0, 0, 0, _input_size.height(), _input_size.width(), _input_size.height());
+            cv::Mat dst = (cv::Mat_<float>(3, 2) << 0, 0, 0, new_height, new_width, new_height);
             auto M = cv::getAffineTransform(element, dst);
-            image::Image *img_dst = new image::Image(_input_size.width(), _input_size.height(), image::FMT_RGB888);
+            image::Image *img_dst = new image::Image(new_width, new_height, image::FMT_RGB888);
             if(!img_dst)
             {
                 log::error("no memory");
                 return nullptr;
             }
-            cv::Mat img_dst2(_input_size.height(), _input_size.width(), CV_8UC3, img_dst->data());
+            cv::Mat img_dst2(new_height, new_width, CV_8UC3, img_dst->data());
             cv::Mat img_src;
             image::image2cv(img, img_src, false, false);
-            cv::warpAffine(img_src, img_dst2, M, cv::Size(_input_size.width(), _input_size.height()), cv::INTER_NEAREST);
+            cv::warpAffine(img_src, img_dst2, M, cv::Size(new_width, new_height), cv::INTER_NEAREST);
             assert(img_dst2.data() == img_dst->data());
 
             cv::invertAffineTransform(M, _M_inverse);

examples/nn_face_emotion/.gitignore

@@ -0,0 +1,9 @@
+build
+dist
+.config.mk
+.flash.conf.json
+data
+
+/CMakeLists.txt
+
+__pycache__

examples/nn_face_emotion/README.md

@@ -0,0 +1,9 @@
+Face emotion recognition demo
+====
+## only run yolov8_pose model
+
+- Use `maixcdk build` to compile binary files.
+- Move files and runtime libraries to device.
+- Use SSH command with parameters to run programs.
+
+` "Usage: "./nn_face_landmarks yolov8_face_model_path face_emotion_model_path <image_path>"; `

examples/nn_face_emotion/main/CMakeLists.txt

@@ -0,0 +1,74 @@
+############### Add include ###################
+list(APPEND ADD_INCLUDE "include"
+    )
+list(APPEND ADD_PRIVATE_INCLUDE "")
+###############################################
+
+############ Add source files #################
+# list(APPEND ADD_SRCS  "src/main.c"
+#                       "src/test.c"
+#     )
+append_srcs_dir(ADD_SRCS "src")       # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
+# FILE(GLOB_RECURSE EXTRA_SRC  "src/*.c")
+# FILE(GLOB EXTRA_SRC  "src/*.c")
+# list(APPEND ADD_SRCS  ${EXTRA_SRC})
+# aux_source_directory(src ADD_SRCS)  # collect all source file in src dir, will set var ADD_SRCS
+# append_srcs_dir(ADD_SRCS "src")     # append source file in src dir to var ADD_SRCS
+# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
+# set(ADD_ASM_SRCS "src/asm.S")
+# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
+# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S  ASM file as C language
+# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
+###############################################
+
+###### Add required/dependent components ######
+list(APPEND ADD_REQUIREMENTS basic nn vision)
+###############################################
+
+###### Add link search path for requirements/libs ######
+# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
+# list(APPEND ADD_REQUIREMENTS pthread m)  # add system libs, pthread and math lib for example here
+# set (OpenCV_DIR opencv/lib/cmake/opencv4)
+# find_package(OpenCV REQUIRED)
+###############################################
+
+############ Add static libs ##################
+# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
+###############################################
+
+#### Add compile option for this component ####
+#### Just for this component, won't affect other 
+#### modules, including component that depend 
+#### on this component
+# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
+
+#### Add compile option for this component
+#### and components depend on this component
+# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
+#                             -DAAAAA333=1)
+###############################################
+
+############ Add static libs ##################
+#### Update parent's variables like CMAKE_C_LINK_FLAGS
+# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
+###############################################
+
+######### Add files need to download #########
+# list(APPEND ADD_FILE_DOWNLOADS "{
+# 'url': 'https://*****/abcde.tar.xz',
+# 'urls': [],  # backup urls, if url failed, will try urls
+# 'sites': [], # download site, user can manually download file and put it into dl_path
+# 'sha256sum': '',
+# 'filename': 'abcde.tar.xz',
+# 'path': 'toolchains/xxxxx',
+# 'check_files': []
+# }"
+# )
+#
+# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
+# you can directly use then, for example use it in add_custom_command
+##############################################
+
+# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
+register_component()

examples/nn_face_emotion/main/include/main.h

@@ -0,0 +1,3 @@
+#pragma once
+
+

examples/nn_face_emotion/main/src/main.cpp

@@ -0,0 +1,224 @@
+
+#include "maix_basic.hpp"
+#include "maix_vision.hpp"
+#include "maix_nn_yolov8.hpp"
+#include "maix_nn_face_landmarks.hpp"
+#include "maix_nn_classifier.hpp"
+#include "main.h"
+
+using namespace maix;
+
+int mode_fileter(int new_idx, size_t labels_num)
+{
+    static std::vector<int> data(10, -1);
+    std::vector<int> count(labels_num, 0);
+    static int p = 0;
+    data[p++] = new_idx;
+    for(size_t i=0; i<data.size(); ++i)
+    {
+        if(data[i] >= 0)
+            ++count[data[i]];
+    }
+    int max_count = count[0];
+    int max_idx = 0;
+    for(size_t i=1; i<labels_num; ++i)
+    {
+        if (count[i] > max_count)
+        {
+            max_count = count[i];
+            max_idx = i;
+        }
+    }
+    return max_idx;
+}
+
+
+int _main(int argc, char *argv[])
+{
+    log::info("Program start");
+    int ret = 0;
+    maix::image::Format img_fmt = maix::image::FMT_RGB888;
+    char tmp_chars[64] = {0};
+
+    std::string help = "Usage: " + std::string(argv[0]) + " detect_model_path emotion_model_path <image_path>";
+
+    if (argc < 2)
+    {
+        log::info(help.c_str());
+        return -1;
+    }
+
+    const char *detect_model_path = argv[1];
+    const char *emotion_model_path = argv[2];
+    float conf_threshold = 0.5;
+    float iou_threshold = 0.45;
+    float crop_scale = 1.2;
+    float emotion_threshold = 0.5;
+
+    nn::YOLOv8 detector(detect_model_path, false);
+    log::info("load yolov8 model %s success", detect_model_path);
+    nn::FaceLandmarks landmarks_detector;
+    nn::Classifier classifier(emotion_model_path, false);
+    log::info("load emotion model %s success", emotion_model_path);
+
+    // for draw result info
+    int max_labels_length = 0;
+    for (size_t j = 0; j < classifier.labels.size(); ++j)
+    {
+        image::Size size = image::string_size(classifier.labels[j]);
+        if (size.width() > max_labels_length)
+            max_labels_length = size.width();
+    }
+
+    if (argc >= 4)
+    {
+        const char *img_path = argv[3];
+        log::info("load image now");
+        maix::image::Image *img = maix::image::load(img_path, img_fmt);
+        err::check_null_raise(img, "load image " + std::string(img_path) + " failed");
+        log::info("load image %s success: %s", img_path, img->to_str().c_str());
+        if (img->width() != detector.input_size().width() || img->height() != detector.input_size().height())
+        {
+            log::warn("image size not match model input size, will auto resize from %dx%d to %dx%d", img->width(), img->height(), detector.input_size().width(), detector.input_size().height());
+        }
+        int max_score_length = img->width() / 4;
+        log::info("detect now");
+        nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold);
+        std::vector<std::vector<std::pair<int, float>> *> results;
+        std::vector<int> idxes;
+        maix::image::Image *img_std_first = nullptr;
+        int i = 0;
+        for (auto &r : *result)
+        {
+            maix::image::Image *img_std = landmarks_detector.crop_image(*img, r->x, r->y, r->w, r->h, r->points, classifier.input_width(), classifier.input_height(), crop_scale);
+            if (img_std)
+            {
+                maix::image::Image *img_std_gray = img_std->to_format(image::Format::FMT_GRAYSCALE);
+                if (i == 0)
+                    img_std_first = img_std;
+                else
+                    delete img_std;
+                std::vector<std::pair<int, float>> *res = classifier.classify(*img_std_gray);
+                results.push_back(res);
+                idxes.push_back(i);
+                delete img_std_gray;
+            }
+            ++i;
+        }
+        i = 0;
+        for (auto r : results)
+        {
+            if (i == 0)
+            {
+                img->draw_image(0, 0, *img_std_first);
+                for (size_t j = 0; j < classifier.labels.size(); ++j)
+                {
+                    int idx = r->at(j).first;
+                    float score = r->at(j).second;
+                    img->draw_string(0, img_std_first->height() + idx * 16, classifier.labels[idx], image::COLOR_WHITE);
+                    img->draw_rect(max_labels_length, img_std_first->height() + idx * 16, score * max_score_length, 8, score >= emotion_threshold ? image::COLOR_GREEN : image::COLOR_RED, -1);
+                    snprintf(tmp_chars, sizeof(tmp_chars), "%.1f", score);
+                    img->draw_string(max_labels_length + score * max_score_length + 2, img_std_first->height() + idx * 16, tmp_chars, image::COLOR_RED);
+                }
+                delete img_std_first;
+            }
+            auto color = r->at(0).second >= emotion_threshold ? image::COLOR_GREEN : image::COLOR_RED;
+            nn::Object &obj = result->at(idxes[i]);
+            img->draw_rect(obj.x, obj.y, obj.w, obj.h, color, 1);
+            snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.1f", classifier.labels[r->at(0).first].c_str(), r->at(0).second);
+            img->draw_string(obj.x, obj.y, tmp_chars, color);
+            ++i;
+            delete r;
+        }
+        img->save("result.jpg");
+        log::info("saved result image to result.jpg");
+        delete img;
+        delete result;
+    }
+    else
+    {
+        log::info("open camera now");
+        maix::image::Size input_size = detector.input_size();
+        camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
+        log::info("open camera success");
+        display::Display disp = display::Display();
+        int max_score_length = cam.width() / 4;
+        while (!app::need_exit())
+        {
+            uint64_t t = time::ticks_ms();
+            maix::image::Image *img = cam.read();
+            err::check_null_raise(img, "read camera failed");
+            uint64_t t2 = time::ticks_ms();
+            nn::Objects *result = detector.detect(*img, conf_threshold, iou_threshold, image::FIT_CONTAIN, 0.5, 1);
+            std::vector<std::vector<std::pair<int, float>> *> results;
+            std::vector<int> idxes;
+            maix::image::Image *img_std_first = nullptr;
+            int i = 0;
+            for (auto &r : *result)
+            {
+                maix::image::Image *img_std = landmarks_detector.crop_image(*img, r->x, r->y, r->w, r->h, r->points, classifier.input_width(), classifier.input_height(), crop_scale);
+                if (img_std)
+                {
+                    maix::image::Image *img_std_gray = img_std->to_format(image::Format::FMT_GRAYSCALE);
+                    if (i == 0)
+                        img_std_first = img_std;
+                    else
+                        delete img_std;
+                    std::vector<std::pair<int, float>> *res = classifier.classify(*img_std_gray);
+                    results.push_back(res);
+                    idxes.push_back(i);
+                    delete img_std_gray;
+                }
+                ++i;
+            }
+            uint64_t t3 = time::ticks_ms();
+            i = 0;
+            for (auto r : results)
+            {
+                // draw fisrt face detailed info
+                if (i == 0)
+                {
+                    img->draw_image(0, 0, *img_std_first);
+                    for (size_t j = 0; j < classifier.labels.size(); ++j)
+                    {
+                        int idx = r->at(j).first;
+                        float score = r->at(j).second;
+                        img->draw_string(0, img_std_first->height() + idx * 16, classifier.labels[idx], image::COLOR_WHITE);
+                        img->draw_rect(max_labels_length, img_std_first->height() + idx * 16, score * max_score_length, 8, score >= emotion_threshold ? image::COLOR_GREEN : image::COLOR_RED, -1);
+                        snprintf(tmp_chars, sizeof(tmp_chars), "%.1f", score);
+                        img->draw_string(max_labels_length + score * max_score_length + 2, img_std_first->height() + idx * 16, tmp_chars, image::COLOR_RED);
+                    }
+                    delete img_std_first;
+                }
+                // draw on all face
+                auto color = r->at(0).second >= emotion_threshold ? image::COLOR_GREEN : image::COLOR_RED;
+                nn::Object &obj = result->at(idxes[i]);
+                img->draw_rect(obj.x, obj.y, obj.w, obj.h, color, 1);
+                snprintf(tmp_chars, sizeof(tmp_chars), "%s: %.1f", classifier.labels[r->at(0).first].c_str(), r->at(0).second);
+                img->draw_string(obj.x, obj.y, tmp_chars, color);
+                ++i;
+                delete r;
+            }
+            disp.show(*img);
+            delete img;
+            delete result;
+            log::info("time: all %d ms, detect %d ms", time::ticks_ms() - t, t3 - t2);
+        }
+    }
+
+    log::info("Program exit");
+
+    return ret;
+}
+
+int main(int argc, char *argv[])
+{
+    // Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+    signal(SIGINT, [](int sig)
+           { app::set_exit_flag(true); });
+
+    // Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
+    // if we don't catch exception, when program throw exception, the objects will not be destructed.
+    // So we catch exception here to let resources be released(call objects' destructor) before exit.
+    CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
+}

examples/nn_face_landmarks/main/src/main.cpp

@@ -60,6 +60,7 @@ int _main(int argc, char *argv[])
             detector.draw_pose(*img, r->points, 4, image::COLOR_RED);
         }
         img->save("result.jpg");
+        log::info("saved result image to result.jpg");
         delete result;
         delete img;
     }