Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-09-10 21:59:54 -05:00
Code Issues Packages Projects Releases 1 Wiki Activity

optimize llm and app classifier/detector ret button size

Browse Source
This commit is contained in:
Neucrack
2025-08-21 14:50:40 +08:00
parent b913a091dd
commit 06027b5c0f
7 changed files with 104 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
components/llm/src/maixcam2/LLM_InternVL.hpp
View File

@@ -14,6 +14,7 @@
#include <ax_sys_api.h>
#include "LLMPostprocess.hpp"
#include "maix_vlm_internvl.hpp"
#include <float.h>
namespace maix::nn::VLM_InternVL
{
@@ -499,7 +500,7 @@ public:
auto &output_post = llama_post.get_output("output");
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = -MAXFLOAT;
float max_val = FLT_MIN;
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;
@@ -601,7 +602,7 @@ public:
auto &output_post = llama_post.get_output("output");
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = -MAXFLOAT;
float max_val = FLT_MIN;
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;

2
components/llm/src/maixcam2/Tokenizer/Tokenizer.cpp
View File

@@ -27,7 +27,7 @@ public:
{
_tokenizer_type = tokenizer_type;
base_url = model_path;
if (!test_connect_http(base_url, 10))
if (!test_connect_http(base_url, 15))
{
ALOGE("connect %s failed", base_url.c_str());
return false;

2
components/llm/src/maixcam2/ax_model_runner/ax_model_runner_ax650.cpp
View File

@@ -262,7 +262,7 @@ int ax_runner_ax650::sub_init()
ret = prepare_io(m_handle->io_info[grpid], &m_handle->io_data[grpid], std::make_pair(AX_ENGINE_ABST_DEFAULT, AX_ENGINE_ABST_CACHED));
if (0 != ret)
{
ALOGE("prepare_io grpid=%d", grpid);
ALOGE("prepare_io grpid=%ld", grpid);
return ret;
}
}

6
components/llm/src/maixcam2/maix_vlm_internvl_maixcam2.cpp
View File

@@ -258,6 +258,12 @@
void InternVL::set_system_prompt(const std::string &prompt)
{
_system_prompt = prompt;
if(_loaded)
{
InternVLObj *obj = (InternVLObj *)_data;
std::vector<int> _token_ids;
obj->lLaMa.SetSystemPrompt(_system_prompt, _token_ids);
}
}
int InternVL::input_width()

21
components/llm/src/maixcam2/utils/memory_utils.hpp
View File

@@ -4,6 +4,8 @@
#include <stdio.h>
#include <fstream>
#include <vector>
#include <string.h>
#include <unistd.h>
bool file_exist(const std::string &path);
@@ -67,8 +69,21 @@ public:
fseek(file_fp, 0, SEEK_END);
*model_size = ftell(file_fp);
fclose(file_fp);
int fd = open(model_file, O_RDWR, 0644);
void *mmap_add = mmap(NULL, *model_size, PROT_WRITE, MAP_SHARED, fd, 0);
return mmap_add;
int fd = open(model_file, O_RDONLY);
if (fd < 0)
{
fprintf(stderr, "[MMap] open failed for file %s: %s\n", model_file, strerror(errno));
return nullptr;
}
void *mmap_addr = mmap(NULL, *model_size, PROT_READ, MAP_SHARED, fd, 0);
if (mmap_addr == MAP_FAILED)
{
fprintf(stderr, "[MMap] mmap failed for file %s: %s\n", model_file, strerror(errno));
close(fd);
return nullptr;
}
close(fd);
return mmap_addr;
}
};

46
projects/app_classifier/main/src/main.cpp
View File

@@ -9,6 +9,35 @@
using namespace maix;
image::Image *get_back_img(int screen_h)
{
image::Image *ret_img = image::load("./assets/ret.png", image::Format::FMT_RGB888);
if (!ret_img)
{
log::error("load ret image failed");
return nullptr;
}
int new_h = screen_h * 0.15;
int new_w = new_h / ret_img->height() * ret_img->width();
new_h = new_h % 2 == 0 ? new_h : new_h + 1; // make sure height is even
new_w = new_w % 2 == 0 ? new_w : new_w + 1; // make sure width is even
if(new_h != ret_img->height() || new_w != ret_img->width())
{
log::info("resize ret image from %dx%d to %dx%d", ret_img->width(), ret_img->height(), new_w, new_h);
image::Image *tmp_img = ret_img->resize(new_w, new_h, image::FIT_CONTAIN);
if (!tmp_img)
{
log::error("resize ret image failed");
delete ret_img;
throw err::Exception(err::ERR_NO_MEM, "resize ret image failed");
}
delete ret_img;
ret_img = tmp_img;
}
return ret_img;
}
int _main(int argc, char *argv[])
{
int ret = 0;
@@ -20,12 +49,6 @@ int _main(int argc, char *argv[])
touchscreen::TouchScreen ts;
int ts_x = 0, ts_y = 0;
bool ts_pressed = false;
image::Image *ret_img = image::load("./assets/ret.png", image::Format::FMT_RGB888);
if (!ret_img)
{
log::error("load ret image failed");
return -12345;
}
log::info("model path: %s", model_path);
nn::Classifier classifier(model_path);
@@ -41,6 +64,11 @@ int _main(int argc, char *argv[])
// image::Size input_size = classifier.input_size();
camera::Camera cam = camera::Camera(w, h, classifier.input_format());
log::info("open camera success");
auto ret_img = get_back_img(cam.height());
int font_scale = 2;
int font_thickness = 2;
uint64_t t, t2, t3, t_show, t_all = 0;
while (!app::need_exit())
{
@@ -59,12 +87,12 @@ int _main(int argc, char *argv[])
t3 = time::ticks_ms();
int max_idx = result->at(0).first;
float max_score = result->at(0).second;
img->draw_rect((w - min_len) / 2, (h - min_len) / 2, min_len, min_len, image::COLOR_WHITE, 2);
img->draw_rect((w - min_len) / 2, (h - min_len) / 2, min_len, min_len, image::COLOR_WHITE, font_thickness);
snprintf(msg, sizeof(msg), "%4.1f %%:\n%s", max_score * 100, classifier.labels[max_idx].c_str());
img->draw_string((w - min_len) / 2, disp.height() - 80, msg, image::COLOR_RED, 2, 2);
img->draw_string((w - min_len) / 2, disp.height() - 80, msg, image::COLOR_RED, font_scale, font_thickness);
img->draw_image(0, 0, *ret_img);
snprintf(tmp_chars, sizeof(tmp_chars), "All: %ldms, cam: %ldms\ndetect: %ldms, show: %ldms", t_all, t2 - t, t3 - t2, t_show);
img->draw_string((w - min_len) / 2, 4, tmp_chars, image::COLOR_RED, 1.5, 2);
img->draw_string((w - min_len) / 2, 4, tmp_chars, image::COLOR_RED, font_scale, font_thickness);
disp.show(*img);
t_show = time::ticks_ms() - t3;
t_all = time::ticks_ms() - t;

47
projects/app_detector/main/src/main.cpp
View File

@@ -15,6 +15,35 @@ bool is_in_button(int x, int y, std::vector<int> btn_pos)
}
image::Image *get_back_img(int screen_h)
{
image::Image *ret_img = image::load("./assets/ret.png", image::Format::FMT_RGB888);
if (!ret_img)
{
log::error("load ret image failed");
return nullptr;
}
int new_h = screen_h * 0.15;
int new_w = new_h / ret_img->height() * ret_img->width();
new_h = new_h % 2 == 0 ? new_h : new_h + 1; // make sure height is even
new_w = new_w % 2 == 0 ? new_w : new_w + 1; // make sure width is even
if(new_h != ret_img->height() || new_w != ret_img->width())
{
log::info("resize ret image from %dx%d to %dx%d", ret_img->width(), ret_img->height(), new_w, new_h);
image::Image *tmp_img = ret_img->resize(new_w, new_h, image::FIT_CONTAIN);
if (!tmp_img)
{
log::error("resize ret image failed");
delete ret_img;
throw err::Exception(err::ERR_NO_MEM, "resize ret image failed");
}
delete ret_img;
ret_img = tmp_img;
}
return ret_img;
}
int _main(int argc, char *argv[])
{
log::info("Program start");
@@ -28,12 +57,6 @@ int _main(int argc, char *argv[])
touchscreen::TouchScreen ts;
int ts_x = 0, ts_y = 0;
bool ts_pressed = false;
image::Image *ret_img = image::load("./assets/ret.png", image::Format::FMT_RGB888);
if (!ret_img)
{
log::error("load ret image failed");
return -12345;
}
const char *model_path = "/root/models/yolov5s.mud";
float conf_threshold = 0.5;
@@ -48,6 +71,12 @@ int _main(int argc, char *argv[])
camera::Camera cam = camera::Camera(input_size.width(), input_size.height(), detector.input_format());
log::info("open camera success");
display::Display disp = display::Display();
auto ret_img = get_back_img(cam.height());
int font_scale = cam.height() >= 480 ? 2 : 1.2;
int font_thickness = cam.height() >= 480 ? 2 : 1;
uint64_t t, t2, t3, t_show, t_all = 0;
int exit_btn_pos[4] = {0, 0, ret_img->width(), ret_img->height()};
std::vector<int> exit_btn_disp_pos = maix::image::resize_map_pos(cam.width(), cam.height(), disp.width(), disp.height(), image::FIT_CONTAIN, exit_btn_pos[0], exit_btn_pos[1], exit_btn_pos[2], exit_btn_pos[3]);
@@ -67,12 +96,12 @@ int _main(int argc, char *argv[])
t3 = time::ticks_ms();
for (auto &r : *result)
{
img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0));
img->draw_string(r.x, r.y, detector.labels[r.class_id], maix::image::Color::from_rgb(255, 0, 0));
img->draw_rect(r.x, r.y, r.w, r.h, maix::image::Color::from_rgb(255, 0, 0), font_thickness + 1);
img->draw_string(r.x + font_thickness, r.y + font_thickness, detector.labels[r.class_id], maix::image::Color::from_rgb(255, 0, 0), font_scale, font_thickness);
}
img->draw_image(0, 0, *ret_img);
snprintf(tmp_chars, sizeof(tmp_chars), "All: %ldms, fps: %ld\ncam: %ldms, detect: %ldms, show: %ldms", t_all, 1000 / t_all, t2 - t, t3 - t2, t_show);
img->draw_string(2, img->height() - 40, tmp_chars, image::COLOR_RED);
img->draw_string(2, img->height() - 40, tmp_chars, image::COLOR_RED, font_scale, font_thickness);
disp.show(*img);
t_show = time::ticks_ms() - t3;
t_all = time::ticks_ms() - t;