Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-09-10 21:59:54 -05:00
Code Issues Packages Projects Releases 1 Wiki Activity

* [maixcam2] Optimize qwen3 vl

Browse Source 
* [maixcam2] Load nt config from /boot/configs
This commit is contained in:
lxowalle
2026-01-07 19:16:33 +08:00
parent 271a06db5e
commit 971a2c16e3
3 changed files with 78 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
components/llm/include/maix_vlm_qwen3.hpp
View File

@@ -9,6 +9,7 @@
#include "maix_basic.hpp"
#include "maix_image.hpp"
#include "maix_llm_qwen.hpp"
#include "maix_nn.hpp"
namespace maix::nn
{
@@ -326,6 +327,8 @@ namespace maix::nn
std::string _tokenizer_type;
std::function<void(Qwen3VL &, const Qwen3VLResp &)> _callback = nullptr;
void *_data; // for implementation
err::Err create_environment_file(nn::MUD &mud);
};
}

75
components/llm/src/maixcam2/maix_vlm_qwen3_maixcam2.cpp
View File

@@ -87,6 +87,53 @@
}
}
err::Err Qwen3VL::create_environment_file(nn::MUD &mud)
{
auto configs = mud.items;
auto work_dir = fs::dirname(mud.model_path);
auto path = fs::join({work_dir, configs["extra"]["service_env_path"]});
auto exec_app_path = fs::join({work_dir, configs["extra"]["exec_app"]});
auto template_filename = fs::join({work_dir, configs["basic"]["model_npu"]});
auto model_num = configs["extra"]["model_num"];
auto encoder_model_path = fs::join({work_dir, configs["extra"]["vpm_resampler_model"]});
auto use_mmap_load_embed = configs["extra"]["use_mmap_load_embed"];
auto tokenizer_url = configs["extra"]["tokenizer_url"];
auto post_model_path = fs::join({work_dir, configs["extra"]["post_model"]});
auto tokens_embed_path = fs::join({work_dir, configs["extra"]["tokens_embed"]});
auto tokens_embed_num = configs["extra"]["tokens_embed_num"];
auto tokens_embed_size = configs["extra"]["tokens_embed_size"];
auto patch_size = configs["extra"]["patch_size"];
auto img_width = configs["extra"]["img_w"];
auto img_height = configs["extra"]["img_h"];
auto vision_start_token_id = configs["extra"]["vision_start_token_id"];
auto post_config_path = fs::join({work_dir, configs["extra"]["post_config_path"]});
auto f = fs::open(path, "w+");
if (!f) {
log::error("open qwen3_vl.service.env failed");
return err::ERR_RUNTIME;
}
f->write("WORK_DIR=" + work_dir + "\n");
f->write("MAIN_API=" + exec_app_path + "\n");
f->write("TEMPLATE_FILENAME_AXMODEL=" + template_filename + "\n");
f->write("AXMODEL_NUM=" + model_num + "\n");
f->write("ENCODER_MODEL_PATH=" + encoder_model_path + "\n");
f->write("USE_MMAP_LOAD_EMBED=" + use_mmap_load_embed + "\n");
f->write("TOKENIZER_URL=" + tokenizer_url + "\n");
f->write("POST_MODEL_PATH=" + post_model_path + "\n");
f->write("TOKENS_EMBED_PATH=" + tokens_embed_path + "\n");
f->write("TOKENS_EMBED_NUM=" + tokens_embed_num + "\n");
f->write("TOKENS_EMBED_SIZE=" + tokens_embed_size + "\n");
f->write("PATCH_SIZE=" + patch_size + "\n");
f->write("IMG_WIDTH=" + img_width + "\n");
f->write("IMG_HEIGHT=" + img_height + "\n");
f->write("VISION_START_TOKEN_ID=" + vision_start_token_id + "\n");
f->write("POST_CONFIG_PATH=" + post_config_path + "\n");
f->close();
return err::ERR_NONE;
}
err::Err Qwen3VL::load(const std::string &model)
{
Qwen3VLObj *obj = (Qwen3VLObj *)_data;
@@ -100,6 +147,13 @@
err::Err e = obj->mud.load(model);
if(e != err::ERR_NONE)
return e;
e = this->create_environment_file(obj->mud);
if(e != err::ERR_NONE) {
log::error("create environment file failed, err:%d", e);
return e;
}
std::string model_dir = fs::dirname(model);
// init llm model
VLM_Qwen3::LLMAttrType attr;
@@ -137,15 +191,18 @@
}
try
{
post_config.enable_temperature = obj->mud.items["post_config"]["enable_temperature"] == "true" ? true : false;
post_config.temperature = std::stof(obj->mud.items["post_config"]["temperature"]);
post_config.enable_repetition_penalty = obj->mud.items["post_config"]["enable_repetition_penalty"] == "true" ? true : false;
post_config.repetition_penalty = std::stof(obj->mud.items["post_config"]["repetition_penalty"]);
post_config.penalty_window = std::stoi(obj->mud.items["post_config"]["penalty_window"]);
post_config.enable_top_p_sampling = obj->mud.items["post_config"]["enable_top_p_sampling"] == "true" ? true : false;
post_config.top_p = std::stof(obj->mud.items["post_config"]["top_p"]);
post_config.enable_top_k_sampling = obj->mud.items["post_config"]["enable_top_k_sampling"] == "true" ? true : false;
post_config.top_k = std::stoi(obj->mud.items["post_config"]["top_k"]);
auto post_config_file = fs::join({model_dir, obj->mud.items["extra"]["post_config_path"]});
std::ifstream file(post_config_file);
nlohmann::json config = nlohmann::json::parse(file);
post_config.enable_temperature = config.value("enable_temperature", true);
post_config.temperature = config.value("temperature", 0.7f);
post_config.enable_repetition_penalty = config.value("enable_repetition_penalty", false);
post_config.repetition_penalty = config.value("repetition_penalty", 1.0f);
post_config.penalty_window = config.value("penalty_window", 30);
post_config.enable_top_p_sampling = config.value("enable_top_p_sampling", false);
post_config.top_p = config.value("top_p", 0.8f);
post_config.enable_top_k_sampling = config.value("enable_top_k_sampling", true);
post_config.top_k = config.value("top_k", 20);
}
catch(...)
{

16
components/vision/port/maixcam2/maix_camera_maixcam2.cpp
View File

@@ -13,8 +13,6 @@
#include <dirent.h>
#include "ax_middleware.hpp"
#define AX_NT_ENABLE (0)
using namespace maix;
using namespace maix::middleware::maixcam2;
#define ALIGN_UP_2(value) ((value + 0x1) & (~0x1))
@@ -599,12 +597,16 @@ namespace maix::camera
log::warn("Height %d is not aligned to 2, will be set to %d", width_tmp, ALIGN_UP_2(height_tmp));
}
#if AX_NT_ENABLE
auto axRet = COMMON_NT_Init(6000, 8082);
if (axRet) {
printf("COMMON_NT_Init fail, ret:0x%x\r\n", axRet);
auto camera_nt = app::get_sys_config_kv("camera", "nt", "0") == "1" ? AX_TRUE : AX_FALSE;
if (camera_nt) {
auto camera_nt_stream_port = atoi(app::get_sys_config_kv("camera", "nt_stream_port", "6000").c_str());
auto camera_nt_ctrl_port = atoi(app::get_sys_config_kv("camera", "nt_ctrl_port", "8082").c_str());
auto axRet = COMMON_NT_Init(camera_nt_stream_port, camera_nt_ctrl_port);
if (axRet) {
printf("COMMON_NT_Init fail, ret:0x%x\r\n", axRet);
}
}
#endif
this->vflip(priv->chn.vflip);
this->hmirror(priv->chn.mirror);
return err::ERR_NONE;