Administrator/NANOKVM-MaixCDK-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/MaixCDK.git synced 2026-09-10 21:59:54 -05:00
Code Issues Packages Projects Releases 1 Wiki Activity

* add smolvlm demo

Browse Source
This commit is contained in:
lxowalle
2025-12-03 15:39:20 +08:00
parent f7dc335456
commit c1e28b827f
10 changed files with 1557 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
components/llm/include/maix_vlm_smolvlm.hpp Normal file
View File

@@ -0,0 +1,309 @@
/**
* VLM SmolVLM
* @license: Apache-2.0
* @author: neucrack@sipeed
* @date: 2025-05-30
*/
#pragma once
#include "maix_basic.hpp"
#include "maix_image.hpp"
#include "maix_llm_qwen.hpp"
namespace maix::nn
{
/**
* SmolVLM model response
* @maixpy maix.nn.SmolVLMResp
*/
class SmolVLMResp
{
public:
SmolVLMResp()
{
err_code = err::ERR_NONE;
err_msg = "";
}
/**
* Model response full message.
* @maixpy maix.nn.SmolVLMResp.msg
*/
std::string msg;
/**
* Model response new message.
* @maixpy maix.nn.SmolVLMResp.msg_new
*/
std::string msg_new;
/**
* Model response error code, maix.Err type, should be err.Err.ERR_NONE if no error.
* @maixpy maix.nn.SmolVLMResp.err_code
*/
err::Err err_code;
/**
* Model response error message.
* @maixpy maix.nn.SmolVLMResp.err_msg
*/
std::string err_msg;
};
/**
* SmolVLM model post config
* @maixpy maix.nn.SmolVLMPostConfig
*/
class SmolVLMPostConfig
{
public:
SmolVLMPostConfig()
{
enable_temperature = true;
temperature = 0.9;
enable_repetition_penalty = false;
repetition_penalty = 1.2;
penalty_window = 20;
enable_top_p_sampling = false;
top_p = 0.8;
enable_top_k_sampling = true;
top_k = 10;
}
/**
* Enable temperature sampling
* @maixpy maix.nn.SmolVLMPostConfig.enable_temperature
*/
bool enable_temperature;
/**
* Temperature sampling value
* @maixpy maix.nn.SmolVLMPostConfig.temperature
*/
float temperature;
/**
* Enable repetition penalty
* @maixpy maix.nn.SmolVLMPostConfig.enable_repetition_penalty
*/
bool enable_repetition_penalty;
/**
* Repetition penalty value
* @maixpy maix.nn.SmolVLMPostConfig.repetition_penalty
*/
float repetition_penalty;
/**
* Repetition penalty window
* @maixpy maix.nn.SmolVLMPostConfig.penalty_window
*/
int penalty_window;
/**
* Enable diversity penalty
* @maixpy maix.nn.SmolVLMPostConfig.enable_top_p_sampling
*/
bool enable_top_p_sampling;
/**
* Diversity penalty value
* @maixpy maix.nn.SmolVLMPostConfig.top_p
*/
float top_p;
/**
* Enable top k sampling
* @maixpy maix.nn.SmolVLMPostConfig.enable_top_k_sampling
*/
bool enable_top_k_sampling;
/**
* Top k sampling value
* @maixpy maix.nn.SmolVLMPostConfig.top_k
*/
int top_k;
};
/**
* SmolVLM model
* @maixpy maix.nn.SmolVLM
*/
class SmolVLM
{
public:
/**
* SmolVLM constructor
* @param[in] model model file path, model format can be MUD(model universal describe file) file.
* If model_path set, will load model from file, load failed will raise err.Exception.
* If model_path not set, you can load model later by load function.
* @maixpy maix.nn.SmolVLM.__init__
* @maixcdk maix.nn.SmolVLM.SmolVLM
*/
SmolVLM(const std::string &model);
~SmolVLM();
/**
* Load model from file
* @param[in] model model file path, model format can be MUD(model universal describe file) file.
* @return error code, if load success, return err::ERR_NONE
* @maixpy maix.nn.SmolVLM.load
*/
err::Err load(const std::string &model);
/**
* Unload model
* @return error code, if unload success, return err::ERR_NONE
* @maixpy maix.nn.SmolVLM.unload
*/
err::Err unload();
/**
* Is model loaded
* @return true if model loaded, else false
* @maixpy maix.nn.SmolVLM.loaded
*/
bool loaded()
{
return _loaded;
}
/**
* Set system prompt
* @param prompt system prompt
* @maixpy maix.nn.SmolVLM.set_system_prompt
*/
void set_system_prompt(const std::string &prompt);
/**
* Get system prompt
* @return system prompt
* @maixpy maix.nn.SmolVLM.get_system_prompt
*/
std::string get_system_prompt()
{
return _system_prompt;
}
/**
* Set log level
* @param level log level, @see maix.log.LogLevel
* @param color true to enable color, false to disable color
* @maixpy maix.nn.SmolVLM.set_log_level
*/
void set_log_level(log::LogLevel level, bool color);
/**
* Set reply callback.
* @param callback reply callback, when token(words) generated, this function will be called,
* so you can get response message in real time in this callback funtion.
* If set to None(nullptr in C++), you can get response after all response message generated.
* @maixpy maix.nn.SmolVLM.set_reply_callback
*/
void set_reply_callback(std::function<void(nn::SmolVLM &, const nn::SmolVLMResp &)> callback = nullptr)
{
_callback = callback;
}
/**
* Get reply callback
* @return reply callback
* @maixpy maix.nn.SmolVLM.get_reply_callback
*/
std::function<void(nn::SmolVLM &, const nn::SmolVLMResp &)> get_reply_callback()
{
return _callback;
}
/**
* Image input width
* @return input width.
* @maixpy maix.nn.SmolVLM.input_width
*/
int input_width();
/**
* Image input height
* @return input height.
* @maixpy maix.nn.SmolVLM.input_height
*/
int input_height();
/**
* Image input format
* @return input format.
* @maixpy maix.nn.SmolVLM.input_format
*/
maix::image::Format input_format();
/**
* Set image and will encode image.
* You can set image once and call send multiple times.
* @param img the image you want to use.
* @param fit Image resize fit method, only used when img size not equal to model input.
* @return err.Err return err.Err.ERR_NONE is no error happen.
* @maixpy maix.nn.SmolVLM.set_image
*/
err::Err set_image(maix::image::Image &img, maix::image::Fit fit = maix::image::Fit::FIT_CONTAIN);
/**
* Clear image, SmolVLM2.5 based on Qwen2.5, so you can clear image and only use LLM function.
* @maixpy maix.nn.SmolVLM.clear_image
*/
void clear_image();
/**
* Whether image set by set_image
* @return Return true if image set by set_image function, or return false.
* @maixpy maix.nn.SmolVLM.is_image_set
*/
bool is_image_set();
/**
* Send message to model
* @param msg message to send
* @return model response
* @maixpy maix.nn.SmolVLM.send
*/
nn::SmolVLMResp send(const std::string &msg);
/**
* Cancel running
* @maixpy maix.nn.SmolVLM.cancel
*/
void cancel();
/**
* Get model version
* @return model version
* @maixpy maix.nn.SmolVLM.version
*/
std::string version()
{
return _version;
}
public:
/**
* SmolVLM post config, default will read config from model mud file, you can also set it manually here.
* @maixpy maix.nn.SmolVLM.post_config
*/
nn::SmolVLMPostConfig post_config;
private:
bool _loaded = false;
std::string _system_prompt;
std::string _model_path;
std::string _version;
std::string _tokenizer_type;
std::function<void(SmolVLM &, const SmolVLMResp &)> _callback = nullptr;
void *_data; // for implementation
};
}

683
components/llm/src/maixcam2/LLM_SmolVLM.hpp Normal file
View File

@@ -0,0 +1,683 @@
#pragma once
#include <string>
#include <algorithm>
#include <cmath>
#include <numeric>
#include "bfloat16.hpp"
#include "Tokenizer/Tokenizer.hpp"
#include "LLMEmbedSelector.hpp"
#include "ax_model_runner/ax_model_runner_ax650.hpp"
#include "ax_cmm_utils.hpp"
#include "cqdm.h"
#include "timer.hpp"
#include <ax_sys_api.h>
#include "LLMPostprocess.hpp"
#include "maix_vlm_smolvlm.hpp"
#include <float.h>
namespace maix::nn::VLM_SmolVLM
{
#define ALIGN_DOWN(x, a) ((x) & ~((a) - 1))
typedef void (*LLMRuningCallback)(int *p_token, int n_token, const char *p_str, float token_per_sec, void *reserve);
struct LLMAttrType
{
std::string system_prompt;
std::string template_filename_axmodel = "tinyllama-int8/tinyllama_l%d.axmodel";
int axmodel_num = 22;
// std::string template_prefill_filename_axmodel = "minicpmv/prefill_axmodel/minicpm_p96_l%d.axmodel";
// int prefill_axmodel_num = 40;
int prefill_token_num = 96; // auto calc
int prefill_max_token_num = 512;
std::string filename_post_axmodel = "tinyllama-int8/tinyllama_post.axmodel";
std::string filename_vpm_encoder_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel";
std::string filename_vpm_resampler_axmodedl = "minicpmv/vpm_resampler_version0_fp16.axmodel";
int vpm_width = 280;
int vpm_height = 280;
bool b_vpm_two_stage = false;
TokenizerType tokenizer_type = TKT_LLaMa;
std::string url_tokenizer_model = "http://127.0.0.1:12345";
bool b_bos = true, b_eos = false;
std::string filename_tokens_embed = "tinyllama.model.embed_tokens.weight.bfloat16.bin";
int tokens_embed_num = 32000;
int tokens_embed_size = 2048;
int max_token_len = 127; // auto calc
int kv_cache_num = 1024; // auto calc
int kv_cache_size = 256; // auto calc
int precompute_len = 1202;
std::vector<int> prefill_max_kv_cache_num_grp;
int prefill_grpid = -1;
bool b_use_mmap_load_embed = false;
int vpm_len;
// bool b_use_mmap_load_layer = true;
// bool b_live_print = true;
LLMRuningCallback runing_callback = nullptr;
void *reserve = nullptr;
};
class LLM
{
private:
std::shared_ptr<BaseTokenizer> tokenizer;
LLaMaEmbedSelector embed_selector;
LLMAttrType _attr;
struct LLMLayer
{
ax_runner_ax650 layer;
std::string filename;
MMap layer_buffer;
std::vector<char> layer_buffer_vec;
};
std::vector<LLMLayer> llama_layers;
ax_runner_ax650 llama_post;
//
int prefill_grpid = 1;
int decode_grpid = 0;
ax_runner_ax650 vpm_resampler;
// std::vector<std::vector<unsigned short>> k_caches, v_caches;
bool b_stop = false;
LLMPostprocess postprocess;
static int post_process(LLMPostprocess &postprocess, unsigned short *p, int n, std::vector<int> &history, float *val = 0)
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
std::vector<float> logits(n);
for (int i = 0; i < n; i++)
{
unsigned int proc = p[i] << 16;
logits[i] = *reinterpret_cast<float *>(&proc);
}
#pragma GCC diagnostic pop
return postprocess.apply(logits, history);
}
public:
bool Init(LLMAttrType attr, maix::nn::QwenPostConfig &post_config, const std::string &tokenizer_type, int &vpm_w, int &vpm_h)
{
ALOGI("LLM init start");
t_cqdm cqdm = create_cqdm(attr.axmodel_num + 4, 32);
this->_attr = attr;
tokenizer = CreateTokenizer(attr.tokenizer_type);
if (!tokenizer->Init(attr.url_tokenizer_model, tokenizer_type))
{
ALOGE("tokenizer.Init(%s) failed", attr.url_tokenizer_model.c_str());
return false;
}
std::vector<int> _token_ids;
tokenizer->Reset(attr.system_prompt, _token_ids);
update_cqdm(&cqdm, 0, "count", "tokenizer init ok");
// test code
// {
// std::vector<int> output;
// tokenizer.Encode("Today is National", output);
// // print output
// for (size_t i = 0; i < output.size(); i++)
// {
// printf("%d ", output[i]);
// }
// printf("\n");
// }
if (!embed_selector.Init(attr.filename_tokens_embed, attr.tokens_embed_num, attr.tokens_embed_size, attr.b_use_mmap_load_embed))
{
ALOGE("embed_selector.Init(%s, %d, %d) failed", attr.filename_tokens_embed.c_str(), attr.tokens_embed_num, attr.tokens_embed_size);
return false;
}
update_cqdm(&cqdm, 1, "count", "embed_selector init ok");
// test code
// {
// std::vector<unsigned short> embed = embed_selector.getByIndex(123);
// printf("embed size: %d\n", embed.size());
// for (int i = 0; i < embed.size(); i++)
// {
// bfloat16 bf16 = bfloat16(embed[i]);
// float val = bf16;
// printf("%d %0.22f\n", embed[i], val);
// }
// }
llama_layers.resize(attr.axmodel_num);
// prefill_layers.resize(attr.prefill_axmodel_num);
char axmodel_path[1024];
for (int i = 0; i < attr.axmodel_num; i++)
{
sprintf(axmodel_path, attr.template_filename_axmodel.c_str(), i);
llama_layers[i].filename = axmodel_path;
int ret = llama_layers[i].layer.init(llama_layers[i].filename.c_str(), false);
if (ret != 0)
{
ALOGE("init axmodel(%s) failed", llama_layers[i].filename.c_str());
return false;
}
int remain_cmm = get_remaining_cmm_size();
sprintf(axmodel_path, "init %d axmodel ok,remain_cmm(%d MB)", i, remain_cmm);
update_cqdm(&cqdm, i + 2, "count", axmodel_path);
}
int ret = llama_post.init(attr.filename_post_axmodel.c_str(), false);
if (ret != 0)
{
ALOGE("init post axmodel(%s) failed", attr.filename_post_axmodel.c_str());
return false;
}
int remain_cmm = get_remaining_cmm_size();
sprintf(axmodel_path, "init post axmodel ok,remain_cmm(%d MB)", remain_cmm);
update_cqdm(&cqdm, attr.axmodel_num + 2, "count", axmodel_path);
// int remain_cmm = get_remaining_cmm_size();
// sprintf(axmodel_path, "init vpm axmodel ok,remain_cmm(%d MB)", remain_cmm);
// update_cqdm(&cqdm, attr.axmodel_num + 2, "count", axmodel_path);
{
ret = vpm_resampler.init(attr.filename_vpm_resampler_axmodedl.c_str(), false);
if (ret != 0)
{
ALOGE("init vpm axmodel(%s) failed", attr.filename_vpm_resampler_axmodedl.c_str());
return false;
}
_attr.vpm_height = vpm_resampler.get_input(0).vShape[1];
_attr.vpm_width = vpm_resampler.get_input(0).vShape[2];
vpm_w = _attr.vpm_width;
vpm_h = _attr.vpm_height;
ALOGD("vpm_width : %d, vpm_height: %d", _attr.vpm_width, _attr.vpm_height);
}
remain_cmm = get_remaining_cmm_size();
sprintf(axmodel_path, "init vpm axmodel ok,remain_cmm(%d MB)", remain_cmm);
update_cqdm(&cqdm, attr.axmodel_num + 3, "count", axmodel_path);
{
_attr.max_token_len = llama_layers[0].layer.get_input("mask").nSize / sizeof(unsigned short) - 1;
ALOGI("max_token_len : %d", _attr.max_token_len);
// auto &input_k_cache = llama_layers[0].layer.get_input("K_cache");
// auto &output_k_cache_out = llama_layers[0].layer.get_output("K_cache_out");
_attr.kv_cache_size = llama_layers[0].layer.get_output("K_cache_out").nSize / sizeof(unsigned short);
_attr.kv_cache_num = llama_layers[0].layer.get_input("K_cache").nSize / _attr.kv_cache_size / sizeof(unsigned short);
ALOGI("kv_cache_size : %d, kv_cache_num: %d", _attr.kv_cache_size, _attr.kv_cache_num);
if (_attr.max_token_len > _attr.kv_cache_num)
{
ALOGE("max_token_len(%d) > kv_cache_num(%d)", _attr.max_token_len, _attr.kv_cache_num);
return false;
}
_attr.prefill_token_num = llama_layers[0].layer.get_input(1, "indices").vShape[1];
ALOGI("prefill_token_num : %d", _attr.prefill_token_num);
}
if (!postprocess.load_config(post_config))
{
ALOGW("load postprocess config failed");
}
// prepare input
for (int i = 0; i < _attr.axmodel_num; i++)
{
memset(llama_layers[i].layer.get_input(prefill_grpid, "K_cache").pVirAddr, 0, llama_layers[i].layer.get_input(prefill_grpid, "K_cache").nSize);
memset(llama_layers[i].layer.get_input(prefill_grpid, "V_cache").pVirAddr, 0, llama_layers[i].layer.get_input(prefill_grpid, "V_cache").nSize);
memset(llama_layers[i].layer.get_input(decode_grpid, "K_cache").pVirAddr, 0, llama_layers[i].layer.get_input(decode_grpid, "K_cache").nSize);
memset(llama_layers[i].layer.get_input(decode_grpid, "V_cache").pVirAddr, 0, llama_layers[i].layer.get_input(decode_grpid, "V_cache").nSize);
}
// Reset();
ALOGI("LLM init ok");
return true;
}
LLMAttrType *getAttr()
{
return &_attr;
}
LLMPostprocess *getPostprocess()
{
return &postprocess;
}
void Deinit()
{
for (int i = 0; i < _attr.axmodel_num; i++)
{
llama_layers[i].layer.release();
}
llama_post.release();
// vpm_encoder.release();
vpm_resampler.release();
embed_selector.Deinit();
}
void Stop()
{
b_stop = true;
}
int SetSystemPrompt(std::string system_prompt, std::vector<int> &_token_ids)
{
tokenizer->Reset(system_prompt, _token_ids);
_attr.system_prompt = system_prompt;
// _attr.prefill_max_token_num = _attr.prefill_max_kv_cache_num_grp[_attr.prefill_max_kv_cache_num_grp.size() - 1];
return 0;
}
int Encode(maix::image::Image &img, std::vector<unsigned short> &img_embed)
{
// encode image
void *data = vpm_resampler.get_input(0).pVirAddr;
memcpy(data, img.data(), img.data_size());
vpm_resampler.inference();
img_embed.resize(vpm_resampler.get_output(0).nSize / sizeof(float));
AX_SYS_MinvalidateCache(vpm_resampler.get_output(0).phyAddr, vpm_resampler.get_output(0).pVirAddr, vpm_resampler.get_output(0).nSize);
float *output_data = (float *)vpm_resampler.get_output(0).pVirAddr;
for (size_t i = 0; i < img_embed.size(); i++)
{
img_embed[i] = bfloat16(output_data[i]).data;
}
return 0;
}
int Encode(std::vector<unsigned short> &out_embed, std::vector<unsigned short> &img_embed, std::string prompt = "What is in the image?")
{
// encode text
std::string last_reply;
std::vector<int> tokens_ids;
std::vector<int> tokens_diff;
bool have_image = !img_embed.empty();
if(!tokenizer->Encode(prompt, last_reply, tokens_ids, tokens_diff, have_image, _attr.vpm_len))
{
ALOGE("encode failed");
return -1;
}
// gen embed
constexpr int IMG_CONTEXT = 49190; // SmolVLM 256M
int offset = 0;
if(have_image)
{
for (size_t i = 0; i < tokens_ids.size(); i++)
{
if (tokens_ids[i] == IMG_CONTEXT)
{
offset = i;
break;
}
}
if (offset == 0)
{
ALOGE("img_context offset == 0, tokenizer error");
return -1;
}
if(tokens_ids[offset + img_embed.size() / _attr.tokens_embed_size - 1] != IMG_CONTEXT ||
tokens_ids[offset + _attr.vpm_len - 1] != IMG_CONTEXT
)
{
ALOGE("vpm encode error, tokenizer return wrong encoded img tag, offset %ld=%d, %d=%d, should be %d",
offset + img_embed.size() / _attr.tokens_embed_size - 1,
tokens_ids[offset + img_embed.size() / _attr.tokens_embed_size - 1],
offset + _attr.vpm_len - 1,
tokens_ids[offset + _attr.vpm_len - 1],
IMG_CONTEXT);
return -1;
}
}
if (tokens_ids.size() > (size_t)_attr.prefill_token_num)
{
ALOGE("tokens_ids(%ld) > prefill_token_num(%d)", tokens_ids.size(), _attr.prefill_token_num);
return -1;
}
out_embed.resize(tokens_ids.size() * _attr.tokens_embed_size);
for (size_t i = 0; i < tokens_ids.size(); i++)
{
if(tokens_ids[i] == IMG_CONTEXT)
continue;
embed_selector.getByIndex(tokens_ids[i], out_embed.data() + i * _attr.tokens_embed_size);
}
if(have_image)
memcpy(out_embed.data() + offset * _attr.tokens_embed_size, img_embed.data(), img_embed.size() * sizeof(unsigned short));
// ALOGI("have_image: %d, offset: %d, img_embed.size(): %ld", have_image, offset, img_embed.size());
return 0;
}
std::string Run(std::vector<unsigned short> test_embed)
{
b_stop = false;
std::string final_out;
bfloat16 bf16 = -65536.f;
std::vector<unsigned short> mask(_attr.kv_cache_num + 1, bf16.data);
std::vector<unsigned short> mask_p(_attr.prefill_token_num * _attr.prefill_token_num, bf16.data);
for (int i = 0; i < _attr.prefill_token_num; i++)
{
for (int j = 0; j < i + 1; j++)
{
mask_p[i * _attr.prefill_token_num + j] = 0;
}
}
std::vector<int> cached_token;
std::vector<int> token_ids;
// std::vector<int> token_ids = tokenizer->Encode(input_str);
// int len_of_input = token_ids.size();
int input_embed_num = test_embed.size() / _attr.tokens_embed_size;
// ALOGI("input_embed_num(%d)", input_embed_num);
mask[_attr.kv_cache_num] = 0;
for (int i = 0; i < input_embed_num; i++)
{
mask[i] = 0;
}
timer t_cost;
timer t_tmp;
timer ttft_timer;
ttft_timer.start();
float decode_t_all = 0;
int decode_req_times = 0;
for (int m = 0; m < _attr.axmodel_num; m++)
{
if (b_stop)
{
break;
}
auto &layer = llama_layers[m];
auto &layer_llama = llama_layers[m];
// if (_attr.b_dynamic_load_axmodel_layer)
// {
// int ret;
// if (_attr.b_use_mmap_load_layer)
// {
// ret = layer.layer.init((char *)layer.layer_buffer.data(), layer.layer_buffer.size());
// }
// else
// {
// ret = layer.layer.init(layer.layer_buffer_vec.data(), layer.layer_buffer_vec.size());
// }
// if (ret != 0)
// {
// ALOGE("init axmodel(%s) failed", layer.filename.c_str());
// }
// }
auto &input_indices = layer.layer.get_input(prefill_grpid, "indices");
unsigned int *input_indices_ptr = (unsigned int *)input_indices.pVirAddr;
for (int i = 0; i < input_embed_num; i++)
{
input_indices_ptr[i] = i;
}
auto &input_mask = layer.layer.get_input(prefill_grpid, "mask");
memcpy(input_mask.pVirAddr, mask_p.data(), mask_p.size() * sizeof(unsigned short));
auto &input_input = layer.layer.get_input(prefill_grpid, "input");
memcpy(input_input.pVirAddr, test_embed.data(), test_embed.size() * sizeof(unsigned short));
if (m == 0)
{
test_embed.resize(_attr.prefill_token_num * _attr.tokens_embed_size);
}
layer.layer.inference(prefill_grpid);
auto &output_k_cache = layer.layer.get_output(prefill_grpid, "K_cache_out");
AX_SYS_MinvalidateCache(output_k_cache.phyAddr, output_k_cache.pVirAddr, output_k_cache.nSize);
auto &input_k_cache = layer_llama.layer.get_input(decode_grpid, "K_cache");
memcpy(input_k_cache.pVirAddr, output_k_cache.pVirAddr, sizeof(unsigned short) * _attr.prefill_token_num * _attr.kv_cache_size);
auto &output_v_cache = layer.layer.get_output(prefill_grpid, "V_cache_out");
AX_SYS_MinvalidateCache(output_v_cache.phyAddr, output_v_cache.pVirAddr, output_v_cache.nSize);
auto &input_v_cache = layer_llama.layer.get_input(decode_grpid, "V_cache");
memcpy(input_v_cache.pVirAddr, output_v_cache.pVirAddr, sizeof(unsigned short) * _attr.prefill_token_num * _attr.kv_cache_size);
auto &output = layer.layer.get_output(prefill_grpid, "output");
AX_SYS_MinvalidateCache(output.phyAddr, output.pVirAddr, output.nSize);
memcpy(test_embed.data(), output.pVirAddr, test_embed.size() * sizeof(unsigned short));
// if (_attr.b_dynamic_load_axmodel_layer)
// {
// layer.layer.deinit();
// }
// ALOGI("%f %f %f %f %f", bfloat16(embed[0]).fp32(), bfloat16(embed[1]).fp32(), bfloat16(embed[2]).fp32(), bfloat16(embed[3]).fp32(), bfloat16(embed[4]).fp32());
}
// ALOGI("prefill time cost: %.2f s", t_cost.cost() / 1000);
// print token_ids
// printf("%s\n", input_str.c_str());
// for (size_t i = 0; i < token_ids.size(); i++)
// {
// printf("%d ", token_ids[i]);
// }
// printf("\n");
int next_token = -1;
t_cqdm cqdm = create_cqdm(_attr.max_token_len, 32);
std::vector<unsigned short> embed(_attr.tokens_embed_size, 0);
memcpy(embed.data(),
test_embed.data() + (input_embed_num - 1) * _attr.tokens_embed_size,
_attr.tokens_embed_size * sizeof(unsigned short));
{
// post process
auto &input = llama_post.get_input("input");
memcpy(input.pVirAddr, embed.data(), embed.size() * sizeof(unsigned short));
llama_post.inference();
int max_index;
// if (_attr.b_use_topk)
// {
// AX_SYS_MinvalidateCache(llama_post.get_output("indices").phyAddr, llama_post.get_output("indices").pVirAddr, llama_post.get_output("indices").nSize);
// max_index = *(int *)llama_post.get_output("indices").pVirAddr;
// }
// else
{
auto &output_post = llama_post.get_output("output");
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = FLT_MIN;
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;
token_ids.push_back(max_index);
cached_token.push_back(max_index);
ALOGI("ttft: %.2f ms, first predict token: %d", ttft_timer.cost(), max_index);
}
t_cost.start();
bool b_hit_eos = false;
for (int indices = input_embed_num; indices < _attr.max_token_len; indices++)
{
if (b_stop)
{
break;
}
// ALOGI("out %d %d", indices, next_token);
embed_selector.getByIndex(next_token, embed);
// ALOGI("%f %f %f %f %f", bfloat16(embed[0]).fp32(), bfloat16(embed[1]).fp32(), bfloat16(embed[2]).fp32(), bfloat16(embed[3]).fp32(), bfloat16(embed[4]).fp32());
for (int m = 0; m < _attr.axmodel_num; m++)
{
if (b_stop)
{
break;
}
auto &layer = llama_layers[m];
// if (_attr.b_dynamic_load_axmodel_layer)
// {
// int ret;
// if (_attr.b_use_mmap_load_layer)
// {
// ret = layer.layer.init((char *)layer.layer_buffer.data(), layer.layer_buffer.size());
// }
// else
// {
// ret = layer.layer.init(layer.layer_buffer_vec.data(), layer.layer_buffer_vec.size());
// }
// if (ret != 0)
// {
// ALOGE("init axmodel(%s) failed", layer.filename.c_str());
// }
// }
auto &input_k_cache = layer.layer.get_input(decode_grpid, "K_cache");
unsigned short *input_k_cache_ptr = (unsigned short *)input_k_cache.pVirAddr;
// memcpy(input_k_cache.pVirAddr, k_caches[m].data(), sizeof(unsigned short) * k_caches[m].size());
auto &input_v_cache = layer.layer.get_input(decode_grpid, "V_cache");
unsigned short *input_v_cache_ptr = (unsigned short *)input_v_cache.pVirAddr;
// memcpy(input_v_cache.pVirAddr, v_caches[m].data(), sizeof(unsigned short) * v_caches[m].size());
auto &input_indices = layer.layer.get_input(decode_grpid, "indices");
memcpy(input_indices.pVirAddr, &indices, sizeof(indices));
auto &input_mask = layer.layer.get_input(decode_grpid, "mask");
memcpy(input_mask.pVirAddr, mask.data(), mask.size() * sizeof(unsigned short));
auto &input_input = layer.layer.get_input(decode_grpid, "input");
memcpy(input_input.pVirAddr, embed.data(), embed.size() * sizeof(unsigned short));
layer.layer.inference(decode_grpid);
auto &output_k_cache = layer.layer.get_output(decode_grpid, "K_cache_out");
AX_SYS_MinvalidateCache(output_k_cache.phyAddr, output_k_cache.pVirAddr, output_k_cache.nSize);
memcpy(input_k_cache_ptr + indices * _attr.kv_cache_size, output_k_cache.pVirAddr, sizeof(unsigned short) * _attr.kv_cache_size);
auto &output_v_cache = layer.layer.get_output(decode_grpid, "V_cache_out");
AX_SYS_MinvalidateCache(output_v_cache.phyAddr, output_v_cache.pVirAddr, output_v_cache.nSize);
memcpy(input_v_cache_ptr + indices * _attr.kv_cache_size, output_v_cache.pVirAddr, sizeof(unsigned short) * _attr.kv_cache_size);
auto &output = layer.layer.get_output(decode_grpid, "output");
AX_SYS_MinvalidateCache(output.phyAddr, output.pVirAddr, output.nSize);
memcpy(embed.data(), output.pVirAddr, embed.size() * sizeof(unsigned short));
// if (_attr.b_dynamic_load_axmodel_layer)
// {
// layer.layer.deinit();
// }
// ALOGI("%f %f %f %f %f", bfloat16(embed[0]).fp32(), bfloat16(embed[1]).fp32(), bfloat16(embed[2]).fp32(), bfloat16(embed[3]).fp32(), bfloat16(embed[4]).fp32());
}
// ALOGI("");
mask[indices] = 0;
{
// post process
auto &input = llama_post.get_input("input");
memcpy(input.pVirAddr, embed.data(), embed.size() * sizeof(unsigned short));
llama_post.inference();
int max_index;
// if (_attr.b_use_topk)
// {
// AX_SYS_MinvalidateCache(llama_post.get_output("indices").phyAddr, llama_post.get_output("indices").pVirAddr, llama_post.get_output("indices").nSize);
// max_index = *(int *)llama_post.get_output("indices").pVirAddr;
// }
// else
{
auto &output_post = llama_post.get_output("output");
AX_SYS_MinvalidateCache(output_post.phyAddr, output_post.pVirAddr, output_post.nSize);
unsigned short *post_out = (unsigned short *)output_post.pVirAddr;
float max_val = FLT_MIN;
max_index = post_process(postprocess, post_out, _attr.tokens_embed_num, token_ids, &max_val);
}
next_token = max_index;
if (tokenizer->isEnd(max_index))
{
if (cached_token.size() && _attr.runing_callback)
{
float t_cost_ms = t_cost.cost();
float token_per_sec = token_ids.size() / (t_cost_ms / 1000);
t_tmp.start();
auto tmp_out = tokenizer->Decode(cached_token);
decode_t_all += t_tmp.cost();
++decode_req_times;
final_out += tmp_out;
_attr.runing_callback(cached_token.data(), cached_token.size(), tmp_out.c_str(), token_per_sec, _attr.reserve);
cached_token.clear();
}
b_hit_eos = true;
break;
}
token_ids.push_back(max_index);
if (_attr.runing_callback)
{
cached_token.push_back(max_index);
if (cached_token.size() >= 3)
{
float t_cost_ms = t_cost.cost();
float token_per_sec = token_ids.size() / (t_cost_ms / 1000);
t_tmp.start();
auto tmp_out = tokenizer->Decode(cached_token);
decode_t_all += t_tmp.cost();
++decode_req_times;
final_out += tmp_out;
_attr.runing_callback(cached_token.data(), cached_token.size(), tmp_out.c_str(), token_per_sec, _attr.reserve);
cached_token.clear();
}
}
}
if (_attr.runing_callback == nullptr)
update_cqdm(&cqdm, indices, "token", "");
if (b_hit_eos)
{
break;
}
}
printf("\n\n");
fflush(stdout);
float t_cost_ms = t_cost.cost();
ALOGN("hit eos, avg %.2f token/s", token_ids.size() / (t_cost_ms / 1000));
if (!_attr.runing_callback)
{
t_tmp.start();
final_out = tokenizer->Decode(token_ids);
decode_t_all += t_tmp.cost();
}
ALOGN("decode cost: total %.0fms, avg %.0fms/req, avg %.0fms/token", decode_t_all, decode_t_all / decode_req_times, decode_t_all / token_ids.size());
// 去掉 len_of_input 那部分
// token_ids.erase(token_ids.begin(), token_ids.begin() + len_of_input);
for (int i = 0; i < _attr.axmodel_num; i++)
{
// memset(llama_layers[i].layer.get_input(prefill_grpid, "K_cache").pVirAddr, 0, llama_layers[i].layer.get_input(prefill_grpid, "K_cache").nSize);
// memset(llama_layers[i].layer.get_input(prefill_grpid, "V_cache").pVirAddr, 0, llama_layers[i].layer.get_input(prefill_grpid, "V_cache").nSize);
memset(llama_layers[i].layer.get_input(decode_grpid, "K_cache").pVirAddr, 0, llama_layers[i].layer.get_input(decode_grpid, "K_cache").nSize);
memset(llama_layers[i].layer.get_input(decode_grpid, "V_cache").pVirAddr, 0, llama_layers[i].layer.get_input(decode_grpid, "V_cache").nSize);
}
return final_out;
}
};
}; // namespace LLM_Qwen

364
components/llm/src/maixcam2/maix_vlm_smolvlm_maixcam2.cpp Normal file
View File

@@ -0,0 +1,364 @@
/**
* LLM SmolVLM implementation on MaixCam2
* @license Apache-2.0
* @author neucrack@sipeed
* @date 2025-06-03
*/
#include "maix_vlm_smolvlm.hpp"
#include "maix_nn.hpp"
#include "LLM_SmolVLM.hpp"
#include "ax_middleware.hpp"
#include "tokenizer_service_util.hpp"
namespace maix::nn
{
class SmolVLMObj
{
public:
MUD mud;
VLM_SmolVLM::LLM lLaMa;
maix::middleware::maixcam2::SYS *ax_sys;
maix::middleware::maixcam2::ENGINE *ax_engine;
std::vector<std::vector<unsigned short>> k_caches, v_caches;
int precompute_len = 0;
SmolVLMResp resp;
SmolVLM *obj;
int image_w;
int image_h;
maix::image::Format image_fmt;
std::vector<unsigned short> img_embed;
};
SmolVLM::SmolVLM(const std::string &model)
{
_data = new SmolVLMObj();
((SmolVLMObj*)_data)->obj = this;
_model_path = model;
_system_prompt = "You are SmolVLM. You are a helpful vision-to-text assistant.";
set_log_level(log::get_log_level(), log::get_log_use_color());
if(!model.empty())
{
err::Err e = load(model);
if(e != err::ERR_NONE)
{
char buf[128];
snprintf(buf, sizeof(buf), "load model %s failed", model.c_str());
throw err::Exception(e, buf);
}
}
}
SmolVLM::~SmolVLM()
{
unload();
if (_data)
{
delete (SmolVLMObj *)_data;
_data = nullptr;
}
}
static void _on_msg(int *p_token, int n_token, const char *p_str, float token_per_sec, void *reserve)
{
SmolVLMObj *obj = (SmolVLMObj *)reserve;
obj->resp.msg_new = p_str;
obj->resp.msg += p_str;
obj->resp.err_code = err::ERR_NONE;
obj->resp.err_msg = "";
auto callback = obj->obj->get_reply_callback();
if (callback)
{
callback(*obj->obj, obj->resp);
}
// fprintf(stdout, "%s", p_str);
// fflush(stdout);
}
void SmolVLM::set_log_level(log::LogLevel level, bool color)
{
ax_log_use_color = color;
switch(level)
{
case log::LogLevel::LEVEL_DEBUG:
ax_log_level = SAMPLE_LOG_DEBUG;
break;
case log::LogLevel::LEVEL_WARN:
ax_log_level = SAMPLE_LOG_WARN;
break;
case log::LogLevel::LEVEL_ERROR:
ax_log_level = SAMPLE_LOG_ERROR;
break;
default:
ax_log_level = SAMPLE_LOG_INFO;
break;
}
}
err::Err SmolVLM::load(const std::string &model)
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
_model_path = model;
err::Err e = obj->mud.load(model);
if(e != err::ERR_NONE)
return e;
std::string model_dir = fs::dirname(model);
// init llm model
VLM_SmolVLM::LLMAttrType attr;
attr.system_prompt = _system_prompt;
attr.tokenizer_type = TKT_HTTP;
bool ai_isp_on = app::get_sys_config_kv("npu", "ai_isp", "0") == "1" ? true : false;
if(ai_isp_on)
{
log::warn("npu_ai_isp_on from config is on, but LLM model only support npu model, please not use camera or turn off ai_isp");
}
try
{
_version = obj->mud.items["extra"]["model_type"];
_tokenizer_type = _version;
attr.url_tokenizer_model = obj->mud.items["extra"]["tokenizer_url"];
attr.filename_tokens_embed = fs::join({model_dir, obj->mud.items["extra"]["tokens_embed"]});
attr.filename_post_axmodel = fs::join({model_dir, obj->mud.items["extra"]["post_model"]});
attr.template_filename_axmodel = fs::join({model_dir, obj->mud.items["basic"]["model_npu"]});
attr.axmodel_num = std::stoi(obj->mud.items["extra"]["model_num"]);
attr.tokens_embed_num = std::stoi(obj->mud.items["extra"]["tokens_embed_num"]);
attr.tokens_embed_size = std::stoi(obj->mud.items["extra"]["tokens_embed_size"]);
attr.b_use_mmap_load_embed = (obj->mud.items["extra"]["use_mmap_load_embed"] == "true" || obj->mud.items["extra"]["use_mmap_load_embed"] == "1") ? true : false;
attr.filename_vpm_resampler_axmodedl = fs::join({model_dir, obj->mud.items["extra"]["vpm_resampler_model"]});
attr.vpm_len = std::stoi(obj->mud.items["extra"]["vpm_len"]);
}
catch(...)
{
log::error("load model failed, key-value error in mud's extra section");
return err::ERR_ARGS;
}
try
{
post_config.enable_temperature = obj->mud.items["post_config"]["enable_temperature"] == "true" ? true : false;
post_config.temperature = std::stof(obj->mud.items["post_config"]["temperature"]);
post_config.enable_repetition_penalty = obj->mud.items["post_config"]["enable_repetition_penalty"] == "true" ? true : false;
post_config.repetition_penalty = std::stof(obj->mud.items["post_config"]["repetition_penalty"]);
post_config.penalty_window = std::stoi(obj->mud.items["post_config"]["penalty_window"]);
post_config.enable_top_p_sampling = obj->mud.items["post_config"]["enable_top_p_sampling"] == "true" ? true : false;
post_config.top_p = std::stof(obj->mud.items["post_config"]["top_p"]);
post_config.enable_top_k_sampling = obj->mud.items["post_config"]["enable_top_k_sampling"] == "true" ? true : false;
post_config.top_k = std::stoi(obj->mud.items["post_config"]["top_k"]);
}
catch(...)
{
log::error("load model failed, key-value error in mud's post_config section");
return err::ERR_ARGS;
}
if(obj->mud.items["extra"].find("tokenizer_type") != obj->mud.items["extra"].end())
{
_tokenizer_type = obj->mud.items["extra"]["tokenizer_type"];
}
attr.runing_callback = nullptr;
attr.reserve = obj;
// init npu engine
log::info("init middleware SYSTEM");
obj->ax_sys = new middleware::maixcam2::SYS();
e = obj->ax_sys->init();
if(e != err::ERR_NONE)
{
log::error("init middleware SYSTEM failed: %s", err::to_str(e).c_str());
delete obj->ax_sys;
obj->ax_sys = nullptr;
return e;
}
log::info("init middleware NPU ENGINE");
obj->ax_engine = new middleware::maixcam2::ENGINE(AX_ENGINE_VIRTUAL_NPU_DISABLE);
e = obj->ax_engine->init();
if(e != err::ERR_NONE)
{
log::error("init middleware ENGINE failed: %s, maybe there's other program using NPU", err::to_str(e).c_str());
delete obj->ax_engine;
obj->ax_engine = nullptr;
delete obj->ax_sys;
obj->ax_sys = nullptr;
return e;
}
// check tokenizer service
// find http://127.0.0.1 in obj->mud.items["extra"]["tokenizer_url"]
e = check_start_tokenizer_service(obj->mud.items["extra"]["tokenizer_url"]);
if(e != err::ERR_NONE)
{
delete obj->ax_engine;
obj->ax_engine = nullptr;
delete obj->ax_sys;
obj->ax_sys = nullptr;
return e;
}
// init llm model
QwenPostConfig config;
config.enable_temperature = post_config.enable_temperature;
config.temperature = post_config.temperature;
config.enable_repetition_penalty = post_config.enable_repetition_penalty;
config.repetition_penalty = post_config.repetition_penalty;
config.penalty_window = post_config.penalty_window;
config.enable_top_p_sampling = post_config.enable_top_p_sampling;
config.top_p = post_config.top_p;
config.enable_top_k_sampling = post_config.enable_top_k_sampling;
config.top_k = post_config.top_k;
if(!obj->lLaMa.Init(attr, config, _tokenizer_type, obj->image_w, obj->image_h))
{
log::error("SmolVLM Init failed");
delete obj->ax_engine;
obj->ax_engine = nullptr;
delete obj->ax_sys;
obj->ax_sys = nullptr;
return err::ERR_RUNTIME;
}
obj->image_fmt = maix::image::Format::FMT_RGB888;
// print params
log::info("model info:");
log::print(log::LogLevel::LEVEL_INFO, "\tmodel type: %s\n", _version.c_str());
log::print(log::LogLevel::LEVEL_INFO, "\tmodel path: %s\n", obj->mud.items["basic"]["model_npu"].c_str());
log::print(log::LogLevel::LEVEL_INFO, "\tpost model path: %s\n", obj->mud.items["extra"]["post_model"].c_str());
log::print(log::LogLevel::LEVEL_INFO, "\ttokens embed path: %s\n", obj->mud.items["extra"]["tokens_embed"].c_str());
log::print(log::LogLevel::LEVEL_INFO, "\tuse_mmap_load_embed: %s\n", obj->mud.items["extra"]["use_mmap_load_embed"].c_str());
log::print(log::LogLevel::LEVEL_INFO, "\tmodel num: %d\n", attr.axmodel_num);
log::print(log::LogLevel::LEVEL_INFO, "\ttokens embed num: %d\n", attr.tokens_embed_num);
log::print(log::LogLevel::LEVEL_INFO, "\ttokens embed size: %d\n", attr.tokens_embed_size);
log::print(log::LogLevel::LEVEL_INFO, "\ttokenizer url: %s\n", attr.url_tokenizer_model.c_str());
log::print(log::LogLevel::LEVEL_INFO, "\tinput image size: %d x %d\n", obj->image_w, obj->image_h);
log::print(log::LogLevel::LEVEL_INFO, "\tinput image format: %s\n", maix::image::format_name(obj->image_fmt).c_str());
log::print(log::LogLevel::LEVEL_INFO, "\n");
_loaded = true;
return err::ERR_NONE;
}
err::Err SmolVLM::unload()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
obj->lLaMa.Stop();
obj->lLaMa.Deinit();
if(obj->ax_engine)
{
delete obj->ax_engine;
obj->ax_engine = nullptr;
}
if(obj->ax_sys)
{
delete obj->ax_sys;
obj->ax_sys = nullptr;
}
_loaded = false;
return err::ERR_NONE;
}
void SmolVLM::set_system_prompt(const std::string &prompt)
{
_system_prompt = prompt;
if(_loaded)
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
std::vector<int> _token_ids;
obj->lLaMa.SetSystemPrompt(_system_prompt, _token_ids);
}
}
int SmolVLM::input_width()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
return obj->image_w;
}
int SmolVLM::input_height()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
return obj->image_h;
}
maix::image::Format SmolVLM::input_format()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
return obj->image_fmt;
}
err::Err SmolVLM::set_image(maix::image::Image &img, maix::image::Fit fit)
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
maix::image::Image *p_img = &img;
bool need_free = false;
if(img.width() != obj->image_w || img.height() != obj->image_h)
{
p_img = img.resize(obj->image_w, obj->image_h, fit);
need_free = true;
}
int ret = obj->lLaMa.Encode(*p_img, obj->img_embed);
if (need_free)
delete p_img;
if(ret != 0)
{
log::error("Encode image failed, ret: %d", ret);
return err::ERR_RUNTIME;
}
return err::ERR_NONE;
}
void SmolVLM::clear_image()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
obj->img_embed.clear();
}
bool SmolVLM::is_image_set()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
return !obj->img_embed.empty();
}
nn::SmolVLMResp SmolVLM::send(const std::string &msg)
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
obj->resp.msg = "";
obj->resp.err_code = err::ERR_NONE;
obj->resp.err_msg = "";
if(msg.empty())
{
obj->resp.err_code = err::ERR_ARGS;
obj->resp.err_msg = "msg is empty";
return obj->resp;
}
// check callback
auto attr = obj->lLaMa.getAttr();
if(_callback)
{
attr->runing_callback = _on_msg;
}
else
{
attr->runing_callback = nullptr;
}
// run LLM model
std::vector<unsigned short> prompt_data;
int ret = obj->lLaMa.Encode(prompt_data, obj->img_embed, msg);
if(ret != 0)
{
log::error("Encode msg failed, ret: %d", ret);
obj->resp.err_code = err::ERR_RUNTIME;
obj->resp.err_msg = "Encode msg failed";
return obj->resp;
}
/*obj->resp.msg = */obj->lLaMa.Run(prompt_data);
return obj->resp;
}
void SmolVLM::cancel()
{
SmolVLMObj *obj = (SmolVLMObj *)_data;
obj->lLaMa.Stop();
}
} // namespace maix::nn

BIN
components/maixcam_lib/lib_maixcam2/libmaixcam_lib.so
View File

Binary file not shown.

9
examples/nn_vlm_smolvlm/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
build
dist
.config.mk
.flash.conf.json
data
/CMakeLists.txt
__pycache__

3
examples/nn_vlm_smolvlm/README.md Normal file
View File

@@ -0,0 +1,3 @@
VLM SmolVLM Project based on MaixCDK
====

74
examples/nn_vlm_smolvlm/main/CMakeLists.txt Normal file
View File

@@ -0,0 +1,74 @@
############### Add include ###################
list(APPEND ADD_INCLUDE "include"
)
list(APPEND ADD_PRIVATE_INCLUDE "")
###############################################
############ Add source files #################
# list(APPEND ADD_SRCS "src/main.c"
# "src/test.c"
# )
append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test2.c")
# FILE(GLOB_RECURSE EXTRA_SRC "src/*.c")
# FILE(GLOB EXTRA_SRC "src/*.c")
# list(APPEND ADD_SRCS ${EXTRA_SRC})
# aux_source_directory(src ADD_SRCS) # collect all source file in src dir, will set var ADD_SRCS
# append_srcs_dir(ADD_SRCS "src") # append source file in src dir to var ADD_SRCS
# list(REMOVE_ITEM COMPONENT_SRCS "src/test.c")
# set(ADD_ASM_SRCS "src/asm.S")
# list(APPEND ADD_SRCS ${ADD_ASM_SRCS})
# SET_PROPERTY(SOURCE ${ADD_ASM_SRCS} PROPERTY LANGUAGE C) # set .S ASM file as C language
# SET_SOURCE_FILES_PROPERTIES(${ADD_ASM_SRCS} PROPERTIES COMPILE_FLAGS "-x assembler-with-cpp -D BBBBB")
###############################################
###### Add required/dependent components ######
list(APPEND ADD_REQUIREMENTS basic llm)
###############################################
###### Add link search path for requirements/libs ######
# list(APPEND ADD_LINK_SEARCH_PATH "${CONFIG_TOOLCHAIN_PATH}/lib")
# list(APPEND ADD_REQUIREMENTS pthread m) # add system libs, pthread and math lib for example here
# set (OpenCV_DIR opencv/lib/cmake/opencv4)
# find_package(OpenCV REQUIRED)
###############################################
############ Add static libs ##################
# list(APPEND ADD_STATIC_LIB "lib/libtest.a")
###############################################
#### Add compile option for this component ####
#### Just for this component, won't affect other
#### modules, including component that depend
#### on this component
# list(APPEND ADD_DEFINITIONS_PRIVATE -DAAAAA=1)
#### Add compile option for this component
#### and components depend on this component
# list(APPEND ADD_DEFINITIONS -DAAAAA222=1
# -DAAAAA333=1)
###############################################
############ Add static libs ##################
#### Update parent's variables like CMAKE_C_LINK_FLAGS
# set(CMAKE_C_LINK_FLAGS "${CMAKE_C_LINK_FLAGS} -Wl,--start-group libmaix/libtest.a -ltest2 -Wl,--end-group" PARENT_SCOPE)
###############################################
######### Add files need to download #########
# list(APPEND ADD_FILE_DOWNLOADS "{
# 'url': 'https://*****/abcde.tar.xz',
# 'urls': [], # backup urls, if url failed, will try urls
# 'sites': [], # download site, user can manually download file and put it into dl_path
# 'sha256sum': '',
# 'filename': 'abcde.tar.xz',
# 'path': 'toolchains/xxxxx',
# 'check_files': []
# }"
# )
#
# then extracted file in ${DL_EXTRACTED_PATH}/toolchains/xxxxx,
# you can directly use then, for example use it in add_custom_command
##############################################
# register component, DYNAMIC or SHARED flags will make component compiled to dynamic(shared) lib
register_component()

0
examples/nn_vlm_smolvlm/main/Kconfig Normal file
View File

3
examples/nn_vlm_smolvlm/main/include/main.h Normal file
View File

@@ -0,0 +1,3 @@
#pragma once

112
examples/nn_vlm_smolvlm/main/src/main.cpp Normal file
View File

@@ -0,0 +1,112 @@
#include "maix_basic.hpp"
#include "main.h"
#include "maix_vlm_smolvlm.hpp"
#include <iostream>
using namespace maix;
void on_reply(nn::SmolVLM &obj, const nn::SmolVLMResp &resp)
{
printf("%s", resp.msg_new.c_str());
fflush(stdout);
}
int _main(int argc, char* argv[])
{
log::info("Program start");
if (argc < 2)
{
log::error("Usage: %s <model_path(.mud)> [system prompt]", argv[0]);
return -1;
}
std::string model_path = argv[1];
log::set_log_level(log::LogLevel::LEVEL_INFO, true);
nn::SmolVLM smolvlm(model_path);
const char *system_prompt = "You are SmolVLM, a multimodal model and a helpful vision-to-text assistant.";
if (argc > 2)
{
system_prompt = argv[2];
}
log::info("System prompt: %s", system_prompt);
smolvlm.set_system_prompt(system_prompt);
smolvlm.set_reply_callback(on_reply);
log::info("Input image path and prompt, you can set image once and ask multiple times.");
log::info("'q' to quit");
std::string image_path;
while(!app::need_exit())
{
// 1. input image path
std::string input;
printf("image path: >> ");
fflush(stdout);
std::getline(std::cin, input);
if (input == "exit" || input == "quit" || input == "q")
break;
if(!input.empty())
{
image::Image *img = image::load(input);
if(!img)
{
log::error("load image %s failed", input.c_str());
continue;
}
err::Err e = smolvlm.set_image(*img, maix::image::Fit::FIT_CONTAIN);
delete img;
if(e != err::Err::ERR_NONE)
{
log::error("set image failed, error: %s", err::to_str(e).c_str());
continue;
}
image_path = input;
}
else if(!smolvlm.is_image_set())
{
log::info("image not set, only text input mode");
}
else
{
log::info("image remain use last set: %s", image_path.c_str());
}
// 2. input prompt
std::string prompt;
printf("prompt: >> ");
fflush(stdout);
std::getline(std::cin, prompt);
if (prompt.empty())
continue;
if (prompt == "exit" || prompt == "quit" || prompt == "q")
break;
// send message to InterVL
nn::SmolVLMResp resp = smolvlm.send(prompt);
if (resp.err_code != err::Err::ERR_NONE)
{
if(resp.err_code == err::Err::ERR_BUFF_FULL)
{
log::error("context buffer full, please clear context");
continue;
}
log::error("Error: %s, %s", err::to_str(resp.err_code).c_str(), resp.err_msg.c_str());
break;
}
// log::info("%s", resp.msg.c_str());
}
log::info("Program exit");
return 0;
}
int main(int argc, char* argv[])
{
// Catch signal and process
sys::register_default_signal_handle();
// Use CATCH_EXCEPTION_RUN_RETURN to catch exception,
// if we don't catch exception, when program throw exception, the objects will not be destructed.
// So we catch exception here to let resources be released(call objects' destructor) before exit.
CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv);
}