From c999a883bd2cfa0353b9ffb086ac6e445e3676c8 Mon Sep 17 00:00:00 2001 From: watermeko Date: Thu, 23 Jul 2026 04:56:17 +0000 Subject: [PATCH] perf(sg2002): integrate zero-copy H.264 capture Move frame ownership and zero-copy VENC handling into the tracked MMF and Vision sources instead of relying on a preload hook. Keep the sensor name mapping aligned with middleware enums and release deferred VI frames safely across encoder lifecycle changes. --- .../additional/kvm_mmf/include/kvm_mmf.hpp | 3 + .../sg2002/additional/kvm_mmf/src/kvm_mmf.cpp | 123 +++++++++++++++--- .../v2/sample/common/sample_common_sensor.c | 4 + .../vision/port/maixcam/maix_camera_mmf.hpp | 22 +++- 4 files changed, 132 insertions(+), 20 deletions(-) diff --git a/support/sg2002/additional/kvm_mmf/include/kvm_mmf.hpp b/support/sg2002/additional/kvm_mmf/include/kvm_mmf.hpp index b98dd12..0636795 100644 --- a/support/sg2002/additional/kvm_mmf/include/kvm_mmf.hpp +++ b/support/sg2002/additional/kvm_mmf/include/kvm_mmf.hpp @@ -46,6 +46,9 @@ void mmf_get_vi_vflip(int ch, bool *en); // get vi frame int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int *format); void mmf_vi_frame_free(int ch); +// Release the current VI frame immediately. mmf_vi_frame_free() defers +// release so the frame can be sent directly to VENC without a second copy. +void mmf_vi_frame_release(int ch); // invert format int mmf_invert_format_to_maix(int mmf_format); diff --git a/support/sg2002/additional/kvm_mmf/src/kvm_mmf.cpp b/support/sg2002/additional/kvm_mmf/src/kvm_mmf.cpp index 89318bd..90de675 100644 --- a/support/sg2002/additional/kvm_mmf/src/kvm_mmf.cpp +++ b/support/sg2002/additional/kvm_mmf/src/kvm_mmf.cpp @@ -134,6 +134,12 @@ typedef struct { uint8_t h265_or_h264_is_used; mmf_vb_pool_t vb_pool[VB_MAX_COMM_POOLS]; + + // Keep all existing member offsets stable for compatibility with the + // vendor middleware; zero-copy state is appended at the end. + bool vi_frame_valid[MMF_VI_MAX_CHN]; + bool vi_frame_mapped[MMF_VI_MAX_CHN]; + bool vi_frame_deferred[MMF_VI_MAX_CHN]; } priv_t; typedef struct { @@ -147,6 +153,60 @@ static g_priv_t g_priv; #define MODULE_NAME "soph_vi" +/* + * Keep the former zero-copy hook's frame lifetime in the MMF implementation. + * A frame released by the image path stays leased until VENC has consumed it. + */ +static void _mmf_release_vi_frame(int ch) +{ + if (ch < 0 || ch >= MMF_VI_MAX_CHN || !priv.vi_frame_valid[ch]) { + return; + } + + VIDEO_FRAME_INFO_S *frame = &priv.vi_frame[ch]; + if (priv.vi_frame_mapped[ch] && frame->stVFrame.pu8VirAddr[0]) { + uint32_t image_size = frame->stVFrame.u32Length[0] + + frame->stVFrame.u32Length[1] + + frame->stVFrame.u32Length[2]; + CVI_SYS_Munmap(frame->stVFrame.pu8VirAddr[0], image_size); + frame->stVFrame.pu8VirAddr[0] = NULL; + } + + if (CVI_VPSS_ReleaseChnFrame(0, ch, frame) != CVI_SUCCESS) { + SAMPLE_PRT("CVI_VPSS_ReleaseChnFrame failed for ch %d\n", ch); + } + + memset(frame, 0, sizeof(*frame)); + priv.vi_frame_valid[ch] = false; + priv.vi_frame_mapped[ch] = false; + priv.vi_frame_deferred[ch] = false; +} + +static void _mmf_release_all_vi_frames(void) +{ + for (int ch = 0; ch < MMF_VI_MAX_CHN; ++ch) { + _mmf_release_vi_frame(ch); + } +} + +static int _mmf_find_deferred_vi_frame(int width, int height, int format, + VIDEO_FRAME_INFO_S **frame_out) +{ + for (int ch = 0; ch < MMF_VI_MAX_CHN; ++ch) { + if (!priv.vi_frame_valid[ch] || !priv.vi_frame_deferred[ch]) { + continue; + } + VIDEO_FRAME_INFO_S *frame = &priv.vi_frame[ch]; + if ((int)frame->stVFrame.u32Width == width + && (int)frame->stVFrame.u32Height == height + && (int)frame->stVFrame.enPixelFormat == format) { + *frame_out = frame; + return ch; + } + } + return -1; +} + static int _is_module_in_use(const char *module_name) { FILE *fp; char buffer[256]; @@ -293,9 +353,9 @@ static int _free_leak_memory_of_ion(void) while (fgets(line, MAX_LINE_LENGTH, fp) != NULL) { if (sscanf(line, "%*d %s %s %*d %s", alloc_buf_size_str, phy_addr_str, buffer_name) == 3) { printf("[ION] %s %s %s\r\n", alloc_buf_size_str, phy_addr_str, buffer_name); - // FIXME: release jpeg_ion - if (strcmp(buffer_name, "VI_DMA_BUF") - && strcmp(buffer_name, "ISP_SHARED_BUFFER_0")) + // ISP_SHARED_BUFFER_0 may still be owned by kvm_system while the + // server starts. Releasing it here makes SAMPLE_PLAT_VI_INIT fail. + if (strcmp(buffer_name, "VI_DMA_BUF")) continue; struct sys_ion_data_new ion_data = { .cached = 1, @@ -1246,6 +1306,7 @@ int mmf_vi_deinit(void) return 0; } + _mmf_release_all_vi_frames(); CVI_S32 s32Ret = CVI_SUCCESS; s32Ret = _mmf_vpss_deinit_new(0); if (s32Ret != CVI_SUCCESS) { @@ -1297,7 +1358,7 @@ static int _mmf_add_vi_channel(int ch, int width, int height, int format) { } CVI_S32 s32Ret = CVI_SUCCESS; - int fps = 30; + int fps = 60; int depth = 2; int width_out = ALIGN(width, DEFAULT_ALIGN); int height_out = height; @@ -1372,6 +1433,7 @@ int mmf_del_vi_channel(int ch) { } CVI_S32 s32Ret = CVI_SUCCESS; + _mmf_release_vi_frame(ch); s32Ret = SAMPLE_COMM_VI_UnBind_VPSS(0, ch, 0); if (s32Ret != CVI_SUCCESS) { SAMPLE_PRT("vi unbind vpss failed. s32Ret: 0x%x !\n", s32Ret); @@ -1418,6 +1480,10 @@ int mmf_vi_aligned_width(int ch) { } int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int *format) { + if (ch < 0 || ch >= MMF_VI_MAX_CHN) { + printf("[%d] invalid ch %d\n", __LINE__, ch); + return -1; + } if (!priv.vi_chn_is_inited[ch]) { // printf("vi ch %d not open\n", ch); return -1; @@ -1433,6 +1499,8 @@ int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int int ret = -1; VIDEO_FRAME_INFO_S *frame = &priv.vi_frame[ch]; + _mmf_release_vi_frame(ch); + memset(frame, 0, sizeof(*frame)); if (CVI_VPSS_GetChnFrame(0, ch, frame, 1000) == 0) { int image_size = frame->stVFrame.u32Length[0] + frame->stVFrame.u32Length[1] @@ -1442,6 +1510,9 @@ int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int CVI_SYS_IonInvalidateCache(frame->stVFrame.u64PhyAddr[0], vir_addr, image_size); frame->stVFrame.pu8VirAddr[0] = (CVI_U8 *)vir_addr; // save virtual address for munmap + priv.vi_frame_valid[ch] = true; + priv.vi_frame_mapped[ch] = true; + priv.vi_frame_deferred[ch] = false; // printf("width: %d, height: %d, total_buf_length: %d, phy:%#lx vir:%p\n", // frame->stVFrame.u32Width, // frame->stVFrame.u32Height, image_size, @@ -1458,14 +1529,17 @@ int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int } void mmf_vi_frame_free(int ch) { - VIDEO_FRAME_INFO_S *frame = &priv.vi_frame[ch]; - int image_size = frame->stVFrame.u32Length[0] - + frame->stVFrame.u32Length[1] - + frame->stVFrame.u32Length[2]; - CVI_SYS_Munmap(frame->stVFrame.pu8VirAddr[0], image_size); - if (CVI_VPSS_ReleaseChnFrame(0, ch, frame) != 0) { - SAMPLE_PRT("CVI_VI_ReleaseChnFrame NG\n"); + if (ch < 0 || ch >= MMF_VI_MAX_CHN || !priv.vi_frame_valid[ch]) { + return; } + priv.vi_frame_deferred[ch] = true; +} + +void mmf_vi_frame_release(int ch) { + if (ch < 0 || ch >= MMF_VI_MAX_CHN) { + return; + } + _mmf_release_vi_frame(ch); } int mmf_region_frame_push(int ch, void *data, int len) @@ -2211,6 +2285,23 @@ int mmf_venc_push(int ch, uint8_t *data, int w, int h, int format) { return -1; } + /* + * CameraCviMmf can return an Image backed by the mapped VPSS buffer. + * When that frame is deferred, send it directly to VENC and skip the + * Image -> VENC buffer copy. Non-camera callers still use the old path. + */ + if (info->type == 2) { + VIDEO_FRAME_INFO_S *vi_frame = NULL; + if (_mmf_find_deferred_vi_frame(w, h, format, &vi_frame) >= 0) { + s32Ret = CVI_VENC_SendFrame(ch, vi_frame, 1000); + if (s32Ret == CVI_SUCCESS) { + info->is_running = 1; + return 0; + } + printf("CVI_VENC_SendFrame zero-copy failed with %#x, fallback to copy\n", s32Ret); + } + } + switch (format) { case PIXEL_FORMAT_NV21: { @@ -2329,13 +2420,16 @@ int mmf_venc_free(int ch) { venc_info_t *info = (venc_info_t *)&priv.venc[ch]; VENC_STREAM_S *venc_stream = (VENC_STREAM_S *)&priv.venc[ch].capture_stream; if (!info->is_running) { + // Encoder reinitialization happens after CameraCviMmf has already + // acquired the next VI frame. With no submitted VENC frame there is + // nothing to release here; releasing all VI frames would invalidate + // the Image that is about to be sent to the new encoder. return s32Ret; } s32Ret = CVI_VENC_ReleaseStream(ch, venc_stream); if (s32Ret != CVI_SUCCESS) { printf("CVI_VENC_ReleaseStream failed with %#x\n", s32Ret); - return s32Ret; } if (venc_stream->pstPack) { @@ -2344,9 +2438,6 @@ int mmf_venc_free(int ch) { } info->is_running = 0; + _mmf_release_all_vi_frames(); return s32Ret; } - - - - \ No newline at end of file diff --git a/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c b/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c index 0044ad2..84738da 100644 --- a/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c +++ b/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c @@ -88,6 +88,7 @@ static const char *snsr_type_name[SAMPLE_SNS_TYPE_BUTT] = { "BYD_BF2253L_MIPI_1200P_30FPS_10BIT", "CVSENS_CV4001_MIPI_4M_1440P_25FPS_12BIT", "GCORE_GC02M1_MIPI_2M_30FPS_10BIT", + "GCORE_GC030A_MIPI_480P_30FPS_10BIT", "GCORE_GC0312_MIPI_480P_20FPS_8BIT", "GCORE_GC0329_MIPI_480P_10FPS_8BIT", "GCORE_GC1054_MIPI_1M_30FPS_10BIT", @@ -219,6 +220,9 @@ static const char *snsr_type_name[SAMPLE_SNS_TYPE_BUTT] = { "VIVO_MCS369Q_4M_30FPS_12BIT", "VIVO_MM308M2_2M_25FPS_8BIT", "LONTIUM_LT6911_2M_60FPS_8BIT", + "GCORE_GC4653_MIPI_720P_60FPS_10BIT", + "GCORE_OV2685_MIPI_1600x1200_30FPS_10BIT", + "OV_OS04A10_MIPI_4M_720P90_12BIT", /* ------ LINEAR END ------*/ /* ------ WDR 2TO1 BEGIN ------*/ diff --git a/support/sg2002/additional/vision/port/maixcam/maix_camera_mmf.hpp b/support/sg2002/additional/vision/port/maixcam/maix_camera_mmf.hpp index 7de5bd8..2f273b4 100644 --- a/support/sg2002/additional/vision/port/maixcam/maix_camera_mmf.hpp +++ b/support/sg2002/additional/vision/port/maixcam/maix_camera_mmf.hpp @@ -161,16 +161,30 @@ namespace maix::camera if (0 == mmf_vi_frame_pop(this->ch, &buffer, &buffer_len, &width, &height, &format)) { if (buffer == NULL) { - mmf_vi_frame_free(this->ch); + mmf_vi_frame_release(this->ch); printf("mmf_vi_frame_free error\r\n"); return NULL; } + bool native_frame = !buff + && this->format == image::FMT_YVU420SP + && !this->align_need + && width == this->width + && height == this->height + && format == mmf_invert_format_to_mmf(this->format) + && buffer_len >= this->width * this->height * 3 / 2; + if (native_frame) { + img = new image::Image(this->width, this->height, + this->format, (uint8_t *)buffer, + this->width * this->height * 3 / 2, false); + mmf_vi_frame_free(this->ch); + return img; + } if(buff) { if(buff_size < (size_t)buffer_len) { log::error("camera read: buff size not enough, need %d, but %d", buffer_len, buff_size); - mmf_vi_frame_free(this->ch); + mmf_vi_frame_release(this->ch); return NULL; } img = new image::Image(width, height, this->format, (uint8_t*)buff, buff_size, false); @@ -212,11 +226,11 @@ namespace maix::camera default: printf("unknown format\n"); delete img; - mmf_vi_frame_free(this->ch); + mmf_vi_frame_release(this->ch); printf("switch (img->format()\r\n"); return NULL; } - mmf_vi_frame_free(this->ch); + mmf_vi_frame_release(this->ch); // printf("mmf_vi_frame_free\r\n"); return img; }