From fa498da900ffced8a09d79793b43d07f5bacf65e Mon Sep 17 00:00:00 2001 From: lxowalle Date: Mon, 7 Sep 2026 10:27:50 +0800 Subject: [PATCH] feat(maixcam2): support OS04A10 high frame rates --- .../include/maixcam2/ax_middleware.hpp | 96 +++- components/vision/include/maix_camera.hpp | 47 +- .../port/maixcam2/maix_camera_maixcam2.cpp | 411 ++++++++++++++++-- examples/camera_display/main/src/main.cpp | 25 +- examples/os04a10_sensor_crop_test/.gitignore | 2 + examples/os04a10_sensor_crop_test/app.yaml | 8 + .../main/CMakeLists.txt | 3 + .../main/src/main.cpp | 111 +++++ examples/video_encode_demo/main/src/main.cpp | 260 +++++++++-- 9 files changed, 874 insertions(+), 89 deletions(-) create mode 100644 examples/os04a10_sensor_crop_test/.gitignore create mode 100644 examples/os04a10_sensor_crop_test/app.yaml create mode 100644 examples/os04a10_sensor_crop_test/main/CMakeLists.txt create mode 100644 examples/os04a10_sensor_crop_test/main/src/main.cpp diff --git a/components/maixcam_lib/include/maixcam2/ax_middleware.hpp b/components/maixcam_lib/include/maixcam2/ax_middleware.hpp index 3a89b2c6..1abb705f 100644 --- a/components/maixcam_lib/include/maixcam2/ax_middleware.hpp +++ b/components/maixcam_lib/include/maixcam2/ax_middleware.hpp @@ -475,9 +475,9 @@ namespace maix::middleware::maixcam2 { case AX_FORMAT_YUV400: return image::FMT_GRAYSCALE; case AX_FORMAT_RGB888: - return image::FMT_BGR888; // actualy is rgb888 + return image::FMT_BGR888; // AX RGB888 stores B at the lowest address case AX_FORMAT_BGR888: - return image::FMT_RGB888; // actualy is bgr888 + return image::FMT_RGB888; // AX BGR888 stores R at the lowest address case AX_FORMAT_ARGB8888: return image::FMT_BGRA8888; // actualy is rgba8888 case AX_FORMAT_ABGR8888: @@ -497,9 +497,9 @@ namespace maix::middleware::maixcam2 { case image::FMT_GRAYSCALE: return AX_FORMAT_YUV400; case image::FMT_RGB888: - return AX_FORMAT_BGR888; // actualy is rgb888 + return AX_FORMAT_BGR888; case image::FMT_BGR888: - return AX_FORMAT_RGB888; // actualy is bgr888 + return AX_FORMAT_RGB888; case image::FMT_RGBA8888: return AX_FORMAT_ABGR8888; // actualy is rgba8888 case image::FMT_BGRA8888: @@ -540,6 +540,14 @@ namespace maix::middleware::maixcam2 { AX_VIN_IVPS_MODE_E eMode; AX_IVPS_ROTATION_E eRotAngle; AX_U32 statDeltaPtsFrmNum; + AX_BOOL bSensorCrop; + AX_S32 nSensorCropX; + AX_S32 nSensorCropY; + AX_S32 nSensorCropW; + AX_S32 nSensorCropH; + AX_S32 nSensorWidth; + AX_S32 nSensorHeight; + AX_S32 nSensorFps; } SAMPLE_VIN_PARAM_T; /* comm pool */ @@ -961,6 +969,43 @@ namespace maix::middleware::maixcam2 { } } + static AX_VOID __apply_os04a10_crop_attrs(AX_CAMERA_T *pCam, const SAMPLE_VIN_PARAM_T *pVinParam) + { + if (pVinParam->nSensorWidth <= 0 || pVinParam->nSensorHeight <= 0) { + return; + } + const AX_U32 w = (AX_U32)pVinParam->nSensorWidth; + const AX_U32 h = (AX_U32)pVinParam->nSensorHeight; + const AX_BAYER_PATTERN_E bayer_pattern = AX_BP_RGGB; + pCam->tSnsAttr.nWidth = w; + pCam->tSnsAttr.nHeight = h; + pCam->tSnsAttr.fFrameRate = pVinParam->nSensorFps; + pCam->tSnsAttr.eBayerPattern = bayer_pattern; + pCam->tDevAttr.eBayerPattern = bayer_pattern; + pCam->tDevAttr.tDevImgRgn[0] = {0, 0, w, h}; + pCam->tDevAttr.tDevImgRgn[1] = {0, 0, w, h}; + pCam->tDevAttr.tDevImgRgn[2] = {0, 0, w, h}; + pCam->tDevAttr.tDevImgRgn[3] = {0, 0, w, h}; + pCam->tPipeAttr[pCam->nPipeId].tPipeImgRgn = {0, 0, w, h}; + pCam->tPipeAttr[pCam->nPipeId].nWidthStride = w; + pCam->tPipeAttr[pCam->nPipeId].eBayerPattern = bayer_pattern; + for (AX_U32 chn = 0; chn < AX_VIN_CHN_ID_MAX; ++chn) { + if (pCam->tChnAttr[chn].nWidth > w) { + pCam->tChnAttr[chn].nWidth = w; + /* VIN FBC channels require a 128-pixel stride. The native + * 2688-wide mode happened to satisfy that constraint, while + * the 1344-wide binned mode did not and produced corrupted + * blocks in the YUV output despite a clean IFE RAW frame. */ + const AX_U32 stride_align = + pCam->tChnAttr[chn].tCompressInfo.enCompressMode == AX_COMPRESS_MODE_NONE ? 2U : 128U; + pCam->tChnAttr[chn].nWidthStride = (w + stride_align - 1U) & ~(stride_align - 1U); + } + if (pCam->tChnAttr[chn].nHeight > h) { + pCam->tChnAttr[chn].nHeight = h; + } + } + } + #ifdef __cplusplus extern "C" { #endif @@ -1041,9 +1086,11 @@ namespace maix::middleware::maixcam2 { AX_U32 j = 0; pCommonArgs->nCamCnt = 1; pCam = &pCamList[0]; + pCam->nPipeId = 0; __vi_get_sns_config(eSnsType, &pCam->tMipiAttr, &pCam->tSnsAttr, &pCam->tSnsClkAttr, &pCam->tDevAttr, &pCam->tPipeAttr[pCam->nPipeId], pCam->tChnAttr); + __apply_os04a10_crop_attrs(pCam, pVinParam); pCam->nDevId = 0; pCam->nRxDev = 0; pCam->nPipeId = 0; @@ -1469,11 +1516,39 @@ namespace maix::middleware::maixcam2 { return 0; } - static SAMPLE_VIN_CASE_E __get_vi_case(char *sensor_name, int &w, int &h, int &fps) { + static SAMPLE_VIN_CASE_E __get_vi_case(char *sensor_name, int &w, int &h, int &fps, + int crop_w = -1, int crop_h = -1) { if (strcmp(sensor_name, "os04a10") == 0) { - w = 2688; - h = 1520; - fps = 30; + const bool has_crop = crop_w > 0 && crop_h > 0; + if (has_crop) { + w = crop_w; + h = crop_h; + const int max_crop_fps = crop_w <= 640 && crop_h <= 360 ? 360 : + crop_w <= 1344 && crop_h <= 760 ? 180 : 60; + /* High-speed crop tiers use the sensor's 108 MHz PLL timing + * (derived from the public Ambarella 90-fps register table). + * The OS04A10 driver enables 2x2 binning first for these + * tiers; the 640x360 tier then additionally crops the + * binned readout window. */ + /* A non-positive fps means "auto": use the conventional + * 60 fps timing. Preserve an explicit request such as + * 30 fps instead of silently promoting it to the tier limit. */ + if (fps <= 0) { + fps = 60; + } else if (fps > max_crop_fps) { + log::error("OS04A10 %dx%d sensor crop supports up to %d fps, requested %d", + crop_w, crop_h, max_crop_fps, fps); + return SAMPLE_VIN_NONE; + } + } else if (fps <= 0 || (fps > 30 && fps <= 60)) { + w = 2688; + h = 1520; + fps = 60; + } else { + w = 2688; + h = 1520; + fps = 30; + } return SAMPLE_VIN_SINGLE_OS04A10; } else if (strcmp(sensor_name, "sc450ai") == 0) { w = 2688; @@ -1836,8 +1911,9 @@ namespace maix::middleware::maixcam2 { return __get_sensor_name(); } - SAMPLE_VIN_CASE_E get_vi_case(char *sensor_name, int &w, int &h, int &fps) { - return __get_vi_case(sensor_name, w, h, fps); + SAMPLE_VIN_CASE_E get_vi_case(char *sensor_name, int &w, int &h, int &fps, + int crop_w = -1, int crop_h = -1) { + return __get_vi_case(sensor_name, w, h, fps, crop_w, crop_h); } AX_U32 config_sample_case(SAMPLE_VIN_PARAM_T *pVinParam, COMMON_SYS_ARGS_T *pCommonArgs, diff --git a/components/vision/include/maix_camera.hpp b/components/vision/include/maix_camera.hpp index d71badfe..075d7f27 100644 --- a/components/vision/include/maix_camera.hpp +++ b/components/vision/include/maix_camera.hpp @@ -85,7 +85,8 @@ namespace maix::camera * @param height camera height, default is -1, means auto, mostly means max height of camera support * @param format camera output format, default is image.Format.FMT_RGB888 * @param device camera device path, you can get devices by list_devices method, by default(value is NULL(None in MaixPy)) means the first device - * @param fps camera fps, default is -1, means auto, mostly means max fps of camera support + * @param fps camera fps, default is -1, means automatic supported timing. + * On MaixCAM2 with OS04A10, automatic timing is 60 FPS. * @param buff_num camera buffer number, default is 3, means 3 buffer, one used by user, one used for cache the next frame, * more than one buffer will accelerate image read speed, but will cost more memory. * @param open If true, camera will automatically call open() after creation. default is true. @@ -108,7 +109,8 @@ namespace maix::camera * @param width camera width, default is -1, means auto, mostly means max width of camera support * @param height camera height, default is -1, means auto, mostly means max height of camera support * @param format camera output format, default same as the constructor's format argument - * @param fps camera fps, default is -1, means auto, mostly means max fps of camera support + * @param fps camera fps, default is -1, means automatic supported timing. + * On MaixCAM2 with OS04A10, automatic timing is 60 FPS. * @param buff_num camera buffer number, default is 3, means 3 buffer, one used by user, one used for cache the next frame, * more than one buffer will accelerate image read speed, but will cost more memory. * @return error code, err::ERR_NONE means success, others means failed @@ -178,7 +180,8 @@ namespace maix::camera * @param width camera width, default is -1, means auto, mostly means max width of camera support * @param height camera height, default is -1, means auto, mostly means max height of camera support * @param format camera output format, default is RGB888 - * @param fps camera fps, default is -1, means auto, mostly means max fps of camera support + * @param fps camera fps, default is -1, means automatic supported timing. + * On MaixCAM2 with OS04A10, automatic timing is 60 FPS. * @param buff_num camera buffer number, default is 3, means 3 buffer, one used by user, one used for cache the next frame, * more than one buffer will accelerate image read speed, but will cost more memory. * @param open If true, camera will automatically call open() after creation. default is true. @@ -320,7 +323,12 @@ namespace maix::camera /** * Set camera fps - * @param fps new fps + * @param fps new fps. On MaixCAM2 with OS04A10 this reconfigures the + * sensor/VIN timing while preserving the current sensor + * crop. Pass a non-positive value to select its automatic + * 60 FPS timing. + * Requests above the current crop mode's supported limit + * are rejected without stopping the current stream. * @return error code, err::ERR_NONE means success, others means failed * @maixpy maix.camera.Camera.set_fps */ @@ -437,6 +445,36 @@ namespace maix::camera * Set window size of camera * @param roi Support two input formats, [x,y,w,h] set the coordinates and size of the window; * [w,h] set the size of the window, when the window is centred. + * On OS04A10 this configures the sensor ROI registers and restarts an opened camera pipeline. + * For outputs up to 1344x760, x/y are native 2704x1536-array coordinates while w/h are + * post-2x2-binning output dimensions. The 1344x760 ROI at [0,4] is the full-FOV binned + * mode; smaller sizes are binned sensor crops. Geometry rules are x/y even, w in + * 256..2688 and a multiple of 16 pixels, and h in 20..1520 and even. The physical + * binned readout must fit within the native 2704x1536 array: + * x + 2 * (w + 8) <= 2704 and y + 2 * h + 8 <= 1536. For a native (non-binned) + * crop the corresponding bounds are x + w + 16 <= 2704 and y + h + 16 <= 1536. + * These are nominal geometry limits. The AX620E OS04A10 pipeline may still produce no + * frames for a sensor/ISP-specific offset and size combination; a dynamic restart then + * times out quickly, restores the previous stream, and returns an error. In particular, + * crop sizes below 1024 wide and above 464 high are rejected before restart; use height + * <= 464 or width >= 1024. + * For example, set_windowing({640,360}) selects a centered native-array ROI at + * [704,404,640,360]; set_windowing({704,404,640,360}) selects the same ROI explicitly. + * The call is synchronous: an opened camera is stopped, sensor registers are rewritten, + * the pipeline is restarted, and initial AE/AWB settling frames are discarded. + * Without set_windowing, an unspecified output size (the public default is 640x480) and + * every output other than an explicit 2688x1520 use the fixed 1344x760 full-FOV binned + * sensor input; VI scales it to the requested output. An explicit 2688x1520 output uses + * the native non-binned sensor image at 30/60fps. If that same output requests more than + * 60fps, the input is automatically downgraded to full-FOV 1344x760 binning. This + * preserves the binned field of view for all smaller outputs but cannot add sensor detail + * beyond 1344x760. A binned sensor ROI itself cannot exceed 1344x760; call set_windowing + * for a smaller crop. In particular, asking for a small Camera output does not crop the + * sensor automatically. + * A binned ROI with width 256..640 and height 20..360 supports up to 360fps. Every other + * binned ROI, i.e. width 256..1344 and height 20..760 but not both within 640x360, + * supports up to 180fps. A non-binned ROI is selected when width is 1360..2688 or height + * is 762..1520; it supports up to 60fps. Width remains a multiple of 16 and height even. * @return error code * @maixpy maix.camera.Camera.set_windowing */ @@ -493,5 +531,6 @@ namespace maix::camera bool _invert_mirror; bool _is_opened; void *_param; + std::vector _windowing; }; } diff --git a/components/vision/port/maixcam2/maix_camera_maixcam2.cpp b/components/vision/port/maixcam2/maix_camera_maixcam2.cpp index 34c009d6..bff0e306 100644 --- a/components/vision/port/maixcam2/maix_camera_maixcam2.cpp +++ b/components/vision/port/maixcam2/maix_camera_maixcam2.cpp @@ -11,6 +11,8 @@ #include "maix_basic.hpp" #include "maix_i2c.hpp" #include +#include +#include #include "ax_middleware.hpp" using namespace maix; @@ -25,6 +27,109 @@ namespace maix::camera static std::vector __sensor_size; static bool __invert_flip = false; static bool __invert_mirror = false; + constexpr int kOs04a10BinnedMaxWidth = 1344; + constexpr int kOs04a10BinnedMaxHeight = 760; + constexpr int kOs04a10HighFpsCropMaxWidth = 640; + constexpr int kOs04a10HighFpsCropMaxHeight = 360; + + static AX_S32 __set_os04a10_sensor_crop(ISP_PIPE_ID pipe, AX_S32 x, AX_S32 y, + AX_S32 width, AX_S32 height, AX_F32 fps, + bool require_api = true) + { + using set_crop_fn = AX_S32 (*)(ISP_PIPE_ID, AX_U32, AX_U32, AX_U32, AX_U32, AX_F32); + static void *sensor_handle = nullptr; + set_crop_fn set_crop = reinterpret_cast(dlsym(RTLD_DEFAULT, "os04a10_set_crop")); + if (!set_crop) { + if (!sensor_handle) { + sensor_handle = dlopen("/opt/lib/libsns_os04a10.so", RTLD_LAZY | RTLD_GLOBAL); + } + if (sensor_handle) { + set_crop = reinterpret_cast(dlsym(sensor_handle, "os04a10_set_crop")); + } + } + if (!set_crop) { + if (require_api) { + log::error("OS04A10 sensor crop API is unavailable: %s", dlerror()); + return -1; + } + /* Old system sensor libraries do not implement the optional crop + * API. They have no process-global crop state to clear. */ + return AX_SUCCESS; + } + return set_crop(pipe, (AX_U32)x, (AX_U32)y, (AX_U32)width, (AX_U32)height, fps); + } + + static int __os04a10_max_fps_for_roi(int width, int height) + { + return width <= kOs04a10HighFpsCropMaxWidth && height <= kOs04a10HighFpsCropMaxHeight ? 360 : + width <= kOs04a10BinnedMaxWidth && height <= kOs04a10BinnedMaxHeight ? 180 : 60; + } + + /* Check timing before tearing down an active OS04A10 pipeline. The + * sensor mode is selected at stream-on, so changing FPS requires a + * close/open; rejecting an impossible request here keeps the old stream + * running. */ + static err::Err __validate_os04a10_fps(const std::vector &windowing, double fps) + { + if (fps <= 0) { + return err::ERR_NONE; // selector-auto is the documented 60 fps mode + } + const int requested_fps = static_cast(fps); + if (requested_fps <= 0 || static_cast(requested_fps) != fps) { + log::error("OS04A10 FPS must be a positive integer (or 0 for automatic 60 fps), requested %.3f", fps); + return err::ERR_ARGS; + } + + const bool has_roi = windowing.size() == 4; + const int max_fps = has_roi ? __os04a10_max_fps_for_roi(windowing[2], windowing[3]) : 180; + if (requested_fps > max_fps) { + if (!has_roi) { + log::error("OS04A10 full-FOV 1344x760 binned mode supports up to 180 fps; " + "set_windowing({x, y, 640, 360}) before requesting %d fps", requested_fps); + } else { + log::error("OS04A10 %dx%d sensor crop supports up to %d fps, requested %d", + windowing[2], windowing[3], max_fps, requested_fps); + } + return err::ERR_NOT_IMPL; + } + return err::ERR_NONE; + } + + /* A bad sensor timing/crop must not turn a command handler into an + * unbounded wait. This helper is intentionally used only by dynamic + * OS04A10 restart paths; ordinary camera reads keep their existing + * blocking semantics. */ + static bool __warmup_os04a10(Camera &camera, int frames) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + for (int i = 0; i < frames; ++i) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + log::error("OS04A10 warmup exceeded its 3 second deadline after %d/%d frames", i, frames); + return false; + } + const int remaining_ms = static_cast(std::chrono::duration_cast(deadline - now).count()); + if (remaining_ms <= 0) { + log::error("OS04A10 warmup exceeded its 3 second deadline after %d/%d frames", i, frames); + return false; + } + const int timeout_ms = i == 0 ? (remaining_ms < 1000 ? remaining_ms : 1000) : + (remaining_ms < 100 ? remaining_ms : 100); + image::Image *img = nullptr; + try { + img = camera.read(true, timeout_ms); + } catch (const err::Exception &) { + log::error("OS04A10 did not produce warmup frame %d/%d", i + 1, frames); + return false; + } + if (!img) { + log::error("OS04A10 warmup frame %d/%d timed out", i + 1, frames); + return false; + } + delete img; + } + return true; + } static AX_S32 __config_os04d10_360p120_ae(AX_U8 pipe) { @@ -110,7 +215,7 @@ namespace maix::camera break; case 0x36: // ov_os04a10 __device_name = "ov_os04a10"; - __sensor_size = {2560, 1440}; + __sensor_size = {2688, 1520}; __invert_flip = true; __invert_mirror = false; break; @@ -442,6 +547,13 @@ namespace maix::camera bool raw; bool flip; bool mirror; + /* Keep the constructor's "no output size supplied" state separate + * from its public 640x480 default. OS04A10 mode selection must not + * infer a small sensor crop merely because its public output happens + * to be 640x480. */ + bool output_size_auto; + bool fps_auto; + double requested_fps; int exptime_max; // unit:us int exptime_min; @@ -482,6 +594,9 @@ namespace maix::camera err::check_null_raise(priv, "camera_priv_t malloc error"); memset(priv, 0, sizeof(camera_priv_t)); priv->raw = raw; + priv->output_size_auto = width == -1 && height == -1; + priv->fps_auto = fps <= 0; + priv->requested_fps = fps; _param = priv; // open camera @@ -529,10 +644,19 @@ namespace maix::camera int width_tmp = (width == -1) ? _width : width; int height_tmp = (height == -1) ? _height : height; image::Format format_tmp = (format == image::FMT_INVALID) ? _format : format; - double fps_tmp = (fps == -1) ? _fps : fps; + camera_priv_t *priv = (camera_priv_t *)_param; + if (fps != -1) { + /* Both 0 and negative constructor FPS values mean selector-auto + * (60 fps for OS04A10). Record an explicit open(…, 0) in the + * same way; otherwise a previous explicit FPS leaks through a + * later auto reopen. */ + priv->fps_auto = fps <= 0; + priv->requested_fps = fps; + } + double fps_tmp = (fps == -1) ? (priv->fps_auto ? -1 : _fps) : + (priv->fps_auto ? -1 : fps); _fps = fps_tmp; int buff_num_tmp =( buff_num == -1) ? _buff_num : buff_num; - camera_priv_t *priv = (camera_priv_t *)_param; // check resolution if (format == image::FMT_RGB888) { @@ -571,6 +695,14 @@ namespace maix::camera .statDeltaPtsFrmNum = 0, }; + if (_windowing.size() == 4) { + tVinParam.bSensorCrop = AX_TRUE; + tVinParam.nSensorCropX = _windowing[0]; + tVinParam.nSensorCropY = _windowing[1]; + tVinParam.nSensorCropW = _windowing[2]; + tVinParam.nSensorCropH = _windowing[3]; + } + // init vi VI *ax_vi = new VI(); @@ -584,10 +716,72 @@ namespace maix::camera return err::ERR_RUNTIME; } + if (get_sensor_res.second == "os04a10" && !tVinParam.bSensorCrop) { + /* Application output and sensor input are intentionally separate. + * The only no-ROI request that selects native readout is an + * explicit 2688x1520 output. The constructor's unspecified size + * (which publicly defaults to 640x480), every smaller explicit + * output, and larger scaled output all retain the full-FOV 2x2 + * binned 1344x760 sensor image. In particular, a small public + * output must never implicitly turn into a 640x360 sensor crop: + * only set_windowing() opts into a smaller sensor ROI. */ + const bool explicit_native_output = !priv->output_size_auto && + _width == 2688 && _height == 1520; + const bool require_binned_input = !explicit_native_output || + (!priv->fps_auto && priv->requested_fps > 60); + if (require_binned_input) { + if (!priv->fps_auto && priv->requested_fps > 180) { + log::error("OS04A10 full-FOV 1344x760 binned mode supports up to 180 fps; " + "for 240 fps, call set_windowing() with a small ROI (recommended: 640x360) before open"); + delete ax_vi; + delete ax_sys; + return err::ERR_NOT_IMPL; + } + if (explicit_native_output) { + log::info("OS04A10 2688x1520 at %.0f fps uses the full-FOV 1344x760 binned sensor input", + priv->requested_fps); + } + tVinParam.bSensorCrop = AX_TRUE; + tVinParam.nSensorCropX = 0; + tVinParam.nSensorCropY = 4; + tVinParam.nSensorCropW = kOs04a10BinnedMaxWidth; + tVinParam.nSensorCropH = kOs04a10BinnedMaxHeight; + } + } + int tmp_w = _width, tmp_h = _height, tmp_fps = _fps; - tVinParam.eSysCase = ax_vi->get_vi_case((char *)get_sensor_res.second.c_str(), tmp_w, tmp_h, tmp_fps); + const int crop_w = tVinParam.bSensorCrop ? tVinParam.nSensorCropW : -1; + const int crop_h = tVinParam.bSensorCrop ? tVinParam.nSensorCropH : -1; + tVinParam.eSysCase = ax_vi->get_vi_case((char *)get_sensor_res.second.c_str(), tmp_w, tmp_h, tmp_fps, + crop_w, crop_h); + if (tVinParam.eSysCase == SAMPLE_VIN_NONE) { + delete ax_vi; + delete ax_sys; + return err::ERR_NOT_IMPL; + } + tVinParam.nSensorWidth = tmp_w; + tVinParam.nSensorHeight = tmp_h; + tVinParam.nSensorFps = tmp_fps; _fps = tmp_fps; + if (get_sensor_res.second == "os04a10") { + const AX_S32 crop_x = tVinParam.bSensorCrop ? tVinParam.nSensorCropX : 0; + const AX_S32 crop_y = tVinParam.bSensorCrop ? tVinParam.nSensorCropY : 0; + const AX_S32 crop_width = tVinParam.bSensorCrop ? tVinParam.nSensorCropW : 0; + const AX_S32 crop_height = tVinParam.bSensorCrop ? tVinParam.nSensorCropH : 0; + if (__set_os04a10_sensor_crop(0, crop_x, crop_y, crop_width, crop_height, + (AX_F32)tVinParam.nSensorFps, + tVinParam.bSensorCrop == AX_TRUE) != AX_SUCCESS) { + delete ax_vi; + delete ax_sys; + return err::ERR_NOT_IMPL; + } + } tVinParam.bAiispEnable = app::get_sys_config_kv("npu", "ai_isp", "1") == "1" ? AX_TRUE : AX_FALSE; + if (tVinParam.bSensorCrop) { + /* The shipped AI-ISP model is calibrated for the native OS04A10 + * frame geometry and cannot safely process arbitrary sensor ROIs. */ + tVinParam.bAiispEnable = AX_FALSE; + } ax_vi->config_sample_case(&tVinParam, &tCommonArgs, &tPrivArgs); err = ax_vi->init(); if (err != err::ERR_NONE) { @@ -873,11 +1067,68 @@ namespace maix::camera } err::Err Camera::set_fps(double fps) { - err::Err ret = err::ERR_NONE; - if (fps > 0) { - this->exposure(1000 / fps * 1000); + camera_priv_t *priv = (camera_priv_t *)_param; + + /* Do not change the established behaviour of other MaixCAM2 + * sensors: their set_fps implementation historically only adjusted + * the exposure limit. OS04A10 timing, however, is programmed during + * VIN/sensor stream-on and must be reopened. */ + if (get_device_name() != "ov_os04a10") { + if (fps > 0) { + this->exposure(1000 / fps * 1000); + } + return err::ERR_NONE; } - return ret; + + if (!this->is_opened()) { + return err::ERR_NOT_OPEN; + } + + const err::Err validation = __validate_os04a10_fps(_windowing, fps); + if (validation != err::ERR_NONE) { + return validation; + } + + const bool old_fps_auto = priv->fps_auto; + const double old_requested_fps = priv->requested_fps; + const image::Format old_format = _format; + const int old_buff_num = _buff_num; + const bool target_fps_auto = fps <= 0; + /* open(..., -1) means "use the already recorded state", whereas 0 + * explicitly records selector-auto. Use 0 for a public auto FPS + * request so a previous explicit value cannot leak into this reopen. */ + const double target_open_fps = target_fps_auto ? 0 : fps; + + this->close(); + err::Err ret = this->open(_width, _height, old_format, target_open_fps, old_buff_num); + if (ret != err::ERR_NONE) { + log::error("OS04A10 failed to apply %.0f fps; restoring the previous stream", fps); + const double restore_fps = old_fps_auto ? 0 : old_requested_fps; + const err::Err restore_ret = this->open(_width, _height, old_format, restore_fps, old_buff_num); + if (restore_ret != err::ERR_NONE) { + log::error("OS04A10 failed to restore the previous stream after FPS reconfiguration"); + } + return ret; + } + + /* A fresh ISP instance needs a short AWB/AE settling window. */ + int warmup_frames = 30; + if (_fps >= 360) { + warmup_frames = 180; + } else if (_fps >= 180) { + warmup_frames = 90; + } + if (!__warmup_os04a10(*this, warmup_frames)) { + log::error("OS04A10 %.0f fps restart produced no usable frames; restoring the previous stream", fps); + this->close(); + const double restore_fps = old_fps_auto ? 0 : old_requested_fps; + const err::Err restore_ret = this->open(_width, _height, old_format, restore_fps, old_buff_num); + if (restore_ret != err::ERR_NONE) { + log::error("OS04A10 failed to restore the previous stream after warmup timeout"); + } + return err::ERR_RUNTIME; + } + return err::ERR_NONE; } int Camera::exposure(int value) { @@ -1350,25 +1601,26 @@ namespace maix::camera } err::Err Camera::set_windowing(std::vector roi) { + camera_priv_t *priv = (camera_priv_t *)_param; auto &mod_param = AxModuleParam::getInstance(); - mod_param.lock(AX_MOD_VI); - auto vi_param = (ax_vi_mod_t *)mod_param.get_param(AX_MOD_VI); - auto ax_cam = vi_param->cams[0]; - mod_param.unlock(AX_MOD_VI); + int max_width = 2688; + int max_height = 1520; + bool os04a10 = false; + if (this->is_opened()) { + mod_param.lock(AX_MOD_VI); + auto vi_param = (ax_vi_mod_t *)mod_param.get_param(AX_MOD_VI); + auto ax_cam = vi_param->cams[0]; + mod_param.unlock(AX_MOD_VI); + os04a10 = ax_cam.eSnsType == OMNIVISION_OS04A10; + if (!os04a10) { + max_width = ax_cam.tSnsAttr.nWidth; + max_height = ax_cam.tSnsAttr.nHeight; + } + } else { + os04a10 = get_device_name() == "ov_os04a10"; + } err::Err ret = err::ERR_NONE; - auto *priv = (camera_priv_t *)_param; - auto ax_vi = priv->ax_vi; - - if (!this->is_opened()) { - return err::ERR_NOT_OPEN; - } - if (!this->is_opened()) { - return err::ERR_NOT_OPEN; - } - - int max_width = ax_cam.tSnsAttr.nWidth; - int max_height = ax_cam.tSnsAttr.nHeight; char log_msg[100]; int x = 0, y = 0, w = 0, h = 0; @@ -1376,23 +1628,118 @@ namespace maix::camera x = roi[0], y = roi[1], w = roi[2], h = roi[3]; } else if (roi.size() == 2) { w = roi[0], h = roi[1]; - x = (max_width - w) / 2; - y = (max_height - h) / 2; + const bool binning = os04a10 && w <= kOs04a10BinnedMaxWidth && h <= kOs04a10BinnedMaxHeight; + if (binning) { + /* x/y are native-array coordinates; width/height are the + * post-binning output. Center the corresponding 2x2 + * physical readout, including the 8-pixel output guard. */ + x = (2704 - 2 * (w + 8)) / 2; + y = (1536 - (2 * h + 8)) / 2; + x &= ~1; + y &= ~1; + } else { + x = (max_width - w) / 2; + y = (max_height - h) / 2; + } } else { err::check_raise(err::ERR_RUNTIME, "roi size must be 4 or 2"); } snprintf(log_msg, sizeof(log_msg), "Width must be a multiple of 2."); err::check_bool_raise(w % 2 == 0, std::string(log_msg)); + if (os04a10) { + const bool binning = w <= kOs04a10BinnedMaxWidth && h <= kOs04a10BinnedMaxHeight; + err::check_bool_raise(x % 2 == 0 && y % 2 == 0, + "sensor crop x and y must be even"); + err::check_bool_raise(w >= 256 && w <= max_width && w % 16 == 0, + "sensor crop width must be 256..2688 and a multiple of 16"); + err::check_bool_raise(h >= 20 && h <= max_height && h % 2 == 0, + "sensor crop height must be 20..1520 and even"); + /* Reject this verified AX ISP failure class before stopping an + * active stream. It is not an alignment requirement. */ + if (w < 1024 && h > 464) { + throw err::Exception(err::ERR_ARGS, + "This OS04A10 crop does not work. Some offset, width, and height combinations are not supported. " + "For this size, use height <= 464 or width >= 1024."); + } + const int physical_w = binning ? 2 * (w + 8) : w + 16; + const int physical_h = binning ? 2 * h + 8 : h + 16; + err::check_bool_raise(x + physical_w <= 2704, + "sensor crop physical readout exceeds 2704 pixels"); + err::check_bool_raise(y + physical_h <= 1536, + "sensor crop physical readout exceeds 1536 lines"); + } snprintf(log_msg, sizeof(log_msg), "the coordinate x range needs to be [0,%d].", max_width - 1); - err::check_bool_raise(x >= 0 || x < max_width, std::string(log_msg)); + err::check_bool_raise(x >= 0 && x < max_width, std::string(log_msg)); snprintf(log_msg, sizeof(log_msg), "the coordinate y range needs to be [0,%d].", max_height - 1); - err::check_bool_raise(y >= 0 || y < max_height, std::string(log_msg)); + err::check_bool_raise(y >= 0 && y < max_height, std::string(log_msg)); snprintf(log_msg, sizeof(log_msg), "the row of the window is larger than the maximum, try x=%d, w=%d.", x, max_width - x); err::check_bool_raise(x + w <= max_width, std::string(log_msg)); snprintf(log_msg, sizeof(log_msg), "the column of the window is larger than the maximum, try y=%d, h=%d.", y, max_height - y); err::check_bool_raise(y + h <= max_height, std::string(log_msg)); + if (os04a10) { + const std::vector old_windowing = _windowing; + const double old_fps = priv->fps_auto ? -1 : priv->requested_fps; + const err::Err validation = __validate_os04a10_fps({x, y, w, h}, old_fps); + if (validation != err::ERR_NONE) { + return validation; + } + _windowing = {x, y, w, h}; + if (!this->is_opened()) { + return ret; + } + /* Sensor timing and VIN dimensions are established during open; + * restart the pipeline so the new ROI is applied before stream-on. */ + auto old_format = _format; + auto old_buff_num = _buff_num; + this->close(); + /* Preserve an explicit caller FPS across the restart. Passing + * -1 re-enters the documented automatic 60 fps mode, so only do + * that for an originally automatic request. */ + ret = this->open(_width, _height, old_format, old_fps, old_buff_num); + if (ret != err::ERR_NONE) { + log::error("OS04A10 failed to apply the new crop; restoring the previous stream"); + _windowing = old_windowing; + const err::Err restore_ret = this->open(_width, _height, old_format, old_fps, old_buff_num); + if (restore_ret != err::ERR_NONE) { + log::error("OS04A10 failed to restore the previous stream after crop reconfiguration"); + } + return ret; + } + { + /* A fresh ISP instance needs a short AWB/AE settling window. + * Without discarding these first frames, a dynamic crop can + * briefly show magenta/green output while the statistics + * block converges. Keep the restart synchronous so the first + * frame returned to the caller is already usable. */ + int warmup_frames = 30; + if (_fps >= 360) { + warmup_frames = 180; + } else if (_fps >= 180) { + warmup_frames = 90; + } + if (!__warmup_os04a10(*this, warmup_frames)) { + log::error("This OS04A10 crop does not work. Some offset, width, and height combinations are not supported. " + "Try another crop area. Restoring the previous stream."); + this->close(); + _windowing = old_windowing; + const err::Err restore_ret = this->open(_width, _height, old_format, old_fps, old_buff_num); + if (restore_ret != err::ERR_NONE) { + log::error("OS04A10 failed to restore the previous stream after crop warmup timeout"); + } + throw err::Exception("This OS04A10 crop does not work. Some offset, width, and height combinations are not supported. " + "Try another crop area."); + } + } + return ret; + } + + if (!this->is_opened()) { + return err::ERR_NOT_OPEN; + } + + auto ax_vi = priv->ax_vi; bool is_vflip = priv->chn.vflip, is_hmirror = priv->chn.mirror; if (!is_vflip) { y = max_height - y - h; @@ -1420,6 +1767,13 @@ namespace maix::camera } else if (ax_cam.eSnsType == OMNIVISION_OS04D10) { max_width = 2560; max_height = 1440; + } else if (ax_cam.eSnsType == OMNIVISION_OS04A10) { + max_width = 2688; + max_height = 1520; + if (_is_opened && ax_cam.tSnsAttr.nWidth > 0 && ax_cam.tSnsAttr.nHeight > 0) { + max_width = ax_cam.tSnsAttr.nWidth; + max_height = ax_cam.tSnsAttr.nHeight; + } } return {max_width, max_height}; } @@ -1471,4 +1825,3 @@ namespace maix::camera return IspSceneParam.tManualParam.nAiWorkMode ? true : false; } } - diff --git a/examples/camera_display/main/src/main.cpp b/examples/camera_display/main/src/main.cpp index 211eb901..5d77c7aa 100644 --- a/examples/camera_display/main/src/main.cpp +++ b/examples/camera_display/main/src/main.cpp @@ -366,7 +366,22 @@ static int cmd_loop(camera::Camera *cam, display::Display *disp) case 11: { std::vector roi = {(int)value, (int)value2, (int)value3, (int)value4}; - err::check_raise(cam->set_windowing(roi), "set error"); + try { + err::check_raise(cam->set_windowing(roi), "set error"); + } catch (const err::Exception &e) { + std::string message = e.what(); + if (message.compare(0, 2, ": ") == 0) { + message.erase(0, 2); + } + const std::string unknown_prefix = "Unknown error: "; + if (message.compare(0, unknown_prefix.size(), unknown_prefix) == 0) { + message.erase(0, unknown_prefix.size()); + } + while (!message.empty() && (message.back() == '\n' || message.back() == '\r')) { + message.pop_back(); + } + log::error("set windowing rejected: %s", message.c_str()); + } break; } case 12: @@ -420,7 +435,12 @@ static int cmd_loop(camera::Camera *cam, display::Display *disp) auto curr_fps = cam->fps(); log::info("curr fps %f", curr_fps); } else { - err::check_raise(cam->set_fps(fps), "set fps error"); + const err::Err ret = cam->set_fps(fps); + if (ret != err::ERR_NONE) { + log::error("set fps rejected (%s); current stream was kept", err::to_str(ret).c_str()); + } else { + log::info("set fps applied: %f", cam->fps()); + } } break; } @@ -510,4 +530,3 @@ int main(int argc, char* argv[]) // So we catch exception here to let resources be released(call objects' destructor) before exit. CATCH_EXCEPTION_RUN_RETURN(_main, -1, argc, argv); } - diff --git a/examples/os04a10_sensor_crop_test/.gitignore b/examples/os04a10_sensor_crop_test/.gitignore new file mode 100644 index 00000000..edd9d60a --- /dev/null +++ b/examples/os04a10_sensor_crop_test/.gitignore @@ -0,0 +1,2 @@ +build/ +dist/ diff --git a/examples/os04a10_sensor_crop_test/app.yaml b/examples/os04a10_sensor_crop_test/app.yaml new file mode 100644 index 00000000..5b05ae5f --- /dev/null +++ b/examples/os04a10_sensor_crop_test/app.yaml @@ -0,0 +1,8 @@ +id: os04a10_sensor_crop_test +name: OS04A10 Sensor Crop Test +name[zh]: OS04A10 传感器裁剪测试 +version: 1.0.0 +author: Sipeed +desc: Verify OS04A10 sensor-level crop and high-frame-rate mode selection +desc[zh]: 验证 OS04A10 传感器级裁剪和高帧率模式选择 +files: {} diff --git a/examples/os04a10_sensor_crop_test/main/CMakeLists.txt b/examples/os04a10_sensor_crop_test/main/CMakeLists.txt new file mode 100644 index 00000000..0593d3ad --- /dev/null +++ b/examples/os04a10_sensor_crop_test/main/CMakeLists.txt @@ -0,0 +1,3 @@ +append_srcs_dir(ADD_SRCS "src") +list(APPEND ADD_REQUIREMENTS vision) +register_component() diff --git a/examples/os04a10_sensor_crop_test/main/src/main.cpp b/examples/os04a10_sensor_crop_test/main/src/main.cpp new file mode 100644 index 00000000..0ebc1de1 --- /dev/null +++ b/examples/os04a10_sensor_crop_test/main/src/main.cpp @@ -0,0 +1,111 @@ +#include "maix_basic.hpp" +#include "maix_camera.hpp" + +#include +#include +#include + +using namespace maix; + +static int run_test(int argc, char *argv[]) +{ + const int x = argc > 1 ? std::strtol(argv[1], nullptr, 10) : 0; + const int y = argc > 2 ? std::strtol(argv[2], nullptr, 10) : 4; + const int crop_width = argc > 3 ? std::strtol(argv[3], nullptr, 10) : 1344; + const int crop_height = argc > 4 ? std::strtol(argv[4], nullptr, 10) : 760; + const int fps = argc > 5 ? std::strtol(argv[5], nullptr, 10) : -1; + const int seconds = argc > 6 ? std::strtol(argv[6], nullptr, 10) : 10; + const bool apply_after_open = argc > 7 && std::strtol(argv[7], nullptr, 10) != 0; + const std::string snapshot_path = argc > 8 ? argv[8] : std::string(); + const int output_width = argc > 9 ? std::strtol(argv[9], nullptr, 10) : 640; + const int output_height = argc > 10 ? std::strtol(argv[10], nullptr, 10) : 360; + const std::string raw_path = argc > 11 ? argv[11] : std::string(); + + // Enable the RAW path when requested so we can distinguish sensor/VI data + // from an ISP-only rendering problem during high-speed crop validation. + camera::Camera cam(output_width, output_height, image::FMT_YVU420SP, nullptr, fps, 3, + false, !raw_path.empty()); + err::Err ret = err::ERR_NONE; + if (apply_after_open) { + ret = cam.open(); + if (ret != err::ERR_NONE) { + log::error("initial open camera failed: %s", err::to_str(ret).c_str()); + return -1; + } + } + if (crop_width > 0 && crop_height > 0) { + ret = cam.set_windowing({x, y, crop_width, crop_height}); + if (ret != err::ERR_NONE) { + log::error("set sensor crop failed: %s", err::to_str(ret).c_str()); + return -1; + } + } + if (!cam.is_opened()) { + ret = cam.open(); + if (ret != err::ERR_NONE) { + log::error("open camera failed: %s", err::to_str(ret).c_str()); + return -1; + } + } + + log::info("camera output=%dx%d fps=%.2f sensor_crop=[%d,%d,%d,%d]", + cam.width(), cam.height(), cam.fps(), x, y, crop_width, crop_height); + const std::vector sensor_size = cam.get_sensor_size(); + log::info("active sensor size=%dx%d", sensor_size[0], sensor_size[1]); + log::info("sensor window regs: start=%02x%02x,%02x%02x end=%02x%02x,%02x%02x output=%02x%02x,%02x%02x vts=%02x%02x", + cam.read_reg(0x3800), cam.read_reg(0x3801), cam.read_reg(0x3802), cam.read_reg(0x3803), + cam.read_reg(0x3804), cam.read_reg(0x3805), cam.read_reg(0x3806), cam.read_reg(0x3807), + cam.read_reg(0x3808), cam.read_reg(0x3809), cam.read_reg(0x380a), cam.read_reg(0x380b), + cam.read_reg(0x380e), cam.read_reg(0x380f)); + log::info("sensor timing/binning regs: pll0305=%02x pll0325=%02x inc=%02x/%02x/%02x/%02x format=%02x/%02x", + cam.read_reg(0x0305), cam.read_reg(0x0325), cam.read_reg(0x3814), cam.read_reg(0x3815), + cam.read_reg(0x3816), cam.read_reg(0x3817), cam.read_reg(0x3820), cam.read_reg(0x3821)); + + uint64_t frames = 0; + uint64_t failures = 0; + bool snapshot_saved = false; + // Do not charge pipeline startup latency to the steady-state frame rate. + cam.skip_frames(30); + if (!raw_path.empty()) { + image::Image *raw = cam.read_raw(); + if (!raw) { + log::error("read raw frame failed"); + return -1; + } + std::ofstream out(raw_path, std::ios::binary); + if (!out) { + log::error("open raw output failed: %s", raw_path.c_str()); + delete raw; + return -1; + } + out.write(reinterpret_cast(raw->data()), raw->data_size()); + log::info("saved raw frame to %s width=%d height=%d size=%d format=%s", + raw_path.c_str(), raw->width(), raw->height(), raw->data_size(), + image::fmt_names[raw->format()].c_str()); + delete raw; + } + const uint64_t start = time::ticks_ms(); + while (time::ticks_ms() - start < static_cast(seconds) * 1000) { + image::Image *img = cam.read(true, 1000); + if (img) { + ++frames; + if (!snapshot_saved && !snapshot_path.empty()) { + img->save(snapshot_path.c_str()); + snapshot_saved = true; + } + delete img; + } else { + ++failures; + } + } + const uint64_t elapsed = time::ticks_ms() - start; + log::info("result frames=%llu failures=%llu elapsed_ms=%llu measured_fps=%.2f", + static_cast(frames), static_cast(failures), + static_cast(elapsed), frames * 1000.0 / elapsed); + return failures == 0 && frames > 0 ? 0 : -1; +} + +int main(int argc, char *argv[]) +{ + CATCH_EXCEPTION_RUN_RETURN(run_test, -1, argc, argv); +} diff --git a/examples/video_encode_demo/main/src/main.cpp b/examples/video_encode_demo/main/src/main.cpp index 165ff103..faace240 100644 --- a/examples/video_encode_demo/main/src/main.cpp +++ b/examples/video_encode_demo/main/src/main.cpp @@ -7,6 +7,13 @@ #include "maix_video.hpp" #include "maix_camera.hpp" #include "list" +#include +#include +#include +#include +#include +#include +#include using namespace maix; #if defined(PLATFORM_MAIXCAM) || defined(PLATFORM_MAIXCAM2) @@ -35,6 +42,86 @@ static double timebase_to_ms(std::vector timebase, uint64_t value) { return value * 1000 / ((double)timebase[1] / timebase[0]); } +static int remux_high_fps_stream(const std::string &input_path, + const std::string &output_path, int framerate) { + AVFormatContext *input = nullptr; + AVFormatContext *output = nullptr; + AVStream *input_stream = nullptr; + AVStream *output_stream = nullptr; + AVPacket *packet = nullptr; + int ret = avformat_open_input(&input, input_path.c_str(), nullptr, nullptr); + if (ret < 0) goto cleanup; + ret = avformat_find_stream_info(input, nullptr); + if (ret < 0) goto cleanup; + ret = avformat_alloc_output_context2(&output, nullptr, "mp4", output_path.c_str()); + if (ret < 0 || !output) goto cleanup; + + { + for (unsigned int i = 0; i < input->nb_streams; ++i) { + if (input->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + input_stream = input->streams[i]; + break; + } + } + if (!input_stream) { + ret = AVERROR_STREAM_NOT_FOUND; + goto cleanup; + } + output_stream = avformat_new_stream(output, nullptr); + if (!output_stream) { + ret = AVERROR(ENOMEM); + goto cleanup; + } + ret = avcodec_parameters_copy(output_stream->codecpar, input_stream->codecpar); + if (ret < 0) goto cleanup; + output_stream->codecpar->codec_tag = 0; + output_stream->time_base = AVRational{1, framerate}; + } + + if (!(output->oformat->flags & AVFMT_NOFILE)) { + ret = avio_open(&output->pb, output_path.c_str(), AVIO_FLAG_WRITE); + if (ret < 0) goto cleanup; + } + ret = avformat_write_header(output, nullptr); + if (ret < 0) goto cleanup; + + packet = av_packet_alloc(); + if (!packet) { + ret = AVERROR(ENOMEM); + goto cleanup; + } + { + int64_t frame_index = 0; + const AVRational frame_time_base = AVRational{1, framerate}; + while ((ret = av_read_frame(input, packet)) >= 0) { + if (packet->stream_index != input_stream->index) { + av_packet_unref(packet); + continue; + } + packet->stream_index = 0; + packet->pts = av_rescale_q(frame_index, frame_time_base, output_stream->time_base); + packet->dts = packet->pts; + packet->duration = av_rescale_q(1, frame_time_base, output_stream->time_base); + packet->pos = -1; + ret = av_interleaved_write_frame(output, packet); + av_packet_unref(packet); + if (ret < 0) goto cleanup; + ++frame_index; + } + if (ret == AVERROR_EOF) ret = 0; + } + if (ret >= 0) ret = av_write_trailer(output); + +cleanup: + av_packet_free(&packet); + if (output) { + if (!(output->oformat->flags & AVFMT_NOFILE) && output->pb) avio_closep(&output->pb); + avformat_free_context(output); + } + if (input) avformat_close_input(&input); + return ret; +} + static int h264_to_mp4(int argc, char *argv[]) { const char *input_filename = "output.h264"; const char *output_filename = "output2.mp4"; @@ -305,7 +392,7 @@ static int h264_to_mp4(int argc, char *argv[]) { return 0; } -static void *_imu_thread_process(void *arg) { +[[maybe_unused]] static void *_imu_thread_process(void *arg) { #define _M_PI (3.14159265358979323846f) int *imu_is_ready = (int *)arg; // gcsv init @@ -410,6 +497,7 @@ static void helper(void) "0 : encode without bind\r\n" "1 : encode with bind\r\n" "2 : time-lapse record\r\n" + "4 : high-fps encode with an in-memory queue\r\n" "5 : encode image to jpeg\r\n" "note:\r\n" "format=%d, NV21\r\n" @@ -417,6 +505,7 @@ static void helper(void) "\r\n" "Example: ./encode_demo 0 output.mp4\r\n" "Example: ./encode_demo 0 output.mp4 640 480 8 30 50 3000000 1000 1\r\n" + "Example: ./video_encode_demo 4 output.mp4 640 360 8 7 360 360 8000000 1000 0 1 360\r\n" "==================================\r\n", image::FMT_YVU420SP, video::VIDEO_H264, video::VIDEO_H265); } @@ -569,13 +658,13 @@ int _main(int argc, char* argv[]) int height = 480; image::Format format = image::Format::FMT_YVU420SP; video::VideoType type = video::VIDEO_H264; - audio::Recorder r = audio::Recorder(); int framerate = 60; int gop = 50; int bitrate = 3000 * 1000; int time_base = 1000; - bool capture = true; + bool capture = false; bool block = true; + int queue_depth = -1; if (argc > 2) path = argv[2]; if (argc > 3) width = atoi(argv[3]); if (argc > 4) height = atoi(argv[4]); @@ -587,47 +676,132 @@ int _main(int argc, char* argv[]) if (argc > 10) time_base = atoi(argv[10]); if (argc > 11) capture = atoi(argv[11]) == 0 ? false : true; if (argc > 12) block = atoi(argv[12]) == 0 ? false : true; - log::info("path:%s width:%d height:%d format:%d type:%d fps:%d gop:%d bitrate:%d time_base:%d capture:%d\r\n", - path.c_str(), width, height, format, type, framerate, gop, bitrate, time_base, capture); - video::Encoder e = video::Encoder(path, width, height, format, type, framerate, gop, bitrate, time_base, capture, block); - camera::Camera cam = camera::Camera(width, height, format, NULL, framerate); - display::Display disp = display::Display(); - - pthread_t imu_thread; - int imu_is_ready = false; - err::check_bool_raise(pthread_create(&imu_thread, NULL, _imu_thread_process, &imu_is_ready) == 0, "create thread failed!"); - while (!imu_is_ready) {time::sleep_ms(10);} - - uint64_t start_time = time::ticks_ms(); - uint64_t last_loop_time = 0; - log::info("camera first read ms:%d", start_time); - while(!app::need_exit()) { - // uint64_t t = time::ticks_ms(); - image::Image *img = cam.read(); - // log::info("camera read use %lld ms", time::ticks_ms() - t); - - // t = time::ticks_ms(); - video::Frame *frame = e.encode(img); - // log::info("encode use %lld ms", time::ticks_ms() - t); - - // t = time::ticks_ms(); - disp.show(*img); - // log::info("show use %lld ms", time::ticks_ms() - t); - - // t = time::ticks_ms(); - // delete pcm; - delete frame; - delete img; - // log::info("free use %lld ms", time::ticks_ms() - t); - - while ((time::ticks_ms() - last_loop_time) * framerate < 1000) { - time::sleep_us(500); - } - last_loop_time = time::ticks_ms(); - - log::info("loop use %lld ms", time::ticks_ms() - start_time, time::ticks_ms()); - start_time = time::ticks_ms(); + if (argc > 13) queue_depth = atoi(argv[13]); + if (queue_depth < 0) queue_depth = framerate; + err::check_bool_raise(queue_depth > 0, "queue_depth must be greater than zero"); + const double queue_mib = static_cast(width) * height * 3 / 2 * queue_depth / + (1024.0 * 1024.0); + log::info("path:%s width:%d height:%d format:%d type:%d fps:%d gop:%d bitrate:%d time_base:%d capture:%d queue_depth:%d\r\n", + path.c_str(), width, height, format, type, framerate, gop, bitrate, time_base, capture, queue_depth); + log::info("high-fps queue capacity: %.1f MiB; oldest frames are dropped if it becomes full", queue_mib); + constexpr int venc_max_fps = 180; + const int venc_fps = std::min(framerate, venc_max_fps); + const bool needs_remux = framerate > venc_max_fps; + const bool is_h265 = type == video::VIDEO_H265 || type == video::VIDEO_H265_CBR || type == video::VIDEO_H265_VBR; + const std::string encoder_path = needs_remux ? path + (is_h265 ? ".cache.h265" : ".cache.h264") : path; + std::unique_ptr encoder(new video::Encoder( + encoder_path, width, height, format, type, venc_fps, gop, bitrate, + time_base, capture, block)); + camera::Camera cam = camera::Camera(width, height, format, NULL, framerate, 3, false); + if (framerate > 180) { + err::check_bool_raise(width <= 640 && height <= 360, + "OS04A10 recording above 180 fps requires a window no larger than 640x360"); + err::check_raise(cam.set_windowing({0, 0, width, height}), + "enable OS04A10 high-speed sensor window failed"); } + err::check_raise(cam.open(), "camera open failed"); + if (needs_remux) { + log::info("sensor fps:%d, VENC RC fps:%d; recording elementary stream for %d-fps remux", + framerate, venc_fps, framerate); + } + + // Keep camera acquisition independent from VENC/file I/O. The image returned + // by Camera::read() retains a scarce VIN pool buffer, so copy its pixels into + // normal heap memory before enqueueing and release the VIN frame immediately. + std::mutex queue_mutex; + std::condition_variable queue_not_empty; + std::deque> image_queue; + std::atomic capture_done(false); + std::atomic stop_requested(false); + std::atomic worker_failed(false); + std::atomic captured_count(0); + std::atomic encoded_count(0); + std::atomic dropped_count(0); + + std::thread capture_thread([&]() { + try { + while (!app::need_exit() && !stop_requested.load()) { + std::unique_ptr camera_img(cam.read()); + if (!camera_img) { + continue; + } + std::unique_ptr img(new image::Image( + camera_img->width(), camera_img->height(), camera_img->format(), + static_cast(camera_img->data()), camera_img->data_size(), true)); + camera_img.reset(); + ++captured_count; + + { + std::lock_guard lock(queue_mutex); + // Do not let a slow encoder throttle sensor acquisition. Retain + // the newest frames, and release a discarded frame immediately. + if (image_queue.size() >= static_cast(queue_depth)) { + image_queue.pop_front(); + ++dropped_count; + } + image_queue.emplace_back(std::move(img)); + } + queue_not_empty.notify_one(); + } + } catch (const std::exception &e) { + log::error("camera capture thread stopped: %s", e.what()); + worker_failed.store(true); + stop_requested.store(true); + } catch (...) { + log::error("camera capture thread stopped by an unknown exception"); + worker_failed.store(true); + stop_requested.store(true); + } + capture_done.store(true); + queue_not_empty.notify_all(); + }); + + std::thread encode_thread([&]() { + try { + while (true) { + std::unique_ptr img; + { + std::unique_lock lock(queue_mutex); + queue_not_empty.wait(lock, [&]() { + return capture_done.load() || !image_queue.empty(); + }); + if (image_queue.empty()) { + if (capture_done.load()) { + break; + } + continue; + } + img = std::move(image_queue.front()); + image_queue.pop_front(); + } + + std::unique_ptr frame(encoder->encode(img.get())); + ++encoded_count; + } + } catch (const std::exception &e) { + log::error("video encode thread stopped: %s", e.what()); + worker_failed.store(true); + stop_requested.store(true); + } catch (...) { + log::error("video encode thread stopped by an unknown exception"); + worker_failed.store(true); + stop_requested.store(true); + } + }); + + capture_thread.join(); + encode_thread.join(); + encoder.reset(); + err::check_bool_raise(!worker_failed.load(), "high-fps recording worker failed"); + if (needs_remux) { + const int remux_ret = remux_high_fps_stream(encoder_path, path, framerate); + err::check_bool_raise(remux_ret >= 0, "high-fps stream remux failed"); + std::remove(encoder_path.c_str()); + } + log::info("high-fps record stopped: captured:%llu encoded:%llu dropped:%llu queue_depth:%d", + static_cast(captured_count.load()), + static_cast(encoded_count.load()), + static_cast(dropped_count.load()), queue_depth); break; } case 5: