refactor(video): use Sophgo SDK for H.264 and MJPEG

Use per-codec capture locks and encoder backpressure, simplify frame ownership, and follow HDMI input dimensions. Refresh prebuilt video libraries while preserving server-only builds and deployment instructions.

Makefile

@@ -18,7 +18,7 @@ DOCKER_RUN_BASE := docker run -e UID=$(UID) -e GID=$(GID) -v $(PWD):/home/build/
 DOCKER_BUILD_ARGS := --build-arg DOCKER_UID=$(UID) --build-arg DOCKER_GID=$(GID)
 
 # Build commands
-GO_BUILD_CMD := cd /home/build/NanoKVM/server && go mod tidy && CGO_ENABLED=1 GOOS=linux GOARCH=riscv64 CC=riscv64-unknown-linux-musl-gcc CGO_CFLAGS="-mcpu=c906fdv -march=rv64imafdcv0p7xthead -mcmodel=medany -mabi=lp64d" go build
+GO_BUILD_CMD := cd /home/build/NanoKVM/server && ./build.sh
 SUPPORT_BUILD_CMD := . ./home/build/MaixCDK/bin/activate && cd /home/build/NanoKVM/support/sg2002 && ./build kvm_system && ./build kvm_system add_to_kvmapp
 VISION_BUILD_CMD := . ./home/build/MaixCDK/bin/activate && cd /home/build/NanoKVM/support/sg2002 && ./build kvm_vision && ./build kvm_vision add_to_kvmapp
 RELEASE_BUILD_CMD := /home/build/NanoKVM/scripts/build-in-container.sh

README.md

@@ -97,7 +97,7 @@ The repository ships a [Dev Container](https://containers.dev) setup (`.devconta
 
 - Requirements: Docker plus an editor or CLI with dev container support (e.g. the VS Code "Dev Containers" extension, or `devcontainer up`).
 - On creation the container prepares itself automatically: `support/sg2002/build update_lib` (resyncs MaixCDK components with the checkout) and `pnpm install` for the frontend.
-- Backend: run `server/build.sh` for a release-equivalent binary (BoringCrypto + `dl_lib` RPATH). `make app` builds a plain development binary without those, so prefer `server/build.sh` whenever the result should match a release.
+- Backend: inside the container, run `./build.sh` from `server/`. On the host, `make app` runs the same script in Docker. Both use the prebuilt libraries in `server/dl_lib/`, enable the BoringCrypto build option, and set the `dl_lib` RPATH.
 - Support layer: `cd support/sg2002 && ./build kvm_system` (the MaixCDK virtualenv is part of the container environment).
 - Frontend: `cd web && pnpm dev` (VS Code forwards port 3001 automatically; other clients may need to map it themselves); see [web/README.md](web/README.md) for pointing it at a device.
 - The base image `ghcr.io/sipeed/nanokvm-builder:latest` is x86-64 only; on Apple Silicon it runs under emulation (works, but C builds are slow). You can also build the image locally first (`make builder-image`) and point the `BASE_IMAGE` build arg at it (e.g. via `build.args` in `devcontainer.json`). On Linux hosts the first container creation also takes a while: the container user is remapped to your uid, which re-chowns its home directory once.

scripts/README.md

@@ -81,22 +81,27 @@ manual download and does not change the on-device `latest.json` contract.
 | `server/web/` | `web/dist/` |
 | `kvm_system/kvm_system` | `support/sg2002` `build kvm_system` |
 | `system/tool/` | prebuilt binaries in `tools/nanokvm_update_edid/` |
-| `kvm/` | default runtime state (resolution, fps, quality) |
+| `kvm/` | default runtime state (input dimensions, fps, quality) |
 | everything else | tracked `kvmapp/` |
 
 Note that `kvm/` ships default runtime state, and `install.go` replaces
 `/kvmapp` wholesale, so an update resets those values on the device. That is
 long-standing behaviour, not something these scripts introduce.
 
-## The two copies of libkvm.so
+## Video library linking
 
-`NanoKVM-Server` is cgo-linked against the **tracked** `server/dl_lib/libkvm.so`
+`make app` and `server/build.sh` compile the server using the prebuilt libraries
-(`server/common/kvm_vision.go`: `-L../dl_lib -lkvm`), but the package ships the
+in `server/dl_lib/`. They do not rebuild the video libraries. The cgo directive
-library the `kvm_vision` build just produced. Those are two different files, and
+in `server/common/kvm_vision.go` links against `libkvm.so`, which depends on
-the tracked one can lag well behind — published releases have shipped this
+`libkvm_mmf.so`.
-mismatch for a long time.
 
-Because a symbol the binary imports could in principle be absent from the
+`make vision` rebuilds the video libraries and stages them in
-shipped library, and that would only surface as a crash on a real device,
+`kvmapp/server/dl_lib/`; `make release-build` does this as part of the full build.
-`build-in-container.sh` checks the shipped `libkvm.so` still exports every
+After changing library interfaces, update both prebuilt files in
-symbol `NanoKVM-Server` actually imports from it, and fails the build otherwise.
+`server/dl_lib/` from the same build. The tracked directory also supplies the
+SDK runtime libraries used by the video build and backfills libraries that
+are absent from the staged directory.
+
+`build-in-container.sh` checks that the staged `libkvm.so` exports every KVM
+symbol imported by the server. See the [server instructions](../server/README.md#compile--deploy)
+for manual compilation and deployment.

scripts/build-in-container.sh

@@ -55,13 +55,8 @@ fi
 # shellcheck disable=SC1091
 . "$MAIXCDK_PATH/bin/activate"
 
-echo "::group::support: sync MaixCDK components"
+# Each support build synchronizes the repository components before compiling.
 cd "$NANOKVM_PATH/support/sg2002"
-# The image bakes support/sg2002/additional/ into MaixCDK's components at image
-# build time, and ./build only re-copies them when components/kvm is missing.
-# Refresh unconditionally so a cached image never yields stale libraries.
-./build update_lib
-echo "::endgroup::"
 
 echo "::group::support: build kvm_system"
 ./build kvm_system
@@ -100,11 +95,8 @@ if [ "$missing" -ne 0 ]; then
     exit 1
 fi
 
-# NanoKVM-Server is cgo-linked against the tracked server/dl_lib/libkvm.so
+# Confirm the shipped video libraries provide every KVM symbol imported by
-# (server/common/kvm_vision.go: -L../dl_lib -lkvm), but the package ships the
+# the server, including the frame ownership API.
-# freshly built one from kvm_vision. Those are different files, so confirm the
-# shipped library still exports every symbol the binary actually imports from
-# it. Without this a dropped symbol only shows up as a crash on a real device.
 echo "[INFO] verifying shipped libkvm.so satisfies NanoKVM-Server"
 if ! command -v readelf >/dev/null 2>&1; then
     echo "[WARN] readelf unavailable, skipping ABI check"
@@ -120,13 +112,10 @@ else
             | sed 's/@.*//' | LC_ALL=C sort -u
     }
 
-    linked_lib="$NANOKVM_PATH/server/dl_lib/libkvm.so"
     shipped_lib="$NANOKVM_PATH/kvmapp/server/dl_lib/libkvm.so"
     server_bin="$NANOKVM_PATH/server/NanoKVM-Server"
 
-    # Symbols the binary imports that the link-time library provides, i.e. the
+    needed=$(imported "$server_bin" | awk '/^(kvmv_|free_kvmv_data$|free_all_kvmv_data$|set_h264_gop$|set_frame_detact$|set_venc_auto_recyc$)/')
-    # ones libkvm.so is actually responsible for at runtime.
-    needed=$(comm -12 <(imported "$server_bin") <(exported "$linked_lib"))
 
     if [ -z "$needed" ]; then
         echo "[WARN] no libkvm symbols resolved; skipping ABI check"

server/common/kvm_vision.go

@@ -35,64 +35,49 @@ func GetKvmVision() *KvmVision {
 	return kvmVision
 }
 
-func (k *KvmVision) ReadMjpeg(width uint16, height uint16, quality uint16) (data []byte, result int) {
+func (k *KvmVision) ReadMjpeg(width uint16, height uint16, quality uint16) ([]byte, int) {
-	k.mutex.RLock()
+	frame := k.AcquireMjpeg(width, height, quality, uint16(GetScreen().FPS))
-	defer k.mutex.RUnlock()
+	defer frame.Release()
-	if k.closed {
+	return frame.Data, frame.Result
-		return nil, -1
-	}
-
-	var (
-		kvmData  *C.uint8_t
-		dataSize C.uint32_t
-	)
-
-	result = int(C.kvmv_read_img(
-		C.uint16_t(width),
-		C.uint16_t(height),
-		C.uint8_t(0),
-		C.uint16_t(quality),
-		&kvmData,
-		&dataSize,
-	))
-	if result < 0 {
-		log.Errorf("failed to read kvm image: %v", result)
-		return
-	}
-	defer C.free_kvmv_data(&kvmData)
-
-	data = C.GoBytes(unsafe.Pointer(kvmData), C.int(dataSize))
-	return
 }
 
-func (k *KvmVision) ReadH264(width uint16, height uint16, bitRate uint16) (data []byte, result int) {
+func (k *KvmVision) AcquireMjpeg(width, height, quality, fps uint16) *VideoFrame {
+	return k.acquireFrame(width, height, quality, fps, 0)
+}
+
+func (k *KvmVision) AcquireH264(width, height, bitRate, fps uint16) *VideoFrame {
+	return k.acquireFrame(width, height, bitRate, fps, 1)
+}
+
+func (k *KvmVision) acquireFrame(width, height, quality, fps uint16, codec uint8) *VideoFrame {
 	k.mutex.RLock()
 	defer k.mutex.RUnlock()
 	if k.closed {
-		return nil, -1
+		return NewVideoFrame(nil, -1, 0, nil)
 	}
-
+	var frame C.kvmv_frame_t
-	var (
+	result := int(C.kvmv_read_img(C.uint16_t(width), C.uint16_t(height),
-		kvmData  *C.uint8_t
+		C.uint8_t(codec), C.uint16_t(quality), C.uint16_t(fps), &frame))
-		dataSize C.uint32_t
+	if frame.data == nil {
-	)
+		return NewVideoFrame(nil, result, int64(frame.timestamp), nil)
-
-	result = int(C.kvmv_read_img(
-		C.uint16_t(width),
-		C.uint16_t(height),
-		C.uint8_t(1),
-		C.uint16_t(bitRate),
-		&kvmData,
-		&dataSize,
-	))
-	if result < 0 {
-		log.Errorf("failed to read kvm image: %v", result)
-		return
 	}
-	defer C.free_kvmv_data(&kvmData)
+	data := C.GoBytes(unsafe.Pointer(frame.data), C.int(frame.size))
+	return NewVideoFrame(data, result, int64(frame.timestamp), func() {
+		k.mutex.RLock()
+		defer k.mutex.RUnlock()
+		C.kvmv_release_frame(&frame)
+	})
+}
 
-	data = C.GoBytes(unsafe.Pointer(kvmData), C.int(dataSize))
+func (k *KvmVision) StopH264()  { k.stopStream(1) }
-	return
+func (k *KvmVision) StopMjpeg() { k.stopStream(0) }
+
+func (k *KvmVision) stopStream(codec uint8) {
+	k.mutex.RLock()
+	defer k.mutex.RUnlock()
+	if !k.closed {
+		C.kvmv_stop_stream(C.uint8_t(codec))
+	}
 }
 
 func (k *KvmVision) SetHDMI(enable bool) int {

server/common/screen.go

@@ -1,10 +1,13 @@
 package common
 
-import "sync"
+import (
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
 
 type Screen struct {
-	Width   uint16
-	Height  uint16
 	FPS     int
 	Quality uint16
 	BitRate uint16
@@ -16,15 +19,6 @@ var (
 	screenOnce sync.Once
 )
 
-// ResolutionMap height to width
-var ResolutionMap = map[uint16]uint16{
-	1080: 1920,
-	720:  1280,
-	600:  800,
-	480:  640,
-	0:    0,
-}
-
 var QualityMap = map[uint16]bool{
 	100: true,
 	80:  true,
@@ -42,8 +36,6 @@ var BitRateMap = map[uint16]bool{
 func GetScreen() *Screen {
 	screenOnce.Do(func() {
 		screen = &Screen{
-			Width:   0,
-			Height:  0,
 			Quality: 80,
 			FPS:     30,
 			BitRate: 3000,
@@ -56,13 +48,6 @@ func GetScreen() *Screen {
 
 func SetScreen(key string, value int) {
 	switch key {
-	case "resolution":
-		height := uint16(value)
-		if width, ok := ResolutionMap[height]; ok {
-			screen.Width = width
-			screen.Height = height
-		}
-
 	case "quality":
 		if value > 100 {
 			screen.BitRate = uint16(value)
@@ -79,11 +64,6 @@ func SetScreen(key string, value int) {
 }
 
 func CheckScreen() {
-	if _, ok := ResolutionMap[screen.Height]; !ok {
-		screen.Width = 1920
-		screen.Height = 1080
-	}
-
 	if _, ok := QualityMap[screen.Quality]; !ok {
 		screen.Quality = 80
 	}
@@ -93,6 +73,21 @@ func CheckScreen() {
 	}
 }
 
+func InputSize() (uint16, uint16) {
+	read := func(path string) uint16 {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return 0
+		}
+		value, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 16)
+		if err != nil {
+			return 0
+		}
+		return uint16(value)
+	}
+	return read("/kvmapp/kvm/width"), read("/kvmapp/kvm/height")
+}
+
 func validateFPS(fps int) int {
 	if fps > 60 {
 		return 60

server/common/video_frame.go

@@ -0,0 +1,26 @@
+package common
+
+import "sync"
+
+type VideoFrame struct {
+	Data      []byte
+	Result    int
+	Timestamp int64
+	release   func()
+	once      sync.Once
+}
+
+func NewVideoFrame(data []byte, result int, timestamp int64, release func()) *VideoFrame {
+	return &VideoFrame{Data: data, Result: result, Timestamp: timestamp, release: release}
+}
+
+func (f *VideoFrame) Release() {
+	if f == nil {
+		return
+	}
+	f.once.Do(func() {
+		if f.release != nil {
+			f.release()
+		}
+	})
+}

server/include/kvm_vision.h

@@ -1,71 +1,43 @@
-
-
 #ifndef KVM_VISION_H_
 #define KVM_VISION_H_
 
+#include <stdint.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include <fcntl.h> /* low-level i/o */
-#include <unistd.h>
-#include <string.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <pthread.h>
-#include <time.h>
 
-#define IMG_BUFFER_FULL			-3
+#define IMG_BUFFER_FULL -3
-#define IMG_VENC_ERROR			-2
+#define IMG_VENC_ERROR  -2
-#define IMG_NOT_EXIST			-1
+#define IMG_NOT_EXIST   -1
-#define IMG_MJPEG_TYPE			0
+#define IMG_MJPEG_TYPE   0
-#define IMG_H264_TYPE_SPS		1
+#define IMG_H264_TYPE_IF 3
-#define IMG_H264_TYPE_PPS		2
+#define IMG_H264_TYPE_PF 4
-#define IMG_H264_TYPE_IF		3
-#define IMG_H264_TYPE_PF		4
 
-#define NORMAL_RES                      0
+typedef struct {
-#define NEW_RES                         1
+    uint8_t *data;
-#define UNSUPPORT_RES                   2
+    uint32_t size;
-#define UNKNOWN_RES                     3
+    uint64_t timestamp;
-#define ERROR_RES                       4
+    uint64_t token;
+    uint8_t codec;
+} kvmv_frame_t;
 
-void kvmv_init(uint8_t _debug_info_en);
+void kvmv_init(uint8_t debug_info);
-void set_venc_auto_recyc(uint8_t _enable);
+void kvmv_deinit(void);
-/**********************************************************************************
+
- * @name    kvmv_read_img
+// type: 0 = MJPEG, 1 = H.264. A zero width/height follows the input size.
- * @author  Sipeed BuGu
+// quality: MJPEG 50..100, H.264 500..10000 Kbps. fps: 1..60.
- * @date    2024/10/25
+// Returns -7 input error, -6 unsupported size, -5 busy, -4 reconfiguring,
- * @version R1.0
+// -3 allocation failure, -2 encoder error, -1 no frame, 0 JPEG, 3 IDR,
- * @brief   Acquire the encoded image with auto init
+// 4 P-frame, or 5 unchanged JPEG. Release every successful frame once.
- * @param	_width				@input: 	Output image width
+int kvmv_read_img(uint16_t width, uint16_t height, uint8_t type,
- * @param	_height				@input: 	Output image height
+                  uint16_t quality, uint16_t fps, kvmv_frame_t *frame);
- * @param	_type				@input: 	Encode type
+// Frees the copied bytes and releases the encoder lease. Safe after shutdown.
- * @param	_qlty				@input: 	MJPEG: (50-100) | H264:  (500-10000)
+void kvmv_release_frame(kvmv_frame_t *frame);
- * @param	_pp_kvm_data		@output: 	Encode data
+void kvmv_stop_stream(uint8_t type);
- * @param	_p_kvmv_data_size	@output: 	Encode data size
+void set_h264_gop(uint8_t gop);
- * @return
+void set_frame_detact(uint8_t interval);
-        -7: HDMI INPUT RES ERROR
+uint8_t kvmv_hdmi_control(uint8_t enable);
-        -6: Unsupported resolution, please modify it in the host settings.
-        -5: Retrieving image, please wait
-        -4: Modifying image resolution, please wait
-        -3: img buffer full
-        -2: VENC Error
-        -1: No images were acquired
-         0: Acquire MJPEG encoded images
-         1: Acquire H264 encoded images(SPS)[Deprecated]
-         2: Acquire H264 encoded images(PPS)[Deprecated]
-         3: Acquire H264 encoded images(I)
-         4: Acquire H264 encoded images(P)
-         5: IMG not changed
- **********************************************************************************/
-int kvmv_read_img(uint16_t _width, uint16_t _height, uint8_t _type, uint16_t _qlty, uint8_t** _pp_kvm_data, uint32_t* _p_kvmv_data_size);
-int free_kvmv_data(uint8_t ** _pp_kvm_data);
-void free_all_kvmv_data();
-void set_h264_gop(uint8_t _gop);
-void set_frame_detact(uint8_t _frame_detact);
-void kvmv_deinit();
-uint8_t kvmv_hdmi_control(uint8_t _en);
 uint8_t kvmv_hdmi_signal_active(void);
 
 #ifdef __cplusplus

server/proto/vm.go

@@ -30,8 +30,8 @@ type GetGpioRsp struct {
 }
 
 type SetScreenReq struct {
-	Type  string `validate:"required"` // resolution / fps / quality
+	Type  string `validate:"required,oneof=type fps quality gop"`
-	Value int    `validate:"number"`   // value
+	Value int    `validate:"number"` // value
 }
 
 type GetScriptsRsp struct {

server/service/mcp/capture/kvm/kvm.go

@@ -8,9 +8,5 @@ import (
 )
 
 func New() mcpservice.Snapshotter {
-	return mcpcapture.NewWithCaptureLease(common.GetKvmVision(), func() (uint16, uint16) {
+	return mcpcapture.NewWithCaptureLease(common.GetKvmVision(), common.InputSize, vm.AcquireHdmiCaptureLeaseForRead)
-		screen := common.GetScreen()
-		common.CheckScreen()
-		return screen.Width, screen.Height
-	}, vm.AcquireHdmiCaptureLeaseForRead)
 }

server/service/picoclaw/screenshot.go

@@ -60,7 +60,7 @@ func (s *Service) Screenshot(c *gin.Context) {
 func (s *Service) captureScreenshot(ctx context.Context, query ScreenshotQuery) ([]byte, ScreenshotMeta, *PicoclawError) {
 	width, height, quality := resolveScreenshotRequest(query)
 
-	screen := common.GetScreen()
+	sourceWidth, sourceHeight := common.InputSize()
 	common.CheckScreen()
 	releaseLease, claimFresh, leaseErr := s.acquireCaptureLease(ctx)
 	if leaseErr != nil {
@@ -94,8 +94,8 @@ func (s *Service) captureScreenshot(ctx context.Context, query ScreenshotQuery)
 				continue
 			}
 			return data, ScreenshotMeta{
-				SourceWidth:   screen.Width,
+				SourceWidth:   sourceWidth,
-				SourceHeight:  screen.Height,
+				SourceHeight:  sourceHeight,
 				CaptureWidth:  width,
 				CaptureHeight: height,
 				Format:        "jpeg",
@@ -108,8 +108,7 @@ func (s *Service) captureScreenshot(ctx context.Context, query ScreenshotQuery)
 
 func resolveScreenshotRequest(query ScreenshotQuery) (uint16, uint16, uint16) {
 	screen := common.GetScreen()
-	width := screen.Width
+	width, height := common.InputSize()
-	height := screen.Height
 	quality := screen.Quality
 
 	if query.Format == "base64" {

server/service/stream/h264_source.go

@@ -3,23 +3,20 @@ package stream
 import (
 	"NanoKVM-Server/common"
 	"sync"
-	"sync/atomic"
 	"time"
 )
 
 type H264Frame struct {
 	Data      []byte
 	Result    int
-	Duration  time.Duration
 	Timestamp int64
 }
 
 type H264Subscription struct {
-	frames             chan H264Frame
+	source *H264Source
-	done               chan struct{}
+	frames chan H264Frame
-	closed             atomic.Bool
+	done   chan struct{}
-	once               sync.Once
+	once   sync.Once
-	waitingForKeyframe bool
 }
 
 type H264Source struct {
@@ -38,7 +35,8 @@ func SubscribeH264() *H264Subscription {
 
 func (s *H264Source) subscribe() *H264Subscription {
 	subscription := &H264Subscription{
-		frames: make(chan H264Frame, 4),
+		source: s,
+		frames: make(chan H264Frame),
 		done:   make(chan struct{}),
 	}
 
@@ -58,6 +56,12 @@ func (s *H264Source) subscribe() *H264Subscription {
 }
 
 func (s *H264Subscription) Next() (H264Frame, bool) {
+	select {
+	case <-s.done:
+		return H264Frame{}, false
+	default:
+	}
+
 	select {
 	case frame := <-s.frames:
 		return frame, true
@@ -68,9 +72,8 @@ func (s *H264Subscription) Next() (H264Frame, bool) {
 
 func (s *H264Subscription) Close() {
 	s.once.Do(func() {
-		s.closed.Store(true)
 		close(s.done)
-		defaultH264Source.remove(s)
+		s.source.remove(s)
 	})
 }
 
@@ -83,18 +86,19 @@ func (s *H264Source) remove(subscription *H264Subscription) {
 func (s *H264Source) run() {
 	screen := common.GetScreen()
 	common.CheckScreen()
-	fps := screen.FPS
-	ticker := time.NewTicker(time.Second / time.Duration(fps))
-	defer ticker.Stop()
-
 	vision := common.GetKvmVision()
-	startTime := time.Now()
+	s.capture(screen, vision.AcquireH264, vision.StopH264)
+}
 
-	for range ticker.C {
+func (s *H264Source) capture(screen *common.Screen, acquire func(uint16, uint16, uint16, uint16) *common.VideoFrame, stop func()) {
+	for {
 		subscribers := s.snapshot()
 		if len(subscribers) == 0 {
 			s.mutex.Lock()
 			if len(s.subscribers) == 0 {
+				if stop != nil {
+					stop()
+				}
 				s.running = false
 				s.mutex.Unlock()
 				return
@@ -103,33 +107,21 @@ func (s *H264Source) run() {
 			continue
 		}
 
-		if screen.FPS != fps && screen.FPS != 0 {
+		current := acquire(0, 0, screen.BitRate, uint16(screen.FPS))
-			fps = screen.FPS
-			ticker.Reset(time.Second / time.Duration(fps))
-		}
-
-		data, result := vision.ReadH264(screen.Width, screen.Height, screen.BitRate)
-		if result < 0 {
-			frame := H264Frame{Result: result}
-			for _, subscription := range subscribers {
-				subscription.send(frame)
-			}
-			continue
-		}
-		if len(data) == 0 {
-			continue
-		}
-
 		frame := H264Frame{
-			Data:      data,
+			Data:      current.Data,
-			Result:    result,
+			Result:    current.Result,
-			Duration:  time.Second / time.Duration(fps),
+			Timestamp: current.Timestamp,
-			Timestamp: time.Since(startTime).Microseconds(),
 		}
 		for _, subscription := range subscribers {
 			subscription.send(frame)
 		}
-		GetFrameRateCounter().Update()
+		current.Release()
+		if frame.Result >= 0 && len(frame.Data) > 0 {
+			GetFrameRateCounter().Update()
+		} else {
+			time.Sleep(time.Second / time.Duration(max(screen.FPS, 1)))
+		}
 	}
 }
 
@@ -145,41 +137,15 @@ func (s *H264Source) snapshot() []*H264Subscription {
 }
 
 func (s *H264Subscription) send(frame H264Frame) bool {
-	if s.closed.Load() {
+	select {
+	case <-s.done:
 		return false
+	default:
 	}
-	if s.waitingForKeyframe {
-		if frame.Result != 3 {
-			return false
-		}
-		s.waitingForKeyframe = false
-	}
-
 	select {
 	case s.frames <- frame:
 		return true
 	case <-s.done:
 		return false
-	default:
-	}
-
-	for {
-		select {
-		case <-s.frames:
-		case <-s.done:
-			return false
-		default:
-			s.waitingForKeyframe = true
-			if frame.Result != 3 {
-				return false
-			}
-			s.waitingForKeyframe = false
-			select {
-			case s.frames <- frame:
-				return true
-			case <-s.done:
-				return false
-			}
-		}
 	}
 }

server/service/stream/mjpeg/streamer.go

@@ -4,7 +4,9 @@ import (
 	"NanoKVM-Server/common"
 	"NanoKVM-Server/service/stream"
 	"NanoKVM-Server/service/vm"
+	"bytes"
 	"fmt"
+	"image/jpeg"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -83,32 +85,69 @@ func (s *Streamer) getClients() []*gin.Context {
 }
 
 func (s *Streamer) run() {
-	defer atomic.StoreInt32(&s.running, 0)
-
 	screen := common.GetScreen()
 	common.CheckScreen()
-	fps := screen.FPS
 
 	vision := common.GetKvmVision()
+	s.streamFrames(screen, vision.AcquireMjpeg, vision.StopMjpeg)
+}
 
-	ticker := time.NewTicker(time.Second / time.Duration(fps))
+func (s *Streamer) streamFrames(screen *common.Screen, acquire func(uint16, uint16, uint16, uint16) *common.VideoFrame, stop func()) {
-	defer ticker.Stop()
+	frames := make(chan *common.VideoFrame)
+	done := make(chan struct{})
+	captureDone := make(chan struct{})
+	go func() {
+		defer close(captureDone)
+		defer stop()
+		for {
+			select {
+			case <-done:
+				return
+			default:
+			}
+			frame := acquire(0, 0, screen.Quality, uint16(screen.FPS))
+			select {
+			case frames <- frame:
+				frame.Release()
+			case <-done:
+				frame.Release()
+				return
+			}
+			if frame.Result < 0 {
+				select {
+				case <-time.After(time.Second / time.Duration(max(screen.FPS, 1))):
+				case <-done:
+					return
+				}
+			}
+		}
+	}()
 
-	for range ticker.C {
+	for {
 		clients := s.getClients()
 		if len(clients) == 0 {
-			log.Debug("mjpeg stream stopped due to no clients")
+			s.mutex.Lock()
-			return
+			if len(s.clients) == 0 {
+				close(done)
+				<-captureDone
+				atomic.StoreInt32(&s.running, 0)
+				s.mutex.Unlock()
+				log.Debug("mjpeg stream stopped due to no clients")
+				return
+			}
+			s.mutex.Unlock()
+			continue
 		}
 
-		data, result := vision.ReadMjpeg(screen.Width, screen.Height, screen.Quality)
+		frame := <-frames
+		data, result := frame.Data, frame.Result
 		stream.UpdateCaptureStatus(stream.CaptureModeMJPEG, result)
 		if result < 0 || result == 5 || len(data) == 0 {
 			continue
 		}
 
 		if s.frameCacheEnabled() {
-			s.setLatestFrame(data, screen.Width, screen.Height)
+			s.setLatestFrame(data)
 		}
 
 		for _, client := range clients {
@@ -118,16 +157,15 @@ func (s *Streamer) run() {
 			}
 		}
 
-		if screen.FPS != fps && screen.FPS != 0 {
-			fps = screen.FPS
-			ticker.Reset(time.Second / time.Duration(fps))
-		}
-
 		stream.GetFrameRateCounter().Update()
 	}
 }
 
-func (s *Streamer) setLatestFrame(data []byte, width uint16, height uint16) {
+func (s *Streamer) setLatestFrame(data []byte) {
+	size, err := jpeg.DecodeConfig(bytes.NewReader(data))
+	if err != nil {
+		return
+	}
 	frameCopy := append([]byte(nil), data...)
 
 	s.frameMutex.Lock()
@@ -135,8 +173,8 @@ func (s *Streamer) setLatestFrame(data []byte, width uint16, height uint16) {
 
 	s.latestFrame = LatestFrame{
 		Data:       frameCopy,
-		Width:      width,
+		Width:      uint16(size.Width),
-		Height:     height,
+		Height:     uint16(size.Height),
 		CapturedAt: time.Now(),
 	}
 }

server/service/stream/webrtc/manager.go

@@ -5,24 +5,14 @@ import (
 	"NanoKVM-Server/service/vm"
 
 	"github.com/gorilla/websocket"
-	"github.com/pion/rtp"
-	"github.com/pion/rtp/codecs"
-	"github.com/pion/webrtc/v4/pkg/media"
 	log "github.com/sirupsen/logrus"
 )
 
 func NewWebRTCManager() *WebRTCManager {
 	m := &WebRTCManager{
-		clients: make(map[*websocket.Conn]*Client),
+		clients:         make(map[*websocket.Conn]*Client),
-		videoPacketizer: rtp.NewPacketizer(
+		videoPacketizer: newH264Packetizer(),
-			1200,
+		videoSending:    false,
-			100,
-			0x1234ABCD,
-			&codecs.H264Payloader{},
-			rtp.NewRandomSequencer(),
-			90000,
-		),
-		videoSending: false,
 	}
 	m.updateClientSnapshotLocked()
 
@@ -85,7 +75,7 @@ func (m *WebRTCManager) StartVideoStream() {
 	m.videoSending = true
 	m.mutex.Unlock()
 
-	go m.sendVideoStream()
+	go m.writeVideoStream()
 	log.Debugf("start sending h264 stream")
 }
 
@@ -101,28 +91,21 @@ func (m *WebRTCManager) stopVideoStreamIfIdle() bool {
 	return true
 }
 
-func (m *WebRTCManager) sendVideoStream() {
+func (m *WebRTCManager) writeVideoStream() {
 	subscription := stream.SubscribeH264()
 	defer subscription.Close()
-	samples, writerDone := m.startVideoWriter()
 
 	for {
 		frame, ok := subscription.Next()
 		if !ok {
-			close(samples)
-			<-writerDone
 			return
 		}
 		clients := m.getClients()
 		if len(clients) == 0 {
-			close(samples)
-			<-writerDone
 			if m.stopVideoStreamIfIdle() {
 				log.Debugf("stop sending h264 stream")
 				return
 			}
-			samples, writerDone = m.startVideoWriter()
-
 			continue
 		}
 
@@ -131,33 +114,13 @@ func (m *WebRTCManager) sendVideoStream() {
 			continue
 		}
 
-		sample := media.Sample{
+		packets := m.videoPacketizer.Packetize(frame.Data, frame.Timestamp)
-			Data:     frame.Data,
+		for _, client := range clients {
-			Duration: frame.Duration,
+			if err := client.track.writeVideoPackets(packets); err != nil {
-		}
+				log.Errorf("failed to write h264 video to client: %s", err)
-
+				m.RemoveClient(client.WsConn())
-		samples <- sample
+				client.Close()
-	}
-}
-
-func (m *WebRTCManager) startVideoWriter() (chan media.Sample, <-chan struct{}) {
-	samples := make(chan media.Sample, 1)
-	done := make(chan struct{})
-
-	go func() {
-		defer close(done)
-		for sample := range samples {
-			packets := m.videoPacketizer.Packetize(sample.Data, uint32(sample.Duration.Seconds()*90000))
-			for _, client := range m.getClients() {
-				err := client.track.writeVideoPackets(packets)
-				if err != nil {
-					log.Errorf("failed to write h264 video to client: %s", err)
-					m.RemoveClient(client.WsConn())
-					client.Close()
-				}
 			}
 		}
-	}()
+	}
-
-	return samples, done
 }

server/service/stream/webrtc/packetizer.go

@@ -0,0 +1,33 @@
+package webrtc
+
+import (
+	"github.com/pion/rtp"
+	"github.com/pion/rtp/codecs"
+)
+
+type h264Packetizer struct {
+	packetizer  rtp.Packetizer
+	lastTicks   uint64
+	initialized bool
+}
+
+func newH264Packetizer() *h264Packetizer {
+	return &h264Packetizer{
+		packetizer: rtp.NewPacketizer(1200, 100, 0x1234ABCD,
+			&codecs.H264Payloader{}, rtp.NewRandomSequencer(), 90000),
+	}
+}
+
+func (p *h264Packetizer) Packetize(data []byte, timestamp int64) []*rtp.Packet {
+	if len(data) == 0 {
+		return nil
+	}
+
+	ticks := uint64(timestamp) * 90 / 1000
+	if p.initialized {
+		p.packetizer.SkipSamples(uint32(ticks - p.lastTicks))
+	}
+	p.lastTicks = ticks
+	p.initialized = true
+	return p.packetizer.Packetize(data, 0)
+}

server/service/stream/webrtc/types.go

@@ -5,14 +5,13 @@ import (
 	"sync/atomic"
 
 	"github.com/gorilla/websocket"
-	"github.com/pion/rtp"
 	"github.com/pion/webrtc/v4"
 )
 
 type WebRTCManager struct {
 	clients         map[*websocket.Conn]*Client
 	clientSnapshot  atomic.Pointer[[]*Client]
-	videoPacketizer rtp.Packetizer
+	videoPacketizer *h264Packetizer
 	videoSending    bool
 	mutex           sync.Mutex
 	viewerVersion   uint64

server/service/vm/screen.go

@@ -13,10 +13,9 @@ import (
 )
 
 var screenFileMap = map[string]string{
-	"type":       "/kvmapp/kvm/type",
+	"type":    "/kvmapp/kvm/type",
-	"fps":        "/kvmapp/kvm/fps",
+	"fps":     "/kvmapp/kvm/fps",
-	"quality":    "/kvmapp/kvm/qlty",
+	"quality": "/kvmapp/kvm/qlty",
-	"resolution": "/kvmapp/kvm/res",
 }
 
 func (s *Service) SetScreen(c *gin.Context) {

support/sg2002/README.md

@@ -1,7 +1,15 @@
 # NanoKVM Support Instructions
 
 ## Environment Preparation
-1. NanoKVM-Lite/Full/PCIe is based on the SG2002 as the main control chip. The projects in the support section are compiled under the [MaixCDK](https://github.com/sipeed/MaixCDK) framework. Before compiling, please ensure that the `MaixCDK` environment is correctly configured. For configuration instructions, click [here](https://github.com/sipeed/MaixCDK/blob/main/docs/doc_zh/README.md).
+
+NanoKVM-Lite/Full/PCIe uses the SG2002. The system daemon and `libkvm.so` wrapper
+build with [MaixCDK](https://github.com/sipeed/MaixCDK); `libkvm_mmf.so` builds
+separately with CMake using the Sophgo sources in that checkout. Configure and
+activate the [MaixCDK environment](https://github.com/sipeed/MaixCDK/blob/main/docs/doc_zh/README.md)
+and add the RISC-V musl toolchain to `PATH`. Run the commands below from
+`support/sg2002/`. Alternatively, `make vision` at the repository root builds
+the video libraries in Docker. `make app` builds the server using the prebuilt
+libraries in `server/dl_lib/`.
 
 ## kvm_system Compilation Instructions
 
@@ -11,16 +19,17 @@
 2. Modify the paths of `MAIXCDK_PATH` and `NanoKVM_PATH` in `./build`.
 3. Execute `./build kvm_system` to compile `kvm_system`.
 4. Use `scp ./kvm_system/dist/kvm_system_release/kvm_system root@192.168.x.x:/kvmapp/kvm_system` to copy it to NanoKVM for testing.
-5. Use `./build add_to_kvmapp` to place the executable file into the `/kvmapp` installation package.
+5. Use `./build kvm_system add_to_kvmapp` to stage the executable in the repository's `kvmapp/` directory.
 6. Use `./build kvm_system clean` to clean the compilation of `kvm_system`.
 
 ## kvm_vision Compilation Instructions
 
-> `kvm_vision` refers to the image acquisition and encoding subsystem of NanoKVM, compiled with MaixCDK to produce dynamic libraries for Go calls. Use `kvm_vision_test` to compile and test the dynamic library.
+> `kvm_vision` builds `libkvm_mmf.so`, then the MaixCDK wrapper `libkvm.so` and the `kvm_vision_test` example.
 
 1. Before compiling, please ensure that the above-mentioned `MaixCDK` environment is correctly configured.
 2. Modify the paths of `MAIXCDK_PATH` and `NanoKVM_PATH` in `./build`.
-3. Execute `./build kvm_vision` to compile `kvm_vision_test`.
+3. Execute `./build kvm_vision` to build both libraries and the example. The library build and install directories are `kvm_vision_test/build/kvm_mmf-build/` and `kvm_vision_test/build/kvm_mmf-install/`.
-4. You can test the dynamic libraries in `kvm_vision_test/dist/kvm_vision_test_release/dl_lib/`.
+4. The resulting libraries are in `kvm_vision_test/dist/kvm_vision_test_release/dl_lib/`.
-5. Use `./build add_to_kvmapp` to place the dynamic libraries into the `/kvmapp` installation package.
+5. Run `./build kvm_vision add_to_kvmapp` to stage them in the repository's `kvmapp/server/dl_lib/`.
-6. Use `./build kvm_vision clean` to clean the compilation of `kvm_vision`.
+6. After changing the video library interfaces, update both prebuilt files `server/dl_lib/libkvm.so` and `server/dl_lib/libkvm_mmf.so` from the same build. Server compilation uses these files directly; see the [server build instructions](../../server/README.md#compile--deploy).
+7. Use `./build kvm_vision clean` to remove the build outputs, including both library build and install directories.

support/sg2002/README_ZH.md

@@ -1,7 +1,14 @@
 # NanoKVM support 说明
 
 ## 环境准备
-1. NanoKVM-Lite/Full/PCIe 以SG2002为主控芯片，support 部分的工程在[MaixCDK](https://github.com/sipeed/MaixCDK) 框架下编译。编译前，请保证`MaixCDK`环境已正确配置，配置教程点击[这里](https://github.com/sipeed/MaixCDK/blob/main/docs/doc_zh/README.md)
+
+NanoKVM-Lite/Full/PCIe 以 SG2002 为主控芯片。系统程序和 `libkvm.so` 包装层借助
+[MaixCDK](https://github.com/sipeed/MaixCDK) 编译，`libkvm_mmf.so` 使用该目录中的
+Sophgo 源码通过 CMake 独立编译。请先配置并激活
+[MaixCDK 环境](https://github.com/sipeed/MaixCDK/blob/main/docs/doc_zh/README.md)，
+将 RISC-V musl 工具链加入 `PATH`。以下命令在 `support/sg2002/` 下执行。
+也可以在仓库根目录执行 `make vision`，通过 Docker 构建视频库。
+`make app` 使用 `server/dl_lib/` 中的预编译库构建服务器。
 
 ## kvm_system 编译说明
 
@@ -11,16 +18,17 @@
 2. 修改 `./build` 中 `MAIXCDK_PATH` 和 `NanoKVM_PATH`的路径
 3. 执行 `./build kvm_system` 编译 kvm_system 
 4. 使用 `scp ./kvm_system/dist/kvm_system_release/kvm_system root@192.168.x.x:/kvmapp/kvm_system` 拷贝入 NanoKVM 测试
-5. 使用 `./build add_to_kvmapp` 可以将可执行文件放入 `/kvmapp` 安装包内
+5. 使用 `./build kvm_system add_to_kvmapp` 将可执行文件放入仓库的 `kvmapp/` 目录
 6. 使用 `./build kvm_system clean` 可以清除 kvm_system 的编译
 
 ## kvm_vision 编译说明
 
-> `kvm_vision` 是 NanoKVM 图像获取编码子系统的统称，使用MaixCDK编译出动态库供Go调用，使用 `kvm_vision_test` 编译和测试动态库
+> `kvm_vision` 先构建 `libkvm_mmf.so`，再构建 MaixCDK 包装层 `libkvm.so` 和 `kvm_vision_test` 示例程序。
 
 1. 编译前，请保证上述 1.`MaixCDK`环境已正确配置
 2. 修改 `./build` 中 `MAIXCDK_PATH` 和 `NanoKVM_PATH`的路径
-3. 执行 `./build kvm_vision` 编译 kvm_vision_test 
+3. 执行 `./build kvm_vision` 编译两份库和示例程序。库的编译和安装目录分别为 `kvm_vision_test/build/kvm_mmf-build/`、`kvm_vision_test/build/kvm_mmf-install/`。
-4. 可使用 `kvm_vision_test/dist/kvm_vision_test_release/dl_lib/` 中的动态库测试
+4. 构建后的动态库位于 `kvm_vision_test/dist/kvm_vision_test_release/dl_lib/`。
-5. 使用 `./build add_to_kvmapp` 可以将动态库放入 `/kvmapp` 安装包内
+5. 执行 `./build kvm_vision add_to_kvmapp`，将库放入仓库的 `kvmapp/server/dl_lib/`。
-6. 使用 `./build kvm_vision clean` 可以清除 kvm_system 的编译
+6. 修改视频库接口后，用同一批构建的两份库更新 `server/dl_lib/libkvm.so` 和 `server/dl_lib/libkvm_mmf.so`。服务器直接使用这些预编译库，编译步骤见[服务器说明](../../server/README_ZH.md#编译部署)。
+7. 执行 `./build kvm_vision clean` 清理构建产物，包括上述库的编译和安装目录。

support/sg2002/additional/kvm/CMakeLists.txt

@@ -1,4 +1,4 @@
-list(APPEND ADD_REQUIREMENTS vision basic peripheral)
+list(APPEND ADD_REQUIREMENTS kvm_mmf basic peripheral)
 list(APPEND ADD_INCLUDE "include")
 list(APPEND ADD_LINK_DEFINITIONS_PRIVATE -Wl,--as-needed)
 

support/sg2002/additional/kvm/include/kvm_vision.h

@@ -1,79 +1,43 @@
-
-
 #ifndef KVM_VISION_H_
 #define KVM_VISION_H_
 
-#include "kvm_mmf.hpp"
+#include <stdint.h>
-#include "maix_camera.hpp"
-#include "maix_basic.hpp"
-
-#include "maix_i2c.hpp"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include <fcntl.h> /* low-level i/o */
-#include <unistd.h>
-#include <string.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <pthread.h>
-#include <time.h>
 
-// #include "kvm_mmf.hpp"
+#define IMG_BUFFER_FULL -3
+#define IMG_VENC_ERROR  -2
+#define IMG_NOT_EXIST   -1
+#define IMG_MJPEG_TYPE   0
+#define IMG_H264_TYPE_IF 3
+#define IMG_H264_TYPE_PF 4
 
-#define IMG_BUFFER_FULL			-3
+typedef struct {
-#define IMG_VENC_ERROR			-2
+    uint8_t *data;
-#define IMG_NOT_EXIST			-1
+    uint32_t size;
-#define IMG_MJPEG_TYPE			0
+    uint64_t timestamp;
-#define IMG_H264_TYPE_SPS		1
+    uint64_t token;
-#define IMG_H264_TYPE_PPS		2
+    uint8_t codec;
-#define IMG_H264_TYPE_IF		3
+} kvmv_frame_t;
-#define IMG_H264_TYPE_PF		4
 
-#define NORMAL_RES                      0
+void kvmv_init(uint8_t debug_info);
-#define NEW_RES                         1
+void kvmv_deinit(void);
-#define UNSUPPORT_RES                   2
-#define UNKNOWN_RES                     3
-#define ERROR_RES                       4
 
-void kvmv_init(uint8_t _debug_info_en = 0);
+// type: 0 = MJPEG, 1 = H.264. A zero width/height follows the input size.
-void set_venc_auto_recyc(uint8_t _enable);
+// quality: MJPEG 50..100, H.264 500..10000 Kbps. fps: 1..60.
-/**********************************************************************************
+// Returns -7 input error, -6 unsupported size, -5 busy, -4 reconfiguring,
- * @name    kvmv_read_img
+// -3 allocation failure, -2 encoder error, -1 no frame, 0 JPEG, 3 IDR,
- * @author  Sipeed BuGu
+// 4 P-frame, or 5 unchanged JPEG. Release every successful frame once.
- * @date    2024/10/25
+int kvmv_read_img(uint16_t width, uint16_t height, uint8_t type,
- * @version R1.0
+                  uint16_t quality, uint16_t fps, kvmv_frame_t *frame);
- * @brief   Acquire the encoded image with auto init
+// Frees the copied bytes and releases the encoder lease. Safe after shutdown.
- * @param	_width				@input: 	Output image width
+void kvmv_release_frame(kvmv_frame_t *frame);
- * @param	_height				@input: 	Output image height
+void kvmv_stop_stream(uint8_t type);
- * @param	_type				@input: 	Encode type
+void set_h264_gop(uint8_t gop);
- * @param	_qlty				@input: 	MJPEG: (50-100) | H264:  (500-10000)
+void set_frame_detact(uint8_t interval);
- * @param	_pp_kvm_data		@output: 	Encode data
+uint8_t kvmv_hdmi_control(uint8_t enable);
- * @param	_p_kvmv_data_size	@output: 	Encode data size
- * @return
-        -7: HDMI INPUT RES ERROR
-        -6: Unsupported resolution, please modify it in the host settings.
-        -5: Retrieving image, please wait
-        -4: Modifying image resolution, please wait
-        -3: img buffer full
-        -2: VENC Errorl
-        -1: No images were acquired
-         0: Acquire MJPEG encoded images
-         1: Acquire H264 encoded images(SPS)[Deprecated]
-         2: Acquire H264 encoded images(PPS)[Deprecated]
-         3: Acquire H264 encoded images(I)
-         4: Acquire H264 encoded images(P)
-         5: IMG not changed
- **********************************************************************************/
-int kvmv_read_img(uint16_t _width, uint16_t _height, uint8_t _type, uint16_t _qlty, uint8_t** _pp_kvm_data, uint32_t* _p_kvmv_data_size);
-int free_kvmv_data(uint8_t ** _pp_kvm_data);
-void free_all_kvmv_data();
-void set_h264_gop(uint8_t _gop);
-void set_frame_detact(uint8_t _frame_detact);
-void kvmv_deinit();
-uint8_t kvmv_hdmi_control(uint8_t _en);
 uint8_t kvmv_hdmi_signal_active(void);
 
 #ifdef __cplusplus

support/sg2002/additional/kvm/src/kvm_vision.cpp

@@ -1,45 +1,29 @@
-/**
- * 待解决的问题:
- * // 分辨率跟随输出
- * // 自动切换分辨率
- * // HDMI分辨率问题
- * // H264输入空图,VENC容易炸问题
- * // MJPEG/H264切换时卡退
- * // 再加一条手动清空全部内存
- * // deinit时free全部内存
- * // free 错内存时会炸的问题
- */
 #include "kvm_vision.h"
+#include "maix_basic.hpp"
+#include "maix_i2c.hpp"
 #include "vi_state_shared.hpp"
 #include "internal/vi_state_writer.hpp"
+#include "kvm_mmf.hpp"
 
+#include <atomic>
+#include <cstdarg>
+#include <mutex>
+#include <pthread.h>
 #include <errno.h>
 #include <sys/stat.h>
 
-#define default_venc_chn        1
-
 #define VENC_MJPEG              0
 #define VENC_H264               1
 
-#define KVMV_MAX_TRY_NUM	   	2
 #define vi_min_width            32
 #define vi_min_height           3
 #define vi_max_width            1920
 #define vi_max_height           1080
 
-#define Farame_sample_size      76800   // 320*240
-
 #define default_vi_width        1920
 #define default_vi_height       1080
-#define default_vpss_width      1920
-#define default_vpss_height     1080
-#define default_venc_type       VENC_MJPEG
-#define default_mjpeg_qlty      60
-#define default_h264_qlty       1000
 #define default_h264_gop        30
-#define fresh_frame_discard_count 5
 
-#define kvmv_data_buffer_size   4
 #define Try_rounds_HDMI_err_res 5
 
 #define vi_width_path           "/kvmapp/kvm/width"
@@ -56,7 +40,9 @@
 #define LT6911_READ 	0xFF
 #define LT6911_WRITE 	0x00
 
-pthread_mutex_t vi_mutex;
+std::mutex capture_mutex[2];
+pthread_t detection_thread, watchdog_thread;
+bool detection_started = false, watchdog_started = false;
 pthread_mutex_t hdmi_signal_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static char NanoKVM_edit[] = {
@@ -75,56 +61,30 @@ using namespace maix::sys;
 using namespace maix::peripheral;
 i2c::I2C LT6911_i2c(4, i2c::Mode::MASTER);
 
-typedef struct {
+enum ResolutionState { NORMAL_RES, NEW_RES, UNSUPPORT_RES, UNKNOWN_RES, ERROR_RES };
-	uint16_t vi_width = default_vi_width;
+
-	uint16_t vi_height = default_vi_height;
+struct {
-	uint16_t vpss_width = default_vpss_width;
+    std::atomic<uint16_t> vi_width{default_vi_width};
-	uint16_t vpss_height = default_vpss_height;
+    std::atomic<uint16_t> vi_height{default_vi_height};
-	uint8_t venc_type;
+    uint8_t frame_detact = 0;
-	uint16_t qlty;
+    std::atomic<bool> reopen_cam_flag{false};
-	uint8_t stream_stop;
+    std::atomic<bool> try_exit_thread{false};
-	uint8_t frame_detact = 0;
-	uint8_t display;
-    uint8_t reinit_flag = 1;
-    uint8_t reopen_cam_flag = 0;
-    uint8_t hdmi_cable_state = 0;
-    uint8_t try_exit_thread = 0;
-    uint8_t thread_is_running = 0;
-    uint8_t Auto_res = 0;
     uint8_t hdmi_version = 0;
     uint8_t hw_version = 0;
-    uint8_t hdmi_stop_flag = 0;
+    std::atomic<bool> hdmi_stop_flag{false};
-    uint8_t hdmi_reading_flag = 0;
+    std::atomic<bool> hdmi_reading_flag{false};
     uint8_t hdmi_mode = 0;
-    uint8_t hdmi_res_type = 0;
+    std::atomic<uint8_t> hdmi_res_type{NORMAL_RES};
-    uint8_t hdmi_res_err = 0;
+    std::atomic<uint8_t> hdmi_res_err{NORMAL_RES};
     uint8_t hdmi_try_rounds = 0;
-    uint8_t vi_detect_state = 0;
+    std::atomic<uint8_t> vi_detect_state{0};
-    uint8_t venc_auto_recyc = 0;
+} kvmv_cfg;
-    uint8_t fresh_frame_count = 0;
-} kvmv_cfg_t;
 
-typedef struct {
+struct {
-	uint8_t* p_img_data = NULL;
+    uint64_t hash = 0;
-    uint8_t img_data_type = 0;
+    uint8_t count = 0;
-    uint32_t img_data_size = 0;
+    bool unchanged = false;
-} kvmv_data_t;
+} jpeg_detection;
-
-typedef struct {
-    uint8_t mmf_venc_chn;
-	uint8_t enc_h264_running;
-	uint8_t enc_h264_init;
-    mmf_venc_cfg_t kvm_venc_cfg;
-} kvm_venc_t;
-
-camera::Camera *cam = new camera::Camera(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
-// camera::Camera *cam = new camera::Camera(320, 240, image::FMT_RGB888);
-
-kvmv_cfg_t kvmv_cfg;
-
-kvmv_data_t kvmv_data_buffer[kvmv_data_buffer_size];
-
-uint8_t kvmv_data_buffer_index = 0;
 
 uint8_t debug_en = 0;
 uint8_t last_vi_state_code = 0;
@@ -200,16 +160,13 @@ static void set_hdmi_capture_enabled(uint8_t enabled)
     pthread_mutex_unlock(&hdmi_signal_mutex);
 }
 
-static void set_hdmi_detection_state(uint8_t active)
-{
-    __atomic_store_n(&kvmv_cfg.hdmi_cable_state, active != 0 ? 1 : 0, __ATOMIC_RELEASE);
-    set_hdmi_signal_state(active);
-}
-
 void debug(const char *format, ...)
 {
     if(debug_en){
-        printf(format);
+        va_list args;
+        va_start(args, format);
+        vprintf(format, args);
+        va_end(args);
     }
 }
 
@@ -219,17 +176,10 @@ uint8_t refresh_vi_state()
 	// that the publication cadence measures wall-clock time, not read time plus it.
     last_vi_state_refresh_ms = vi_state_shared::monotonic_ms();
     last_vi_state_code = vi_state_shared::refresh();
-    set_hdmi_detection_state(last_vi_state_code == 1 || last_vi_state_code >= 3);
+    set_hdmi_signal_state(last_vi_state_code == 1 || last_vi_state_code >= 3);
     return last_vi_state_code;
 }
 
-uint8_t to_roll(int8_t _input)
-{
-    if(_input <                      0) return _input + kvmv_data_buffer_size;
-    if(_input >= kvmv_data_buffer_size) return _input - kvmv_data_buffer_size;
-    return _input;
-}
-
 int maxmin_data(int _max, int _min, int _data)
 {
 	if(_data > _max) return _max;
@@ -237,17 +187,6 @@ int maxmin_data(int _max, int _min, int _data)
 	return _data;
 }
 
-kvmv_data_t* get_save_buffer()
-{
-    kvmv_data_buffer_index = to_roll(kvmv_data_buffer_index + 1);
-    // debug("[kvmv]kvmv_data_buffer_index = %d\n", kvmv_data_buffer_index);
-    // debug("[kvmv]kvmv_data_buffer.p_img_data = %d\n", kvmv_data_buffer[kvmv_data_buffer_index].p_img_data);
-    if(kvmv_data_buffer[kvmv_data_buffer_index].p_img_data == NULL){
-        return &kvmv_data_buffer[kvmv_data_buffer_index];
-    }
-    return NULL;
-}
-
 // ====HDMI RES==================================================
 
 uint16_t hdmi_res_list[][2] = {
@@ -435,12 +374,12 @@ int get_manual_resolution(void)
     // res change ?
     if(kvmv_cfg.vi_width != tmp_width){
         kvmv_cfg.vi_width = tmp_width;
-        printf("[kvmk] get new width = %d\n", kvmv_cfg.vi_width);
+        printf("[kvmk] get new width = %d\n", kvmv_cfg.vi_width.load());
         res = 1;
     }
     if(kvmv_cfg.vi_height != tmp_height){
         kvmv_cfg.vi_height = tmp_height;
-        printf("[kvmk] get new height = %d\n", kvmv_cfg.vi_height);
+        printf("[kvmk] get new height = %d\n", kvmv_cfg.vi_height.load());
         res = 1;
     }
     return res;
@@ -452,12 +391,12 @@ uint8_t auto_try_res()
     uint8_t err_code;
     uint8_t auto_trying_times = 0;
 
-    for (auto_trying_times = 0; auto_trying_times < sizeof(hdmi_res_list)/4; auto_trying_times++){
+    for (auto_trying_times = 0; auto_trying_times < sizeof(hdmi_res_list)/4 && !kvmv_cfg.try_exit_thread.load(); auto_trying_times++){
         err_code = refresh_vi_state();
         switch(err_code){
         case 0:
             // shouldn't be possible to run here
-            cam->restart(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
+            kvm_mmf::restart();
             printf("[kvmv] VI not init\n");
             break;
         case 1:
@@ -484,8 +423,8 @@ uint8_t auto_try_res()
 
             kvmv_cfg.vi_width = hdmi_res_list[auto_trying_times][0];
             kvmv_cfg.vi_height = hdmi_res_list[auto_trying_times][1];
-            printf("[kvmv] restart cam...\n");
+            printf("[kvmv] restart video...\n");
-            cam->restart(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
+            kvm_mmf::restart();
             time::sleep_ms(50);
             break;
         case 7: // Unknown reason
@@ -1121,7 +1060,7 @@ void* watchdog_sf_feed(void * arg)
 {
     while(true)
     {
-        if(kvmv_cfg.try_exit_thread == 1)
+        if(kvmv_cfg.try_exit_thread.load() == 1)
             break;
         time::sleep_ms(500);
         if (watchdog_sf_is_open()){
@@ -1133,6 +1072,7 @@ void* watchdog_sf_feed(void * arg)
             vision_update_watchdog();
         }
     }
+    return nullptr;
 }
 
 void get_hdmi_version()
@@ -1172,6 +1112,16 @@ void get_hdmi_version()
     }
 }
 
+static uint8_t update_hdmi_resolution()
+{
+    uint16_t width = kvmv_cfg.vi_width;
+    uint16_t height = kvmv_cfg.vi_height;
+    uint8_t result = lt6911_get_csi_res(&width, &height);
+    kvmv_cfg.vi_width = width;
+    kvmv_cfg.vi_height = height;
+    return result;
+}
+
 void* vi_subsystem_detection(void * arg)
 {
 	uint64_t __attribute__((unused)) int_time;
@@ -1184,7 +1134,6 @@ void* vi_subsystem_detection(void * arg)
     uint8_t falling_times = 0;
 	uint8_t cam_need_restart = 0;
     uint8_t auto_change_mode = 0;
-    kvmv_cfg.thread_is_running = 1;
 	if(access("/proc/lt_int", F_OK) != 0){
 		time::sleep_ms(10);
 		debug("[hdmi]/proc/lt_int not ok\n");
@@ -1196,7 +1145,7 @@ void* vi_subsystem_detection(void * arg)
     last_vi_state_refresh_ms = vi_state_shared::monotonic_ms() - vi_state_publish_interval_ms;
     while(true)
     {
-        if(kvmv_cfg.try_exit_thread == 1)
+        if(kvmv_cfg.try_exit_thread.load() == 1)
             break;
 
         uint8_t get_new_hdmi_mode = get_hdmi_mode();
@@ -1258,8 +1207,8 @@ void* vi_subsystem_detection(void * arg)
                                 if(lt6911_get_hdmi_res()){
                                     // hdmi get res
                                     debug("[hdmi] C HDMI cable insertion!\n");
-                                    set_hdmi_detection_state(1);
+                                    set_hdmi_signal_state(1);
-                                    kvmv_cfg.hdmi_res_type = lt6911_get_csi_res(&kvmv_cfg.vi_width, &kvmv_cfg.vi_height);
+                                    kvmv_cfg.hdmi_res_type = update_hdmi_resolution();
                                     if (kvmv_cfg.hdmi_res_type == NEW_RES) kvmv_cfg.reopen_cam_flag = 1;
                                     else if (kvmv_cfg.hdmi_res_type == UNKNOWN_RES){
                                         /* Move HDMI resolution modification directly
@@ -1270,7 +1219,7 @@ void* vi_subsystem_detection(void * arg)
                                 } else {
                                     // HDMI res = 0*0/x*0
                                     debug("[hdmi] C HDMI cable unplugged!\n");
-                                    set_hdmi_detection_state(0);
+                                    set_hdmi_signal_state(0);
                                 }
                                 lt6911_disable();
                             }
@@ -1283,8 +1232,8 @@ void* vi_subsystem_detection(void * arg)
                                 if(lt6911_get_hdmi_res()){
                                     // hdmi get res
                                     debug("[hdmi] UXC HDMI cable insertion!\n");
-                                    set_hdmi_detection_state(1);
+                                    set_hdmi_signal_state(1);
-                                    kvmv_cfg.hdmi_res_type = lt6911_get_csi_res(&kvmv_cfg.vi_width, &kvmv_cfg.vi_height);
+                                    kvmv_cfg.hdmi_res_type = update_hdmi_resolution();
                                     if (kvmv_cfg.hdmi_res_type == NEW_RES) kvmv_cfg.reopen_cam_flag = 1;
                                     else if (kvmv_cfg.hdmi_res_type == UNKNOWN_RES){
                                         /* Move HDMI resolution modification directly
@@ -1295,7 +1244,7 @@ void* vi_subsystem_detection(void * arg)
                                 } else {
                                     // HDMI res = 0*0/x*0
                                     debug("[hdmi] UXC HDMI cable unplugged!\n");
-                                    set_hdmi_detection_state(0);
+                                    set_hdmi_signal_state(0);
                                 }
                                 lt6911_disable();
                             }
@@ -1308,8 +1257,8 @@ void* vi_subsystem_detection(void * arg)
                                 if(lt6911_get_hdmi_res()){
                                     // hdmi get res
                                     debug("[hdmi] D HDMI cable insertion!\n");
-                                    set_hdmi_detection_state(1);
+                                    set_hdmi_signal_state(1);
-                                    kvmv_cfg.hdmi_res_type = lt6911_get_csi_res(&kvmv_cfg.vi_width, &kvmv_cfg.vi_height);
+                                    kvmv_cfg.hdmi_res_type = update_hdmi_resolution();
                                     if (kvmv_cfg.hdmi_res_type == NEW_RES) kvmv_cfg.reopen_cam_flag = 1;
                                     else if (kvmv_cfg.hdmi_res_type == UNKNOWN_RES){
                                         /* Move HDMI resolution modification directly
@@ -1320,7 +1269,7 @@ void* vi_subsystem_detection(void * arg)
                                 } else {
                                     // HDMI res = 0*0/x*0
                                     debug("[hdmi] D HDMI cable unplugged!\n");
-                                    set_hdmi_detection_state(0);
+                                    set_hdmi_signal_state(0);
                                 }
                                 lt6911_disable();
                             }
@@ -1392,8 +1341,8 @@ void* vi_subsystem_detection(void * arg)
                 if (kvmv_cfg.vi_detect_state == 1){
                     // detect_res
                     if (get_manual_resolution()) {
-                        debug("[kvmv] restart cam...\n");
+                        debug("[kvmv] restart video...\n");
-                        cam->restart(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
+                        kvm_mmf::restart();
                     }
 
                     // Sample after a manual resolution change.
@@ -1442,554 +1391,121 @@ void* vi_subsystem_detection(void * arg)
 
 		time::sleep_ms(10);
     }
-    kvmv_cfg.thread_is_running = 0;
+    return nullptr;
-}
-
-int sync_vi_res()
-{
-    int res = 0;
-    uint8_t RW_Data[35];
-    FILE *fp;
-    int file_size;
-    uint16_t tmp16;
-
-    // vi_width:
-    if (access(vi_width_path, F_OK) != 0){
-        kvmv_cfg.vi_width = default_vi_width;
-        kvmv_cfg.vi_height = default_vi_height;
-        res = -1;
-        return res;
-    } else {
-        fp = fopen(vi_width_path, "r");
-        fseek(fp, 0, SEEK_END);
-        file_size = ftell(fp);
-        fseek(fp, 0, SEEK_SET);
-        fread(RW_Data, sizeof(char), file_size, fp);
-        fclose(fp);
-        RW_Data[file_size] = 0;
-        tmp16 = atoi((char*)RW_Data);
-        if(tmp16 != kvmv_cfg.vi_width){
-            kvmv_cfg.vi_width = tmp16;
-            debug("[hdmi] Get new HDMI width = %d\r\n", kvmv_cfg.vi_width);
-            res = 1;
-        }
-    }
-    // vi_height:
-    if (access(vi_height_path, F_OK) != 0){
-        kvmv_cfg.vi_height = default_vi_height;
-        res = -1;
-        return res;
-    } else {
-        fp = fopen(vi_height_path, "r");
-        fseek(fp, 0, SEEK_END);
-        file_size = ftell(fp);
-        fseek(fp, 0, SEEK_SET);
-        fread(RW_Data, sizeof(char), file_size, fp);
-        fclose(fp);
-        RW_Data[file_size] = 0;
-        tmp16 = atoi((char*)RW_Data);
-        if(tmp16 != kvmv_cfg.vi_height){
-            kvmv_cfg.vi_height = tmp16;
-            debug("[hdmi] Get new HDMI height = %d\r\n", kvmv_cfg.vi_height);
-            res = 1;
-        }
-    }
-    return res;
-}
-
-uint8_t frame_changed(image::Image *raw)
-{
-    static int raw_size = 0;
-    static uint8_t Farame_sample[Farame_sample_size] = {0};
-    uint8_t ret = 0;
-    uint8_t sample_byte;
-    if(raw->data_size() != raw_size){
-        raw_size = raw->data_size();
-        ret = 1;
-    }
-    int Detection_Pixel_Interval = raw_size/(Farame_sample_size*1.5);
-    for(int i = 0; i < Farame_sample_size; i ++){
-        // printf("[kvmv] i = %d\n", i);
-        if(i >= raw_size){
-            ret = 0;
-        }
-        sample_byte = *(uint8_t*)(raw->data()+(i*Detection_Pixel_Interval));
-        if(sample_byte != Farame_sample[i]){
-            Farame_sample[i] = sample_byte;
-            ret = 1;
-        }
-    }
-    return ret;
-}
-
-bool jpg_dump(kvmv_data_t* dump_to, image::Image *raw)
-{
-    if(dump_to == NULL || raw == NULL || raw->data() == NULL || raw->data_size() == 0){
-        return false;
-    }
-    dump_to->p_img_data = (uint8_t *)malloc(raw->data_size());
-    if(dump_to->p_img_data == NULL){
-        dump_to->img_data_size = 0;
-        dump_to->img_data_type = 0;
-        return false;
-    }
-    dump_to->img_data_size = raw->data_size();
-    dump_to->img_data_type = VENC_MJPEG;
-    memcpy(dump_to->p_img_data, (uint8_t *)raw->data(), raw->data_size());
-    return true;
 }
 
 uint8_t kvmvenc_gop = default_h264_gop;
-kvm_venc_t kvm_venc;
+
-mmf_venc_cfg_t cfg;
+void set_h264_gop(uint8_t gop)
-void init_venc_h264(uint16_t _width, uint16_t _height, uint16_t _qlty)
 {
-    cfg.type = 2; //1, h265, 2, h264
+    __atomic_store_n(&kvmvenc_gop, (uint8_t)maxmin_data(100, 1, gop), __ATOMIC_RELEASE);
-    cfg.w = _width;
-    cfg.h = _height;
-    cfg.fmt = mmf_invert_format_to_mmf(image::Format::FMT_YVU420SP);
-    cfg.jpg_quality = 0;       // unused
-    cfg.gop = kvmvenc_gop;
-    cfg.intput_fps = 60;
-    cfg.output_fps = 60;
-    cfg.bitrate = _qlty;  // 码率
-
-    kvm_venc.mmf_venc_chn = default_venc_chn;
-    kvm_venc.enc_h264_init = 0;
-    kvm_venc.kvm_venc_cfg = cfg;
-
-	// if(mmf_vdec_is_used(kvm_venc.mmf_venc_chn)){
-		mmf_del_venc_channel(kvm_venc.mmf_venc_chn);
-	// }
-    if (0 != mmf_add_venc_channel(kvm_venc.mmf_venc_chn, &kvm_venc.kvm_venc_cfg)) {
-        err::check_raise(err::ERR_RUNTIME, "mmf venc init failed!");
-    }
-    kvm_venc.enc_h264_init = 1;
-
-	// init_kvm_h264_stream(&kvm_h264_stream, mmf_stream_buf);
-	// init_h264_stream_struct(&kvm_h264_stream);
 }
 
-int h264_stream_dump(kvmv_data_t* dump_to, mmf_stream_t* dump_from)
+void set_frame_detact(uint8_t interval)
 {
-    static int8_t I_Frame_index = -1;
+    std::lock_guard<std::mutex> lock(capture_mutex[VENC_MJPEG]);
-    // debug("[kvmv]dump_from->count = %d\n", dump_from->count);
+    kvmv_cfg.frame_detact = maxmin_data(100, 0, interval);
-    if (dump_from->count == 3) {
+    jpeg_detection = {};
+}
 
-        dump_to->p_img_data = (uint8_t *)malloc(dump_from->data_size[0]+dump_from->data_size[1]+dump_from->data_size[2]);
+void kvmv_init(uint8_t debug_info)
-        dump_to->img_data_size = dump_from->data_size[0]+dump_from->data_size[1]+dump_from->data_size[2];
+{
-        dump_to->img_data_type = IMG_H264_TYPE_IF;
+    debug_en = debug_info != 0;
-        memcpy(dump_to->p_img_data, dump_from->data[0], dump_from->data_size[0]);
+    kvmv_cfg.try_exit_thread = false;
-        memcpy(dump_to->p_img_data+dump_from->data_size[0], dump_from->data[1], dump_from->data_size[1]);
+    if (!kvm_mmf::initialize()) {
-        memcpy(dump_to->p_img_data+dump_from->data_size[0]+dump_from->data_size[1], dump_from->data[2], dump_from->data_size[2]);
+        fprintf(stderr, "[kvmv] video initialization failed\n");
-
+    }
-        debug("[kvmv]SPS size = %d\n", dump_from->data_size[0]);
+    if (!detection_started) {
-        debug("[kvmv]PPS size = %d\n", dump_from->data_size[1]);
+        detection_started = pthread_create(&detection_thread, nullptr, vi_subsystem_detection, nullptr) == 0;
-        debug("[kvmv]I-Frame size = %d\n", dump_from->data_size[2]);
+    }
-
+    if (!watchdog_started) {
-        return IMG_H264_TYPE_IF;
+        watchdog_started = pthread_create(&watchdog_thread, nullptr, watchdog_sf_feed, nullptr) == 0;
-
-    } else if (dump_from->count == 1) {
-        // debug("[kvmv]dump P-Frame\r\n");
-        I_Frame_index = -1;
-        dump_to->p_img_data = (uint8_t *)malloc(dump_from->data_size[0]);
-        dump_to->img_data_size = dump_from->data_size[0];
-        dump_to->img_data_type = IMG_H264_TYPE_PF;
-        memcpy(dump_to->p_img_data, dump_from->data[0], dump_from->data_size[0]);
-        return IMG_H264_TYPE_PF;
-    } else {
-        debug("[kvmv]venc error!\r\n");
-        return IMG_VENC_ERROR;
     }
 }
 
-void set_h264_gop(uint8_t _gop)
+int kvmv_read_img(uint16_t width, uint16_t height, uint8_t type,
+                  uint16_t quality, uint16_t fps, kvmv_frame_t *out)
 {
-    kvm_venc.enc_h264_init = 0; // call
+    if (!out) return IMG_VENC_ERROR;
-    kvmvenc_gop = maxmin_data(100, 1, (int)_gop);
+    *out = {};
-    debug("[kvmv] set_h264_gop = %d\n", kvmvenc_gop);
+    if (type > VENC_H264) return IMG_VENC_ERROR;
-}
+    std::lock_guard<std::mutex> lock(capture_mutex[type]);
-
+    if (kvmv_cfg.try_exit_thread || kvmv_cfg.hdmi_stop_flag) return IMG_NOT_EXIST;
-void set_frame_detact(uint8_t _frame_detact)
+    if (kvmv_cfg.hdmi_res_err == ERROR_RES) return -7;
-{
+    if (kvmv_cfg.hdmi_res_type == UNSUPPORT_RES) return -6;
-    uint8_t frame_detact = maxmin_data(100, 0, (int)_frame_detact);
+    if (kvmv_cfg.vi_detect_state == 1) return -4;
-    kvmv_cfg.frame_detact = frame_detact;
+    if (kvmv_cfg.reopen_cam_flag.exchange(false) && !kvm_mmf::restart()) return IMG_VENC_ERROR;
-    debug("[kvmv] set_frame_detact = %d\n", kvmv_cfg.frame_detact);
+    if (width == 0 || height == 0) {
-    kvmv_cfg.stream_stop = 0;
+        width = kvmv_cfg.vi_width;
-}
+        height = kvmv_cfg.vi_height;
-
-int8_t raw_to_h264(image::Image *raw, kvmv_data_t* ret_stream, uint16_t _qlty)
-{
-	uint64_t __attribute__((unused)) start_time;
-	uint64_t __attribute__((unused)) frame_time;
-	int8_t ret = 0;
-	static uint8_t P_Frame_Count = 0;
-		// start_time = time::time_ms();
-		// log::info("getimg: %d \r\n", (int)(time::time_ms()));
-    mmf_stream_t _stream = {0};
-    if(kvm_venc.enc_h264_init != 1 || raw->width() != kvm_venc.kvm_venc_cfg.w || raw->height() != kvm_venc.kvm_venc_cfg.h || _qlty != kvm_venc.kvm_venc_cfg.bitrate){
-		debug("[kvmv]init_venc_h264	enc_h264_init = %d; raw->width() = %d | %d raw->height() = %d | %d \n",
-				kvm_venc.enc_h264_init,
-				raw->width(), kvm_venc.kvm_venc_cfg.w,
-				raw->height(), kvm_venc.kvm_venc_cfg.h);
-
-		init_venc_h264(raw->width(), raw->height(), _qlty);
-        debug("[kvmv]init_venc_h264 finish enc_h264_init = %d; raw->width() = %d | %d raw->height() = %d | %d \n",
-				kvm_venc.enc_h264_init,
-				raw->width(), kvm_venc.kvm_venc_cfg.w,
-				raw->height(), kvm_venc.kvm_venc_cfg.h);
-        // if(kvm_venc.enc_h264_init == 1){
-		// 	init_venc_h264(raw->width(), raw->height(), _qlty);
-		// } else {
-		// 	init_venc_h264(default_vpss_width, default_vpss_height, default_h264_qlty);
-		// }
     }
-	// log::info("init(): %d \r\n", (int)(time::time_ms() - start_time));
+    quality = type == VENC_H264 ? maxmin_data(10000, 500, quality) : maxmin_data(100, 50, quality);
-    if (mmf_venc_push(kvm_venc.mmf_venc_chn, (uint8_t *)raw->data(), raw->width(), raw->height(), mmf_invert_format_to_mmf(raw->format()))) {
+    fps = maxmin_data(60, 1, fps);
-        mmf_del_venc_channel(kvm_venc.mmf_venc_chn);
+    auto codec = static_cast<kvm_mmf::Codec>(type);
-        kvm_venc.enc_h264_init = 0;
+    kvm_mmf::Frame frame;
-        // rtmp->unlock();
+    int result = kvm_mmf::acquire(codec, width, height, quality, fps,
-		debug("[kvmv]mmf venc push failed!\n");
+                                __atomic_load_n(&kvmvenc_gop, __ATOMIC_ACQUIRE), &frame);
-        // err::check_raise(err::ERR_RUNTIME, "mmf venc push failed!\r\n");
+    if (result < 0) return result;
-        return -1;
-    }
-	// log::info("push(): %d \r\n", (int)(time::time_ms() - start_time));
-    if (mmf_venc_pop(kvm_venc.mmf_venc_chn, &_stream)) {
-        // log::error("mmf_venc_pop failed\n");
-        mmf_venc_free(kvm_venc.mmf_venc_chn);
-        mmf_del_venc_channel(kvm_venc.mmf_venc_chn);
-        kvm_venc.enc_h264_init = 0;
-		debug("[kvmv]mmf venc push failed!\n");
-        // rtmp->unlock();
-        return -1;
-    }
-	// log::info("pop(): %d \r\n", (int)(time::time_ms() - start_time));
-    ret = h264_stream_dump(ret_stream, &_stream);
-    mmf_venc_free(kvm_venc.mmf_venc_chn);
-	// log::info("dump(): %d \r\n", (int)(time::time_ms() - start_time));
 
-	// debug("[kvmv]_stream.data[0][4] = %d;\n", _stream.data[0][4]);
+    // Detect unchanged JPEGs without reading raw VPSS pixels.
-	debug("[kvmv]Frame size = %d;\n", ret_stream->img_data_size);
+    if (codec == kvm_mmf::MJPEG) {
-
+        uint8_t interval = kvmv_cfg.frame_detact;
-    if(ret == IMG_H264_TYPE_IF){
+        if (interval == 0) {
-		debug("[kvmv]================ GOP = %d ================\n", kvm_venc.kvm_venc_cfg.gop);
+            jpeg_detection = {};
-		debug("[kvmv]SPS; PPS; I-Frame, I-Frame size = %d\n", ret_stream->img_data_size);
+        } else if (jpeg_detection.unchanged || ++jpeg_detection.count >= interval) {
-		P_Frame_Count = 0;
+            jpeg_detection.count = 0;
-
+            uint64_t hash = 14695981039346656037ULL;
-    } else if(ret == IMG_H264_TYPE_PF){
+            for (uint32_t i = 0; i < frame.size; ++i) hash = (hash ^ frame.data[i]) * 1099511628211ULL;
-		debug("[kvmv]P-Frame size = %d, P-count = %d\n", ret_stream->img_data_size, P_Frame_Count);
+            jpeg_detection.unchanged = hash == jpeg_detection.hash;
-		P_Frame_Count++;
+            jpeg_detection.hash = hash;
-    }
+            if (jpeg_detection.unchanged) {
-    debug("[kvmv]dump ret = %d\n", ret);
+                kvm_mmf::release(codec, frame.token);
-    return ret;
+                free(frame.data);
-}
+                return 5;
-
-void kvmv_init(uint8_t _debug_info_en)
-{
-    pthread_t thread;
-    pthread_mutex_init(&vi_mutex, NULL);
-    if(_debug_info_en == 0) debug_en = 0;
-    else                    debug_en = 1;
-
-    // debug("[kvmv]kvmv_init - 1\r\n");
-
-    cam->hmirror(1);
-    cam->vflip(1);
-    cam->restart(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
-    for(int i = 0; i < kvmv_data_buffer_size; i++){
-        kvmv_data_buffer[i].p_img_data = NULL;
-    }
-
-    kvmv_cfg.try_exit_thread = 0;
-    // debug("[kvmv]kvmv_init - 2\r\n");
-
-    if(kvmv_cfg.thread_is_running == 1){
-        debug("[kvmv]thread is running!\r\n");
-    } else {
-        if (0 != pthread_create(&thread, NULL, vi_subsystem_detection, NULL)) {
-            debug("[kvmv]create vi_subsystem_detection thread failed!\r\n");
-            // return -1;
-        }
-
-        if (0 != pthread_create(&thread, NULL, watchdog_sf_feed, NULL)) {
-            debug("[kvmv]create watchdog_sf_feed thread failed!\r\n");
-            // return -1;
-        }
-    }
-    // debug("[kvmv]kvmv_init - 3\r\n");
-}
-
-uint8_t check_kvmv(uint8_t _try_num)
-{
-    if(__atomic_load_n(&kvmv_cfg.hdmi_cable_state, __ATOMIC_ACQUIRE) == 0){
-        debug("[kvmv]HDMI Cable not exist!\n");
-        return 0;
-    }
-    if(_try_num >= KVMV_MAX_TRY_NUM){
-        debug("[kvmv]try_num >= KVMV_MAX_TRY_NUM!\n");
-        return 0;
-    }
-    // if(sync_vi_res() != 0){
-    if(kvmv_cfg.reopen_cam_flag == 1){
-        // vi size changed
-        kvmv_cfg.reopen_cam_flag = 0;
-        // cam->open(kvmv_cfg.vpss_width, kvmv_cfg.vpss_height, image::FMT_YVU420SP, 3);
-        cam->restart(default_vpss_width, default_vpss_height, image::FMT_YVU420SP);
-        debug("[kvmv]vi size changed, try again\n");
-        return 1;
-    }
-    debug("[kvmv]just try again\n");
-    return 1;
-}
-
-void set_venc_auto_recyc(uint8_t _enable)
-{
-    if(_enable) kvmv_cfg.venc_auto_recyc = 1;
-    else kvmv_cfg.venc_auto_recyc = 0;
-}
-
-/**********************************************************************************
- * @name    kvmv_read_img
- * @author  Sipeed BuGu
- * @date    2024/10/25
- * @version R1.0
- * @brief   Acquire the encoded image with auto init
- * @param	_width				@input: 	Output image width
- * @param	_height				@input: 	Output image height
- * @param	_type				@input: 	Encode type
- * @param	_qlty				@input: 	MJPEG: (50-100) | H264:  (500-10000)
- * @param	_pp_kvm_data		@output: 	Encode data
- * @param	_p_kvmv_data_size	@output: 	Encode data size
- * @return
-        -7: HDMI INPUT RES ERROR
-        -6: Unsupported resolution, please modify it in the host settings.
-        -5: Retrieving image, please wait
-        -4: Modifying image resolution, please wait
-        -3: img buffer full
-        -2: VENC Errorl
-        -1: No images were acquired
-         0: Acquire MJPEG encoded images
-         1: Acquire H264 encoded images(SPS)[Deprecated]
-         2: Acquire H264 encoded images(PPS)[Deprecated]
-         3: Acquire H264 encoded images(I)
-         4: Acquire H264 encoded images(P)
-         5: IMG not changed
- **********************************************************************************/
-int kvmv_read_img(uint16_t _width, uint16_t _height, uint8_t _type, uint16_t _qlty, uint8_t** _pp_kvm_data, uint32_t* _p_kvmv_data_size)
-{
-    *_pp_kvm_data = NULL;
-    *_p_kvmv_data_size = 0;
-    static uint8_t frame_undetact_count = 0;
-	// uint64_t __attribute__((unused)) start_time = time::time_ms();
-    debug("[kvmv]kvmv_read_img type = %d...\n", _type);
-    struct timespec ts;
-    clock_gettime(CLOCK_REALTIME, &ts);
-    ts.tv_sec += 1;
-    // pthread_mutex_lock(&vi_mutex);         // Add lock
-    int mutex_res = pthread_mutex_timedlock(&vi_mutex, &ts);
-    if(mutex_res != 0){
-        return -5;
-    }
-    if (kvmv_cfg.hdmi_res_err == ERROR_RES){
-        pthread_mutex_unlock(&vi_mutex);
-        return -7;
-    }
-    if (kvmv_cfg.hdmi_res_type == UNSUPPORT_RES){
-        pthread_mutex_unlock(&vi_mutex);
-        return -6;
-    }
-    if (kvmv_cfg.vi_detect_state == 1){
-        pthread_mutex_unlock(&vi_mutex);
-        return -4;
-    }
-    uint8_t try_num = 0;
-    do {
-        if(kvmv_cfg.vpss_width != _width || kvmv_cfg.vpss_height != _height){
-            if(_width == 0 || _height == 0){
-                // Follow the HDMI output
-                kvmv_cfg.vpss_width = kvmv_cfg.vi_width;
-                kvmv_cfg.vpss_height = kvmv_cfg.vi_height;
-
-                if(kvmv_cfg.Auto_res == 0){
-                    kvmv_cfg.Auto_res = 1;
-                    cam->set_resolution(kvmv_cfg.vpss_width, kvmv_cfg.vpss_height);
-                    kvmv_cfg.reinit_flag = 1;
-                }
-
-            } else {
-                kvmv_cfg.Auto_res = 0;
-                // Set the output
-                kvmv_cfg.vpss_width = _width;
-                kvmv_cfg.vpss_height = _height;
-
-                cam->set_resolution(kvmv_cfg.vpss_width, kvmv_cfg.vpss_height);
-                kvmv_cfg.reinit_flag = 1;
-            }
-        }
-
-        //
-        if (kvmv_cfg.reinit_flag == 1) {
-            cam->hmirror(1);
-            cam->vflip(1);
-			kvmv_cfg.reinit_flag = 0;
-        }
-        // debug("[kvmv]befor read img: %d \r\n", (int)(time::time_ms() - start_time));
-        image::Image *img = cam->read();
-        // debug("[kvmv]read img: %d \r\n", (int)(time::time_ms() - start_time));
-
-        if(img != NULL){
-            if(kvmv_cfg.fresh_frame_count != 0){
-                // Do not restart VI after HDMI idle. Reopening the MMF
-                // channel can exhaust the carveout heap when the detector
-                // thread is also transitioning. Consume queued frames
-                // instead; the camera buffer contains at most three frames.
-                kvmv_cfg.fresh_frame_count--;
-                delete img;
-                continue;
-            }
-
-            // frame detect
-            if(_type == VENC_MJPEG && kvmv_cfg.frame_detact != 0){
-                if(kvmv_cfg.stream_stop == 0){
-                    frame_undetact_count++;
-                } else {
-                    frame_undetact_count = kvmv_cfg.frame_detact;
-                }
-
-                if(frame_undetact_count == kvmv_cfg.frame_detact){
-                    frame_undetact_count = 0;
-
-                    if(frame_changed(img) == 0){
-                        debug("[kvmv]frame not changed...\n");
-                        kvmv_cfg.stream_stop = 1;
-                        delete img;
-                        pthread_mutex_unlock(&vi_mutex);
-                        return 5;
-                    } else {
-                        kvmv_cfg.stream_stop = 0;
-                    }
-                }
-            }
-        } else {
-            delete img;
-            debug("[kvmv]can`t get img...\n");
-            continue;
-            // pthread_mutex_unlock(&vi_mutex);
-            // return IMG_NOT_EXIST;
-        }
-        // debug("[kvmv]cheak img null?: %d \r\n", (int)(time::time_ms() - start_time));
-
-        // img exist
-        // Encode
-        if(kvmv_cfg.venc_type == VENC_MJPEG && kvmv_cfg.venc_type != _type){
-            if(kvmv_cfg.venc_auto_recyc == 1){
-                mmf_enc_jpg_deinit(0);
-            }
-            kvm_venc.enc_h264_init = 1;
-        }
-        if(kvmv_cfg.venc_type == VENC_H264 && kvmv_cfg.venc_type != _type){
-            if(kvmv_cfg.venc_auto_recyc == 1){
-                mmf_del_venc_channel(kvm_venc.mmf_venc_chn);
-            }
-            kvm_venc.enc_h264_init = 0;
-        }
-
-        kvmv_cfg.venc_type = _type;
-
-        if(kvmv_cfg.venc_type == VENC_MJPEG){
-            kvmv_data_t* p_kvmv_data = get_save_buffer();
-            if(p_kvmv_data == NULL){
-                // buffer full
-			    delete img;
-                debug("[kvmv]jpg buffer full\n");
-                pthread_mutex_unlock(&vi_mutex);
-                return IMG_BUFFER_FULL;
-            }
-            image::Image *jpg = img->to_jpeg(maxmin_data(99, 51, (int)_qlty));
-            if(jpg == NULL || !jpg_dump(p_kvmv_data, jpg)){
-                delete jpg;
-			    delete img;
-                debug("[kvmv]failed to allocate jpg buffer\n");
-                pthread_mutex_unlock(&vi_mutex);
-                return IMG_BUFFER_FULL;
-            }
-            delete jpg;
-			delete img;
-            *_pp_kvm_data = p_kvmv_data->p_img_data;
-            *_p_kvmv_data_size = p_kvmv_data->img_data_size;
-            pthread_mutex_unlock(&vi_mutex);
-            return IMG_MJPEG_TYPE;
-        } else if (kvmv_cfg.venc_type == VENC_H264){
-            int ret;
-            kvmv_data_t* p_kvmv_data = get_save_buffer();
-            if(p_kvmv_data == NULL){
-                // buffer full
-			    delete img;
-                *_pp_kvm_data = NULL;
-                pthread_mutex_unlock(&vi_mutex);
-                return IMG_BUFFER_FULL;
-            }
-            // debug("[kvmv]get_save_buffer: %d \r\n", (int)(time::time_ms() - start_time));
-            ret = raw_to_h264(img, p_kvmv_data, maxmin_data(10000, 500, (int)_qlty));
-            // debug("[kvmv]venc raw_to_h264: %d \r\n", (int)(time::time_ms() - start_time));
-			delete img;
-            *_pp_kvm_data = p_kvmv_data->p_img_data;
-            *_p_kvmv_data_size = p_kvmv_data->img_data_size;
-            pthread_mutex_unlock(&vi_mutex);
-            return ret;
-        }
-    } while (check_kvmv(try_num++));
-    // debug("[kvmv]return: %d \r\n", (int)(time::time_ms() - start_time));
-    *_pp_kvm_data = NULL;
-    pthread_mutex_unlock(&vi_mutex);
-    return IMG_NOT_EXIST;
-}
-
-int free_kvmv_data(uint8_t ** _pp_kvm_data)
-{
-        // debug("[kvmv]free_kvmv_data - 1\r\n");
-    for(int i = 0; i < kvmv_data_buffer_size; i++){
-        if(*_pp_kvm_data == kvmv_data_buffer[i].p_img_data){
-            // debug("[kvmv]free buffer : %d\n", *_pp_kvm_data);
-            if (*_pp_kvm_data != NULL){
-        // debug("[kvmv]free_kvmv_data - 2\r\n");
-                free(*_pp_kvm_data);
-        // debug("[kvmv]free_kvmv_data - 3\r\n");
-                kvmv_data_buffer[i].p_img_data = NULL;
-                uint8_t _type = kvmv_data_buffer[i].img_data_type;
-                return _type;
-            } else {
-                return IMG_NOT_EXIST;
             }
         }
     }
-    return IMG_NOT_EXIST;
+
+    *out = {frame.data, frame.size, frame.timestamp, frame.token, type};
+    return codec == kvm_mmf::MJPEG ? IMG_MJPEG_TYPE :
+           (frame.keyframe ? IMG_H264_TYPE_IF : IMG_H264_TYPE_PF);
 }
 
-void free_all_kvmv_data()
+void kvmv_release_frame(kvmv_frame_t *frame)
 {
-    for(int i = 0; i < kvmv_data_buffer_size; i++){
+    if (!frame || !frame->data) return;
-        if(kvmv_data_buffer[i].p_img_data != NULL){
+    kvm_mmf::release(static_cast<kvm_mmf::Codec>(frame->codec), frame->token);
-            free(kvmv_data_buffer[i].p_img_data);
+    free(frame->data);
-            kvmv_data_buffer[i].p_img_data = NULL;
+    *frame = {};
-        }
+}
-    }
+
+void kvmv_stop_stream(uint8_t type)
+{
+    if (type > VENC_H264) return;
+    std::lock_guard<std::mutex> lock(capture_mutex[type]);
+    kvm_mmf::stop(static_cast<kvm_mmf::Codec>(type));
+    if (type == VENC_MJPEG) jpeg_detection = {};
 }
 
 void kvmv_deinit()
 {
     set_hdmi_capture_enabled(0);
-    pthread_mutex_destroy(&vi_mutex);
+    kvmv_cfg.try_exit_thread = true;
-    kvmv_cfg.try_exit_thread = 1;
+    if (detection_started) { pthread_join(detection_thread, nullptr); detection_started = false; }
-    cam->close();
+    if (watchdog_started) { pthread_join(watchdog_thread, nullptr); watchdog_started = false; }
-    mmf_deinit();
+    std::unique_lock<std::mutex> h264(capture_mutex[VENC_H264], std::defer_lock);
-    free_all_kvmv_data();
+    std::unique_lock<std::mutex> mjpeg(capture_mutex[VENC_MJPEG], std::defer_lock);
+    std::lock(h264, mjpeg);
+    kvm_mmf::deinitialize();
+    jpeg_detection = {};
 }
 
 uint8_t kvmv_hdmi_control(uint8_t _en)
 {
     if(_en == 0){
         set_hdmi_capture_enabled(0);
+        kvm_mmf::stop(kvm_mmf::H264);
+        kvm_mmf::stop(kvm_mmf::MJPEG);
     }
 
     if(kvmv_cfg.hw_version == 0){
@@ -2030,11 +1546,6 @@ uint8_t kvmv_hdmi_control(uint8_t _en)
     } else {
         kvmv_cfg.hdmi_stop_flag = 0;
         system("echo 1 > /sys/class/gpio/gpio451/value");
-
-        // Keep the existing VI channel and consume queued frames after idle.
-        // Reopening it here or from kvmv_read_img can exhaust the carveout
-        // heap while the HDMI detector is transitioning.
-        kvmv_cfg.fresh_frame_count = fresh_frame_discard_count;
         set_hdmi_capture_enabled(1);
         return 0;
     }

support/sg2002/additional/kvm_mmf/CMakeLists.txt

@@ -1,66 +1,71 @@
+cmake_minimum_required(VERSION 3.16)
 
-list(APPEND ADD_REQUIREMENTS basic ini)
+if(COMMAND register_component)
-list(APPEND ADD_INCLUDE "include")
+    if(NOT EXISTS "$ENV{KVM_MMF_PREFIX}/lib/libkvm_mmf.so")
-
+        message(FATAL_ERROR "Build standalone kvm_mmf first and set KVM_MMF_PREFIX (support/sg2002/build does this)")
-# sophgo-middleware
+    endif()
-set(middleware_src_path "${CMAKE_CURRENT_SOURCE_DIR}/../3rd_party/sophgo-middleware/sophgo-middleware")
+    list(APPEND ADD_INCLUDE "$ENV{KVM_MMF_PREFIX}/include")
-
+    file(GLOB video_libraries "$ENV{KVM_MMF_PREFIX}/lib/*.so")
-
+    list(APPEND ADD_DYNAMIC_LIB ${video_libraries})
-set(source_dir "src")
+    register_component()
-append_srcs_dir(ADD_SRCS "${source_dir}")
+    return()
-
-# middleware
-set(middleware_include_dir .
-    ${middleware_src_path}/v2/component/panel/${CONFIG_SOPHGO_MIDDLEWARE_CHIP}
-    ${middleware_src_path}/v2/include
-    ${middleware_src_path}/v2/include/isp/${CONFIG_SOPHGO_MIDDLEWARE_CHIP}
-    ${middleware_src_path}/v2/sample/common
-    ${middleware_src_path}/v2/uapi
-    ${middleware_src_path}/v2/3rdparty/inih
-    ${middleware_src_path}/v2/modules/ive/include/
-)
-list(APPEND ADD_INCLUDE ${middleware_include_dir})
-set_property(SOURCE ${middleware_include_dir} PROPERTY GENERATED 1)
-
-append_srcs_dir(middleware_src_dir  ${middleware_src_path}/v2/sample/common
-                                    ${middleware_src_path}/v2/component/isp/sensor/sg200x/lontium_lt6911
-                                    )
-list(APPEND ADD_SRCS        ${middleware_src_dir}
-                            "${source_dir}/kvm_mmf.cpp")
-set_property(SOURCE ${middleware_src_dir} PROPERTY GENERATED 1)
-
-set(middleware_static_lib_file  "")
-    list(APPEND ADD_STATIC_LIB ${middleware_static_lib_file})
-set_property(SOURCE ${middleware_static_lib_file} PROPERTY GENERATED 1)
-
-set(mmf_lib_dir ${middleware_src_path}/v2/lib)
-
-if(CONFIG_SOPHGO_MIDDLEWARE_C_LIBRARY STREQUAL "musl")
-    set(middleware_dynamic_lib_file     ${mmf_lib_dir}/libcvi_bin.so
-                                        ${mmf_lib_dir}/libae.so
-                                        ${mmf_lib_dir}/libaf.so
-                                        ${mmf_lib_dir}/libawb.so
-                                        ${mmf_lib_dir}/libcvi_bin_isp.so
-                                        ${mmf_lib_dir}/libisp_algo.so
-                                        ${mmf_lib_dir}/libisp.so
-                                        ${mmf_lib_dir}/libsys.so
-                                        ${mmf_lib_dir}/libvdec.so
-                                        ${mmf_lib_dir}/libvenc.so
-                                        ${mmf_lib_dir}/libvpu.so
-                                        ${mmf_lib_dir}/3rd/libini.so)
-elseif(CONFIG_SOPHGO_MIDDLEWARE_C_LIBRARY STREQUAL "glibc")
-    set(middleware_dynamic_lib_file     ${mmf_lib_dir}/libcvi_vb.so
-                                        ${mmf_lib_dir}/libcvi_sys.so
-                                        ${mmf_lib_dir}/libcvi_debug.so
-    )
-else()
-    message(FATAL_ERROR "No sophgo middleware c library support, please check menuoconfig to select c library")
 endif()
-list(APPEND ADD_DYNAMIC_LIB ${middleware_dynamic_lib_file})
-set_property(SOURCE ${middleware_dynamic_lib_file} PROPERTY GENERATED 1)
 
-list(APPEND ADD_DEFINITIONS_PRIVATE -DSENSOR_LONTIUM_LT6911
+project(kvm_mmf VERSION 1.0 LANGUAGES C CXX)
-                                -DSENSOR2_TYPE=LONTIUM_LT6911_2M_60FPS_8BIT)
+include(GNUInstallDirs)
+find_package(Threads REQUIRED)
 
+set(SOPHGO_MIDDLEWARE_DIR "" CACHE PATH "CVI SDK middleware root containing v2/")
+set(SOPHGO_RUNTIME_DIR "" CACHE PATH "Firmware-matched SDK .so directory; defaults to middleware v2/lib")
+set(SOPHGO_CHIP cv181x CACHE STRING "CVI SDK chip family")
+option(KVM_MMF_INSTALL_SDK_LIBS "Install the SDK runtime libraries alongside kvm_mmf" ON)
+if(NOT EXISTS "${SOPHGO_MIDDLEWARE_DIR}/v2/include/cvi_venc.h")
+    message(FATAL_ERROR "Set SOPHGO_MIDDLEWARE_DIR to the CVI SDK middleware root containing v2/include/cvi_venc.h")
+endif()
+get_filename_component(middleware "${SOPHGO_MIDDLEWARE_DIR}/v2" REALPATH)
+if(SOPHGO_RUNTIME_DIR)
+    get_filename_component(runtime "${SOPHGO_RUNTIME_DIR}" REALPATH)
+else()
+    set(runtime "${middleware}/lib")
+endif()
+set(common "${middleware}/sample/common")
+set(sensor "${middleware}/component/isp/sensor/sg200x/lontium_lt6911")
 
-register_component(DYNAMIC)
+add_library(kvm_mmf SHARED
+    src/kvm_mmf.cpp
+    "${common}/sample_common_sys.c"
+    "${common}/sample_common_vi.c"
+    "${common}/sample_common_isp.c"
+    "${common}/sample_common_bin.c"
+    "${common}/sample_common_platform.c"
+    "${common}/sample_common_sensor.c"
+    "${sensor}/lt6911_cmos.c" "${sensor}/lt6911_sensor_ctl.c"
+)
+target_compile_features(kvm_mmf PRIVATE cxx_std_17)
+target_compile_definitions(kvm_mmf PRIVATE SENSOR_LONTIUM_LT6911 SENSOR2_TYPE=LONTIUM_LT6911_2M_60FPS_8BIT)
+target_compile_options(kvm_mmf PRIVATE -Wall -ffunction-sections -fdata-sections)
+target_include_directories(kvm_mmf PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)
+target_include_directories(kvm_mmf PRIVATE
+    "${common}" "${middleware}/include" "${middleware}/uapi"
+    "${middleware}/include/isp/${SOPHGO_CHIP}"
+    "${middleware}/component/panel/${SOPHGO_CHIP}"
+    "${middleware}/3rdparty/inih"
+)
+foreach(name cvi_bin ae af awb cvi_bin_isp isp isp_algo sys venc vpu ini)
+    unset(sdk_${name} CACHE)
+    find_library(sdk_${name} NAMES ${name} PATHS "${runtime}" "${runtime}/3rd" NO_DEFAULT_PATH REQUIRED)
+    target_link_libraries(kvm_mmf PRIVATE "${sdk_${name}}")
+    if(KVM_MMF_INSTALL_SDK_LIBS AND sdk_${name} MATCHES "\\.so$")
+        install(FILES "${sdk_${name}}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+    endif()
+endforeach()
+target_link_libraries(kvm_mmf PRIVATE Threads::Threads m atomic)
+target_link_options(kvm_mmf PRIVATE -Wl,--as-needed -Wl,--gc-sections -Wl,-z,defs -Wl,--no-allow-shlib-undefined)
+set_target_properties(kvm_mmf PROPERTIES
+    C_VISIBILITY_PRESET hidden CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN YES
+    BUILD_WITH_INSTALL_RPATH YES INSTALL_RPATH "\$ORIGIN")
+install(TARGETS kvm_mmf LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES include/kvm_mmf.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

support/sg2002/additional/kvm_mmf/cmake/riscv64-musl.cmake

@@ -0,0 +1,7 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR riscv64)
+
+set(CMAKE_C_COMPILER riscv64-unknown-linux-musl-gcc)
+set(CMAKE_CXX_COMPILER riscv64-unknown-linux-musl-g++)
+set(CMAKE_C_FLAGS_INIT "-mcpu=c906fdv -march=rv64imafdcv0p7xthead -mcmodel=medany -mabi=lp64d")
+set(CMAKE_CXX_FLAGS_INIT "${CMAKE_C_FLAGS_INIT}")

support/sg2002/additional/kvm_mmf/include/kvm_mmf.hpp

@@ -1,71 +1,27 @@
-#ifndef __KVM_MMF_HPP__
+#pragma once
-#define __KVM_MMF_HPP__
 
-#include "stdint.h"
+#include <stdint.h>
 
-typedef struct {
+// VI -> VPSS -> VENC, with no application queue of encoded frames.
-    uint8_t *data[8];
+namespace kvm_mmf __attribute__((visibility("default"))) {
-    int data_size[8];
-    int count;
-} mmf_stream_t;
 
-typedef struct {
+enum Codec { MJPEG = 0, H264 = 1 };
-    uint8_t type;           // 0, jpg; 1, h265; 2, h264
-    int w;
-	int h;
-	int fmt;
-	uint8_t jpg_quality;	// jpeg
-	int gop;				// h264
-	int intput_fps;			// h264
-	int output_fps;			// h264
-	int bitrate;			// h264
-} mmf_venc_cfg_t;
 
-// init sys
+struct Frame {
-int mmf_init(void);
+    uint8_t *data = nullptr;
-int mmf_deinit(void);
+    uint32_t size = 0;
-int mmf_try_deinit(bool force);
+    uint64_t timestamp = 0; // SDK capture PTS, in microseconds.
-bool mmf_is_init(void);
+    uint64_t token = 0;
+    bool keyframe = false;
+};
 
-// manage vi channels(vi->vpssgroup->vpss->frame)
+bool initialize();
-int mmf_get_vi_unused_channel(void);
+bool restart();
-int mmf_vi_init(void);
+void deinitialize();
-int mmf_vi_deinit(void);
+void stop(Codec codec);
-int mmf_add_vi_channel_with_enc(int ch, int width, int height, int format);
-int mmf_add_vi_channel(int ch, int width, int height, int format);
-int mmf_del_vi_channel(int ch);
-int mmf_del_vi_channel_all(void);
-int mmf_reset_vi_channel(int ch, int width, int height, int format);
-bool mmf_vi_chn_is_open(int ch);
-int mmf_vi_aligned_width(int ch);
-void mmf_set_vi_hmirror(int ch, bool en);
-void mmf_set_vi_vflip(int ch, bool en);
-void mmf_get_vi_hmirror(int ch, bool *en);
-void mmf_get_vi_vflip(int ch, bool *en);
 
-// get vi frame
+int acquire(Codec codec, uint16_t width, uint16_t height, uint16_t quality,
-int mmf_vi_frame_pop(int ch, void **data, int *len, int *width, int *height, int *format);
+            uint16_t fps, uint8_t gop, Frame *frame);
-void mmf_vi_frame_free(int ch);
+void release(Codec codec, uint64_t token);
-// Release the current VI frame immediately. mmf_vi_frame_free() defers
-// release so the frame can be sent directly to VENC without a second copy.
-void mmf_vi_frame_release(int ch);
 
-// invert format
+} // namespace kvm_mmf
-int mmf_invert_format_to_maix(int mmf_format);
-int mmf_invert_format_to_mmf(int maix_format);
-
-// venc
-int mmf_enc_jpg_init(int ch, int w, int h, int format, int quality);
-int mmf_enc_jpg_deinit(int ch);
-int mmf_enc_jpg_push(int ch, uint8_t *data, int w, int h, int format);
-int mmf_enc_jpg_push_with_quality(int ch, uint8_t *data, int w, int h, int format, int quality);
-int mmf_enc_jpg_pop(int ch, uint8_t **data, int *size);
-int mmf_enc_jpg_free(int ch);
-int mmf_add_venc_channel(int ch, mmf_venc_cfg_t *cfg);
-int mmf_del_venc_channel(int ch);
-int mmf_del_venc_channel_all();
-int mmf_venc_push(int ch, uint8_t *data, int w, int h, int format);
-int mmf_venc_pop(int ch, mmf_stream_t *stream);
-int mmf_venc_free(int ch);
-
-#endif // __KVM_MMF_HPP__

support/sg2002/build

@@ -13,7 +13,7 @@ maixcdk_build() {
 
   DIST_PATH="./dist"
   rm -rf $DIST_PATH
-  
+
   if [ "$input_value" = "from_zero" ]
   then
     maixcdk build <<EOF
@@ -24,8 +24,31 @@ EOF
   fi
 }
 
+build_kvm_mmf() {
+  local component="$NanoKVM_PATH/support/sg2002/additional/kvm_mmf"
+  local middleware="${SOPHGO_MIDDLEWARE_DIR:-$MAIXCDK_PATH/components/3rd_party/sophgo-middleware/sophgo-middleware}"
+  local runtime="${SOPHGO_RUNTIME_DIR:-$NanoKVM_PATH/server/dl_lib}"
+  local build_dir="$Project_PATH/build/kvm_mmf-build"
+  export KVM_MMF_PREFIX="$Project_PATH/build/kvm_mmf-install"
+  cmake -S "$component" -B "$build_dir" \
+    -DCMAKE_TOOLCHAIN_FILE="$component/cmake/riscv64-musl.cmake" \
+    -DSOPHGO_MIDDLEWARE_DIR="$middleware" -DSOPHGO_RUNTIME_DIR="$runtime" \
+    -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$KVM_MMF_PREFIX" || return 1
+  cmake --build "$build_dir" --parallel || return 1
+  cmake --install "$build_dir" || return 1
+}
+
+update_components() {
+  cp -rf "$NanoKVM_PATH/support/sg2002/additional/kvm" "$MAIXCDK_PATH/components/" || return 1
+  cp -rf "$NanoKVM_PATH/support/sg2002/additional/kvm_mmf" "$MAIXCDK_PATH/components/" || return 1
+  cp -f "$NanoKVM_PATH/support/sg2002/additional/peripheral/port/maixcam/maix_i2c.cpp" \
+    "$MAIXCDK_PATH/components/peripheral/port/maixcam/maix_i2c.cpp" || return 1
+  cp -f "$NanoKVM_PATH/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c" \
+    "$MAIXCDK_PATH/components/3rd_party/sophgo-middleware/sophgo-middleware/v2/sample/common/" || return 1
+}
+
 chack_build(){
-  DIST_PATH="./dist"  
+  DIST_PATH="./dist"
   if [ -d "$DIST_PATH" ]
   then
     echo "SUCCESS"
@@ -41,15 +64,7 @@ maixcdk_clean(){
 
 case "$1" in
   update_lib)
-    # This script does not run under "set -e", and a release build relies on
+    update_components || exit 1
-    # update_lib to guarantee the SDK components match the source tree. A copy
-    # that fails silently here would produce stale libraries, so fail loudly.
-    set -e
-    cp -rf $NanoKVM_PATH/support/sg2002/additional/kvm $MAIXCDK_PATH/components
-    cp -rf $NanoKVM_PATH/support/sg2002/additional/kvm_mmf $MAIXCDK_PATH/components
-    cp -rf $NanoKVM_PATH/support/sg2002/additional/vision $MAIXCDK_PATH/components
-    cp -f $NanoKVM_PATH/support/sg2002/additional/peripheral/port/maixcam/maix_i2c.cpp $MAIXCDK_PATH/components/peripheral/port/maixcam/maix_i2c.cpp
-    cp -f $NanoKVM_PATH/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c $MAIXCDK_PATH/components/3rd_party/sophgo-middleware/sophgo-middleware/v2/sample/common/
 
     echo -e "\e[36m[build] \e[32mupdate_lib ok!\e[0m"
     exit 0
@@ -106,37 +121,34 @@ then
       if [ ! -e "$MAIXCDK_PATH/components/kvm" ]
       then
         echo -e "\e[36m[build] \e[32mDetected that the library has not been modified; first, move the library\e[0m"
-        
+
         mkdir $NanoKVM_PATH/support/sg2002/additional/original
-        mv -f $MAIXCDK_PATH/components/vision $NanoKVM_PATH/support/sg2002/additional/original
         # mv $MAIXCDK_PATH/components/maixcam_lib $NanoKVM_PATH/support/sg2002/additional/original
         mv -f $MAIXCDK_PATH/components/peripheral/port/maixcam/maix_i2c.cpp $NanoKVM_PATH/support/sg2002/additional/original
-        mv -f $MAIXCDK_PATH/components/3rd_party/sophgo-middleware/sophgo-middleware/v2/sample/common/sample_common_sensor.c $NanoKVM_PATH/support/sg2002/additional/original
-      
-        cp -rf $NanoKVM_PATH/support/sg2002/additional/kvm $MAIXCDK_PATH/components
-        cp -rf $NanoKVM_PATH/support/sg2002/additional/kvm_mmf $MAIXCDK_PATH/components
-        cp -rf $NanoKVM_PATH/support/sg2002/additional/vision $MAIXCDK_PATH/components
-        cp -f $NanoKVM_PATH/support/sg2002/additional/peripheral/port/maixcam/maix_i2c.cpp $MAIXCDK_PATH/components/peripheral/port/maixcam/maix_i2c.cpp
-        cp -f $NanoKVM_PATH/support/sg2002/additional/sophgo-middleware/v2/sample/common/sample_common_sensor.c $MAIXCDK_PATH/components/3rd_party/sophgo-middleware/sophgo-middleware/v2/sample/common/
 
         echo -e "\e[36m[build] \e[32mMove the library completed!\e[0m"
       fi
 
-      cd $Project_PATH
+      update_components || exit 1
-      if [ -f $JSON_PATH ]
+
+      cd "$Project_PATH"
+      if [ -f "$JSON_PATH" ] && ! grep -q "$Platform_Char" "$JSON_PATH"; then
+        maixcdk_clean || exit 1
+      fi
+
+      if [ "$Project_Name" = "kvm_vision" ]; then
+        build_kvm_mmf || exit 1
+      fi
+
+      if [ -f "$JSON_PATH" ]
       then
-        if grep -q $Platform_Char $JSON_PATH; then
+        maixcdk_build
-          maixcdk_build
-        else
-          maixcdk_clean
-          maixcdk_build "from_zero"
-        fi
       else
         maixcdk_build "from_zero"
       fi
 
       RES=$(chack_build)
-      
+
       if [ "$RES" = "SUCCESS" ]
       then
         echo -e "\e[36m[build] \e[32mBuild completed! Use \e[34m./build $Project_Name add_to_kvmapp\e[32m to add it to kvmapp.\e[0m"
@@ -173,7 +185,7 @@ then
         else
           echo -e "\e[36m[build] \e[32mkvm_vision has been moved to kvmapp.\e[0m"
         fi
-        
+
       else
         echo -e "\e[36m[build] \e[31mThe source file does not exist. Please compile using \e[34m./build kvm_system \e[31mfirst.\e[0m"
         exit 1

support/sg2002/kvm_system/main/lib/system_init/system_init.cpp

@@ -134,7 +134,6 @@ void new_app_init(void)
 	system("echo 0 > /kvmapp/kvm/now_fps");
 	system("echo 30 > /kvmapp/kvm/fps");
 	system("echo 2000 > /kvmapp/kvm/qlty");
-	system("echo 720 > /kvmapp/kvm/res");
 	system("echo h264 > /kvmapp/kvm/type");
 	write_hdmi_state(0);
 	system("touch /etc/kvm/frame_detact");

support/sg2002/kvm_vision_test/main/src/main.cpp

@@ -1,61 +1,23 @@
-#include "kvm_vision.h"
+#include "kvm_vision.h"
-#include "maix_basic.hpp"
+#include "maix_basic.hpp"
-
+
-using namespace maix;
+using namespace maix;
-using namespace maix::sys;
+
-
+int main()
-// #define NOT_GET_IMG
+{
-
+    signal(SIGINT, [](int) { app::set_exit_flag(true); });
-int main(int argc, char* argv[])
+    kvmv_hdmi_control(1);
-{
+    kvmv_init(0);
-	uint64_t __attribute__((unused)) start_time;
+    set_h264_gop(30);
-	// Catch SIGINT signal(e.g. Ctrl + C), and set exit flag to true.
+
-    signal(SIGINT, [](int sig){ app::set_exit_flag(true); 
+    uint8_t codec = 0;
-	log::info("========================\n");
+    while (!app::need_exit()) {
-	});
+        kvmv_frame_t frame = {};
-
+        int result = kvmv_read_img(1920, 1080, codec, codec ? 3000 : 60, 60, &frame);
-	kvmv_hdmi_control(0);
+        printf("codec=%u result=%d size=%u pts=%llu\n", codec, result, frame.size,
-	kvmv_hdmi_control(1);
+               static_cast<unsigned long long>(frame.timestamp));
-
+        kvmv_release_frame(&frame);
-	kvmv_init(0);
+        codec ^= 1;
-	set_h264_gop(30);
+    }
-
+    kvmv_deinit();
-	uint16_t get_fream_count = 0;
+}
-
-	while(!app::need_exit()){
-#ifdef NOT_GET_IMG
-		printf("NOT_GET_IMG ...\n");
-		time::sleep_ms(1000);
-#else
-		uint8_t* p_kvmv_img_data;
-		uint32_t kvmv_img_data_size;
-		int ret;
-
-		printf("KVM-Vison Get Fream ...\n");
-		
-		start_time = time::time_ms();
-		if(get_fream_count < 1){
-			get_fream_count ++;
-			ret = kvmv_read_img(1920, 1080, 1, 3000, &p_kvmv_img_data, &kvmv_img_data_size);
-		} else if(get_fream_count >= 1 && get_fream_count < 2){
-			get_fream_count ++;
-			ret = kvmv_read_img(0, 0, 0, 60, &p_kvmv_img_data, &kvmv_img_data_size);
-		} else {
-			get_fream_count = 0;
-		}
-		
-		// printf("kvmv_read_img(): %d \r\n", (int)(time::time_ms() - start_time));
-
-		// ret = kvmv_read_img(1920, 1080, 1, 3000, &p_kvmv_img_data, &kvmv_img_data_size);
-		// ret = kvmv_read_img(1920, 1080, 0, 60, &p_kvmv_img_data, &kvmv_img_data_size);
-
-		printf("kvmv_read_img ret = %d\n", ret);
-
-		// send...
-		// if(ret >= 0){
-			free_kvmv_data(&p_kvmv_img_data);
-		// }
-#endif
-	}
-	kvmv_deinit();
-}

web/src/jotai/screen.ts

@@ -1,6 +1,6 @@
 import { atom } from 'jotai';
 
-import { ControlRegionMode, InputRegion, Resolution } from '@/types';
+import { ControlRegionMode, InputRegion } from '@/types';
 
 export const isHdmiEnabledAtom = atom(true);
 
@@ -12,9 +12,6 @@ export const videoModeAtom = atom('');
 
 export const videoScaleAtom = atom<number>(1.0);
 
-// browser screen resolution
-export const resolutionAtom = atom<Resolution | null>(null);
-
 // currently effective absolute mouse input region
 export const inputRegionAtom = atom<InputRegion | null>(null);
 export const manualInputRegionAtom = atom<InputRegion | null>(null);

web/src/lib/localstorage.ts

@@ -1,9 +1,6 @@
-import { Resolution } from '@/types';
-
 const LANGUAGE_KEY = 'nano-kvm-language';
 const VIDEO_MODE_KEY = 'nano-kvm-vide-mode';
 const VIDEO_SCALE_KEY = 'nano-kvm-video-scale';
-const WEB_RESOLUTION_KEY = 'nano-kvm-web-resolution';
 const FPS_KEY = 'nano-kvm-fps';
 const QUALITY_KEY = 'nano-kvm-quality';
 const GOP_KEY = 'nano-kvm-gop';
@@ -81,20 +78,6 @@ export function setVideoScale(scale: number): void {
   localStorage.setItem(VIDEO_SCALE_KEY, String(scale));
 }
 
-export function getResolution(): Resolution | null {
-  const resolution = localStorage.getItem(WEB_RESOLUTION_KEY);
-  if (resolution) {
-    const obj = JSON.parse(window.atob(resolution));
-    return obj as Resolution;
-  }
-
-  return null;
-}
-
-export function setResolution(resolution: Resolution) {
-  localStorage.setItem(WEB_RESOLUTION_KEY, window.btoa(JSON.stringify(resolution)));
-}
-
 export function getFps() {
   const fps = localStorage.getItem(FPS_KEY);
   return fps ? Number(fps) : null;

web/src/pages/desktop/index.tsx

@@ -16,7 +16,6 @@ import {
   inputRegionAtom,
   manualInputRegionAtom,
   manualRegionsAtom,
-  resolutionAtom,
   selectedManualRegionAtom,
   selectedOriginalResolutionAtom,
   videoModeAtom
@@ -65,7 +64,6 @@ const Console = () => {
   const captureStatus = useCaptureStatus(activeVideoMode);
 
   const [videoMode, setVideoMode] = useAtom(videoModeAtom);
-  const [resolution, setResolution] = useAtom(resolutionAtom);
   const [inputRegion, setInputRegion] = useAtom(inputRegionAtom);
   const [controlRegionMode, setControlRegionModeState] = useAtom(controlRegionModeAtom);
   const setManualInputRegion = useSetAtom(manualInputRegionAtom);
@@ -94,8 +92,6 @@ const Console = () => {
 
     setVideoMode(activeVideoMode);
 
-    const res = storage.getResolution() || { width: 0, height: 0 };
-    setResolution(res);
     setInputRegion(null);
     setManualInputRegion(null);
     setManualRegions([]);
@@ -141,7 +137,6 @@ const Console = () => {
     setInputRegion,
     setManualInputRegion,
     setManualRegions,
-    setResolution,
     setSelectedManualRegion,
     setSelectedOriginalResolution,
     setVideoMode
@@ -172,7 +167,7 @@ const Console = () => {
         clearTimeout(validationTimer);
       }
       validationTimer = setTimeout(() => {
-        const mediaSize = getMediaSize(target, resolution);
+        const mediaSize = getMediaSize(target);
         if (!mediaSize || !isMediaReady(target) || isInputRegionCompatible(region, mediaSize)) {
           return;
         }
@@ -214,7 +209,6 @@ const Console = () => {
     canControl,
     controlRegionMode,
     inputRegion,
-    resolution,
     setControlRegionModeState,
     setInputRegion,
     videoMode
@@ -234,7 +228,7 @@ const Console = () => {
       {isBigScreen && <Notification />}
       <H264ModeNotification />
 
-      {videoMode && resolution && (
+      {videoMode && (
         <div className="relative flex h-full min-h-0 w-full min-w-0">
           <Menu />
           <div className="h-full min-h-0 w-full min-w-0">

web/src/pages/desktop/menu/screen/index.tsx

@@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next';
 
 import { updateScreen } from '@/api/vm';
 import * as ls from '@/lib/localstorage';
-import { resolutionAtom, videoModeAtom } from '@/jotai/screen.ts';
+import { videoModeAtom } from '@/jotai/screen.ts';
 import { MenuItem } from '@/components/menu-item.tsx';
 
 import { getQualityMap, getScreenType } from './constants.ts';
@@ -14,7 +14,6 @@ import { FrameDetect } from './frame-detect';
 import { Gop } from './gop.tsx';
 import { Quality } from './quality';
 import { Reset } from './reset.tsx';
-import { Resolution } from './resolution';
 import { Scale } from './scale';
 import { VideoMode } from './video-mode.tsx';
 
@@ -22,7 +21,6 @@ export const Screen = () => {
   const { t } = useTranslation();
 
   const videoMode = useAtomValue(videoModeAtom);
-  const resolution = useAtomValue(resolutionAtom);
   const isMjpeg = videoMode === 'mjpeg';
   const [fps, setFps] = useState(30);
   const [quality, setQuality] = useState(2);
@@ -33,7 +31,6 @@ export const Screen = () => {
     if (screenType !== null) {
       updateScreen('type', screenType);
     }
-    updateScreen('resolution', resolution!.height);
     updateQuality();
     updateFps();
     updateGop();
@@ -77,7 +74,6 @@ export const Screen = () => {
   const content = (
     <div className="flex flex-col space-y-1">
       <VideoMode />
-      <Resolution />
       <Quality quality={quality} setQuality={setQuality} />
       <Fps fps={fps} setFps={setFps} />
       <Scale />

web/src/pages/desktop/menu/screen/resolution.tsx

@@ -1,76 +0,0 @@
-import { Popover, Tooltip } from 'antd';
-import { useAtom } from 'jotai';
-import { CheckIcon, CircleHelpIcon, RatioIcon } from 'lucide-react';
-import { useTranslation } from 'react-i18next';
-
-import { updateScreen } from '@/api/vm';
-import { Resolution as TypeResolution } from '@/types';
-import { setResolution as setCookie } from '@/lib/localstorage';
-import { resolutionAtom } from '@/jotai/screen.ts';
-
-const resolutions: TypeResolution[] = [
-  { width: 0, height: 0 },
-  { width: 1920, height: 1080 },
-  { width: 1280, height: 720 },
-  { width: 800, height: 600 },
-  { width: 640, height: 480 }
-];
-
-export const Resolution = () => {
-  const { t } = useTranslation();
-  const [resolution, setResolution] = useAtom(resolutionAtom);
-
-  async function update(item: TypeResolution) {
-    const rsp = await updateScreen('resolution', item.height);
-    if (rsp.code !== 0) {
-      return;
-    }
-
-    setResolution(item);
-    setCookie(item);
-  }
-
-  const content = (
-    <>
-      {resolutions.map((res) => (
-        <div
-          key={res.height}
-          className="flex cursor-pointer select-none items-center rounded py-1.5 pl-1 pr-5 hover:bg-neutral-700/70"
-          onClick={() => update(res)}
-        >
-          <div className="flex h-[14px] w-[20px] items-end text-blue-500">
-            {res.height === resolution?.height && <CheckIcon size={15} />}
-          </div>
-
-          {res.height === 0 ? (
-            <div className="flex items-center justify-between space-x-2">
-              <span>{t('screen.auto')}</span>
-              <Tooltip
-                title={t('screen.autoTips')}
-                placement="right"
-                overlayInnerStyle={{ width: '300px' }}
-              >
-                <CircleHelpIcon size={14} />
-              </Tooltip>
-            </div>
-          ) : (
-            <>
-              <span>{res.width}</span>
-              <span className="px-1">x</span>
-              <span>{res.height}</span>
-            </>
-          )}
-        </div>
-      ))}
-    </>
-  );
-
-  return (
-    <Popover content={content} placement="rightTop" arrow={false} align={{ offset: [14, 0] }}>
-      <div className="flex h-[30px] cursor-pointer items-center space-x-2 rounded px-3 text-neutral-300 hover:bg-neutral-700/70">
-        <RatioIcon size={18} />
-        <span className="select-none text-sm">{t('screen.resolution')}</span>
-      </div>
-    </Popover>
-  );
-};

web/src/pages/desktop/mouse/absolute.tsx

@@ -4,7 +4,7 @@ import { useAtomValue } from 'jotai';
 import { MouseReportAbsolute } from '@/lib/mouse.ts';
 import { client, MessageEvent } from '@/lib/websocket.ts';
 import { scrollDirectionAtom, scrollIntervalAtom } from '@/jotai/mouse.ts';
-import { inputRegionAtom, resolutionAtom } from '@/jotai/screen.ts';
+import { inputRegionAtom } from '@/jotai/screen.ts';
 
 import {
   FrameContent,
@@ -25,7 +25,6 @@ enum MouseButton {
 }
 
 export const Absolute = () => {
-  const resolution = useAtomValue(resolutionAtom);
   const inputRegion = useAtomValue(inputRegionAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
@@ -426,9 +425,7 @@ export const Absolute = () => {
         return null;
       }
 
-      const mediaSize =
+      const mediaSize = getMediaSize(target);
-        getMediaSize(target) ||
-        (resolution && resolution.width > 0 && resolution.height > 0 ? resolution : null);
       if (!mediaSize) {
         const x = (clientX - rect.left) / rect.width;
         const y = (clientY - rect.top) / rect.height;
@@ -531,7 +528,7 @@ export const Absolute = () => {
         clearTimeout(longPressTimerRef.current);
       }
     };
-  }, [inputRegion, resolution, scrollDirection, scrollInterval]);
+  }, [inputRegion, scrollDirection, scrollInterval]);
 
   // disable default events
   function disableEvent(event: Event) {

web/src/pages/desktop/mouse/relative.tsx

@@ -6,7 +6,6 @@ import { useTranslation } from 'react-i18next';
 import { MouseReportRelative } from '@/lib/mouse.ts';
 import { client, MessageEvent } from '@/lib/websocket.ts';
 import { scrollDirectionAtom, scrollIntervalAtom } from '@/jotai/mouse.ts';
-import { resolutionAtom } from '@/jotai/screen.ts';
 
 import { MouseRelativeEvent } from './types.ts';
 
@@ -14,7 +13,6 @@ export const Relative = () => {
   const { t } = useTranslation();
   const [messageApi, contextHolder] = message.useMessage();
 
-  const resolution = useAtomValue(resolutionAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
 
@@ -328,7 +326,7 @@ export const Relative = () => {
       target.removeEventListener('touchcancel', handleTouchCancel, touchOptions.capture);
       clearTouchLongPressTimer();
     };
-  }, [resolution, scrollDirection, scrollInterval]);
+  }, [scrollDirection, scrollInterval]);
 
   // Mouse handler
   function handleMouseEvent(event: MouseRelativeEvent) {

web/src/pages/desktop/screen/auto-region.tsx

@@ -4,8 +4,7 @@ import { useAtomValue, useSetAtom } from 'jotai';
 import {
   controlRegionModeAtom,
   inputRegionAtom,
-  inputRegionSelectingAtom,
+  inputRegionSelectingAtom
-  resolutionAtom
 } from '@/jotai/screen.ts';
 
 import { detectFrameContent, getMediaSize } from './geometry.ts';
@@ -13,7 +12,6 @@ import { detectFrameContent, getMediaSize } from './geometry.ts';
 export const AutoRegion = () => {
   const mode = useAtomValue(controlRegionModeAtom);
   const selecting = useAtomValue(inputRegionSelectingAtom);
-  const resolution = useAtomValue(resolutionAtom);
   const setInputRegion = useSetAtom(inputRegionAtom);
 
   useEffect(() => {
@@ -37,7 +35,7 @@ export const AutoRegion = () => {
         confirmations = 0;
       }
 
-      const mediaSize = getMediaSize(screen, resolution);
+      const mediaSize = getMediaSize(screen);
       if (!mediaSize) return;
       const content = detectFrameContent(screen, mediaSize);
       const key = [
@@ -74,7 +72,7 @@ export const AutoRegion = () => {
       stopped = true;
       window.clearInterval(timer);
     };
-  }, [mode, resolution, selecting, setInputRegion]);
+  }, [mode, selecting, setInputRegion]);
 
   useEffect(() => {
     if (mode !== 'auto') setInputRegion(null);

web/src/pages/desktop/screen/geometry.ts

@@ -27,7 +27,7 @@ export const fullFrameContent = (mediaSize: MediaSize): FrameContent => ({
   height: mediaSize.height
 });
 
-export function getMediaSize(screen: Element, fallback?: MediaSize | null): MediaSize | null {
+export function getMediaSize(screen: Element): MediaSize | null {
   const dataWidth = Number(screen.getAttribute(mediaWidthAttribute));
   const dataHeight = Number(screen.getAttribute(mediaHeightAttribute));
   if (dataWidth > 0 && dataHeight > 0) {
@@ -42,7 +42,7 @@ export function getMediaSize(screen: Element, fallback?: MediaSize | null): Medi
     return { width: screen.naturalWidth, height: screen.naturalHeight };
   }
 
-  return fallback && fallback.width > 0 && fallback.height > 0 ? fallback : null;
+  return null;
 }
 
 export function isMediaReady(screen: Element) {

web/src/pages/desktop/screen/input-region-overlay.tsx

@@ -13,7 +13,6 @@ import {
   inputRegionSelectingAtom,
   manualInputRegionAtom,
   manualRegionsAtom,
-  resolutionAtom,
   selectedManualRegionAtom,
   selectedOriginalResolutionAtom,
   videoScaleAtom
@@ -87,7 +86,6 @@ export const InputRegionOverlay = () => {
   const [, setInputRegionState] = useAtom(inputRegionAtom);
   const setManualInputRegion = useSetAtom(manualInputRegionAtom);
   const [manualRegions, setManualRegions] = useAtom(manualRegionsAtom);
-  const resolution = useAtomValue(resolutionAtom);
   const setSelectedManualRegion = useSetAtom(selectedManualRegionAtom);
   const setSelectedOriginalResolution = useSetAtom(selectedOriginalResolutionAtom);
   const videoScale = useAtomValue(videoScaleAtom);
@@ -166,7 +164,7 @@ export const InputRegionOverlay = () => {
         frameRectRef.current = null;
         return;
       }
-      const nextMediaSize = getMediaSize(target, resolution);
+      const nextMediaSize = getMediaSize(target);
       const bounds = target.getBoundingClientRect();
       if (!nextMediaSize || bounds.width <= 0 || bounds.height <= 0) {
         setFrameRect(null);
@@ -206,7 +204,7 @@ export const InputRegionOverlay = () => {
       window.removeEventListener('scroll', updateFrame, true);
       window.clearInterval(timer);
     };
-  }, [resolution, selecting, videoScale]);
+  }, [selecting, videoScale]);
 
   useEffect(() => {
     if (!selecting) {

web/src/pages/desktop/screen/manual-region.tsx

@@ -5,7 +5,6 @@ import {
   controlRegionModeAtom,
   inputRegionAtom,
   manualRegionsAtom,
-  resolutionAtom,
   selectedManualRegionAtom,
   selectedOriginalResolutionAtom
 } from '@/jotai/screen.ts';
@@ -17,7 +16,6 @@ export const ManualRegion = () => {
   const selectedOriginal = useAtomValue(selectedOriginalResolutionAtom);
   const selectedManual = useAtomValue(selectedManualRegionAtom);
   const manualRegions = useAtomValue(manualRegionsAtom);
-  const resolution = useAtomValue(resolutionAtom);
   const setInputRegion = useSetAtom(inputRegionAtom);
 
   useEffect(() => {
@@ -44,7 +42,7 @@ export const ManualRegion = () => {
     const update = () => {
       const target = document.getElementById('screen');
       if (!target) return;
-      const mediaSize = getMediaSize(target, resolution);
+      const mediaSize = getMediaSize(target);
       setInputRegion(
         mediaSize ? getCenteredInputRegionByAspectRatio(width, height, mediaSize) : null
       );
@@ -54,7 +52,7 @@ export const ManualRegion = () => {
     return () => {
       window.clearInterval(timer);
     };
-  }, [manualRegions, mode, resolution, selectedManual, selectedOriginal, setInputRegion]);
+  }, [manualRegions, mode, selectedManual, selectedOriginal, setInputRegion]);
 
   return null;
 };

web/src/pages/desktop/screen/mjpeg.tsx

@@ -6,17 +6,14 @@ import { stopFrameDetect } from '@/api/stream.ts';
 import { getFrameDetect } from '@/lib/localstorage.ts';
 import { getBaseUrl } from '@/lib/service.ts';
 import { mouseStyleAtom } from '@/jotai/mouse.ts';
-import { resolutionAtom } from '@/jotai/screen.ts';
 
 import { ScreenViewport } from './viewport.tsx';
 
 export const Mjpeg = () => {
-  const resolution = useAtomValue(resolutionAtom);
   const mouseStyle = useAtomValue(mouseStyleAtom);
   const [hasError, setHasError] = useState(false);
-  const [streamNonce, setStreamNonce] = useState(0);
   const streamURL = `${getBaseUrl('http')}/api/stream/mjpeg`;
-  const streamSrc = hasError ? undefined : `${streamURL}?v=${streamNonce}`;
+  const streamSrc = hasError ? undefined : streamURL;
 
   useEffect(() => {
     // stop frame detect for a while
@@ -24,9 +21,7 @@ export const Mjpeg = () => {
     if (enabled) {
       stopFrameDetect(10);
     }
-    setHasError(false);
+  }, []);
-    setStreamNonce((current) => current + 1);
-  }, [resolution]);
 
   return (
     <ScreenViewport>