mirror of
https://github.com/sipeed/NanoKVM.git
synced 2026-09-11 00:22:56 -05:00
perf: improve H264 streaming client handling
Use client snapshots for stream fanout, reduce frame queue latency, expose backend ICE server configuration to the WebRTC client, and clean up disconnected WebRTC clients more aggressively.
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
|||||||
var (
|
var (
|
||||||
streamer = newStreamer()
|
streamer = newStreamer()
|
||||||
upgrader = websocket.Upgrader{
|
upgrader = websocket.Upgrader{
|
||||||
|
WriteBufferSize: 256 * 1024,
|
||||||
CheckOrigin: func(r *http.Request) bool {
|
CheckOrigin: func(r *http.Request) bool {
|
||||||
return true
|
return true
|
||||||
},
|
},
|
||||||
@@ -36,7 +37,7 @@ func Connect(c *gin.Context) {
|
|||||||
defer streamer.removeClient(ws)
|
defer streamer.removeClient(ws)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if _, _, err := ws.ReadMessage(); err != nil {
|
if _, _, err := ws.NextReader(); err != nil {
|
||||||
log.Debugf("failed to read message (client disconnected): %s", err)
|
log.Debugf("failed to read message (client disconnected): %s", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,20 +14,25 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Streamer struct {
|
type Streamer struct {
|
||||||
mutex sync.RWMutex
|
mutex sync.Mutex
|
||||||
clients map[*websocket.Conn]bool
|
clients map[*websocket.Conn]bool
|
||||||
running int32
|
clientSnapshot atomic.Pointer[[]*websocket.Conn]
|
||||||
|
running int32
|
||||||
}
|
}
|
||||||
|
|
||||||
func newStreamer() *Streamer {
|
func newStreamer() *Streamer {
|
||||||
return &Streamer{
|
s := &Streamer{
|
||||||
clients: make(map[*websocket.Conn]bool),
|
clients: make(map[*websocket.Conn]bool),
|
||||||
}
|
}
|
||||||
|
s.updateClientSnapshotLocked()
|
||||||
|
|
||||||
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) addClient(ws *websocket.Conn) {
|
func (s *Streamer) addClient(ws *websocket.Conn) {
|
||||||
s.mutex.Lock()
|
s.mutex.Lock()
|
||||||
s.clients[ws] = true
|
s.clients[ws] = true
|
||||||
|
s.updateClientSnapshotLocked()
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
|
|
||||||
if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||||||
@@ -39,31 +44,47 @@ func (s *Streamer) addClient(ws *websocket.Conn) {
|
|||||||
func (s *Streamer) removeClient(ws *websocket.Conn) {
|
func (s *Streamer) removeClient(ws *websocket.Conn) {
|
||||||
s.mutex.Lock()
|
s.mutex.Lock()
|
||||||
delete(s.clients, ws)
|
delete(s.clients, ws)
|
||||||
|
count := s.updateClientSnapshotLocked()
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
|
|
||||||
log.Debugf("h264 websocket disconnected, remaining clients: %d", len(s.clients))
|
log.Debugf("h264 websocket disconnected, remaining clients: %d", count)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) getClientCount() int {
|
func (s *Streamer) updateClientSnapshotLocked() int {
|
||||||
s.mutex.RLock()
|
clients := make([]*websocket.Conn, 0, len(s.clients))
|
||||||
defer s.mutex.RUnlock()
|
for client := range s.clients {
|
||||||
|
clients = append(clients, client)
|
||||||
|
}
|
||||||
|
s.clientSnapshot.Store(&clients)
|
||||||
|
|
||||||
return len(s.clients)
|
return len(clients)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Streamer) getClients() []*websocket.Conn {
|
||||||
|
clients := s.clientSnapshot.Load()
|
||||||
|
if clients == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return *clients
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) run() {
|
func (s *Streamer) run() {
|
||||||
defer atomic.StoreInt32(&s.running, 0)
|
defer atomic.StoreInt32(&s.running, 0)
|
||||||
|
|
||||||
duration := time.Second / time.Duration(120)
|
screen := common.GetScreen()
|
||||||
ticker := time.NewTicker(duration)
|
common.CheckScreen()
|
||||||
|
fps := screen.FPS
|
||||||
|
|
||||||
|
ticker := time.NewTicker(time.Second / time.Duration(fps))
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
screen := common.GetScreen()
|
|
||||||
vision := common.GetKvmVision()
|
vision := common.GetKvmVision()
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
|
|
||||||
for range ticker.C {
|
for range ticker.C {
|
||||||
if s.getClientCount() == 0 {
|
clients := s.getClients()
|
||||||
|
if len(clients) == 0 {
|
||||||
log.Debug("h264 stream stopped due to no clients")
|
log.Debug("h264 stream stopped due to no clients")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -81,28 +102,34 @@ func (s *Streamer) run() {
|
|||||||
|
|
||||||
timestamp := time.Since(startTime).Microseconds()
|
timestamp := time.Since(startTime).Microseconds()
|
||||||
|
|
||||||
if err := s.send(isKeyFrame, timestamp, data); err != nil {
|
if err := s.send(clients, isKeyFrame, timestamp, data); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if screen.FPS != fps && screen.FPS != 0 {
|
||||||
|
fps = screen.FPS
|
||||||
|
ticker.Reset(time.Second / time.Duration(fps))
|
||||||
|
}
|
||||||
|
|
||||||
stream.GetFrameRateCounter().Update()
|
stream.GetFrameRateCounter().Update()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) send(isKeyFrame byte, timestamp int64, data []byte) error {
|
func (s *Streamer) send(clients []*websocket.Conn, isKeyFrame byte, timestamp int64, data []byte) error {
|
||||||
buf := BufferPool.Get().(*bytes.Buffer)
|
buf := BufferPool.Get().(*bytes.Buffer)
|
||||||
defer BufferPool.Put(buf)
|
defer BufferPool.Put(buf)
|
||||||
|
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
|
buf.Grow(1 + 8 + len(data))
|
||||||
|
|
||||||
if err := buf.WriteByte(isKeyFrame); err != nil {
|
if err := buf.WriteByte(isKeyFrame); err != nil {
|
||||||
log.Errorf("failed to write keyframe flag: %s", err)
|
log.Errorf("failed to write keyframe flag: %s", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
tsBytes := make([]byte, 8)
|
var tsBytes [8]byte
|
||||||
binary.LittleEndian.PutUint64(tsBytes, uint64(timestamp))
|
binary.LittleEndian.PutUint64(tsBytes[:], uint64(timestamp))
|
||||||
if _, err := buf.Write(tsBytes); err != nil {
|
if _, err := buf.Write(tsBytes[:]); err != nil {
|
||||||
log.Errorf("failed to write timestamp: %s", err)
|
log.Errorf("failed to write timestamp: %s", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -112,7 +139,7 @@ func (s *Streamer) send(isKeyFrame byte, timestamp int64, data []byte) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for client := range s.clients {
|
for _, client := range clients {
|
||||||
if err := client.WriteMessage(websocket.BinaryMessage, buf.Bytes()); err != nil {
|
if err := client.WriteMessage(websocket.BinaryMessage, buf.Bytes()); err != nil {
|
||||||
log.Errorf("failed to write message to client %s: %s.", client.RemoteAddr(), err)
|
log.Errorf("failed to write message to client %s: %s.", client.RemoteAddr(), err)
|
||||||
|
|
||||||
|
|||||||
@@ -16,9 +16,8 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type FrameRateCounter struct {
|
type FrameRateCounter struct {
|
||||||
frameCount int32
|
frameCount atomic.Int32
|
||||||
fps int32
|
fps atomic.Int32
|
||||||
mutex sync.Mutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetFrameRateCounter() *FrameRateCounter {
|
func GetFrameRateCounter() *FrameRateCounter {
|
||||||
@@ -30,16 +29,10 @@ func GetFrameRateCounter() *FrameRateCounter {
|
|||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for range ticker.C {
|
for range ticker.C {
|
||||||
counter.mutex.Lock()
|
currentCount := counter.frameCount.Swap(0)
|
||||||
|
counter.fps.Store(currentCount / 3)
|
||||||
|
|
||||||
currentCount := atomic.LoadInt32(&counter.frameCount)
|
data := fmt.Sprintf("%d", counter.fps.Load())
|
||||||
|
|
||||||
counter.fps = currentCount / 3
|
|
||||||
atomic.StoreInt32(&counter.frameCount, 0)
|
|
||||||
|
|
||||||
counter.mutex.Unlock()
|
|
||||||
|
|
||||||
data := fmt.Sprintf("%d", counter.fps)
|
|
||||||
err := os.WriteFile("/kvmapp/kvm/now_fps", []byte(data), 0o666)
|
err := os.WriteFile("/kvmapp/kvm/now_fps", []byte(data), 0o666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed to write fps: %s", err)
|
log.Errorf("failed to write fps: %s", err)
|
||||||
@@ -52,12 +45,9 @@ func GetFrameRateCounter() *FrameRateCounter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (f *FrameRateCounter) Update() {
|
func (f *FrameRateCounter) Update() {
|
||||||
atomic.AddInt32(&f.frameCount, 1)
|
f.frameCount.Add(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *FrameRateCounter) GetFPS() int32 {
|
func (f *FrameRateCounter) GetFPS() int32 {
|
||||||
f.mutex.Lock()
|
return f.fps.Load()
|
||||||
defer f.mutex.Unlock()
|
|
||||||
|
|
||||||
return f.fps
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,24 +13,31 @@ import (
|
|||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var crlf = []byte("\r\n")
|
||||||
|
|
||||||
type Streamer struct {
|
type Streamer struct {
|
||||||
mutex sync.RWMutex
|
mutex sync.Mutex
|
||||||
clients map[*gin.Context]bool
|
clients map[*gin.Context]bool
|
||||||
running int32
|
clientSnapshot atomic.Pointer[[]*gin.Context]
|
||||||
frameMutex sync.RWMutex
|
running int32
|
||||||
latestFrame LatestFrame
|
frameMutex sync.RWMutex
|
||||||
cacheRefs int32
|
latestFrame LatestFrame
|
||||||
|
cacheRefs int32
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStreamer() *Streamer {
|
func NewStreamer() *Streamer {
|
||||||
return &Streamer{
|
s := &Streamer{
|
||||||
clients: make(map[*gin.Context]bool),
|
clients: make(map[*gin.Context]bool),
|
||||||
}
|
}
|
||||||
|
s.updateClientSnapshotLocked()
|
||||||
|
|
||||||
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) AddClient(c *gin.Context) {
|
func (s *Streamer) AddClient(c *gin.Context) {
|
||||||
s.mutex.Lock()
|
s.mutex.Lock()
|
||||||
s.clients[c] = true
|
s.clients[c] = true
|
||||||
|
s.updateClientSnapshotLocked()
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
|
|
||||||
if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
|
||||||
@@ -42,28 +49,29 @@ func (s *Streamer) AddClient(c *gin.Context) {
|
|||||||
func (s *Streamer) RemoveClient(c *gin.Context) {
|
func (s *Streamer) RemoveClient(c *gin.Context) {
|
||||||
s.mutex.Lock()
|
s.mutex.Lock()
|
||||||
delete(s.clients, c)
|
delete(s.clients, c)
|
||||||
|
count := s.updateClientSnapshotLocked()
|
||||||
s.mutex.Unlock()
|
s.mutex.Unlock()
|
||||||
|
|
||||||
log.Debugf("mjpeg connection removed, remaining clients: %d", len(s.clients))
|
log.Debugf("mjpeg connection removed, remaining clients: %d", count)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) getClients() []*gin.Context {
|
func (s *Streamer) updateClientSnapshotLocked() int {
|
||||||
s.mutex.RLock()
|
|
||||||
defer s.mutex.RUnlock()
|
|
||||||
|
|
||||||
clients := make([]*gin.Context, 0, len(s.clients))
|
clients := make([]*gin.Context, 0, len(s.clients))
|
||||||
for c := range s.clients {
|
for c := range s.clients {
|
||||||
clients = append(clients, c)
|
clients = append(clients, c)
|
||||||
}
|
}
|
||||||
|
s.clientSnapshot.Store(&clients)
|
||||||
|
|
||||||
return clients
|
return len(clients)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) getClientCount() int {
|
func (s *Streamer) getClients() []*gin.Context {
|
||||||
s.mutex.RLock()
|
clients := s.clientSnapshot.Load()
|
||||||
defer s.mutex.RUnlock()
|
if clients == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
return len(s.clients)
|
return *clients
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Streamer) run() {
|
func (s *Streamer) run() {
|
||||||
@@ -79,7 +87,8 @@ func (s *Streamer) run() {
|
|||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for range ticker.C {
|
for range ticker.C {
|
||||||
if s.getClientCount() == 0 {
|
clients := s.getClients()
|
||||||
|
if len(clients) == 0 {
|
||||||
log.Debug("mjpeg stream stopped due to no clients")
|
log.Debug("mjpeg stream stopped due to no clients")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -94,7 +103,6 @@ func (s *Streamer) run() {
|
|||||||
s.setLatestFrame(data, screen.Width, screen.Height)
|
s.setLatestFrame(data, screen.Width, screen.Height)
|
||||||
}
|
}
|
||||||
|
|
||||||
clients := s.getClients()
|
|
||||||
for _, client := range clients {
|
for _, client := range clients {
|
||||||
if err := writeFrame(client, data); err != nil {
|
if err := writeFrame(client, data); err != nil {
|
||||||
log.Errorf("failed to write mjpeg frame for client %s: %s", client.Request.RemoteAddr, err)
|
log.Errorf("failed to write mjpeg frame for client %s: %s", client.Request.RemoteAddr, err)
|
||||||
@@ -195,7 +203,7 @@ func writeFrame(c *gin.Context, data []byte) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = c.Writer.Write([]byte("\r\n")); err != nil {
|
if _, err = c.Writer.Write(crlf); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,20 @@ func NewClient(ws *websocket.Conn, videoConn *webrtc.PeerConnection) *Client {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) Close() {
|
||||||
|
if c.video != nil {
|
||||||
|
if err := c.video.Close(); err != nil {
|
||||||
|
log.Debugf("failed to close video peer connection: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.ws != nil {
|
||||||
|
if err := c.ws.Close(); err != nil {
|
||||||
|
log.Debugf("failed to close websocket: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Client) WriteMessage(event string, data string) error {
|
func (c *Client) WriteMessage(event string, data string) error {
|
||||||
c.mutex.Lock()
|
c.mutex.Lock()
|
||||||
defer c.mutex.Unlock()
|
defer c.mutex.Unlock()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package webrtc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"NanoKVM-Server/config"
|
"NanoKVM-Server/config"
|
||||||
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -15,6 +16,7 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
upgrader = websocket.Upgrader{
|
upgrader = websocket.Upgrader{
|
||||||
|
WriteBufferSize: 256 * 1024,
|
||||||
CheckOrigin: func(r *http.Request) bool {
|
CheckOrigin: func(r *http.Request) bool {
|
||||||
return true
|
return true
|
||||||
},
|
},
|
||||||
@@ -79,6 +81,10 @@ func Connect(c *gin.Context) {
|
|||||||
// handle signaling
|
// handle signaling
|
||||||
signalingHandler := NewSignalingHandler(client)
|
signalingHandler := NewSignalingHandler(client)
|
||||||
signalingHandler.RegisterCallbacks()
|
signalingHandler.RegisterCallbacks()
|
||||||
|
if err := sendICEServers(client, iceServers); err != nil {
|
||||||
|
log.Errorf("failed to send ICE servers: %s", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// read and wait
|
// read and wait
|
||||||
for {
|
for {
|
||||||
@@ -116,6 +122,30 @@ func createICEServers() []webrtc.ICEServer {
|
|||||||
return iceServers
|
return iceServers
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type clientICEServer struct {
|
||||||
|
URLs []string `json:"urls"`
|
||||||
|
Username string `json:"username,omitempty"`
|
||||||
|
Credential interface{} `json:"credential,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func sendICEServers(client *Client, iceServers []webrtc.ICEServer) error {
|
||||||
|
clientServers := make([]clientICEServer, 0, len(iceServers))
|
||||||
|
for _, server := range iceServers {
|
||||||
|
clientServers = append(clientServers, clientICEServer{
|
||||||
|
URLs: server.URLs,
|
||||||
|
Username: server.Username,
|
||||||
|
Credential: server.Credential,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(clientServers)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return client.WriteMessage("ice-servers", string(data))
|
||||||
|
}
|
||||||
|
|
||||||
func createMediaEngine() (*webrtc.MediaEngine, error) {
|
func createMediaEngine() (*webrtc.MediaEngine, error) {
|
||||||
mediaEngine := &webrtc.MediaEngine{}
|
mediaEngine := &webrtc.MediaEngine{}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package webrtc
|
|||||||
import (
|
import (
|
||||||
"NanoKVM-Server/common"
|
"NanoKVM-Server/common"
|
||||||
"NanoKVM-Server/service/stream"
|
"NanoKVM-Server/service/stream"
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -13,11 +12,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func NewWebRTCManager() *WebRTCManager {
|
func NewWebRTCManager() *WebRTCManager {
|
||||||
return &WebRTCManager{
|
m := &WebRTCManager{
|
||||||
clients: make(map[*websocket.Conn]*Client),
|
clients: make(map[*websocket.Conn]*Client),
|
||||||
videoSending: 0,
|
videoSending: 0,
|
||||||
mutex: sync.RWMutex{},
|
|
||||||
}
|
}
|
||||||
|
m.updateClientSnapshotLocked()
|
||||||
|
|
||||||
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) {
|
func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) {
|
||||||
@@ -25,24 +26,42 @@ func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) {
|
|||||||
|
|
||||||
m.mutex.Lock()
|
m.mutex.Lock()
|
||||||
m.clients[ws] = client
|
m.clients[ws] = client
|
||||||
|
count := m.updateClientSnapshotLocked()
|
||||||
m.mutex.Unlock()
|
m.mutex.Unlock()
|
||||||
|
|
||||||
log.Debugf("added client %s, total clients: %d", ws.RemoteAddr(), len(m.clients))
|
log.Debugf("added client %s, total clients: %d", ws.RemoteAddr(), count)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *WebRTCManager) RemoveClient(ws *websocket.Conn) {
|
func (m *WebRTCManager) RemoveClient(ws *websocket.Conn) {
|
||||||
m.mutex.Lock()
|
m.mutex.Lock()
|
||||||
delete(m.clients, ws)
|
delete(m.clients, ws)
|
||||||
|
count := m.updateClientSnapshotLocked()
|
||||||
m.mutex.Unlock()
|
m.mutex.Unlock()
|
||||||
|
|
||||||
log.Debugf("removed client %s, total clients: %d", ws.RemoteAddr(), len(m.clients))
|
log.Debugf("removed client %s, total clients: %d", ws.RemoteAddr(), count)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *WebRTCManager) GetClientCount() int {
|
func (m *WebRTCManager) GetClientCount() int {
|
||||||
m.mutex.RLock()
|
return len(m.getClients())
|
||||||
defer m.mutex.RUnlock()
|
}
|
||||||
|
|
||||||
return len(m.clients)
|
func (m *WebRTCManager) updateClientSnapshotLocked() int {
|
||||||
|
clients := make([]*Client, 0, len(m.clients))
|
||||||
|
for _, client := range m.clients {
|
||||||
|
clients = append(clients, client)
|
||||||
|
}
|
||||||
|
m.clientSnapshot.Store(&clients)
|
||||||
|
|
||||||
|
return len(clients)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *WebRTCManager) getClients() []*Client {
|
||||||
|
clients := m.clientSnapshot.Load()
|
||||||
|
if clients == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return *clients
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *WebRTCManager) StartVideoStream() {
|
func (m *WebRTCManager) StartVideoStream() {
|
||||||
@@ -66,7 +85,8 @@ func (m *WebRTCManager) sendVideoStream() {
|
|||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for range ticker.C {
|
for range ticker.C {
|
||||||
if m.GetClientCount() == 0 {
|
clients := m.getClients()
|
||||||
|
if len(clients) == 0 {
|
||||||
log.Debugf("stop sending h264 stream")
|
log.Debugf("stop sending h264 stream")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -82,8 +102,12 @@ func (m *WebRTCManager) sendVideoStream() {
|
|||||||
Duration: duration,
|
Duration: duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, client := range m.clients {
|
for _, client := range clients {
|
||||||
client.track.writeVideo(sample)
|
if err := client.track.writeVideoSample(sample); err != nil {
|
||||||
|
log.Errorf("failed to write h264 video to client: %s", err)
|
||||||
|
m.RemoveClient(client.WsConn())
|
||||||
|
client.Close()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if screen.FPS != fps && screen.FPS != 0 {
|
if screen.FPS != fps && screen.FPS != 0 {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ func (t *Track) updateExtension() {
|
|||||||
t.playoutDelayExtensionID = 5
|
t.playoutDelayExtensionID = 5
|
||||||
}
|
}
|
||||||
|
|
||||||
if t.playoutDelayExtensionData == nil || len(t.playoutDelayExtensionData) == 0 {
|
if len(t.playoutDelayExtensionData) == 0 {
|
||||||
playoutDelay := &rtp.PlayoutDelayExtension{
|
playoutDelay := &rtp.PlayoutDelayExtension{
|
||||||
MinDelay: 0,
|
MinDelay: 0,
|
||||||
MaxDelay: 0,
|
MaxDelay: 0,
|
||||||
@@ -44,10 +44,3 @@ func (t *Track) writeVideoSample(sample media.Sample) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *Track) writeVideo(sample media.Sample) {
|
|
||||||
err := t.writeVideoSample(sample)
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("failed to write h264 video: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package webrtc
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/gorilla/websocket"
|
"github.com/gorilla/websocket"
|
||||||
"github.com/pion/rtp"
|
"github.com/pion/rtp"
|
||||||
@@ -9,9 +10,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type WebRTCManager struct {
|
type WebRTCManager struct {
|
||||||
clients map[*websocket.Conn]*Client
|
clients map[*websocket.Conn]*Client
|
||||||
videoSending int32
|
clientSnapshot atomic.Pointer[[]*Client]
|
||||||
mutex sync.RWMutex
|
videoSending int32
|
||||||
|
mutex sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
type Client struct {
|
type Client struct {
|
||||||
@@ -21,6 +23,10 @@ type Client struct {
|
|||||||
mutex sync.Mutex
|
mutex sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) WsConn() *websocket.Conn {
|
||||||
|
return c.ws
|
||||||
|
}
|
||||||
|
|
||||||
type SignalingHandler struct {
|
type SignalingHandler struct {
|
||||||
client *Client
|
client *Client
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,17 @@ import Queue from 'yocto-queue';
|
|||||||
let canvas: OffscreenCanvas | null = null;
|
let canvas: OffscreenCanvas | null = null;
|
||||||
let ctx: OffscreenCanvasRenderingContext2D | null = null;
|
let ctx: OffscreenCanvasRenderingContext2D | null = null;
|
||||||
let rendering: boolean = false;
|
let rendering: boolean = false;
|
||||||
|
let flushScheduled: boolean = false;
|
||||||
let decoder: VideoDecoder | null = null;
|
let decoder: VideoDecoder | null = null;
|
||||||
|
|
||||||
|
const maxQueuedFrames = 3;
|
||||||
const frameQueue = new Queue<VideoFrame>();
|
const frameQueue = new Queue<VideoFrame>();
|
||||||
|
const frameChannel = new MessageChannel();
|
||||||
|
|
||||||
|
frameChannel.port1.onmessage = () => {
|
||||||
|
flushScheduled = false;
|
||||||
|
processFrameQueue();
|
||||||
|
};
|
||||||
|
|
||||||
self.onmessage = (event: MessageEvent) => {
|
self.onmessage = (event: MessageEvent) => {
|
||||||
const { type, data, canvas: offscreenCanvas } = event.data;
|
const { type, data, canvas: offscreenCanvas } = event.data;
|
||||||
@@ -61,13 +69,13 @@ function initializeDecoder() {
|
|||||||
const init = {
|
const init = {
|
||||||
output: (frame: VideoFrame) => {
|
output: (frame: VideoFrame) => {
|
||||||
frameQueue.enqueue(frame);
|
frameQueue.enqueue(frame);
|
||||||
if (frameQueue.size >= 10) {
|
while (frameQueue.size > maxQueuedFrames) {
|
||||||
frameQueue.dequeue()?.close();
|
frameQueue.dequeue()?.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!rendering) {
|
if (!rendering) {
|
||||||
rendering = true;
|
rendering = true;
|
||||||
processFrameQueue();
|
scheduleFrameQueue();
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
error: () => {
|
error: () => {
|
||||||
@@ -110,12 +118,21 @@ function processFrameQueue() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (frameQueue.size > 0) {
|
if (frameQueue.size > 0) {
|
||||||
setTimeout(processFrameQueue, 0);
|
scheduleFrameQueue();
|
||||||
} else {
|
} else {
|
||||||
rendering = false;
|
rendering = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function scheduleFrameQueue() {
|
||||||
|
if (flushScheduled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
flushScheduled = true;
|
||||||
|
frameChannel.port2.postMessage(null);
|
||||||
|
}
|
||||||
|
|
||||||
function renderFrame(frame: VideoFrame) {
|
function renderFrame(frame: VideoFrame) {
|
||||||
if (!canvas || !ctx) {
|
if (!canvas || !ctx) {
|
||||||
frame.close();
|
frame.close();
|
||||||
@@ -142,6 +159,7 @@ function resetDecoder() {
|
|||||||
|
|
||||||
decoder = null;
|
decoder = null;
|
||||||
rendering = false;
|
rendering = false;
|
||||||
|
flushScheduled = false;
|
||||||
|
|
||||||
Array.from(frameQueue.drain()).forEach((frame) => frame.close());
|
Array.from(frameQueue.drain()).forEach((frame) => frame.close());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,19 @@ import { getBaseUrl } from '@/lib/service.ts';
|
|||||||
import { mouseStyleAtom } from '@/jotai/mouse.ts';
|
import { mouseStyleAtom } from '@/jotai/mouse.ts';
|
||||||
import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts';
|
import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts';
|
||||||
|
|
||||||
|
type SignalingMessage = {
|
||||||
|
event?: string;
|
||||||
|
data?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseSignalingData = <T,>(data?: string): T | null => {
|
||||||
|
if (!data) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return JSON.parse(data) as T;
|
||||||
|
};
|
||||||
|
|
||||||
export const H264Webrtc = () => {
|
export const H264Webrtc = () => {
|
||||||
const resolution = useAtomValue(resolutionAtom);
|
const resolution = useAtomValue(resolutionAtom);
|
||||||
const mouseStyle = useAtomValue(mouseStyleAtom);
|
const mouseStyle = useAtomValue(mouseStyleAtom);
|
||||||
@@ -21,120 +34,11 @@ export const H264Webrtc = () => {
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const url = `${getBaseUrl('ws')}/api/stream/h264`;
|
const url = `${getBaseUrl('ws')}/api/stream/h264`;
|
||||||
const ws = new W3cWebSocket(url);
|
const ws = new W3cWebSocket(url);
|
||||||
|
const videoElement = videoRef.current;
|
||||||
|
|
||||||
const iceServers = [{ urls: ['stun:stun.l.google.com:19302'] }];
|
let video: RTCPeerConnection | null = null;
|
||||||
const video = new RTCPeerConnection({ iceServers });
|
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
let disposed = false;
|
||||||
// --- Init Video ---
|
|
||||||
video.onnegotiationneeded = async () => {
|
|
||||||
if (videoOfferSent.current || video.signalingState !== 'stable') {
|
|
||||||
console.log('Skipping video negotiation - Waiting for answer or state unstable');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
videoOfferSent.current = true;
|
|
||||||
const offer = await video.createOffer({
|
|
||||||
offerToReceiveVideo: true,
|
|
||||||
offerToReceiveAudio: false
|
|
||||||
});
|
|
||||||
|
|
||||||
await video.setLocalDescription(offer);
|
|
||||||
|
|
||||||
sendMsg('video-offer', JSON.stringify(video.localDescription));
|
|
||||||
} catch (error) {
|
|
||||||
videoOfferSent.current = false;
|
|
||||||
console.error('Video negotiation failed:', error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
video.onconnectionstatechange = () => {
|
|
||||||
if (video.iceConnectionState === 'connected') {
|
|
||||||
setIsLoading(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
video.ontrack = (event) => {
|
|
||||||
if (videoRef.current && event.track.kind === 'video') {
|
|
||||||
videoRef.current.srcObject = new MediaStream([event.track]);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
ws.onopen = () => {
|
|
||||||
videoOfferSent.current = false;
|
|
||||||
|
|
||||||
video.onicecandidate = (event) => {
|
|
||||||
if (event.candidate) {
|
|
||||||
sendMsg('video-candidate', JSON.stringify(event.candidate));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
video.addTransceiver('video', { direction: 'recvonly' });
|
|
||||||
};
|
|
||||||
|
|
||||||
ws.onmessage = (event) => {
|
|
||||||
try {
|
|
||||||
const msg = JSON.parse(event.data as string);
|
|
||||||
if (!msg?.data) return;
|
|
||||||
|
|
||||||
const data = JSON.parse(msg.data);
|
|
||||||
if (!data) return;
|
|
||||||
|
|
||||||
switch (msg.event) {
|
|
||||||
case 'video-answer':
|
|
||||||
handleVideoAnswer(data);
|
|
||||||
break;
|
|
||||||
case 'video-candidate':
|
|
||||||
handleVideoCandidate(data);
|
|
||||||
break;
|
|
||||||
case 'heartbeat':
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
console.log('Unhandled event:', msg.event);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Message processing error:', err);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleVideoAnswer = (data: any) => {
|
|
||||||
if (video.signalingState !== 'have-local-offer') {
|
|
||||||
videoOfferSent.current = false;
|
|
||||||
console.warn(`Video signaling state incorrect for answer: ${video.signalingState}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
video
|
|
||||||
.setRemoteDescription(new RTCSessionDescription(data))
|
|
||||||
.then(() => {
|
|
||||||
videoOfferSent.current = false;
|
|
||||||
videoIceCandidates.current.forEach((candidate) => {
|
|
||||||
video
|
|
||||||
.addIceCandidate(candidate)
|
|
||||||
.catch((e) => console.error('Video candidate failed to add:', e.message));
|
|
||||||
});
|
|
||||||
videoIceCandidates.current = [];
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
console.error('Video answer set failed:', error);
|
|
||||||
videoOfferSent.current = false;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleVideoCandidate = (data: any) => {
|
|
||||||
if (!data.candidate) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const candidate = new RTCIceCandidate(data);
|
|
||||||
if (video.remoteDescription) {
|
|
||||||
video
|
|
||||||
.addIceCandidate(candidate)
|
|
||||||
.catch((e) => console.error('Video candidate failed to add:', e.message));
|
|
||||||
} else {
|
|
||||||
videoIceCandidates.current.push(candidate);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const sendMsg = (event: string, data: string) => {
|
const sendMsg = (event: string, data: string) => {
|
||||||
if (ws.readyState !== WebSocket.OPEN) {
|
if (ws.readyState !== WebSocket.OPEN) {
|
||||||
@@ -148,25 +52,173 @@ export const H264Webrtc = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const heartbeatTimer = setInterval(() => {
|
const startVideo = (iceServers: RTCIceServer[]) => {
|
||||||
sendMsg('heartbeat', '');
|
if (video || disposed) {
|
||||||
}, 60 * 1000);
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
setTimeout(() => {
|
const peer = new RTCPeerConnection({ iceServers });
|
||||||
|
video = peer;
|
||||||
|
videoOfferSent.current = false;
|
||||||
|
videoIceCandidates.current = [];
|
||||||
|
|
||||||
|
// --- Init Video ---
|
||||||
|
peer.onnegotiationneeded = async () => {
|
||||||
|
if (videoOfferSent.current || peer.signalingState !== 'stable') {
|
||||||
|
console.log('Skipping video negotiation - Waiting for answer or state unstable');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
videoOfferSent.current = true;
|
||||||
|
const offer = await peer.createOffer({
|
||||||
|
offerToReceiveVideo: true,
|
||||||
|
offerToReceiveAudio: false
|
||||||
|
});
|
||||||
|
|
||||||
|
await peer.setLocalDescription(offer);
|
||||||
|
|
||||||
|
sendMsg('video-offer', JSON.stringify(peer.localDescription));
|
||||||
|
} catch (error) {
|
||||||
|
videoOfferSent.current = false;
|
||||||
|
console.error('Video negotiation failed:', error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
peer.onconnectionstatechange = () => {
|
||||||
|
if (peer.iceConnectionState === 'connected' || peer.connectionState === 'connected') {
|
||||||
|
setIsLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
peer.ontrack = (event) => {
|
||||||
|
if (videoElement && event.track.kind === 'video') {
|
||||||
|
videoElement.srcObject = new MediaStream([event.track]);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
peer.onicecandidate = (event) => {
|
||||||
|
if (event.candidate) {
|
||||||
|
sendMsg('video-candidate', JSON.stringify(event.candidate));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
peer.addTransceiver('video', { direction: 'recvonly' });
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleVideoAnswer = (data: RTCSessionDescriptionInit) => {
|
||||||
|
const peer = video;
|
||||||
|
if (!peer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (peer.signalingState !== 'have-local-offer') {
|
||||||
|
videoOfferSent.current = false;
|
||||||
|
console.warn(`Video signaling state incorrect for answer: ${peer.signalingState}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
peer
|
||||||
|
.setRemoteDescription(new RTCSessionDescription(data))
|
||||||
|
.then(() => {
|
||||||
|
videoOfferSent.current = false;
|
||||||
|
videoIceCandidates.current.forEach((candidate) => {
|
||||||
|
peer
|
||||||
|
.addIceCandidate(candidate)
|
||||||
|
.catch((e) => console.error('Video candidate failed to add:', e.message));
|
||||||
|
});
|
||||||
|
videoIceCandidates.current = [];
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error('Video answer set failed:', error);
|
||||||
|
videoOfferSent.current = false;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleVideoCandidate = (data: RTCIceCandidateInit) => {
|
||||||
|
const peer = video;
|
||||||
|
if (!peer || !data.candidate) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const candidate = new RTCIceCandidate(data);
|
||||||
|
if (peer.remoteDescription) {
|
||||||
|
peer
|
||||||
|
.addIceCandidate(candidate)
|
||||||
|
.catch((e) => console.error('Video candidate failed to add:', e.message));
|
||||||
|
} else {
|
||||||
|
videoIceCandidates.current.push(candidate);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
ws.onopen = () => {
|
||||||
|
if (disposed) {
|
||||||
|
ws.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
heartbeatTimer = setInterval(() => {
|
||||||
|
sendMsg('heartbeat', '');
|
||||||
|
}, 60 * 1000);
|
||||||
|
};
|
||||||
|
|
||||||
|
ws.onmessage = (event) => {
|
||||||
|
try {
|
||||||
|
const msg = JSON.parse(event.data as string) as SignalingMessage;
|
||||||
|
|
||||||
|
switch (msg.event) {
|
||||||
|
case 'ice-servers': {
|
||||||
|
const iceServers = parseSignalingData<RTCIceServer[]>(msg.data);
|
||||||
|
startVideo(Array.isArray(iceServers) ? iceServers : []);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'video-answer': {
|
||||||
|
const data = parseSignalingData<RTCSessionDescriptionInit>(msg.data);
|
||||||
|
if (data) {
|
||||||
|
handleVideoAnswer(data);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'video-candidate': {
|
||||||
|
const data = parseSignalingData<RTCIceCandidateInit>(msg.data);
|
||||||
|
if (data) {
|
||||||
|
handleVideoCandidate(data);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'heartbeat':
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
console.log('Unhandled event:', msg.event);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Message processing error:', err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const loadingTimer = setTimeout(() => {
|
||||||
setIsLoading(false);
|
setIsLoading(false);
|
||||||
}, 5 * 1000);
|
}, 5 * 1000);
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
if (ws.readyState === WebSocket.OPEN) {
|
disposed = true;
|
||||||
|
|
||||||
|
if (ws.readyState !== WebSocket.CLOSING && ws.readyState !== WebSocket.CLOSED) {
|
||||||
ws.close();
|
ws.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
video.close();
|
video?.close();
|
||||||
|
video = null;
|
||||||
|
if (videoElement) {
|
||||||
|
videoElement.srcObject = null;
|
||||||
|
}
|
||||||
videoOfferSent.current = false;
|
videoOfferSent.current = false;
|
||||||
|
videoIceCandidates.current = [];
|
||||||
|
|
||||||
if (heartbeatTimer) {
|
if (heartbeatTimer) {
|
||||||
clearInterval(heartbeatTimer);
|
clearInterval(heartbeatTimer);
|
||||||
}
|
}
|
||||||
|
clearTimeout(loadingTimer);
|
||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user