perf: improve H264 streaming client handling

Use client snapshots for stream fanout, reduce frame queue latency, expose backend ICE server configuration to the WebRTC client, and clean up disconnected WebRTC clients more aggressively.
This commit is contained in:
wenjie
2026-05-19 17:04:57 +08:00
parent af19795c29
commit b653f905e4
11 changed files with 364 additions and 201 deletions

View File

@@ -12,6 +12,7 @@ import (
var ( var (
streamer = newStreamer() streamer = newStreamer()
upgrader = websocket.Upgrader{ upgrader = websocket.Upgrader{
WriteBufferSize: 256 * 1024,
CheckOrigin: func(r *http.Request) bool { CheckOrigin: func(r *http.Request) bool {
return true return true
}, },
@@ -36,7 +37,7 @@ func Connect(c *gin.Context) {
defer streamer.removeClient(ws) defer streamer.removeClient(ws)
for { for {
if _, _, err := ws.ReadMessage(); err != nil { if _, _, err := ws.NextReader(); err != nil {
log.Debugf("failed to read message (client disconnected): %s", err) log.Debugf("failed to read message (client disconnected): %s", err)
return return
} }

View File

@@ -14,20 +14,25 @@ import (
) )
type Streamer struct { type Streamer struct {
mutex sync.RWMutex mutex sync.Mutex
clients map[*websocket.Conn]bool clients map[*websocket.Conn]bool
running int32 clientSnapshot atomic.Pointer[[]*websocket.Conn]
running int32
} }
func newStreamer() *Streamer { func newStreamer() *Streamer {
return &Streamer{ s := &Streamer{
clients: make(map[*websocket.Conn]bool), clients: make(map[*websocket.Conn]bool),
} }
s.updateClientSnapshotLocked()
return s
} }
func (s *Streamer) addClient(ws *websocket.Conn) { func (s *Streamer) addClient(ws *websocket.Conn) {
s.mutex.Lock() s.mutex.Lock()
s.clients[ws] = true s.clients[ws] = true
s.updateClientSnapshotLocked()
s.mutex.Unlock() s.mutex.Unlock()
if atomic.CompareAndSwapInt32(&s.running, 0, 1) { if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
@@ -39,31 +44,47 @@ func (s *Streamer) addClient(ws *websocket.Conn) {
func (s *Streamer) removeClient(ws *websocket.Conn) { func (s *Streamer) removeClient(ws *websocket.Conn) {
s.mutex.Lock() s.mutex.Lock()
delete(s.clients, ws) delete(s.clients, ws)
count := s.updateClientSnapshotLocked()
s.mutex.Unlock() s.mutex.Unlock()
log.Debugf("h264 websocket disconnected, remaining clients: %d", len(s.clients)) log.Debugf("h264 websocket disconnected, remaining clients: %d", count)
} }
func (s *Streamer) getClientCount() int { func (s *Streamer) updateClientSnapshotLocked() int {
s.mutex.RLock() clients := make([]*websocket.Conn, 0, len(s.clients))
defer s.mutex.RUnlock() for client := range s.clients {
clients = append(clients, client)
}
s.clientSnapshot.Store(&clients)
return len(s.clients) return len(clients)
}
func (s *Streamer) getClients() []*websocket.Conn {
clients := s.clientSnapshot.Load()
if clients == nil {
return nil
}
return *clients
} }
func (s *Streamer) run() { func (s *Streamer) run() {
defer atomic.StoreInt32(&s.running, 0) defer atomic.StoreInt32(&s.running, 0)
duration := time.Second / time.Duration(120) screen := common.GetScreen()
ticker := time.NewTicker(duration) common.CheckScreen()
fps := screen.FPS
ticker := time.NewTicker(time.Second / time.Duration(fps))
defer ticker.Stop() defer ticker.Stop()
screen := common.GetScreen()
vision := common.GetKvmVision() vision := common.GetKvmVision()
startTime := time.Now() startTime := time.Now()
for range ticker.C { for range ticker.C {
if s.getClientCount() == 0 { clients := s.getClients()
if len(clients) == 0 {
log.Debug("h264 stream stopped due to no clients") log.Debug("h264 stream stopped due to no clients")
return return
} }
@@ -81,28 +102,34 @@ func (s *Streamer) run() {
timestamp := time.Since(startTime).Microseconds() timestamp := time.Since(startTime).Microseconds()
if err := s.send(isKeyFrame, timestamp, data); err != nil { if err := s.send(clients, isKeyFrame, timestamp, data); err != nil {
continue continue
} }
if screen.FPS != fps && screen.FPS != 0 {
fps = screen.FPS
ticker.Reset(time.Second / time.Duration(fps))
}
stream.GetFrameRateCounter().Update() stream.GetFrameRateCounter().Update()
} }
} }
func (s *Streamer) send(isKeyFrame byte, timestamp int64, data []byte) error { func (s *Streamer) send(clients []*websocket.Conn, isKeyFrame byte, timestamp int64, data []byte) error {
buf := BufferPool.Get().(*bytes.Buffer) buf := BufferPool.Get().(*bytes.Buffer)
defer BufferPool.Put(buf) defer BufferPool.Put(buf)
buf.Reset() buf.Reset()
buf.Grow(1 + 8 + len(data))
if err := buf.WriteByte(isKeyFrame); err != nil { if err := buf.WriteByte(isKeyFrame); err != nil {
log.Errorf("failed to write keyframe flag: %s", err) log.Errorf("failed to write keyframe flag: %s", err)
return err return err
} }
tsBytes := make([]byte, 8) var tsBytes [8]byte
binary.LittleEndian.PutUint64(tsBytes, uint64(timestamp)) binary.LittleEndian.PutUint64(tsBytes[:], uint64(timestamp))
if _, err := buf.Write(tsBytes); err != nil { if _, err := buf.Write(tsBytes[:]); err != nil {
log.Errorf("failed to write timestamp: %s", err) log.Errorf("failed to write timestamp: %s", err)
return err return err
} }
@@ -112,7 +139,7 @@ func (s *Streamer) send(isKeyFrame byte, timestamp int64, data []byte) error {
return err return err
} }
for client := range s.clients { for _, client := range clients {
if err := client.WriteMessage(websocket.BinaryMessage, buf.Bytes()); err != nil { if err := client.WriteMessage(websocket.BinaryMessage, buf.Bytes()); err != nil {
log.Errorf("failed to write message to client %s: %s.", client.RemoteAddr(), err) log.Errorf("failed to write message to client %s: %s.", client.RemoteAddr(), err)

View File

@@ -16,9 +16,8 @@ var (
) )
type FrameRateCounter struct { type FrameRateCounter struct {
frameCount int32 frameCount atomic.Int32
fps int32 fps atomic.Int32
mutex sync.Mutex
} }
func GetFrameRateCounter() *FrameRateCounter { func GetFrameRateCounter() *FrameRateCounter {
@@ -30,16 +29,10 @@ func GetFrameRateCounter() *FrameRateCounter {
defer ticker.Stop() defer ticker.Stop()
for range ticker.C { for range ticker.C {
counter.mutex.Lock() currentCount := counter.frameCount.Swap(0)
counter.fps.Store(currentCount / 3)
currentCount := atomic.LoadInt32(&counter.frameCount) data := fmt.Sprintf("%d", counter.fps.Load())
counter.fps = currentCount / 3
atomic.StoreInt32(&counter.frameCount, 0)
counter.mutex.Unlock()
data := fmt.Sprintf("%d", counter.fps)
err := os.WriteFile("/kvmapp/kvm/now_fps", []byte(data), 0o666) err := os.WriteFile("/kvmapp/kvm/now_fps", []byte(data), 0o666)
if err != nil { if err != nil {
log.Errorf("failed to write fps: %s", err) log.Errorf("failed to write fps: %s", err)
@@ -52,12 +45,9 @@ func GetFrameRateCounter() *FrameRateCounter {
} }
func (f *FrameRateCounter) Update() { func (f *FrameRateCounter) Update() {
atomic.AddInt32(&f.frameCount, 1) f.frameCount.Add(1)
} }
func (f *FrameRateCounter) GetFPS() int32 { func (f *FrameRateCounter) GetFPS() int32 {
f.mutex.Lock() return f.fps.Load()
defer f.mutex.Unlock()
return f.fps
} }

View File

@@ -13,24 +13,31 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
var crlf = []byte("\r\n")
type Streamer struct { type Streamer struct {
mutex sync.RWMutex mutex sync.Mutex
clients map[*gin.Context]bool clients map[*gin.Context]bool
running int32 clientSnapshot atomic.Pointer[[]*gin.Context]
frameMutex sync.RWMutex running int32
latestFrame LatestFrame frameMutex sync.RWMutex
cacheRefs int32 latestFrame LatestFrame
cacheRefs int32
} }
func NewStreamer() *Streamer { func NewStreamer() *Streamer {
return &Streamer{ s := &Streamer{
clients: make(map[*gin.Context]bool), clients: make(map[*gin.Context]bool),
} }
s.updateClientSnapshotLocked()
return s
} }
func (s *Streamer) AddClient(c *gin.Context) { func (s *Streamer) AddClient(c *gin.Context) {
s.mutex.Lock() s.mutex.Lock()
s.clients[c] = true s.clients[c] = true
s.updateClientSnapshotLocked()
s.mutex.Unlock() s.mutex.Unlock()
if atomic.CompareAndSwapInt32(&s.running, 0, 1) { if atomic.CompareAndSwapInt32(&s.running, 0, 1) {
@@ -42,28 +49,29 @@ func (s *Streamer) AddClient(c *gin.Context) {
func (s *Streamer) RemoveClient(c *gin.Context) { func (s *Streamer) RemoveClient(c *gin.Context) {
s.mutex.Lock() s.mutex.Lock()
delete(s.clients, c) delete(s.clients, c)
count := s.updateClientSnapshotLocked()
s.mutex.Unlock() s.mutex.Unlock()
log.Debugf("mjpeg connection removed, remaining clients: %d", len(s.clients)) log.Debugf("mjpeg connection removed, remaining clients: %d", count)
} }
func (s *Streamer) getClients() []*gin.Context { func (s *Streamer) updateClientSnapshotLocked() int {
s.mutex.RLock()
defer s.mutex.RUnlock()
clients := make([]*gin.Context, 0, len(s.clients)) clients := make([]*gin.Context, 0, len(s.clients))
for c := range s.clients { for c := range s.clients {
clients = append(clients, c) clients = append(clients, c)
} }
s.clientSnapshot.Store(&clients)
return clients return len(clients)
} }
func (s *Streamer) getClientCount() int { func (s *Streamer) getClients() []*gin.Context {
s.mutex.RLock() clients := s.clientSnapshot.Load()
defer s.mutex.RUnlock() if clients == nil {
return nil
}
return len(s.clients) return *clients
} }
func (s *Streamer) run() { func (s *Streamer) run() {
@@ -79,7 +87,8 @@ func (s *Streamer) run() {
defer ticker.Stop() defer ticker.Stop()
for range ticker.C { for range ticker.C {
if s.getClientCount() == 0 { clients := s.getClients()
if len(clients) == 0 {
log.Debug("mjpeg stream stopped due to no clients") log.Debug("mjpeg stream stopped due to no clients")
return return
} }
@@ -94,7 +103,6 @@ func (s *Streamer) run() {
s.setLatestFrame(data, screen.Width, screen.Height) s.setLatestFrame(data, screen.Width, screen.Height)
} }
clients := s.getClients()
for _, client := range clients { for _, client := range clients {
if err := writeFrame(client, data); err != nil { if err := writeFrame(client, data); err != nil {
log.Errorf("failed to write mjpeg frame for client %s: %s", client.Request.RemoteAddr, err) log.Errorf("failed to write mjpeg frame for client %s: %s", client.Request.RemoteAddr, err)
@@ -195,7 +203,7 @@ func writeFrame(c *gin.Context, data []byte) (err error) {
return err return err
} }
if _, err = c.Writer.Write([]byte("\r\n")); err != nil { if _, err = c.Writer.Write(crlf); err != nil {
return err return err
} }

View File

@@ -21,6 +21,20 @@ func NewClient(ws *websocket.Conn, videoConn *webrtc.PeerConnection) *Client {
} }
} }
func (c *Client) Close() {
if c.video != nil {
if err := c.video.Close(); err != nil {
log.Debugf("failed to close video peer connection: %s", err)
}
}
if c.ws != nil {
if err := c.ws.Close(); err != nil {
log.Debugf("failed to close websocket: %s", err)
}
}
}
func (c *Client) WriteMessage(event string, data string) error { func (c *Client) WriteMessage(event string, data string) error {
c.mutex.Lock() c.mutex.Lock()
defer c.mutex.Unlock() defer c.mutex.Unlock()

View File

@@ -2,6 +2,7 @@ package webrtc
import ( import (
"NanoKVM-Server/config" "NanoKVM-Server/config"
"encoding/json"
"net/http" "net/http"
"sync" "sync"
"time" "time"
@@ -15,6 +16,7 @@ import (
var ( var (
upgrader = websocket.Upgrader{ upgrader = websocket.Upgrader{
WriteBufferSize: 256 * 1024,
CheckOrigin: func(r *http.Request) bool { CheckOrigin: func(r *http.Request) bool {
return true return true
}, },
@@ -79,6 +81,10 @@ func Connect(c *gin.Context) {
// handle signaling // handle signaling
signalingHandler := NewSignalingHandler(client) signalingHandler := NewSignalingHandler(client)
signalingHandler.RegisterCallbacks() signalingHandler.RegisterCallbacks()
if err := sendICEServers(client, iceServers); err != nil {
log.Errorf("failed to send ICE servers: %s", err)
return
}
// read and wait // read and wait
for { for {
@@ -116,6 +122,30 @@ func createICEServers() []webrtc.ICEServer {
return iceServers return iceServers
} }
type clientICEServer struct {
URLs []string `json:"urls"`
Username string `json:"username,omitempty"`
Credential interface{} `json:"credential,omitempty"`
}
func sendICEServers(client *Client, iceServers []webrtc.ICEServer) error {
clientServers := make([]clientICEServer, 0, len(iceServers))
for _, server := range iceServers {
clientServers = append(clientServers, clientICEServer{
URLs: server.URLs,
Username: server.Username,
Credential: server.Credential,
})
}
data, err := json.Marshal(clientServers)
if err != nil {
return err
}
return client.WriteMessage("ice-servers", string(data))
}
func createMediaEngine() (*webrtc.MediaEngine, error) { func createMediaEngine() (*webrtc.MediaEngine, error) {
mediaEngine := &webrtc.MediaEngine{} mediaEngine := &webrtc.MediaEngine{}

View File

@@ -3,7 +3,6 @@ package webrtc
import ( import (
"NanoKVM-Server/common" "NanoKVM-Server/common"
"NanoKVM-Server/service/stream" "NanoKVM-Server/service/stream"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
@@ -13,11 +12,13 @@ import (
) )
func NewWebRTCManager() *WebRTCManager { func NewWebRTCManager() *WebRTCManager {
return &WebRTCManager{ m := &WebRTCManager{
clients: make(map[*websocket.Conn]*Client), clients: make(map[*websocket.Conn]*Client),
videoSending: 0, videoSending: 0,
mutex: sync.RWMutex{},
} }
m.updateClientSnapshotLocked()
return m
} }
func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) { func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) {
@@ -25,24 +26,42 @@ func (m *WebRTCManager) AddClient(ws *websocket.Conn, client *Client) {
m.mutex.Lock() m.mutex.Lock()
m.clients[ws] = client m.clients[ws] = client
count := m.updateClientSnapshotLocked()
m.mutex.Unlock() m.mutex.Unlock()
log.Debugf("added client %s, total clients: %d", ws.RemoteAddr(), len(m.clients)) log.Debugf("added client %s, total clients: %d", ws.RemoteAddr(), count)
} }
func (m *WebRTCManager) RemoveClient(ws *websocket.Conn) { func (m *WebRTCManager) RemoveClient(ws *websocket.Conn) {
m.mutex.Lock() m.mutex.Lock()
delete(m.clients, ws) delete(m.clients, ws)
count := m.updateClientSnapshotLocked()
m.mutex.Unlock() m.mutex.Unlock()
log.Debugf("removed client %s, total clients: %d", ws.RemoteAddr(), len(m.clients)) log.Debugf("removed client %s, total clients: %d", ws.RemoteAddr(), count)
} }
func (m *WebRTCManager) GetClientCount() int { func (m *WebRTCManager) GetClientCount() int {
m.mutex.RLock() return len(m.getClients())
defer m.mutex.RUnlock() }
return len(m.clients) func (m *WebRTCManager) updateClientSnapshotLocked() int {
clients := make([]*Client, 0, len(m.clients))
for _, client := range m.clients {
clients = append(clients, client)
}
m.clientSnapshot.Store(&clients)
return len(clients)
}
func (m *WebRTCManager) getClients() []*Client {
clients := m.clientSnapshot.Load()
if clients == nil {
return nil
}
return *clients
} }
func (m *WebRTCManager) StartVideoStream() { func (m *WebRTCManager) StartVideoStream() {
@@ -66,7 +85,8 @@ func (m *WebRTCManager) sendVideoStream() {
defer ticker.Stop() defer ticker.Stop()
for range ticker.C { for range ticker.C {
if m.GetClientCount() == 0 { clients := m.getClients()
if len(clients) == 0 {
log.Debugf("stop sending h264 stream") log.Debugf("stop sending h264 stream")
return return
} }
@@ -82,8 +102,12 @@ func (m *WebRTCManager) sendVideoStream() {
Duration: duration, Duration: duration,
} }
for _, client := range m.clients { for _, client := range clients {
client.track.writeVideo(sample) if err := client.track.writeVideoSample(sample); err != nil {
log.Errorf("failed to write h264 video to client: %s", err)
m.RemoveClient(client.WsConn())
client.Close()
}
} }
if screen.FPS != fps && screen.FPS != 0 { if screen.FPS != fps && screen.FPS != 0 {

View File

@@ -11,7 +11,7 @@ func (t *Track) updateExtension() {
t.playoutDelayExtensionID = 5 t.playoutDelayExtensionID = 5
} }
if t.playoutDelayExtensionData == nil || len(t.playoutDelayExtensionData) == 0 { if len(t.playoutDelayExtensionData) == 0 {
playoutDelay := &rtp.PlayoutDelayExtension{ playoutDelay := &rtp.PlayoutDelayExtension{
MinDelay: 0, MinDelay: 0,
MaxDelay: 0, MaxDelay: 0,
@@ -44,10 +44,3 @@ func (t *Track) writeVideoSample(sample media.Sample) error {
return nil return nil
} }
func (t *Track) writeVideo(sample media.Sample) {
err := t.writeVideoSample(sample)
if err != nil {
log.Errorf("failed to write h264 video: %s", err)
}
}

View File

@@ -2,6 +2,7 @@ package webrtc
import ( import (
"sync" "sync"
"sync/atomic"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/pion/rtp" "github.com/pion/rtp"
@@ -9,9 +10,10 @@ import (
) )
type WebRTCManager struct { type WebRTCManager struct {
clients map[*websocket.Conn]*Client clients map[*websocket.Conn]*Client
videoSending int32 clientSnapshot atomic.Pointer[[]*Client]
mutex sync.RWMutex videoSending int32
mutex sync.Mutex
} }
type Client struct { type Client struct {
@@ -21,6 +23,10 @@ type Client struct {
mutex sync.Mutex mutex sync.Mutex
} }
func (c *Client) WsConn() *websocket.Conn {
return c.ws
}
type SignalingHandler struct { type SignalingHandler struct {
client *Client client *Client
} }

View File

@@ -3,9 +3,17 @@ import Queue from 'yocto-queue';
let canvas: OffscreenCanvas | null = null; let canvas: OffscreenCanvas | null = null;
let ctx: OffscreenCanvasRenderingContext2D | null = null; let ctx: OffscreenCanvasRenderingContext2D | null = null;
let rendering: boolean = false; let rendering: boolean = false;
let flushScheduled: boolean = false;
let decoder: VideoDecoder | null = null; let decoder: VideoDecoder | null = null;
const maxQueuedFrames = 3;
const frameQueue = new Queue<VideoFrame>(); const frameQueue = new Queue<VideoFrame>();
const frameChannel = new MessageChannel();
frameChannel.port1.onmessage = () => {
flushScheduled = false;
processFrameQueue();
};
self.onmessage = (event: MessageEvent) => { self.onmessage = (event: MessageEvent) => {
const { type, data, canvas: offscreenCanvas } = event.data; const { type, data, canvas: offscreenCanvas } = event.data;
@@ -61,13 +69,13 @@ function initializeDecoder() {
const init = { const init = {
output: (frame: VideoFrame) => { output: (frame: VideoFrame) => {
frameQueue.enqueue(frame); frameQueue.enqueue(frame);
if (frameQueue.size >= 10) { while (frameQueue.size > maxQueuedFrames) {
frameQueue.dequeue()?.close(); frameQueue.dequeue()?.close();
} }
if (!rendering) { if (!rendering) {
rendering = true; rendering = true;
processFrameQueue(); scheduleFrameQueue();
} }
}, },
error: () => { error: () => {
@@ -110,12 +118,21 @@ function processFrameQueue() {
} }
if (frameQueue.size > 0) { if (frameQueue.size > 0) {
setTimeout(processFrameQueue, 0); scheduleFrameQueue();
} else { } else {
rendering = false; rendering = false;
} }
} }
function scheduleFrameQueue() {
if (flushScheduled) {
return;
}
flushScheduled = true;
frameChannel.port2.postMessage(null);
}
function renderFrame(frame: VideoFrame) { function renderFrame(frame: VideoFrame) {
if (!canvas || !ctx) { if (!canvas || !ctx) {
frame.close(); frame.close();
@@ -142,6 +159,7 @@ function resetDecoder() {
decoder = null; decoder = null;
rendering = false; rendering = false;
flushScheduled = false;
Array.from(frameQueue.drain()).forEach((frame) => frame.close()); Array.from(frameQueue.drain()).forEach((frame) => frame.close());
} }

View File

@@ -8,6 +8,19 @@ import { getBaseUrl } from '@/lib/service.ts';
import { mouseStyleAtom } from '@/jotai/mouse.ts'; import { mouseStyleAtom } from '@/jotai/mouse.ts';
import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts'; import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts';
type SignalingMessage = {
event?: string;
data?: string;
};
const parseSignalingData = <T,>(data?: string): T | null => {
if (!data) {
return null;
}
return JSON.parse(data) as T;
};
export const H264Webrtc = () => { export const H264Webrtc = () => {
const resolution = useAtomValue(resolutionAtom); const resolution = useAtomValue(resolutionAtom);
const mouseStyle = useAtomValue(mouseStyleAtom); const mouseStyle = useAtomValue(mouseStyleAtom);
@@ -21,120 +34,11 @@ export const H264Webrtc = () => {
useEffect(() => { useEffect(() => {
const url = `${getBaseUrl('ws')}/api/stream/h264`; const url = `${getBaseUrl('ws')}/api/stream/h264`;
const ws = new W3cWebSocket(url); const ws = new W3cWebSocket(url);
const videoElement = videoRef.current;
const iceServers = [{ urls: ['stun:stun.l.google.com:19302'] }]; let video: RTCPeerConnection | null = null;
const video = new RTCPeerConnection({ iceServers }); let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
let disposed = false;
// --- Init Video ---
video.onnegotiationneeded = async () => {
if (videoOfferSent.current || video.signalingState !== 'stable') {
console.log('Skipping video negotiation - Waiting for answer or state unstable');
return;
}
try {
videoOfferSent.current = true;
const offer = await video.createOffer({
offerToReceiveVideo: true,
offerToReceiveAudio: false
});
await video.setLocalDescription(offer);
sendMsg('video-offer', JSON.stringify(video.localDescription));
} catch (error) {
videoOfferSent.current = false;
console.error('Video negotiation failed:', error);
}
};
video.onconnectionstatechange = () => {
if (video.iceConnectionState === 'connected') {
setIsLoading(false);
}
};
video.ontrack = (event) => {
if (videoRef.current && event.track.kind === 'video') {
videoRef.current.srcObject = new MediaStream([event.track]);
}
};
ws.onopen = () => {
videoOfferSent.current = false;
video.onicecandidate = (event) => {
if (event.candidate) {
sendMsg('video-candidate', JSON.stringify(event.candidate));
}
};
video.addTransceiver('video', { direction: 'recvonly' });
};
ws.onmessage = (event) => {
try {
const msg = JSON.parse(event.data as string);
if (!msg?.data) return;
const data = JSON.parse(msg.data);
if (!data) return;
switch (msg.event) {
case 'video-answer':
handleVideoAnswer(data);
break;
case 'video-candidate':
handleVideoCandidate(data);
break;
case 'heartbeat':
break;
default:
console.log('Unhandled event:', msg.event);
}
} catch (err) {
console.error('Message processing error:', err);
}
};
const handleVideoAnswer = (data: any) => {
if (video.signalingState !== 'have-local-offer') {
videoOfferSent.current = false;
console.warn(`Video signaling state incorrect for answer: ${video.signalingState}`);
return;
}
video
.setRemoteDescription(new RTCSessionDescription(data))
.then(() => {
videoOfferSent.current = false;
videoIceCandidates.current.forEach((candidate) => {
video
.addIceCandidate(candidate)
.catch((e) => console.error('Video candidate failed to add:', e.message));
});
videoIceCandidates.current = [];
})
.catch((error) => {
console.error('Video answer set failed:', error);
videoOfferSent.current = false;
});
};
const handleVideoCandidate = (data: any) => {
if (!data.candidate) {
return;
}
const candidate = new RTCIceCandidate(data);
if (video.remoteDescription) {
video
.addIceCandidate(candidate)
.catch((e) => console.error('Video candidate failed to add:', e.message));
} else {
videoIceCandidates.current.push(candidate);
}
};
const sendMsg = (event: string, data: string) => { const sendMsg = (event: string, data: string) => {
if (ws.readyState !== WebSocket.OPEN) { if (ws.readyState !== WebSocket.OPEN) {
@@ -148,25 +52,173 @@ export const H264Webrtc = () => {
} }
}; };
const heartbeatTimer = setInterval(() => { const startVideo = (iceServers: RTCIceServer[]) => {
sendMsg('heartbeat', ''); if (video || disposed) {
}, 60 * 1000); return;
}
setTimeout(() => { const peer = new RTCPeerConnection({ iceServers });
video = peer;
videoOfferSent.current = false;
videoIceCandidates.current = [];
// --- Init Video ---
peer.onnegotiationneeded = async () => {
if (videoOfferSent.current || peer.signalingState !== 'stable') {
console.log('Skipping video negotiation - Waiting for answer or state unstable');
return;
}
try {
videoOfferSent.current = true;
const offer = await peer.createOffer({
offerToReceiveVideo: true,
offerToReceiveAudio: false
});
await peer.setLocalDescription(offer);
sendMsg('video-offer', JSON.stringify(peer.localDescription));
} catch (error) {
videoOfferSent.current = false;
console.error('Video negotiation failed:', error);
}
};
peer.onconnectionstatechange = () => {
if (peer.iceConnectionState === 'connected' || peer.connectionState === 'connected') {
setIsLoading(false);
}
};
peer.ontrack = (event) => {
if (videoElement && event.track.kind === 'video') {
videoElement.srcObject = new MediaStream([event.track]);
}
};
peer.onicecandidate = (event) => {
if (event.candidate) {
sendMsg('video-candidate', JSON.stringify(event.candidate));
}
};
peer.addTransceiver('video', { direction: 'recvonly' });
};
const handleVideoAnswer = (data: RTCSessionDescriptionInit) => {
const peer = video;
if (!peer) {
return;
}
if (peer.signalingState !== 'have-local-offer') {
videoOfferSent.current = false;
console.warn(`Video signaling state incorrect for answer: ${peer.signalingState}`);
return;
}
peer
.setRemoteDescription(new RTCSessionDescription(data))
.then(() => {
videoOfferSent.current = false;
videoIceCandidates.current.forEach((candidate) => {
peer
.addIceCandidate(candidate)
.catch((e) => console.error('Video candidate failed to add:', e.message));
});
videoIceCandidates.current = [];
})
.catch((error) => {
console.error('Video answer set failed:', error);
videoOfferSent.current = false;
});
};
const handleVideoCandidate = (data: RTCIceCandidateInit) => {
const peer = video;
if (!peer || !data.candidate) {
return;
}
const candidate = new RTCIceCandidate(data);
if (peer.remoteDescription) {
peer
.addIceCandidate(candidate)
.catch((e) => console.error('Video candidate failed to add:', e.message));
} else {
videoIceCandidates.current.push(candidate);
}
};
ws.onopen = () => {
if (disposed) {
ws.close();
return;
}
heartbeatTimer = setInterval(() => {
sendMsg('heartbeat', '');
}, 60 * 1000);
};
ws.onmessage = (event) => {
try {
const msg = JSON.parse(event.data as string) as SignalingMessage;
switch (msg.event) {
case 'ice-servers': {
const iceServers = parseSignalingData<RTCIceServer[]>(msg.data);
startVideo(Array.isArray(iceServers) ? iceServers : []);
break;
}
case 'video-answer': {
const data = parseSignalingData<RTCSessionDescriptionInit>(msg.data);
if (data) {
handleVideoAnswer(data);
}
break;
}
case 'video-candidate': {
const data = parseSignalingData<RTCIceCandidateInit>(msg.data);
if (data) {
handleVideoCandidate(data);
}
break;
}
case 'heartbeat':
break;
default:
console.log('Unhandled event:', msg.event);
}
} catch (err) {
console.error('Message processing error:', err);
}
};
const loadingTimer = setTimeout(() => {
setIsLoading(false); setIsLoading(false);
}, 5 * 1000); }, 5 * 1000);
return () => { return () => {
if (ws.readyState === WebSocket.OPEN) { disposed = true;
if (ws.readyState !== WebSocket.CLOSING && ws.readyState !== WebSocket.CLOSED) {
ws.close(); ws.close();
} }
video.close(); video?.close();
video = null;
if (videoElement) {
videoElement.srcObject = null;
}
videoOfferSent.current = false; videoOfferSent.current = false;
videoIceCandidates.current = [];
if (heartbeatTimer) { if (heartbeatTimer) {
clearInterval(heartbeatTimer); clearInterval(heartbeatTimer);
} }
clearTimeout(loadingTimer);
}; };
}, []); }, []);