perf: reduce latency in 60 Hz mode (#844)

- Add decode-driven flow control and bounded GOP-aware queues to prevent stale H.264 frames from accumulating in TCP and WebSocket buffers.
- Isolate slow clients with dedicated writer goroutines, write deadlines, and safer connection lifecycle handling.
- Move the Direct H.264 socket into the worker, simplify the low-latency decode path, and improve reconnect behavior.

web/src/lib/websocket.ts

@@ -22,7 +22,7 @@ const DEFAULT_OPTIONS: Required<WsClientOptions> = {
   url: `${getBaseUrl('ws')}/api/ws`,
   heartbeatInterval: 10 * 1000,
   reconnectInterval: 3 * 1000,
-  maxReconnectAttempts: 1
+  maxReconnectAttempts: Number.POSITIVE_INFINITY
 };
 
 export class WsClient {

web/src/pages/desktop/screen/direct.worker.ts

@@ -5,38 +5,147 @@ let ctx: OffscreenCanvasRenderingContext2D | null = null;
 let rendering: boolean = false;
 let flushScheduled: boolean = false;
 let decoder: VideoDecoder | null = null;
-let displayDecoder: VideoDecoder | null = null;
-let replacementDecoder: VideoDecoder | null = null;
-let resyncAtNextKeyFrame: boolean = false;
+let streamUrl: string | null = null;
+let socket: WebSocket | null = null;
+let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
+let reconnectDelayMs = 250;
+let stopped = false;
+let resyncRequested = false;
+let pendingAckTimestamp: number | null = null;
+let decodeBackpressured = false;
 
 const maxQueuedFrames = 1;
-const maxDecoderQueueSize = 6;
+const maxReconnectDelayMs = 5_000;
+const frameAckMessage = 2;
+const streamResyncMessage = 3;
+const flowControlWindow = 8;
+const decoderHighWatermark = 6;
+const decoderLowWatermark = 3;
 const frameQueue = new Queue<VideoFrame>();
 const frameChannel = new MessageChannel();
 
+type WorkerMessage = {
+  type: 'h264' | 'stop';
+  canvas?: OffscreenCanvas;
+  url?: string;
+};
+
 frameChannel.port1.onmessage = () => {
   flushScheduled = false;
   processFrameQueue();
 };
 
-self.onmessage = (event: MessageEvent) => {
-  const { type, data, canvas: offscreenCanvas } = event.data;
+self.onmessage = (event: MessageEvent<WorkerMessage>) => {
+  const { type, canvas: offscreenCanvas, url } = event.data;
 
   switch (type) {
     case 'h264':
+      if (!offscreenCanvas || !url) {
+        return;
+      }
+
       canvas = offscreenCanvas;
-      ctx = canvas!.getContext('2d') as OffscreenCanvasRenderingContext2D;
+      ctx = canvas!.getContext('2d', {
+        alpha: false,
+        desynchronized: true
+      }) as OffscreenCanvasRenderingContext2D;
+      streamUrl = url;
+      stopped = false;
+      connect();
       break;
-    case 'ws_message':
-      handleWsMessage(data);
-      break;
-    case 'error':
-    case 'close':
+    case 'stop':
+      stopped = true;
+      clearReconnectTimer();
+      disconnect();
       resetDecoder();
       break;
   }
 };
 
+function connect() {
+  if (stopped || !streamUrl || socket) {
+    return;
+  }
+
+  try {
+    const url = new URL(streamUrl);
+    url.searchParams.set('flow', String(flowControlWindow));
+    const nextSocket = new WebSocket(url);
+    nextSocket.binaryType = 'arraybuffer';
+    socket = nextSocket;
+
+    nextSocket.onopen = () => {
+      if (socket !== nextSocket || stopped) {
+        return;
+      }
+
+      reconnectDelayMs = 250;
+      resyncRequested = false;
+      pendingAckTimestamp = null;
+      decodeBackpressured = false;
+    };
+
+    nextSocket.onmessage = (event) => {
+      if (socket !== nextSocket || stopped || !(event.data instanceof ArrayBuffer)) {
+        return;
+      }
+
+      handleWsMessage(event.data);
+    };
+
+    nextSocket.onerror = () => {
+      if (socket === nextSocket) {
+        nextSocket.close();
+      }
+    };
+
+    nextSocket.onclose = () => {
+      if (socket !== nextSocket) {
+        return;
+      }
+
+      socket = null;
+      resyncRequested = false;
+      pendingAckTimestamp = null;
+      decodeBackpressured = false;
+      resetDecoder();
+      scheduleReconnect();
+    };
+  } catch (error) {
+    console.error('Failed to create Direct H264 WebSocket:', error);
+    scheduleReconnect();
+  }
+}
+
+function disconnect() {
+  const currentSocket = socket;
+  socket = null;
+
+  if (currentSocket && currentSocket.readyState !== WebSocket.CLOSED) {
+    currentSocket.close();
+  }
+}
+
+function scheduleReconnect() {
+  if (stopped || reconnectTimer !== null || !streamUrl) {
+    return;
+  }
+
+  const delay = reconnectDelayMs;
+  reconnectDelayMs = Math.min(reconnectDelayMs * 2, maxReconnectDelayMs);
+  reconnectTimer = setTimeout(() => {
+    reconnectTimer = null;
+    connect();
+  }, delay);
+}
+
+function clearReconnectTimer() {
+  if (reconnectTimer !== null) {
+    clearTimeout(reconnectTimer);
+    reconnectTimer = null;
+  }
+}
+
 function handleWsMessage(message: ArrayBuffer) {
   try {
     if (message.byteLength < 9) {
@@ -48,33 +157,21 @@ function handleWsMessage(message: ArrayBuffer) {
     const timestamp = Number(view.getBigUint64(1, true));
     const data = new Uint8Array(message, 9);
 
-    if (
-      !replacementDecoder &&
-      decoder?.state === 'configured' &&
-      decoder.decodeQueueSize >= maxDecoderQueueSize
-    ) {
-      resyncAtNextKeyFrame = true;
-    }
-
-    if (resyncAtNextKeyFrame && isKeyFrame && displayDecoder) {
-      const replacement = createDecoder();
-      if (replacement) {
-        replacementDecoder = replacement;
-        decoder = replacement;
-        resyncAtNextKeyFrame = false;
-      }
-    }
-
     if (!decoder) {
       if (!isKeyFrame) {
+        requestStreamResync();
         return;
       }
+
+      resyncRequested = false;
       const initial = createDecoder();
       if (!initial) {
+        requestStreamResync();
         return;
       }
+
       decoder = initial;
-      displayDecoder = initial;
+      resyncRequested = false;
     }
 
     if (decoder?.state === 'configured') {
@@ -97,12 +194,19 @@ function createDecoder(): VideoDecoder | null {
       handleDecodedFrame(instance, frame);
     },
     error: () => {
-      resetDecoder();
+      if (decoder === instance) {
+        requestStreamResync();
+        resetDecoder();
+      }
     }
   };
 
   try {
     instance = new VideoDecoder(init);
+    const configuredDecoder = instance;
+    instance.ondequeue = () => {
+      releaseDecodeBackpressure(configuredDecoder);
+    };
     instance.configure({
       codec: 'avc1.42E02A',
       hardwareAcceleration: 'prefer-hardware',
@@ -110,6 +214,9 @@ function createDecoder(): VideoDecoder | null {
     });
     return instance;
   } catch (err) {
+    if (instance && instance.state !== 'closed') {
+      instance.close();
+    }
     console.log(err);
     return null;
   }
@@ -121,24 +228,17 @@ function handleDecodedFrame(source: VideoDecoder | null, frame: VideoFrame) {
     return;
   }
 
-  if (source === replacementDecoder) {
-    const previous = displayDecoder;
-    displayDecoder = source;
-    decoder = source;
-    replacementDecoder = null;
-
-    Array.from(frameQueue.drain()).forEach((queuedFrame) => queuedFrame.close());
-    if (previous && previous !== source && previous.state !== 'closed') {
-      previous.close();
-    }
-  } else if (source !== displayDecoder) {
+  if (source !== decoder) {
     frame.close();
     return;
   }
 
   frameQueue.enqueue(frame);
   while (frameQueue.size > maxQueuedFrames) {
-    frameQueue.dequeue()?.close();
+    const droppedFrame = frameQueue.dequeue();
+    if (droppedFrame) {
+      droppedFrame.close();
+    }
   }
 
   if (!rendering) {
@@ -156,8 +256,14 @@ function decode(target: VideoDecoder, isKeyFrame: boolean, timestamp: number, da
 
   try {
     target.decode(chunk);
+    pendingAckTimestamp = timestamp;
+    if (target.decodeQueueSize >= decoderHighWatermark) {
+      decodeBackpressured = true;
+    }
+    releaseDecodeBackpressure(target);
   } catch (err: any) {
     if (err.name === 'TypeError' || err.message.includes('configured')) {
+      requestStreamResync();
       resetDecoder();
     }
   }
@@ -166,7 +272,14 @@ function decode(target: VideoDecoder, isKeyFrame: boolean, timestamp: number, da
 function processFrameQueue() {
   const frame = frameQueue.dequeue();
   if (frame) {
-    renderFrame(frame);
+    try {
+      renderFrame(frame);
+    } catch (error) {
+      console.error('Failed to render Direct H264 frame:', error);
+      requestStreamResync();
+      resetDecoder();
+      return;
+    }
   }
 
   if (frameQueue.size > 0) {
@@ -191,33 +304,81 @@ function renderFrame(frame: VideoFrame) {
     return;
   }
 
-  if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
-    canvas.width = frame.displayWidth;
-    canvas.height = frame.displayHeight;
-  }
+  try {
+    if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+      canvas.width = frame.displayWidth;
+      canvas.height = frame.displayHeight;
+    }
 
-  ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
-  frame.close();
+    ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+  } finally {
+    frame.close();
+  }
 }
 
 function resetDecoder() {
-  const decoders = new Set([decoder, displayDecoder, replacementDecoder]);
-  decoders.forEach((item) => {
-    if (item && item.state !== 'closed') {
-      try {
-        item.close();
-      } catch (err) {
-        console.log(err);
-      }
+  if (decoder && decoder.state !== 'closed') {
+    try {
+      decoder.close();
+    } catch (err) {
+      console.log(err);
     }
-  });
+  }
 
   decoder = null;
-  displayDecoder = null;
-  replacementDecoder = null;
-  resyncAtNextKeyFrame = false;
+  pendingAckTimestamp = null;
+  decodeBackpressured = false;
   rendering = false;
   flushScheduled = false;
 
   Array.from(frameQueue.drain()).forEach((frame) => frame.close());
 }
+
+function releaseDecodeBackpressure(source: VideoDecoder) {
+  if (source !== decoder || pendingAckTimestamp === null) {
+    return;
+  }
+
+  if (decodeBackpressured && source.decodeQueueSize > decoderLowWatermark) {
+    return;
+  }
+
+  decodeBackpressured = false;
+  acknowledgeFrame(pendingAckTimestamp);
+  pendingAckTimestamp = null;
+}
+
+function acknowledgeFrame(timestamp: number) {
+  const currentSocket = socket;
+  if (!currentSocket || currentSocket.readyState !== WebSocket.OPEN) {
+    return;
+  }
+
+  const message = new ArrayBuffer(9);
+  const view = new DataView(message);
+  view.setUint8(0, frameAckMessage);
+  view.setBigUint64(1, BigInt(timestamp), true);
+  try {
+    currentSocket.send(message);
+  } catch {
+    currentSocket.close();
+  }
+}
+
+function requestStreamResync() {
+  if (resyncRequested) {
+    return;
+  }
+
+  const currentSocket = socket;
+  if (!currentSocket || currentSocket.readyState !== WebSocket.OPEN) {
+    return;
+  }
+
+  try {
+    currentSocket.send(new Uint8Array([streamResyncMessage]));
+    resyncRequested = true;
+  } catch {
+    currentSocket.close();
+  }
+}

web/src/pages/desktop/screen/h264-direct.tsx

@@ -1,7 +1,6 @@
 import { useEffect, useRef } from 'react';
 import clsx from 'clsx';
 import { useAtom, useAtomValue } from 'jotai';
-import { w3cwebsocket as W3cWebSocket } from 'websocket';
 
 import * as storage from '@/lib/localstorage.ts';
 import { getBaseUrl } from '@/lib/service.ts';
@@ -38,32 +37,11 @@ export const H264Direct = () => {
     workerRef.current = worker;
 
     const offscreen = canvasRef.current.transferControlToOffscreen();
-    worker.postMessage({ type: 'h264', canvas: offscreen }, [offscreen]);
-
     const url = `${getBaseUrl('ws')}/api/stream/h264/direct`;
-    const ws = new W3cWebSocket(url);
-    ws.binaryType = 'arraybuffer';
-
-    ws.onmessage = (event) => {
-      try {
-        worker.postMessage({ type: 'ws_message', data: event.data }, [event.data]);
-      } catch (error) {
-        console.error('Error processing WebSocket message:', error);
-      }
-    };
-
-    ws.onerror = () => {
-      worker.postMessage({ type: 'error' });
-    };
-
-    ws.onclose = () => {
-      worker.postMessage({ type: 'close' });
-    };
+    worker.postMessage({ type: 'h264', canvas: offscreen, url }, [offscreen]);
 
     return () => {
-      if (ws.readyState === 1) {
-        ws.close();
-      }
+      worker.postMessage({ type: 'stop' });
       worker.terminate();
     };
   }, []);