fix(web): correct mouse coordinates for direct h264 frames

web/src/pages/desktop/mouse/absolute.tsx

@@ -8,6 +8,25 @@ import { resolutionAtom } from '@/jotai/screen.ts';
 
 import { MouseAbsoluteEvent } from './types.ts';
 
+type MediaSize = {
+  width: number;
+  height: number;
+};
+
+type FrameContent = {
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+};
+
+const fullFrameContent = (mediaSize: MediaSize): FrameContent => ({
+  left: 0,
+  top: 0,
+  width: mediaSize.width,
+  height: mediaSize.height
+});
+
 enum MouseButton {
   Left = 0,
   Middle = 1,
@@ -52,6 +71,7 @@ export const Absolute = () => {
     const target = screen;
     const mouse = mouseRef.current;
     const pressedMouseButtons = new Set<number>();
+    let frameContentCache: { key: string; checkedAt: number; content: FrameContent } | null = null;
     let pendingMove: { x: number; y: number } | null = null;
     let moveFrame: number | null = null;
 
@@ -67,6 +87,9 @@ export const Absolute = () => {
     target.addEventListener('touchmove', handleTouchMove, touchOptions);
     target.addEventListener('touchend', handleTouchEnd, touchOptions);
     target.addEventListener('touchcancel', handleTouchCancel, touchOptions);
+    target.addEventListener('load', invalidateFrameContent);
+    target.addEventListener('loadedmetadata', invalidateFrameContent);
+    target.addEventListener('canplay', invalidateFrameContent);
 
     // Mouse event handler
     function handleMouseEvent(event: MouseAbsoluteEvent) {
@@ -423,23 +446,45 @@ export const Absolute = () => {
         offsetX = (rect.width - renderedWidth) / 2;
       }
 
-      const contentLeft = rect.left + offsetX;
-      const contentTop = rect.top + offsetY;
+      const frameContent = getFrameContent(mediaSize);
+      const frameScaleX = renderedWidth / mediaSize.width;
+      const frameScaleY = renderedHeight / mediaSize.height;
+      const contentLeft = rect.left + offsetX + frameContent.left * frameScaleX;
+      const contentTop = rect.top + offsetY + frameContent.top * frameScaleY;
+      const contentWidth = frameContent.width * frameScaleX;
+      const contentHeight = frameContent.height * frameScaleY;
+
       if (
         clientX < contentLeft ||
-        clientX > contentLeft + renderedWidth ||
+        clientX > contentLeft + contentWidth ||
         clientY < contentTop ||
-        clientY > contentTop + renderedHeight
+        clientY > contentTop + contentHeight
       ) {
         return null;
       }
 
-      const x = (clientX - contentLeft) / renderedWidth;
-      const y = (clientY - contentTop) / renderedHeight;
+      const x = (clientX - contentLeft) / contentWidth;
+      const y = (clientY - contentTop) / contentHeight;
 
       return { x, y };
     }
 
+    function invalidateFrameContent() {
+      frameContentCache = null;
+    }
+
+    function getFrameContent(mediaSize: MediaSize): FrameContent {
+      const key = `${mediaSize.width}x${mediaSize.height}`;
+      const now = performance.now();
+      if (frameContentCache?.key === key && now - frameContentCache.checkedAt < 1000) {
+        return frameContentCache.content;
+      }
+
+      const content = detectFrameContent(target, mediaSize);
+      frameContentCache = { key, checkedAt: now, content };
+      return content;
+    }
+
     function queueMouseMove(x: number, y: number) {
       pendingMove = { x, y };
       if (moveFrame !== null) {
@@ -500,6 +545,9 @@ export const Absolute = () => {
       target.removeEventListener('touchmove', handleTouchMove, touchOptions.capture);
       target.removeEventListener('touchend', handleTouchEnd, touchOptions.capture);
       target.removeEventListener('touchcancel', handleTouchCancel, touchOptions.capture);
+      target.removeEventListener('load', invalidateFrameContent);
+      target.removeEventListener('loadedmetadata', invalidateFrameContent);
+      target.removeEventListener('canplay', invalidateFrameContent);
 
       if (longPressTimerRef.current) {
         clearTimeout(longPressTimerRef.current);
@@ -516,7 +564,102 @@ export const Absolute = () => {
   return <></>;
 };
 
-function getMediaSize(screen: Element) {
+function detectFrameContent(screen: Element, mediaSize: MediaSize): FrameContent {
+  // Frame metadata includes letterbox pixels; infer stable side borders until the capture pipeline exposes an active rect.
+  // ponytail: replace pixel inference with active-content metadata when available.
+  const sampleScale = Math.min(1, 640 / mediaSize.width, 360 / mediaSize.height);
+  const sampleWidth = Math.max(1, Math.round(mediaSize.width * sampleScale));
+  const sampleHeight = Math.max(1, Math.round(mediaSize.height * sampleScale));
+  const canvas = document.createElement('canvas');
+  canvas.width = sampleWidth;
+  canvas.height = sampleHeight;
+
+  const context = canvas.getContext('2d', { willReadFrequently: true });
+  if (!context || !drawMediaFrame(context, screen, sampleWidth, sampleHeight)) {
+    return fullFrameContent(mediaSize);
+  }
+
+  try {
+    const pixels = context.getImageData(0, 0, sampleWidth, sampleHeight).data;
+    const left = findBorderInset(pixels, sampleWidth, sampleHeight, true);
+    const right = findBorderInset(pixels, sampleWidth, sampleHeight, false);
+    const minHorizontalBorder = Math.max(4, Math.round(sampleWidth * 0.02));
+    const horizontalBorder =
+      left >= minHorizontalBorder &&
+      right >= minHorizontalBorder &&
+      left + right < sampleWidth * 0.4 &&
+      Math.abs(left - right) <= Math.max(4, Math.round(sampleWidth * 0.05));
+    const frameLeft = horizontalBorder ? (left / sampleWidth) * mediaSize.width : 0;
+    const frameRight = horizontalBorder ? (right / sampleWidth) * mediaSize.width : 0;
+
+    return {
+      left: frameLeft,
+      top: 0,
+      width: mediaSize.width - frameLeft - frameRight,
+      height: mediaSize.height
+    };
+  } catch {
+    return fullFrameContent(mediaSize);
+  }
+}
+
+function drawMediaFrame(
+  context: CanvasRenderingContext2D,
+  screen: Element,
+  width: number,
+  height: number
+) {
+  if (
+    screen instanceof HTMLVideoElement ||
+    screen instanceof HTMLImageElement ||
+    screen instanceof HTMLCanvasElement
+  ) {
+    try {
+      context.drawImage(screen, 0, 0, width, height);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  return false;
+}
+
+function findBorderInset(
+  pixels: Uint8ClampedArray,
+  width: number,
+  height: number,
+  fromStart: boolean
+) {
+  const lines = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9].map((ratio) =>
+    Math.round((height - 1) * ratio)
+  );
+  const limit = Math.floor(width * 0.45);
+  let inset = 0;
+
+  for (let offset = 0; offset < limit; offset++) {
+    const position = fromStart ? offset : width - offset - 1;
+    const samples = lines.map((line) => getPixel(pixels, width, position, line));
+    if (!samples.every(isBlackPixel)) {
+      break;
+    }
+
+    inset = offset + 1;
+  }
+
+  return inset;
+}
+
+function getPixel(pixels: Uint8ClampedArray, width: number, x: number, y: number) {
+  const offset = (y * width + x) * 4;
+  return [pixels[offset], pixels[offset + 1], pixels[offset + 2]];
+}
+
+function isBlackPixel(pixel: number[]) {
+  return pixel[0] === 0 && pixel[1] === 0 && pixel[2] === 0;
+}
+
+function getMediaSize(screen: Element): MediaSize | null {
   if (screen instanceof HTMLVideoElement && screen.videoWidth > 0 && screen.videoHeight > 0) {
     return { width: screen.videoWidth, height: screen.videoHeight };
   }
@@ -525,8 +668,12 @@ function getMediaSize(screen: Element) {
     return { width: screen.naturalWidth, height: screen.naturalHeight };
   }
 
-  if (screen instanceof HTMLCanvasElement && screen.width > 0 && screen.height > 0) {
-    return { width: screen.width, height: screen.height };
+  if (screen instanceof HTMLCanvasElement) {
+    const width = Number(screen.dataset.mediaWidth);
+    const height = Number(screen.dataset.mediaHeight);
+    if (width > 0 && height > 0) {
+      return { width, height };
+    }
   }
 
   return null;

web/src/pages/desktop/screen/direct.worker.ts

@@ -13,6 +13,8 @@ let stopped = false;
 let resyncRequested = false;
 let pendingAckTimestamp: number | null = null;
 let decodeBackpressured = false;
+let reportedFrameWidth = 0;
+let reportedFrameHeight = 0;
 
 const maxQueuedFrames = 1;
 const maxReconnectDelayMs = 5_000;
@@ -45,6 +47,8 @@ self.onmessage = (event: MessageEvent<WorkerMessage>) => {
       }
 
       canvas = offscreenCanvas;
+      reportedFrameWidth = 0;
+      reportedFrameHeight = 0;
       ctx = canvas!.getContext('2d', {
         alpha: false,
         desynchronized: true
@@ -309,6 +313,15 @@ function renderFrame(frame: VideoFrame) {
       canvas.width = frame.displayWidth;
       canvas.height = frame.displayHeight;
     }
+    if (reportedFrameWidth !== frame.displayWidth || reportedFrameHeight !== frame.displayHeight) {
+      reportedFrameWidth = frame.displayWidth;
+      reportedFrameHeight = frame.displayHeight;
+      self.postMessage({
+        type: 'frame-size',
+        width: reportedFrameWidth,
+        height: reportedFrameHeight
+      });
+    }
 
     ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
   } finally {

web/src/pages/desktop/screen/h264-direct.tsx

@@ -37,6 +37,17 @@ export const H264Direct = () => {
 
     const offscreen = canvasRef.current.transferControlToOffscreen();
     const url = `${getBaseUrl('ws')}/api/stream/h264/direct`;
+    worker.onmessage = (
+      event: MessageEvent<{ type?: string; width?: number; height?: number }>
+    ) => {
+      const { type, width, height } = event.data;
+      if (type !== 'frame-size' || !width || !height || !canvasRef.current) {
+        return;
+      }
+
+      canvasRef.current.dataset.mediaWidth = String(width);
+      canvasRef.current.dataset.mediaHeight = String(height);
+    };
     worker.postMessage({ type: 'h264', canvas: offscreen, url }, [offscreen]);
 
     return () => {
@@ -50,7 +61,7 @@ export const H264Direct = () => {
       <canvas
         id="screen"
         ref={canvasRef}
-        className={clsx('block select-none touch-none', mouseStyle)}
+        className={clsx('block touch-none select-none', mouseStyle)}
         style={{
           transform: `scale(${videoScale})`,
           transformOrigin: 'center',