From 9415afb3591ee7a83abc947601ba1f2aeb6c8f10 Mon Sep 17 00:00:00 2001 From: watermeko <61347352+watermeko@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:47:04 +0800 Subject: [PATCH] fix(mouse): improve touch interaction and screen fitting (#857) - Add touch gestures for relative mouse mode, including tap, drag, long press, double-tap drag, and two-finger scrolling. - Improve absolute mouse touch handling and prevent black letterbox areas from generating remote mouse events. - Fit H.264 and MJPEG screens with contain scaling and disable browser touch handling on the screen elements. - Move golang.org/x/sys to the direct dependency list. --- web/src/pages/desktop/mouse/absolute.tsx | 311 +++++++++++++------ web/src/pages/desktop/mouse/relative.tsx | 285 ++++++++++++++++- web/src/pages/desktop/screen/h264-direct.tsx | 11 +- web/src/pages/desktop/screen/h264-webrtc.tsx | 11 +- web/src/pages/desktop/screen/mjpeg.tsx | 8 +- 5 files changed, 503 insertions(+), 123 deletions(-) diff --git a/web/src/pages/desktop/mouse/absolute.tsx b/web/src/pages/desktop/mouse/absolute.tsx index e369c6a..30aa893 100644 --- a/web/src/pages/desktop/mouse/absolute.tsx +++ b/web/src/pages/desktop/mouse/absolute.tsx @@ -1,6 +1,5 @@ import { useEffect, useRef } from 'react'; import { useAtomValue } from 'jotai'; -import { useMediaQuery } from 'react-responsive'; import { MouseReportAbsolute } from '@/lib/mouse.ts'; import { client, MessageEvent } from '@/lib/websocket.ts'; @@ -18,8 +17,6 @@ enum MouseButton { } export const Absolute = () => { - const isBigScreen = useMediaQuery({ minWidth: 650 }); - const resolution = useAtomValue(resolutionAtom); const scrollDirection = useAtomValue(scrollDirectionAtom); const scrollInterval = useAtomValue(scrollIntervalAtom); @@ -29,24 +26,32 @@ export const Absolute = () => { const lastScrollTimeRef = useRef(0); // For touch events - const touchStartTimeRef = useRef(0); const lastTouchYRef = useRef(0); const longPressTimerRef = useRef | null>(null); const isLongPressRef = useRef(false); const hasMoveRef = useRef(false); const isDraggingRef = useRef(false); + const isMultiTouchRef = useRef(false); + const isTouchActiveRef = useRef(false); const pressedButtonRef = useRef(null); const touchStartPosRef = useRef({ x: 0, y: 0 }); + const lastTapTimeRef = useRef(0); + const lastTapPosRef = useRef({ x: 0, y: 0 }); + const isDoubleTapCandidateRef = useRef(false); + const doubleTapDragArmedUntilRef = useRef(0); const TAP_THRESHOLD = 8; const DRAG_THRESHOLD = 10; - const VELOCITY_THRESHOLD = 0.3; + const DOUBLE_TAP_DELAY = 400; + const DOUBLE_TAP_DRAG_WINDOW = 800; + const DOUBLE_TAP_DISTANCE = 24; useEffect(() => { - const screen = document.getElementById('screen') as HTMLVideoElement | null; + const screen = document.getElementById('screen'); if (!screen) return; const target = screen; const mouse = mouseRef.current; + const pressedMouseButtons = new Set(); let pendingMove: { x: number; y: number } | null = null; let moveFrame: number | null = null; @@ -57,12 +62,11 @@ export const Absolute = () => { target.addEventListener('click', disableEvent); target.addEventListener('contextmenu', disableEvent); - if (isBigScreen) { - target.addEventListener('touchstart', handleTouchStart); - target.addEventListener('touchmove', handleTouchMove); - target.addEventListener('touchend', handleTouchEnd); - target.addEventListener('touchcancel', handleTouchCancel); - } + const touchOptions: AddEventListenerOptions = { passive: false }; + target.addEventListener('touchstart', handleTouchStart, touchOptions); + target.addEventListener('touchmove', handleTouchMove, touchOptions); + target.addEventListener('touchend', handleTouchEnd, touchOptions); + target.addEventListener('touchcancel', handleTouchCancel, touchOptions); // Mouse event handler function handleMouseEvent(event: MouseAbsoluteEvent) { @@ -100,21 +104,36 @@ export const Absolute = () => { // Mouse down event function handleMouseDown(e: MouseEvent) { disableEvent(e); + if (!getCorrectedCoords(e.clientX, e.clientY)) { + return; + } + flushMouseMove(); + pressedMouseButtons.add(e.button); handleMouseEvent({ type: 'mousedown', button: e.button }); } // Mouse up event function handleMouseUp(e: MouseEvent) { disableEvent(e); + if (!getCorrectedCoords(e.clientX, e.clientY) && !pressedMouseButtons.has(e.button)) { + return; + } + flushMouseMove(); + pressedMouseButtons.delete(e.button); handleMouseEvent({ type: 'mouseup', button: e.button }); } // Mouse move event function handleMouseMove(e: MouseEvent) { disableEvent(e); - const { x, y } = getCoordinate(e); + const coordinate = getCoordinate(e); + if (!coordinate) { + return; + } + + const { x, y } = coordinate; queueMouseMove(x, y); } @@ -122,7 +141,7 @@ export const Absolute = () => { function handleWheel(e: WheelEvent) { disableEvent(e); - if (Math.floor(e.deltaY) === 0) { + if (Math.floor(e.deltaY) === 0 || !getCorrectedCoords(e.clientX, e.clientY)) { return; } @@ -147,36 +166,68 @@ export const Absolute = () => { const touch = e.touches[0]; + if (!getCorrectedCoords(touch.clientX, touch.clientY)) { + isTouchActiveRef.current = false; + isMultiTouchRef.current = e.touches.length > 1; + hasMoveRef.current = true; + clearLongPressTimer(); + releasePressedButton(); + isDoubleTapCandidateRef.current = false; + doubleTapDragArmedUntilRef.current = 0; + lastTouchYRef.current = touch.clientY; + return; + } + + isTouchActiveRef.current = true; + + if (e.touches.length > 1) { + isMultiTouchRef.current = true; + hasMoveRef.current = true; + clearLongPressTimer(); + releasePressedButton(); + isDoubleTapCandidateRef.current = false; + lastTouchYRef.current = touch.clientY; + return; + } + + const currentTime = Date.now(); + const tapDistance = Math.hypot( + touch.clientX - lastTapPosRef.current.x, + touch.clientY - lastTapPosRef.current.y + ); + isDoubleTapCandidateRef.current = + (currentTime - lastTapTimeRef.current <= DOUBLE_TAP_DELAY && + tapDistance <= DOUBLE_TAP_DISTANCE) || + (currentTime <= doubleTapDragArmedUntilRef.current && tapDistance <= DOUBLE_TAP_DISTANCE); + // Reset states - touchStartTimeRef.current = Date.now(); lastTouchYRef.current = touch.clientY; isLongPressRef.current = false; hasMoveRef.current = false; isDraggingRef.current = false; - pressedButtonRef.current = null; + isMultiTouchRef.current = false; + releasePressedButton(); touchStartPosRef.current = { x: touch.clientX, y: touch.clientY }; - if (longPressTimerRef.current) { - clearTimeout(longPressTimerRef.current); - } + clearLongPressTimer(); - const { x, y } = getCoordinate(touch); - handleMouseEvent({ type: 'move', x, y }); - - if (e.touches.length > 1) { - return; + const coordinate = getCoordinate(touch); + if (coordinate) { + handleMouseEvent({ type: 'move', x: coordinate.x, y: coordinate.y }); } // Start long press - longPressTimerRef.current = setTimeout(() => { - isLongPressRef.current = true; - pressedButtonRef.current = MouseButton.Right; - if (navigator.vibrate) { - navigator.vibrate(50); - } + if (!isDoubleTapCandidateRef.current) { + longPressTimerRef.current = setTimeout(() => { + isLongPressRef.current = true; + pressedButtonRef.current = MouseButton.Right; + if (navigator.vibrate) { + navigator.vibrate(50); + } - handleMouseEvent({ type: 'mousedown', button: MouseButton.Right }); - }, 800); + handleMouseEvent({ type: 'mousedown', button: MouseButton.Right }); + }, 800); + } } // Mouse touch move event @@ -186,10 +237,27 @@ export const Absolute = () => { if (e.touches.length === 0) { return; } + if (!isTouchActiveRef.current) { + return; + } const touch = e.touches[0]; + const coordinate = getCoordinate(touch); + + if (!coordinate) { + hasMoveRef.current = true; + clearLongPressTimer(); + releasePressedButton(); + isDoubleTapCandidateRef.current = false; + return; + } // Handle two-finger scroll first if (e.touches.length > 1) { + isMultiTouchRef.current = true; + hasMoveRef.current = true; + clearLongPressTimer(); + releasePressedButton(); + const currentTime = Date.now(); if (currentTime - lastScrollTimeRef.current < scrollInterval) { return; @@ -203,40 +271,35 @@ export const Absolute = () => { return; } - const deltaX = Math.abs(touch.clientX - touchStartPosRef.current.x); - const deltaY = Math.abs(touch.clientY - touchStartPosRef.current.y); - const distance = Math.sqrt(deltaX * deltaX + deltaY * deltaY); - - const timeDelta = Date.now() - touchStartTimeRef.current; - const velocity = timeDelta > 0 ? distance / timeDelta : 0; - - const shouldStartDrag = - distance > DRAG_THRESHOLD || (distance > TAP_THRESHOLD && velocity > VELOCITY_THRESHOLD); - - if (shouldStartDrag && !isDraggingRef.current && !isLongPressRef.current) { - if (!hasMoveRef.current) { - hasMoveRef.current = true; - } - - if (longPressTimerRef.current) { - clearTimeout(longPressTimerRef.current); - longPressTimerRef.current = null; - } - - if (pressedButtonRef.current === null) { - isDraggingRef.current = true; - pressedButtonRef.current = MouseButton.Left; - handleMouseEvent({ type: 'mousedown', button: MouseButton.Left }); - } + if (isMultiTouchRef.current) { + return; } - if (distance > TAP_THRESHOLD && !hasMoveRef.current) { + const deltaX = touch.clientX - touchStartPosRef.current.x; + const deltaY = touch.clientY - touchStartPosRef.current.y; + const distance = Math.hypot(deltaX, deltaY); + + if (distance > TAP_THRESHOLD) { hasMoveRef.current = true; + clearLongPressTimer(); } - if (isDraggingRef.current || isLongPressRef.current) { - const { x, y } = getCoordinate(touch); - handleMouseEvent({ type: 'move', x, y }); + if ( + isDoubleTapCandidateRef.current && + distance > DRAG_THRESHOLD && + !isDraggingRef.current && + !isLongPressRef.current && + coordinate !== null + ) { + isDraggingRef.current = true; + pressedButtonRef.current = MouseButton.Left; + handleMouseEvent({ type: 'mousedown', button: MouseButton.Left }); + } + + if (!isDoubleTapCandidateRef.current || isDraggingRef.current || isLongPressRef.current) { + if (coordinate !== null) { + handleMouseEvent({ type: 'move', x: coordinate.x, y: coordinate.y }); + } } } @@ -244,50 +307,80 @@ export const Absolute = () => { function handleTouchEnd(e: TouchEvent) { disableEvent(e); - if (longPressTimerRef.current) { - clearTimeout(longPressTimerRef.current); - longPressTimerRef.current = null; + if (!isTouchActiveRef.current) { + if (e.touches.length === 0) { + isTouchActiveRef.current = false; + isMultiTouchRef.current = false; + } + return; } - if (!hasMoveRef.current && !isLongPressRef.current) { + if (e.touches.length > 0) { + isMultiTouchRef.current = true; + return; + } + + clearLongPressTimer(); + + const endTouch = e.changedTouches[0]; + const isTap = + !isMultiTouchRef.current && + !hasMoveRef.current && + !isLongPressRef.current && + !isDraggingRef.current && + !!endTouch && + !!getCorrectedCoords(endTouch.clientX, endTouch.clientY); + + if (isTap) { handleMouseEvent({ type: 'mousedown', button: MouseButton.Left }); - setTimeout(() => { - handleMouseEvent({ type: 'mouseup', button: MouseButton.Left }); - }, 50); + handleMouseEvent({ type: 'mouseup', button: MouseButton.Left }); + const currentTime = Date.now(); + lastTapTimeRef.current = currentTime; + lastTapPosRef.current = { x: endTouch.clientX, y: endTouch.clientY }; + if (isDoubleTapCandidateRef.current) { + doubleTapDragArmedUntilRef.current = currentTime + DOUBLE_TAP_DRAG_WINDOW; + } } else if (pressedButtonRef.current !== null) { flushMouseMove(); - handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! }); + releasePressedButton(); + doubleTapDragArmedUntilRef.current = 0; } isLongPressRef.current = false; hasMoveRef.current = false; isDraggingRef.current = false; - pressedButtonRef.current = null; + isMultiTouchRef.current = false; + isTouchActiveRef.current = false; + isDoubleTapCandidateRef.current = false; } // Mouse touch cancel event - function handleTouchCancel(e: any) { + function handleTouchCancel(e: TouchEvent) { disableEvent(e); - if (longPressTimerRef.current) { - clearTimeout(longPressTimerRef.current); - longPressTimerRef.current = null; - } + clearLongPressTimer(); - if (pressedButtonRef.current !== null) { - flushMouseMove(); - handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! }); - } + releasePressedButton(); isLongPressRef.current = false; hasMoveRef.current = false; isDraggingRef.current = false; + isMultiTouchRef.current = false; + isTouchActiveRef.current = false; pressedButtonRef.current = null; + isDoubleTapCandidateRef.current = false; + doubleTapDragArmedUntilRef.current = 0; + lastTapTimeRef.current = 0; } // get mouse coordinate - function getCoordinate(event: any) { - const { x, y } = getCorrectedCoords(event.clientX, event.clientY); + function getCoordinate(event: MouseEvent | Touch): { x: number; y: number } | null { + const correctedCoords = getCorrectedCoords(event.clientX, event.clientY); + if (!correctedCoords) { + return null; + } + + const { x, y } = correctedCoords; const finalX = Math.max(0, Math.min(1, x)); const finalY = Math.max(0, Math.min(1, y)); @@ -298,10 +391,16 @@ export const Absolute = () => { return { x: hexX, y: hexY }; } - function getCorrectedCoords(clientX: number, clientY: number) { + function getCorrectedCoords(clientX: number, clientY: number): { x: number; y: number } | null { const rect = target.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) { + return null; + } - const mediaSize = getMediaSize(target); + const mediaSize = + resolution && resolution.width > 0 && resolution.height > 0 + ? resolution + : getMediaSize(target); if (!mediaSize) { const x = (clientX - rect.left) / rect.width; const y = (clientY - rect.top) / rect.height; @@ -324,8 +423,19 @@ export const Absolute = () => { offsetX = (rect.width - renderedWidth) / 2; } - const x = (clientX - rect.left - offsetX) / renderedWidth; - const y = (clientY - rect.top - offsetY) / renderedHeight; + const contentLeft = rect.left + offsetX; + const contentTop = rect.top + offsetY; + if ( + clientX < contentLeft || + clientX > contentLeft + renderedWidth || + clientY < contentTop || + clientY > contentTop + renderedHeight + ) { + return null; + } + + const x = (clientX - contentLeft) / renderedWidth; + const y = (clientY - contentTop) / renderedHeight; return { x, y }; } @@ -356,6 +466,25 @@ export const Absolute = () => { handleMouseEvent({ type: 'move', x: move.x, y: move.y }); } + function clearLongPressTimer() { + if (longPressTimerRef.current) { + clearTimeout(longPressTimerRef.current); + longPressTimerRef.current = null; + } + } + + function releasePressedButton() { + if (pressedButtonRef.current === null) { + return; + } + + flushMouseMove(); + handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current }); + pressedButtonRef.current = null; + isDraggingRef.current = false; + isLongPressRef.current = false; + } + return () => { if (moveFrame !== null) { cancelAnimationFrame(moveFrame); @@ -367,19 +496,19 @@ export const Absolute = () => { target.removeEventListener('wheel', handleWheel); target.removeEventListener('click', disableEvent); target.removeEventListener('contextmenu', disableEvent); - target.removeEventListener('touchstart', handleTouchStart); - target.removeEventListener('touchmove', handleTouchMove); - target.removeEventListener('touchend', handleTouchEnd); - target.removeEventListener('touchcancel', handleTouchCancel); + target.removeEventListener('touchstart', handleTouchStart, touchOptions.capture); + target.removeEventListener('touchmove', handleTouchMove, touchOptions.capture); + target.removeEventListener('touchend', handleTouchEnd, touchOptions.capture); + target.removeEventListener('touchcancel', handleTouchCancel, touchOptions.capture); if (longPressTimerRef.current) { clearTimeout(longPressTimerRef.current); } }; - }, [isBigScreen, resolution, scrollDirection, scrollInterval]); + }, [resolution, scrollDirection, scrollInterval]); // disable default events - function disableEvent(event: any) { + function disableEvent(event: Event) { event.preventDefault(); event.stopPropagation(); } diff --git a/web/src/pages/desktop/mouse/relative.tsx b/web/src/pages/desktop/mouse/relative.tsx index 4c2c611..8bad9bd 100644 --- a/web/src/pages/desktop/mouse/relative.tsx +++ b/web/src/pages/desktop/mouse/relative.tsx @@ -21,28 +21,65 @@ export const Relative = () => { const mouseRef = useRef(new MouseReportRelative()); const isLockedRef = useRef(false); const lastScrollTimeRef = useRef(0); + const lastTouchPosRef = useRef({ x: 0, y: 0 }); + const touchStartPosRef = useRef({ x: 0, y: 0 }); + const touchIdentifierRef = useRef(null); + const pendingTouchDeltaRef = useRef({ x: 0, y: 0 }); + const touchLongPressTimerRef = useRef | null>(null); + const isTouchLongPressRef = useRef(false); + const hasTouchMoveRef = useRef(false); + const isMultiTouchRef = useRef(false); + const pressedTouchButtonRef = useRef(null); + const lastTouchEndTimeRef = useRef(0); + const lastTapTimeRef = useRef(0); + const lastTapPosRef = useRef({ x: 0, y: 0 }); + const isDoubleTapCandidateRef = useRef(false); + const doubleTapDragArmedUntilRef = useRef(0); + + const TAP_THRESHOLD = 8; + const DRAG_THRESHOLD = 10; + const LONG_PRESS_DELAY = 800; + const DOUBLE_TAP_DELAY = 400; + const DOUBLE_TAP_DRAG_WINDOW = 800; + const DOUBLE_TAP_DISTANCE = 24; useEffect(() => { const screen = document.getElementById('screen'); if (!screen) return; + const target = screen; const mouse = mouseRef.current; showMessage(); - screen.addEventListener('click', handleMouseClick); - screen.addEventListener('mousedown', handleMouseDown); - screen.addEventListener('mouseup', handleMouseUp); - screen.addEventListener('mousemove', handleMouseMove); - screen.addEventListener('wheel', handleMouseWheel, { passive: false }); - screen.addEventListener('contextmenu', disableEvent); + target.addEventListener('click', handleMouseClick); + target.addEventListener('mousedown', handleMouseDown); + target.addEventListener('mouseup', handleMouseUp); + target.addEventListener('mousemove', handleMouseMove); + target.addEventListener('wheel', handleMouseWheel, { passive: false }); + target.addEventListener('contextmenu', disableEvent); document.addEventListener('pointerlockchange', handlePointerLockChange); + const touchOptions: AddEventListenerOptions = { passive: false }; + target.addEventListener('touchstart', handleTouchStart, touchOptions); + target.addEventListener('touchmove', handleTouchMove, touchOptions); + target.addEventListener('touchend', handleTouchEnd, touchOptions); + target.addEventListener('touchcancel', handleTouchCancel, touchOptions); + // Mouse click event function handleMouseClick(event: MouseEvent) { disableEvent(event); - if (!isLockedRef.current) { - screen?.requestPointerLock(); + if (Date.now() - lastTouchEndTimeRef.current < 500) { + return; + } + + if (!isLockedRef.current && 'requestPointerLock' in target) { + const requestPointerLock = ( + target as HTMLElement & { + requestPointerLock?: () => Promise | void; + } + ).requestPointerLock; + requestPointerLock?.call(target)?.catch?.(() => undefined); } } @@ -72,6 +109,187 @@ export const Relative = () => { handleMouseEvent({ type: 'move', deltaX, deltaY }); } + // Touch start event + function handleTouchStart(e: TouchEvent) { + disableEvent(e); + + if (e.touches.length === 0) { + return; + } + + if (e.touches.length > 1) { + isMultiTouchRef.current = true; + hasTouchMoveRef.current = true; + clearTouchLongPressTimer(); + releasePressedTouchButton(); + isDoubleTapCandidateRef.current = false; + lastTouchPosRef.current = { + x: e.touches[0].clientX, + y: e.touches[0].clientY + }; + return; + } + + const touch = e.touches[0]; + const currentTime = Date.now(); + const tapDistance = Math.hypot( + touch.clientX - lastTapPosRef.current.x, + touch.clientY - lastTapPosRef.current.y + ); + isDoubleTapCandidateRef.current = + currentTime - lastTapTimeRef.current <= DOUBLE_TAP_DELAY && + tapDistance <= DOUBLE_TAP_DISTANCE; + if (currentTime <= doubleTapDragArmedUntilRef.current && tapDistance <= DOUBLE_TAP_DISTANCE) { + isDoubleTapCandidateRef.current = true; + } + if (!isDoubleTapCandidateRef.current) { + doubleTapDragArmedUntilRef.current = 0; + } + + touchIdentifierRef.current = touch.identifier; + lastTouchPosRef.current = { x: touch.clientX, y: touch.clientY }; + touchStartPosRef.current = { x: touch.clientX, y: touch.clientY }; + pendingTouchDeltaRef.current = { x: 0, y: 0 }; + isTouchLongPressRef.current = false; + hasTouchMoveRef.current = false; + isMultiTouchRef.current = e.touches.length > 1; + releasePressedTouchButton(); + pressedTouchButtonRef.current = null; + + clearTouchLongPressTimer(); + + if (!isDoubleTapCandidateRef.current) { + touchLongPressTimerRef.current = setTimeout(() => { + isTouchLongPressRef.current = true; + pressedTouchButtonRef.current = 2; + navigator.vibrate?.(50); + handleMouseEvent({ type: 'mousedown', button: 2 }); + }, LONG_PRESS_DELAY); + } + } + + // Touch move event + function handleTouchMove(e: TouchEvent) { + disableEvent(e); + + if (e.touches.length === 0) { + return; + } + + if (e.touches.length > 1) { + isMultiTouchRef.current = true; + hasTouchMoveRef.current = true; + pendingTouchDeltaRef.current = { x: 0, y: 0 }; + clearTouchLongPressTimer(); + releasePressedTouchButton(); + } + + const touch = + Array.from(e.touches).find((item) => item.identifier === touchIdentifierRef.current) ?? + e.touches[0]; + const deltaX = touch.clientX - lastTouchPosRef.current.x; + const deltaY = touch.clientY - lastTouchPosRef.current.y; + lastTouchPosRef.current = { x: touch.clientX, y: touch.clientY }; + + if (e.touches.length > 1) { + const currentTime = Date.now(); + if (currentTime - lastScrollTimeRef.current >= scrollInterval && deltaY !== 0) { + const wheel = (deltaY > 0 ? 1 : -1) * scrollDirection; + handleMouseEvent({ type: 'wheel', deltaY: wheel }); + lastScrollTimeRef.current = currentTime; + } + return; + } + + if (isMultiTouchRef.current) { + return; + } + + const totalDeltaX = touch.clientX - touchStartPosRef.current.x; + const totalDeltaY = touch.clientY - touchStartPosRef.current.y; + const distance = Math.hypot(totalDeltaX, totalDeltaY); + + pendingTouchDeltaRef.current.x += deltaX; + pendingTouchDeltaRef.current.y += deltaY; + + if (distance > TAP_THRESHOLD) { + hasTouchMoveRef.current = true; + clearTouchLongPressTimer(); + } + + if ( + isDoubleTapCandidateRef.current && + distance > DRAG_THRESHOLD && + pressedTouchButtonRef.current === null + ) { + pressedTouchButtonRef.current = 0; + handleMouseEvent({ type: 'mousedown', button: 0 }); + } + + if ( + (!isDoubleTapCandidateRef.current || + pressedTouchButtonRef.current !== null || + isTouchLongPressRef.current) && + (pendingTouchDeltaRef.current.x !== 0 || pendingTouchDeltaRef.current.y !== 0) + ) { + handleMouseEvent({ + type: 'move', + deltaX: scaleTouchMovement(pendingTouchDeltaRef.current.x), + deltaY: scaleTouchMovement(pendingTouchDeltaRef.current.y) + }); + pendingTouchDeltaRef.current = { x: 0, y: 0 }; + } + } + + // Touch end event + function handleTouchEnd(e: TouchEvent) { + disableEvent(e); + lastTouchEndTimeRef.current = Date.now(); + + if (e.touches.length > 0) { + isMultiTouchRef.current = true; + return; + } + + clearTouchLongPressTimer(); + + const isTap = + !isMultiTouchRef.current && + !hasTouchMoveRef.current && + !isTouchLongPressRef.current && + pressedTouchButtonRef.current === null; + + if (isTap) { + handleMouseEvent({ type: 'mousedown', button: 0 }); + handleMouseEvent({ type: 'mouseup', button: 0 }); + lastTapTimeRef.current = Date.now(); + lastTapPosRef.current = { + x: e.changedTouches[0].clientX, + y: e.changedTouches[0].clientY + }; + if (isDoubleTapCandidateRef.current) { + doubleTapDragArmedUntilRef.current = Date.now() + DOUBLE_TAP_DRAG_WINDOW; + } + } else if (pressedTouchButtonRef.current !== null) { + releasePressedTouchButton(); + doubleTapDragArmedUntilRef.current = 0; + } + + resetTouchState(); + isDoubleTapCandidateRef.current = false; + } + + // Touch cancel event + function handleTouchCancel(e: TouchEvent) { + disableEvent(e); + clearTouchLongPressTimer(); + + releasePressedTouchButton(); + + resetTouchState(); + isDoubleTapCandidateRef.current = false; + } + // Mouse wheel event function handleMouseWheel(e: WheelEvent) { disableEvent(e); @@ -91,19 +309,24 @@ export const Relative = () => { } function handlePointerLockChange() { - isLockedRef.current = document.pointerLockElement === screen; + isLockedRef.current = document.pointerLockElement === target; } return () => { const release = mouse.reset(); client.send(new Uint8Array([MessageEvent.Mouse, ...release])); - screen.removeEventListener('click', handleMouseClick); - screen.removeEventListener('mousemove', handleMouseMove); - screen.removeEventListener('mousedown', handleMouseDown); - screen.removeEventListener('mouseup', handleMouseUp); - screen.removeEventListener('wheel', handleMouseWheel); - screen.removeEventListener('contextmenu', disableEvent); + target.removeEventListener('click', handleMouseClick); + target.removeEventListener('mousemove', handleMouseMove); + target.removeEventListener('mousedown', handleMouseDown); + target.removeEventListener('mouseup', handleMouseUp); + target.removeEventListener('wheel', handleMouseWheel); + target.removeEventListener('contextmenu', disableEvent); document.removeEventListener('pointerlockchange', handlePointerLockChange); + target.removeEventListener('touchstart', handleTouchStart, touchOptions.capture); + target.removeEventListener('touchmove', handleTouchMove, touchOptions.capture); + target.removeEventListener('touchend', handleTouchEnd, touchOptions.capture); + target.removeEventListener('touchcancel', handleTouchCancel, touchOptions.capture); + clearTouchLongPressTimer(); }; }, [resolution, scrollDirection, scrollInterval]); @@ -150,10 +373,40 @@ export const Relative = () => { } // disable default events - function disableEvent(event: any) { + function disableEvent(event: Event) { event.preventDefault(); event.stopPropagation(); } + function scaleTouchMovement(value: number) { + return Math.abs(value * window.devicePixelRatio) < 10 ? value * 2 : value; + } + + function clearTouchLongPressTimer() { + if (touchLongPressTimerRef.current) { + clearTimeout(touchLongPressTimerRef.current); + touchLongPressTimerRef.current = null; + } + } + + function resetTouchState() { + touchIdentifierRef.current = null; + pendingTouchDeltaRef.current = { x: 0, y: 0 }; + isTouchLongPressRef.current = false; + hasTouchMoveRef.current = false; + isMultiTouchRef.current = false; + pressedTouchButtonRef.current = null; + } + + function releasePressedTouchButton() { + if (pressedTouchButtonRef.current === null) { + return; + } + + handleMouseEvent({ type: 'mouseup', button: pressedTouchButtonRef.current }); + pressedTouchButtonRef.current = null; + isTouchLongPressRef.current = false; + } + return <>{contextHolder}; }; diff --git a/web/src/pages/desktop/screen/h264-direct.tsx b/web/src/pages/desktop/screen/h264-direct.tsx index 09f1c77..acadd7a 100644 --- a/web/src/pages/desktop/screen/h264-direct.tsx +++ b/web/src/pages/desktop/screen/h264-direct.tsx @@ -5,12 +5,11 @@ import { useAtom, useAtomValue } from 'jotai'; import * as storage from '@/lib/localstorage.ts'; import { getBaseUrl } from '@/lib/service.ts'; import { mouseStyleAtom } from '@/jotai/mouse'; -import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts'; +import { videoScaleAtom } from '@/jotai/screen.ts'; import DirectWorker from './direct.worker.ts?worker'; export const H264Direct = () => { - const resolution = useAtomValue(resolutionAtom); const mouseStyle = useAtomValue(mouseStyleAtom); const [videoScale, setVideoScale] = useAtom(videoScaleAtom); @@ -51,13 +50,13 @@ export const H264Direct = () => { diff --git a/web/src/pages/desktop/screen/h264-webrtc.tsx b/web/src/pages/desktop/screen/h264-webrtc.tsx index 42bc9ac..a54ba79 100644 --- a/web/src/pages/desktop/screen/h264-webrtc.tsx +++ b/web/src/pages/desktop/screen/h264-webrtc.tsx @@ -7,7 +7,7 @@ import { w3cwebsocket as W3cWebSocket } from 'websocket'; import * as storage from '@/lib/localstorage.ts'; import { getBaseUrl } from '@/lib/service.ts'; import { mouseStyleAtom } from '@/jotai/mouse.ts'; -import { resolutionAtom, videoScaleAtom } from '@/jotai/screen.ts'; +import { videoScaleAtom } from '@/jotai/screen.ts'; type SignalingMessage = { event?: string; @@ -23,7 +23,6 @@ const parseSignalingData = (data?: string): T | null => { }; export const H264Webrtc = () => { - const resolution = useAtomValue(resolutionAtom); const mouseStyle = useAtomValue(mouseStyleAtom); const [videoScale, setVideoScale] = useAtom(videoScaleAtom); const [isLoading, setIsLoading] = useState(true); @@ -230,13 +229,13 @@ export const H264Webrtc = () => {