From 287cbb5dccdcf7efe7a0d84187011a24913e88c7 Mon Sep 17 00:00:00 2001 From: wj-xiao Date: Mon, 19 Jan 2026 16:47:10 +0800 Subject: [PATCH] refactor(desktop): refactor mouse module, add touchscreen support --- .../src/components/menu/mouse/direction.tsx | 4 +- desktop/src/common/ipc-events.ts | 3 +- desktop/src/main/device/index.ts | 33 +-- desktop/src/main/device/proto.ts | 6 +- desktop/src/main/device/utils.ts | 18 -- desktop/src/main/events/serial-port.ts | 31 +-- desktop/src/main/index.ts | 2 +- .../src/components/mouse/absolute.tsx | 252 +++++++++--------- .../src/components/mouse/relative.tsx | 194 ++++++++------ .../renderer/src/components/mouse/touchpad.ts | 204 ++++++++++++++ .../renderer/src/components/mouse/types.ts | 33 +++ desktop/src/renderer/src/jotai/mouse.ts | 2 +- .../renderer/src/libs/mouse-jiggler/index.ts | 26 +- desktop/src/renderer/src/libs/mouse/index.ts | 147 ++++++++-- 14 files changed, 632 insertions(+), 323 deletions(-) delete mode 100644 desktop/src/main/device/utils.ts create mode 100644 desktop/src/renderer/src/components/mouse/touchpad.ts create mode 100644 desktop/src/renderer/src/components/mouse/types.ts diff --git a/browser/src/components/menu/mouse/direction.tsx b/browser/src/components/menu/mouse/direction.tsx index add2f1c..4d66837 100644 --- a/browser/src/components/menu/mouse/direction.tsx +++ b/browser/src/components/menu/mouse/direction.tsx @@ -14,8 +14,8 @@ export const Direction = (): ReactElement => { const [scrollDirection, setScrollDirection] = useAtom(scrollDirectionAtom); const directions = [ - { name: t('mouse.scrollUp'), value: '1' }, - { name: t('mouse.scrollDown'), value: '-1' } + { name: t('mouse.scrollUp'), value: '-1' }, + { name: t('mouse.scrollDown'), value: '1' } ]; function update(direction: string): void { diff --git a/desktop/src/common/ipc-events.ts b/desktop/src/common/ipc-events.ts index 8b13a29..2ad4431 100644 --- a/desktop/src/common/ipc-events.ts +++ b/desktop/src/common/ipc-events.ts @@ -10,8 +10,7 @@ export enum IpcEvents { OPEN_SERIAL_PORT_RSP = 'open-serial-port-rsp', CLOSE_SERIAL_PORT = 'close-serial-port', SEND_KEYBOARD = 'send-keyboard', - SEND_MOUSE_RELATIVE = 'send-mouse-relative', - SEND_MOUSE_ABSOLUTE = 'send-mouse-absolute', + SEND_MOUSE = 'send-mouse', UPDATE_AVAILABLE = 'update-available', UPDATE_NOT_AVAILABLE = 'update-not-available', diff --git a/desktop/src/main/device/index.ts b/desktop/src/main/device/index.ts index 7313931..17255f4 100644 --- a/desktop/src/main/device/index.ts +++ b/desktop/src/main/device/index.ts @@ -1,6 +1,5 @@ import { CmdEvent, CmdPacket, InfoPacket } from './proto' import { SerialPort } from './serial-port' -import { intToByte, intToLittleEndianList } from './utils' export class Device { addr: number @@ -20,36 +19,16 @@ export class Device { return new InfoPacket(rspPacket.DATA) } - async sendKeyboardData(data: number[]): Promise { - const cmdData = new CmdPacket(this.addr, CmdEvent.SEND_KB_GENERAL_DATA, data).encode() + async sendKeyboardData(report: number[]): Promise { + const cmdData = new CmdPacket(this.addr, CmdEvent.SEND_KB_GENERAL_DATA, report).encode() await this.serialPort.write(cmdData) } - async sendMouseRelativeData(key: number, x: number, y: number, scroll: number): Promise { - const xByte = intToByte(x) - const yByte = intToByte(y) + async sendMouseData(report: number[]): Promise { + if (report.length === 0) return - const data = [0x01, key, xByte, yByte, scroll] - const cmdData = new CmdPacket(this.addr, CmdEvent.SEND_MS_REL_DATA, data).encode() - await this.serialPort.write(cmdData) - } - - async sendMouseAbsoluteData( - key: number, - width: number, - height: number, - x: number, - y: number, - scroll: number - ): Promise { - const xAbs = width === 0 ? 0 : Math.floor((x * 4096) / width) - const xLittle = intToLittleEndianList(xAbs) - - const yAbs = width === 0 ? 0 : Math.floor((y * 4096) / height) - const yLittle = intToLittleEndianList(yAbs) - - const data = [0x02, key, ...xLittle, ...yLittle, scroll] - const cmdData = new CmdPacket(this.addr, CmdEvent.SEND_MS_ABS_DATA, data).encode() + const cmdEvent = report[0] === 0x01 ? CmdEvent.SEND_MS_REL_DATA : CmdEvent.SEND_MS_ABS_DATA + const cmdData = new CmdPacket(this.addr, cmdEvent, report).encode() await this.serialPort.write(cmdData) } } diff --git a/desktop/src/main/device/proto.ts b/desktop/src/main/device/proto.ts index 8dc1884..63e82b4 100644 --- a/desktop/src/main/device/proto.ts +++ b/desktop/src/main/device/proto.ts @@ -1,5 +1,3 @@ -import { getBit } from './utils' - export enum CmdEvent { GET_INFO = 0x01, SEND_KB_GENERAL_DATA = 0x02, @@ -132,3 +130,7 @@ export class InfoPacket { this.SCROLL_LOCK = getBit(data[2], 2) === 1 } } + +function getBit(number: number, bitPosition: number): number { + return (number >> bitPosition) & 1 +} diff --git a/desktop/src/main/device/utils.ts b/desktop/src/main/device/utils.ts deleted file mode 100644 index 67d5e08..0000000 --- a/desktop/src/main/device/utils.ts +++ /dev/null @@ -1,18 +0,0 @@ -export function getBit(number: number, bitPosition: number): number { - return (number >> bitPosition) & 1 -} - -export function intToByte(value: number): number { - if (value < -128 || value > 127) { - throw new Error('value must be in range -128 to 127 for a signed byte') - } - return (value + 256) % 256 -} - -export function intToLittleEndianList(number: number): number[] { - const byteList: number[] = [] - for (let i = 0; i < 2; i++) { - byteList.push((number >> (i * 8)) & 0xff) - } - return byteList -} diff --git a/desktop/src/main/events/serial-port.ts b/desktop/src/main/events/serial-port.ts index aaa8c32..cf7246a 100644 --- a/desktop/src/main/events/serial-port.ts +++ b/desktop/src/main/events/serial-port.ts @@ -9,8 +9,7 @@ export function registerSerialPort(): void { ipcMain.handle(IpcEvents.OPEN_SERIAL_PORT, openSerialPort) ipcMain.handle(IpcEvents.CLOSE_SERIAL_PORT, closeSerialPort) ipcMain.handle(IpcEvents.SEND_KEYBOARD, sendKeyboard) - ipcMain.handle(IpcEvents.SEND_MOUSE_ABSOLUTE, sendMouseAbsolute) - ipcMain.handle(IpcEvents.SEND_MOUSE_RELATIVE, sendMouseRelative) + ipcMain.handle(IpcEvents.SEND_MOUSE, sendMouse) } async function getSerialPorts(): Promise { @@ -58,32 +57,10 @@ async function sendKeyboard(_: IpcMainInvokeEvent, report: number[]): Promise { +async function sendMouse(_: IpcMainInvokeEvent, report: number[]): Promise { try { - await device.sendMouseRelativeData(key, x, y, scroll) + await device.sendMouseData(report) } catch (error) { - console.error('Error sending mouse relative data:', error) - } -} - -async function sendMouseAbsolute( - _: IpcMainInvokeEvent, - key: number, - width: number, - height: number, - x: number, - y: number, - scroll: number -): Promise { - try { - await device.sendMouseAbsoluteData(key, width, height, x, y, scroll) - } catch (error) { - console.error('Error sending mouse absolute data:', error) + console.error('Error sending mouse data:', error) } } diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index 0aff80a..d3469ee 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -44,7 +44,7 @@ app.whenReady().then(() => { electronApp.setAppUserModelId('com.sipeed.usbkvm') session.defaultSession.setPermissionRequestHandler((_, permission, callback) => { - const allowedPermissions = ['media', 'clipboard-read'] + const allowedPermissions = ['media', 'clipboard-read', 'pointerLock'] callback(allowedPermissions.includes(permission)) }) diff --git a/desktop/src/renderer/src/components/mouse/absolute.tsx b/desktop/src/renderer/src/components/mouse/absolute.tsx index 1187c18..6039483 100644 --- a/desktop/src/renderer/src/components/mouse/absolute.tsx +++ b/desktop/src/renderer/src/components/mouse/absolute.tsx @@ -1,128 +1,43 @@ import { ReactElement, useEffect, useRef } from 'react' import { useAtomValue } from 'jotai' +import { useMediaQuery } from 'react-responsive' import { IpcEvents } from '@common/ipc-events' -import { resolutionAtom } from '@renderer/jotai/device' import { scrollDirectionAtom, scrollIntervalAtom } from '@renderer/jotai/mouse' -import { Key } from '@renderer/libs/mouse' +import { MouseAbsoluteRelative } from '@renderer/libs/mouse' import { mouseJiggler } from '@renderer/libs/mouse-jiggler' +import { createInitialTouchState, createTouchHandlers } from './touchpad' +import { MouseAbsoluteEvent } from './types' + export const Absolute = (): ReactElement => { - const resolution = useAtomValue(resolutionAtom) + const isBigScreen = useMediaQuery({ minWidth: 650 }) + const scrollDirection = useAtomValue(scrollDirectionAtom) const scrollInterval = useAtomValue(scrollIntervalAtom) - const keyRef = useRef(new Key()) + const mouseRef = useRef(new MouseAbsoluteRelative()) + const lastPosRef = useRef({ x: 0.5, y: 0.5 }) const lastScrollTimeRef = useRef(0) + const touchStateRef = useRef(createInitialTouchState()) useEffect(() => { - const canvas = document.getElementById('video') - if (!canvas) return + const screen = document.getElementById('video') as HTMLVideoElement + if (!screen) return - canvas.addEventListener('mousedown', handleMouseDown) - canvas.addEventListener('mouseup', handleMouseUp) - canvas.addEventListener('mousemove', handleMouseMove) - canvas.addEventListener('wheel', handleWheel) - canvas.addEventListener('click', disableEvent) - canvas.addEventListener('contextmenu', disableEvent) + function getCoordinate(event: { clientX: number; clientY: number }): { x: number; y: number } { + const rect = screen.getBoundingClientRect() - // press button - async function handleMouseDown(event: MouseEvent): Promise { - disableEvent(event) + const clientX = event.clientX + const clientY = event.clientY - switch (event.button) { - case 0: - keyRef.current.left = true - break - case 1: - keyRef.current.mid = true - break - case 2: - keyRef.current.right = true - break - default: - console.log(`unknown button ${event.button}`) - return - } - - await send(event) - } - - // release button - async function handleMouseUp(event: MouseEvent): Promise { - disableEvent(event) - - switch (event.button) { - case 0: - keyRef.current.left = false - break - case 1: - keyRef.current.mid = false - break - case 2: - keyRef.current.right = false - break - default: - console.log(`unknown button ${event.button}`) - return - } - - await send(event) - } - - // mouse move - async function handleMouseMove(event: MouseEvent): Promise { - disableEvent(event) - await send(event) - - mouseJiggler.moveEventCallback() - } - - // mouse scroll - async function handleWheel(event: WheelEvent): Promise { - disableEvent(event) - - const currentTime = Date.now() - if (currentTime - lastScrollTimeRef.current < scrollInterval) { - return - } - - const delta = Math.floor(event.deltaY) - if (delta === 0) return - - await send(event, delta > 0 ? -1 * scrollDirection : scrollDirection) - - lastScrollTimeRef.current = currentTime - } - - async function send(event: MouseEvent, scroll: number = 0): Promise { - const { x, y } = getCorrectedCoords(event.clientX, event.clientY) - await window.electron.ipcRenderer.invoke( - IpcEvents.SEND_MOUSE_ABSOLUTE, - keyRef.current.encode(), - 1, - 1, - x, - y, - scroll - ) - } - - function getCorrectedCoords(clientX: number, clientY: number) { - if (!canvas) { - return { x: 0, y: 0 } - } - - const rect = canvas.getBoundingClientRect() - const videoElement = canvas as HTMLVideoElement - - if (!videoElement.videoWidth || !videoElement.videoHeight) { + if (!screen.videoWidth || !screen.videoHeight) { const x = (clientX - rect.left) / rect.width const y = (clientY - rect.top) / rect.height return { x, y } } - const videoRatio = videoElement.videoWidth / videoElement.videoHeight + const videoRatio = screen.videoWidth / screen.videoHeight const elementRatio = rect.width / rect.height let renderedWidth = rect.width @@ -140,27 +55,124 @@ export const Absolute = (): ReactElement => { const x = (clientX - rect.left - offsetX) / renderedWidth const y = (clientY - rect.top - offsetY) / renderedHeight - - const finalX = Math.max(0, Math.min(1, x)) - const finalY = Math.max(0, Math.min(1, y)) - - return { x: finalX, y: finalY } + return { x, y } } - return (): void => { - canvas.removeEventListener('mousemove', handleMouseMove) - canvas.removeEventListener('mousedown', handleMouseDown) - canvas.removeEventListener('mouseup', handleMouseUp) - canvas.removeEventListener('wheel', handleWheel) - canvas.removeEventListener('click', disableEvent) - canvas.removeEventListener('contextmenu', disableEvent) + // Disable default events + function disableEvent(event: Event): void { + event.preventDefault() + event.stopPropagation() } - }, [resolution, scrollDirection, scrollInterval]) - function disableEvent(event: MouseEvent): void { - event.preventDefault() - event.stopPropagation() - } + // Mouse event handler + function handleMouseEvent(event: MouseAbsoluteEvent): void { + let report: number[] + const mouse = mouseRef.current + + switch (event.type) { + case 'mousedown': + mouse.buttonDown(event.button) + report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y) + break + case 'mouseup': + mouse.buttonUp(event.button) + report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y) + break + case 'wheel': + report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y, event.deltaY) + break + case 'move': + report = mouse.buildReport(event.x, event.y) + lastPosRef.current = { x: event.x, y: event.y } + break + default: + report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y) + break + } + + window.electron.ipcRenderer.invoke(IpcEvents.SEND_MOUSE, [0x02, ...report]) + + mouseJiggler.moveEventCallback() + } + + // Mouse down event + function handleMouseDown(e: MouseEvent): void { + disableEvent(e) + handleMouseEvent({ type: 'mousedown', button: e.button }) + } + + // Mouse up event + function handleMouseUp(e: MouseEvent): void { + disableEvent(e) + handleMouseEvent({ type: 'mouseup', button: e.button }) + } + + // Mouse move event + function handleMouseMove(e: MouseEvent): void { + disableEvent(e) + const { x, y } = getCoordinate(e) + handleMouseEvent({ type: 'move', x, y }) + } + + // Mouse wheel event + function handleWheel(e: WheelEvent): void { + disableEvent(e) + + if (Math.floor(e.deltaY) === 0) { + return + } + + const currentTime = Date.now() + if (currentTime - lastScrollTimeRef.current < scrollInterval) { + return + } + + const deltaY = (e.deltaY > 0 ? 1 : -1) * scrollDirection + handleMouseEvent({ type: 'wheel', deltaY }) + lastScrollTimeRef.current = currentTime + } + + // Add mouse event listeners + screen.addEventListener('mousedown', handleMouseDown) + screen.addEventListener('mouseup', handleMouseUp) + screen.addEventListener('mousemove', handleMouseMove) + screen.addEventListener('wheel', handleWheel) + screen.addEventListener('click', disableEvent) + screen.addEventListener('contextmenu', disableEvent) + + // Create touch handlers + const touchState = touchStateRef.current + const touchHandlers = createTouchHandlers(touchState, { + scrollDirection, + scrollInterval, + getCoordinate, + handleMouseEvent, + disableEvent + }) + + // Add touch event listeners (only on big screens) + if (isBigScreen) { + screen.addEventListener('touchstart', touchHandlers.handleTouchStart) + screen.addEventListener('touchmove', touchHandlers.handleTouchMove) + screen.addEventListener('touchend', touchHandlers.handleTouchEnd) + screen.addEventListener('touchcancel', touchHandlers.handleTouchCancel) + } + + return () => { + screen.removeEventListener('mousedown', handleMouseDown) + screen.removeEventListener('mouseup', handleMouseUp) + screen.removeEventListener('mousemove', handleMouseMove) + screen.removeEventListener('wheel', handleWheel) + screen.removeEventListener('click', disableEvent) + screen.removeEventListener('contextmenu', disableEvent) + screen.removeEventListener('touchstart', touchHandlers.handleTouchStart) + screen.removeEventListener('touchmove', touchHandlers.handleTouchMove) + screen.removeEventListener('touchend', touchHandlers.handleTouchEnd) + screen.removeEventListener('touchcancel', touchHandlers.handleTouchCancel) + + touchHandlers.cleanup() + } + }, [isBigScreen, scrollDirection, scrollInterval]) return <> } diff --git a/desktop/src/renderer/src/components/mouse/relative.tsx b/desktop/src/renderer/src/components/mouse/relative.tsx index 5d46b2c..4240be9 100644 --- a/desktop/src/renderer/src/components/mouse/relative.tsx +++ b/desktop/src/renderer/src/components/mouse/relative.tsx @@ -1,140 +1,160 @@ import { ReactElement, useEffect, useRef } from 'react' +import { message } from 'antd' import { useAtomValue } from 'jotai' +import { useTranslation } from 'react-i18next' import { IpcEvents } from '@common/ipc-events' -import { resolutionAtom } from '@renderer/jotai/device' import { scrollDirectionAtom, scrollIntervalAtom } from '@renderer/jotai/mouse' -import { Key } from '@renderer/libs/mouse' +import { MouseReportRelative } from '@renderer/libs/mouse' import { mouseJiggler } from '@renderer/libs/mouse-jiggler' +import type { MouseRelativeEvent } from './types' + export const Relative = (): ReactElement => { - const resolution = useAtomValue(resolutionAtom) + const { t } = useTranslation() + const [messageApi, contextHolder] = message.useMessage() + const scrollDirection = useAtomValue(scrollDirectionAtom) const scrollInterval = useAtomValue(scrollIntervalAtom) + const mouseRef = useRef(new MouseReportRelative()) const isLockedRef = useRef(false) - const keyRef = useRef(new Key()) const lastScrollTimeRef = useRef(0) useEffect(() => { - const canvas = document.getElementById('video') - if (!canvas) return + const screen = document.getElementById('video') + if (!screen) return + showMessage() + + screen.addEventListener('click', handleClick) + screen.addEventListener('mousedown', handleMouseDown) + screen.addEventListener('mouseup', handleMouseUp) + screen.addEventListener('mousemove', handleMouseMove) + screen.addEventListener('wheel', handleMouseWheel) + screen.addEventListener('contextmenu', disableEvent) document.addEventListener('pointerlockchange', handlePointerLockChange) - canvas.addEventListener('click', handleClick) - canvas.addEventListener('mousedown', handleMouseDown) - canvas.addEventListener('mouseup', handleMouseUp) - canvas.addEventListener('mousemove', handleMouseMove) - canvas.addEventListener('wheel', handleWheel) - canvas.addEventListener('contextmenu', disableEvent) - - function handlePointerLockChange(): void { - isLockedRef.current = document.pointerLockElement === canvas - } + // Click to request pointer lock function handleClick(event: MouseEvent): void { disableEvent(event) if (!isLockedRef.current) { - canvas!.requestPointerLock() + screen?.requestPointerLock() } } - async function handleMouseDown(event: MouseEvent): Promise { - disableEvent(event) - - switch (event.button) { - case 0: - keyRef.current.left = true - break - case 1: - keyRef.current.mid = true - break - case 2: - keyRef.current.right = true - break - default: - console.log(`unknown button ${event.button}`) - return - } - - await send(0, 0, 0) + // Mouse down event + function handleMouseDown(e: MouseEvent): void { + disableEvent(e) + handleMouseEvent({ type: 'mousedown', button: e.button }) } - async function handleMouseUp(event: MouseEvent): Promise { - disableEvent(event) - - switch (event.button) { - case 0: - keyRef.current.left = false - break - case 1: - keyRef.current.mid = false - break - case 2: - keyRef.current.right = false - break - default: - console.log(`unknown button ${event.button}`) - return - } - - await send(0, 0, 0) + // Mouse up event + function handleMouseUp(e: MouseEvent): void { + disableEvent(e) + handleMouseEvent({ type: 'mouseup', button: e.button }) } - async function handleMouseMove(event: MouseEvent): Promise { - disableEvent(event) + // Mouse move event + function handleMouseMove(e: MouseEvent): void { + disableEvent(e) - const x = event.movementX || 0 - const y = event.movementY || 0 + const x = e.movementX || 0 + const y = e.movementY || 0 if (x === 0 && y === 0) return - await send(Math.abs(x) < 10 ? x * 2 : x, Math.abs(y) < 10 ? y * 2 : y, 0) + const deltaX = Math.abs(x * window.devicePixelRatio) < 10 ? x * 2 : x + const deltaY = Math.abs(y * window.devicePixelRatio) < 10 ? y * 2 : y - mouseJiggler.moveEventCallback() + handleMouseEvent({ type: 'move', deltaX, deltaY }) } - async function handleWheel(event: WheelEvent): Promise { - disableEvent(event) + // Mouse wheel event + function handleMouseWheel(e: WheelEvent): void { + disableEvent(e) + + if (Math.floor(e.deltaY) === 0) { + return + } const currentTime = Date.now() if (currentTime - lastScrollTimeRef.current < scrollInterval) { return } - const delta = Math.floor(event.deltaY) - if (delta === 0) return - - await send(0, 0, delta > 0 ? -1 * scrollDirection : scrollDirection) - + const deltaY = (e.deltaY > 0 ? 1 : -1) * scrollDirection + handleMouseEvent({ type: 'wheel', deltaY }) lastScrollTimeRef.current = currentTime } - async function send(x: number, y: number, scroll: number): Promise { - await window.electron.ipcRenderer.invoke( - IpcEvents.SEND_MOUSE_RELATIVE, - keyRef.current.encode(), - x, - y, - scroll - ) + // Pointer lock state change + function handlePointerLockChange(): void { + isLockedRef.current = document.pointerLockElement === screen } return (): void => { - document.removeEventListener('pointerlockchange', handlePointerLockChange) - canvas.removeEventListener('click', handleClick) - canvas.removeEventListener('mousemove', handleMouseMove) - canvas.removeEventListener('mousedown', handleMouseDown) - canvas.removeEventListener('mouseup', handleMouseUp) - canvas.removeEventListener('wheel', handleWheel) - canvas.removeEventListener('contextmenu', disableEvent) - } - }, [resolution, scrollDirection, scrollInterval]) + // Exit pointer lock when component unmounts + if (document.pointerLockElement === screen) { + document.exitPointerLock() + } - function disableEvent(event: MouseEvent): void { + screen.removeEventListener('click', handleClick) + screen.removeEventListener('mousedown', handleMouseDown) + screen.removeEventListener('mouseup', handleMouseUp) + screen.removeEventListener('mousemove', handleMouseMove) + screen.removeEventListener('wheel', handleMouseWheel) + screen.removeEventListener('contextmenu', disableEvent) + document.removeEventListener('pointerlockchange', handlePointerLockChange) + } + }, [scrollDirection, scrollInterval]) + + // Mouse handler + function handleMouseEvent(event: MouseRelativeEvent): void { + let report: number[] + const mouse = mouseRef.current + + switch (event.type) { + case 'mousedown': + mouse.buttonDown(event.button) + report = mouse.buildButtonReport() + break + case 'mouseup': + mouse.buttonUp(event.button) + report = mouse.buildButtonReport() + break + case 'wheel': + report = mouse.buildReport(0, 0, event.deltaY) + break + case 'move': + report = mouse.buildReport(event.deltaX, event.deltaY) + break + default: + report = mouse.buildReport(0, 0) + break + } + + window.electron.ipcRenderer.invoke(IpcEvents.SEND_MOUSE, [0x01, ...report]) + + mouseJiggler.moveEventCallback() + } + + function showMessage(): void { + messageApi.open({ + key: 'requestPointer', + type: 'info', + content: t('mouse.requestPointer'), + duration: 3, + style: { + marginTop: '40vh' + } + }) + } + + function disableEvent(event: Event): void { event.preventDefault() event.stopPropagation() } - return <> + return <>{contextHolder} } diff --git a/desktop/src/renderer/src/components/mouse/touchpad.ts b/desktop/src/renderer/src/components/mouse/touchpad.ts new file mode 100644 index 0000000..6645c34 --- /dev/null +++ b/desktop/src/renderer/src/components/mouse/touchpad.ts @@ -0,0 +1,204 @@ +import { MouseButton } from './types' +import type { MouseAbsoluteEvent } from './types' + +// Touch event thresholds +const TAP_THRESHOLD = 8 +const DRAG_THRESHOLD = 10 +const VELOCITY_THRESHOLD = 0.3 +const LONG_PRESS_DELAY = 800 + +export interface TouchHandlerOptions { + scrollDirection: number + scrollInterval: number + getCoordinate: (event: { clientX: number; clientY: number }) => { x: number; y: number } + handleMouseEvent: (event: MouseAbsoluteEvent) => void + disableEvent: (event: Event) => void +} + +export interface TouchState { + touchStartTime: number + lastTouchY: number + longPressTimer: ReturnType | null + isLongPress: boolean + hasMove: boolean + isDragging: boolean + pressedButton: MouseButton | null + touchStartPos: { x: number; y: number } + lastScrollTime: number +} + +export function createInitialTouchState(): TouchState { + return { + touchStartTime: 0, + lastTouchY: 0, + longPressTimer: null, + isLongPress: false, + hasMove: false, + isDragging: false, + pressedButton: null, + touchStartPos: { x: 0, y: 0 }, + lastScrollTime: 0 + } +} + +export function createTouchHandlers(state: TouchState, options: TouchHandlerOptions) { + const { scrollDirection, scrollInterval, getCoordinate, handleMouseEvent, disableEvent } = options + + function handleTouchStart(e: TouchEvent): void { + disableEvent(e) + + if (e.touches.length === 0) { + return + } + + const touch = e.touches[0] + + // Reset states + state.touchStartTime = Date.now() + state.lastTouchY = touch.clientY + state.isLongPress = false + state.hasMove = false + state.isDragging = false + state.pressedButton = null + state.touchStartPos = { x: touch.clientX, y: touch.clientY } + + if (state.longPressTimer) { + clearTimeout(state.longPressTimer) + } + + const { x, y } = getCoordinate(touch) + handleMouseEvent({ type: 'move', x, y }) + + if (e.touches.length > 1) { + return + } + + // Start long press timer + state.longPressTimer = setTimeout(() => { + state.isLongPress = true + state.pressedButton = MouseButton.Right + if (navigator.vibrate) { + navigator.vibrate(50) + } + + handleMouseEvent({ type: 'mousedown', button: MouseButton.Right }) + }, LONG_PRESS_DELAY) + } + + function handleTouchMove(e: TouchEvent): void { + disableEvent(e) + + if (e.touches.length === 0) { + return + } + const touch = e.touches[0] + + // Handle two-finger scroll first + if (e.touches.length > 1) { + const currentTime = Date.now() + if (currentTime - state.lastScrollTime < scrollInterval) { + return + } + + const deltaY = (touch.clientY - state.lastTouchY > 0 ? 1 : -1) * scrollDirection + handleMouseEvent({ type: 'wheel', deltaY }) + + state.lastTouchY = touch.clientY + state.lastScrollTime = currentTime + return + } + + const deltaX = Math.abs(touch.clientX - state.touchStartPos.x) + const deltaY = Math.abs(touch.clientY - state.touchStartPos.y) + const distance = Math.sqrt(deltaX * deltaX + deltaY * deltaY) + + const timeDelta = Date.now() - state.touchStartTime + const velocity = timeDelta > 0 ? distance / timeDelta : 0 + + const shouldStartDrag = + distance > DRAG_THRESHOLD || (distance > TAP_THRESHOLD && velocity > VELOCITY_THRESHOLD) + + if (shouldStartDrag && !state.isDragging && !state.isLongPress) { + if (!state.hasMove) { + state.hasMove = true + } + + if (state.longPressTimer) { + clearTimeout(state.longPressTimer) + state.longPressTimer = null + } + + if (state.pressedButton === null) { + state.isDragging = true + state.pressedButton = MouseButton.Left + handleMouseEvent({ type: 'mousedown', button: MouseButton.Left }) + } + } + + if (distance > TAP_THRESHOLD && !state.hasMove) { + state.hasMove = true + } + + if (state.isDragging || state.isLongPress) { + const { x, y } = getCoordinate(touch) + handleMouseEvent({ type: 'move', x, y }) + } + } + + function handleTouchEnd(e: TouchEvent): void { + disableEvent(e) + + if (state.longPressTimer) { + clearTimeout(state.longPressTimer) + state.longPressTimer = null + } + + if (!state.hasMove && !state.isLongPress) { + handleMouseEvent({ type: 'mousedown', button: MouseButton.Left }) + setTimeout(() => { + handleMouseEvent({ type: 'mouseup', button: MouseButton.Left }) + }, 50) + } else if (state.pressedButton !== null) { + handleMouseEvent({ type: 'mouseup', button: state.pressedButton }) + } + + resetTouchState() + } + + function handleTouchCancel(e: TouchEvent): void { + disableEvent(e) + + if (state.longPressTimer) { + clearTimeout(state.longPressTimer) + state.longPressTimer = null + } + + if (state.pressedButton !== null) { + handleMouseEvent({ type: 'mouseup', button: state.pressedButton }) + } + + resetTouchState() + } + + function resetTouchState(): void { + state.isLongPress = false + state.hasMove = false + state.isDragging = false + state.pressedButton = null + } + + function cleanup(): void { + if (state.longPressTimer) { + clearTimeout(state.longPressTimer) + state.longPressTimer = null + } + } + + return { + handleTouchStart, + handleTouchMove, + handleTouchEnd, + handleTouchCancel, + cleanup + } +} diff --git a/desktop/src/renderer/src/components/mouse/types.ts b/desktop/src/renderer/src/components/mouse/types.ts new file mode 100644 index 0000000..79b644b --- /dev/null +++ b/desktop/src/renderer/src/components/mouse/types.ts @@ -0,0 +1,33 @@ +export enum MouseButton { + Left = 0, + Middle = 1, + Right = 2, + Back = 3, + Forward = 4 +} + +interface MouseMoveAbsoluteEvent { + type: 'move' + x: number + y: number +} + +interface MouseMoveRelativeEvent { + type: 'move' + deltaX: number + deltaY: number +} + +interface MouseButtonEvent { + type: 'mousedown' | 'mouseup' + button: number +} + +interface MouseWheelEvent { + type: 'wheel' + deltaY: number +} + +export type MouseAbsoluteEvent = MouseMoveAbsoluteEvent | MouseButtonEvent | MouseWheelEvent + +export type MouseRelativeEvent = MouseMoveRelativeEvent | MouseButtonEvent | MouseWheelEvent diff --git a/desktop/src/renderer/src/jotai/mouse.ts b/desktop/src/renderer/src/jotai/mouse.ts index 01f3620..4feca82 100644 --- a/desktop/src/renderer/src/jotai/mouse.ts +++ b/desktop/src/renderer/src/jotai/mouse.ts @@ -7,7 +7,7 @@ export const mouseStyleAtom = atom('cursor-default') export const mouseModeAtom = atom('absolute') // mouse scroll direction: 1 or -1 -export const scrollDirectionAtom = atom(1) +export const scrollDirectionAtom = atom(-1) // mouse scroll interval (unit: ms) export const scrollIntervalAtom = atom(0) diff --git a/desktop/src/renderer/src/libs/mouse-jiggler/index.ts b/desktop/src/renderer/src/libs/mouse-jiggler/index.ts index f693bba..fb706cf 100644 --- a/desktop/src/renderer/src/libs/mouse-jiggler/index.ts +++ b/desktop/src/renderer/src/libs/mouse-jiggler/index.ts @@ -1,18 +1,19 @@ import { IpcEvents } from '@common/ipc-events' -import { Key } from '@renderer/libs/mouse' +import { MouseReportRelative } from '@renderer/libs/mouse' const MOUSE_JIGGLER_INTERVAL = 15_000 -const EMPTY_KEY: Key = new Key(false, false, false) class MouseJiggler { private lastMoveTime: number private timer: NodeJS.Timeout | null private mode: 'enable' | 'disable' + private mouseReport: MouseReportRelative constructor() { this.lastMoveTime = Date.now() this.timer = null this.mode = 'disable' + this.mouseReport = new MouseReportRelative() } // enable or disable mouse jiggler @@ -43,20 +44,13 @@ class MouseJiggler { } async sendJiggle(): Promise { - await window.electron.ipcRenderer.invoke( - IpcEvents.SEND_MOUSE_RELATIVE, - EMPTY_KEY.encode(), - 10, - 10, - 0 - ) - await window.electron.ipcRenderer.invoke( - IpcEvents.SEND_MOUSE_RELATIVE, - EMPTY_KEY.encode(), - -10, - -10, - 0 - ) + // Build reports directly using the report builder + const report1 = this.mouseReport.buildReport(10, 10, 0) + const report2 = this.mouseReport.buildReport(-10, -10, 0) + + await window.electron.ipcRenderer.invoke(IpcEvents.SEND_MOUSE, [0x01, report1]) + await window.electron.ipcRenderer.invoke(IpcEvents.SEND_MOUSE, [0x01, report2]) } } + export const mouseJiggler = new MouseJiggler() diff --git a/desktop/src/renderer/src/libs/mouse/index.ts b/desktop/src/renderer/src/libs/mouse/index.ts index bb13965..1ef22b0 100644 --- a/desktop/src/renderer/src/libs/mouse/index.ts +++ b/desktop/src/renderer/src/libs/mouse/index.ts @@ -1,27 +1,134 @@ -export class Key { - left: boolean - right: boolean - mid: boolean +// Maximum absolute coordinate value +const MAX_ABS_COORD = 4096 - constructor(left: boolean = false, right: boolean = false, mid: boolean = false) { - this.left = left - this.right = right - this.mid = mid - } +// Button bit positions +const MouseButtons = { + Left: 1 << 0, + Right: 1 << 1, + Middle: 1 << 2, + Back: 1 << 3, + Forward: 1 << 4 +} as const - public encode(): number { - let b = 0x00 - b = setBit(b, 0, this.left) - b = setBit(b, 1, this.right) - b = setBit(b, 2, this.mid) - return b +// Map browser button index to HID bit +function getMouseButtonBit(button: number): number { + switch (button) { + case 0: + return MouseButtons.Left + case 1: + return MouseButtons.Middle + case 2: + return MouseButtons.Right + case 3: + return MouseButtons.Back + case 4: + return MouseButtons.Forward + default: + return 0 } } -function setBit(number: number, bitPosition: number, value: boolean): number { - if (value) { - return number | (1 << bitPosition) - } else { - return number & ~(1 << bitPosition) +/** + * Relative Mouse Report (4 bytes) + * + * Byte 0: Buttons + * Byte 1: X movement (-127 to 127) + * Byte 2: Y movement (-127 to 127) + * Byte 3: Wheel (-127 to 127) + */ +export class MouseReportRelative { + private buttons: number = 0 + + buttonDown(button: number): void { + this.buttons |= getMouseButtonBit(button) + } + + buttonUp(button: number): void { + this.buttons &= ~getMouseButtonBit(button) + } + + /** + * Build relative mouse report + * @param deltaX X movement (-127 to 127) + * @param deltaY Y movement (-127 to 127) + * @param wheel Scroll wheel (-127 to 127, negative = down) + */ + buildReport(deltaX: number, deltaY: number, wheel: number = 0): number[] { + const x = this.clamp(Math.round(deltaX), -127, 127) & 0xff + const y = this.clamp(Math.round(deltaY), -127, 127) & 0xff + const scroll = this.clamp(Math.round(wheel), -127, 127) & 0xff + return [this.buttons, x, y, scroll] + } + + /** + * Build button-only report (no movement) + */ + buildButtonReport(): number[] { + return this.buildReport(0, 0, 0) + } + + reset(): number[] { + this.buttons = 0 + return this.buildReport(0, 0, 0) + } + + private clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)) + } +} + +/** + * Absolute Mouse Report (6 bytes) + * + * Byte 0: Buttons + * Byte 1-2: X position (0 to 32767, Little Endian) + * Byte 3-4: Y position (0 to 32767, Little Endian) + * Byte 5: Wheel + */ +export class MouseAbsoluteRelative { + private buttons: number = 0 + + buttonDown(button: number): void { + this.buttons |= getMouseButtonBit(button) + } + + buttonUp(button: number): void { + this.buttons &= ~getMouseButtonBit(button) + } + + /** + * Build absolute mouse report + * @param x X position (0.0 to 1.0, normalized) + * @param y Y position (0.0 to 1.0, normalized) + * @param wheel Scroll wheel (-127 to 127) + */ + buildReport(x: number, y: number, wheel: number = 0): number[] { + // Convert normalized coordinates (0-1) to absolute coordinates (0-4096) + const xAbs = Math.floor(Math.max(0, Math.min(1, x)) * MAX_ABS_COORD) + const yAbs = Math.floor(Math.max(0, Math.min(1, y)) * MAX_ABS_COORD) + + const x1 = xAbs & 0xff + const x2 = (xAbs >> 8) & 0xff + const y1 = yAbs & 0xff + const y2 = (yAbs >> 8) & 0xff + const scroll = this.clamp(Math.round(wheel), -127, 127) & 0xff + + return [this.buttons, x1, x2, y1, y2, scroll] + } + + /** + * Build button-only report (keeps last position) + */ + buildButtonReport(lastX: number, lastY: number): number[] { + return this.buildReport(lastX, lastY, 0) + } + + reset(): number[] { + this.buttons = 0 + return this.buildReport(0, 0, 0) + } + + private clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)) } }