Administrator/NanoKVM-MIRROR
1
0
Fork 1
mirror of https://github.com/sipeed/NanoKVM.git synced 2026-09-11 00:22:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: refactor keyboard, mouse, and WebSocket

Browse Source
This commit is contained in:
wj-xiao
2026-01-05 16:49:17 +08:00
parent 687dd3bfc4
commit 1b7021d64b
25 changed files with 1443 additions and 759 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
kvmapp/system/init.d/S03usbdev
View File

@@ -60,7 +60,7 @@ start_usb_dev(){
fi
echo 1 > functions/hid.GS0/protocol
echo 8 > functions/hid.GS0/report_length
echo -ne \\x05\\x01\\x09\\x06\\xa1\\x01\\x05\\x07\\x19\\xe0\\x29\\xe7\\x15\\x00\\x25\\x01\\x75\\x01\\x95\\x08\\x81\\x02\\x95\\x01\\x75\\x08\\x81\\x03\\x95\\x05\\x75\\x01\\x05\\x08\\x19\\x01\\x29\\x05\\x91\\x02\\x95\\x01\\x75\\x03\\x91\\x03\\x95\\x06\\x75\\x08\\x15\\x00\\x25\\x65\\x05\\x07\\x19\\x00\\x29\\x65\\x81\\x00\\xc0 > functions/hid.GS0/report_desc
echo -ne \\x05\\x01\\x09\\x06\\xa1\\x01\\x05\\x07\\x19\\xe0\\x29\\xe7\\x15\\x00\\x25\\x01\\x75\\x01\\x95\\x08\\x81\\x02\\x95\\x01\\x75\\x08\\x81\\x03\\x95\\x05\\x75\\x01\\x05\\x08\\x19\\x01\\x29\\x05\\x91\\x02\\x95\\x01\\x75\\x03\\x91\\x03\\x95\\x06\\x75\\x08\\x15\\x00\\x25\\xE7\\x05\\x07\\x19\\x00\\x29\\xE7\\x81\\x00\\xc0 > functions/hid.GS0/report_desc
ln -s functions/hid.GS0 configs/c.1
# mouse
@@ -90,7 +90,7 @@ start_usb_dev(){
fi
echo 2 > functions/hid.GS2/protocol
echo 6 > functions/hid.GS2/report_length
echo -ne \\x05\\x01\\x09\\x02\\xa1\\x01\\x09\\x01\\xa1\\x00\\x05\\x09\\x19\\x01\\x29\\x03\\x15\\x00\\x25\\x01\\x95\\x03\\x75\\x01\\x81\\x02\\x95\\x01\\x75\\x05\\x81\\x01\\x05\\x01\\x09\\x30\\x09\\x31\\x15\\x00\\x26\\xff\\x7f\\x35\\x00\\x46\\xff\\x7f\\x75\\x10\\x95\\x02\\x81\\x02\\x05\\x01\\x09\\x38\\x15\\x81\\x25\\x7f\\x35\\x00\\x45\\x00\\x75\\x08\\x95\\x01\\x81\\x06\\xc0\\xc0 > functions/hid.GS2/report_desc
echo -ne \\x05\\x01\\x09\\x02\\xa1\\x01\\x09\\x01\\xa1\\x00\\x05\\x09\\x19\\x01\\x29\\x05\\x15\\x00\\x25\\x01\\x95\\x05\\x75\\x01\\x81\\x02\\x95\\x01\\x75\\x03\\x81\\x01\\x05\\x01\\x09\\x30\\x09\\x31\\x15\\x00\\x26\\xff\\x7f\\x35\\x00\\x46\\xff\\x7f\\x75\\x10\\x95\\x02\\x81\\x02\\x05\\x01\\x09\\x38\\x15\\x81\\x25\\x7f\\x35\\x00\\x45\\x00\\x75\\x08\\x95\\x01\\x81\\x06\\xc0\\xc0 > functions/hid.GS2/report_desc
ln -s functions/hid.GS2 configs/c.1
fi

18
server/service/hid/hid.go
View File

@@ -18,9 +18,9 @@ type Hid struct {
}
const (
HID0 = "/dev/hidg0"
HID1 = "/dev/hidg1"
HID2 = "/dev/hidg2"
HID0 = "/dev/hidg0" // Keyboard
HID1 = "/dev/hidg1" // Mouse (Relative Mode)
HID2 = "/dev/hidg2" // Touchpad (Absolute Mode)
)
var (
@@ -95,16 +95,22 @@ func (h *Hid) Close() {
}
func (h *Hid) WriteHid0(data []byte) {
deadline := time.Now().Add(8 * time.Millisecond)
h.kbMutex.Lock()
_ = h.g0.SetWriteDeadline(deadline)
_, err := h.g0.Write(data)
h.kbMutex.Unlock()
if err != nil {
if errors.Is(err, os.ErrClosed) {
switch {
case errors.Is(err, os.ErrClosed):
log.Errorf("hid already closed, reopen it...")
h.OpenNoLock()
} else {
log.Debugf("write to %s failed: %s", HID0, err)
case errors.Is(err, os.ErrDeadlineExceeded):
log.Debugf("write to %s timeout", HID0)
default:
log.Errorf("write to %s failed: %s", HID0, err)
}
return
}

17
server/service/hid/keyboard.go
View File

@@ -1,15 +1,16 @@
package hid
func (h *Hid) Keyboard(queue <-chan []int) {
for event := range queue {
code := byte(event[0])
import (
log "github.com/sirupsen/logrus"
)
var modifier byte = 0x00
if code > 0 {
modifier = byte(event[1]) | byte(event[2]) | byte(event[3]) | byte(event[4])
func (h *Hid) Keyboard(queue <-chan []byte) {
for event := range queue {
if len(event) != 8 {
log.Debugf("invalid keyboard event: %v", event)
continue
}
data := []byte{modifier, 0x00, code, 0x00, 0x00, 0x00, 0x00, 0x00}
h.WriteHid0(data)
h.WriteHid0(event)
}
}

76
server/service/hid/mouse.go
View File

@@ -1,82 +1,18 @@
package hid
import (
"encoding/binary"
log "github.com/sirupsen/logrus"
)
const (
MouseUp = iota
MouseDown
MouseMoveAbsolute
MouseMoveRelative
MouseScroll
)
var mouseButtonMap = map[byte]bool{
0x01: true,
0x02: true,
0x04: true,
}
func (h *Hid) Mouse(queue <-chan []int) {
func (h *Hid) Mouse(queue <-chan []byte) {
for event := range queue {
switch event[0] {
case MouseDown:
h.mouseDown(event)
case MouseUp:
h.mouseUp()
case MouseMoveAbsolute:
h.mouseMoveAbsolute(event)
case MouseMoveRelative:
h.mouseMoveRelative(event)
case MouseScroll:
h.mouseScroll(event)
switch len(event) {
case 4:
h.WriteHid1(event)
case 6:
h.WriteHid2(event)
default:
log.Debugf("invalid mouse event: %v", event)
}
}
}
func (h *Hid) mouseDown(event []int) {
button := byte(event[1])
if _, ok := mouseButtonMap[button]; !ok {
log.Errorf("invalid mouse button: %v", event)
return
}
data := []byte{button, 0, 0, 0}
h.WriteHid1(data)
}
func (h *Hid) mouseUp() {
data := []byte{0, 0, 0, 0}
h.WriteHid1(data)
}
func (h *Hid) mouseScroll(event []int) {
direction := 0x01
if event[3] < 0 {
direction = -0x1
}
data := []byte{0, 0, 0, byte(direction)}
h.WriteHid1(data)
}
func (h *Hid) mouseMoveAbsolute(event []int) {
x := make([]byte, 2)
y := make([]byte, 2)
binary.LittleEndian.PutUint16(x, uint16(event[2]))
binary.LittleEndian.PutUint16(y, uint16(event[3]))
data := []byte{0, x[0], x[1], y[0], y[1], 0}
h.WriteHid2(data)
}
func (h *Hid) mouseMoveRelative(event []int) {
data := []byte{byte(event[1]), byte(event[2]), byte(event[3]), 0}
h.WriteHid1(data)
}

109
server/service/ws/client.go Normal file
View File

@@ -0,0 +1,109 @@
package ws
import (
"encoding/json"
"time"
"NanoKVM-Server/service/hid"
"NanoKVM-Server/service/vm/jiggler"
"github.com/gorilla/websocket"
log "github.com/sirupsen/logrus"
)
const (
Heartbeat = iota
KeyboardEvent
MouseEvent
)
func NewClient(ws *websocket.Conn) *Client {
client := &Client{
ws: ws,
hid: hid.GetHid(),
keyboard: make(chan []byte, 200),
mouse: make(chan []byte, 200),
lastHeartbeat: time.Time{},
}
client.hid.Open()
return client
}
func (c *Client) Start() {
defer c.Close()
go c.hid.Keyboard(c.keyboard)
go c.hid.Mouse(c.mouse)
_ = c.Read()
}
func (c *Client) Read() error {
var zeroTime time.Time
_ = c.ws.SetReadDeadline(zeroTime)
for {
messageType, data, err := c.ws.ReadMessage()
if err != nil {
return err
}
log.Debugf("received message %d: %v", messageType, data)
switch data[0] {
case Heartbeat:
c.UpdateHeartbeat()
case KeyboardEvent:
writeQueue(c.keyboard, data[1:])
case MouseEvent:
writeQueue(c.mouse, data[1:])
}
}
}
func (c *Client) Write(event string, data string) error {
message := &Message{
Type: event,
Data: data,
}
messageByte, err := json.Marshal(message)
if err != nil {
log.Errorf("failed to marshal message: %s", err)
return err
}
c.mutex.Lock()
defer c.mutex.Unlock()
_ = c.ws.SetWriteDeadline(time.Now().Add(10 * time.Second))
return c.ws.WriteMessage(websocket.TextMessage, messageByte)
}
func (c *Client) UpdateHeartbeat() {
c.mutex.Lock()
defer c.mutex.Unlock()
c.lastHeartbeat = time.Now()
}
func (c *Client) Close() {
_ = c.ws.Close()
closeQueue(c.keyboard)
closeQueue(c.mouse)
log.Debug("websocket disconnected")
}
func writeQueue(queue chan []byte, data []byte) {
queue <- data
jiggler.GetJiggler().Update()
}
func closeQueue(queue chan []byte) {
for range queue {
}
close(queue)
}

46
server/service/ws/manager.go Normal file
View File

@@ -0,0 +1,46 @@
package ws
import (
"sync"
"github.com/gorilla/websocket"
)
var (
globalManager *Manager
managerOnce sync.Once
)
func GetManager() *Manager {
managerOnce.Do(func() {
globalManager = &Manager{
clients: make(map[*websocket.Conn]*Client),
mutex: sync.RWMutex{},
}
})
return globalManager
}
func (m *Manager) AddClient(ws *websocket.Conn, client *Client) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.clients[ws] = client
}
func (m *Manager) RemoveClient(ws *websocket.Conn) {
m.mutex.Lock()
defer m.mutex.Unlock()
delete(m.clients, ws)
}
func (m *Manager) GetClients() []*Client {
m.mutex.Lock()
defer m.mutex.Unlock()
clients := make([]*Client, 0, len(m.clients))
for _, c := range m.clients {
clients = append(clients, c)
}
return clients
}

6
server/service/ws/message.go
View File

@@ -1,6 +0,0 @@
package ws
type Stream struct {
Type string `json:"type"`
State int `json:"state"`
}

34
server/service/ws/service.go
View File

@@ -1,7 +1,41 @@
package ws
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
log "github.com/sirupsen/logrus"
)
type Service struct{}
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true
},
}
func NewService() *Service {
return &Service{}
}
func (s *Service) Connect(c *gin.Context) {
ws, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
log.Errorf("create websocket failed: %s", err)
return
}
log.Debug("websocket connected")
client := NewClient(ws)
manager := GetManager()
manager.AddClient(ws, client)
defer manager.RemoveClient(ws)
client.Start()
}

29
server/service/ws/types.go Normal file
View File

@@ -0,0 +1,29 @@
package ws
import (
"sync"
"time"
"github.com/gorilla/websocket"
"NanoKVM-Server/service/hid"
)
type Manager struct {
clients map[*websocket.Conn]*Client
mutex sync.RWMutex
}
type Client struct {
ws *websocket.Conn
hid *hid.Hid
keyboard chan []byte
mouse chan []byte
lastHeartbeat time.Time
mutex sync.Mutex
}
type Message struct {
Type string `json:"type"`
Data string `json:"data"`
}

117
server/service/ws/ws.go
View File

@@ -1,117 +0,0 @@
package ws
import (
"encoding/json"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
log "github.com/sirupsen/logrus"
"NanoKVM-Server/service/hid"
"NanoKVM-Server/service/vm/jiggler"
)
const (
KeyboardEvent int = 1
MouseEvent int = 2
)
type WsClient struct {
conn *websocket.Conn
hid *hid.Hid
keyboard chan []int
mouse chan []int
}
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true
},
}
func (s *Service) Connect(c *gin.Context) {
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
log.Errorf("create websocket failed: %s", err)
return
}
log.Debug("websocket connected")
client := &WsClient{
hid: hid.GetHid(),
conn: conn,
keyboard: make(chan []int, 200),
mouse: make(chan []int, 200),
}
go client.Start()
}
func (c *WsClient) Start() {
defer c.Clean()
c.hid.Open()
go c.hid.Keyboard(c.keyboard)
go c.hid.Mouse(c.mouse)
_ = c.Read()
}
func (c *WsClient) Read() error {
var zeroTime time.Time
_ = c.conn.SetReadDeadline(zeroTime)
for {
_, message, err := c.conn.ReadMessage()
if err != nil {
return err
}
log.Debugf("received message: %s", message)
var event []int
err = json.Unmarshal(message, &event)
if err != nil {
log.Debugf("received invalid message: %s", message)
continue
}
if event[0] == KeyboardEvent {
c.keyboard <- event[1:]
} else if event[0] == MouseEvent {
c.mouse <- event[1:]
}
// update latest HID operation time
jiggler.GetJiggler().Update()
}
}
func (c *WsClient) Write(message []byte) error {
_ = c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
return c.conn.WriteMessage(websocket.TextMessage, message)
}
func (c *WsClient) Clean() {
_ = c.conn.Close()
go clearQueue(c.keyboard)
close(c.keyboard)
go clearQueue(c.mouse)
close(c.mouse)
c.hid.Close()
log.Debug("websocket disconnected")
}
func clearQueue(queue chan []int) {
for range queue {
}
}

65
web/src/lib/keyboard.ts Normal file
View File

@@ -0,0 +1,65 @@
import { getKeycode, getModifierBit, isModifier } from './keymap';
const MAX_KEYS = 6;
export class KeyboardReport {
private modifier: number = 0;
private pressedKeys: Map<string, number> = new Map();
keyDown(code: string): Uint8Array {
if (isModifier(code)) {
this.modifier |= getModifierBit(code);
} else {
const keycode = getKeycode(code);
if (keycode !== undefined && this.pressedKeys.size < MAX_KEYS) {
this.pressedKeys.set(code, keycode);
}
}
return this.buildReport();
}
keyUp(code: string): Uint8Array {
if (isModifier(code)) {
this.modifier &= ~getModifierBit(code);
} else {
this.pressedKeys.delete(code);
}
return this.buildReport();
}
reset(): Uint8Array {
this.modifier = 0;
this.pressedKeys.clear();
return this.buildReport();
}
/**
* Build the 8-byte HID keyboard report
* Byte 0: Modifier keys bitmap
* Byte 1: Reserved (0x00)
* Bytes 2-7: Up to 6 keycodes
*/
private buildReport(): Uint8Array {
const report = new Uint8Array(8);
report[0] = this.modifier;
report[1] = 0x00;
let i = 2;
for (const keycode of this.pressedKeys.values()) {
if (i >= 8) break;
report[i++] = keycode;
}
return report;
}
getModifier(): number {
return this.modifier;
}
getPressedKeyCount(): number {
return this.pressedKeys.size;
}
}
export const keyboard = new KeyboardReport();

269
web/src/lib/keymap.ts Normal file
View File

@@ -0,0 +1,269 @@
// Modifier key bit positions
export const ModifierBits = {
LeftCtrl: 1 << 0,
LeftShift: 1 << 1,
LeftAlt: 1 << 2,
LeftMeta: 1 << 3,
RightCtrl: 1 << 4,
RightShift: 1 << 5,
RightAlt: 1 << 6,
RightMeta: 1 << 7
} as const;
// Map event.code to HID modifier bit
export const ModifierMap: Record<string, number> = {
ControlLeft: ModifierBits.LeftCtrl,
ShiftLeft: ModifierBits.LeftShift,
AltLeft: ModifierBits.LeftAlt,
MetaLeft: ModifierBits.LeftMeta,
ControlRight: ModifierBits.RightCtrl,
ShiftRight: ModifierBits.RightShift,
AltRight: ModifierBits.RightAlt,
MetaRight: ModifierBits.RightMeta
};
// Map event.code to HID keycode
export const KeycodeMap: Record<string, number> = {
// Letters
KeyA: 0x04,
KeyB: 0x05,
KeyC: 0x06,
KeyD: 0x07,
KeyE: 0x08,
KeyF: 0x09,
KeyG: 0x0a,
KeyH: 0x0b,
KeyI: 0x0c,
KeyJ: 0x0d,
KeyK: 0x0e,
KeyL: 0x0f,
KeyM: 0x10,
KeyN: 0x11,
KeyO: 0x12,
KeyP: 0x13,
KeyQ: 0x14,
KeyR: 0x15,
KeyS: 0x16,
KeyT: 0x17,
KeyU: 0x18,
KeyV: 0x19,
KeyW: 0x1a,
KeyX: 0x1b,
KeyY: 0x1c,
KeyZ: 0x1d,
// Numbers
Digit1: 0x1e,
Digit2: 0x1f,
Digit3: 0x20,
Digit4: 0x21,
Digit5: 0x22,
Digit6: 0x23,
Digit7: 0x24,
Digit8: 0x25,
Digit9: 0x26,
Digit0: 0x27,
// Special keys
Enter: 0x28,
Escape: 0x29,
Backspace: 0x2a,
Tab: 0x2b,
Space: 0x2c,
Minus: 0x2d,
Equal: 0x2e,
BracketLeft: 0x2f,
BracketRight: 0x30,
Backslash: 0x31,
Semicolon: 0x33,
Quote: 0x34,
Backquote: 0x35,
Comma: 0x36,
Period: 0x37,
Slash: 0x38,
CapsLock: 0x39,
// Function keys
F1: 0x3a,
F2: 0x3b,
F3: 0x3c,
F4: 0x3d,
F5: 0x3e,
F6: 0x3f,
F7: 0x40,
F8: 0x41,
F9: 0x42,
F10: 0x43,
F11: 0x44,
F12: 0x45,
// Control keys
PrintScreen: 0x46,
ScrollLock: 0x47,
Pause: 0x48,
Insert: 0x49,
Home: 0x4a,
PageUp: 0x4b,
Delete: 0x4c,
End: 0x4d,
PageDown: 0x4e,
// Arrow keys
ArrowRight: 0x4f,
ArrowLeft: 0x50,
ArrowDown: 0x51,
ArrowUp: 0x52,
// Numpad
NumLock: 0x53,
NumpadDivide: 0x54,
NumpadMultiply: 0x55,
NumpadSubtract: 0x56,
NumpadAdd: 0x57,
NumpadEnter: 0x58,
Numpad1: 0x59,
Numpad2: 0x5a,
Numpad3: 0x5b,
Numpad4: 0x5c,
Numpad5: 0x5d,
Numpad6: 0x5e,
Numpad7: 0x5f,
Numpad8: 0x60,
Numpad9: 0x61,
Numpad0: 0x62,
NumpadDecimal: 0x63,
// International / Non-US keyboard keys
IntlBackslash: 0x64,
ContextMenu: 0x65,
Power: 0x66,
NumpadEqual: 0x67,
// Extended function keys
F13: 0x68,
F14: 0x69,
F15: 0x6a,
F16: 0x6b,
F17: 0x6c,
F18: 0x6d,
F19: 0x6e,
F20: 0x6f,
F21: 0x70,
F22: 0x71,
F23: 0x72,
F24: 0x73,
// System / Edit keys
Execute: 0x74,
Help: 0x75,
Props: 0x76,
Select: 0x77,
Stop: 0x78,
Again: 0x79,
Undo: 0x7a,
Cut: 0x7b,
Copy: 0x7c,
Paste: 0x7d,
Find: 0x7e,
// Media / Volume keys
AudioVolumeMute: 0x7f,
AudioVolumeUp: 0x80,
AudioVolumeDown: 0x81,
VolumeMute: 0x7f, // Alias
VolumeUp: 0x80, // Alias
VolumeDown: 0x81, // Alias
// Locking keys (for keyboards with physical lock keys)
LockingCapsLock: 0x82,
LockingNumLock: 0x83,
LockingScrollLock: 0x84,
// Numpad additional
NumpadComma: 0x85,
NumpadEqual2: 0x86, // AS/400 keyboard equal key
// International keys - Japanese
IntlRo: 0x87, // Japanese Ro key (ろ)
KanaMode: 0x88, // Katakana/Hiragana toggle
IntlYen: 0x89, // Japanese Yen (¥)
Convert: 0x8a, // Japanese Henkan (変換)
NonConvert: 0x8b, // Japanese Muhenkan (無変換)
// International keys - Additional Japanese
International6: 0x8c,
International7: 0x8d,
International8: 0x8e,
International9: 0x8f,
// Language keys - Korean/Japanese/Chinese
Lang1: 0x90, // Korean Hangul/English toggle
Lang2: 0x91, // Korean Hanja
Lang3: 0x92, // Japanese Katakana
Lang4: 0x93, // Japanese Hiragana
Lang5: 0x94, // Japanese Zenkaku/Hankaku
Lang6: 0x95,
Lang7: 0x96,
Lang8: 0x97,
Lang9: 0x98,
// ISO keyboard specific
IntlHash: 0x32, // Non-US # and ~ (ISO keyboards)
// Numpad extended
NumpadParenLeft: 0xb6,
NumpadParenRight: 0xb7,
NumpadBackspace: 0xbb,
NumpadMemoryStore: 0xd0,
NumpadMemoryRecall: 0xd1,
NumpadMemoryClear: 0xd2,
NumpadMemoryAdd: 0xd3,
NumpadMemorySubtract: 0xd4,
NumpadClear: 0xd8,
NumpadClearEntry: 0xd9,
// Additional browser/system keys
BrowserSearch: 0xf0,
BrowserHome: 0xf1,
BrowserBack: 0xf2,
BrowserForward: 0xf3,
BrowserStop: 0xf4,
BrowserRefresh: 0xf5,
BrowserFavorites: 0xf6,
// Media keys
MediaPlayPause: 0xe8,
MediaStop: 0xe9,
MediaTrackPrevious: 0xea,
MediaTrackNext: 0xeb,
Eject: 0xec,
MediaSelect: 0xed,
// Application launch keys
LaunchMail: 0xee,
LaunchApp1: 0xef,
LaunchApp2: 0xf0,
// Sleep/Wake keys
Sleep: 0xf8,
Wake: 0xf9,
// Accessibility keys
MediaRewind: 0xfa,
MediaFastForward: 0xfb
};
// Check if code is a modifier key
export function isModifier(code: string): boolean {
return code in ModifierMap;
}
// Get modifier bit for code
export function getModifierBit(code: string): number {
return ModifierMap[code] ?? 0;
}
// Get keycode for code
export function getKeycode(code: string): number | undefined {
return KeycodeMap[code];
}

138
web/src/lib/mouse.ts Normal file
View File

@@ -0,0 +1,138 @@
// Button bit positions
const MouseButtons = {
Left: 1 << 0,
Right: 1 << 1,
Middle: 1 << 2,
Back: 1 << 3,
Forward: 1 << 4
} as const;
// Map browser button index to HID bit
function getMouseButtonBit(button: number): number {
switch (button) {
case 0:
return MouseButtons.Left;
case 1:
return MouseButtons.Middle;
case 2:
return MouseButtons.Right;
case 3:
return MouseButtons.Back;
case 4:
return MouseButtons.Forward;
default:
return 0;
}
}
/**
* Relative Mouse Report (4 bytes)
* Used with /dev/hidg1 (relative mouse)
*
* Byte 0: Buttons
* Byte 1: X movement (-127 to 127)
* Byte 2: Y movement (-127 to 127)
* Byte 3: Wheel (-127 to 127)
*/
export class MouseReportRelative {
private buttons: number = 0;
buttonDown(button: number): void {
this.buttons |= getMouseButtonBit(button);
}
buttonUp(button: number): void {
this.buttons &= ~getMouseButtonBit(button);
}
/**
* Build relative mouse report
* @param deltaX X movement (-127 to 127)
* @param deltaY Y movement (-127 to 127)
* @param wheel Scroll wheel (-127 to 127, negative = down)
*/
buildReport(deltaX: number, deltaY: number, wheel: number = 0): Uint8Array {
const report = new Uint8Array(4);
report[0] = this.buttons;
report[1] = this.clamp(Math.round(deltaX), -127, 127) & 0xff;
report[2] = this.clamp(Math.round(deltaY), -127, 127) & 0xff;
report[3] = this.clamp(Math.round(wheel), -127, 127) & 0xff;
return report;
}
/**
* Build button-only report (no movement)
*/
buildButtonReport(): Uint8Array {
return this.buildReport(0, 0, 0);
}
reset(): Uint8Array {
this.buttons = 0;
return this.buildReport(0, 0, 0);
}
private clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
}
/**
* Absolute Mouse Report (6 bytes)
* Used with /dev/hidg2 (absolute mouse/tablet)
*
* Byte 0: Buttons
* Byte 1-2: X position (0 to 32767, Little Endian)
* Byte 3-4: Y position (0 to 32767, Little Endian)
* Byte 5: Wheel
*/
export class MouseReportAbsolute {
private buttons: number = 0;
buttonDown(button: number): void {
this.buttons |= getMouseButtonBit(button);
}
buttonUp(button: number): void {
this.buttons &= ~getMouseButtonBit(button);
}
/**
* Build absolute mouse report
* @param x X position (0.0 to 1.0, normalized)
* @param y Y position (0.0 to 1.0, normalized)
* @param wheel Scroll wheel (-127 to 127)
*/
buildReport(x: number, y: number, wheel: number = 0): Uint8Array {
const report = new Uint8Array(6);
report[0] = this.buttons;
report[1] = x & 0xff;
report[2] = (x >> 8) & 0xff;
report[3] = y & 0xff;
report[4] = (y >> 8) & 0xff;
report[5] = this.clamp(Math.round(wheel), -127, 127) & 0xff;
return report;
}
/**
* Build button-only report (keeps last position)
*/
buildButtonReport(lastX: number, lastY: number): Uint8Array {
return this.buildReport(lastX, lastY, 0);
}
reset(): Uint8Array {
this.buttons = 0;
return this.buildReport(0, 0, 0);
}
private clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
}
// Singleton instances
export const mouseRelative = new MouseReportRelative();
export const mouseAbsolute = new MouseReportAbsolute();

212
web/src/lib/websocket.ts
View File

@@ -2,64 +2,188 @@ import { IMessageEvent, w3cwebsocket as W3cWebSocket } from 'websocket';
import { getBaseUrl } from '@/lib/service.ts';
type Event = (message: IMessageEvent) => void;
type MessageHandler = (message: IMessageEvent) => void;
type SendData = number[] | ArrayBuffer | Uint8Array;
const eventMap: Map<string, Event> = new Map<string, Event>();
export enum MessageEvent {
Heartbeat = 0,
Keyboard = 1,
Mouse = 2
}
class WsClient {
private readonly url: string;
private instance: W3cWebSocket;
interface WsClientOptions {
url?: string;
heartbeatInterval?: number;
reconnectInterval?: number;
maxReconnectAttempts?: number;
}
constructor() {
this.url = `${getBaseUrl('ws')}/api/ws`;
this.instance = new W3cWebSocket(this.url);
this.setEvents();
const DEFAULT_OPTIONS: Required<WsClientOptions> = {
url: `${getBaseUrl('ws')}/api/ws`,
heartbeatInterval: 10 * 1000,
reconnectInterval: 3 * 1000,
maxReconnectAttempts: 1
};
export class WsClient {
private readonly options: Required<WsClientOptions>;
private instance: W3cWebSocket | null = null;
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private reconnectAttempts = 0;
private shouldReconnect = true;
private readonly eventHandlers = new Map<string, Set<MessageHandler>>();
constructor(options: WsClientOptions = {}) {
this.options = { ...DEFAULT_OPTIONS, ...options };
}
public connect() {
this.close();
this.instance = new W3cWebSocket(this.url);
this.setEvents();
public connect(): void {
this.shouldReconnect = true;
this.reconnectAttempts = 0;
this.createConnection();
}
public send(data: number[]) {
if (this.instance.readyState !== W3cWebSocket.OPEN) {
public close(): void {
this.shouldReconnect = false;
this.cleanup();
if (this.instance && this.instance.readyState === W3cWebSocket.OPEN) {
this.instance.close();
}
this.instance = null;
}
public on(type: string, handler: MessageHandler): () => void {
if (!this.eventHandlers.has(type)) {
this.eventHandlers.set(type, new Set());
}
this.eventHandlers.get(type)!.add(handler);
return () => {
const handlers = this.eventHandlers.get(type);
if (handlers) {
handlers.delete(handler);
if (handlers.size === 0) {
this.eventHandlers.delete(type);
}
}
};
}
public off(type: string, handler?: MessageHandler): void {
if (handler) {
const handlers = this.eventHandlers.get(type);
if (handlers) {
handlers.delete(handler);
if (handlers.size === 0) {
this.eventHandlers.delete(type);
}
}
} else {
this.eventHandlers.delete(type);
}
}
public send(data: SendData): boolean {
if (!this.instance || !this.isConnected) {
return false;
}
if (data instanceof ArrayBuffer || (data as unknown) instanceof Uint8Array) {
this.instance.send(data);
} else {
this.instance.send(JSON.stringify(data));
}
return true;
}
public get isConnected(): boolean {
return this.instance?.readyState === W3cWebSocket.OPEN;
}
private createConnection(): void {
this.cleanup();
this.instance = new W3cWebSocket(this.options.url);
this.instance.binaryType = 'arraybuffer';
this.instance.onopen = this.handleOpen.bind(this);
this.instance.onclose = this.handleClose.bind(this);
this.instance.onerror = this.handleError.bind(this);
this.instance.onmessage = this.handleMessage.bind(this);
}
private handleOpen(): void {
this.reconnectAttempts = 0;
this.startHeartbeat();
}
private handleClose(): void {
this.stopHeartbeat();
this.scheduleReconnect();
}
private handleError(error: Error): void {
console.error('[WebSocket] Error:', error);
}
private handleMessage(message: IMessageEvent): void {
try {
const data = JSON.parse(message.data as string);
const handlers = this.eventHandlers.get(data.type);
if (handlers) {
handlers.forEach((handler) => handler(message));
}
} catch (err) {
console.log(err);
}
}
private startHeartbeat(): void {
this.stopHeartbeat();
this.heartbeatTimer = setInterval(() => {
this.send(new Uint8Array([MessageEvent.Heartbeat]));
}, this.options.heartbeatInterval);
}
private stopHeartbeat(): void {
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
}
private scheduleReconnect(): void {
if (!this.shouldReconnect) {
return;
}
const message = JSON.stringify(data);
this.instance.send(message);
}
public close() {
if (this.instance.readyState === W3cWebSocket.OPEN) {
this.instance.close();
if (this.reconnectAttempts >= this.options.maxReconnectAttempts) {
console.error('[WebSocket] Max reconnect attempts reached');
return;
}
this.reconnectAttempts++;
console.log(`[WebSocket] Reconnecting... (attempt ${this.reconnectAttempts})`);
this.reconnectTimer = setTimeout(() => {
this.createConnection();
}, this.options.reconnectInterval);
}
public register(type: string, fn: (message: IMessageEvent) => void) {
eventMap.set(type, fn);
private cleanup(): void {
this.stopHeartbeat();
this.setEvents();
}
public unregister(type: string) {
eventMap.delete(type);
this.setEvents();
}
private setEvents() {
this.instance.onmessage = (message) => {
const data = JSON.parse(message.data as string);
if (!data) return;
const fn = eventMap.get(data.type);
if (!fn) return;
fn(message);
};
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
}
}

16
web/src/pages/desktop/index.tsx
View File

@@ -1,20 +1,19 @@
import { useEffect } from 'react';
import { useAtom, useAtomValue } from 'jotai';
import { useAtom } from 'jotai';
import { useTranslation } from 'react-i18next';
import { useMediaQuery } from 'react-responsive';
import * as storage from '@/lib/localstorage.ts';
import { client } from '@/lib/websocket.ts';
import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts';
import { resolutionAtom, videoModeAtom } from '@/jotai/screen.ts';
import { Head } from '@/components/head.tsx';
import { Keyboard } from './keyboard';
import { VirtualKeyboard } from './keyboard/virtual-keyboard';
import { Menu } from './menu';
import { Mouse } from './mouse';
import { Notification } from './notification.tsx';
import { Screen } from './screen';
import { VirtualKeyboard } from './virtual-keyboard';
export const Desktop = () => {
const { t } = useTranslation();
@@ -22,22 +21,17 @@ export const Desktop = () => {
const [videoMode, setVideoMode] = useAtom(videoModeAtom);
const [resolution, setResolution] = useAtom(resolutionAtom);
const isKeyboardEnable = useAtomValue(isKeyboardEnableAtom);
useEffect(() => {
client.connect();
const mode = getVideoMode();
setVideoMode(mode);
const res = storage.getResolution() || { width: 0, height: 0 };
setResolution(res);
const timer = setInterval(() => {
client.send([0]);
}, 60 * 1000);
return () => {
clearInterval(timer);
client.unregister('stream');
client.close();
};
}, []);
@@ -67,7 +61,7 @@ export const Desktop = () => {
<Menu />
<Screen />
<Mouse />
{isKeyboardEnable && <Keyboard />}
<Keyboard />
</>
)}

164
web/src/pages/desktop/keyboard/index.tsx
View File

@@ -1,102 +1,98 @@
import { useEffect, useRef } from 'react';
import { useAtomValue } from 'jotai';
import { client } from '@/lib/websocket.ts';
import { KeyboardCodes } from './mappings.ts';
import { KeyboardReport } from '@/lib/keyboard.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts';
export const Keyboard = () => {
const pressedKeys = useRef<Set<string>>(new Set());
const isKeyboardEnabled = useAtomValue(isKeyboardEnableAtom);
const keyboardRef = useRef(new KeyboardReport());
const pressedKeys = useRef(new Set<string>());
// listen keyboard events
useEffect(() => {
window.addEventListener('keydown', handleKeyDown);
window.addEventListener('keyup', handleKeyUp);
window.addEventListener('blur', releaseAllKeys);
document.addEventListener('visibilitychange', handleVisibilityChange);
// press button
function handleKeyDown(event: KeyboardEvent) {
disableEvent(event);
if (!pressedKeys.current.has(event.code)) {
pressedKeys.current.add(event.code);
}
sendKeyDown(event);
}
// release button
function handleKeyUp(event: KeyboardEvent) {
disableEvent(event);
if (pressedKeys.current.has(event.code)) {
pressedKeys.current.delete(event.code);
}
sendKeyUp();
}
function releaseAllKeys() {
if (pressedKeys.current.size === 0) {
return;
}
sendKeyUp();
pressedKeys.current.clear();
}
function handleVisibilityChange() {
if (document.hidden) {
releaseAllKeys();
}
}
return () => {
window.removeEventListener('keydown', handleKeyDown);
window.removeEventListener('keyup', handleKeyUp);
window.removeEventListener('blur', releaseAllKeys);
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, []);
function sendKeyDown(event: KeyboardEvent) {
const code = KeyboardCodes.get(event.code);
if (!code) {
console.log('unknown code: ', event.code);
if (!isKeyboardEnabled) {
releaseKeys();
return;
}
let ctrl = 0;
if (event.ctrlKey) {
if (pressedKeys.current.has('ControlLeft')) {
ctrl = 1;
} else if (pressedKeys.current.has('ControlRight')) {
ctrl = 16;
} else if (pressedKeys.current.has('AltRight')) {
ctrl = 0;
} else {
ctrl = 1;
document.addEventListener('keydown', handleKeyDown);
document.addEventListener('keyup', handleKeyUp);
window.addEventListener('blur', handleBlur);
document.addEventListener('visibilitychange', handleVisibilityChange);
// Key down event
function handleKeyDown(event: KeyboardEvent) {
if (!isKeyboardEnabled) return;
event.preventDefault();
event.stopPropagation();
const code = event.code;
if (pressedKeys.current.has(code)) {
return;
}
pressedKeys.current.add(code);
handleKeyEvent({ type: 'keydown', code });
}
// Key up event
function handleKeyUp(event: KeyboardEvent) {
if (!isKeyboardEnabled) return;
event.preventDefault();
event.stopPropagation();
const code = event.code;
pressedKeys.current.delete(code);
handleKeyEvent({ type: 'keyup', code });
}
// Release all keys when window loses focus
function handleBlur() {
releaseKeys();
}
// Release all keys before window closes
function handleVisibilityChange() {
if (document.hidden) {
releaseKeys();
}
}
const modifiers = [
ctrl,
event.shiftKey ? (pressedKeys.current.has('ShiftRight') ? 32 : 2) : 0,
event.altKey ? (pressedKeys.current.has('AltRight') ? 64 : 4) : 0,
event.metaKey ? (pressedKeys.current.has('MetaRight') ? 128 : 8) : 0
];
// Release all keys
function releaseKeys() {
pressedKeys.current.forEach((code) => {
handleKeyEvent({ type: 'keyup', code });
});
client.send([1, code, ...modifiers]);
pressedKeys.current.clear();
const report = keyboardRef.current.reset();
sendReport(report);
}
return () => {
document.removeEventListener('keydown', handleKeyDown);
document.removeEventListener('keyup', handleKeyUp);
window.removeEventListener('blur', handleBlur);
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, [isKeyboardEnabled]);
// Keyboard handler
function handleKeyEvent(event: { type: 'keydown' | 'keyup'; code: string }) {
const kb = keyboardRef.current;
const report = event.type === 'keydown' ? kb.keyDown(event.code) : kb.keyUp(event.code);
sendReport(report);
}
function sendKeyUp() {
client.send([1, 0, 0, 0, 0, 0]);
}
// disable the default keyboard events
function disableEvent(event: KeyboardEvent) {
event.preventDefault();
event.stopPropagation();
// Send keyboard report
function sendReport(report: Uint8Array) {
const data = new Uint8Array([MessageEvent.Keyboard, ...report]);
client.send(data);
}
return <></>;

168
web/src/pages/desktop/keyboard/mappings.ts
View File

@@ -1,168 +0,0 @@
export const KeyboardCodes: Map<string, number> = new Map([
['KeyA', 4],
['KeyB', 5],
['KeyC', 6],
['KeyD', 7],
['KeyE', 8],
['KeyF', 9],
['KeyG', 10],
['KeyH', 11],
['KeyI', 12],
['KeyJ', 13],
['KeyK', 14],
['KeyL', 15],
['KeyM', 16],
['KeyN', 17],
['KeyO', 18],
['KeyP', 19],
['KeyQ', 20],
['KeyR', 21],
['KeyS', 22],
['KeyT', 23],
['KeyU', 24],
['KeyV', 25],
['KeyW', 26],
['KeyX', 27],
['KeyY', 28],
['KeyZ', 29],
['RusA', 4],
['RusB', 5],
['RusC', 6],
['RusD', 7],
['RusE', 8],
['RusF', 9],
['RusG', 10],
['RusH', 11],
['RusI', 12],
['RusJ', 13],
['RusK', 14],
['RusL', 15],
['RusM', 16],
['RusN', 17],
['RusO', 18],
['RusP', 19],
['RusQ', 20],
['RusR', 21],
['RusS', 22],
['RusT', 23],
['RusU', 24],
['RusV', 25],
['RusW', 26],
['RusX', 27],
['RusY', 28],
['RusZ', 29],
['RusBracketLeft', 47],
['RusBracketRight', 48],
['RusBackslash', 49],
['RusSemicolon', 51],
['RusQuote', 52],
['RusComma', 54],
['RusPeriod', 55],
['RusSlash', 56],
['Digit1', 30],
['Digit2', 31],
['Digit3', 32],
['Digit4', 33],
['Digit5', 34],
['Digit6', 35],
['Digit7', 36],
['Digit8', 37],
['Digit9', 38],
['Digit0', 39],
['Enter', 40],
['Escape', 41],
['Backspace', 42],
['Tab', 43],
['Space', 44],
['Minus', 45],
['Equal', 46],
['BracketLeft', 47],
['BracketRight', 48],
['Backslash', 49],
['IntlBackslash', 49],
['Backquote_azerty', 100],
['IntlBackslash_qwertz', 100],
['Semicolon', 51],
['Quote', 52],
['Backquote', 53],
['KeyTilde', 53],
['Comma', 54],
['Period', 55],
['KeyDot', 55],
['Slash', 56],
['CapsLock', 57],
['F1', 58],
['F2', 59],
['F3', 60],
['F4', 61],
['F5', 62],
['F6', 63],
['F7', 64],
['F8', 65],
['F9', 66],
['F10', 67],
['F11', 68],
['F12', 69],
['F13', 70],
['PrintScreen', 70],
['ScrollLock', 71],
['Pause', 72],
['Insert', 73],
['Home', 74],
['PageUp', 75],
['Delete', 76],
['End', 77],
['PageDown', 78],
['ArrowRight', 79],
['ArrowLeft', 80],
['ArrowDown', 81],
['ArrowUp', 82],
['NumLock', 83],
['NumpadDivide', 84],
['NumpadMultiply', 85],
['NumpadSubtract', 86],
['NumpadAdd', 87],
['NumpadEnter', 88],
['Numpad1', 89],
['Numpad2', 90],
['Numpad3', 91],
['Numpad4', 92],
['Numpad5', 93],
['Numpad6', 94],
['Numpad7', 95],
['Numpad8', 96],
['Numpad9', 97],
['Numpad0', 98],
['NumpadDecimal', 99],
['KeyKpDot', 99],
['Menu', 118],
['ControlLeft', 224],
['ShiftLeft', 225],
['AltLeft', 226],
['MetaLeft', 227],
['ControlRight', 228],
['ShiftRight', 229],
['AltRight', 230],
['MetaRight', 231]
]);
export const ModifierCodes: Map<string, number> = new Map([
['ControlLeft', 1],
['ShiftLeft', 2],
['AltLeft', 4],
['MetaLeft', 8],
['ControlRight', 16],
['ShiftRight', 32],
['AltRight', 64],
['MetaRight', 128]
]);

29
web/src/pages/desktop/menu/keyboard/ctrl-alt-del.tsx
View File

@@ -1,26 +1,33 @@
import clsx from 'clsx';
import { useTranslation } from 'react-i18next';
import { OctagonMinus } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { KeyboardCodes, ModifierCodes } from '@/pages/desktop/keyboard/mappings.ts';
import { client } from '@/lib/websocket.ts';
import { getKeycode, getModifierBit } from '@/lib/keymap.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
export const CtrlAltDel = () => {
const { t } = useTranslation();
function sendCtrlAltDel() {
const ctrl = ModifierCodes.get('ControlLeft')!;
const alt = ModifierCodes.get('AltLeft')!;
const del = KeyboardCodes.get('Delete')!;
const ctrl = getModifierBit('ControlLeft')!;
const alt = getModifierBit('AltLeft')!;
const modifier = ctrl | alt;
client.send([1, del, ctrl, 0, alt, 0]);
client.send([1, 0, 0, 0, 0, 0]);
};
const del = getKeycode('Delete')!;
send(modifier, del);
send(0, 0);
}
function send(modifier: number, code: number) {
const data = new Uint8Array([MessageEvent.Keyboard, modifier, 0, code, 0, 0, 0, 0, 0]);
client.send(data);
}
return (
<div
className={clsx(
"flex cursor-pointer select-none items-center space-x-2 rounded py-1 pl-2 pr-5 hover:bg-neutral-700/70"
'flex cursor-pointer select-none items-center space-x-2 rounded py-1 pl-2 pr-5 hover:bg-neutral-700/70'
)}
onClick={sendCtrlAltDel}
>
@@ -28,4 +35,4 @@ export const CtrlAltDel = () => {
<span>{t('keyboard.ctrlaltdel')}</span>
</div>
);
}
};

7
web/src/pages/desktop/menu/mouse/reset-hid.tsx
View File

@@ -4,7 +4,7 @@ import { RefreshCwIcon } from 'lucide-react';
import { useTranslation } from 'react-i18next';
import * as api from '@/api/hid.ts';
import { client } from '@/lib/websocket.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
export const ResetHid = () => {
const { t } = useTranslation();
@@ -15,7 +15,10 @@ export const ResetHid = () => {
if (isResetting) return;
setIsResetting(true);
client.send([1, 0, 0, 0, 0, 0]);
// Release keyboard keys
const data = new Uint8Array([MessageEvent.Keyboard, 0, 0, 0, 0, 0, 0, 0, 0]);
client.send(data);
client.close();
api.reset().finally(() => {

338
web/src/pages/desktop/mouse/absolute.tsx
View File

@@ -1,95 +1,267 @@
import { useEffect, useRef } from 'react';
import { useAtomValue } from 'jotai';
import { useMediaQuery } from 'react-responsive';
import { client } from '@/lib/websocket.ts';
import { MouseReportAbsolute } from '@/lib/mouse.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
import { scrollDirectionAtom, scrollIntervalAtom } from '@/jotai/mouse.ts';
import { resolutionAtom } from '@/jotai/screen.ts';
import { MouseButton, MouseEvent } from './constants';
import { MouseAbsoluteEvent } from './types.ts';
enum MouseButton {
Left = 0,
Middle = 1,
Right = 2,
Back = 3,
Forward = 4
}
export const Absolute = () => {
const isBigScreen = useMediaQuery({ minWidth: 650 });
const resolution = useAtomValue(resolutionAtom);
const scrollDirection = useAtomValue(scrollDirectionAtom);
const scrollInterval = useAtomValue(scrollIntervalAtom);
const mouseRef = useRef(new MouseReportAbsolute());
const lastPosRef = useRef({ x: 0.5, y: 0.5 });
const lastScrollTimeRef = useRef(0);
const mouseButtonMapping = (button: number) => {
const mappings = [MouseButton.Left, MouseButton.Wheel, MouseButton.Right];
return mappings[button] || MouseButton.None;
};
// For touch events
const touchStartTimeRef = useRef(0);
const lastTouchYRef = useRef(0);
const longPressTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const isLongPressRef = useRef(false);
const hasMoveRef = useRef(false);
const isDraggingRef = useRef(false);
const pressedButtonRef = useRef<MouseButton | null>(null);
const touchStartPosRef = useRef({ x: 0, y: 0 });
const TAP_THRESHOLD = 8;
const DRAG_THRESHOLD = 10;
const VELOCITY_THRESHOLD = 0.3;
// listen mouse events
useEffect(() => {
const canvas = document.getElementById('screen') as HTMLVideoElement;
if (!canvas) return;
const screen = document.getElementById('screen') as HTMLVideoElement;
if (!screen) return;
canvas.addEventListener('mousedown', handleMouseDown);
canvas.addEventListener('mouseup', handleMouseUp);
canvas.addEventListener('mousemove', handleMouseMove);
canvas.addEventListener('wheel', handleWheel);
canvas.addEventListener('click', disableEvent);
canvas.addEventListener('contextmenu', disableEvent);
screen.addEventListener('mousedown', handleMouseDown);
screen.addEventListener('mouseup', handleMouseUp);
screen.addEventListener('mousemove', handleMouseMove);
screen.addEventListener('wheel', handleWheel);
screen.addEventListener('click', disableEvent);
screen.addEventListener('contextmenu', disableEvent);
// press button
function handleMouseDown(event: any) {
disableEvent(event);
const button: MouseButton = mouseButtonMapping(event.button);
if (button === MouseButton.None) return;
const data = [2, MouseEvent.Down, button, 0, 0];
client.send(data);
if (isBigScreen) {
screen.addEventListener('touchstart', handleTouchStart);
screen.addEventListener('touchmove', handleTouchMove);
screen.addEventListener('touchend', handleTouchEnd);
screen.addEventListener('touchcancel', handleTouchCancel);
}
// release button
function handleMouseUp(event: any) {
disableEvent(event);
const data = [2, MouseEvent.Up, MouseButton.None, 0, 0];
client.send(data);
// Mouse down event
function handleMouseDown(e: MouseEvent) {
disableEvent(e);
handleMouseEvent({ type: 'mousedown', button: e.button });
}
// mouse move
function handleMouseMove(event: any) {
disableEvent(event);
const { x, y } = getCoordinate(event);
const data = [2, MouseEvent.MoveAbsolute, MouseButton.None, x, y];
client.send(data);
// Mouse up event
function handleMouseUp(e: MouseEvent) {
disableEvent(e);
handleMouseEvent({ type: 'mouseup', button: e.button });
}
// mouse scroll
function handleWheel(event: any) {
disableEvent(event);
// Mouse move event
function handleMouseMove(e: MouseEvent) {
disableEvent(e);
const { x, y } = getCoordinate(e);
handleMouseEvent({ type: 'move', x, y });
}
if (Math.floor(event.deltaY) === 0) return;
// Mouse wheel event
function handleWheel(e: WheelEvent) {
disableEvent(e);
if (Math.floor(e.deltaY) === 0) {
return;
}
const currentTime = Date.now();
if (currentTime - lastScrollTimeRef.current < scrollInterval) {
return;
}
lastScrollTimeRef.current = currentTime;
const deltaY = (event.deltaY > 0 ? 1 : -1) * scrollDirection;
const data = [2, MouseEvent.Scroll, 0, 0, deltaY];
client.send(data);
const deltaY = (e.deltaY > 0 ? 1 : -1) * scrollDirection;
handleMouseEvent({ type: 'wheel', deltaY });
lastScrollTimeRef.current = currentTime;
}
// Mouse touch start event
function handleTouchStart(e: TouchEvent) {
disableEvent(e);
if (e.touches.length === 0) {
return;
}
const touch = e.touches[0];
// Reset states
touchStartTimeRef.current = Date.now();
lastTouchYRef.current = touch.clientY;
isLongPressRef.current = false;
hasMoveRef.current = false;
isDraggingRef.current = false;
pressedButtonRef.current = null;
touchStartPosRef.current = { x: touch.clientX, y: touch.clientY };
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
}
const { x, y } = getCoordinate(touch);
handleMouseEvent({ type: 'move', x, y });
if (e.touches.length > 1) {
return;
}
// Start long press
longPressTimerRef.current = setTimeout(() => {
isLongPressRef.current = true;
pressedButtonRef.current = MouseButton.Right;
if (navigator.vibrate) {
navigator.vibrate(50);
}
handleMouseEvent({ type: 'mousedown', button: MouseButton.Right });
}, 800);
}
// Mouse touch move event
function handleTouchMove(e: TouchEvent) {
disableEvent(e);
if (e.touches.length === 0) {
return;
}
const touch = e.touches[0];
const { x, y } = getCoordinate(touch);
// Handle two-finger scroll first
if (e.touches.length > 1) {
const deltaY = (touch.clientY - lastTouchYRef.current > 0 ? 1 : -1) * scrollDirection;
lastTouchYRef.current = touch.clientY;
if (Math.abs(deltaY) > 2) {
handleMouseEvent({ type: 'wheel', deltaY });
}
return;
}
const deltaX = Math.abs(touch.clientX - touchStartPosRef.current.x);
const deltaY = Math.abs(touch.clientY - touchStartPosRef.current.y);
const distance = Math.sqrt(deltaX * deltaX + deltaY * deltaY);
const timeDelta = Date.now() - touchStartTimeRef.current;
const velocity = timeDelta > 0 ? distance / timeDelta : 0;
const shouldStartDrag =
distance > DRAG_THRESHOLD || (distance > TAP_THRESHOLD && velocity > VELOCITY_THRESHOLD);
if (shouldStartDrag && !isDraggingRef.current && !isLongPressRef.current) {
if (!hasMoveRef.current) {
hasMoveRef.current = true;
}
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
longPressTimerRef.current = null;
}
if (pressedButtonRef.current === null) {
isDraggingRef.current = true;
pressedButtonRef.current = MouseButton.Left;
handleMouseEvent({ type: 'mousedown', button: MouseButton.Left });
}
}
if (distance > TAP_THRESHOLD && !hasMoveRef.current) {
hasMoveRef.current = true;
}
if (isDraggingRef.current || isLongPressRef.current) {
handleMouseEvent({ type: 'move', x, y });
}
}
// Mouse touch end event
function handleTouchEnd(e: TouchEvent) {
disableEvent(e);
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
longPressTimerRef.current = null;
}
if (!hasMoveRef.current && !isLongPressRef.current) {
handleMouseEvent({ type: 'mousedown', button: MouseButton.Left });
setTimeout(() => {
handleMouseEvent({ type: 'mouseup', button: MouseButton.Left });
}, 50);
} else if (pressedButtonRef.current !== null) {
handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! });
}
isLongPressRef.current = false;
hasMoveRef.current = false;
isDraggingRef.current = false;
pressedButtonRef.current = null;
}
// Mouse touch cancel event
function handleTouchCancel(e: any) {
disableEvent(e);
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
longPressTimerRef.current = null;
}
if (pressedButtonRef.current) {
handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! });
}
isLongPressRef.current = false;
hasMoveRef.current = false;
isDraggingRef.current = false;
pressedButtonRef.current = null;
}
// get mouse coordinate
function getCoordinate(event: any) {
const { x, y } = getCorrectedCoords(event.clientX, event.clientY);
const finalX = Math.max(0, Math.min(1, x));
const finalY = Math.max(0, Math.min(1, y));
const hexX = Math.floor(0x7fff * finalX) + 0x0001;
const hexY = Math.floor(0x7fff * finalY) + 0x0001;
return { x: hexX, y: hexY };
}
function getCorrectedCoords(clientX: number, clientY: number) {
if (!canvas) {
return { x: 0, y: 0 };
}
const rect = screen.getBoundingClientRect();
const rect = canvas.getBoundingClientRect();
if (!canvas.videoWidth || !canvas.videoHeight) {
if (!screen.videoWidth || !screen.videoHeight) {
const x = (clientX - rect.left) / rect.width;
const y = (clientY - rect.top) / rect.height;
return { x, y };
}
const videoRatio = canvas.videoWidth / canvas.videoHeight;
const videoRatio = screen.videoWidth / screen.videoHeight;
const elementRatio = rect.width / rect.height;
let renderedWidth = rect.width;
@@ -111,27 +283,57 @@ export const Absolute = () => {
return { x, y };
}
function getCoordinate(event: any): { x: number; y: number } {
const { x, y } = getCorrectedCoords(event.clientX, event.clientY);
return () => {
screen.removeEventListener('mousemove', handleMouseMove);
screen.removeEventListener('mousedown', handleMouseDown);
screen.removeEventListener('mouseup', handleMouseUp);
screen.removeEventListener('wheel', handleWheel);
screen.removeEventListener('click', disableEvent);
screen.removeEventListener('contextmenu', disableEvent);
screen.removeEventListener('touchstart', handleTouchStart);
screen.removeEventListener('touchmove', handleTouchMove);
screen.removeEventListener('touchend', handleTouchEnd);
screen.removeEventListener('touchcancel', handleTouchCancel);
const finalX = Math.max(0, Math.min(1, x));
const finalY = Math.max(0, Math.min(1, y));
if (longPressTimerRef.current) {
clearTimeout(longPressTimerRef.current);
}
};
}, [isBigScreen, resolution, scrollDirection, scrollInterval]);
const hexX = Math.floor(0x7fff * finalX) + 0x0001;
const hexY = Math.floor(0x7fff * finalY) + 0x0001;
// Mouse event handler
function handleMouseEvent(event: MouseAbsoluteEvent) {
let report: Uint8Array;
const mouse = mouseRef.current;
return { x: hexX, y: hexY };
switch (event.type) {
case 'mousedown':
mouse.buttonDown(event.button);
report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y);
break;
case 'mouseup':
mouse.buttonUp(event.button);
report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y);
break;
case 'wheel':
report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y, event.deltaY);
break;
case 'move':
report = mouse.buildReport(event.x, event.y);
lastPosRef.current = { x: event.x, y: event.y };
break;
default:
report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y);
break;
}
return () => {
canvas.removeEventListener('mousemove', handleMouseMove);
canvas.removeEventListener('mousedown', handleMouseDown);
canvas.removeEventListener('mouseup', handleMouseUp);
canvas.removeEventListener('wheel', handleWheel);
canvas.removeEventListener('click', disableEvent);
canvas.removeEventListener('contextmenu', disableEvent);
};
}, [resolution, scrollDirection, scrollInterval]);
sendReport(report);
}
function sendReport(report: Uint8Array) {
const data = new Uint8Array([MessageEvent.Mouse, ...report]);
client.send(data);
}
// disable default events
function disableEvent(event: any) {

14
web/src/pages/desktop/mouse/constants.ts
View File

@@ -1,14 +0,0 @@
export enum MouseEvent {
Up = 0,
Down = 1,
MoveAbsolute = 2,
MoveRelative = 3,
Scroll = 4
}
export enum MouseButton {
None = 0,
Left = 1,
Right = 2,
Wheel = 4
}

226
web/src/pages/desktop/mouse/relative.tsx
View File

@@ -3,11 +3,12 @@ import { message } from 'antd';
import { useAtomValue } from 'jotai';
import { useTranslation } from 'react-i18next';
import { client } from '@/lib/websocket.ts';
import { MouseReportRelative } from '@/lib/mouse.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
import { scrollDirectionAtom, scrollIntervalAtom } from '@/jotai/mouse.ts';
import { resolutionAtom } from '@/jotai/screen.ts';
import { MouseButton, MouseEvent } from './constants';
import { MouseRelativeEvent } from './types.ts';
export const Relative = () => {
const { t } = useTranslation();
@@ -17,17 +18,125 @@ export const Relative = () => {
const scrollDirection = useAtomValue(scrollDirectionAtom);
const scrollInterval = useAtomValue(scrollIntervalAtom);
const mouseRef = useRef(new MouseReportRelative());
const isLockedRef = useRef(false);
const buttonRef = useRef<MouseButton>(MouseButton.None);
const lastScrollTimeRef = useRef(0);
// listen mouse events
useEffect(() => {
const canvas = document.getElementById('screen');
if (!canvas) return;
const screen = document.getElementById('screen');
if (!screen) return;
showMessage();
screen.addEventListener('click', handleMouseClick);
screen.addEventListener('mousedown', handleMouseDown);
screen.addEventListener('mouseup', handleMouseUp);
screen.addEventListener('mousemove', handleMouseMove);
screen.addEventListener('wheel', handleMouseWheel, { passive: false });
screen.addEventListener('contextmenu', (e) => e.preventDefault());
document.addEventListener('pointerlockchange', handlePointerLockChange);
// Mouse click event
function handleMouseClick(event: MouseEvent) {
disableEvent(event);
if (!isLockedRef.current) {
screen?.requestPointerLock();
}
}
// Mouse down event
function handleMouseDown(e: MouseEvent) {
disableEvent(e);
handleMouseEvent({ type: 'mousedown', button: e.button });
}
// Mouse up event
function handleMouseUp(e: MouseEvent) {
disableEvent(e);
handleMouseEvent({ type: 'mouseup', button: e.button });
}
// Mouse move event
function handleMouseMove(e: any) {
disableEvent(e);
const x = e.movementX || e.mozMovementX || e.webkitMovementX || 0;
const y = e.movementY || e.mozMovementY || e.webkitMovementY || 0;
if (x === 0 && y === 0) return;
const deltaX = Math.abs(x * window.devicePixelRatio) < 10 ? x * 2 : x;
const deltaY = Math.abs(y * window.devicePixelRatio) < 10 ? y * 2 : y;
handleMouseEvent({ type: 'move', deltaX, deltaY });
}
// Mouse wheel event
function handleMouseWheel(e: WheelEvent) {
disableEvent(e);
if (Math.floor(e.deltaY) === 0) {
return;
}
const currentTime = Date.now();
if (currentTime - lastScrollTimeRef.current < scrollInterval) {
return;
}
const deltaY = (e.deltaY > 0 ? 1 : -1) * scrollDirection;
handleMouseEvent({ type: 'wheel', deltaY });
lastScrollTimeRef.current = currentTime;
}
function handlePointerLockChange() {
isLockedRef.current = document.pointerLockElement === screen;
}
return () => {
screen.removeEventListener('click', handleMouseClick);
screen.removeEventListener('mousemove', handleMouseMove);
screen.removeEventListener('mousedown', handleMouseDown);
screen.removeEventListener('mouseup', handleMouseUp);
screen.removeEventListener('wheel', handleMouseWheel);
screen.removeEventListener('contextmenu', disableEvent);
document.removeEventListener('pointerlockchange', handlePointerLockChange);
};
}, [resolution, scrollDirection, scrollInterval]);
// Mouse handler
function handleMouseEvent(event: MouseRelativeEvent) {
let report: Uint8Array;
const mouse = mouseRef.current;
switch (event.type) {
case 'mousedown':
mouse.buttonDown(event.button);
report = mouse.buildButtonReport();
break;
case 'mouseup':
mouse.buttonUp(event.button);
report = mouse.buildButtonReport();
break;
case 'wheel':
report = mouse.buildReport(0, 0, event.deltaY);
break;
case 'move':
report = mouse.buildReport(event.deltaX, event.deltaY);
break;
default:
report = mouse.buildReport(0, 0);
break;
}
const data = new Uint8Array([MessageEvent.Mouse, ...report]);
client.send(data);
}
// show message
function showMessage() {
messageApi.open({
key: 'no_mouse_relative',
key: 'requestPointer',
type: 'info',
content: t('mouse.requestPointer'),
duration: 3,
@@ -35,108 +144,7 @@ export const Relative = () => {
marginTop: '40vh'
}
});
document.addEventListener('pointerlockchange', handlePointerLockChange);
canvas.addEventListener('click', handleClick);
canvas.addEventListener('mousedown', handleMouseDown);
canvas.addEventListener('mouseup', handleMouseUp);
canvas.addEventListener('mousemove', handleMouseMove);
canvas.addEventListener('wheel', handleWheel);
canvas.addEventListener('contextmenu', disableEvent);
function handleClick(event: any) {
disableEvent(event);
if (!isLockedRef.current) {
canvas!.requestPointerLock();
}
}
function handlePointerLockChange() {
isLockedRef.current = document.pointerLockElement === canvas;
}
// press button
function handleMouseDown(event: any) {
disableEvent(event);
let button: MouseButton;
switch (event.button) {
case 0:
button = MouseButton.Left;
break;
case 1:
button = MouseButton.Wheel;
break;
case 2:
button = MouseButton.Right;
break;
default:
console.log(`unknown button ${event.button}`);
return;
}
buttonRef.current = button;
const data = [2, MouseEvent.Down, button, 0, 0];
client.send(data);
}
// release button
function handleMouseUp(event: any) {
disableEvent(event);
buttonRef.current = MouseButton.None;
const data = [2, MouseEvent.Up, MouseButton.None, 0, 0];
client.send(data);
}
// mouse move
function handleMouseMove(event: any) {
disableEvent(event);
const x = event.movementX || event.mozMovementX || event.webkitMovementX || 0;
const y = event.movementY || event.mozMovementY || event.webkitMovementY || 0;
if (x === 0 && y === 0) return;
const data = [
2,
MouseEvent.MoveRelative,
buttonRef.current,
Math.abs(x) < 10 ? x * 2 : x,
Math.abs(y) < 10 ? y * 2 : y
];
client.send(data);
}
// mouse scroll
function handleWheel(event: any) {
disableEvent(event);
if (Math.floor(event.deltaY) === 0) return;
const currentTime = Date.now();
if (currentTime - lastScrollTimeRef.current < scrollInterval) {
return;
}
lastScrollTimeRef.current = currentTime;
const deltaY = (event.deltaY > 0 ? 1 : -1) * scrollDirection;
const data = [2, MouseEvent.Scroll, 0, 0, deltaY];
client.send(data);
}
return () => {
document.removeEventListener('pointerlockchange', handlePointerLockChange);
canvas.removeEventListener('click', handleClick);
canvas.removeEventListener('mousemove', handleMouseMove);
canvas.removeEventListener('mousedown', handleMouseDown);
canvas.removeEventListener('mouseup', handleMouseUp);
canvas.removeEventListener('wheel', handleWheel);
canvas.removeEventListener('contextmenu', disableEvent);
};
}, [resolution, scrollDirection, scrollInterval]);
}
// disable default events
function disableEvent(event: any) {

25
web/src/pages/desktop/mouse/types.ts Normal file
View File

@@ -0,0 +1,25 @@
interface MouseMoveAbsoluteEvent {
type: 'move';
x: number;
y: number;
}
interface MouseMoveRelativeEvent {
type: 'move';
deltaX: number;
deltaY: number;
}
interface MouseButtonEvent {
type: 'mousedown' | 'mouseup';
button: number;
}
interface MouseWheelEvent {
type: 'wheel';
deltaY: number;
}
export type MouseAbsoluteEvent = MouseMoveAbsoluteEvent | MouseButtonEvent | MouseWheelEvent;
export type MouseRelativeEvent = MouseMoveRelativeEvent | MouseButtonEvent | MouseWheelEvent;

75
web/src/pages/desktop/keyboard/virtual-keyboard.tsx → web/src/pages/desktop/virtual-keyboard/index.tsx
View File

@@ -12,11 +12,11 @@ import '@/assets/styles/keyboard.css';
import { ConfigProvider, Segmented, Select, theme } from 'antd';
import { useMediaQuery } from 'react-responsive';
import { getKeycode, getModifierBit } from '@/lib/keymap.ts';
import * as storage from '@/lib/localstorage.ts';
import { client } from '@/lib/websocket.ts';
import { client, MessageEvent } from '@/lib/websocket.ts';
import { isKeyboardOpenAtom } from '@/jotai/keyboard.ts';
import { KeyboardCodes, ModifierCodes } from './mappings.ts';
import {
doubleKeys,
keyboardArrowsOptions,
@@ -83,6 +83,7 @@ export const VirtualKeyboard = () => {
setKeyboardLayout('default');
}, [keyboardSystem, keyboardLanguage]);
// Press key
function onKeyPress(key: string) {
if (modifierKeys.includes(key)) {
if (activeModifierKeys.includes(key)) {
@@ -97,6 +98,7 @@ export const VirtualKeyboard = () => {
sendKeydown(key);
}
// Release key
function onKeyReleased(key: string) {
if (modifierKeys.includes(key)) {
return;
@@ -105,92 +107,87 @@ export const VirtualKeyboard = () => {
sendKeyup();
}
// Send all keys
function sendKeydown(key: string) {
const code = getKeyCode(key);
const code = getKeyboardCode(key);
if (!code) {
console.log('unknown code: ', key);
return;
}
const modifiers = sendModifierKeyDown();
const modifier = sendModifierKeyDown();
client.send([1, code, ...modifiers]);
send(modifier, code);
}
function getKeyCode(key: string) {
function getKeyboardCode(key: string) {
// AZERTY: swap A↔Q and Z↔W on French physical positions
if (keyboardLanguage === 'fr' && key.endsWith('_azerty')) {
const base = key.replace('_azerty', '');
if (base === 'KeyA') return KeyboardCodes.get('KeyQ');
if (base === 'KeyQ') return KeyboardCodes.get('KeyA');
if (base === 'KeyZ') return KeyboardCodes.get('KeyW');
if (base === 'KeyW') return KeyboardCodes.get('KeyZ');
if (base === 'KeyA') return getKeycode('KeyQ');
if (base === 'KeyQ') return getKeycode('KeyA');
if (base === 'KeyZ') return getKeycode('KeyW');
if (base === 'KeyW') return getKeycode('KeyZ');
// all other labels use their own code
return KeyboardCodes.get(base);
return getKeycode(base);
}
if (keyboardLanguage === 'de' && key.endsWith('_qwertz')) {
const base = key.replace('_qwertz', '');
// Tausch
if (base === 'KeyZ') return KeyboardCodes.get('KeyY');
if (base === 'KeyY') return KeyboardCodes.get('KeyZ');
if (base === 'IntlBackslash') return KeyboardCodes.get('IntlBackslash_qwertz');
if (base === 'KeyZ') return getKeycode('KeyY');
if (base === 'KeyY') return getKeycode('KeyZ');
if (base === 'IntlBackslash') return getKeycode('IntlBackslash_qwertz');
// all other labels use their own code
return KeyboardCodes.get(base);
return getKeycode(base);
}
const specialKey = specialKeyMap.get(key);
if (specialKey) {
return KeyboardCodes.get(specialKey);
return getKeycode(specialKey);
}
return KeyboardCodes.get(key);
return getKeycode(key);
}
// Release all keys
function sendKeyup() {
sendModifierKeyUp();
client.send([1, 0, 0, 0, 0, 0]);
send(0, 0);
}
// Send modifier keys
function sendModifierKeyDown() {
let ctrl = 0;
let shift = 0;
let alt = 0;
let meta = 0;
let modifier = 0;
activeModifierKeys.forEach((modifierKey) => {
const key = specialKeyMap.get(modifierKey)!;
const code = KeyboardCodes.get(key)!;
const modifier = ModifierCodes.get(key)!;
modifier |= getModifierBit(key)!;
const code = getKeycode(key)!;
if ([1, 16].includes(modifier)) {
ctrl = modifier;
} else if ([2, 32].includes(modifier)) {
shift = modifier;
} else if ([4, 64].includes(modifier)) {
alt = modifier;
} else if ([8, 128].includes(modifier)) {
meta = modifier;
}
client.send([1, code, ctrl, shift, alt, meta]);
send(modifier, code);
});
return [ctrl, shift, alt, meta];
return modifier;
}
// Release modifier keys
function sendModifierKeyUp() {
if (activeModifierKeys.length === 0) return;
activeModifierKeys.forEach(() => {
client.send([1, 0, 0, 0, 0, 0]);
send(0, 0);
});
setActiveModifierKeys([]);
}
function send(modifier: number, code: number) {
const data = new Uint8Array([MessageEvent.Keyboard, modifier, 0, code, 0, 0, 0, 0, 0]);
client.send(data);
}
function selectSystem(system: string) {
setKeyboardSystem(system);
storage.setKeyboardSystem(system);

0
web/src/pages/desktop/keyboard/virtual-keys.ts → web/src/pages/desktop/virtual-keyboard/virtual-keys.ts
View File