fix: harden HID recovery and cleanup

- Open HID gadget handles lazily in nonblocking mode and clear stale handles on write failures
- Release keyboard and mouse state when HID queues close or writes fail
- Reopen HID devices with retry after USB PHY reset
- Make websocket client close idempotent and close HID queues safely
- Expire and renew PicoClaw session locks during gateway relay

server/service/hid/hid.go

@@ -2,8 +2,11 @@ package hid
 
 import (
 	"errors"
+	"fmt"
+	"io"
 	"os"
 	"sync"
+	"syscall"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -23,6 +26,24 @@ const (
 	HID2 = "/dev/hidg2" // Touchpad (Absolute Mode)
 )
 
+const (
+	hidWriteTimeout     = 50 * time.Millisecond
+	hidWriteRetryDelay  = time.Millisecond
+	hidReopenTimeout    = 2 * time.Second
+	hidReopenRetryDelay = 100 * time.Millisecond
+)
+
+type hidWriter interface {
+	Write([]byte) (int, error)
+}
+
+type hidDevice struct {
+	path string
+	mu   *sync.Mutex
+	get  func() *os.File
+	set  func(*os.File)
+}
+
 var (
 	hid     *Hid
 	hidOnce sync.Once
@@ -45,33 +66,168 @@ func (h *Hid) Unlock() {
 	h.mouseMutex.Unlock()
 }
 
-func (h *Hid) OpenNoLock() {
-	var err error
-	h.CloseNoLock()
-
-	h.g0, err = os.OpenFile(HID0, os.O_WRONLY, 0o666)
-	if err != nil {
-		log.Errorf("open %s failed: %s", HID0, err)
-	}
-
-	h.g1, err = os.OpenFile(HID1, os.O_WRONLY, 0o666)
-	if err != nil {
-		log.Errorf("open %s failed: %s", HID1, err)
-	}
-
-	h.g2, err = os.OpenFile(HID2, os.O_WRONLY, 0o666)
-	if err != nil {
-		log.Errorf("open %s failed: %s", HID2, err)
+func (h *Hid) keyboardDevice(path string) hidDevice {
+	return hidDevice{
+		path: path,
+		mu:   &h.kbMutex,
+		get: func() *os.File {
+			return h.g0
+		},
+		set: func(file *os.File) {
+			h.g0 = file
+		},
 	}
 }
 
-func (h *Hid) CloseNoLock() {
-	for _, file := range []*os.File{h.g0, h.g1, h.g2} {
-		if file != nil {
-			_ = file.Sync()
-			_ = file.Close()
+func (h *Hid) relativeMouseDevice(path string) hidDevice {
+	return hidDevice{
+		path: path,
+		mu:   &h.mouseMutex,
+		get: func() *os.File {
+			return h.g1
+		},
+		set: func(file *os.File) {
+			h.g1 = file
+		},
+	}
+}
+
+func (h *Hid) absoluteMouseDevice(path string) hidDevice {
+	return hidDevice{
+		path: path,
+		mu:   &h.mouseMutex,
+		get: func() *os.File {
+			return h.g2
+		},
+		set: func(file *os.File) {
+			h.g2 = file
+		},
+	}
+}
+
+func (h *Hid) devices() []hidDevice {
+	return []hidDevice{
+		h.keyboardDevice(HID0),
+		h.relativeMouseDevice(HID1),
+		h.absoluteMouseDevice(HID2),
+	}
+}
+
+func (h *Hid) OpenNoLock() error {
+	h.CloseNoLock()
+
+	var errs []error
+	for _, device := range h.devices() {
+		if err := h.openDeviceNoLock(device); err != nil {
+			log.Errorf("open %s failed: %s", device.path, err)
+			errs = append(errs, err)
 		}
 	}
+
+	return errors.Join(errs...)
+}
+
+func (h *Hid) OpenNoLockWithRetry(timeout, delay time.Duration) error {
+	return openNoLockWithRetry(h.OpenNoLock, timeout, delay)
+}
+
+func openNoLockWithRetry(open func() error, timeout, delay time.Duration) error {
+	if timeout <= 0 {
+		return open()
+	}
+	if delay <= 0 {
+		delay = hidReopenRetryDelay
+	}
+
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for {
+		if err := open(); err != nil {
+			lastErr = err
+		} else {
+			return nil
+		}
+
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		if remaining > delay {
+			remaining = delay
+		}
+		time.Sleep(remaining)
+	}
+
+	return fmt.Errorf("open HID devices within %s: %w", timeout, lastErr)
+}
+
+func (h *Hid) CloseNoLock() {
+	for _, device := range h.devices() {
+		h.closeDeviceNoLock(device)
+	}
+}
+
+func (h *Hid) openDeviceNoLock(device hidDevice) error {
+	if device.get() != nil {
+		return nil
+	}
+
+	file, err := os.OpenFile(device.path, os.O_WRONLY|syscall.O_NONBLOCK, 0o666)
+	if err != nil {
+		return fmt.Errorf("%s: %w", device.path, err)
+	}
+
+	device.set(file)
+	return nil
+}
+
+func (h *Hid) closeDeviceNoLock(device hidDevice) {
+	file := device.get()
+	if file == nil {
+		return
+	}
+
+	device.set(nil)
+	if err := file.Close(); err != nil {
+		log.Debugf("close %s failed: %s", device.path, err)
+	}
+}
+
+// writeWithTimeout bounds how long callers hold HID locks when writing to a
+// nonblocking descriptor. EAGAIN means the host is not accepting HID reports
+// yet, so retry until the caller's deadline expires.
+func writeWithTimeout(writer hidWriter, data []byte, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	for {
+		n, err := writer.Write(data)
+		if err == nil {
+			if n != len(data) {
+				return io.ErrShortWrite
+			}
+			return nil
+		}
+
+		if n != 0 {
+			return io.ErrShortWrite
+		}
+		if !isRetryableWriteError(err) {
+			return err
+		}
+
+		remaining := time.Until(deadline)
+		if timeout <= 0 || remaining <= 0 {
+			return os.ErrDeadlineExceeded
+		}
+		if remaining > hidWriteRetryDelay {
+			remaining = hidWriteRetryDelay
+		}
+		time.Sleep(remaining)
+	}
+}
+
+func isRetryableWriteError(err error) bool {
+	return errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EWOULDBLOCK)
 }
 
 func (h *Hid) Open() {
@@ -80,8 +236,6 @@ func (h *Hid) Open() {
 	h.mouseMutex.Lock()
 	defer h.mouseMutex.Unlock()
 
-	h.CloseNoLock()
-
 	h.OpenNoLock()
 }
 
@@ -95,73 +249,55 @@ func (h *Hid) Close() {
 }
 
 func (h *Hid) WriteHid0(data []byte) {
-	deadline := time.Now().Add(8 * time.Millisecond)
-
-	h.kbMutex.Lock()
-	_ = h.g0.SetWriteDeadline(deadline)
-	_, err := h.g0.Write(data)
-	h.kbMutex.Unlock()
-
-	if err != nil {
-		switch {
-		case errors.Is(err, os.ErrClosed):
-			log.Errorf("hid already closed, reopen it...")
-			h.OpenNoLock()
-		case errors.Is(err, os.ErrDeadlineExceeded):
-			log.Debugf("write to %s timeout", HID0)
-		default:
-			log.Errorf("write to %s failed: %s", HID0, err)
-		}
-		return
-	}
-
-	log.Debugf("write to %s: %v", HID0, data)
+	h.writeHIDReport(h.keyboardDevice(HID0), data)
 }
 
 func (h *Hid) WriteHid1(data []byte) {
-	deadline := time.Now().Add(8 * time.Millisecond)
-
-	h.mouseMutex.Lock()
-	_ = h.g1.SetWriteDeadline(deadline)
-	_, err := h.g1.Write(data)
-	h.mouseMutex.Unlock()
-
-	if err != nil {
-		switch {
-		case errors.Is(err, os.ErrClosed):
-			log.Errorf("hid already closed, reopen it...")
-			h.OpenNoLock()
-		case errors.Is(err, os.ErrDeadlineExceeded):
-			log.Debugf("write to %s timeout", HID1)
-		default:
-			log.Errorf("write to %s failed: %s", HID1, err)
-		}
-		return
-	}
-
-	log.Debugf("write to %s: %v", HID1, data)
+	h.writeHIDReport(h.relativeMouseDevice(HID1), data)
 }
 
 func (h *Hid) WriteHid2(data []byte) {
-	deadline := time.Now().Add(8 * time.Millisecond)
+	h.writeHIDReport(h.absoluteMouseDevice(HID2), data)
+}
 
-	h.mouseMutex.Lock()
-	_ = h.g2.SetWriteDeadline(deadline)
-	_, err := h.g2.Write(data)
-	h.mouseMutex.Unlock()
+func (h *Hid) writeHIDReport(device hidDevice, data []byte) bool {
+	if err := h.writeHID(device, data); err != nil {
+		log.Errorf("write to %s failed: %s", device.path, err)
+		return false
+	}
+	return true
+}
 
-	if err != nil {
-		switch {
-		case errors.Is(err, os.ErrClosed):
-			log.Errorf("hid already closed, reopen it...")
-			h.OpenNoLock()
-		case errors.Is(err, os.ErrDeadlineExceeded):
-			log.Debugf("write to %s timeout", HID2)
-		default:
-			log.Errorf("write to %s failed: %s", HID2, err)
-		}
-		return
+func (h *Hid) writeHID(device hidDevice, data []byte) error {
+	device.mu.Lock()
+	defer device.mu.Unlock()
+
+	if err := h.openDeviceNoLock(device); err != nil {
+		return err
 	}
 
-	log.Debugf("write to %s: %v", HID2, data)
+	file := device.get()
+	if file == nil {
+		return fmt.Errorf("%s: hid handle is nil", device.path)
+	}
+
+	deadline := time.Now().Add(hidWriteTimeout)
+	if err := file.SetWriteDeadline(deadline); err != nil {
+		log.Debugf("set write deadline for %s failed: %s", device.path, err)
+	}
+
+	if err := writeWithTimeout(file, data, hidWriteTimeout); err != nil {
+		h.closeDeviceNoLock(device)
+		switch {
+		case errors.Is(err, os.ErrClosed):
+			return fmt.Errorf("hid already closed: %w", err)
+		case errors.Is(err, os.ErrDeadlineExceeded):
+			return fmt.Errorf("timeout after %s: %w", hidWriteTimeout, err)
+		default:
+			return err
+		}
+	}
+
+	log.Debugf("write to %s: %v", device.path, data)
+	return nil
 }

server/service/hid/keyboard.go

@@ -5,12 +5,46 @@ import (
 )
 
 func (h *Hid) Keyboard(queue <-chan []byte) {
+	h.keyboard(queue, HID0)
+}
+
+func (h *Hid) keyboard(queue <-chan []byte, path string) {
+	defer h.releaseKeyboard(path)
+
 	for event := range queue {
 		if len(event) != 8 {
 			log.Debugf("invalid keyboard event: %v", event)
 			continue
 		}
 
-		h.WriteHid0(event)
+		if !h.writeHIDReport(h.keyboardDevice(path), event) {
+			if dropped := drainHIDQueue(queue); dropped > 0 {
+				log.Debugf("dropped %d stale keyboard HID reports after write failure", dropped)
+			}
+			h.releaseKeyboard(path)
+		}
+	}
+}
+
+func (h *Hid) releaseKeyboard(path string) {
+	h.writeHIDReport(h.keyboardDevice(path), keyboardReleaseReport())
+}
+
+func keyboardReleaseReport() []byte {
+	return []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
+}
+
+func drainHIDQueue(queue <-chan []byte) int {
+	dropped := 0
+	for {
+		select {
+		case _, ok := <-queue:
+			if !ok {
+				return dropped
+			}
+			dropped++
+		default:
+			return dropped
+		}
 	}
 }

server/service/hid/mouse.go

@@ -5,14 +5,67 @@ import (
 )
 
 func (h *Hid) Mouse(queue <-chan []byte) {
+	h.mouse(queue, HID1, HID2)
+}
+
+func (h *Hid) mouse(queue <-chan []byte, relativePath string, absolutePath string) {
+	defer h.releaseRelativeMouse(relativePath)
+
+	absoluteButtonsActive := false
+	var absoluteReleaseReport []byte
+	defer func() {
+		if absoluteButtonsActive {
+			h.releaseAbsoluteMouse(absolutePath, absoluteReleaseReport)
+		}
+	}()
+
 	for event := range queue {
 		switch len(event) {
 		case 4:
-			h.WriteHid1(event)
+			if !h.writeHIDReport(h.relativeMouseDevice(relativePath), event) {
+				if dropped := drainHIDQueue(queue); dropped > 0 {
+					log.Debugf("dropped %d stale mouse HID reports after relative write failure", dropped)
+				}
+				h.releaseRelativeMouse(relativePath)
+			}
 		case 6:
-			h.WriteHid2(event)
+			if !h.writeHIDReport(h.absoluteMouseDevice(absolutePath), event) {
+				if dropped := drainHIDQueue(queue); dropped > 0 {
+					log.Debugf("dropped %d stale mouse HID reports after absolute write failure", dropped)
+				}
+				if absoluteButtonsActive {
+					if h.releaseAbsoluteMouse(absolutePath, absoluteReleaseReport) {
+						absoluteButtonsActive = false
+					}
+				} else if event[0] != 0 {
+					h.releaseAbsoluteMouse(absolutePath, absoluteMouseReleaseReport(event))
+				}
+				continue
+			}
+			absoluteReleaseReport = absoluteMouseReleaseReport(event)
+			absoluteButtonsActive = event[0] != 0
 		default:
 			log.Debugf("invalid mouse event: %v", event)
 		}
 	}
 }
+
+func (h *Hid) releaseRelativeMouse(path string) {
+	h.writeHIDReport(h.relativeMouseDevice(path), relativeMouseReleaseReport())
+}
+
+func (h *Hid) releaseAbsoluteMouse(path string, report []byte) bool {
+	return h.writeHIDReport(h.absoluteMouseDevice(path), report)
+}
+
+func relativeMouseReleaseReport() []byte {
+	return []byte{0x00, 0x00, 0x00, 0x00}
+}
+
+func absoluteMouseReleaseReport(positionReport []byte) []byte {
+	report := []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
+	if len(positionReport) >= 5 {
+		copy(report[1:5], positionReport[1:5])
+	}
+	return report
+}

server/service/hid/status.go

@@ -118,15 +118,15 @@ func ResetUSBPHY() error {
 	h := GetHid()
 	h.Lock()
 	h.CloseNoLock()
-	defer func() {
-		h.OpenNoLock()
-		h.Unlock()
-	}()
+	defer h.Unlock()
 
 	command := fmt.Sprintf("%s restart_phy", USBDevScript)
-	err := exec.Command("sh", "-c", command).Run()
-	if err != nil {
-		return err
+	if err := exec.Command("sh", "-c", command).Run(); err != nil {
+		return fmt.Errorf("restart usb phy: %w", err)
+	}
+
+	if err := h.OpenNoLockWithRetry(hidReopenTimeout, hidReopenRetryDelay); err != nil {
+		return fmt.Errorf("reopen HID devices after usb phy reset: %w", err)
 	}
 
 	return nil

server/service/picoclaw/gateway_proxy.go

@@ -88,8 +88,8 @@ func (s *Service) ConnectGateway(c *gin.Context) {
 	s.configureRelayConn(upstream, cfg)
 
 	wg.Add(4)
-	go s.runPingLoop("downstream", downstream, cfg, &wg)
-	go s.runPingLoop("upstream", upstream, cfg, &wg)
+	go s.runPingLoop("downstream", session.SessionID, downstream, cfg, &wg)
+	go s.runPingLoop("upstream", session.SessionID, upstream, cfg, &wg)
 	go s.proxyMessages("downstream", session, downstream, cfg, &wg, results)
 	go s.proxyMessages("upstream", session, upstream, cfg, &wg, results)
 
@@ -134,6 +134,15 @@ func (s *Service) proxyMessages(source string, session *GatewaySession, src *web
 			return
 		}
 
+		if !s.lock.Renew(session.SessionID) {
+			results <- relayResult{
+				Source:    source,
+				CloseCode: CloseCodePicoclawTakenOver,
+				Reason:    "session lock lost",
+			}
+			return
+		}
+
 		var writeErr error
 		switch source {
 		case "downstream":
@@ -152,13 +161,19 @@ func (s *Service) proxyMessages(source string, session *GatewaySession, src *web
 	}
 }
 
-func (s *Service) runPingLoop(name string, conn *websocket.Conn, cfg Config, wg *sync.WaitGroup) {
+func (s *Service) runPingLoop(name string, sessionID string, conn *websocket.Conn, cfg Config, wg *sync.WaitGroup) {
 	defer wg.Done()
 
 	ticker := time.NewTicker(time.Duration(cfg.PingIntervalMs) * time.Millisecond)
 	defer ticker.Stop()
 
 	for range ticker.C {
+		if !s.lock.Renew(sessionID) {
+			writeGatewayClose(conn, CloseCodePicoclawTakenOver, "session lock lost")
+			_ = conn.Close()
+			return
+		}
+
 		if err := conn.WriteControl(
 			websocket.PingMessage,
 			[]byte(name),

server/service/picoclaw/session_lock.go

@@ -10,6 +10,8 @@ var (
 	sessionLock     *SessionLock
 )
 
+const sessionLockDuration = 30 * time.Minute
+
 type SessionLock struct {
 	mu             sync.Mutex
 	ownerSessionID string
@@ -34,6 +36,35 @@ func (l *SessionLock) AcquireTemporary(sessionID string) (bool, *PicoclawError)
 	return l.acquire(sessionID)
 }
 
+func (l *SessionLock) Renew(sessionID string) bool {
+	if sessionID == "" {
+		return false
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if l.ownerSessionID != sessionID {
+		return false
+	}
+
+	l.expiresAt = time.Now().Add(sessionLockDuration)
+	return true
+}
+
+func (l *SessionLock) clearExpiredLocked(now time.Time) {
+	if l.ownerSessionID == "" {
+		return
+	}
+	if l.expiresAt.IsZero() || now.Before(l.expiresAt) {
+		return
+	}
+
+	l.ownerSessionID = ""
+	l.acquiredAt = time.Time{}
+	l.expiresAt = time.Time{}
+}
+
 func (l *SessionLock) acquire(sessionID string) (bool, *PicoclawError) {
 	if sessionID == "" {
 		return false, newPicoclawError(CodeSessionIDInvalid, "invalid PicoClaw session")
@@ -42,14 +73,16 @@ func (l *SessionLock) acquire(sessionID string) (bool, *PicoclawError) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
+	now := time.Now()
+	l.clearExpiredLocked(now)
+
 	if l.ownerSessionID == "" || l.ownerSessionID == sessionID {
 		acquired := l.ownerSessionID == ""
-		now := time.Now()
 		l.ownerSessionID = sessionID
 		if l.acquiredAt.IsZero() {
 			l.acquiredAt = now
 		}
-		l.expiresAt = now.Add(30 * time.Minute)
+		l.expiresAt = now.Add(sessionLockDuration)
 		return acquired, nil
 	}
 
@@ -79,19 +112,24 @@ func (l *SessionLock) ForceTakeover(sessionID string) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
+	now := time.Now()
 	l.ownerSessionID = sessionID
-	l.acquiredAt = time.Now()
-	l.expiresAt = time.Now().Add(30 * time.Minute)
+	l.acquiredAt = now
+	l.expiresAt = now.Add(sessionLockDuration)
 }
 
 func (l *SessionLock) Owner() string {
 	l.mu.Lock()
 	defer l.mu.Unlock()
+
+	l.clearExpiredLocked(time.Now())
 	return l.ownerSessionID
 }
 
 func (l *SessionLock) BlocksManualInput() bool {
 	l.mu.Lock()
 	defer l.mu.Unlock()
+
+	l.clearExpiredLocked(time.Now())
 	return l.ownerSessionID != ""
 }

server/service/ws/client.go

@@ -102,21 +102,42 @@ func (c *Client) UpdateHeartbeat() {
 }
 
 func (c *Client) Close() {
-	_ = c.ws.Close()
+	c.closeOnce.Do(func() {
+		if c.ws != nil {
+			_ = c.ws.Close()
+		}
 
-	closeQueue(c.keyboard)
-	closeQueue(c.mouse)
+		if c.keyboard != nil {
+			close(c.keyboard)
+		}
+		if c.mouse != nil {
+			close(c.mouse)
+		}
 
-	log.Debug("websocket disconnected")
+		log.Debug("websocket disconnected")
+	})
 }
 
 func writeQueue(queue chan []byte, data []byte) {
-	queue <- data
+	if !sendQueue(queue, data) {
+		log.Debug("hid event dropped because websocket queue is closed")
+		return
+	}
+
 	jiggler.GetJiggler().Update()
 }
 
-func closeQueue(queue chan []byte) {
-	for range queue {
+func sendQueue(queue chan []byte, data []byte) (ok bool) {
+	if queue == nil {
+		return false
 	}
-	close(queue)
+
+	defer func() {
+		if r := recover(); r != nil {
+			ok = false
+		}
+	}()
+
+	queue <- data
+	return true
 }

server/service/ws/types.go

@@ -21,6 +21,7 @@ type Client struct {
 	mouse         chan []byte
 	lastHeartbeat time.Time
 	mutex         sync.Mutex
+	closeOnce     sync.Once
 }
 
 type Message struct {