From 6b98f5415f496a0bd19a83a350d202e0c613ef78 Mon Sep 17 00:00:00 2001 From: SiYue-ZO <2835601846@qq.com> Date: Mon, 20 Jul 2026 18:01:30 +0800 Subject: [PATCH] feat(web): add MCP settings and control handoff UI Add the MCP settings view and frontend API bindings for configuring remote-control access. Expose AI-control ownership state in PicoClaw, lock conflicting keyboard input paths, and preserve chat state when control is released. --- web/src/api/mcp.ts | 20 + web/src/api/picoclaw.ts | 12 + web/src/jotai/ai-control.ts | 74 ++ web/src/jotai/keyboard.ts | 21 +- web/src/jotai/picoclaw.ts | 23 + web/src/lib/mouse.ts | 4 +- web/src/lib/picoclaw-gateway.ts | 47 +- web/src/pages/desktop/menu/download.tsx | 93 ++- .../desktop/menu/keyboard/leader-key.tsx | 8 +- web/src/pages/desktop/menu/keyboard/paste.tsx | 90 ++- .../menu/keyboard/shortcuts/recorder.tsx | 9 +- .../settings/device/advanced/autostart.tsx | 14 +- web/src/pages/desktop/menu/settings/index.tsx | 11 +- .../pages/desktop/menu/settings/mcp/index.tsx | 320 ++++++++ .../desktop/menu/terminal/serial-port.tsx | 22 +- web/src/pages/desktop/menu/wol/index.tsx | 13 +- web/src/pages/desktop/mouse/absolute.tsx | 107 ++- web/src/pages/desktop/mouse/relative.tsx | 3 + .../pages/desktop/picoclaw/action-overlay.tsx | 2 +- .../pages/desktop/picoclaw/keyboard-lock.ts | 17 + .../pages/desktop/picoclaw/message-input.tsx | 51 +- .../pages/desktop/picoclaw/runtime-view.ts | 165 ++++- .../pages/desktop/picoclaw/sidebar-actions.ts | 700 +++++++++++++++--- .../pages/desktop/picoclaw/sidebar-effects.ts | 276 ++++++- .../pages/desktop/picoclaw/sidebar-header.tsx | 29 +- .../desktop/picoclaw/sidebar-install.tsx | 3 + .../desktop/picoclaw/sidebar-model-config.tsx | 35 +- .../picoclaw/sidebar-session-actions.ts | 60 +- web/src/pages/desktop/picoclaw/sidebar.tsx | 345 ++++++++- web/src/pages/desktop/picoclaw/use-sidebar.ts | 70 +- 30 files changed, 2289 insertions(+), 355 deletions(-) create mode 100644 web/src/api/mcp.ts create mode 100644 web/src/jotai/ai-control.ts create mode 100644 web/src/pages/desktop/menu/settings/mcp/index.tsx create mode 100644 web/src/pages/desktop/picoclaw/keyboard-lock.ts diff --git a/web/src/api/mcp.ts b/web/src/api/mcp.ts new file mode 100644 index 0000000..d2e0046 --- /dev/null +++ b/web/src/api/mcp.ts @@ -0,0 +1,20 @@ +import { http } from '@/lib/http.ts'; + +export type MCPConfig = { + enabled: boolean; + apiKey: string; + controlMode: 'off' | 'mcp' | 'picoclaw'; + transitioning: boolean; +}; + +export function getMCPConfig() { + return http.get('/api/mcp/config'); +} + +export function setMCPEnabled(enabled: boolean) { + return http.post('/api/mcp/config', { enabled }); +} + +export function regenerateMCPAPIKey() { + return http.post('/api/mcp/key/regenerate'); +} diff --git a/web/src/api/picoclaw.ts b/web/src/api/picoclaw.ts index b833a18..98150ef 100644 --- a/web/src/api/picoclaw.ts +++ b/web/src/api/picoclaw.ts @@ -71,6 +71,18 @@ export function stopRuntime() { return http.post('/api/picoclaw/runtime/stop'); } +export function getAIControlStatus() { + return http.get('/api/ai/control/status'); +} + +export function setAIControlMode(mode: 'off' | 'mcp' | 'picoclaw') { + return http.request({ + method: 'put', + url: '/api/ai/control/mode', + data: { mode } + }); +} + export function installRuntime() { return http.post('/api/picoclaw/runtime/install'); } diff --git a/web/src/jotai/ai-control.ts b/web/src/jotai/ai-control.ts new file mode 100644 index 0000000..028239d --- /dev/null +++ b/web/src/jotai/ai-control.ts @@ -0,0 +1,74 @@ +import { atom } from 'jotai'; + +export type AIControlMode = 'off' | 'mcp' | 'picoclaw'; + +export type AIControlStatus = { + mode: AIControlMode; + transitioning: boolean; + canControlPicoclaw: boolean; + changedAt?: string; + lastError?: string; + source?: string; +}; + +export const aiControlStatusAtom = atom(null); + +export function normalizeAIControlStatus(value: unknown, source?: string): AIControlStatus | null { + const record = asRecord(value); + if (!record) { + return null; + } + + const control = asRecord(record.control); + const capabilities = asRecord(record.capabilities); + const mode = parseAIControlMode( + control?.mode ?? record.mode ?? record.controlMode ?? record.control_mode + ); + + if (!mode) { + return null; + } + + const transitioning = readBoolean(control?.transitioning ?? record.transitioning) ?? false; + const explicitCanControl = readBoolean( + control?.can_control ?? record.can_control ?? record.canControlPicoclaw + ); + const deviceWrite = readBoolean(capabilities?.device_write); + const changedAt = readString(control?.changed_at ?? control?.changedAt ?? record.changed_at); + const lastError = readString(control?.last_error ?? control?.lastError ?? record.last_error); + const payloadSource = readString(control?.source ?? record.source); + + return { + mode, + transitioning, + canControlPicoclaw: + explicitCanControl ?? deviceWrite ?? (mode === 'picoclaw' && transitioning !== true), + changedAt, + lastError, + source: source ?? payloadSource + }; +} + +function parseAIControlMode(value: unknown): AIControlMode | null { + if (value === 'off' || value === 'mcp' || value === 'picoclaw') { + return value; + } + + return null; +} + +function readBoolean(value: unknown) { + return typeof value === 'boolean' ? value : undefined; +} + +function readString(value: unknown) { + if (typeof value !== 'string' || value.trim() === '') { + return undefined; + } + + return value; +} + +function asRecord(value: unknown): Record | null { + return typeof value === 'object' && value !== null ? (value as Record) : null; +} diff --git a/web/src/jotai/keyboard.ts b/web/src/jotai/keyboard.ts index c4955d6..a371910 100644 --- a/web/src/jotai/keyboard.ts +++ b/web/src/jotai/keyboard.ts @@ -1,7 +1,26 @@ import { atom } from 'jotai'; +export type KeyboardLockSource = string; + +type KeyboardLockAction = { + source: KeyboardLockSource; + locked: boolean; +}; + +const keyboardLocksAtom = atom>(new Set()); + // is the keyboard enabled (Disable keyboard events when input is required) -export const isKeyboardEnableAtom = atom(true); +export const isKeyboardEnableAtom = atom((get) => get(keyboardLocksAtom).size === 0); + +export const keyboardLockAtom = atom(null, (get, set, action: KeyboardLockAction) => { + const locks = new Set(get(keyboardLocksAtom)); + if (action.locked) { + locks.add(action.source); + } else { + locks.delete(action.source); + } + set(keyboardLocksAtom, locks); +}); // is the virtual keyboard opened export const isKeyboardOpenAtom = atom(false); diff --git a/web/src/jotai/picoclaw.ts b/web/src/jotai/picoclaw.ts index 2fea45f..a768a8f 100644 --- a/web/src/jotai/picoclaw.ts +++ b/web/src/jotai/picoclaw.ts @@ -21,6 +21,29 @@ export type PicoclawRuntimeStatus = { last_error?: string; checked_at?: string; current_session?: string; + restoring?: boolean; + runtime_intent?: { + desired_running: boolean; + updated_at?: string; + updated_by?: string; + last_started_at?: string; + last_stopped_at?: string; + last_error?: string; + }; + control_mode: 'off' | 'mcp' | 'picoclaw'; + transitioning?: boolean; + control?: { + mode: 'off' | 'mcp' | 'picoclaw'; + transitioning: boolean; + can_control: boolean; + last_error?: string; + changed_at?: string; + }; + capabilities?: { + chat: boolean; + read_only_tools: boolean; + device_write: boolean; + }; }; export type PicoclawRuntimeStartResult = { diff --git a/web/src/lib/mouse.ts b/web/src/lib/mouse.ts index ef94e27..1c9c081 100644 --- a/web/src/lib/mouse.ts +++ b/web/src/lib/mouse.ts @@ -123,9 +123,9 @@ export class MouseReportAbsolute { return this.buildReport(lastX, lastY, 0); } - reset(): Uint8Array { + reset(x: number = 0, y: number = 0): Uint8Array { this.buttons = 0; - return this.buildReport(0, 0, 0); + return this.buildReport(x, y, 0); } private clamp(value: number, min: number, max: number): number { diff --git a/web/src/lib/picoclaw-gateway.ts b/web/src/lib/picoclaw-gateway.ts index 18779d2..bf16895 100644 --- a/web/src/lib/picoclaw-gateway.ts +++ b/web/src/lib/picoclaw-gateway.ts @@ -19,6 +19,23 @@ export type GatewayRuntimeStatus = { last_error?: string; checked_at?: string; current_session?: string; + control_mode: 'off' | 'mcp' | 'picoclaw'; + transitioning?: boolean; + control?: GatewayControlStatus; + capabilities?: { + chat: boolean; + read_only_tools: boolean; + device_write: boolean; + }; +}; + +export type GatewayControlStatus = { + mode: 'off' | 'mcp' | 'picoclaw'; + transitioning: boolean; + can_control: boolean; + last_error?: string; + changed_at?: string; + source?: string; }; export type GatewayAssistantMessage = { @@ -62,6 +79,7 @@ type GatewayEventMap = { observation: GatewayObservation; error: GatewayError; close: GatewayClose; + control_mode_changed: GatewayControlStatus; }; type EventName = keyof GatewayEventMap; @@ -84,7 +102,9 @@ const CLOSE_ERRORS: Record = { 4002: 'RUNTIME_UNAVAILABLE', 4003: 'AUTH_FAILED', 4004: 'AI_TAKEN_OVER', - 4005: 'UPSTREAM_CLOSED' + 4005: 'UPSTREAM_CLOSED', + 4006: 'AI_MODE_CONFLICT', + 4007: 'RUNTIME_STOPPED' }; export function generateUUIDv4() { @@ -165,7 +185,7 @@ class PicoClawGateway { } this.ws = null; - if (event.code !== 1000) { + if (event.code !== 1000 && event.code !== 4006 && event.code !== 4007) { const code = CLOSE_ERRORS[event.code] || 'UNKNOWN'; if (code !== 'UNKNOWN') { this.emit('error', { @@ -180,12 +200,16 @@ class PicoClawGateway { reason: event.reason }); - this.setTransportState(event.code === 1000 ? 'disconnected' : 'error'); + this.setTransportState( + event.code === 1000 || event.code === 4006 || event.code === 4007 + ? 'disconnected' + : 'error' + ); this.setRunState('idle'); if (!this.explicitClose) { reject(new Error(event.reason || 'gateway closed')); - if (this.autoReconnect) { + if (this.autoReconnect && event.code !== 4006 && event.code !== 4007) { this.scheduleReconnect(); } } @@ -350,6 +374,21 @@ class PicoClawGateway { }); return; } + if (type === 'control.mode_changed') { + const payload = (message.payload || {}) as Record; + const mode = String(payload.mode || 'off'); + if (mode === 'off' || mode === 'mcp' || mode === 'picoclaw') { + this.emit('control_mode_changed', { + mode, + transitioning: payload.transitioning === true, + can_control: payload.can_control === true, + last_error: typeof payload.last_error === 'string' ? payload.last_error : undefined, + changed_at: typeof payload.changed_at === 'string' ? payload.changed_at : undefined, + source: typeof payload.source === 'string' ? payload.source : undefined + }); + } + return; + } if (type === 'message.create' || type === 'message.update') { this.setRunState('idle'); this.emit('assistant_message', { diff --git a/web/src/pages/desktop/menu/download.tsx b/web/src/pages/desktop/menu/download.tsx index 93b0eec..368cee9 100644 --- a/web/src/pages/desktop/menu/download.tsx +++ b/web/src/pages/desktop/menu/download.tsx @@ -6,20 +6,15 @@ import { useSetAtom } from 'jotai'; import { DownloadIcon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { - cancelDownloadImage, - downloadImage, - imageEnabled, - statusImage -} from '@/api/download.ts'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { cancelDownloadImage, downloadImage, imageEnabled, statusImage } from '@/api/download.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; import { MenuItem } from '@/components/menu-item.tsx'; const imageUpdatedEvent = 'nanokvm:image-updated'; export const DownloadImage = () => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [input, setInput] = useState(''); const [sha256sum, setSha256sum] = useState(''); @@ -58,10 +53,10 @@ export const DownloadImage = () => { if (open) { checkDiskEnabled(); startStatusPolling(); - setIsKeyboardEnable(false); + setKeyboardLock({ source: 'download-popover', locked: true }); setPopoverKey((prevKey) => prevKey + 1); // Force re-render } else { - setIsKeyboardEnable(true); + setKeyboardLock({ source: 'download-popover', locked: false }); // Keep monitoring an active remote download after the popover closes so // completion can still refresh an already-open image list. @@ -243,7 +238,7 @@ export const DownloadImage = () => { function handleFileChange(e: React.ChangeEvent) { const file = e.target.files?.[0] ?? null; - if (!file || !file.name.toLowerCase().endsWith(".iso")) { + if (!file || !file.name.toLowerCase().endsWith('.iso')) { setStatus('failed'); setLog(t('download.NoISO')); return; @@ -259,7 +254,7 @@ export const DownloadImage = () => { function upload(file: File | null) { if (!file) return; - if (!file || !file.name.toLowerCase().endsWith(".iso")) { + if (!file || !file.name.toLowerCase().endsWith('.iso')) { setStatus('failed'); setLog(t('download.NoISO')); return; @@ -275,20 +270,19 @@ export const DownloadImage = () => { setLog('Downloading: ' + file.name); const formData = new FormData(); - formData.append("file", file); + formData.append('file', file); - fetch("/api/download/file", { - method: "POST", + fetch('/api/download/file', { + method: 'POST', headers: { 'X-SHA256-Sum': checksum }, - body: formData, + body: formData }) .then(async (response) => { const rsp = await response.json(); if (!response.ok || rsp.code !== 0) { - const message = - rsp.msg === 'sha256 mismatch' ? t('download.checksumFailed') : rsp.msg; + const message = rsp.msg === 'sha256 mismatch' ? t('download.checksumFailed') : rsp.msg; throw new Error(message || t('download.failed')); } @@ -303,7 +297,6 @@ export const DownloadImage = () => { }); startStatusPolling(); - } const content = ( @@ -337,9 +330,7 @@ export const DownloadImage = () => { ? cancelDownload() : download(input) } - disabled={ - isCancelling || (status === 'in_progress' && !isRemoteDownloading) - } + disabled={isCancelling || (status === 'in_progress' && !isRemoteDownloading)} > {isRemoteDownloading && status === 'in_progress' ? t('download.cancel') @@ -368,35 +359,35 @@ export const DownloadImage = () => { ? 'cursor-not-allowed bg-neutral-700 opacity-50' : 'cursor-pointer hover:bg-neutral-500' )} - onDrop={(e) => { - if (status === "in_progress") return; // deaktiviert - e.preventDefault(); - setIsDragging(false); - const file = e.dataTransfer.files?.[0] ?? null; - if (!file || !file.name.toLowerCase().endsWith(".iso")) { - setStatus('failed'); - setLog(t('download.NoISO')); - return; - } - setStatus('idle'); - setLog(''); - setSelectedFile(file); - }} - onDragOver={(e) => { - if (status === "in_progress") return; // deaktiviert - e.preventDefault(); - setIsDragging(true); // Datei wird über den Bereich gezogen - }} - onDragLeave={(e) => { - if (status === "in_progress") return; // deaktiviert - e.preventDefault(); - setIsDragging(false); // Maus verlässt Bereich - }} - onClick={() => { - if (status === "in_progress") return; // deaktiviert - document.getElementById("file-upload")?.click() - }} - > + onDrop={(e) => { + if (status === 'in_progress') return; // deaktiviert + e.preventDefault(); + setIsDragging(false); + const file = e.dataTransfer.files?.[0] ?? null; + if (!file || !file.name.toLowerCase().endsWith('.iso')) { + setStatus('failed'); + setLog(t('download.NoISO')); + return; + } + setStatus('idle'); + setLog(''); + setSelectedFile(file); + }} + onDragOver={(e) => { + if (status === 'in_progress') return; // deaktiviert + e.preventDefault(); + setIsDragging(true); // Datei wird über den Bereich gezogen + }} + onDragLeave={(e) => { + if (status === 'in_progress') return; // deaktiviert + e.preventDefault(); + setIsDragging(false); // Maus verlässt Bereich + }} + onClick={() => { + if (status === 'in_progress') return; // deaktiviert + document.getElementById('file-upload')?.click(); + }} + > {selectedFile ? selectedFile.name : t('download.uploadbox')} diff --git a/web/src/pages/desktop/menu/keyboard/leader-key.tsx b/web/src/pages/desktop/menu/keyboard/leader-key.tsx index 0aba5bf..2a5e9a5 100644 --- a/web/src/pages/desktop/menu/keyboard/leader-key.tsx +++ b/web/src/pages/desktop/menu/keyboard/leader-key.tsx @@ -13,12 +13,12 @@ import { import { useTranslation } from 'react-i18next'; import * as api from '@/api/hid.ts'; -import { isKeyboardEnableAtom, leaderKeyAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom, leaderKeyAtom } from '@/jotai/keyboard.ts'; export const LeaderKey = () => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [leaderKey, setLeaderKey] = useAtom(leaderKeyAtom); const [isModalOpen, setIsModalOpen] = useState(false); @@ -64,7 +64,7 @@ export const LeaderKey = () => { }, [isFocused]); function openModal() { - setIsKeyboardEnable(false); + setKeyboardLock({ source: 'leader-key-recorder', locked: true }); setIsLeaderKeyEnable(!!leaderKey); setTempLeaderKey(leaderKey); setIsDocCollapsed(true); @@ -72,7 +72,7 @@ export const LeaderKey = () => { } function closeModal() { - setIsKeyboardEnable(true); + setKeyboardLock({ source: 'leader-key-recorder', locked: false }); setIsModalOpen(false); } diff --git a/web/src/pages/desktop/menu/keyboard/paste.tsx b/web/src/pages/desktop/menu/keyboard/paste.tsx index 217088c..8218b85 100644 --- a/web/src/pages/desktop/menu/keyboard/paste.tsx +++ b/web/src/pages/desktop/menu/keyboard/paste.tsx @@ -7,13 +7,13 @@ import { ClipboardIcon, ClipboardPasteIcon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { paste } from '@/api/hid'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; type InputStatus = '' | 'error'; export const Paste = () => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [isModalOpen, setIsModalOpen] = useState(false); const [inputValue, setInputValue] = useState(''); @@ -63,16 +63,43 @@ export const Paste = () => { setIsReadingClipboard(false); } } - + // Extended RU → EN translation including punctuation function translateRuToEnWithPunctuation(value: string): string { const letterMap: Record = { - 'ё': '`', - 'й': 'q', 'ц': 'w', 'у': 'e', 'к': 'r', 'е': 't', 'н': 'y', 'г': 'u', 'ш': 'i', - 'щ': 'o', 'з': 'p', 'х': '[', 'ъ': ']', - 'ф': 'a', 'ы': 's', 'в': 'd', 'а': 'f', 'п': 'g', 'р': 'h', 'о': 'j', 'л': 'k', - 'д': 'l', 'ж': ';', 'э': '\'', - 'я': 'z', 'ч': 'x', 'с': 'c', 'м': 'v', 'и': 'b', 'т': 'n', 'ь': 'm', 'б': ',', 'ю': '.', + ё: '`', + й: 'q', + ц: 'w', + у: 'e', + к: 'r', + е: 't', + н: 'y', + г: 'u', + ш: 'i', + щ: 'o', + з: 'p', + х: '[', + ъ: ']', + ф: 'a', + ы: 's', + в: 'd', + а: 'f', + п: 'g', + р: 'h', + о: 'j', + л: 'k', + д: 'l', + ж: ';', + э: "'", + я: 'z', + ч: 'x', + с: 'c', + м: 'v', + и: 'b', + т: 'n', + ь: 'm', + б: ',', + ю: '.' }; const punctuationMap: Record = { @@ -81,23 +108,25 @@ export const Paste = () => { ';': '$', ':': '^', '?': '&', - 'Ё': '~', + Ё: '~', '/': '|', '.': '/', - ',': '?', + ',': '?' }; - return Array.from(value).map((ch) => { - const lower = ch.toLowerCase(); - if (letterMap[lower]) { - const translated = letterMap[lower]; - return ch === lower ? translated : translated.toUpperCase(); - } - if (punctuationMap[ch]) { - return punctuationMap[ch]; - } - return ch; - }).join(''); + return Array.from(value) + .map((ch) => { + const lower = ch.toLowerCase(); + if (letterMap[lower]) { + const translated = letterMap[lower]; + return ch === lower ? translated : translated.toUpperCase(); + } + if (punctuationMap[ch]) { + return punctuationMap[ch]; + } + return ch; + }) + .join(''); } function submit() { @@ -128,7 +157,7 @@ export const Paste = () => { inputRef.current?.focus(); } - setIsKeyboardEnable(!open); + setKeyboardLock({ source: 'paste-modal', locked: open }); } function isValidForLanguage(value: string, selectedLangue: string) { @@ -142,11 +171,14 @@ export const Paste = () => { // For Russian language, allow Cyrillic letters and special characters // that can be typed on Russian keyboard (but not English letters) if ( - (code >= 0x0410 && code <= 0x042F) || // (А-Я) - (code >= 0x0430 && code <= 0x044F) || // (а-я) + (code >= 0x0410 && code <= 0x042f) || // (А-Я) + (code >= 0x0430 && code <= 0x044f) || // (а-я) code === 0x0401 || // (Ё) code === 0x0451 || // (ё) - (code >= 0x20 && code <= 0x7E && !(code >= 0x41 && code <= 0x5A) && !(code >= 0x61 && code <= 0x7A)) // Special chars, digits, space, but not English letters + (code >= 0x20 && + code <= 0x7e && + !(code >= 0x41 && code <= 0x5a) && + !(code >= 0x61 && code <= 0x7a)) // Special chars, digits, space, but not English letters ) { continue; } @@ -154,12 +186,12 @@ export const Paste = () => { } else if (isFrench) { // For French, allow ASCII and Latin-1/Extended accented characters // (é è ê ë à â ù û ç î ï ô œ æ ° µ £ § ¨ etc.) - if (code <= 0x7F) continue; - if (code >= 0x00A0 && code <= 0x017E) continue; + if (code <= 0x7f) continue; + if (code >= 0x00a0 && code <= 0x017e) continue; return false; } else { // For English/German, only allow ASCII - if (code <= 0x7F) continue; + if (code <= 0x7f) continue; return false; } } diff --git a/web/src/pages/desktop/menu/keyboard/shortcuts/recorder.tsx b/web/src/pages/desktop/menu/keyboard/shortcuts/recorder.tsx index 40ab864..d93f16e 100644 --- a/web/src/pages/desktop/menu/keyboard/shortcuts/recorder.tsx +++ b/web/src/pages/desktop/menu/keyboard/shortcuts/recorder.tsx @@ -5,7 +5,7 @@ import { KeyboardIcon, Trash2Icon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { isModifier } from '@/lib/keymap.ts'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; import { Kbd, KbdGroup } from '@/components/ui/kbd.tsx'; import { ScrollArea } from '@/components/ui/scroll-area.tsx'; @@ -60,7 +60,7 @@ export const Recorder = ({ setIsRecording }: RecorderProps) => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [isModalOpen, setIsModalOpen] = useState(false); const [isFocused, setIsFocused] = useState(false); @@ -70,7 +70,7 @@ export const Recorder = ({ const recordedKeysRef = useRef([]); useEffect(() => { - setIsKeyboardEnable(!isModalOpen); + setKeyboardLock({ source: 'shortcut-recorder', locked: isModalOpen }); setIsRecording(isModalOpen); if (!isModalOpen) return; @@ -81,8 +81,9 @@ export const Recorder = ({ return () => { clearTimeout(timer); + setKeyboardLock({ source: 'shortcut-recorder', locked: false }); }; - }, [isModalOpen]); + }, [isModalOpen, setIsRecording, setKeyboardLock]); useEffect(() => { if (!isFocused) return; diff --git a/web/src/pages/desktop/menu/settings/device/advanced/autostart.tsx b/web/src/pages/desktop/menu/settings/device/advanced/autostart.tsx index c326678..a43a1d5 100644 --- a/web/src/pages/desktop/menu/settings/device/advanced/autostart.tsx +++ b/web/src/pages/desktop/menu/settings/device/advanced/autostart.tsx @@ -7,7 +7,7 @@ import { useSetAtom } from 'jotai'; import { useTranslation } from 'react-i18next'; import * as api from '@/api/autostart.ts'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; export const Autostart = () => { interface AutostartItem { @@ -16,7 +16,7 @@ export const Autostart = () => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [isEditAutostartOpen, setIsEditAutostartOpen] = useState(false); const [isManageAutostartOpen, setIsManageAutostartOpen] = useState(false); @@ -52,13 +52,13 @@ export const Autostart = () => { ]; useEffect(() => { - setIsKeyboardEnable(false); + setKeyboardLock({ source: 'autostart-settings', locked: true }); getAutostart(); return () => { - setIsKeyboardEnable(true); + setKeyboardLock({ source: 'autostart-settings', locked: false }); }; - }, []); + }, [setKeyboardLock]); function getAutostart() { api.getAutostart().then((rsp) => { @@ -166,8 +166,8 @@ export const Autostart = () => { footer="" onCancel={() => setIsManageAutostartOpen(false)} title={t('settings.device.autostart.title')} - > -
+ > +
+
+ )} + + + ); +}; + +type CredentialRowProps = { + label: string; + value: string; + copied: boolean; + onCopy: () => void; + disabled?: boolean; + actions?: ReactNode; +}; + +const CredentialRow = ({ + label, + value, + copied, + onCopy, + disabled = false, + actions +}: CredentialRowProps) => ( +
+ {label} +
+ + {value} + +
+ {actions} +
+
+
+); diff --git a/web/src/pages/desktop/menu/terminal/serial-port.tsx b/web/src/pages/desktop/menu/terminal/serial-port.tsx index 09a9aa4..279016b 100644 --- a/web/src/pages/desktop/menu/terminal/serial-port.tsx +++ b/web/src/pages/desktop/menu/terminal/serial-port.tsx @@ -4,11 +4,11 @@ import { useSetAtom } from 'jotai'; import { SquareTerminalIcon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; export const SerialPort = () => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [isModalOpen, setIsModalOpen] = useState(false); const [port, setPort] = useState(''); @@ -19,13 +19,12 @@ export const SerialPort = () => { const [stopBits, setStopBits] = useState(1); function openModal() { - setIsKeyboardEnable(false); - + setKeyboardLock({ source: 'serial-port-modal', locked: true }); setIsModalOpen(true); } function closeModal() { - setIsKeyboardEnable(true); + setKeyboardLock({ source: 'serial-port-modal', locked: false }); setIsModalOpen(false); } @@ -48,8 +47,12 @@ export const SerialPort = () => { return; } + setKeyboardLock({ source: 'serial-port-modal', locked: false }); setIsModalOpen(false); - window.open(`/#terminal?port=${port}&baud=${baudrate}&parity=${parity}&flowControl=${flowControl}&dataBits=${dataBits}&stopBits=${stopBits}`, '_blank'); + window.open( + `/#terminal?port=${port}&baud=${baudrate}&parity=${parity}&flowControl=${flowControl}&dataBits=${dataBits}&stopBits=${stopBits}`, + '_blank' + ); } return ( @@ -113,7 +116,9 @@ export const SerialPort = () => {
-
{t('terminal.flowControl')}
+
+ {t('terminal.flowControl')} +
setIsKeyboardEnable(false)} - onBlur={() => setIsKeyboardEnable(true)} + onFocus={() => setKeyboardLock({ source: 'wol-edit-input', locked: true })} + onBlur={() => setKeyboardLock({ source: 'wol-edit-input', locked: false })} defaultValue={item.name} onPressEnter={(e) => setMacName(e, item.mac)} /> diff --git a/web/src/pages/desktop/mouse/absolute.tsx b/web/src/pages/desktop/mouse/absolute.tsx index 74357e6..e369c6a 100644 --- a/web/src/pages/desktop/mouse/absolute.tsx +++ b/web/src/pages/desktop/mouse/absolute.tsx @@ -46,6 +46,9 @@ export const Absolute = () => { const screen = document.getElementById('screen') as HTMLVideoElement | null; if (!screen) return; const target = screen; + const mouse = mouseRef.current; + let pendingMove: { x: number; y: number } | null = null; + let moveFrame: number | null = null; target.addEventListener('mousedown', handleMouseDown); target.addEventListener('mouseup', handleMouseUp); @@ -61,15 +64,50 @@ export const Absolute = () => { target.addEventListener('touchcancel', handleTouchCancel); } + // Mouse event handler + function handleMouseEvent(event: MouseAbsoluteEvent) { + let report: Uint8Array; + + switch (event.type) { + case 'mousedown': + mouse.buttonDown(event.button); + report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y); + break; + case 'mouseup': + mouse.buttonUp(event.button); + report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y); + break; + case 'wheel': + report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y, event.deltaY); + break; + case 'move': + report = mouse.buildReport(event.x, event.y); + lastPosRef.current = { x: event.x, y: event.y }; + break; + default: + report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y); + break; + } + + sendReport(report); + } + + function sendReport(report: Uint8Array) { + const data = new Uint8Array([MessageEvent.Mouse, ...report]); + client.send(data); + } + // Mouse down event function handleMouseDown(e: MouseEvent) { disableEvent(e); + flushMouseMove(); handleMouseEvent({ type: 'mousedown', button: e.button }); } // Mouse up event function handleMouseUp(e: MouseEvent) { disableEvent(e); + flushMouseMove(); handleMouseEvent({ type: 'mouseup', button: e.button }); } @@ -77,7 +115,7 @@ export const Absolute = () => { function handleMouseMove(e: MouseEvent) { disableEvent(e); const { x, y } = getCoordinate(e); - handleMouseEvent({ type: 'move', x, y }); + queueMouseMove(x, y); } // Mouse wheel event @@ -93,6 +131,7 @@ export const Absolute = () => { return; } + flushMouseMove(); const deltaY = (e.deltaY > 0 ? 1 : -1) * scrollDirection; handleMouseEvent({ type: 'wheel', deltaY }); lastScrollTimeRef.current = currentTime; @@ -216,6 +255,7 @@ export const Absolute = () => { handleMouseEvent({ type: 'mouseup', button: MouseButton.Left }); }, 50); } else if (pressedButtonRef.current !== null) { + flushMouseMove(); handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! }); } @@ -235,6 +275,7 @@ export const Absolute = () => { } if (pressedButtonRef.current !== null) { + flushMouseMove(); handleMouseEvent({ type: 'mouseup', button: pressedButtonRef.current! }); } @@ -289,7 +330,37 @@ export const Absolute = () => { return { x, y }; } + function queueMouseMove(x: number, y: number) { + pendingMove = { x, y }; + if (moveFrame !== null) { + return; + } + + moveFrame = requestAnimationFrame(() => { + moveFrame = null; + flushMouseMove(); + }); + } + + function flushMouseMove() { + if (moveFrame !== null) { + cancelAnimationFrame(moveFrame); + moveFrame = null; + } + if (pendingMove === null) { + return; + } + + const move = pendingMove; + pendingMove = null; + handleMouseEvent({ type: 'move', x: move.x, y: move.y }); + } + return () => { + if (moveFrame !== null) { + cancelAnimationFrame(moveFrame); + } + sendReport(mouse.reset(lastPosRef.current.x, lastPosRef.current.y)); target.removeEventListener('mousemove', handleMouseMove); target.removeEventListener('mousedown', handleMouseDown); target.removeEventListener('mouseup', handleMouseUp); @@ -307,40 +378,6 @@ export const Absolute = () => { }; }, [isBigScreen, resolution, scrollDirection, scrollInterval]); - // Mouse event handler - function handleMouseEvent(event: MouseAbsoluteEvent) { - let report: Uint8Array; - const mouse = mouseRef.current; - - switch (event.type) { - case 'mousedown': - mouse.buttonDown(event.button); - report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y); - break; - case 'mouseup': - mouse.buttonUp(event.button); - report = mouse.buildButtonReport(lastPosRef.current.x, lastPosRef.current.y); - break; - case 'wheel': - report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y, event.deltaY); - break; - case 'move': - report = mouse.buildReport(event.x, event.y); - lastPosRef.current = { x: event.x, y: event.y }; - break; - default: - report = mouse.buildReport(lastPosRef.current.x, lastPosRef.current.y); - break; - } - - sendReport(report); - } - - function sendReport(report: Uint8Array) { - const data = new Uint8Array([MessageEvent.Mouse, ...report]); - client.send(data); - } - // disable default events function disableEvent(event: any) { event.preventDefault(); diff --git a/web/src/pages/desktop/mouse/relative.tsx b/web/src/pages/desktop/mouse/relative.tsx index ac0bafa..4c2c611 100644 --- a/web/src/pages/desktop/mouse/relative.tsx +++ b/web/src/pages/desktop/mouse/relative.tsx @@ -25,6 +25,7 @@ export const Relative = () => { useEffect(() => { const screen = document.getElementById('screen'); if (!screen) return; + const mouse = mouseRef.current; showMessage(); @@ -94,6 +95,8 @@ export const Relative = () => { } return () => { + const release = mouse.reset(); + client.send(new Uint8Array([MessageEvent.Mouse, ...release])); screen.removeEventListener('click', handleMouseClick); screen.removeEventListener('mousemove', handleMouseMove); screen.removeEventListener('mousedown', handleMouseDown); diff --git a/web/src/pages/desktop/picoclaw/action-overlay.tsx b/web/src/pages/desktop/picoclaw/action-overlay.tsx index 1a12e1a..ffafe04 100644 --- a/web/src/pages/desktop/picoclaw/action-overlay.tsx +++ b/web/src/pages/desktop/picoclaw/action-overlay.tsx @@ -48,7 +48,7 @@ export const ActionOverlay = () => { }; }, [takeover.active]); - if (!takeover.active || !rect || rect.width <= 0 || rect.height <= 0) { + if (!takeover.active || !overlay.visible || !rect || rect.width <= 0 || rect.height <= 0) { return null; } diff --git a/web/src/pages/desktop/picoclaw/keyboard-lock.ts b/web/src/pages/desktop/picoclaw/keyboard-lock.ts new file mode 100644 index 0000000..431c363 --- /dev/null +++ b/web/src/pages/desktop/picoclaw/keyboard-lock.ts @@ -0,0 +1,17 @@ +export const PICOCLAW_INPUT_KEYBOARD_LOCK_SOURCE = 'picoclaw-input'; +export const PICOCLAW_MODEL_CONFIG_KEYBOARD_LOCK_SOURCE = 'picoclaw-model-config'; + +export function releasePicoclawInputFocus() { + if (typeof document === 'undefined') { + return; + } + + const activeElement = document.activeElement; + if (!(activeElement instanceof HTMLElement)) { + return; + } + + if (activeElement.closest('[data-picoclaw-sidebar="true"]')) { + activeElement.blur(); + } +} diff --git a/web/src/pages/desktop/picoclaw/message-input.tsx b/web/src/pages/desktop/picoclaw/message-input.tsx index 83d6930..fb4ac4b 100644 --- a/web/src/pages/desktop/picoclaw/message-input.tsx +++ b/web/src/pages/desktop/picoclaw/message-input.tsx @@ -4,52 +4,75 @@ import { useSetAtom } from 'jotai'; import { PlusIcon, SendIcon } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts'; +import { keyboardLockAtom } from '@/jotai/keyboard.ts'; import type { PicoclawTransportState } from '@/jotai/picoclaw.ts'; +import { PICOCLAW_INPUT_KEYBOARD_LOCK_SOURCE } from './keyboard-lock.ts'; + type MessageInputProps = { transportState: PicoclawTransportState; - onSend: (content: string) => void | Promise; + disabled?: boolean; + onSend: (content: string) => boolean | void | Promise; onNewConversation: () => void | Promise; disableNewConversation?: boolean; }; export const MessageInput = ({ transportState, + disabled, onSend, onNewConversation, disableNewConversation }: MessageInputProps) => { const { t } = useTranslation(); - const setIsKeyboardEnable = useSetAtom(isKeyboardEnableAtom); + const setKeyboardLock = useSetAtom(keyboardLockAtom); const [value, setValue] = useState(''); + const textareaRef = useRef(null); const isComposingRef = useRef(false); - const isReady = transportState === 'connected'; + const isConnecting = transportState === 'connecting'; + const canSubmit = !disabled && !isConnecting; useEffect(() => { - setIsKeyboardEnable(false); return () => { - setIsKeyboardEnable(true); + setKeyboardLock({ source: PICOCLAW_INPUT_KEYBOARD_LOCK_SOURCE, locked: false }); }; - }, [setIsKeyboardEnable]); + }, [setKeyboardLock]); + + useEffect(() => { + if (!isConnecting) { + return; + } + + textareaRef.current?.blur(); + setKeyboardLock({ source: PICOCLAW_INPUT_KEYBOARD_LOCK_SOURCE, locked: false }); + }, [isConnecting, setKeyboardLock]); async function submit() { const content = value.trim(); - if (!content || !isReady) return; - await onSend(content); - setValue(''); + if (!content || !canSubmit) return; + const sent = await onSend(content); + if (sent !== false) { + setValue(''); + } } return (