refactor(desktop): split camera and microphone permissions

desktop/src/common/ipc-events.ts

@@ -2,7 +2,8 @@ export enum IpcEvents {
   GET_APP_VERSION = 'get-app-version',
   GET_PLATFORM = 'get-platform',
   OPEN_EXTERNAL_RUL = 'open-external-url',
-  REQUEST_MEDIA_PERMISSIONS = 'request-media-permissions',
+  CHECK_MEDIA_PERMISSION = 'check-media-permission',
+  REQUEST_MEDIA_PERMISSION = 'request-media-permission',
   SET_FULL_SCREEN = 'set-full-screen',
 
   GET_SERIAL_PORTS = 'get-serial-ports',

desktop/src/main/events/app.ts

@@ -1,4 +1,4 @@
-import { app, BrowserWindow, ipcMain, shell, systemPreferences } from 'electron'
+import { app, BrowserWindow, ipcMain, IpcMainInvokeEvent, shell, systemPreferences } from 'electron'
 import type { IpcMainEvent, OpenExternalOptions } from 'electron'
 
 import { IpcEvents } from '../../common/ipc-events'
@@ -7,7 +7,8 @@ export function registerApp(): void {
   ipcMain.handle(IpcEvents.GET_APP_VERSION, getAppVersion)
   ipcMain.handle(IpcEvents.GET_PLATFORM, getPlatform)
   ipcMain.on(IpcEvents.OPEN_EXTERNAL_RUL, openExternalUrl)
-  ipcMain.handle(IpcEvents.REQUEST_MEDIA_PERMISSIONS, requestMediaPermissions)
+  ipcMain.handle(IpcEvents.CHECK_MEDIA_PERMISSION, checkMediaPermission)
+  ipcMain.handle(IpcEvents.REQUEST_MEDIA_PERMISSION, requestMediaPermission)
   ipcMain.on(IpcEvents.SET_FULL_SCREEN, setFullScreen)
 }
 
@@ -23,17 +24,15 @@ function openExternalUrl(_: IpcMainEvent, url: string, options?: OpenExternalOpt
   shell.openExternal(url, options).catch(console.error)
 }
 
-async function requestMediaPermissions(): Promise<{
-  camera: boolean
-  microphone: boolean
-}> {
-  const camera = await grant('camera')
-  const microphone = await grant('microphone')
-
-  return { camera, microphone }
+function checkMediaPermission(_: IpcMainInvokeEvent, media: 'camera' | 'microphone'): boolean {
+  const status = systemPreferences.getMediaAccessStatus(media)
+  return status === 'granted'
 }
 
-async function grant(media: 'camera' | 'microphone'): Promise<boolean> {
+async function requestMediaPermission(
+  _: IpcMainInvokeEvent,
+  media: 'camera' | 'microphone'
+): Promise<boolean> {
   try {
     const status = systemPreferences.getMediaAccessStatus(media)
     if (status === 'granted') {

desktop/src/renderer/src/App.tsx

@@ -19,7 +19,8 @@ import {
 } from '@renderer/jotai/device'
 import { isKeyboardEnableAtom } from '@renderer/jotai/keyboard'
 import { mouseModeAtom, mouseStyleAtom } from '@renderer/jotai/mouse'
-import { camera } from '@renderer/libs/camera'
+import { camera } from '@renderer/libs/media/camera'
+import { requestCameraPermission } from '@renderer/libs/media/permission'
 import { getVideoResolution } from '@renderer/libs/storage'
 import type { Resolution } from '@renderer/types'
 
@@ -55,34 +56,9 @@ const App = (): ReactElement => {
 
   async function requestMediaPermissions(resolution?: Resolution): Promise<void> {
     try {
-      const platform = await window.electron.ipcRenderer.invoke(IpcEvents.GET_PLATFORM)
-      if (platform === 'darwin') {
-        const res = await window.electron.ipcRenderer.invoke(IpcEvents.REQUEST_MEDIA_PERMISSIONS)
-
-        if (!res.camera) {
-          setState('failed')
-          return
-        }
-      } else {
-        const stream = await navigator.mediaDevices.getUserMedia({
-          video: {
-            width: { ideal: resolution?.width || 1920 },
-            height: { ideal: resolution?.height || 1080 },
-            frameRate: { ideal: 60 }
-          },
-          audio: {
-            echoCancellation: false,
-            noiseSuppression: false,
-            autoGainControl: false,
-            sampleRate: 48000
-          }
-        })
-        stream.getTracks().forEach((track) => track.stop())
-      }
-
-      setState('success')
+      const granted = await requestCameraPermission(resolution)
+      setState(granted ? 'success' : 'failed')
     } catch (err) {
-      console.log('failed to request media permissions: ', err)
       if (err instanceof Error && ['NotAllowedError', 'PermissionDeniedError'].includes(err.name)) {
         setState('failed')
       } else {

desktop/src/renderer/src/components/device/video.tsx

@@ -4,7 +4,7 @@ import { useAtom, useAtomValue } from 'jotai'
 import { useTranslation } from 'react-i18next'
 
 import { resolutionAtom, videoDeviceIdAtom, videoStateAtom } from '@renderer/jotai/device'
-import { camera } from '@renderer/libs/camera'
+import { camera } from '@renderer/libs/media/camera'
 import * as storage from '@renderer/libs/storage'
 import type { MediaDevice } from '@renderer/types'
 

desktop/src/renderer/src/components/menu/audio/index.tsx

@@ -0,0 +1,79 @@
+import { useEffect, useState } from 'react'
+import { Button, Modal } from 'antd'
+import { useSetAtom } from 'jotai'
+import { VolumeOffIcon } from 'lucide-react'
+import { useTranslation } from 'react-i18next'
+
+import { videoDeviceIdAtom, videoStateAtom } from '@renderer/jotai/device'
+import { camera } from '@renderer/libs/media/camera'
+import { checkPermission, requestMicrophonePermission } from '@renderer/libs/media/permission'
+
+export const Audio = () => {
+  const { t } = useTranslation()
+
+  const setVideoState = useSetAtom(videoStateAtom)
+  const setVideoDeviceId = useSetAtom(videoDeviceIdAtom)
+
+  const [isGranted, setIsGranted] = useState(false)
+  const [isModalOpen, setIsModalOpen] = useState(false)
+
+  useEffect(() => {
+    checkPermission('microphone').then((granted) => {
+      setIsGranted(granted)
+    })
+  }, [])
+
+  async function requestPermission(): Promise<void> {
+    try {
+      const granted = await requestMicrophonePermission()
+      if (!granted) {
+        setIsModalOpen(true)
+        return
+      }
+
+      setVideoDeviceId('')
+      setVideoState('disconnected')
+      setIsGranted(granted)
+
+      camera.close()
+    } catch (err: any) {
+      console.log('failed to request media permissions: ', err)
+    }
+  }
+
+  function closeModal(): void {
+    setIsModalOpen(false)
+  }
+
+  if (isGranted) {
+    return null
+  }
+
+  return (
+    <>
+      <div
+        className="flex h-[28px] w-[28px] cursor-pointer items-center justify-center rounded text-neutral-300 hover:bg-neutral-700/70 hover:text-white"
+        onClick={requestPermission}
+      >
+        <VolumeOffIcon size={18} />
+      </div>
+
+      <Modal open={isModalOpen} title={t('audio.tip')} footer={null} onCancel={closeModal}>
+        <div className="py-5 whitespace-pre-line">{t('audio.permission')}</div>
+        <a
+          href="https://wiki.sipeed.com/hardware/en/kvm/NanoKVM_USB/quick_start.html#Authorization"
+          target="_blank"
+          rel="noreferrer"
+        >
+          {t('audio.viewDoc')}
+        </a>
+
+        <div className="flex w-full justify-center pt-8">
+          <Button type="primary" className="min-w-20" onClick={closeModal}>
+            {t('audio.ok')}
+          </Button>
+        </div>
+      </Modal>
+    </>
+  )
+}

desktop/src/renderer/src/components/menu/index.tsx

@@ -6,6 +6,7 @@ import Draggable from 'react-draggable'
 
 import * as storage from '@renderer/libs/storage'
 
+import { Audio } from './audio'
 import { Keyboard } from './keyboard'
 import { Mouse } from './mouse'
 import { Recorder } from './recorder'
@@ -81,6 +82,7 @@ export const Menu = (): ReactElement => {
             <Divider type="vertical" />
 
             <Video />
+            <Audio />
             <SerialPort />
             <Divider type="vertical" className="px-0.5" />
 

desktop/src/renderer/src/components/menu/recorder/index.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useRef, useState } from 'react'
 import { Video } from 'lucide-react'
 
-import { camera } from '@renderer/libs/camera'
+import { camera } from '@renderer/libs/media/camera'
 
 export const Recorder = () => {
   const [isRecording, setIsRecording] = useState(false)

desktop/src/renderer/src/components/menu/video/device.tsx

@@ -6,7 +6,7 @@ import { VideoIcon } from 'lucide-react'
 import { useTranslation } from 'react-i18next'
 
 import { resolutionAtom, videoDeviceIdAtom } from '@renderer/jotai/device'
-import { camera } from '@renderer/libs/camera'
+import { camera } from '@renderer/libs/media/camera'
 import * as storage from '@renderer/libs/storage'
 import type { MediaDevice } from '@renderer/types'
 

desktop/src/renderer/src/components/menu/video/resolution.tsx

@@ -7,7 +7,7 @@ import { useTranslation } from 'react-i18next'
 
 import { resolutionAtom } from '@renderer/jotai/device'
 import { isKeyboardEnableAtom } from '@renderer/jotai/keyboard'
-import { camera } from '@renderer/libs/camera'
+import { camera } from '@renderer/libs/media/camera'
 import * as storage from '@renderer/libs/storage'
 import type { Resolution as VideoResolution } from '@renderer/types'
 

desktop/src/renderer/src/i18n/locales/en.ts

@@ -37,6 +37,13 @@ const en = {
         cancel: 'Cancel'
       }
     },
+    audio: {
+      tip: 'Tip',
+      permission:
+        'Microphone access is required to connect your USB audio device. The operating system classifies USB inputs as microphones, so this permission is necessary.\n\nThis action is solely for device connectivity and does not enable audio recording.',
+      viewDoc: 'View document.',
+      ok: 'Ok'
+    },
     keyboard: {
       paste: 'Paste',
       virtualKeyboard: 'Keyboard',

desktop/src/renderer/src/i18n/locales/ja.ts

@@ -65,7 +65,8 @@ const ja = {
       speed: 'ホイール速度',
       fast: '速い',
       slow: '遅い',
-      requestPointer: '相対モードを使用中です。マウスポインターを取得するにはデスクトップをクリックしてください。',
+      requestPointer:
+        '相対モードを使用中です。マウスポインターを取得するにはデスクトップをクリックしてください。',
       jiggler: {
         title: 'マウスジグラー',
         enable: '有効',
@@ -102,8 +103,7 @@ const ja = {
         warningDescription: 'この操作は元に戻せません。すべてのカスタム設定が失われます。',
         button: 'すべての設定をリセット',
         confirmTitle: 'リセットの確認',
-        confirmMessage:
-          'すべての設定をリセットしてもよろしいですか？この操作は元に戻せません。',
+        confirmMessage: 'すべての設定をリセットしてもよろしいですか？この操作は元に戻せません。',
         confirm: 'リセット',
         cancel: 'キャンセル'
       }

desktop/src/renderer/src/i18n/locales/zh.ts

@@ -36,6 +36,13 @@ const zh = {
         cancel: '取消'
       }
     },
+    audio: {
+      tip: '提示',
+      permission:
+        '需要麦克风权限来获取 USB 设备的音频信号。因为电脑系统会将 USB 音频输入设备识别为麦克风，而非扬声器。\n\n此操作仅用于设备连接，不会录制任何声音。',
+      viewDoc: '查看文档。',
+      ok: '确定'
+    },
     kkeyboard: {
       paste: '粘贴',
       virtualKeyboard: '虚拟键盘',

desktop/src/renderer/src/libs/media/camera.ts

@@ -1,3 +1,5 @@
+import { checkPermission } from '@renderer/libs/media/permission'
+
 class Camera {
   id: string = ''
   width: number = 1920
@@ -12,16 +14,18 @@ class Camera {
 
     this.close()
 
-    const constraints = {
-      video: {
-        deviceId: { exact: id },
-        width: { ideal: width },
-        height: { ideal: height },
-        frameRate: { ideal: 60 },
-        latency: { ideal: 0 },
-        resizeMode: 'none'
-      },
-      audio: audioId
+    const video = {
+      deviceId: { exact: id },
+      width: { ideal: width },
+      height: { ideal: height },
+      frameRate: { ideal: 60 },
+      latency: { ideal: 0 },
+      resizeMode: 'none'
+    }
+
+    const isMicGranted = await checkPermission('microphone')
+    const audio =
+      isMicGranted && audioId
         ? {
             deviceId: { exact: audioId },
             echoCancellation: false,
@@ -31,13 +35,12 @@ class Camera {
             latency: 0
           }
         : false
-    }
 
     this.id = id
     this.width = width
     this.height = height
     if (audioId) this.audioId = audioId
-    this.stream = await navigator.mediaDevices.getUserMedia(constraints)
+    this.stream = await navigator.mediaDevices.getUserMedia({ video, audio })
   }
 
   public async updateResolution(width: number, height: number): Promise<void> {

desktop/src/renderer/src/libs/media/permission.ts

@@ -0,0 +1,75 @@
+import { IpcEvents } from '@common/ipc-events'
+import type { Resolution } from '@renderer/types'
+
+export async function checkPermission(device: 'camera' | 'microphone'): Promise<boolean> {
+  try {
+    const platform = await window.electron.ipcRenderer.invoke(IpcEvents.GET_PLATFORM)
+    if (platform === 'darwin') {
+      return await window.electron.ipcRenderer.invoke(IpcEvents.CHECK_MEDIA_PERMISSION, device)
+    }
+
+    const status = await navigator.permissions.query({
+      name: device as PermissionName
+    })
+    return status.state === 'granted'
+  } catch (error) {
+    console.error(error)
+    return false
+  }
+}
+
+export async function requestCameraPermission(resolution?: Resolution): Promise<boolean> {
+  try {
+    const platform = await window.electron.ipcRenderer.invoke(IpcEvents.GET_PLATFORM)
+    if (platform === 'darwin') {
+      return await window.electron.ipcRenderer.invoke(IpcEvents.REQUEST_MEDIA_PERMISSION, 'camera')
+    }
+
+    const granted = await checkPermission('camera')
+    if (granted) {
+      return true
+    }
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+      video: {
+        width: { ideal: resolution?.width || 1920 },
+        height: { ideal: resolution?.height || 1080 },
+        frameRate: { ideal: 60 }
+      }
+    })
+    stream.getTracks().forEach((track) => track.stop())
+    return true
+  } catch (err: any) {
+    return !(err.name === 'NotAllowedError' || err.name === 'PermissionDeniedError')
+  }
+}
+
+export async function requestMicrophonePermission(): Promise<boolean> {
+  try {
+    const platform = await window.electron.ipcRenderer.invoke(IpcEvents.GET_PLATFORM)
+    if (platform === 'darwin') {
+      return await window.electron.ipcRenderer.invoke(
+        IpcEvents.REQUEST_MEDIA_PERMISSION,
+        'microphone'
+      )
+    }
+
+    const granted = await checkPermission('microphone')
+    if (granted) {
+      return true
+    }
+
+    const stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: false,
+        noiseSuppression: false,
+        autoGainControl: false,
+        sampleRate: 48000
+      }
+    })
+    stream.getTracks().forEach((track) => track.stop())
+    return true
+  } catch (err: any) {
+    return !(err.name === 'NotAllowedError' || err.name === 'PermissionDeniedError')
+  }
+}