feat(xo-server): server status become a persistant property

@vates/types/src/xo.mts

@@ -602,7 +602,7 @@ export type XoServer = {
   poolNameDescription?: string
   poolNameLabel?: string
   readOnly: boolean
-  status: 'connected' | 'disconnected' | 'connecting'
+  status: 'connected' | 'disconnecting' | 'disconnected' | 'connecting'
   username: string
 }
 

@xen-orchestra/web-core/lib/components/status/VtsStatus.vue

@@ -14,6 +14,7 @@ export type Status =
   | 'connecting'
   | 'connected'
   | 'disconnected'
+  | 'disconnecting'
   | 'partially-connected'
   | 'disconnected-from-physical-device'
   | 'physically-disconnected'
@@ -43,6 +44,7 @@ const currentStatus = useMapper<Status, { text: string; accent: InfoAccent }>(
     ['connecting', { text: t('connecting'), accent: 'info' }],
     ['connected', { text: t('connected'), accent: 'success' }],
     ['disconnected', { text: t('disconnected'), accent: 'danger' }],
+    ['disconnecting', { text: t('disconnecting'), accent: 'info' }],
     ['partially-connected', { text: t('partially-connected'), accent: 'warning' }],
     ['disconnected-from-physical-device', { text: t('disconnected-from-physical-device'), accent: 'warning' }],
     ['physically-disconnected', { text: t('disconnected-from-physical-device'), accent: 'danger' }],

@xen-orchestra/web-core/lib/locales/en.json

@@ -356,6 +356,7 @@
   "disabled": "Disabled",
   "disconnect-vifs-in-xo-5-to-delete-network": "To delete a network, you must first disconnect all VIFs attached to it. You can manage your VMs' VIFs in {xo-5}.",
   "disconnected": "Disconnected",
+  "disconnecting": "Disconnecting",
   "disconnected-from-physical-device": "Disconnected from physical device",
   "disconnected-pbd-number": "Disconnected PBD #{n}",
   "disk-name": "Disk name",

@xen-orchestra/web-core/lib/locales/fr.json

@@ -356,6 +356,7 @@
   "disabled": "Désactivé | Désactivés",
   "disconnect-vifs-in-xo-5-to-delete-network": "Pour supprimer un réseau, vous devez d'abord déconnecter tous les VIFs qui y sont attachés. Vous pouvez gérer les VIFs de vos VMs dans {xo-5}.",
   "disconnected": "Déconnecté",
+  "disconnecting": "Déconnexion",
   "disconnected-from-physical-device": "Déconnecté de l'appareil physique",
   "disconnected-pbd-number": "PBD déconnecté #{n}",
   "disk-name": "Nom du disque",

@xen-orchestra/web/src/modules/pool/components/actions/connection/PoolConnectionToggleButton.vue

@@ -1,7 +1,7 @@
 <template>
   <template v-if="server">
     <UiButton
-      v-if="server.status === 'connected'"
+      v-if="server.status === 'connected' || server.status === 'connecting'"
       v-tooltip="!canDisconnectServer ? disconnectServerErrorMessage : undefined"
       left-icon="action:disconnect"
       variant="tertiary"

@xen-orchestra/web/src/modules/pool/components/list/PoolsTable.vue

@@ -74,6 +74,7 @@ const schema = useQueryBuilderSchema<PoolFilterableData>({
   poolStatus: useStringSchema(t('status'), {
     connected: t('connected'),
     connecting: t('connecting'),
+    disconnecting: t('disconnecting'),
     disconnected: t('disconnected'),
     'unable-to-connect-to-the-pool': t('unable-to-connect-to-the-pool'),
   }),
@@ -157,7 +158,7 @@ const { HeadCells, BodyCells } = useServerColumns({
         r({
           onClick: () => (selectedServerId.value = server.id),
           actions: [
-            server.status === 'connected'
+            server.status === 'connected' || server.status === 'connecting'
               ? {
                   label: t('action:disconnect-pool'),
                   icon: 'action:disconnect',

@xen-orchestra/web/src/modules/pool/components/list/panel/PoolSidePanel.vue

@@ -226,6 +226,7 @@ const connectionStatus = useMapper(
   {
     error: { accent: 'danger', text: t('unable-to-connect-to-the-pool') },
     disconnected: { accent: 'muted', text: t('disconnected') },
+    disconnecting: { accent: 'info', text: t('disconnecting') }, // TODO: fix when designed
     connected: { accent: 'success', text: t('connected') },
     connecting: { accent: 'info', text: t('connecting') },
   },

packages/xo-server/src/_xenServerAutoReconnect.mjs

@@ -13,14 +13,13 @@ export const MAX_DELAY = 60e3
 // - connect(id): attempt the connection, throws on failure
 // - delay(ms): wait before the next attempt
 // - getServer(id): server record ({ enabled }), throws if the server was deleted
-// - getStatus(id): 'disconnected' | 'connecting' | 'connected'
 // - isFatal(error): optional, true for permanent errors not worth retrying
 // - isGone(error): optional, true when a getServer error means the server was deleted
 //
 // Resolves with the reason why the loop stopped, never rejects.
 export async function autoReconnect(
   id,
-  { connect, delay, getServer, getStatus, isFatal = () => false, isGone = () => true, log }
+  { connect, delay, getServer, isFatal = () => false, isGone = () => true, log }
 ) {
   for (const ms of fibonacci()
     .toMs()
@@ -41,7 +40,7 @@ export async function autoReconnect(
     if (!server.enabled) {
       return 'server disabled'
     }
-    if (getStatus(id) !== 'disconnected') {
+    if (server.status !== 'disconnected') {
       return 'already connected'
     }
 

packages/xo-server/src/_xenServerAutoReconnect.test.mjs

@@ -9,8 +9,7 @@ function makeDeps(overrides) {
   return {
     connect: async () => {},
     delay: () => Promise.resolve(),
-    getServer: async () => ({ enabled: true }),
-    getStatus: () => 'disconnected',
+    getServer: async () => ({ enabled: true, status: 'disconnected' }),
     log: noopLog,
     ...overrides,
   }
@@ -61,7 +60,7 @@ describe('autoReconnect', () => {
           if (++reads === 1) {
             throw new Error('database hiccup')
           }
-          return { enabled: true }
+          return { enabled: true, status: 'disconnected' }
         },
         isGone: () => false,
         connect: async () => {
@@ -79,7 +78,7 @@ describe('autoReconnect', () => {
     const outcome = await autoReconnect(
       's1',
       makeDeps({
-        getStatus: () => 'connected',
+        getServer: () => ({ enabled: true, status: 'connected' }),
         connect: async () => {
           connectCalled = true
         },

packages/xo-server/src/xo-mixins/_xen-servers.test.mjs

@@ -3,17 +3,44 @@ import { describe, it } from 'node:test'
 import { configure } from '@xen-orchestra/log/configure'
 
 import XenServers from './xen-servers.mjs'
+import EventEmitter from 'node:events'
 
 configure({ level: 'FATAL', transport: () => {} })
 
-// Minimal XoApp mock: the constructor only registers hooks and watches a config
-// duration, none of which are fired by these tests
 const createMockApp = () => ({
   config: { watchDuration: () => {} },
-  hooks: { on: () => {} },
+  hooks: new EventEmitter(),
+  _redis: {
+    sAdd() {
+      return new Promise(resolve => resolve())
+    },
+    sMembers() {
+      return new Promise(resolve => resolve())
+    },
+    get() {
+      return new Promise(resolve => resolve({}))
+    },
+    set() {
+      return new Promise(resolve => resolve())
+    },
+    del() {
+      return new Promise(resolve => resolve())
+    },
+    keys() {
+      return new Promise(resolve => resolve([]))
+    },
+  },
+  addConfigManager() {},
 })
 
-const SERVER = { id: 'server-1', enabled: true, host: '192.0.2.1', password: 'secret', username: 'root' }
+const SERVER = {
+  id: 'server-1',
+  enabled: true,
+  host: '192.0.2.1',
+  password: 'secret',
+  username: 'root',
+  status: 'disconnected',
+}
 
 // `poolId` defaults to the pool the connection was opened with, `$id` is the
 // one it currently reports (they differ after a pool UUID change)
@@ -26,7 +53,9 @@ const createXapi = ({ $id = 'pool-1' } = {}) => ({
 // `_servers` is normally created on the `core started` hook, and
 // `_autoReconnectXenServer` would start a real reconnection loop
 const createXenServers = (...servers) => {
-  const xenServers = new XenServers(createMockApp(), { safeMode: true })
+  const app = createMockApp()
+  const xenServers = new XenServers(app, { safeMode: true })
+  app.hooks.emit('core started')
 
   const stored = { __proto__: null }
   const serversToInitialize = servers.length === 0 ? [SERVER] : servers
@@ -107,14 +136,15 @@ describe('updateXenServer', function () {
 })
 
 describe('disconnectXenServer', function () {
-  const connect = (xenServers, { poolId = 'pool-1', serverId = 'server-1' } = {}) => {
+  const connect = async (xenServers, { poolId = 'pool-1', serverId = 'server-1' } = {}) => {
     xenServers._xapis[serverId] = createXapi({ $id: poolId })
     xenServers._serverIdsByPool[poolId] = serverId
+    await xenServers.updateXenServer(serverId, { status: 'connected' })
   }
 
   it('forgets the pool of the server', async function () {
     const { xenServers } = createXenServers()
-    connect(xenServers)
+    await connect(xenServers)
 
     await xenServers.disconnectXenServer('server-1')
 
@@ -124,7 +154,7 @@ describe('disconnectXenServer', function () {
 
   it('forgets the pool of the server even after a pool UUID change', async function () {
     const { xenServers } = createXenServers()
-    connect(xenServers, { poolId: 'pool-1' })
+    await connect(xenServers, { poolId: 'pool-1' })
     // `_onXenAdd` re-registers the server under the new identifier
     delete xenServers._serverIdsByPool['pool-1']
     xenServers._serverIdsByPool['pool-2'] = 'server-1'
@@ -138,21 +168,23 @@ describe('disconnectXenServer', function () {
     // a second entry registered on an already connected pool stays `enabled`
     // but disconnected, with a `PoolAlreadyConnected` error: disconnecting or
     // deleting it must not release the pool of the server which owns it
-    const { xenServers } = createXenServers(SERVER, { ...SERVER, id: 'server-2' })
-    connect(xenServers)
+    const { reconnected, xenServers } = createXenServers(SERVER, { ...SERVER, id: 'server-2' })
+    await connect(xenServers)
     const xapi = xenServers._xapis['server-1']
 
     await xenServers.disconnectXenServer('server-2')
 
     assert.deepEqual({ ...xenServers._serverIdsByPool }, { 'pool-1': 'server-1' })
     assert.equal(xenServers._xapis['server-1'], xapi)
-    assert.equal(xenServers._getXenServerStatus('server-1'), 'connected')
+    const server1 = await xenServers.getXenServer('server-1')
+    assert.equal(server1.status, 'connected')
+    assert.deepEqual(reconnected, [])
   })
 
   it('leaves the pools of the other servers alone', async function () {
-    const { xenServers } = createXenServers()
-    connect(xenServers)
-    connect(xenServers, { poolId: 'pool-2', serverId: 'server-2' })
+    const { xenServers } = createXenServers(SERVER, { ...SERVER, id: 'server-2' })
+    await connect(xenServers)
+    await connect(xenServers, { poolId: 'pool-2', serverId: 'server-2' })
 
     await xenServers.disconnectXenServer('server-1')
 

packages/xo-server/src/xo-mixins/xen-servers.mjs

@@ -43,15 +43,6 @@ const MAX_TIMER_DELAY = 2 ** 31 - 1
 const DEFAULT_LOAD_BALANCER_RE_ENABLE_DELAY = 30 * 60 * 1000 // 30 minutes, same as config.toml
 const synchronizedLoadBalancerOperation = synchronized()(operation => operation())
 
-// Server is disconnected:
-// - _xapis[server.id] is undefined
-
-// Server is connecting:
-// - _xapis[server.id] is defined
-
-// Server is connected:
-// - _xapis[server.id] id defined
-// - _serverIdsByPool[xapi.pool.$id] is server.id
 export default class XenServers {
   constructor(app, { safeMode }) {
     this._objectConflicts = { __proto__: null } // TODO: clean when a server is disconnected.
@@ -81,18 +72,38 @@ export default class XenServers {
         }
       }
     }
-    app.hooks.on('core started', () => {
+    let timeoutId
+    let promiseSetupServersStatus = new Promise((_resolve, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new Error('unable to setup the server collection. Timed out'))
+      }, 1000)
+    })
+    app.hooks.on('core started', async () => {
       const serversDb = (this._servers = new Servers({
         connection: app._redis,
         namespace: 'server',
         indexes: ['host'],
         crypto: app.cryptoCredentials,
       }))
+      // at XO startup, set all servers as disconnected
+      clearTimeout(timeoutId)
+      promiseSetupServersStatus = (async () => {
+        const servers = await serversDb.get()
+        await serversDb.update(servers.map(server => ({ ...server, status: 'disconnected' })))
+      })()
+      await promiseSetupServersStatus
+
+      app.hooks.emit('registerCollection', {
+        collection: serversDb,
+        type: 'server',
+        decorate: this._decorateXenServer.bind(this),
+      })
 
       app.addConfigManager(
         'xenServers',
         () => serversDb.get(),
-        servers => serversDb.update(servers).then(connectServers)
+        // when importing an XO config, set all servers as disconnected
+        servers => serversDb.update(servers.map(server => ({ ...server, status: 'disconnected' }))).then(connectServers)
       )
     })
     app.hooks.on('start', async () => {
@@ -109,6 +120,7 @@ export default class XenServers {
       }
 
       if (!safeMode) {
+        await promiseSetupServersStatus
         await connectServers()
       }
     })
@@ -136,6 +148,7 @@ export default class XenServers {
       password,
       readOnly,
       username,
+      status: 'disconnected',
     })
 
     return server
@@ -180,6 +193,7 @@ export default class XenServers {
       'poolNameDescription',
       'poolNameLabel',
       'username',
+      'status',
     ]) {
       let value = properties[key]
       if (value !== undefined) {
@@ -245,7 +259,7 @@ export default class XenServers {
       canFixConnection &&
       server.enabled &&
       !this._connectingXenServers.has(id) &&
-      this._getXenServerStatus(id) === 'disconnected'
+      server.status === 'disconnected'
     ) {
       this._autoReconnectXenServer(id)
     }
@@ -397,7 +411,6 @@ export default class XenServers {
       connect: id => this.connectXenServer(id, { enable: false }),
       delay: pDelay,
       getServer: id => this.getXenServer(id),
-      getStatus: id => this._getXenServerStatus(id),
       isFatal: error => error instanceof PoolAlreadyConnected,
       isGone: error => noSuchObject.is(error),
       log,
@@ -415,7 +428,7 @@ export default class XenServers {
 
   async connectXenServer(id, { enable = true } = {}) {
     const server = await this.getXenServerWithCredentials(id)
-    const serverStatus = this._getXenServerStatus(id)
+    const serverStatus = server.status
     // `_connectingXenServers` also guards against a concurrent connection
     // attempt for the same server, which would overwrite `_xapis[id]` and leak
     // a live connection
@@ -437,9 +450,7 @@ export default class XenServers {
   }
 
   async _connectXenServer(id, server, { enable }) {
-    if (enable) {
-      await this.updateXenServer(id, { enabled: true })
-    }
+    await this.updateXenServer(id, { status: 'connecting', enabled: enable ? true : undefined })
 
     const { config } = this._app
 
@@ -473,7 +484,8 @@ export default class XenServers {
 
       // requesting disconnection on the connecting server
       if (this._xapis[server.id] === undefined) {
-        xapi.disconnect()::ignoreErrors()
+        await xapi.disconnect()::ignoreErrors()
+        await this.updateXenServer(id, { status: 'disconnected' })
         return
       }
 
@@ -617,7 +629,9 @@ export default class XenServers {
       xapi.xo.install()
       xapi.watchEvents()
 
-      this.updateXenServer(id, { error: null })::ignoreErrors()
+      const _updateXenServer = (id, props) => {
+        this.updateXenServer(id, props)::ignoreErrors()
+      }
 
       const onEventFetchingError = () => {
         const timeout = setTimeout(() => {
@@ -625,6 +639,7 @@ export default class XenServers {
 
           // switch server status from connected to connecting
           this._forgetXenServerPool(server.id)
+          _updateXenServer(server.id, { status: 'connecting' })
         }, this._xapiMarkDisconnectedDelay)
         xapi.once('eventFetchingSuccess', () => {
           xapi.once('eventFetchingError', onEventFetchingError)
@@ -633,6 +648,7 @@ export default class XenServers {
             // replays the pool object which registers it under the right one
             serverIdsByPool[poolId] = server.id
             xapi.xo.install()
+            _updateXenServer(server.id, { error: null, status: 'connected' })
           } else {
             clearTimeout(timeout)
           }
@@ -640,27 +656,33 @@ export default class XenServers {
       }
       xapi.once('eventFetchingError', onEventFetchingError)
 
-      xapi.once('disconnected', () => {
+      xapi.once('disconnected', async () => {
         xapi.xo.uninstall()
         delete this._xapis[server.id]
         this._forgetXenServerPool(server.id)
         this._app.emit('server:disconnected', { server, xapi })
+        const _server = await this.getXenServerWithCredentials(id)
+        if (_server.status !== 'disconnected') {
+          await this.updateXenServer(server.id, { status: 'disconnected' })
+        }
 
         // deliberate disconnections set `enabled` to false beforehand, in
         // which case the loop stops on its own
         this._autoReconnectXenServer(server.id)
       })
       this._app.emit('server:connected', { server, xapi })
+      await this.updateXenServer(id, { error: null, status: 'connected' })::ignoreErrors()
     } catch (error) {
       delete this._xapis[server.id]
-      xapi.disconnect()::ignoreErrors()
+      await xapi.disconnect()::ignoreErrors()
+      await this.updateXenServer(id, { status: 'disconnected' })
 
       const serializedError = serializeError(error)
 
       // avoid a database write per auto-reconnect attempt when the error did not change
       const previousError = server.error
       if (previousError?.code !== serializedError.code || previousError?.message !== serializedError.message) {
-        this.updateXenServer(id, { error: serializedError })::ignoreErrors()
+        await this.updateXenServer(id, { error: serializedError })::ignoreErrors()
       }
 
       // permanent errors: retrying is pointless, do not start the loop
@@ -690,7 +712,7 @@ export default class XenServers {
   async disconnectXenServer(id) {
     // throw no such object if the server does not exist
     const server = await this.getXenServer(id)
-    const status = this._getXenServerStatus(id)
+    const status = server.status
     if (status === 'disconnected' && !server.enabled) {
       throw incorrectState({
         actual: status,
@@ -699,7 +721,11 @@ export default class XenServers {
         property: 'status',
       })
     }
-    await this.updateXenServer(id, { enabled: false })
+
+    await this.updateXenServer(id, {
+      enabled: false,
+      status: status === 'disconnected' ? undefined : 'disconnecting',
+    })
 
     /**
      * if the server is enabled but disconnected, xapi is undefined
@@ -710,7 +736,10 @@ export default class XenServers {
 
     this._forgetXenServerPool(id)
 
-    return xapi?.disconnect()
+    await xapi?.disconnect()
+    if (status !== 'disconnected') {
+      await this.updateXenServer(id, { status: 'disconnected' })
+    }
   }
 
   getAllXapis() {
@@ -728,15 +757,6 @@ export default class XenServers {
     return this.getXapi(xoObject).getObjectByRef(xoObject._xapiRef)
   }
 
-  _getXenServerStatus(id) {
-    const xapi = this._xapis[id]
-    return xapi === undefined
-      ? 'disconnected'
-      : this._serverIdsByPool[xapi.pool?.$id] === id
-        ? 'connected'
-        : 'connecting'
-  }
-
   _decorateXenServer(server) {
     const xapis = this._xapis
 
@@ -747,9 +767,8 @@ export default class XenServers {
     ) {
       server.error = serializeError(xapis[server.id].watchEventsError)
     }
-    server.status = this._getXenServerStatus(server.id)
-    if (server.status === 'connected') {
-      const xapi = xapis[server.id]
+    const xapi = xapis[server.id]
+    if (server.status === 'connected' && xapi !== undefined) {
       server.poolId = xapi.pool.uuid
       try {
         server.master = xapi.getObjectByRef(xapi.pool.master).uuid