fix(backups): return backups repository info one by one instead of erroring on first failure (#10205)

@vates/types/src/xo-app.mts

@@ -328,10 +328,11 @@ export type XoApp = {
     xo: Record<XoBackupRepository['id'], XoConfigBackupArchive[]>
     pool: Record<XoBackupRepository['id'], Record<XoPool['id'], XoPoolBackupArchive[]>>
   }>
+  /** `null` when the listing of a backup repository failed */
   listVmBackupsNg(
     backupRepositoryIds: XoBackupRepository['id'][],
-    opts?: { _forceRefresh?: boolean; vmId: XoVm['id'] }
-  ): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]>>>
+    opts?: { _forceRefresh?: boolean; vmId?: XoVm['id'] }
+  ): Promise<Record<XoBackupRepository['id'], Record<XoVm['id'], XoVmBackupArchive[]> | null>>
   pingRemote(id: XoBackupRepository['id']): Promise<{ success: true }>
   /** Allow to add a new server in the DB (XCP-ng/XenServer) */
   registerXenServer(

@xen-orchestra/rest-api/src/backup-archives/backup-archive.controller.mts

@@ -56,7 +56,7 @@ export class BackupArchiveController extends XoController<XoVmBackupArchive> {
 
     const backupArchivesByRemote = await this.restApi.xoApp.listVmBackupsNg(backupRepositoryIds)
     const vmBackupArchives = Object.values(backupArchivesByRemote)
-      .filter(backupsByVm => backupsByVm !== undefined)
+      .filter(backupsByVm => backupsByVm !== null)
       .map(backupsByVm => Object.values(backupsByVm))
       .flat(2)
 

@xen-orchestra/rest-api/src/vms/vm.service.mts

@@ -421,7 +421,7 @@ export class VmService {
     const backupArchivesByVmByBr = await this.#restApi.xoApp.listVmBackupsNg(brIds, { vmId: vm.id })
 
     return Object.values(backupArchivesByVmByBr)
-      .filter(backupArchiveByVm => backupArchiveByVm !== undefined)
+      .filter(backupArchiveByVm => backupArchiveByVm !== null)
       .flatMap(backupArchiveByVm => backupArchiveByVm[vm.id] ?? [])
       .sort((a, b) => b.timestamp - a.timestamp)
       .splice(0, 3)

CHANGELOG.unreleased.md

@@ -22,6 +22,7 @@
 
 > Users must be able to say: "I had this issue, happy to know it's fixed"
 
+- [Backup/File restore, Backup/Health] An unreachable backup repository no longer slows down every listing: it is skipped after a delay and retried with an increasing backoff (PR [#10205](https://github.com/vatesfr/xen-orchestra/pull/10205))
 - [REST API] Fix `/users/:id/authentication_tokens` sometimes did not return the token used to make the request (PR [#10233](https://github.com/vatesfr/xen-orchestra/pull/10233))
 - [XO server] Fix a random behavior regarding `coresPerSocket` update (PR [#10201](https://github.com/vatesfr/xen-orchestra/pull/10201))
 - [Warm migration] Fix `Vm target of warm migration not found` error at the end of a migration (PR [#10210](https://github.com/vatesfr/xen-orchestra/pull/10210))
@@ -42,6 +43,7 @@
 
 <!--packages-start-->
 
+- @vates/types patch
 - @xen-orchestra/acl minor
 - @xen-orchestra/async-map patch
 - @xen-orchestra/proxy-cli patch
@@ -55,5 +57,6 @@
 - xo-server patch
 - xo-server-ipmi-sensors minor
 - xo-server-netbox patch
+- xo-web patch
 
 <!--packages-end-->

packages/xo-server/src/xo-mixins/backups-ng/index.mjs

@@ -3,6 +3,7 @@ import Disposable from 'promise-toolbox/Disposable'
 import forOwn from 'lodash/forOwn.js'
 import groupBy from 'lodash/groupBy.js'
 import merge from 'lodash/merge.js'
+import { asyncEach } from '@vates/async-each'
 import { createLogger } from '@xen-orchestra/log'
 import { createPredicate } from 'value-matcher'
 import { decorateWith } from '@vates/decorate-with'
@@ -11,6 +12,7 @@ import { HealthCheckVmBackup } from '@xen-orchestra/backups/HealthCheckVmBackup.
 import { ImportVmBackup } from '@xen-orchestra/backups/ImportVmBackup.mjs'
 import { createRunner } from '@xen-orchestra/backups/Backup.mjs'
 import { invalidParameters, noMatchingVm } from 'xo-common/api-errors.js'
+import { timeout } from 'promise-toolbox'
 import { runBackupWorker } from '@xen-orchestra/backups/runBackupWorker.mjs'
 import { Task } from '@vates/task'
 
@@ -21,6 +23,38 @@ import { waitAll } from '../../_waitAll.mjs'
 
 const logger = createLogger('xo:xo-mixins:backups-ng')
 
+/**
+ * @typedef {import('@vates/types').XoBackupRepository} XoBackupRepository
+ * @typedef {import('@vates/types').XoVm} XoVm
+ * @typedef {import('@vates/types').XoVmBackupArchive} XoVmBackupArchive
+ *
+ * @typedef {Record<XoVm['id'], XoVmBackupArchive[]>} BackupsByVm
+ * @typedef {{ backupsByVm?: BackupsByVm, error?: Error }} RemoteListingResult
+ * @typedef {{ _forceRefresh?: boolean, vmId?: XoVm['id'] }} ListVmBackupsOpts
+ * @typedef {{ attempt: number, nextAttemptAt: number, error: Error }} ListingRetryState
+ */
+
+// a remote whose listing failed is not listed again before this delay
+const LISTING_RETRY_DELAY = 30e3
+const LISTING_TIMEOUT = 30e3
+const LISTING_RETRY_MAX_DELAY = 60 * 60 * 1e3 // cap at 1h
+/**
+ * @param {number} attempt number of consecutive failures so far, `0` for the first one
+ * @returns {number} delay in ms before the next attempt is allowed
+ */
+export function backupsListingRetryDelay(attempt) {
+  let prev = 1
+  let curr = 1
+
+  for (let i = 0; i < attempt; i++) {
+    const next = prev + curr
+    prev = curr
+    curr = next
+  }
+
+  return Math.min(prev * LISTING_RETRY_DELAY, LISTING_RETRY_MAX_DELAY)
+}
+
 const parseVmBackupId = id => {
   const i = id.indexOf('/')
   return {
@@ -67,6 +101,11 @@ export default class BackupNg {
     this._app = app
     this._runningRestores = new Set()
 
+    /** @type {Record<XoBackupRepository['id'], ListingRetryState>} */
+    this._backupsListingRetry = { __proto__: null }
+    /** @type {Record<XoBackupRepository['id'], Promise<BackupsByVm>>} */
+    this._trackedBackupsListings = { __proto__: null }
+
     app.hooks.on('start', async () => {
       const executor = async ({
         cancelToken,
@@ -356,7 +395,7 @@ export default class BackupNg {
             return result
           }
         } finally {
-          targetRemoteIds.forEach(id => this._listVmBackupsOnRemote(REMOVE_CACHE_ENTRY, id))
+          targetRemoteIds.forEach(id => this.invalidateVmBackupsListing(id))
         }
       }
       app.registerJobExecutor('backup', executor)
@@ -465,7 +504,7 @@ export default class BackupNg {
         await Disposable.use(app.getBackupsRemoteAdapter(remote), adapter => adapter.deleteVmBackups(filenames))
       }
 
-      this._listVmBackupsOnRemote(REMOVE_CACHE_ENTRY, remoteId)
+      this.invalidateVmBackupsListing(remoteId)
     })
   }
 
@@ -584,59 +623,151 @@ export default class BackupNg {
       return [this, remoteId]
     }
   )
-  async _listVmBackupsOnRemote(remoteId, { vmId } = {}) {
-    const app = this._app
-    try {
-      const remote = await app.getRemoteWithCredentials(remoteId)
-
-      let backupsByVm
-      if (remote.proxy !== undefined) {
-        ;({ [remoteId]: backupsByVm } = await app.callProxyMethod(remote.proxy, 'backup.listVmBackups', {
-          remotes: {
-            [remoteId]: {
-              url: remote.url,
-              options: remote.options,
-            },
-          },
-          vmId,
-        }))
-      } else {
-        backupsByVm = await Disposable.use(app.getBackupsRemoteAdapter(remote), async adapter => {
-          let vmBackups
-          if (vmId !== undefined) {
-            vmBackups = { [vmId]: await adapter.listVmBackups(vmId) }
-          } else {
-            vmBackups = await adapter.listAllVmBackups()
-          }
-
-          return formatVmBackups(vmBackups, remote.id)
-        })
-      }
-
-      // inject the remote id on the backup which is needed for importVmBackupNg()
-      forOwn(backupsByVm, backups =>
-        backups.forEach(backup => {
-          backup.id = `${remoteId}/${backup.id}`
-        })
-      )
-      return backupsByVm
-    } catch (error) {
-      logger.warn(`listVmBackups for remote ${remoteId}:`, { error })
-    }
+  /**
+   * rejects when the listing failed, `_listVmBackupsWithBackoff()` is in charge of handling it
+   *
+   * the timeout must be *inside* the debounced call: a listing which never settles (it can happen
+   * on NFS/SMB) would otherwise stay cached forever and each caller would pay `LISTING_TIMEOUT`
+   * again
+   *
+   * @param {XoBackupRepository['id']} remoteId
+   * @param {{ vmId?: XoVm['id'] }} [opts]
+   * @returns {Promise<BackupsByVm>}
+   */
+  _listVmBackupsOnRemote(remoteId, opts) {
+    return timeout.call(this._listVmBackupsOnRemoteUncached(remoteId, opts), LISTING_TIMEOUT)
   }
 
-  async listVmBackupsNg(remotes, { _forceRefresh = false, vmId } = {}) {
-    const backupsByVmByRemote = {}
+  /**
+   * @param {XoBackupRepository['id']} remoteId
+   * @param {{ vmId?: XoVm['id'] }} [opts]
+   * @returns {Promise<BackupsByVm>}
+   */
+  async _listVmBackupsOnRemoteUncached(remoteId, { vmId } = {}) {
+    const app = this._app
+    const remote = await app.getRemoteWithCredentials(remoteId)
 
-    await Promise.all(
-      remotes.map(async remoteId => {
-        if (_forceRefresh) {
-          this._listVmBackupsOnRemote(REMOVE_CACHE_ENTRY, remoteId)
+    let backupsByVm
+    if (remote.proxy !== undefined) {
+      ;({ [remoteId]: backupsByVm } = await app.callProxyMethod(remote.proxy, 'backup.listVmBackups', {
+        remotes: {
+          [remoteId]: {
+            url: remote.url,
+            options: remote.options,
+          },
+        },
+        vmId,
+      }))
+    } else {
+      backupsByVm = await Disposable.use(app.getBackupsRemoteAdapter(remote), async adapter => {
+        let vmBackups
+        if (vmId !== undefined) {
+          vmBackups = { [vmId]: await adapter.listVmBackups(vmId) }
+        } else {
+          vmBackups = await adapter.listAllVmBackups()
         }
 
-        backupsByVmByRemote[remoteId] = await this._listVmBackupsOnRemote(remoteId, { vmId })
+        return formatVmBackups(vmBackups, remote.id)
+      })
+    }
+
+    // inject the remote id on the backup which is needed for importVmBackupNg()
+    forOwn(backupsByVm, backups =>
+      backups.forEach(backup => {
+        backup.id = `${remoteId}/${backup.id}`
       })
     )
+    return backupsByVm
+  }
+
+  /**
+   * @param {XoBackupRepository['id']} remoteId
+   * @param {Error} error
+   */
+  _scheduleVmBackupsListingRetry(remoteId, error) {
+    const retries = this._backupsListingRetry
+    let state = retries[remoteId]
+    if (state === undefined) {
+      state = retries[remoteId] = { attempt: 0 }
+    }
+
+    const attempt = state.attempt++
+    const delay = backupsListingRetryDelay(attempt)
+    state.error = error
+    state.nextAttemptAt = Date.now() + delay
+
+    // warn on the first failure so that an unreachable backup repository is visible in the
+    // logs, then debug to avoid flooding them while it keeps failing
+    const log = attempt === 0 ? logger.warn : logger.debug
+    log(`listVmBackups for remote ${remoteId} failed, not retrying before ${delay}ms`, { error })
+  }
+
+  /**
+   * never rejects: a failed listing is reported as `error` so that a caller can tell it apart
+   * from a remote which has no backups
+   *
+   * @param {XoBackupRepository['id']} remoteId
+   * @param {{ vmId?: XoVm['id'] }} [opts]
+   * @returns {Promise<RemoteListingResult>} `error` is set when the listing failed, took longer
+   * than `LISTING_TIMEOUT`, or was skipped because the remote is in its retry delay
+   */
+  _listVmBackupsWithBackoff(remoteId, { vmId } = {}) {
+    const state = this._backupsListingRetry[remoteId]
+    if (state !== undefined && state.nextAttemptAt > Date.now()) {
+      // report the failure which put this remote in its retry delay
+      return Promise.resolve({ error: state.error })
+    }
+
+    const promise = this._listVmBackupsOnRemote(remoteId, { vmId })
+
+    // this promise is returned to every caller of the debounce window, even after it has settled:
+    // only track its outcome once, and keep tracking it after it settles or a late caller would
+    // count it a second time
+    if (this._trackedBackupsListings[remoteId] !== promise) {
+      this._trackedBackupsListings[remoteId] = promise
+      promise.then(
+        () => {
+          // ignore the outcome of a listing which has been invalidated in the meantime
+          if (this._trackedBackupsListings[remoteId] === promise) {
+            delete this._backupsListingRetry[remoteId]
+          }
+        },
+        error => {
+          if (this._trackedBackupsListings[remoteId] === promise) {
+            this._scheduleVmBackupsListingRetry(remoteId, error)
+          }
+        }
+      )
+    }
+
+    return promise.then(
+      backupsByVm => ({ backupsByVm }),
+      error => ({ error })
+    )
+  }
+
+  /**
+   * a backup repository whose listing failed is reported as `null` so that a slow or unreachable
+   * one does not prevent the others from being listed
+   *
+   * @param {XoBackupRepository['id'][]} remotes
+   * @param {ListVmBackupsOpts} [opts]
+   * @returns {Promise<Record<XoBackupRepository['id'], BackupsByVm | null>>}
+   */
+  async listVmBackupsNg(remotes, { _forceRefresh = false, vmId } = {}) {
+    /** @type {Record<XoBackupRepository['id'], BackupsByVm | null>} */
+    const backupsByVmByRemote = {}
+
+    await asyncEach(remotes, async remoteId => {
+      if (_forceRefresh) {
+        this.invalidateVmBackupsListing(remoteId)
+      }
+
+      const { backupsByVm, error } = await this._listVmBackupsWithBackoff(remoteId, { vmId })
+
+      // `null` = the listing failed, an empty object = this repository has no backups
+      backupsByVmByRemote[remoteId] = error === undefined ? backupsByVm : null
+    })
 
     return backupsByVmByRemote
   }
@@ -667,4 +798,21 @@ export default class BackupNg {
         }
       })
   }
+  /**
+   * drops the cached listing of a backup repository and its retry state, so that it is listed
+   * again on the next call instead of waiting for the current backoff delay
+   *
+   * the outcome of a listing which is still running is ignored: it no longer represents the
+   * current state of the repository
+   *
+   * public because it is also called by the remotes mixin when a backup repository is updated
+   * or removed
+   *
+   * @param {XoBackupRepository['id']} remoteId
+   */
+  invalidateVmBackupsListing(remoteId) {
+    this._listVmBackupsOnRemote(REMOVE_CACHE_ENTRY, remoteId)
+    delete this._trackedBackupsListings[remoteId]
+    delete this._backupsListingRetry[remoteId]
+  }
 }

packages/xo-server/src/xo-mixins/backups-ng/index.test.mjs

@@ -0,0 +1,185 @@
+import assert from 'assert/strict'
+import test from 'node:test'
+import TimeoutError from 'promise-toolbox/TimeoutError'
+
+import BackupNg, { backupsListingRetryDelay } from './index.mjs'
+
+const { afterEach, beforeEach, describe, it, mock } = test
+
+// must be kept in sync with `index.mjs`
+const LISTING_TIMEOUT = 30e3
+const LISTING_DEBOUNCE = 60e3
+
+// `mock.timers.tick()` is synchronous: yield to the event loop so that the promise continuations
+// it scheduled have run before the assertions
+const tick = async ms => {
+  mock.timers.tick(ms)
+  await new Promise(resolve => setImmediate(resolve))
+}
+
+const createBackupNg = () =>
+  new BackupNg({
+    config: { getDuration: () => LISTING_DEBOUNCE },
+    hooks: { on() {} },
+  })
+
+describe('backupsListingRetryDelay()', () => {
+  it('follows the Fibonacci sequence', () => {
+    assert.deepEqual([0, 1, 2, 3, 4, 5].map(backupsListingRetryDelay), [30e3, 30e3, 60e3, 90e3, 150e3, 240e3])
+  })
+
+  it('is capped at 1h', () => {
+    assert.equal(backupsListingRetryDelay(100), 60 * 60 * 1e3)
+  })
+})
+
+describe('_listVmBackupsWithBackoff()', () => {
+  beforeEach(() => {
+    mock.timers.enable({ apis: ['setTimeout', 'Date'] })
+  })
+
+  afterEach(() => {
+    mock.timers.reset()
+  })
+
+  it('starts a new listing when the previous one never settled', async () => {
+    const backupNg = createBackupNg()
+    let calls = 0
+    backupNg._listVmBackupsOnRemoteUncached = () => {
+      ++calls
+      return new Promise(() => {})
+    }
+
+    const first = backupNg._listVmBackupsWithBackoff('remote')
+    await tick(LISTING_TIMEOUT)
+    assert.ok((await first).error instanceof TimeoutError)
+    assert.equal(calls, 1)
+
+    // the repository is in its retry delay, it is reported as failing without being listed again
+    assert.ok((await backupNg._listVmBackupsWithBackoff('remote')).error instanceof TimeoutError)
+    assert.equal(calls, 1)
+
+    // the debounce entry has been dropped even though the underlying listing is still running:
+    // this is what putting the timeout *inside* the debounced call buys
+    await tick(LISTING_DEBOUNCE)
+    backupNg._listVmBackupsWithBackoff('remote')
+    assert.equal(calls, 2)
+  })
+
+  it('counts a single attempt for all the callers of a listing', async () => {
+    const backupNg = createBackupNg()
+    const error = new Error('unreachable')
+    let calls = 0
+    backupNg._listVmBackupsOnRemoteUncached = () => {
+      ++calls
+      return Promise.reject(error)
+    }
+
+    const results = await Promise.all([
+      backupNg._listVmBackupsWithBackoff('remote'),
+      backupNg._listVmBackupsWithBackoff('remote'),
+      backupNg._listVmBackupsWithBackoff('remote'),
+    ])
+
+    assert.deepEqual(results, [{ error }, { error }, { error }])
+    assert.equal(calls, 1)
+    assert.equal(backupNg._backupsListingRetry.remote.attempt, 1)
+  })
+
+  it('does not count a caller which arrives after the listing failed', async () => {
+    const backupNg = createBackupNg()
+    const error = new Error('unreachable')
+    let calls = 0
+    backupNg._listVmBackupsOnRemoteUncached = () => {
+      ++calls
+      return Promise.reject(error)
+    }
+
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { error })
+    assert.equal(backupNg._backupsListingRetry.remote.attempt, 1)
+
+    // past the retry delay but still inside the debounce window: the caller gets the cached
+    // rejection back and it must not be counted a second time
+    await tick(backupsListingRetryDelay(0) + 1e3)
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { error })
+    assert.equal(calls, 1)
+    assert.equal(backupNg._backupsListingRetry.remote.attempt, 1)
+  })
+
+  it('increases the retry delay on each consecutive failure', async () => {
+    const backupNg = createBackupNg()
+    const error = new Error('unreachable')
+    backupNg._listVmBackupsOnRemoteUncached = () => Promise.reject(error)
+
+    await backupNg._listVmBackupsWithBackoff('remote')
+    assert.equal(backupNg._backupsListingRetry.remote.nextAttemptAt, Date.now() + backupsListingRetryDelay(0))
+
+    // past both the retry delay and the debounce window
+    await tick(LISTING_DEBOUNCE + 1e3)
+    await backupNg._listVmBackupsWithBackoff('remote')
+    assert.equal(backupNg._backupsListingRetry.remote.attempt, 2)
+    assert.equal(backupNg._backupsListingRetry.remote.nextAttemptAt, Date.now() + backupsListingRetryDelay(1))
+  })
+
+  it('clears the retry state when the listing succeeds again', async () => {
+    const backupNg = createBackupNg()
+    const backupsByVm = { vm: [] }
+    const error = new Error('unreachable')
+    let failing = true
+    backupNg._listVmBackupsOnRemoteUncached = () => (failing ? Promise.reject(error) : Promise.resolve(backupsByVm))
+
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { error })
+    failing = false
+
+    // the repository is not listed again before its retry delay has passed
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { error })
+
+    await tick(LISTING_DEBOUNCE + 1e3)
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { backupsByVm })
+    assert.equal(backupNg._backupsListingRetry.remote, undefined)
+  })
+})
+
+describe('invalidateVmBackupsListing()', () => {
+  beforeEach(() => {
+    mock.timers.enable({ apis: ['setTimeout', 'Date'] })
+  })
+
+  afterEach(() => {
+    mock.timers.reset()
+  })
+
+  it('lists the repository again instead of waiting for its retry delay', async () => {
+    const backupNg = createBackupNg()
+    const backupsByVm = { vm: [] }
+    let failing = true
+    backupNg._listVmBackupsOnRemoteUncached = () =>
+      failing ? Promise.reject(new Error('unreachable')) : Promise.resolve(backupsByVm)
+
+    await backupNg._listVmBackupsWithBackoff('remote')
+    failing = false
+
+    backupNg.invalidateVmBackupsListing('remote')
+
+    assert.deepEqual(await backupNg._listVmBackupsWithBackoff('remote'), { backupsByVm })
+  })
+
+  it('ignores the outcome of a listing which is still running', async () => {
+    const backupNg = createBackupNg()
+    let rejectListing
+    backupNg._listVmBackupsOnRemoteUncached = () =>
+      new Promise((resolve, reject) => {
+        rejectListing = reject
+      })
+
+    const pending = backupNg._listVmBackupsWithBackoff('remote')
+    backupNg.invalidateVmBackupsListing('remote')
+
+    rejectListing(new Error('unreachable'))
+    await pending
+
+    // this listing no longer represents the state of the repository, it must not put it in a
+    // retry delay
+    assert.equal(backupNg._backupsListingRetry.remote, undefined)
+  })
+})

packages/xo-server/src/xo-mixins/remotes.mjs

@@ -13,7 +13,7 @@ import Disposable from 'promise-toolbox/Disposable'
 
 // ===================================================================
 
-const { warn, logError } = createLogger('xo:mixins:remotes')
+const { warn, debug } = createLogger('xo:mixins:remotes')
 
 const obfuscateRemote = ({ url, ...remote }) => {
   const parsedUrl = parse(url)
@@ -229,19 +229,23 @@ export default class {
       if (_isRetryableRemoteError(error)) {
         this._scheduleRemoteInfoRetry(remote, error)
       } else {
-        logError('failed to get remote info, will NOT retry', { id: remote.id, error, code: error.code })
+        debug('failed to get remote info, will NOT retry', { id: remote.id, name: remote.name, error })
         this._cancelRemoteInfoRetry(remote.id)
       }
     }
   }
 
-  _scheduleRemoteInfoRetry({ id }, error) {
-    warn('failed to get remote info, will retry', { id, error })
+  _scheduleRemoteInfoRetry({ id, name }, error) {
     let state = this._remotesInfoRetry[id]
     if (!state) {
       state = this._remotesInfoRetry[id] = { attempt: 0 }
     }
-    const delay = remoteInfoRetryDelay(state.attempt++)
+    const attempt = state.attempt++
+    const delay = remoteInfoRetryDelay(attempt)
+
+    const log = attempt === 1 ? warn : debug
+    log('failed to get remote info, will retry', { id, name, delay, error })
+
     state.timer = setTimeout(() => ignoreErrors.call(this._retryRemoteInfo(id)), delay)
     state.timer.unref?.()
   }
@@ -337,6 +341,7 @@ export default class {
     }
 
     this._cancelRemoteInfoRetry(id)
+    this._app.invalidateVmBackupsListing(id)
     if (enabled === false) {
       delete this._remotesInfo[id]
     }
@@ -370,6 +375,7 @@ export default class {
 
   async removeRemote(id) {
     this._cancelRemoteInfoRetry(id)
+    this._app.invalidateVmBackupsListing(id)
     delete this._remotesInfo[id]
 
     const handlers = this._handlers

packages/xo-web/src/xo-app/backup/file-restore/index.js

@@ -8,7 +8,7 @@ import { addSubscriptions, noop, NumericDate } from 'utils'
 import { confirm } from 'modal'
 import { error } from 'notification'
 import { deleteBackups, fetchFiles, listVmBackups, subscribeBackupNgJobs, subscribeRemotes } from 'xo'
-import { filter, find, flatMap, forEach, keyBy, map, orderBy, reduce, toArray } from 'lodash'
+import { filter, find, flatMap, forEach, map, orderBy, reduce } from 'lodash'
 
 import DeleteBackupsModalBody from '../restore/delete-backups-modal-body'
 import RestoreFileModalBody from './restore-file-modal'
@@ -57,62 +57,73 @@ export default class Restore extends Component {
     backupDataByVm: {},
   }
 
+  _refreshId = 0
+
   componentWillReceiveProps(props) {
     if (props.remotes !== this.props.remotes || props.jobs !== this.props.jobs) {
       this._refreshBackupList(props.remotes, props.jobs)
     }
   }
 
-  _refreshBackupList = async (_remotes = this.props.remotes, jobs = this.props.jobs) => {
-    const remotes = keyBy(
-      filter(_remotes, remote => remote.enabled),
-      'id'
-    )
-    const backupsByRemote = await listVmBackups(toArray(remotes))
+  _summarizeBackups = (backups, vmId) => {
+    const sortedBackups = orderBy(backups, 'timestamp', 'desc')
 
-    const backupDataByVm = {}
-    forEach(backupsByRemote, (backups, remoteId) => {
-      const remote = remotes[remoteId]
-      forEach(backups, (vmBackups, vmId) => {
+    return {
+      backups: sortedBackups,
+      first: sortedBackups[sortedBackups.length - 1],
+      last: sortedBackups[0],
+      count: sortedBackups.length, // Number since there's only 1 mode in file restore
+      id: vmId,
+    }
+  }
+
+  _refreshBackupListOnRemote = async (remote, jobs, refreshId) => {
+    const backupsByRemote = await listVmBackups([remote.id])
+
+    if (refreshId !== this._refreshId) {
+      return // a newer refresh has started; discard these stale results
+    }
+
+    this.setState(({ backupDataByVm }) => {
+      const newBackupDataByVm = { ...backupDataByVm }
+
+      forEach(backupsByRemote[remote.id], (vmBackups, vmId) => {
         vmBackups = filter(vmBackups, { mode: 'delta' })
         if (vmBackups.length === 0) {
           return
         }
-        if (backupDataByVm[vmId] === undefined) {
-          backupDataByVm[vmId] = { backups: [] }
-        }
 
-        backupDataByVm[vmId].backups.push(
-          ...map(vmBackups, bkp => {
-            const job = find(jobs, { id: bkp.jobId })
-            return { ...bkp, remote, jobName: job && job.name }
-          })
+        newBackupDataByVm[vmId] = this._summarizeBackups(
+          [
+            ...(newBackupDataByVm[vmId]?.backups ?? []),
+            ...map(vmBackups, bkp => {
+              const job = find(jobs, { id: bkp.jobId })
+              return { ...bkp, remote, jobName: job && job.name }
+            }),
+          ],
+          vmId
         )
       })
-    })
-    let first, last
-    forEach(backupDataByVm, (data, vmId) => {
-      first = { timestamp: Infinity }
-      last = { timestamp: 0 }
-      let count = 0 // Number since there's only 1 mode in file restore
-      forEach(data.backups, backup => {
-        if (backup.timestamp > last.timestamp) {
-          last = backup
-        }
-        if (backup.timestamp < first.timestamp) {
-          first = backup
-        }
-        count++
-      })
 
-      Object.assign(data, { first, last, count, id: vmId })
+      return { backupDataByVm: newBackupDataByVm }
     })
+  }
 
-    forEach(backupDataByVm, ({ backups }, vmId) => {
-      backupDataByVm[vmId].backups = orderBy(backups, 'timestamp', 'desc')
+  _refreshBackupList = (_remotes = this.props.remotes, jobs = this.props.jobs) => {
+    const refreshId = ++this._refreshId
+    return new Promise((resolve, reject) => {
+      this.setState({ backupDataByVm: {} }, () =>
+        Promise.all(
+          map(
+            filter(_remotes, remote => remote.enabled),
+            remote =>
+              this._refreshBackupListOnRemote(remote, jobs, refreshId).catch(() =>
+                error(_('remoteLoadBackupsFailure'), _('remoteLoadBackupsFailureMessage', { name: remote.name }))
+              )
+          )
+        ).then(resolve, reject)
+      )
     })
-
-    this.setState({ backupDataByVm })
   }
 
   // Actions -------------------------------------------------------------------

packages/xo-web/src/xo-app/backup/health/index.js

@@ -176,8 +176,18 @@ const Health = decorate([
       initialize({ fetchBackupList }) {
         return fetchBackupList()
       },
+      // fetch each remote independently so that its backups are displayed as
+      // soon as they are available, instead of waiting for the slowest remote
       async fetchBackupList() {
-        this.state.backupsByRemote = await listVmBackups(toArray(await getRemotes()))
+        const remotes = toArray(await getRemotes())
+        this.state.backupsByRemote = {}
+
+        await Promise.all(
+          remotes.map(async remote => {
+            const backupsByRemote = await listVmBackups([remote])
+            this.state.backupsByRemote = { ...this.state.backupsByRemote, ...backupsByRemote }
+          })
+        )
       },
     },
     computed: {