feat(vmware-explorer): Use NFS datastore from XO Remote to bypass VMFS6 lock

- Migrate VM to NFS on VMWare side
- mount NFS in XO , named [VMWARE]{datastoreName}
- launch import and it will read through the direct NFS instead of using the soap access layer

@xen-orchestra/vmware-explorer/DatastoreSoapEsxi.mjs

@@ -0,0 +1,22 @@
+import { Datastore } from './_Datastore.mjs'
+
+export class DatastoreSoapEsxi extends Datastore {
+  #esxi
+  constructor(datastoreName, { esxi, ...otherOptions } = {}) {
+    super(datastoreName, otherOptions)
+    this.#esxi = esxi
+  }
+  async getStream(path, start, end) {
+    const res = await this.#esxi.download(this.datastoreName, path, start || end ? `${start}-${end}` : undefined)
+    return res.body
+  }
+  async getBuffer(path, start, end) {
+    return (
+      await this.#esxi.download(this.datastoreName, path, start || end ? `${start}-${end - 1}` : undefined)
+    ).buffer()
+  }
+  async getSize(path) {
+    const res = await this.#esxi.download(this.datastoreName, path)
+    return Number(res.headers.get('content-length'))
+  }
+}

@xen-orchestra/vmware-explorer/DatastoreXoRemote.mjs

@@ -0,0 +1,22 @@
+import assert from 'node:assert'
+import { Datastore } from './_Datastore.mjs'
+
+export const DatastoreXoRemote = class DatastoreXoRemote extends Datastore {
+  #handler
+  constructor(datastoreName, { handler, ...otherOptions }) {
+    super(datastoreName, otherOptions)
+    this.#handler = handler
+  }
+  async getStream(path, start, end) {
+    return this.#handler.createReadStream(path, { start, end })
+  }
+  async getBuffer(path, start, end) {
+    const buffer = Buffer.alloc(end - start)
+    const { bytesRead } = await this.#handler.read(path, buffer, start)
+    assert.strictEqual(bytesRead, end - start)
+    return buffer
+  }
+  async getSize(path) {
+    return this.#handler.getSize(path)
+  }
+}

@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs

@@ -4,9 +4,9 @@ import { FOOTER_SIZE } from 'vhd-lib/_constants.js'
 import { notEqual, strictEqual } from 'node:assert'
 import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
 import { VhdAbstract } from 'vhd-lib'
+import { openDatastore } from './_openDatastore.mjs'
 
 export default class VhdEsxiCowd extends VhdAbstract {
-  #esxi
   #datastore
   #parentVhd
   #path
@@ -17,14 +17,14 @@ export default class VhdEsxiCowd extends VhdAbstract {
 
   #grainDirectory
 
-  static async open(esxi, datastore, path, parentVhd, opts) {
-    const vhd = new VhdEsxiCowd(esxi, datastore, path, parentVhd, opts)
+  static async open(datastoreName, path, parentVhd, opts) {
+    const datastore = openDatastore(datastoreName, opts)
+    const vhd = new VhdEsxiCowd(datastore, path, parentVhd, opts)
     await vhd.readHeaderAndFooter()
     return vhd
   }
-  constructor(esxi, datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) {
+  constructor(datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) {
     super()
-    this.#esxi = esxi
     this.#path = path
     this.#datastore = datastore
     this.#parentVhd = parentVhd
@@ -54,7 +54,7 @@ export default class VhdEsxiCowd extends VhdAbstract {
   }
 
   async #read(start, length) {
-    return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${start + length - 1}`)).buffer()
+    return this.#datastore.getBuffer(this.#path, start, start + length)
   }
 
   async readHeaderAndFooter() {

@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs

@@ -6,6 +6,7 @@ import { Task } from '@vates/task'
 import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
 import { VhdAbstract } from 'vhd-lib'
 import assert from 'node:assert'
+import { openDatastore } from './_openDatastore.mjs'
 
 /* eslint-disable no-console */
 
@@ -13,7 +14,6 @@ import assert from 'node:assert'
 const VHD_BLOCK_LENGTH = 2 * 1024 * 1024
 export default class VhdEsxiRaw extends VhdAbstract {
   #datastore
-  #esxi
   #path
   #thin
 
@@ -25,8 +25,9 @@ export default class VhdEsxiRaw extends VhdAbstract {
   #stream
   #reading = false
 
-  static async open(esxi, datastore, path, opts) {
-    const vhd = new VhdEsxiRaw(esxi, datastore, path, opts)
+  static async open(datastoreName, path, opts) {
+    const datastore = openDatastore(datastoreName, opts)
+    const vhd = new VhdEsxiRaw(datastore, path, opts)
     await vhd.readHeaderAndFooter()
     return vhd
   }
@@ -39,17 +40,15 @@ export default class VhdEsxiRaw extends VhdAbstract {
     return this.#footer
   }
 
-  constructor(esxi, datastore, path, { thin = true } = {}) {
+  constructor(datastore, path, { thin = true } = {}) {
     super()
-    this.#esxi = esxi
     this.#path = path
     this.#datastore = datastore
     this.#thin = thin
   }
 
   async readHeaderAndFooter() {
-    const res = await this.#esxi.download(this.#datastore, this.#path)
-    const length = res.headers.get('content-length')
+    const length = await this.#datastore.getSize(this.#path)
 
     this.#header = unpackHeader(createHeader(length / VHD_BLOCK_LENGTH))
     const geometry = _computeGeometryForSize(length)
@@ -85,8 +84,7 @@ export default class VhdEsxiRaw extends VhdAbstract {
       // no stream
       if (this.#stream === undefined) {
         const end = this.footer.currentSize - 1
-        const res = await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)
-        this.#stream = res.body
+        this.#stream = await this.#datastore.getStream(this.#path, start, end)
         this.#streamOffset = start
       }
 
@@ -201,8 +199,7 @@ export default class VhdEsxiRaw extends VhdAbstract {
   }
 
   rawContent() {
-    return this.#esxi.download(this.#datastore, this.#path).then(res => {
-      const stream = res.body
+    return this.#datastore.getStream(this.#path).then(stream => {
       stream.length = this.footer.currentSize
       return stream
     })

@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs

@@ -5,6 +5,7 @@ import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js'
 import { notEqual, strictEqual } from 'node:assert'
 import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
 import { VhdAbstract } from 'vhd-lib'
+import { openDatastore } from './_openDatastore.mjs'
 
 // one big difference with the other versions of VMDK is that the grain tables are actually sparse, they are pre-allocated but not used in grain order,
 // so we have to read the grain directory to know where to find the grain tables
@@ -53,7 +54,6 @@ function readSeSparseTable(buffer, index) {
 }
 
 export default class VhdEsxiSeSparse extends VhdAbstract {
-  #esxi
   #datastore
   #parentVhd
   #path
@@ -73,8 +73,9 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
   #stream
   #streamOffset = 0
 
-  static async open(esxi, datastore, path, parentVhd, opts) {
-    const vhd = new VhdEsxiSeSparse(esxi, datastore, path, parentVhd, opts)
+  static async open(datastoreName, path, parentVhd, opts) {
+    const datastore = openDatastore(datastoreName, opts)
+    const vhd = new VhdEsxiSeSparse(datastore, path, parentVhd, opts)
     await vhd.readHeaderAndFooter()
     return vhd
   }
@@ -82,9 +83,8 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
   get path() {
     return this.#path
   }
-  constructor(esxi, datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) {
+  constructor(datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) {
     super()
-    this.#esxi = esxi
     this.#path = path
     this.#datastore = datastore
     this.#parentVhd = parentVhd
@@ -114,7 +114,7 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
   async #read(start, length) {
     if (!this.#footer) {
       // we need to be able before the footer is loaded, to read the header and footer
-      return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${start + length - 1}`)).buffer()
+      return this.#datastore.getBuffer(this.#path, start, start + length)
     }
     if (this.#reading) {
       throw new Error('reading must be done sequentially')
@@ -132,8 +132,8 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
       }
       // no stream
       if (this.#stream === undefined) {
-        const end = this.footer.currentSize - 1
-        const res = await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)
+        const end = this.footer.currentSize
+        const res = await this.#datastore.getStream(this.#path, start, end)
         this.#stream = res.body
         this.#streamOffset = start
       }

@xen-orchestra/vmware-explorer/_Datastore.mjs

@@ -0,0 +1,17 @@
+export class Datastore {
+  #datastoreName
+
+  get datastoreName() {
+    return this.#datastoreName
+  }
+  constructor(dataStoreName) {
+    this.#datastoreName = dataStoreName
+  }
+  async getStream(path, start, end) {
+    throw new Error('Not implemented')
+  }
+  async getBuffer(path, start, end) {
+    throw new Error('Not implemented')
+  }
+  async getSize(path) {}
+}

@xen-orchestra/vmware-explorer/_openDatastore.mjs

@@ -0,0 +1,10 @@
+import { DatastoreSoapEsxi } from './DatastoreSoapEsxi.mjs'
+import { DatastoreXoRemote } from './DatastoreXoRemote.mjs'
+
+export function openDatastore(dataStoreName, { esxi, remotes = {}, ...otherOptions }) {
+  const handler = remotes[dataStoreName]
+  if (handler === undefined) {
+    return new DatastoreSoapEsxi(dataStoreName, { esxi, ...otherOptions })
+  }
+  return new DatastoreXoRemote(dataStoreName, { handler, ...otherOptions })
+}

@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs

@@ -1,13 +1,15 @@
 import VHDEsxiSeSparse from './VhdEsxiSeSparse.mjs'
 import VhdEsxiCowd from './VhdEsxiCowd.mjs'
+import { openDatastore } from './_openDatastore.mjs'
 
-export default async function openDeltaVmdkasVhd(esxi, datastore, path, parentVhd, opts) {
+export default async function openDeltaVmdkasVhd(datastoreName, path, parentVhd, opts) {
   let vhd
+  const datastore = openDatastore(datastoreName, opts)
   if (path.endsWith('-sesparse.vmdk')) {
-    vhd = new VHDEsxiSeSparse(esxi, datastore, path, parentVhd, opts)
+    vhd = new VHDEsxiSeSparse(datastore, path, parentVhd, opts)
   } else {
     if (path.endsWith('-delta.vmdk')) {
-      vhd = new VhdEsxiCowd(esxi, datastore, path, parentVhd, opts)
+      vhd = new VhdEsxiCowd(datastore, path, parentVhd, opts)
     } else {
       throw new Error(`Vmdk ${path} does not seems to be a delta vmdk.`)
     }

CHANGELOG.unreleased.md

@@ -8,6 +8,7 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - [VMWare/Migration] Make one pass for the cold base disk and snapshots (PR [#7487](https://github.com/vatesfr/xen-orchestra/pull/7487))
+- [VMWare/Migration] Use NFS datastore from XO Remote to bypass VMFS6 lock (PR [#7487](https://github.com/vatesfr/xen-orchestra/pull/7487))
 
 ### Bug fixes
 

packages/xo-server/src/api/vm.mjs

@@ -17,6 +17,7 @@ import { invalidParameters, noSuchObject, unauthorized } from 'xo-common/api-err
 import { Ref } from 'xen-api'
 
 import { forEach, map, mapFilter, parseSize, safeDateFormat } from '../utils.mjs'
+import { getHandler } from '@xen-orchestra/fs'
 
 const log = createLogger('xo:vm')
 
@@ -1384,7 +1385,29 @@ export { import_ as import }
 
 export async function importFromEsxi({ host, network, password, sr, sslVerify = true, stopSource = false, user, vm }) {
   const task = await this.tasks.create({ name: `importing vm ${vm}` })
-  return task.run(() => this.migrationfromEsxi({ host, user, password, sslVerify, vm, sr, network, stopSource }))
+  const remotes = {}
+  await asyncEach(await this.getAllRemotes(), async remote => {
+    if (remote.name.startsWith('[VMWARE]')) {
+      const handler = getHandler(remote)
+      await handler.sync()
+      remotes[remote.name.substring('[VMWARE]'.length)] = handler
+      // todo : dispose
+    }
+  })
+
+  return task.run(() =>
+    this.migrationfromEsxi({
+      host,
+      user,
+      password,
+      sslVerify,
+      vm,
+      sr,
+      network,
+      stopSource,
+      dataStoreToRemotes: remotes,
+    })
+  )
 }
 
 importFromEsxi.params = {