fix(v2v): improve vddk import (#8911)

* fix(nbd-client): better handling of error during getMap
* fix(xo-server): esxi check errors are objects instead of text
* fix(xo-web): V2v/check improve alignment
* feat(xo-web): add button to install nbdinfo/nbdkit if needed

@vates/nbd-client/index.mjs

@@ -378,11 +378,13 @@ export default class NbdClient {
         `nbd://${this.#serverAddress}:${this.#serverPort}/${encodeURIComponent(this.#exportName)}`,
       ])
       let text = ''
+      let errText = ''
       process.stdout.on('data', data => (text += data))
+      process.stderr.on('data', data => (errText += data))
       process.on('error', reject)
       process.on('close', code => {
         if (code !== 0) {
-          return reject(new Error(`process ended with code ${code}`))
+          return reject(new Error(`Error during getMap (code: ${code}): ${errText}`))
         }
         try {
           const json = JSON.parse(text)

@xen-orchestra/vmware-explorer/checks.mjs

@@ -2,18 +2,19 @@ import { exec } from 'node:child_process'
 import semver from 'semver'
 import fs from 'node:fs/promises'
 
+const NBDKIT_VERSION_VDDK9 = '1.42.5'
+
 /**
  *
  * @returns {Promise<Object>}
  */
-/* async */ function nbdInfos() {
+/* async */ function nbdInfo() {
   return new Promise(function (resolve, reject) {
     const expectedVersion = '1.23.4'
     exec('nbdinfo --version', (error, stdout, stderr) => {
       if (error) {
         return resolve({
-          installed: false,
-          error,
+          error: `exit code ${error.code}`,
           status: 'error',
         })
       }
@@ -28,32 +29,43 @@ import fs from 'node:fs/promises'
     })
   })
 }
-/**
- *
- * @returns {Promise<Object>}
- */
-/* async */ function nbdKit() {
-  const expectedVersion = '1.45'
+
+async function getNbdKitVersion() {
   return new Promise(function (resolve) {
     exec('nbdkit --version', (error, stdout, stderr) => {
       if (error) {
         return resolve({
-          installed: false,
-          error,
+          error: `exit code ${error.code}`,
           status: 'error',
         })
       }
       const matches = stdout.match(/nbdkit ([0-9.]+)/)
       const version = matches?.[1] ?? ''
-      resolve({
-        installed: true,
-        version,
-        status: semver.satisfies(version, `>=${expectedVersion}`) ? 'success' : 'alarm',
-        expectedVersion,
-      })
+      resolve(version)
     })
   })
 }
+/**
+ *
+ * @returns {Promise<Object>}
+ */
+async function nbdKit() {
+  const expectedVersion = '1.42'
+  try {
+    const version = await getNbdKitVersion()
+    return {
+      installed: true,
+      version,
+      status: semver.satisfies(version, `>=${expectedVersion}`) ? 'success' : 'alarm',
+      expectedVersion,
+    }
+  } catch (error) {
+    return {
+      error: `exit code ${error.code}`,
+      status: 'error',
+    }
+  }
+}
 
 /**
  *
@@ -82,7 +94,27 @@ import fs from 'node:fs/promises'
 async function vddk() {
   try {
     await fs.stat('/usr/local/lib/vddk/vmware-vix-disklib-distrib/lib64/libvixDiskLib.so')
-    return { status: 'success' }
+
+    try {
+      const isV9 = await fs.exists('/usr/local/lib/vddk/vmware-vix-disklib-distrib/lib64/libvixDiskLib.so.9')
+      const nbdKitVersion = await getNbdKitVersion()
+      if (isV9) {
+        if (!semver.satisfies(nbdKitVersion, `>=${NBDKIT_VERSION_VDDK9}`)) {
+          return {
+            status: 'warning',
+            expectedVersion: '1.42.5',
+            version: nbdKitVersion,
+          }
+        }
+      }
+      return { status: 'success' }
+    } catch (error) {
+      return {
+        status: 'warning',
+        expectedVersion: '1.42.5',
+        version: 'unknown',
+      }
+    }
   } catch (error) {
     return {
       status: 'error',
@@ -97,9 +129,9 @@ async function vddk() {
  */
 export async function checkVddkDependencies() {
   return {
-    nbdInfos: await nbdInfos(),
+    nbdinfo: await nbdInfo(),
     nbdkit: await nbdKit(),
-    nbdKitVddk: await nbdKitVddk(),
+    nbdkitPluginVddk: await nbdKitVddk(),
     vddk: await vddk(),
   }
 }

CHANGELOG.unreleased.md

@@ -11,6 +11,8 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- [V2V] Auto install library (PR [#8911](https://github.com/vatesfr/pull/8911))
+
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
@@ -31,7 +33,11 @@
 
 <!--packages-start-->
 
+- @vates/nbd-client patch
 - @xen-orchestra/web minor
 - @xen-orchestra/web-core minor
+- @xen-orchestra/vmware-explorer patch
+- xo-server minor
+- xo-web minor
 
 <!--packages-end-->

packages/xo-server/src/api/esxi.mjs

@@ -5,6 +5,18 @@ import { createWriteStream } from 'node:fs'
 import { VDDK_LIB_DIR } from '@xen-orchestra/vmware-explorer/esxi.mjs'
 import { exec } from 'node:child_process'
 
+function execPromise(command, opts = []) {
+  return new Promise((resolve, reject) => {
+    exec(command, { maxBuffer: 10 * 1024 * 1024, ...opts }, (error, stdout) => {
+      if (error) {
+        reject(error)
+      } else {
+        resolve(stdout)
+      }
+    })
+  })
+}
+
 export function listVms({ host, password, sslVerify = true, user }) {
   return this.connectToEsxiAndList({ host, user, password, sslVerify })
 }
@@ -35,16 +47,7 @@ async function handleImport(req, res) {
     req.on('end', resolve)
     req.on('error', reject)
   })
-  return new Promise((resolve, reject) => {
-    exec(`tar xfz ${tempFilePath} -C ${VDDK_LIB_DIR}`, { maxBuffer: 10 * 1024 * 1024 }, err => {
-      if (err) {
-        reject(err)
-      } else {
-        res.end()
-        return resolve()
-      }
-    })
-  })
+  await execPromise(`tar xfz ${tempFilePath} -C ${VDDK_LIB_DIR}`)
 }
 export async function installVddkLib() {
   return {
@@ -54,3 +57,50 @@ export async function installVddkLib() {
 installVddkLib.params = {}
 
 installVddkLib.permission = 'admin'
+
+async function install({ repository, version, executable }) {
+  const tmpDir = await fs.mkdtemp(join(tmpdir(), 'xo-server'))
+  const libPath = await execPromise(`which ${executable}`).catch(() => {})
+  if (libPath) {
+    throw new Error(
+      `${executable} is already installed at ${libPath} on this system please uninstall before rerunning this script`
+    )
+  }
+  let id
+  if ((id = (await execPromise('id -u')).trim() !== '0')) {
+    throw new Error(`install script can only be run as root ${id}`)
+  }
+  try {
+    await execPromise('apt-get --version')
+  } catch (err) {
+    const error = new Error(`The ${executable} auto install can only be run on a debian based system`)
+    error.cause = err
+    throw err
+  }
+
+  await execPromise('apt-get install -y git dh-autoreconf pkg-config make libxml2-dev ocaml libc-bin')
+
+  await execPromise(`git clone ${repository} ${tmpDir}`)
+  await execPromise(`git checkout ${version} `, { cwd: tmpDir })
+
+  await execPromise('autoreconf -i > autoreconf.log', { cwd: tmpDir })
+  await execPromise('./configure > autoreconf.log', { cwd: tmpDir })
+  await execPromise('make', { cwd: tmpDir })
+  await execPromise('make install', { cwd: tmpDir })
+  await execPromise('ldconfig', { cwd: tmpDir })
+  await execPromise(`${executable} --version`)
+}
+
+export async function installNbdInfoFromSource() {
+  await install({ executable: 'nbdinfo', repository: 'https://gitlab.com/nbdkit/libnbd.git', version: 'v1.23.4' })
+}
+
+installNbdInfoFromSource.params = {}
+installNbdInfoFromSource.permission = 'admin'
+
+export async function installNbdKitFromSource() {
+  await install({ executable: 'nbdkit', repository: 'https://gitlab.com/nbdkit/nbdkit.git', version: 'v1.44.3' })
+}
+
+installNbdKitFromSource.params = {}
+installNbdKitFromSource.permission = 'admin'

packages/xo-web/src/common/intl/messages.js

@@ -20,14 +20,20 @@ const messages = {
   esxiLibraryInfo: 'The V2V tool need the vddk library, provided by Broadcom. Please download it from their website',
   esxiLibraryLink: 'Donwload link',
 
-  esxiLibrary: 'Drop the tar.gz file of the vddk library (linux)',
-  esxiLibraryImport: 'Import and install library',
+  esxiVddkLibrary: 'Drop the tar.gz file of the vddk library (linux)',
+  esxiVddkLibraryImport: 'Import and install the Vddk library. VDDK9 need nbdkit 1.42+',
+  esxiLibraryManualInstall:
+    'For other systems, you can install manually from https://gitlab.com/nbdkit/nbdkit . For reference the list of packages need for a debian 13 is **git dh-autoreconf pkg-config make libxml2-dev ocaml libc-bin**',
+  esxiLibraryAutoInstall: 'install {library} (debian based system)',
+  esxiLibraryNotInstalled: '{library} is not installed',
+
+  esxiLibraryOutdated:
+    '{library} library is outdated expecting {expectedVersion}, got {version}. Please uninstall it and install the required version.',
   esxiCheckingPrerequisite: 'Checking prerequisite on XO',
   esxiCheckedPrerequisite: 'Result of the prerequisite check on XO',
 
   esxiCheckingPrerequisiteError: 'Must be corrected before importing VM',
-  // eslint-disable-next-line no-template-curly-in-string
-  esxiCheckedPrerequisiteVersion: 'expected version ${expectedVersion} , ${version} installed',
+  esxiCheckedPrerequisiteVersion: 'expected version {expectedVersion} , {version} installed',
   esxiImportSslCertificate: 'Skip SSL check',
   esxiImportThin: 'Thin mode',
   esxiImportThinDescription:

packages/xo-web/src/common/xo/index.js

@@ -4316,6 +4316,24 @@ export const importVddkLib = file => {
       })
   })
 }
+export const installNbdInfo = file => {
+  return _call('esxi.installNbdInfoFromSource')
+    .then(() => {
+      success('nbdInfo successfullly installed successfully installed')
+    })
+    .catch(err => {
+      error('fail to install nbdInfo', err)
+    })
+}
+export const installNbdKit = file => {
+  return _call('esxi.installNbdKitFromSource')
+    .then(() => {
+      success('nbdkit successfullly installed successfully installed')
+    })
+    .catch(err => {
+      error('fail to install nbdkit', err)
+    })
+}
 // GitHub API ---------------------------------------------------------------
 const _callGithubApi = async (endpoint = '') => {
   const url = new URL('https://api.github.com/repos/vatesfr/xen-orchestra')

packages/xo-web/src/xo-app/vm-import/esxi/esxi-import.js

@@ -8,7 +8,15 @@ import Icon from 'icon'
 import React from 'react'
 import { connectStore, resolveId } from 'utils'
 import { createGetObjectsOfType, createSelector } from 'selectors'
-import { esxiCheckInstall, esxiListVms, importVddkLib, importVmsFromEsxi, isSrWritable } from 'xo'
+import {
+  esxiCheckInstall,
+  esxiListVms,
+  importVddkLib,
+  importVmsFromEsxi,
+  installNbdInfo,
+  installNbdKit,
+  isSrWritable,
+} from 'xo'
 import { find, isEmpty, keyBy, map, pick } from 'lodash'
 import { injectIntl } from 'react-intl'
 import { Input } from 'debounce-input-decorator'
@@ -27,9 +35,11 @@ function EsxiCheckResults({ esxiCheck }) {
         const { status, error, version, expectedVersion } = value
         return (
           <li key={name}>
-            <Icon icon={status} />
-            {name} :{status === 'success' && ' ok'}
-            {status === 'error' && error}
+            <Icon icon={status} size='lg' fixedWidth />
+            &nbsp;
+            {name} : {status === 'success' && ' ok'}
+            {status === 'error' && `"${error}"`}
+            &nbsp;
             {status === 'error' && _('esxiCheckingPrerequisiteError')}
             {version && status === 'alarm' && _('esxiCheckedPrerequisiteVersion', { version, expectedVersion })}
           </li>
@@ -79,6 +89,14 @@ class EsxiImport extends Component {
     }
     return this._esxiCheck()
   }
+  _installNbdInfo = async () => {
+    await installNbdInfo()
+    return this._esxiCheck()
+  }
+  _installNbKit = async () => {
+    await installNbdKit()
+    return this._esxiCheck()
+  }
   _getDefaultNetwork = createSelector(
     () => this.state.pool?.master,
     () => this.props.hostsById,
@@ -184,6 +202,37 @@ class EsxiImport extends Component {
       return <div>checking</div>
     }
 
+    // cehck nbdkit, nbdinfo, nbdkit plugin vddk
+    for (const [library, fn] of [
+      ['nbdinfo', this._installNbdInfo],
+      ['nbdkit', this._installNbKit],
+      ['nbdkitPluginVddk', this._installNbKit],
+    ]) {
+      const check = esxiCheck[library]
+      if (check.status !== 'success') {
+        return (
+          <div>
+            {check.version === undefined && (
+              <div>
+                <p>{_('esxiLibraryNotInstalled', { library })}</p>
+                <div className='form-group pull-right'>
+                  <ActionButton btnStyle='primary' className='mr-1' handler={fn} icon='import'>
+                    {_('esxiLibraryAutoInstall', { library })}
+                  </ActionButton>
+                  <p>{_('esxiLibraryManualInstall')}</p>
+                </div>
+              </div>
+            )}
+            {check.version !== undefined && (
+              <p>
+                {_('esxiLibraryOutdated', { library, expectedVersion: check.expectedVersion, version: check.version })}
+              </p>
+            )}
+          </div>
+        )
+      }
+    }
+
     if (esxiCheck.vddk?.status === 'error') {
       return (
         <div>
@@ -197,21 +246,17 @@ class EsxiImport extends Component {
               {_('esxiLibraryLink')}
             </a>
           </p>
-          <Dropzone multiple={false} onDrop={this._handleDropVddk} message={_('esxiLibrary')} accept='.tar.gz' />
+          <Dropzone multiple={false} onDrop={this._handleDropVddk} message={_('esxiVddkLibrary')} accept='.tar.gz' />
           {vddkFile && (
             <div className='form-group pull-right'>
               <ActionButton btnStyle='primary' className='mr-1' handler={this._handleImportVddk} icon='import'>
-                {_('esxiLibraryImport')}
+                {_('esxiVddkLibraryImport')}
               </ActionButton>
             </div>
           )}
         </div>
       )
     }
-    if (Object.values(esxiCheck).some(({ status }) => status === 'error')) {
-      // do not show connection form is some prerequisites are in error
-      return <EsxiCheckResults esxiCheck={esxiCheck} />
-    }
 
     if (!isConnected) {
       return (