feat(xo-server-audit): audit import (#8474)

@xen-orchestra/audit-core/index.js

@@ -98,6 +98,17 @@ class AuditCore {
     return record
   }
 
+  async _importRecord(record) {
+    if (!record.id || !record.previousId) {
+      throw new Error('Invalid record')
+    }
+    await this._storage.put(record)
+    return {
+      id: record.id,
+      isValid: record.id === createHash(record),
+    }
+  }
+
   async checkIntegrity(oldest, newest) {
     const storage = this._storage
 

CHANGELOG.unreleased.md

@@ -25,6 +25,7 @@
 - [REST API] Expose `/rest/v0/schedules` and `/rest/v0/schedules/<schedule-id>` enpoints (PR [#8477](https://github.com/vatesfr/xen-orchestra/pull/8477))
 - [REST API] Expose the possibility to run a schedule `/rest/v0/schedules/<schedule-id>/actions/run` (PR [#8477](https://github.com/vatesfr/xen-orchestra/pull/8477))
 - [Host/Networks] PIFs can now be filtered by network names
+- [Plugins/audit] Add an option to import audit logs from an XOA to another (PR [#8474](https://github.com/vatesfr/xen-orchestra/pull/8474))
 
 ### Bug fixes
 
@@ -55,12 +56,14 @@
 <!--packages-start-->
 
 - @vates/types minor
+- @xen-orchestra/audit-core minor
 - @xen-orchestra/backups minor
 - @xen-orchestra/fs minor
 - @xen-orchestra/rest-api minor
 - @xen-orchestra/web minor
 - @xen-orchestra/web-core minor
 - xo-server patch
+- xo-server-audit minor
 - xo-web minor
 
 <!--packages-end-->

packages/xo-server-audit/package.json

@@ -44,13 +44,16 @@
     "prepublishOnly": "yarn run build"
   },
   "dependencies": {
+    "@vates/decorate-with": "^2.1.0",
     "@xen-orchestra/audit-core": "^0.3.1",
     "@xen-orchestra/cron": "^1.0.6",
     "@xen-orchestra/log": "^0.7.1",
     "async-iterator-to-stream": "^1.1.0",
     "complex-matcher": "^0.7.1",
+    "golike-defer": "^0.5.1",
+    "ndjson": "^2.0.0",
     "promise-toolbox": "^0.21.0",
-    "readable-stream": "^4.1.0",
+    "readable-stream": "^4.7.0",
     "xo-common": "^0.8.0"
   },
   "private": true

packages/xo-server-audit/src/index.js

@@ -1,12 +1,15 @@
 import * as CM from 'complex-matcher'
 import asyncIteratorToStream from 'async-iterator-to-stream'
+import ndjson from 'ndjson'
 import { alteredAuditRecord, missingAuditRecord } from 'xo-common/api-errors'
-import { createGzip } from 'zlib'
+import { createGzip, createUnzip } from 'zlib'
 import { createLogger } from '@xen-orchestra/log'
 import { createSchedule } from '@xen-orchestra/cron'
+import { decorateClass } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
 import { fromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
 import { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } from '@xen-orchestra/audit-core'
+import { PassThrough, pipeline, promises } from 'readable-stream'
 
 const log = createLogger('xo:xo-server-audit')
 
@@ -171,13 +174,24 @@ class Db extends Storage {
       db.createKeyStream().on('data', deleteEntry).on('end', cb).on('error', reject)
     })
   }
+
+  async isEmpty() {
+    const readStream = this._db.createReadStream({ limit: 1 })
+    return new Promise(function (resolve, reject) {
+      readStream
+        .on('data', () => resolve(false))
+        .on('end', () => resolve(true))
+        .on('error', reject)
+    })
+  }
 }
 
 export const configurationSchema = {
   type: 'object',
   properties: {
     active: {
-      description: 'Whether to save user actions in the audit log',
+      description:
+        'Whether to save user actions in the audit log (do not enable if you are planning to perform an audit log import)',
       type: 'boolean',
     },
   },
@@ -231,6 +245,10 @@ class AuditXoPlugin {
     const exportRecords = this._exportRecords.bind(this)
     exportRecords.permission = 'admin'
 
+    const importRecords = this._importRecords.bind(this)
+    importRecords.description = 'Import audit logs from a file'
+    importRecords.permission = 'admin'
+
     const getRecords = this._getRecords.bind(this)
     getRecords.description = 'Get records from a passed record ID'
     getRecords.permission = 'admin'
@@ -286,6 +304,7 @@ class AuditXoPlugin {
           exportRecords,
           generateFingerprint,
           getRecords,
+          importRecords,
         },
       })
     )
@@ -407,6 +426,76 @@ class AuditXoPlugin {
       }))
   }
 
+  async _handleImportRecords($defer, { fileStream, zipped }) {
+    const fnReleaseLock = await this._storage.acquireLock()
+    $defer(fnReleaseLock)
+
+    if (!(await this._storage.isEmpty())) {
+      throw new Error('Audit log database must be empty')
+    }
+
+    const processRecord = async source => {
+      let nInvalidRecords = 0
+      let nMissingRecords = 0
+      // entries are listed backwards, from newest to oldest. nextLog is the last log we imported
+      let lastLogId, lastId, nextLog
+      for await (const record of source) {
+        const { id, isValid } = await this._auditCore._importRecord(record)
+        if (lastId === undefined) {
+          lastId = id
+          await this._storage.setLastId(lastId)
+        }
+        if (!isValid) {
+          nInvalidRecords++
+        }
+        if (nextLog !== undefined && nextLog.previousId !== record.id) {
+          nMissingRecords++
+          lastLogId ??= nextLog.id
+        }
+        nextLog = record
+      }
+      lastLogId ??= nextLog.id // in case no log was missing
+      return { lastLogId, nInvalidRecords, nMissingRecords }
+    }
+
+    try {
+      const res = await promises.pipeline(
+        fileStream,
+        zipped ? createUnzip() : new PassThrough(),
+        ndjson.parse(),
+        processRecord
+      )
+
+      if (this._uploadLastHashJob !== undefined) {
+        await this._uploadLastHash()
+      }
+
+      return res
+    } catch (err) {
+      // Cleaning database because logs were only partially imported
+      await this._storage.clean()
+      throw err
+    }
+  }
+
+  async _importRecords({ zipped }) {
+    if (!(await this._storage.isEmpty())) {
+      throw new Error('Audit log database must be empty')
+    }
+
+    return {
+      $sendTo: await this._xo.registerHttpRequest(async (req, res) => {
+        try {
+          const importResults = await this._handleImportRecords({ fileStream: req, zipped })
+          res.end(JSON.stringify(importResults))
+        } catch (err) {
+          res.writeHead(500)
+          res.end(err.message)
+        }
+      }),
+    }
+  }
+
   // See www-xo#344
   async _uploadLastHash() {
     const xo = this._xo
@@ -518,4 +607,8 @@ AuditXoPlugin.prototype._getRecordsStream = asyncIteratorToStream(async function
   }
 })
 
+decorateClass(AuditXoPlugin, {
+  _handleImportRecords: defer,
+})
+
 export default opts => new AuditXoPlugin(opts)

packages/xo-web/src/common/intl/messages.js

@@ -2793,6 +2793,16 @@ const messages = {
   noAuditRecordAvailable: 'No audit record available',
   refreshAuditRecordsList: 'Refresh records list',
   auditInactiveUserActionsRecord: 'User actions recording is currently inactive',
+  importAuditRecords: 'Import records',
+  importRecordsTip: 'Try dropping a .ndjson.gz or .ndjson file here or click to select a file.',
+  importAuditRecordsCleanList: 'Reset',
+  importAuditRecordsButton: 'Import',
+  importAuditRecordsTooltip: 'Import audit record from another XOA. Audit log database must be empty.',
+  importAuditRecordsSuccess: 'Audit records successfully imported',
+  importAuditRecordsSuccessWithProblems:
+    "Audit records successfully imported, but {nInvalidRecords} invalid records were imported and at least {nMissingRecords} records were missing. Logs prior to the missing entries will not appear. The oldest visible log's ID is {lastLogId}",
+  importAuditRecordsError: 'Error while importing audit records: {importError}',
+  noAuditRecordsFile: 'No audit records file selected',
 
   // Licenses
   allHostsMustBeBound: 'All hosts must be bound to a license',

packages/xo-web/src/common/xo/index.js

@@ -4240,6 +4240,22 @@ export const exportAuditRecords = () =>
     window.open(`.${url}`)
   })
 
+export const importAuditRecords = async recordsFile => {
+  const { $sendTo } = await _call('audit.importRecords', { zipped: recordsFile.type === 'application/gzip' })
+  const response = await post($sendTo, recordsFile)
+  const text = await response.text()
+
+  if (response.status !== 200) {
+    throw new Error(text)
+  }
+
+  try {
+    return JSON.parse(text)
+  } catch (error) {
+    throw new Error(`Body is not a JSON, original message is : ${text}`)
+  }
+}
+
 export const checkAuditRecordsIntegrity = (oldest, newest) => _call('audit.checkIntegrity', { oldest, newest })
 
 export const generateAuditFingerprint = oldest => _call('audit.generateFingerprint', { oldest })

packages/xo-web/src/xo-app/settings/audit/index.js

@@ -4,6 +4,7 @@ import Button from 'button'
 import Copiable from 'copiable'
 import CopyToClipboard from 'react-copy-to-clipboard'
 import decorate from 'apply-decorators'
+import Dropzone from 'dropzone'
 import Icon from 'icon'
 import Link from 'link'
 import NoObjects from 'no-objects'
@@ -17,7 +18,7 @@ import { injectIntl } from 'react-intl'
 import { get } from '@xen-orchestra/defined'
 import { injectState, provideState } from 'reaclette'
 import { noop, startCase } from 'lodash'
-import { NumericDate } from 'utils'
+import { NumericDate, formatSize } from 'utils'
 import { PREMIUM } from 'xoa-plans'
 import { User } from 'render-xo-item'
 import {
@@ -26,6 +27,7 @@ import {
   fetchAuditRecords,
   generateAuditFingerprint,
   getPlugin,
+  importAuditRecords,
 } from 'xo'
 import RichText from 'rich-text'
 
@@ -171,6 +173,34 @@ const displayRecord = record =>
     <RichText copiable message={JSON.stringify(record, null, 2)} />
   )
 
+const renderImportStatus = ({
+  recordsFile,
+  importError,
+  importResults: { nInvalidRecords, nMissingRecords, lastLogId } = {},
+  importStatus,
+}) => {
+  switch (importStatus) {
+    case 'noFile':
+      return _('noAuditRecordsFile')
+    case 'selectedFile':
+      return <span>{`${recordsFile?.name} (${formatSize(recordsFile?.size)})`}</span>
+    case 'start':
+      return <Icon icon='loading' />
+    case 'end':
+      if (nInvalidRecords === 0 && nMissingRecords === 0) {
+        return <span className='text-success'>{_('importAuditRecordsSuccess')}</span>
+      } else {
+        return (
+          <span className='text-warning'>
+            {_('importAuditRecordsSuccessWithProblems', { nInvalidRecords, nMissingRecords, lastLogId })}
+          </span>
+        )
+      }
+    case 'importError':
+      return <span className='text-danger'>{_('importAuditRecordsError', { importError: importError || '' })}</span>
+  }
+}
+
 const INDIVIDUAL_ACTIONS = [
   {
     handler: displayRecord,
@@ -252,7 +282,12 @@ export default decorate([
       _records: undefined,
       checkedRecords: {},
       goTo: undefined,
+      importError: undefined,
+      importStatus: 'noFile',
+      importResults: undefined,
       missingRecord: undefined,
+      recordsFile: undefined,
+      showImportDropzone: false,
     }),
     effects: {
       initialize({ fetchRecords }) {
@@ -261,6 +296,35 @@ export default decorate([
       async fetchRecords() {
         this.state._records = await fetchAuditRecords()
       },
+      showImportDropzone() {
+        this.state.showImportDropzone = !this.state.showImportDropzone
+      },
+      startImportRecords() {
+        this.state.importStatus = 'start'
+
+        return importAuditRecords(this.state.recordsFile)
+          .then(
+            importResults => {
+              this.state.recordsFile = undefined
+              this.state.importStatus = 'end'
+              this.state.importResults = importResults
+            },
+            error => {
+              this.state.recordsFile = undefined
+              this.state.importStatus = 'importError'
+              this.state.importError = error?.message
+            }
+          )
+          .finally(this.effects.fetchRecords)
+      },
+      handleDrop(_, files) {
+        this.state.recordsFile = files && files[0]
+        this.state.importStatus = 'selectedFile'
+      },
+      unselectFile() {
+        this.state.recordsFile = undefined
+        this.state.importStatus = 'noFile'
+      },
       handleRef(_, ref) {
         if (ref !== null) {
           const component = ref.getWrappedInstance()
@@ -339,8 +403,39 @@ export default decorate([
             size='large'
           >
             {_('auditCheckIntegrity')}
+          </ActionButton>{' '}
+          <ActionButton
+            btnStyle='warning'
+            disabled={state.records?.length > 0}
+            handler={effects.showImportDropzone}
+            icon='upload'
+            size='large'
+            tooltip={_('importAuditRecordsTooltip')}
+          >
+            {_('importAuditRecords')}
           </ActionButton>
         </div>
+
+        {!!state.showImportDropzone && (
+          <div>
+            <Dropzone onDrop={effects.handleDrop} message={_('importRecordsTip')} />
+            {renderImportStatus(state)}
+            <div className='form-group pull-right'>
+              <ActionButton
+                btnStyle='primary'
+                className='mr-1'
+                disabled={!state.recordsFile}
+                handler={effects.startImportRecords}
+                icon='import'
+                type='submit'
+              >
+                {_('importAuditRecordsButton')}
+              </ActionButton>
+              <Button onClick={effects.unselectFile}>{_('importAuditRecordsCleanList')}</Button>
+            </div>
+          </div>
+        )}
+
         {state.isUserActionsRecordInactive && (
           <p>
             <Link

yarn.lock

@@ -17700,7 +17700,7 @@ read-pkg@^3.0.0:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
-"readable-stream@>= 0.3.0", readable-stream@^4.1.0, readable-stream@^4.5.2:
+"readable-stream@>= 0.3.0", readable-stream@^4.1.0, readable-stream@^4.5.2, readable-stream@^4.7.0:
   version "4.7.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.7.0.tgz#cedbd8a1146c13dfff8dab14068028d58c15ac91"
   integrity sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==