diff --git a/.github/workflows/vectura.yml b/.github/workflows/vectura.yml index 18b6879690..5e26df703a 100644 --- a/.github/workflows/vectura.yml +++ b/.github/workflows/vectura.yml @@ -45,8 +45,8 @@ jobs: - name: Hold the coverage floor run: cargo test --locked --test coverage -- --ignored - # builds the binary as a release does, then checks it and the one in git, - # which xo-server runs + # builds the binary as release.sh does, checks it, then checks the one in git, + # which xo-server runs, is that build binary: runs-on: ubuntu-latest steps: @@ -65,4 +65,9 @@ jobs: - run: scripts/build.sh "$RUNNER_TEMP/build" - run: scripts/check-binary.sh "$RUNNER_TEMP/build/vectura" - - run: scripts/check-binary.sh vectura + - name: The binary in git is the build of these sources + run: | + cmp "$RUNNER_TEMP/build/vectura" vectura || { + echo '::error::vectura is not the build of these sources: run scripts/build.sh . in vectura/ and commit the binary' + exit 1 + } diff --git a/@xen-orchestra/vmware-explorer/vectura/README.md b/@xen-orchestra/vmware-explorer/vectura/README.md index a1ffdf68d0..30bfc128dd 100644 --- a/@xen-orchestra/vmware-explorer/vectura/README.md +++ b/@xen-orchestra/vmware-explorer/vectura/README.md @@ -85,25 +85,44 @@ Explanation, the reasoning: touches this directory. The pre-commit hook runs `cargo fmt --check` when a `.rs` file is staged. -A change to vectura needs its own line in the packages list of -`CHANGELOG.unreleased.md`: +A pull request that changes the sources, `Cargo.lock` or a tool version bumps +vectura once, at its end, from this directory, then commits `Cargo.toml`, +`Cargo.lock` and `vectura`: -```md -- vectura patch +```sh +scripts/release.sh patch # or minor, major +``` + +It bumps the version in `Cargo.toml` and `Cargo.lock` and rebuilds the binary +in place of the previous one. If the build or the check of the binary fails, +it puts the version back and can run again. vectura is not released on its +own: the pull request lists the package which bundles the binary in the +packages list of `CHANGELOG.unreleased.md`, and the release of that package +ships the binary as is: + +```md +- @xen-orchestra/vmware-explorer patch ``` -`scripts/gen-deps-list.js` then lists `./scripts/bump-pkg vectura `, -followed by a release of `@xen-orchestra/vmware-explorer`, which bundles the -binary. `bump-pkg` runs `scripts/release.sh`: it bumps the version in -`Cargo.toml` and `Cargo.lock`, rebuilds the binary in place of the previous -one and commits both as `feat(vectura): `. `tests/bundled_binary.rs` fails when the binary and `Cargo.toml` disagree, or when the binary lost its executable bit. +The binary in git is the build of its sources: the workflow builds it again +and fails when the two differ by a single byte. A change pushed after the bump +does not bump again: rebuild the binary from this directory and commit it: + +```sh +scripts/build.sh . +``` + +The build is reproducible: `scripts/build.sh` keeps the paths of the machine +out of the binary and refuses any zig or cargo-zigbuild but the versions the +workflow installs. Linux x86_64 machines build the same binary; macOS and +aarch64 hosts are untested. + The build links against glibc 2.31 through zig, whatever the glibc of the machine, so the binary runs on Debian 11, which XOA appliances still run, -and Ubuntu 20.04. It needs -[zig](https://ziglang.org/download/) (0.16.0 tested) and: +and Ubuntu 20.04. It needs [zig](https://ziglang.org/download/) 0.16.0 and: ```sh cargo install --locked cargo-zigbuild@0.23.4 diff --git a/@xen-orchestra/vmware-explorer/vectura/scripts/build.sh b/@xen-orchestra/vmware-explorer/vectura/scripts/build.sh index 828d737269..f953929c65 100755 --- a/@xen-orchestra/vmware-explorer/vectura/scripts/build.sh +++ b/@xen-orchestra/vmware-explorer/vectura/scripts/build.sh @@ -6,11 +6,39 @@ # XOA appliances still run Debian 11. # Needs zig and cargo-zigbuild, see the README. The Vectura workflow and # release.sh run it. +# +# The build is reproducible: with the zig and cargo-zigbuild below and the +# toolchain of rust-toolchain.toml, any machine builds the binary in git byte +# for byte, which the Vectura workflow checks. set -eu out=$(mkdir -p "$1" && cd "$1" && pwd) cd "$(dirname "$0")/.." +# the versions the Vectura workflow installs: another one may build another +# binary +zig=0.16.0 +zigbuild=0.23.4 +if [ "$(zig version)" != "$zig" ]; then + echo "needs zig $zig, found $(zig version)" >&2 + exit 1 +fi +if [ "$(cargo-zigbuild --version)" != "cargo-zigbuild $zigbuild" ]; then + echo "needs cargo-zigbuild $zigbuild, found $(cargo-zigbuild --version)" >&2 + exit 1 +fi + +# rustc writes the paths of the sources in the panic messages. Those of the +# dependencies lie in CARGO_HOME, those of std in the sysroot when the rust-src +# component is installed, /rustc/ otherwise: both become what a +# machine without rust-src writes. RUSTFLAGS from the environment would change +# the binary, they are replaced. +sysroot=$(rustc --print sysroot) +commit=$(rustc -vV | sed -n 's/^commit-hash: //p') +RUSTFLAGS="--remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=/cargo --remap-path-prefix=$sysroot/lib/rustlib/src/rust=/rustc/$commit" +export RUSTFLAGS +unset CARGO_ENCODED_RUSTFLAGS + cargo zigbuild --locked --release --target x86_64-unknown-linux-gnu.2.31 # the glibc suffix is for zig only: the binary lands under the plain triple install -m 755 target/x86_64-unknown-linux-gnu/release/vectura "$out" diff --git a/@xen-orchestra/vmware-explorer/vectura/scripts/release.sh b/@xen-orchestra/vmware-explorer/vectura/scripts/release.sh index bfb330f4a7..b115d0b817 100755 --- a/@xen-orchestra/vmware-explorer/vectura/scripts/release.sh +++ b/@xen-orchestra/vmware-explorer/vectura/scripts/release.sh @@ -1,9 +1,10 @@ #!/bin/sh -# Releases vectura: `scripts/bump-pkg vectura `, at the -# root of the repository, runs it for the `- vectura ` line of -# CHANGELOG.unreleased.md. Bumps the version of the crate, rebuilds its -# binary in place of the previous one and commits both. See build.sh for the -# tools it needs. +# Bumps vectura: run it once, at the end of the pull request which changes +# vectura, then commit Cargo.toml, Cargo.lock and vectura. Bumps the version of +# the crate and rebuilds its binary in place of the previous one. vectura is +# not released on its own: the pull request lists +# `- @xen-orchestra/vmware-explorer ` in CHANGELOG.unreleased.md, whose +# release ships this binary as is. See build.sh for the tools it needs. set -eu case ${1-} in @@ -31,6 +32,13 @@ new=$(echo "$old" | awk -F. -v type="$type" '{ }') echo "vectura: $old -> $new ($type)" +tmp=$(mktemp -d) +# Until the binary is checked, a failure puts the version back, so that the +# script can run again. A copy rather than `git checkout`: it keeps a change +# not committed yet, such as a new dependency in Cargo.lock. +cp Cargo.toml Cargo.lock "$tmp" +trap 'cp "$tmp/Cargo.toml" "$tmp/Cargo.lock" .' EXIT + # The first `version` line of Cargo.toml is the one of [package], and the # line after `name = "vectura"` in Cargo.lock is its entry there. awk -v new="$new" '!done && /^version = / { $0 = "version = \"" new "\""; done = 1 } 1' Cargo.toml > Cargo.toml.tmp @@ -38,12 +46,12 @@ mv Cargo.toml.tmp Cargo.toml awk -v new="$new" 'prev == "name = \"vectura\"" && /^version = / { $0 = "version = \"" new "\"" } { print; prev = $0 }' Cargo.lock > Cargo.lock.tmp mv Cargo.lock.tmp Cargo.lock -tmp=$(mktemp -d) scripts/build.sh "$tmp" +scripts/check-binary.sh "$tmp/vectura" # mv keeps the mode build.sh set, which git records: the binary stays # executable in the npm package mv "$tmp/vectura" vectura -rmdir "$tmp" -git add -- Cargo.toml Cargo.lock vectura -git commit --message "feat(vectura): $new" +trap - EXIT +rm -r "$tmp" +echo "vectura $new built: commit Cargo.toml, Cargo.lock and vectura" diff --git a/@xen-orchestra/vmware-explorer/vectura/tests/bundled_binary.rs b/@xen-orchestra/vmware-explorer/vectura/tests/bundled_binary.rs index b3a5330565..04ee9cdaf2 100644 --- a/@xen-orchestra/vmware-explorer/vectura/tests/bundled_binary.rs +++ b/@xen-orchestra/vmware-explorer/vectura/tests/bundled_binary.rs @@ -3,7 +3,7 @@ //! `@xen-orchestra/vmware-explorer` runs the `vectura` binary of the crate //! root, so it is the build of the version of Cargo.toml and it stays //! executable. scripts/release.sh bumps the version and rebuilds the binary -//! in the same commit. +//! together. #![cfg(all(target_os = "linux", target_arch = "x86_64"))] @@ -22,6 +22,6 @@ fn the_binary_at_the_crate_root_runs_as_this_version() { assert_eq!( String::from_utf8_lossy(&output.stdout), format!("vectura {}\n", env!("CARGO_PKG_VERSION")), - "release with `scripts/bump-pkg vectura ` rather than editing the version by hand" + "bump with `scripts/release.sh ` rather than editing the version by hand" ); } diff --git a/@xen-orchestra/vmware-explorer/vectura/vectura b/@xen-orchestra/vmware-explorer/vectura/vectura index 12fbd9fb2b..b53c2362ce 100755 Binary files a/@xen-orchestra/vmware-explorer/vectura/vectura and b/@xen-orchestra/vmware-explorer/vectura/vectura differ diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 85362706a8..83e15e1457 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -101,7 +101,6 @@ - @xen-orchestra/web minor - @xen-orchestra/web-core minor - @xen-orchestra/xapi minor -- vectura patch - xen-api minor - xo-cli patch - xo-server minor diff --git a/scripts/bump-pkg b/scripts/bump-pkg index 3a5af3a06c..0aff5a4016 100755 --- a/scripts/bump-pkg +++ b/scripts/bump-pkg @@ -14,11 +14,6 @@ cd "$(dirname "$(which "$0")")/.." pkg=$1 case "$pkg" in - vectura) - # a Rust crate, which @xen-orchestra/vmware-explorer bundles - shift - exec @xen-orchestra/vmware-explorer/vectura/scripts/release.sh "$@" - ;; @*/*) cd "$pkg" ;; diff --git a/scripts/check-changed-pkgs.js b/scripts/check-changed-pkgs.js index 550a54d598..a60009b9f6 100755 --- a/scripts/check-changed-pkgs.js +++ b/scripts/check-changed-pkgs.js @@ -33,10 +33,6 @@ const IGNORED_PACKAGES = new Set([ '@xen-orchestra/lite', ]) -// vectura is a Rust crate, released on its own: @xen-orchestra/vmware-explorer -// bundles its binary and `gen-deps-list.js` releases it after vectura -const VECTURA_DIR = '@xen-orchestra/vmware-explorer/vectura/' - // files which don't change what is shipped to users: a package whose changes // are limited to them does not need to be released const IGNORED_FILES = [ @@ -76,7 +72,7 @@ function getChangedPackages(changedFiles, packageNamesByDir) { continue } - const name = file.startsWith(VECTURA_DIR) ? 'vectura' : packageNamesByDir.get(`${parts[0]}/${parts[1]}`) + const name = packageNamesByDir.get(`${parts[0]}/${parts[1]}`) if (name === undefined || IGNORED_PACKAGES.has(name) || isIgnoredFile(file)) { continue } diff --git a/scripts/check-changed-pkgs.test.mjs b/scripts/check-changed-pkgs.test.mjs index 1699723738..8f4c2fb880 100644 --- a/scripts/check-changed-pkgs.test.mjs +++ b/scripts/check-changed-pkgs.test.mjs @@ -115,13 +115,13 @@ describe('getChangedPackages', function () { ) }) - it('maps a vectura file to vectura, not to the package which bundles it', function () { + it('maps a vectura file to the package which bundles its binary', function () { assert.deepEqual( changedPackageNames([ '@xen-orchestra/vmware-explorer/vectura/src/main.rs', '@xen-orchestra/vmware-explorer/vectura/tests/cli.rs', ]), - ['vectura'] + ['@xen-orchestra/vmware-explorer'] ) }) diff --git a/scripts/gen-deps-list.js b/scripts/gen-deps-list.js index b87f05b8b2..352d1eafd9 100755 --- a/scripts/gen-deps-list.js +++ b/scripts/gen-deps-list.js @@ -2,8 +2,7 @@ 'use strict' const semver = require('semver') -const { join } = require('path') -const { getPackages, readFile } = require('./utils') +const { getPackages } = require('./utils') const { readChangelogPackages } = require('./_readChangelogPackages.js') const invert = require('lodash/invert') const keyBy = require('lodash/keyBy') @@ -80,15 +79,6 @@ async function main(args, scriptName) { allPackages = keyBy(await getPackages(true), 'name') - // vectura is a Rust crate, not an npm package: @xen-orchestra/vmware-explorer - // bundles its binary, so a vectura release is followed by one of - // vmware-explorer, as if it depended on this exact version - const vecturaVersion = /^version = "(.+)"$/m.exec( - await readFile(join(__dirname, '../@xen-orchestra/vmware-explorer/vectura/Cargo.toml')) - )[1] - allPackages.vectura = { name: 'vectura', package: { name: 'vectura', version: vecturaVersion } } - allPackages['@xen-orchestra/vmware-explorer'].package.dependencies.vectura = vecturaVersion - const releaseOrder = computeDepOrder(allPackages) Object.entries(toRelease).forEach(([packageName, releaseType]) => {