diff --git a/.github/workflows/builder-image.yml b/.github/workflows/builder-image.yml new file mode 100644 index 0000000..ed1ef8b --- /dev/null +++ b/.github/workflows/builder-image.yml @@ -0,0 +1,132 @@ +name: Builder Image + +# Builds the nanokvm-builder image (riscv64 musl toolchain, Go, and a MaixCDK +# checkout patched with support/sg2002/additional) and publishes it to GHCR so +# the release workflow does not have to rebuild the SDK on every tag. +# +# The image is deliberately not rebuilt on every push: it only carries the +# toolchain. Release builds re-sync support/sg2002/additional into MaixCDK and +# recompile from the checked-out source, so a slightly stale image is harmless. + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - docker/** + - .github/workflows/builder-image.yml + +concurrency: + group: builder-image + cancel-in-progress: false + +jobs: + build: + name: Build and publish nanokvm-builder + runs-on: ubuntu-latest + timeout-minutes: 240 + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Free up disk space + run: | + echo "Before:" + df -h / + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/boost /usr/local/share/powershell + sudo docker image prune -af + echo "After:" + df -h / + + - name: Resolve image reference + id: image + env: + OWNER: ${{ github.repository_owner }} + run: | + # GHCR only accepts lowercase repository paths. + owner=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]') + echo "ref=ghcr.io/$owner/nanokvm-builder" >> "$GITHUB_OUTPUT" + + # Logging in before the build means a credential or registry problem fails + # in seconds instead of after a full SDK build. + - name: Log in to GHCR + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ok=0 + for attempt in 1 2 3; do + if printf '%s' "$GHCR_TOKEN" \ + | docker login ghcr.io -u "$GHCR_USER" --password-stdin; then + ok=1 + break + fi + echo "login attempt $attempt failed, retrying in 15s" + sleep 15 + done + if [ "$ok" -ne 1 ]; then + echo "::error::could not log in to ghcr.io after 3 attempts" + exit 1 + fi + + - name: Build image + run: | + # The Dockerfile chowns MaixCDK to DOCKER_UID/DOCKER_GID and the + # entrypoint gosu's to the caller's ids, so bake in the ids the + # release job will actually run as. Otherwise the SDK tree belongs to + # 1000 and syncing components into it fails with permission denied. + docker build \ + --build-arg DOCKER_UID="$(id -u)" \ + --build-arg DOCKER_GID="$(id -g)" \ + --tag "${{ steps.image.outputs.ref }}:latest" \ + --tag "${{ steps.image.outputs.ref }}:${{ github.sha }}" \ + --file docker/Dockerfile \ + . + + - name: Verify toolchain + run: | + docker run --rm -i "${{ steps.image.outputs.ref }}:latest" go version + docker run --rm -i "${{ steps.image.outputs.ref }}:latest" \ + riscv64-unknown-linux-musl-gcc --version | head -1 + docker run --rm -i "${{ steps.image.outputs.ref }}:latest" patchelf --version + + - name: Push image + env: + IMAGE_REF: ${{ steps.image.outputs.ref }} + SHA: ${{ github.sha }} + run: | + # ghcr.io occasionally times out; a retry is cheaper than rebuilding. + # Push the immutable sha tag first so a failure there cannot leave + # :latest — which release.yml pulls — already moved. + for tag in "$SHA" latest; do + ok=0 + for attempt in 1 2 3; do + if docker push "$IMAGE_REF:$tag"; then + ok=1 + break + fi + echo "push of $tag attempt $attempt failed, retrying in 15s" + sleep 15 + done + if [ "$ok" -ne 1 ]; then + echo "::error::could not push $IMAGE_REF:$tag after 3 attempts" + exit 1 + fi + done + + - name: Summary + run: | + { + echo "### Builder image published" + echo + echo '```' + echo "${{ steps.image.outputs.ref }}:latest" + echo "${{ steps.image.outputs.ref }}:${{ github.sha }}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ba20ee6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,343 @@ +name: NanoKVM Package + +# Builds nanokvm_.tar.gz and the latest.json manifest that the +# on-device updater consumes (server/service/application/). Pull requests get a +# uniquely identified, short-lived test artifact; version tags also attach the +# same package to a GitHub release. +# +# This workflow does NOT publish to cdn.sipeed.com. Uploading latest.json is what +# actually offers the update to every device in the field, so that step stays +# manual and deliberate. + +on: + pull_request: + branches: + - main + paths: + - .github/workflows/release.yml + - kvmapp/** + - scripts/** + - server/** + - support/** + - tools/nanokvm_update_edid/** + - web/** + - Makefile + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + inputs: + version: + description: Version to package, e.g. 2.4.4 + required: true + type: string + +concurrency: + group: package-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + package: + name: Build package + runs-on: ubuntu-latest + timeout-minutes: 180 + outputs: + artifact_name: ${{ steps.version.outputs.artifact_name }} + version: ${{ steps.version.outputs.version }} + permissions: + contents: read + packages: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # package.sh derives SOURCE_DATE_EPOCH from the commit date. + fetch-depth: 0 + # Build scripts from a pull request must not inherit checkout's token. + persist-credentials: false + + - name: Resolve version + id: version + env: + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + INPUT_VERSION: ${{ inputs.version }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REF_NAME: ${{ github.ref_name }} + run: | + if [ "$EVENT_NAME" = "pull_request" ]; then + VERSION="0.${PR_NUMBER}.${GITHUB_RUN_NUMBER}" + SOURCE_SHA="$HEAD_SHA" + elif [ -n "$INPUT_VERSION" ]; then + VERSION="$INPUT_VERSION" + SOURCE_SHA="$GITHUB_SHA" + else + VERSION="$REF_NAME" + SOURCE_SHA="$GITHUB_SHA" + fi + if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::invalid version '$VERSION', expected MAJOR.MINOR.PATCH" + exit 1 + fi + if [ -z "$SOURCE_SHA" ]; then + echo "::error::could not resolve source commit" + exit 1 + fi + SHORT_SHA=$(printf '%s' "$SOURCE_SHA" | cut -c1-12) + if [ "$EVENT_NAME" = "pull_request" ]; then + ARTIFACT_NAME="nanokvm-pr-${PR_NUMBER}-${SHORT_SHA}-run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + else + ARTIFACT_NAME="nanokvm-${VERSION}-${SHORT_SHA}-run-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" + fi + echo "artifact_name=$ARTIFACT_NAME" >> "$GITHUB_OUTPUT" + echo "source_sha=$SOURCE_SHA" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Packaging version $VERSION as $ARTIFACT_NAME" + + - name: Free up disk space + run: | + df -h / + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + df -h / + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Set up pnpm + run: npm install --global pnpm@11 + + - name: Build frontend + run: make web + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ok=0 + for attempt in 1 2 3; do + if printf '%s' "$GHCR_TOKEN" \ + | docker login ghcr.io -u "$GHCR_USER" --password-stdin; then + ok=1 + break + fi + echo "login attempt $attempt failed, retrying in 15s" + sleep 15 + done + if [ "$ok" -ne 1 ]; then + echo "::error::could not log in to ghcr.io after 3 attempts" + exit 1 + fi + + - name: Pull builder image + id: image + env: + OWNER: ${{ github.repository_owner }} + run: | + # GHCR only accepts lowercase repository paths. + owner=$(echo "$OWNER" | tr '[:upper:]' '[:lower:]') + IMAGE_REPO="ghcr.io/$owner/nanokvm-builder" + TAGGED_REF="${IMAGE_REPO}:latest" + ok=0 + for attempt in 1 2 3; do + if docker pull "$TAGGED_REF"; then + ok=1 + break + fi + echo "pull attempt $attempt failed, retrying in 15s" + sleep 15 + done + if [ "$ok" -ne 1 ]; then + echo "::error::could not pull $TAGGED_REF - run the 'Builder Image' workflow first and ensure PR builds can pull it without credentials" + exit 1 + fi + RESOLVED_REF=$(docker image inspect --format='{{index .RepoDigests 0}}' "$TAGGED_REF") + case "$RESOLVED_REF" in + "$IMAGE_REPO"@sha256:*) ;; + *) + echo "::error::could not resolve immutable digest for $TAGGED_REF (got '$RESOLVED_REF')" + exit 1 + ;; + esac + echo "Pulled $RESOLVED_REF" + echo "ref=$RESOLVED_REF" >> "$GITHUB_OUTPUT" + + - name: Build riscv64 artifacts + run: | + make release-build \ + DOCKER_TTY= \ + IMAGE_NAME="${{ steps.image.outputs.ref }}" + + - name: Assemble package + run: make package VERSION="${{ steps.version.outputs.version }}" + + - name: Compare against the published release + # Informational only: highlights what changed relative to what devices + # are currently running. Never blocks the build. + continue-on-error: true + run: | + ./scripts/compare-release.sh \ + "build/release/nanokvm_${{ steps.version.outputs.version }}.tar.gz" + + - name: Write build provenance + env: + ARTIFACT_NAME: ${{ steps.version.outputs.artifact_name }} + BUILDER_IMAGE: ${{ steps.image.outputs.ref }} + BUILD_SHA: ${{ github.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + SOURCE_SHA: ${{ steps.version.outputs.source_sha }} + VERSION: ${{ steps.version.outputs.version }} + run: | + TARBALL="build/release/nanokvm_${VERSION}.tar.gz" + TARBALL_NAME=$(basename "$TARBALL") + SHA256_HEX=$(sha256sum "$TARBALL" | cut -d' ' -f1) + SHA512_HEX=$(sha512sum "$TARBALL" | cut -d' ' -f1) + SHA512_BASE64=$(jq -er '.sha512 | select(type == "string" and length > 0)' \ + build/release/latest.json) + ACTUAL_BASE64=$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A) + if [ "$SHA512_BASE64" != "$ACTUAL_BASE64" ]; then + echo "::error::latest.json sha512 does not match $TARBALL" + exit 1 + fi + printf '%s %s\n' "$SHA256_HEX" "$TARBALL_NAME" > build/release/sha256.txt + { + echo "artifact=${ARTIFACT_NAME}" + echo "version=${VERSION}" + echo "event=${GITHUB_EVENT_NAME}" + echo "pull_request=${PR_NUMBER}" + echo "source_sha=${SOURCE_SHA}" + echo "build_sha=${BUILD_SHA}" + echo "builder_image=${BUILDER_IMAGE}" + echo "run_id=${GITHUB_RUN_ID}" + echo "run_attempt=${GITHUB_RUN_ATTEMPT}" + echo "run_url=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "tarball=${TARBALL_NAME}" + echo "sha256_hex=${SHA256_HEX}" + echo "sha512_hex=${SHA512_HEX}" + echo "sha512_base64=${SHA512_BASE64}" + } > build/release/BUILD_INFO.txt + + - name: Summary + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + { + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "> [!WARNING]" + echo "> PR TEST ONLY — DO NOT PUBLISH TO CDN" + echo + fi + echo "### nanokvm_${VERSION}.tar.gz" + echo + echo '```text' + cat build/release/BUILD_INFO.txt + echo '```' + echo + echo '```json' + cat build/release/latest.json + echo '```' + echo + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "Upload only the inner tarball through NanoKVM's manual offline update UI." + else + echo "Publishing to \`cdn.sipeed.com/nanokvm/\` is a separate manual step." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.version.outputs.artifact_name }} + path: | + build/release/nanokvm_${{ steps.version.outputs.version }}.tar.gz + build/release/latest.json + build/release/sha256.txt + build/release/BUILD_INFO.txt + if-no-files-found: error + retention-days: ${{ github.event_name == 'pull_request' && 7 || 90 }} + + publish-release: + name: Attach package to GitHub release + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + needs: package + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download package artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.package.outputs.artifact_name }} + path: build/release + + - name: Attach to GitHub release + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ needs.package.outputs.version }} + run: | + TARBALL="build/release/nanokvm_${VERSION}.tar.gz" + TARBALL_NAME=$(basename "$TARBALL") + SHA256=$(sha256sum "$TARBALL" | cut -d' ' -f1) + EXPECTED_SHA256_LINE="${SHA256} ${TARBALL_NAME}" + ACTUAL_SHA256_LINE=$(cat build/release/sha256.txt) + if [ "$ACTUAL_SHA256_LINE" != "$EXPECTED_SHA256_LINE" ]; then + echo "::error::sha256.txt does not match $TARBALL" + exit 1 + fi + SHA512=$(jq -er '.sha512 | select(type == "string" and length > 0)' \ + build/release/latest.json) + ACTUAL_SHA512=$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A) + if [ "$SHA512" != "$ACTUAL_SHA512" ]; then + echo "::error::latest.json sha512 does not match $TARBALL" + exit 1 + fi + + CHECKSUM_BLOCK=$(printf '%s\n' \ + "" \ + "### Checksums" \ + "" \ + "\`SHA-256\` (hex):" \ + "" \ + " ${SHA256}" \ + "" \ + "\`SHA-512\` (base64, as expected by the on-device updater):" \ + "" \ + " ${SHA512}" \ + "") + + NOTES=$(printf '%s\n' \ + "Application package for NanoKVM ${VERSION}." \ + "" \ + "${CHECKSUM_BLOCK}" \ + "" \ + "To offer this build over OTA, upload the tarball and \`latest.json\` to \`cdn.sipeed.com/nanokvm/\`.") + + if gh release view "$VERSION" >/dev/null 2>&1; then + echo "Release $VERSION exists; updating notes and assets" + CURRENT_NOTES=$(gh release view "$VERSION" --json body --jq .body) + PRESERVED_NOTES=$(printf '%s\n' "$CURRENT_NOTES" | awk ' + $0 == "" { skip = 1; next } + $0 == "" { skip = 0; next } + !skip { print } + ') + if [ -n "$PRESERVED_NOTES" ]; then + UPDATED_NOTES=$(printf '%s\n\n%s\n' "$PRESERVED_NOTES" "$CHECKSUM_BLOCK") + else + UPDATED_NOTES="$CHECKSUM_BLOCK" + fi + gh release upload "$VERSION" \ + "$TARBALL" build/release/latest.json build/release/sha256.txt --clobber + # Publish the new checksums only after every asset upload succeeds. + gh release edit "$VERSION" --notes "$UPDATED_NOTES" + else + gh release create "$VERSION" \ + --title "$VERSION" \ + --notes "$NOTES" \ + "$TARBALL" build/release/latest.json build/release/sha256.txt + fi diff --git a/.gitignore b/.gitignore index 2fe22c5..db77876 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ support/sg2002/kvm_vision_test/CMakeLists.txt support/sg2002/additional/original kvmapp/server/dl_lib kvmapp/kvm_system/kvm_system + +# Release packages assembled by scripts/package.sh +/build/ diff --git a/Makefile b/Makefile index 79f1bf4..dc32083 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ GID := $(shell id -g) # ?= so a prebuilt image can be used instead, e.g. a GHCR-cached builder in CI. IMAGE_NAME ?= nanokvm-builder-local-$(UID)-$(GID) PWD := $(shell pwd) +VERSION ?= # Docker run common parameters. Allocating a TTY breaks in environments without # one, so it can be overridden with DOCKER_TTY= @@ -19,8 +20,11 @@ DOCKER_BUILD_ARGS := --build-arg DOCKER_UID=$(UID) --build-arg DOCKER_GID=$(GID) # Build commands GO_BUILD_CMD := cd /home/build/NanoKVM/server && go mod tidy && CGO_ENABLED=1 GOOS=linux GOARCH=riscv64 CC=riscv64-unknown-linux-musl-gcc CGO_CFLAGS="-mcpu=c906fdv -march=rv64imafdcv0p7xthead -mcmodel=medany -mabi=lp64d" go build SUPPORT_BUILD_CMD := . ./home/build/MaixCDK/bin/activate && cd /home/build/NanoKVM/support/sg2002 && ./build kvm_system && ./build kvm_system add_to_kvmapp +VISION_BUILD_CMD := . ./home/build/MaixCDK/bin/activate && cd /home/build/NanoKVM/support/sg2002 && ./build kvm_vision && ./build kvm_vision add_to_kvmapp +RELEASE_BUILD_CMD := /home/build/NanoKVM/scripts/build-in-container.sh -.PHONY: help check-root builder-image rebuild-image check-image shell app support all clean +.PHONY: help check-root builder-image rebuild-image check-image shell app support vision \ + web release-build package release all clean # Default target all: app support @@ -36,8 +40,13 @@ help: @echo " rebuild-image - Force rebuild Docker image" @echo " shell - Enter interactive builder environment" @echo " app - Build Go application server" - @echo " support - Build hardware support libraries" + @echo " support - Build kvm_system daemon" + @echo " vision - Build video libraries (libkvm.so)" + @echo " web - Build the frontend into web/dist" @echo " all - Build both app and support (default)" + @echo " release-build - Build every riscv64 release artifact in one pass" + @echo " package - Assemble nanokvm_.tar.gz + latest.json" + @echo " release - release-build + web + package (needs VERSION=x.y.z)" @echo " clean - Clean build artifacts" @echo "" @echo "Prerequisites:" @@ -90,6 +99,37 @@ support: check-root builder-image @echo "Building support..." @$(DOCKER_RUN_BASE) $(DOCKER_TTY) $(IMAGE_NAME) /bin/bash -c '$(SUPPORT_BUILD_CMD)' +# Build video libraries (libkvm.so) into kvmapp/server/dl_lib +vision: check-root builder-image + @echo "Building vision..." + @$(DOCKER_RUN_BASE) $(DOCKER_TTY) $(IMAGE_NAME) /bin/bash -c '$(VISION_BUILD_CMD)' + +# Build every riscv64 release artifact in one container pass +release-build: check-root builder-image + @echo "Building release artifacts..." + @$(DOCKER_RUN_BASE) $(DOCKER_TTY) $(IMAGE_NAME) /bin/bash -c '$(RELEASE_BUILD_CMD)' + +# Build the frontend (runs on the host; the builder image has no Node) +web: + @echo "Building web..." + @cd web && pnpm install --frozen-lockfile && pnpm build + +# Assemble the release package and its manifest +package: + @if [ -z "$(VERSION)" ]; then \ + echo "VERSION is required, e.g. make package VERSION=2.4.4"; \ + exit 1; \ + fi + @scripts/package.sh "$(VERSION)" + +# Full local release: riscv64 artifacts + frontend + package. +# Invoked as sub-makes rather than prerequisites: packaging asserts on what the +# earlier steps produce, so the order matters even under make -j. +release: + @$(MAKE) release-build + @$(MAKE) web + @$(MAKE) package VERSION="$(VERSION)" + # Clean build artifacts clean: @echo "Cleaning build artifacts..." @@ -101,4 +141,12 @@ clean: rm -rf support/sg2002/build; \ echo "Removed support/sg2002/build"; \ fi + @if [ -d build/release ]; then \ + rm -rf build/release; \ + echo "Removed build/release"; \ + fi + @if [ -d web/dist ]; then \ + rm -rf web/dist; \ + echo "Removed web/dist"; \ + fi @echo "Clean completed." diff --git a/README.md b/README.md index 435d205..4a57a2d 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ Start with the guide that matches the part of NanoKVM you want to work on: - **System support modules:** Build and update the low-level hardware support components in [support/sg2002/README.md](support/sg2002/README.md). - **Backend service:** Set up, build, and understand the Go service in [server/README.md](server/README.md). - **Frontend UI:** Develop, lint, and build the React interface in [web/README.md](web/README.md). +- **Release packaging:** Assemble the `nanokvm_.tar.gz` update package in [scripts/README.md](scripts/README.md). > Backend compilation and runtime validation require the target toolchain or a NanoKVM device. See the module-specific guides above for the latest development workflow. diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..4c2e668 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,92 @@ +# Release packaging + +These scripts assemble the application update package that NanoKVM devices pull +over the air: `nanokvm_.tar.gz` plus its `latest.json` manifest. + +| Script | Purpose | +|---|---| +| `build-in-container.sh` | Builds every riscv64 artifact (`kvm_system`, `libkvm.so`, `NanoKVM-Server`). Runs inside the `nanokvm-builder` image only. | +| `package.sh` | Stages the package tree, creates the tarball, and writes `latest.json`. | +| `compare-release.sh` | Diffs a freshly built package against the currently published one. Informational. | + +## What the updater expects + +The format is not arbitrary — it is fixed by the on-device updater in +`server/service/application/`: + +- **One root directory.** `install.go` untars the package and moves the single + top-level directory over `/kvmapp`, so the tarball must contain exactly + `nanokvm_/`. +- **`name` is the file name.** `version.go` builds the download URL as + `/`, where `` is `https://cdn.sipeed.com/nanokvm` (or + `.../preview` when `/etc/kvm/preview_updates` exists). +- **`sha512` is base64, not hex.** `update.go` compares against + `base64(raw sha512 digest)`. A hex digest will fail verification on device. +- **`/kvmapp/version`** is what the device reports as its installed version, so + it must match the `version` field. + +`size` is parsed into `Latest.Size` but never read anywhere, so nothing on the +device depends on its units. Published manifests have carried a kilobyte-ish +value; `package.sh` writes the exact byte count instead. + +## Building a release + +The frontend needs Node and pnpm on the host; everything riscv64 needs the +builder image (see `docker/Dockerfile`). + +```bash +make release VERSION=2.4.4 +``` + +That is equivalent to: + +```bash +make release-build # kvm_system + libkvm.so + NanoKVM-Server, in Docker +make web # web/dist +make package VERSION=2.4.4 # build/release/{nanokvm_2.4.4.tar.gz,latest.json} +``` + +In CI this runs as the **NanoKVM Package** workflow. Pull requests get a +short-lived Actions artifact for device testing; pushing a `MAJOR.MINOR.PATCH` +tag creates or updates a GitHub release with three assets: + +- `nanokvm_.tar.gz` +- `latest.json` +- `sha256.txt` + +The Actions artifact also includes `BUILD_INFO.txt` with the source commit, +workflow run, immutable builder image digest, and both checksum encodings; that +provenance file is not attached to the GitHub release. Uploading the tarball and +`latest.json` to the CDN is what actually offers the update to devices in the +field, so that step stays manual. `sha256.txt` is only for users verifying a +manual download and does not change the on-device `latest.json` contract. + +## Where each file in the package comes from + +| Package path | Source | +|---|---| +| `version` | the requested version | +| `server/NanoKVM-Server` | `server/build.sh` (BoringCrypto, RPATH `$ORIGIN/dl_lib`) | +| `server/dl_lib/` | whatever the `kvm_vision` build emits, with any library it does not emit backfilled from the tracked `server/dl_lib/` | +| `server/web/` | `web/dist/` | +| `kvm_system/kvm_system` | `support/sg2002` `build kvm_system` | +| `system/tool/` | prebuilt binaries in `tools/nanokvm_update_edid/` | +| `kvm/` | default runtime state (resolution, fps, quality) | +| everything else | tracked `kvmapp/` | + +Note that `kvm/` ships default runtime state, and `install.go` replaces +`/kvmapp` wholesale, so an update resets those values on the device. That is +long-standing behaviour, not something these scripts introduce. + +## The two copies of libkvm.so + +`NanoKVM-Server` is cgo-linked against the **tracked** `server/dl_lib/libkvm.so` +(`server/common/kvm_vision.go`: `-L../dl_lib -lkvm`), but the package ships the +library the `kvm_vision` build just produced. Those are two different files, and +the tracked one can lag well behind — published releases have shipped this +mismatch for a long time. + +Because a symbol the binary imports could in principle be absent from the +shipped library, and that would only surface as a crash on a real device, +`build-in-container.sh` checks the shipped `libkvm.so` still exports every +symbol `NanoKVM-Server` actually imports from it, and fails the build otherwise. diff --git a/scripts/build-in-container.sh b/scripts/build-in-container.sh new file mode 100755 index 0000000..048ee26 --- /dev/null +++ b/scripts/build-in-container.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Build every riscv64 artifact of a release, from inside the nanokvm-builder +# image (docker/Dockerfile). Not meant to be run on the host: it needs the +# riscv64 musl toolchain and the patched MaixCDK checkout that the image ships. +# +# Produces: +# kvmapp/kvm_system/kvm_system kvm_system daemon +# kvmapp/server/dl_lib/ video libraries, including a fresh libkvm.so +# server/NanoKVM-Server Go server (BoringCrypto, RPATH $ORIGIN/dl_lib) + +set -euo pipefail + +# docker/entrypoint sets HOME for the build user. support/sg2002/build resolves +# both MaixCDK and the source tree through ~, so a wrong HOME would send the +# whole build at /root; say so plainly rather than emitting "directory missing". +BUILD_HOME="${BUILD_HOME:-/home/build}" +if [ "${HOME:-}" != "$BUILD_HOME" ]; then + echo "[ERROR] HOME is '${HOME:-unset}', expected $BUILD_HOME" >&2 + echo " run this inside the builder image via the Makefile" >&2 + exit 1 +fi + +MAIXCDK_PATH="$HOME/MaixCDK" +NANOKVM_PATH="$HOME/NanoKVM" + +for dir in "$MAIXCDK_PATH" "$NANOKVM_PATH"; do + if [ ! -d "$dir" ]; then + echo "[ERROR] $dir not found - is this running inside nanokvm-builder?" >&2 + exit 1 + fi +done + +# Check the toolchain up front. builder-image reuses any existing image, so an +# image built before a tooling change silently lacks it - and without this the +# SDK build would run for minutes before server/build.sh failed at the end. +for tool in go riscv64-unknown-linux-musl-gcc patchelf; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "[ERROR] '$tool' is missing from the builder image" >&2 + echo " the image is out of date; rebuild it with: make rebuild-image" >&2 + exit 1 + fi +done + +# The SDK tree is owned by the ids baked into the image (DOCKER_UID/DOCKER_GID), +# while the entrypoint drops to the caller's ids. If they disagree, syncing +# components fails partway through a long build; catch it here instead. +if [ ! -w "$MAIXCDK_PATH/components" ]; then + echo "[ERROR] $MAIXCDK_PATH/components is not writable by $(id -u):$(id -g)" >&2 + echo " the image was built for different ids; rebuild it with:" >&2 + echo " make rebuild-image" >&2 + exit 1 +fi + +# shellcheck disable=SC1091 +. "$MAIXCDK_PATH/bin/activate" + +echo "::group::support: sync MaixCDK components" +cd "$NANOKVM_PATH/support/sg2002" +# The image bakes support/sg2002/additional/ into MaixCDK's components at image +# build time, and ./build only re-copies them when components/kvm is missing. +# Refresh unconditionally so a cached image never yields stale libraries. +./build update_lib +echo "::endgroup::" + +echo "::group::support: build kvm_system" +./build kvm_system +./build kvm_system add_to_kvmapp +echo "::endgroup::" + +echo "::group::support: build kvm_vision (libkvm.so)" +./build kvm_vision +./build kvm_vision add_to_kvmapp +echo "::endgroup::" + +echo "::group::server: cross-compile NanoKVM-Server" +cd "$NANOKVM_PATH/server" +./build.sh +echo "::endgroup::" + +# support/sg2002/build reports "Build Error!" without a non-zero exit status for +# some failure paths, so assert on the artifacts rather than trusting $?. +echo "[INFO] verifying artifacts" +missing=0 +for artifact in \ + "$NANOKVM_PATH/kvmapp/kvm_system/kvm_system" \ + "$NANOKVM_PATH/kvmapp/server/dl_lib/libkvm.so" \ + "$NANOKVM_PATH/server/NanoKVM-Server" +do + if [ -f "$artifact" ]; then + echo " ok ${artifact#$NANOKVM_PATH/}" + else + echo " MISSING ${artifact#$NANOKVM_PATH/}" >&2 + missing=1 + fi +done + +if [ "$missing" -ne 0 ]; then + echo "[ERROR] build did not produce all expected artifacts" >&2 + exit 1 +fi + +# NanoKVM-Server is cgo-linked against the tracked server/dl_lib/libkvm.so +# (server/common/kvm_vision.go: -L../dl_lib -lkvm), but the package ships the +# freshly built one from kvm_vision. Those are different files, so confirm the +# shipped library still exports every symbol the binary actually imports from +# it. Without this a dropped symbol only shows up as a crash on a real device. +echo "[INFO] verifying shipped libkvm.so satisfies NanoKVM-Server" +if ! command -v readelf >/dev/null 2>&1; then + echo "[WARN] readelf unavailable, skipping ABI check" +else + exported() { + readelf --dyn-syms --wide "$1" 2>/dev/null \ + | awk '$7 != "UND" && $8 != "" { print $8 }' \ + | sed 's/@.*//' | LC_ALL=C sort -u + } + imported() { + readelf --dyn-syms --wide "$1" 2>/dev/null \ + | awk '$7 == "UND" && $8 != "" { print $8 }' \ + | sed 's/@.*//' | LC_ALL=C sort -u + } + + linked_lib="$NANOKVM_PATH/server/dl_lib/libkvm.so" + shipped_lib="$NANOKVM_PATH/kvmapp/server/dl_lib/libkvm.so" + server_bin="$NANOKVM_PATH/server/NanoKVM-Server" + + # Symbols the binary imports that the link-time library provides, i.e. the + # ones libkvm.so is actually responsible for at runtime. + needed=$(comm -12 <(imported "$server_bin") <(exported "$linked_lib")) + + if [ -z "$needed" ]; then + echo "[WARN] no libkvm symbols resolved; skipping ABI check" + else + absent=$(comm -23 <(printf '%s\n' "$needed") <(exported "$shipped_lib")) + if [ -n "$absent" ]; then + echo "[ERROR] shipped libkvm.so is missing symbols NanoKVM-Server needs:" >&2 + printf ' %s\n' $absent >&2 + exit 1 + fi + echo " ok $(printf '%s\n' "$needed" | wc -l | tr -d ' ') libkvm symbols resolve" + fi +fi + +echo "[DONE] riscv64 artifacts built" diff --git a/scripts/compare-release.sh b/scripts/compare-release.sh new file mode 100755 index 0000000..0c76dbc --- /dev/null +++ b/scripts/compare-release.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Compare a freshly built package against the currently published release. +# +# Purely informational: it exits 0 whatever it finds. The point is to make the +# delta reviewable before the tarball is published, since the package is what +# every device pulls over OTA. Unexpected entries here (a library that vanished, +# an init script that changed without a matching commit) are worth a look. +# +# Usage: scripts/compare-release.sh [base-url] + +set -uo pipefail + +NEW_TARBALL="${1:-}" +BASE_URL="${2:-https://cdn.sipeed.com/nanokvm}" + +if [ -z "$NEW_TARBALL" ] || [ ! -f "$NEW_TARBALL" ]; then + echo "Usage: $0 [base-url]" >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "[INFO] fetching published manifest from $BASE_URL/latest.json" +if ! curl -fsSL "$BASE_URL/latest.json" -o "$WORK/latest.json"; then + echo "[WARN] could not fetch latest.json - skipping comparison" + exit 0 +fi + +cat "$WORK/latest.json" + +OLD_NAME="$(sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$WORK/latest.json")" +if [ -z "$OLD_NAME" ]; then + echo "[WARN] no \"name\" in latest.json - skipping comparison" + exit 0 +fi + +echo "[INFO] downloading published package $OLD_NAME" +if ! curl -fsSL "$BASE_URL/$OLD_NAME" -o "$WORK/$OLD_NAME"; then + echo "[WARN] could not download $OLD_NAME - skipping comparison" + exit 0 +fi + +mkdir -p "$WORK/old" "$WORK/new" +if ! tar xzf "$WORK/$OLD_NAME" -C "$WORK/old"; then + echo "[WARN] could not extract $OLD_NAME - skipping comparison" + exit 0 +fi +if ! tar xzf "$NEW_TARBALL" -C "$WORK/new"; then + echo "[WARN] could not extract $(basename "$NEW_TARBALL") - skipping comparison" + exit 0 +fi + +# Strip the nanokvm_/ prefix so the two trees are comparable. +OLD_ROOT="$(find "$WORK/old" -mindepth 1 -maxdepth 1 -type d | head -1)" +NEW_ROOT="$(find "$WORK/new" -mindepth 1 -maxdepth 1 -type d | head -1)" + +if [ -z "$OLD_ROOT" ] || [ -z "$NEW_ROOT" ]; then + echo "[WARN] could not locate a package root directory - skipping comparison" + exit 0 +fi + +( cd "$OLD_ROOT" && find . -type f | sed 's|^\./||' | LC_ALL=C sort ) > "$WORK/old.list" +( cd "$NEW_ROOT" && find . -type f | sed 's|^\./||' | LC_ALL=C sort ) > "$WORK/new.list" + +# An empty inventory would make every section below print "(none)", which reads +# identically to "verified, nothing changed". Refuse to imply that. +if [ ! -s "$WORK/old.list" ] || [ ! -s "$WORK/new.list" ]; then + echo "[WARN] one of the packages produced an empty file list - comparison unreliable, skipping" + exit 0 +fi + +echo +echo "==============================================================" +echo " $(basename "$OLD_ROOT") -> $(basename "$NEW_ROOT")" +echo "==============================================================" +printf ' files: %s -> %s\n' "$(wc -l < "$WORK/old.list" | tr -d ' ')" \ + "$(wc -l < "$WORK/new.list" | tr -d ' ')" + +echo +echo "--- added ----------------------------------------------------" +comm -13 "$WORK/old.list" "$WORK/new.list" | sed 's/^/ + /' || true + +echo +echo "--- removed --------------------------------------------------" +comm -23 "$WORK/old.list" "$WORK/new.list" | sed 's/^/ - /' || true + +echo +echo "--- changed --------------------------------------------------" +changed=0 +while IFS= read -r file; do + if ! cmp -s "$OLD_ROOT/$file" "$NEW_ROOT/$file"; then + old_size="$(wc -c < "$OLD_ROOT/$file" | tr -d ' ')" + new_size="$(wc -c < "$NEW_ROOT/$file" | tr -d ' ')" + printf ' ~ %-52s %s -> %s bytes\n' "$file" "$old_size" "$new_size" + changed=$((changed + 1)) + fi +done < <(comm -12 "$WORK/old.list" "$WORK/new.list") +[ "$changed" -eq 0 ] && echo " (none)" + +echo +echo "[INFO] comparison complete" diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100755 index 0000000..419e4f8 --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# +# Assemble a NanoKVM release package: nanokvm_.tar.gz + latest.json. +# +# The layout and the manifest fields are dictated by the on-device updater: +# - server/service/application/version.go parses latest.json and derives the +# download URL as "/", so "name" must be the tarball file name. +# - server/service/application/update.go verifies the download against +# "sha512", which is the *base64* encoding of the raw SHA-512 digest. +# - server/service/application/install.go untars the package and moves the +# single top-level directory over /kvmapp, so the tarball must contain +# exactly one root directory: nanokvm_/. +# +# Build artifacts are expected to be in place already (see +# scripts/build-in-container.sh and the "web" target in the Makefile): +# server/NanoKVM-Server riscv64 server binary +# kvmapp/kvm_system/kvm_system kvm_system daemon +# kvmapp/server/dl_lib/libkvm.so freshly built video library (optional) +# web/dist/ built frontend +# +# Usage: scripts/package.sh + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +VERSION="${1:-}" + +if [ -z "$VERSION" ]; then + echo "Usage: $0 e.g. $0 2.4.4" >&2 + exit 1 +fi + +# The updater rejects file names outside [a-zA-Z0-9._-] (update_offline.go), +# and the version string ends up in the file name, so validate it up front. +if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "[ERROR] invalid version '$VERSION', expected MAJOR.MINOR.PATCH" >&2 + exit 1 +fi + +OUT="$ROOT/build/release" +STAGE="$OUT/nanokvm_$VERSION" +TARBALL="$OUT/nanokvm_$VERSION.tar.gz" +MANIFEST="$OUT/latest.json" + +require_file() { + if [ ! -f "$1" ]; then + echo "[ERROR] missing build artifact: ${1#"$ROOT"/}" >&2 + echo " $2" >&2 + exit 1 + fi +} + +# A host-architecture build is indistinguishable from a cross-compiled one by +# name alone, and NanoKVM-Server is gitignored, so nothing else would catch it. +# Refuse to package anything that is not riscv64 (ELF e_machine 243). +require_riscv64() { + local path="$1" magic machine + magic=$(od -An -tx1 -N4 "$path" | tr -d ' \n') + if [ "$magic" != "7f454c46" ]; then + echo "[ERROR] not an ELF object: ${path#"$ROOT"/}" >&2 + exit 1 + fi + machine=$(od -An -tu1 -j18 -N1 "$path" | tr -d ' \n') + if [ "$machine" != "243" ]; then + echo "[ERROR] ${path#"$ROOT"/} is not riscv64 (ELF e_machine=$machine)" >&2 + echo " a host-architecture build must never be packaged" >&2 + exit 1 + fi +} + +require_file "$ROOT/server/NanoKVM-Server" "run: make app" +require_file "$ROOT/kvmapp/kvm_system/kvm_system" "run: make support" +require_file "$ROOT/web/dist/index.html" "run: make web" +# libkvm.so is the one runtime library that must come from the build: the copy +# tracked in server/dl_lib/ exists for cgo link time and lags far behind. If it +# were optional here, "make package" on its own would quietly ship the stale one. +require_file "$ROOT/kvmapp/server/dl_lib/libkvm.so" "run: make vision" + +require_riscv64 "$ROOT/server/NanoKVM-Server" +require_riscv64 "$ROOT/kvmapp/kvm_system/kvm_system" +require_riscv64 "$ROOT/kvmapp/server/dl_lib/libkvm.so" + +echo "[INFO] staging nanokvm_$VERSION" +# Clear the whole output directory: release.yml uploads build/release/nanokvm_* +# by glob, and a leftover tarball from an earlier version would ride along. +rm -rf "$OUT" +mkdir -p "$STAGE" + +# 1. Everything tracked under kvmapp/ (init scripts, picoclaw bundle, kernel +# module, jpg_stream, kvm_stream) plus whatever the support build dropped in +# (kvm_system, server/dl_lib). -a carries the exec bits over from git, which +# only matters for reading the archive by hand: install.go chmods the whole +# tree to 0755 on the device regardless. +cp -a "$ROOT/kvmapp/." "$STAGE/" + +# 2. Version marker. The updater reads /kvmapp/version to report the currently +# installed version (version.go). +printf '%s\n' "$VERSION" > "$STAGE/version" + +# 3. Go server binary. +mkdir -p "$STAGE/server" +cp -a "$ROOT/server/NanoKVM-Server" "$STAGE/server/NanoKVM-Server" + +# 4. Runtime shared libraries. "build kvm_vision add_to_kvmapp" copies its whole +# dist/dl_lib into kvmapp/server/dl_lib, so freshly built libraries are +# already staged; the tracked server/dl_lib/ backfills the rest. Never +# clobber a fresh library with the tracked (link-time) copy. +mkdir -p "$STAGE/server/dl_lib" +for lib in "$ROOT"/server/dl_lib/*; do + [ -e "$lib" ] || continue + name="$(basename "$lib")" + if [ ! -e "$STAGE/server/dl_lib/$name" ]; then + cp -a "$lib" "$STAGE/server/dl_lib/$name" + fi +done + +# The SDK is built from an unpinned MaixCDK checkout, so its dist could one +# day add or rename a library (e.g. an soname bump leaving both .409 and +# .410 behind) and we would ship a library set nobody reviewed. Require the +# shipped names to be exactly the tracked set, and fail loudly otherwise. +staged_libs=$(cd "$STAGE/server/dl_lib" && ls -1 | LC_ALL=C sort) +tracked_libs=$(cd "$ROOT/server/dl_lib" && ls -1 | LC_ALL=C sort) +if [ "$staged_libs" != "$tracked_libs" ]; then + echo "[ERROR] shipped server/dl_lib does not match the tracked library set" >&2 + echo " unexpected (in package, not tracked):" >&2 + comm -23 <(printf '%s\n' "$staged_libs") <(printf '%s\n' "$tracked_libs") \ + | sed 's/^/ + /' >&2 + echo " absent (tracked, not in package):" >&2 + comm -13 <(printf '%s\n' "$staged_libs") <(printf '%s\n' "$tracked_libs") \ + | sed 's/^/ - /' >&2 + echo " if this change is intended, update server/dl_lib/ to match." >&2 + exit 1 +fi + +# A library arrives either from the SDK dist (mode 755) or from the tracked +# backfill (git stores them 644), so without this the archive would depend on +# which path populated each file. 644 is what published releases have always +# carried, and install.go chmods the installed tree to 0755 anyway. +chmod 644 "$STAGE"/server/dl_lib/* + +# 5. Frontend. router.go serves /web. +rm -rf "$STAGE/server/web" +mkdir -p "$STAGE/server/web" +cp -a "$ROOT/web/dist/." "$STAGE/server/web/" + +# 6. EDID helper shipped under system/tool/ (prebuilt riscv64 binary in tools/). +mkdir -p "$STAGE/system/tool" +cp -a "$ROOT/tools/nanokvm_update_edid/nanokvm_update_edid" "$STAGE/system/tool/" +cp -a "$ROOT/tools/nanokvm_update_edid/E21_NanoKVM.bin" "$STAGE/system/tool/" + +# 7. Default runtime state read by common.GetScreen() on first boot. +mkdir -p "$STAGE/kvm" +printf '30\n' > "$STAGE/kvm/fps" +printf '0\n' > "$STAGE/kvm/now_fps" +printf '60\n' > "$STAGE/kvm/qlty" +printf '1920\n' > "$STAGE/kvm/width" +printf '1080\n' > "$STAGE/kvm/height" +printf '1\n' > "$STAGE/kvm/state" +printf 'mjpeg\n' > "$STAGE/kvm/type" +printf '0' > "$STAGE/kvm/res" + +# 8. Directories the released package has always carried, kept so the layout +# matches previous releases exactly (system_init.cpp probes inside them). +mkdir -p "$STAGE/jpg_stream/dl_lib" "$STAGE/kvm_system/dl_lib" + +# --- archive ----------------------------------------------------------------- +# Normalise owner and timestamps so the same source tree yields the same +# tarball, which makes the published sha512 verifiable after the fact. +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -C "$ROOT" log -1 --format=%ct 2>/dev/null || echo 0)}" + +find "$STAGE" -exec touch -d "@$SOURCE_DATE_EPOCH" {} + 2>/dev/null \ + || find "$STAGE" -exec touch -t "$(date -r "$SOURCE_DATE_EPOCH" +%Y%m%d%H%M.%S)" {} + 2>/dev/null \ + || echo "[WARN] could not normalise timestamps" + +echo "[INFO] creating $(basename "$TARBALL")" +rm -f "$TARBALL" + +# Read the version into a variable first: piping through head can hand tar a +# SIGPIPE, and under "set -o pipefail" that would silently select the bsdtar +# branch on a GNU system, quietly losing reproducibility. +tar_version="$(tar --version 2>/dev/null || true)" + +if [ "${tar_version#*GNU}" != "$tar_version" ]; then + tar --format=gnu --sort=name \ + --owner=0 --group=0 --numeric-owner \ + --mtime="@$SOURCE_DATE_EPOCH" \ + -C "$OUT" -cf - "nanokvm_$VERSION" \ + | gzip -n -9 > "$TARBALL" +else + # bsdtar (macOS): no --sort/--mtime, so the archive is not byte-reproducible. + echo "[WARN] GNU tar not found; archive will not be byte-reproducible" + tar --uid 0 --gid 0 --uname '' --gname '' \ + -C "$OUT" -cf - "nanokvm_$VERSION" \ + | gzip -n -9 > "$TARBALL" +fi + +# --- manifest ---------------------------------------------------------------- +# update.go compares base64(raw sha512), not the hex digest. +SHA512="$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)" +SIZE="$(wc -c < "$TARBALL" | tr -d ' ')" + +cat > "$MANIFEST" <