Skip to content

Commit 35f3d45

Browse files
ludfjigludfjig
authored
Add snapshot golden tests (#1446)
* Add golden snapshot tests Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> * Downgrade oras version Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> * PR feedback Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --------- Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>
1 parent 82f07d5 commit 35f3d45

28 files changed

Lines changed: 2888 additions & 173 deletions

‎.github/workflows/DailyArm64.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@ jobs:
4141
# single-driver steps that are skipped on PRs.
4242
build-test:
4343
needs: build-guests
44+
permissions:
45+
# checkout in the called workflow
46+
contents: read
47+
# pull goldens from GHCR in the called workflow
48+
packages: read
4449
strategy:
4550
fail-fast: false
4651
matrix:

‎.github/workflows/RegenSnapshotGoldens.yml‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
2+
3+
# Publish snapshot goldens to
4+
# ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens.
5+
#
6+
# Runs automatically when a merge to main changes GOLDENS_VERSION (the
7+
# version string lives in
8+
# src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs). The check-published
9+
# job reads that version and checks GHCR for its `{version}-complete`
10+
# marker. If the marker is absent, the matrix walks every (arch, hv,
11+
# cpu, config) combination, dumps the canonical snapshot, and uploads it
12+
# as a workflow artifact. A single publish job then downloads every
13+
# artifact, pushes each as a tag named `{version}-{arch}-{hv}-{cpu}-{profile}`,
14+
# and pushes the marker last. Publishing the whole set from one job means a
15+
# partial run leaves no marker and is republished on the next run.
16+
#
17+
# A version whose marker exists is left untouched, so a merge that does
18+
# not bump the version, or a re-run of the same version, is a no-op.
19+
# Manual dispatch with `force: true` overwrites an existing version and
20+
# exists for recovery only.
21+
#
22+
# See docs/snapshot-versioning.md
23+
24+
name: Regenerate Snapshot Goldens
25+
26+
on:
27+
push:
28+
branches: [main]
29+
paths:
30+
- src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs
31+
workflow_dispatch:
32+
inputs:
33+
version:
34+
description: Goldens version string. Must match GOLDENS_VERSION in source (e.g. "v1.0").
35+
required: true
36+
type: string
37+
force:
38+
description: Overwrite tags even if the version is already published (recovery only).
39+
type: boolean
40+
default: false
41+
42+
env:
43+
CARGO_TERM_COLOR: always
44+
RUST_BACKTRACE: full
45+
GHCR_IMAGE: ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens
46+
47+
permissions:
48+
contents: read
49+
packages: write
50+
51+
concurrency:
52+
group: regen-snapshot-goldens-${{ github.ref }}
53+
cancel-in-progress: false
54+
55+
defaults:
56+
run:
57+
shell: bash
58+
59+
jobs:
60+
check-published:
61+
runs-on: ubuntu-latest
62+
permissions:
63+
contents: read
64+
packages: read
65+
outputs:
66+
version: ${{ steps.decide.outputs.version }}
67+
needs_publish: ${{ steps.decide.outputs.needs_publish }}
68+
steps:
69+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
70+
71+
- name: Install oras
72+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
73+
with:
74+
version: 1.3.1
75+
76+
- name: Decide version and whether to publish
77+
id: decide
78+
env:
79+
EVENT_NAME: ${{ github.event_name }}
80+
INPUT_VERSION: ${{ inputs.version }}
81+
FORCE: ${{ inputs.force }}
82+
GHCR_USER: ${{ github.actor }}
83+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
84+
run: |
85+
set -euo pipefail
86+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
87+
if ! [[ "${SRC}" =~ ^v[0-9]+\.[0-9]+$ ]]; then
88+
echo "::error::GOLDENS_VERSION in source must match ^v[0-9]+\.[0-9]+$ (e.g. v1.0), found '${SRC}'"
89+
exit 1
90+
fi
91+
92+
# On manual dispatch the input must name the version that the
93+
# dispatched ref actually carries. This catches a stale input.
94+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${INPUT_VERSION}" != "${SRC}" ]; then
95+
echo "::error::version input '${INPUT_VERSION}' does not match GOLDENS_VERSION in source '${SRC}'"
96+
exit 1
97+
fi
98+
99+
echo "version=${SRC}" >> "$GITHUB_OUTPUT"
100+
101+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${FORCE}" = "true" ]; then
102+
echo "force requested: will publish ${SRC} even if it already exists"
103+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
104+
exit 0
105+
fi
106+
107+
# A version is frozen once its completion marker exists on
108+
# GHCR. The marker is pushed only after every matrix job has
109+
# uploaded its tag, so a partial push (some jobs failed)
110+
# leaves no marker and the next run republishes the missing
111+
# combinations. Publishing only when the marker is absent makes the
112+
# workflow idempotent and never clobbers a complete baseline.
113+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
114+
if oras repo tags "${GHCR_IMAGE}" 2>/dev/null | grep -qxF "${SRC}-complete"; then
115+
echo "${SRC} already published (marker ${SRC}-complete present). Nothing to do."
116+
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
117+
else
118+
echo "${SRC} not fully published yet. Will publish."
119+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
120+
fi
121+
122+
build-guests:
123+
needs: check-published
124+
if: needs.check-published.outputs.needs_publish == 'true'
125+
strategy:
126+
matrix:
127+
arch: [X64, arm64]
128+
config: [debug, release]
129+
uses: ./.github/workflows/dep_build_guests.yml
130+
with:
131+
arch: ${{ matrix.arch }}
132+
config: ${{ matrix.config }}
133+
secrets: inherit
134+
135+
generate-snapshots:
136+
needs: [check-published, build-guests]
137+
if: needs.check-published.outputs.needs_publish == 'true'
138+
strategy:
139+
fail-fast: false
140+
matrix:
141+
hypervisor: [kvm, mshv3, hyperv-ws2025]
142+
cpu: [amd, intel, apple]
143+
arch: [X64, arm64]
144+
config: [debug, release]
145+
exclude:
146+
# aarch64 covers Apple under KVM only.
147+
- cpu: apple
148+
hypervisor: mshv3
149+
- cpu: apple
150+
hypervisor: hyperv-ws2025
151+
- cpu: apple
152+
arch: X64
153+
- cpu: amd
154+
arch: arm64
155+
- cpu: intel
156+
arch: arm64
157+
runs-on: ${{ fromJson(
158+
format('["self-hosted", "{0}", "{1}"{2}]',
159+
matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
160+
matrix.arch,
161+
matrix.arch == 'X64'
162+
&& format(', "1ES.Pool=hld-{0}-{1}", "JobId=regen-goldens-{2}-{3}-{4}-{5}"',
163+
matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
164+
matrix.cpu,
165+
matrix.config,
166+
github.run_id,
167+
github.run_number,
168+
github.run_attempt)
169+
|| ', "kvm", "ubuntu-24.04"')) }}
170+
steps:
171+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
172+
173+
- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
174+
with:
175+
rust-toolchain: "1.94"
176+
env:
177+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
178+
179+
- name: Fix cargo home permissions
180+
if: runner.os == 'Linux'
181+
run: sudo chown -R $(id -u):$(id -g) /opt/cargo || true
182+
183+
- name: Download Rust guests
184+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
185+
with:
186+
name: rust-guests-${{ matrix.arch }}-${{ matrix.config }}
187+
path: src/tests/rust_guests/bin/${{ matrix.config }}/
188+
189+
- name: Confirm source matches resolved version
190+
env:
191+
RESOLVED_VERSION: ${{ needs.check-published.outputs.version }}
192+
run: |
193+
set -euo pipefail
194+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
195+
if [ "${SRC}" != "${RESOLVED_VERSION}" ]; then
196+
echo "::error::source GOLDENS_VERSION '${SRC}' does not match resolved '${RESOLVED_VERSION}'"
197+
exit 1
198+
fi
199+
200+
- name: Generate snapshots
201+
run: just snapshot-goldens-generate ${{ matrix.config }} "$RUNNER_TEMP/snapshot-goldens"
202+
203+
- name: Resolve produced tag
204+
id: tag
205+
env:
206+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
207+
run: |
208+
set -euo pipefail
209+
shopt -s nullglob
210+
layouts=("$RUNNER_TEMP/snapshot-goldens/${GOLDENS_VERSION}-"*/)
211+
if [ "${#layouts[@]}" -ne 1 ]; then
212+
echo "::error::expected exactly one golden layout under $RUNNER_TEMP/snapshot-goldens, found ${#layouts[@]}: ${layouts[*]:-none}"
213+
exit 1
214+
fi
215+
layout="${layouts[0]%/}"
216+
echo "tag=$(basename "${layout}")" >> "$GITHUB_OUTPUT"
217+
echo "dir=${layout}" >> "$GITHUB_OUTPUT"
218+
219+
- name: Upload golden layout
220+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
221+
with:
222+
name: golden-${{ steps.tag.outputs.tag }}
223+
path: ${{ steps.tag.outputs.dir }}/
224+
if-no-files-found: error
225+
retention-days: 1
226+
227+
# Push every matrix job's snapshot from this single job, so the published set is
228+
# whole or absent. `generate-snapshots` runs `fail-fast: false` and uploads each
229+
# snapshot as an artifact, so this job's `needs` succeeds only when
230+
# all matrix jobs did. It downloads every artifact, pushes each tag, then
231+
# pushes the `{version}-complete` marker that `check-published` gates on. A
232+
# push that dies partway leaves no marker, so the next run republishes.
233+
publish:
234+
needs: [check-published, generate-snapshots]
235+
if: needs.check-published.outputs.needs_publish == 'true'
236+
runs-on: ubuntu-latest
237+
steps:
238+
- name: Install oras
239+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
240+
with:
241+
version: 1.3.1
242+
243+
- name: Download all golden layouts
244+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
245+
with:
246+
pattern: golden-*
247+
path: layouts
248+
249+
- name: Push goldens and completion marker
250+
env:
251+
GHCR_USER: ${{ github.actor }}
252+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
253+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
254+
run: |
255+
set -euo pipefail
256+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
257+
for layout in layouts/golden-*/; do
258+
tag=$(basename "${layout%/}")
259+
tag=${tag#golden-}
260+
echo "::group::push ${tag}"
261+
oras cp --from-oci-layout "${layout%/}:${tag}" "${GHCR_IMAGE}:${tag}"
262+
echo "::endgroup::"
263+
done
264+
printf '%s' "${GOLDENS_VERSION}" > complete.txt
265+
oras push "${GHCR_IMAGE}:${GOLDENS_VERSION}-complete" \
266+
--artifact-type application/vnd.hyperlight.goldens.complete.v1 \
267+
complete.txt:text/plain

‎.github/workflows/ValidatePullRequest.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,15 +81,74 @@ jobs:
8181
with:
8282
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
8383

84+
# Pick the goldens mode. The `regen-goldens` label means regenerate. No label means pull.
85+
# Also validate the label against the goldens state so a stale or missing label fails fast.
86+
check-golden-label:
87+
runs-on: ubuntu-latest
88+
permissions:
89+
contents: read
90+
pull-requests: read
91+
packages: read
92+
outputs:
93+
should_regen_goldens: ${{ steps.check.outputs.should_regen_goldens || 'false' }}
94+
steps:
95+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
96+
97+
- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
98+
with:
99+
rust-toolchain: "1.94"
100+
env:
101+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
102+
103+
- name: Install oras
104+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
105+
with:
106+
version: 1.3.1
107+
108+
- name: Log in to GHCR
109+
env:
110+
GHCR_USER: ${{ github.actor }}
111+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
112+
run: |
113+
# login technically not needed for reading public packages, but might avoid potential rate limiting
114+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
115+
116+
- id: check
117+
if: github.event_name == 'pull_request'
118+
env:
119+
GH_TOKEN: ${{ github.token }}
120+
run: |
121+
labels="$(gh pr view ${{ github.event.pull_request.number }} \
122+
--repo ${{ github.repository }} --json labels -q '.labels[].name')"
123+
if grep -qx regen-goldens <<<"$labels"; then
124+
echo "should_regen_goldens=true" >> "$GITHUB_OUTPUT"
125+
else
126+
echo "should_regen_goldens=false" >> "$GITHUB_OUTPUT"
127+
fi
128+
129+
- name: Validate regen-goldens label
130+
if: github.event_name == 'pull_request'
131+
env:
132+
BASE_REF: ${{ github.event.pull_request.base.ref }}
133+
run: |
134+
git fetch --no-tags --depth=1 origin "${BASE_REF}"
135+
just snapshot-goldens-check-label "${{ steps.check.outputs.should_regen_goldens }}" FETCH_HEAD
136+
84137
# Build and test - needs guest artifacts
85138
build-test:
86139
needs:
87140
- docs-pr
88141
- build-guests
142+
- check-golden-label
89143
# Required because update-guest-locks is skipped on non-dependabot PRs,
90144
# and a skipped dependency transitively skips all downstream jobs.
91145
# See: https://github.com/actions/runner/issues/2205
92146
if: ${{ !cancelled() && !failure() }}
147+
permissions:
148+
# checkout in the called workflow
149+
contents: read
150+
# pull goldens from GHCR in the called workflow
151+
packages: read
93152
strategy:
94153
fail-fast: true
95154
matrix:
@@ -116,6 +175,7 @@ jobs:
116175
cpu_vendor: ${{ matrix.cpu_vendor }}
117176
arch: ${{ matrix.arch }}
118177
config: ${{ matrix.config }}
178+
should_regen_goldens: ${{ needs.check-golden-label.outputs.should_regen_goldens }}
119179

120180
# Run examples - needs guest artifacts, runs in parallel with build-test
121181
run-examples:
@@ -191,6 +251,7 @@ jobs:
191251
- update-guest-locks
192252
- build-guests
193253
- code-checks
254+
- check-golden-label
194255
- build-test
195256
- run-examples
196257
- fuzzing

0 commit comments

Comments
 (0)