Skip to content

Commit 0658445

Browse files
ludfjigludfjig
committed
Golden tests
Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>
1 parent e9fe886 commit 0658445

23 files changed

Lines changed: 2566 additions & 26 deletions

‎.github/workflows/RegenSnapshotGoldens.yml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
2+
3+
# Publish snapshot goldens to
4+
# ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens.
5+
#
6+
# Runs automatically when a merge to main changes GOLDENS_VERSION (the
7+
# version string lives in
8+
# src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs). The check-published
9+
# job reads that version and checks GHCR for its `{version}-complete`
10+
# marker. If the marker is absent, the matrix walks every (hv, cpu,
11+
# config) combination, dumps the canonical snapshot, and uploads it as a
12+
# workflow artifact. A single publish job then downloads every artifact,
13+
# pushes each as a tag named `{version}-{hv}-{cpu}-{profile}`, and
14+
# pushes the marker last. Publishing the whole set from one job means a
15+
# partial run leaves no marker and is republished on the next run.
16+
#
17+
# A version whose marker exists is left untouched, so a merge that does
18+
# not bump the version, or a re-run of the same version, is a no-op.
19+
# Manual dispatch with `force: true` overwrites an existing version and
20+
# exists for recovery only.
21+
#
22+
# See docs/snapshot-versioning.md
23+
24+
name: Regenerate Snapshot Goldens
25+
26+
on:
27+
push:
28+
branches: [main]
29+
paths:
30+
- src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs
31+
workflow_dispatch:
32+
inputs:
33+
version:
34+
description: Goldens version string. Must match GOLDENS_VERSION in source (e.g. "v1.0").
35+
required: true
36+
type: string
37+
force:
38+
description: Overwrite tags even if the version is already published (recovery only).
39+
type: boolean
40+
default: false
41+
42+
env:
43+
CARGO_TERM_COLOR: always
44+
RUST_BACKTRACE: full
45+
GHCR_IMAGE: ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens
46+
47+
permissions:
48+
contents: read
49+
packages: write
50+
51+
concurrency:
52+
group: regen-snapshot-goldens-${{ github.ref }}
53+
cancel-in-progress: false
54+
55+
defaults:
56+
run:
57+
shell: bash
58+
59+
jobs:
60+
check-published:
61+
runs-on: ubuntu-latest
62+
permissions:
63+
contents: read
64+
packages: read
65+
outputs:
66+
version: ${{ steps.decide.outputs.version }}
67+
needs_publish: ${{ steps.decide.outputs.needs_publish }}
68+
steps:
69+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
70+
71+
- name: Install oras
72+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
73+
with:
74+
version: 1.3.2
75+
76+
- name: Decide version and whether to publish
77+
id: decide
78+
env:
79+
EVENT_NAME: ${{ github.event_name }}
80+
INPUT_VERSION: ${{ inputs.version }}
81+
FORCE: ${{ inputs.force }}
82+
GHCR_USER: ${{ github.actor }}
83+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
84+
run: |
85+
set -euo pipefail
86+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
87+
if ! [[ "${SRC}" =~ ^v[0-9]+\.[0-9]+$ ]]; then
88+
echo "::error::GOLDENS_VERSION in source must match ^v[0-9]+\.[0-9]+$ (e.g. v1.0), found '${SRC}'"
89+
exit 1
90+
fi
91+
92+
# On manual dispatch the input must name the version that the
93+
# dispatched ref actually carries. This catches a stale input.
94+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${INPUT_VERSION}" != "${SRC}" ]; then
95+
echo "::error::version input '${INPUT_VERSION}' does not match GOLDENS_VERSION in source '${SRC}'"
96+
exit 1
97+
fi
98+
99+
echo "version=${SRC}" >> "$GITHUB_OUTPUT"
100+
101+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${FORCE}" = "true" ]; then
102+
echo "force requested: will publish ${SRC} even if it already exists"
103+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
104+
exit 0
105+
fi
106+
107+
# A version is frozen once its completion marker exists on
108+
# GHCR. The marker is pushed only after every matrix job has
109+
# uploaded its tag, so a partial push (some jobs failed)
110+
# leaves no marker and the next run republishes the missing
111+
# combinations. Publishing only when the marker is absent makes the
112+
# workflow idempotent and never clobbers a complete baseline.
113+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
114+
if oras repo tags "${GHCR_IMAGE}" 2>/dev/null | grep -qxF "${SRC}-complete"; then
115+
echo "${SRC} already published (marker ${SRC}-complete present). Nothing to do."
116+
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
117+
else
118+
echo "${SRC} not fully published yet. Will publish."
119+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
120+
fi
121+
122+
build-guests:
123+
needs: check-published
124+
if: needs.check-published.outputs.needs_publish == 'true'
125+
strategy:
126+
matrix:
127+
config: [debug, release]
128+
uses: ./.github/workflows/dep_build_guests.yml
129+
with:
130+
config: ${{ matrix.config }}
131+
secrets: inherit
132+
133+
generate-snapshots:
134+
needs: [check-published, build-guests]
135+
if: needs.check-published.outputs.needs_publish == 'true'
136+
strategy:
137+
fail-fast: false
138+
matrix:
139+
hypervisor: [kvm, mshv3, hyperv-ws2025]
140+
cpu: [amd, intel]
141+
config: [debug, release]
142+
runs-on: ${{ fromJson(
143+
format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=regen-goldens-{3}-{4}-{5}-{6}"]',
144+
matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
145+
matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
146+
matrix.cpu,
147+
matrix.config,
148+
github.run_id,
149+
github.run_number,
150+
github.run_attempt)) }}
151+
steps:
152+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
153+
154+
- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
155+
with:
156+
rust-toolchain: "1.94"
157+
env:
158+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
159+
160+
- name: Fix cargo home permissions
161+
if: runner.os == 'Linux'
162+
run: sudo chown -R $(id -u):$(id -g) /opt/cargo || true
163+
164+
- name: Download Rust guests
165+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
166+
with:
167+
name: rust-guests-${{ matrix.config }}
168+
path: src/tests/rust_guests/bin/${{ matrix.config }}/
169+
170+
- name: Confirm source matches resolved version
171+
env:
172+
RESOLVED_VERSION: ${{ needs.check-published.outputs.version }}
173+
run: |
174+
set -euo pipefail
175+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
176+
if [ "${SRC}" != "${RESOLVED_VERSION}" ]; then
177+
echo "::error::source GOLDENS_VERSION '${SRC}' does not match resolved '${RESOLVED_VERSION}'"
178+
exit 1
179+
fi
180+
181+
- name: Generate snapshots
182+
run: just snapshot-goldens-generate ${{ matrix.config }} "$RUNNER_TEMP/snapshot-goldens"
183+
184+
- name: Resolve produced tag
185+
id: tag
186+
env:
187+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
188+
run: |
189+
set -euo pipefail
190+
shopt -s nullglob
191+
layouts=("$RUNNER_TEMP/snapshot-goldens/${GOLDENS_VERSION}-"*/)
192+
if [ "${#layouts[@]}" -ne 1 ]; then
193+
echo "::error::expected exactly one golden layout under $RUNNER_TEMP/snapshot-goldens, found ${#layouts[@]}: ${layouts[*]:-none}"
194+
exit 1
195+
fi
196+
layout="${layouts[0]%/}"
197+
echo "tag=$(basename "${layout}")" >> "$GITHUB_OUTPUT"
198+
echo "dir=${layout}" >> "$GITHUB_OUTPUT"
199+
200+
- name: Upload golden layout
201+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
202+
with:
203+
name: golden-${{ steps.tag.outputs.tag }}
204+
path: ${{ steps.tag.outputs.dir }}/
205+
if-no-files-found: error
206+
retention-days: 1
207+
208+
# Push every matrix job's snapshot from this single job, so the published set is
209+
# whole or absent. `generate-snapshots` runs `fail-fast: false` and uploads each
210+
# snapshot as an artifact, so this job's `needs` succeeds only when
211+
# all matrix jobs did. It downloads every artifact, pushes each tag, then
212+
# pushes the `{version}-complete` marker that `check-published` gates on. A
213+
# push that dies partway leaves no marker, so the next run republishes.
214+
publish:
215+
needs: [check-published, generate-snapshots]
216+
if: needs.check-published.outputs.needs_publish == 'true'
217+
runs-on: ubuntu-latest
218+
steps:
219+
- name: Install oras
220+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
221+
with:
222+
version: 1.3.2
223+
224+
- name: Download all golden layouts
225+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
226+
with:
227+
pattern: golden-*
228+
path: layouts
229+
230+
- name: Push goldens and completion marker
231+
env:
232+
GHCR_USER: ${{ github.actor }}
233+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
234+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
235+
run: |
236+
set -euo pipefail
237+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
238+
for layout in layouts/golden-*/; do
239+
tag=$(basename "${layout%/}")
240+
tag=${tag#golden-}
241+
echo "::group::push ${tag}"
242+
oras cp --from-oci-layout "${layout%/}:${tag}" "${GHCR_IMAGE}:${tag}"
243+
echo "::endgroup::"
244+
done
245+
printf '%s' "${GOLDENS_VERSION}" > complete.txt
246+
oras push "${GHCR_IMAGE}:${GOLDENS_VERSION}-complete" \
247+
--artifact-type application/vnd.hyperlight.goldens.complete.v1 \
248+
complete.txt:text/plain

‎.github/workflows/ValidatePullRequest.yml‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,17 +79,37 @@ jobs:
7979
with:
8080
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
8181

82+
# Pick the goldens mode. The `regen-goldens` label means regenerate. No label means pull.
83+
goldens-mode:
84+
runs-on: ubuntu-latest
85+
outputs:
86+
regen: ${{ steps.check.outputs.regen || 'false' }}
87+
steps:
88+
- id: check
89+
if: github.event_name == 'pull_request'
90+
env:
91+
GH_TOKEN: ${{ github.token }}
92+
run: |
93+
labels="$(gh pr view ${{ github.event.pull_request.number }} \
94+
--repo ${{ github.repository }} --json labels -q '.labels[].name')"
95+
if grep -qx regen-goldens <<<"$labels"; then
96+
echo "regen=true" >> "$GITHUB_OUTPUT"
97+
else
98+
echo "regen=false" >> "$GITHUB_OUTPUT"
99+
fi
100+
82101
# Build and test - needs guest artifacts
83102
build-test:
84103
needs:
85104
- docs-pr
86105
- build-guests
106+
- goldens-mode
87107
# Required because update-guest-locks is skipped on non-dependabot PRs,
88108
# and a skipped dependency transitively skips all downstream jobs.
89109
# See: https://github.com/actions/runner/issues/2205
90110
if: ${{ !cancelled() && !failure() }}
91111
strategy:
92-
fail-fast: true
112+
fail-fast: false
93113
matrix:
94114
hypervisor: ['hyperv-ws2025', mshv3, kvm]
95115
cpu: [amd, intel]
@@ -101,6 +121,31 @@ jobs:
101121
hypervisor: ${{ matrix.hypervisor }}
102122
cpu: ${{ matrix.cpu }}
103123
config: ${{ matrix.config }}
124+
regen_goldens: ${{ needs.goldens-mode.outputs.regen }}
125+
126+
# Cross-CPU snapshot check for regenerated baselines. Only the regen
127+
# path needs it: the pull path already runs self+cross per cell.
128+
# Each build-test cell uploads its generated layout, then this job
129+
# loads the peer CPU's layout and verifies the opposite vendor.
130+
snapshot-cross-verify:
131+
needs:
132+
- docs-pr
133+
- build-test
134+
- goldens-mode
135+
if: ${{ !cancelled() && !failure() && needs.goldens-mode.outputs.regen == 'true' }}
136+
strategy:
137+
fail-fast: false
138+
matrix:
139+
hypervisor: ['hyperv-ws2025', mshv3, kvm]
140+
cpu: [amd, intel]
141+
config: [debug, release]
142+
uses: ./.github/workflows/dep_snapshot_cross_verify.yml
143+
secrets: inherit
144+
with:
145+
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
146+
hypervisor: ${{ matrix.hypervisor }}
147+
cpu: ${{ matrix.cpu }}
148+
config: ${{ matrix.config }}
104149

105150
# Run examples - needs guest artifacts, runs in parallel with build-test
106151
run-examples:
@@ -164,7 +209,9 @@ jobs:
164209
- update-guest-locks
165210
- build-guests
166211
- code-checks
212+
- goldens-mode
167213
- build-test
214+
- snapshot-cross-verify
168215
- run-examples
169216
- fuzzing
170217
- spelling

0 commit comments

Comments
 (0)