Skip to content

Commit e8bcea9

Browse files
ludfjigludfjig
committed
Golden tests
Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>
1 parent c0f549f commit e8bcea9

22 files changed

Lines changed: 2077 additions & 26 deletions

File tree

‎.github/workflows/RegenSnapshotGoldens.yml‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
2+
3+
# Publish snapshot goldens to
4+
# ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens.
5+
#
6+
# Runs automatically when a merge to main changes GOLDENS_VERSION (the
7+
# version string lives in
8+
# src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs). The check-published
9+
# job reads that version and checks GHCR for its `{version}-complete`
10+
# marker. If the marker is absent, the matrix walks every (hv, cpu,
11+
# config) combination, dumps the canonical snapshot, and uploads it as a
12+
# workflow artifact. A single publish job then downloads every artifact,
13+
# pushes each as a tag named `{version}-{hv}-{cpu}-{profile}`, and
14+
# pushes the marker last. Publishing the whole set from one job means a
15+
# partial run leaves no marker and is republished on the next run.
16+
#
17+
# A version whose marker exists is left untouched, so a merge that does
18+
# not bump the version, or a re-run of the same version, is a no-op.
19+
# Manual dispatch with `force: true` overwrites an existing version and
20+
# exists for recovery only.
21+
#
22+
# See docs/snapshot-versioning.md
23+
24+
name: Regenerate Snapshot Goldens
25+
26+
on:
27+
push:
28+
branches: [main]
29+
paths:
30+
- src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs
31+
workflow_dispatch:
32+
inputs:
33+
version:
34+
description: Goldens version string. Must match GOLDENS_VERSION in source (e.g. "v1.0").
35+
required: true
36+
type: string
37+
force:
38+
description: Overwrite tags even if the version is already published (recovery only).
39+
type: boolean
40+
default: false
41+
42+
env:
43+
CARGO_TERM_COLOR: always
44+
RUST_BACKTRACE: full
45+
GHCR_IMAGE: ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens
46+
47+
permissions:
48+
contents: read
49+
packages: write
50+
51+
concurrency:
52+
group: regen-snapshot-goldens-${{ github.ref }}
53+
cancel-in-progress: false
54+
55+
defaults:
56+
run:
57+
shell: bash
58+
59+
jobs:
60+
check-published:
61+
runs-on: ubuntu-latest
62+
permissions:
63+
contents: read
64+
packages: read
65+
outputs:
66+
version: ${{ steps.decide.outputs.version }}
67+
needs_publish: ${{ steps.decide.outputs.needs_publish }}
68+
steps:
69+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
70+
71+
- name: Install oras
72+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
73+
with:
74+
version: 1.3.2
75+
76+
- name: Decide version and whether to publish
77+
id: decide
78+
env:
79+
EVENT_NAME: ${{ github.event_name }}
80+
INPUT_VERSION: ${{ inputs.version }}
81+
FORCE: ${{ inputs.force }}
82+
GHCR_USER: ${{ github.actor }}
83+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
84+
run: |
85+
set -euo pipefail
86+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
87+
if ! [[ "${SRC}" =~ ^v[0-9]+\.[0-9]+$ ]]; then
88+
echo "::error::GOLDENS_VERSION in source must match ^v[0-9]+\.[0-9]+$ (e.g. v1.0), found '${SRC}'"
89+
exit 1
90+
fi
91+
92+
# On manual dispatch the input must name the version that the
93+
# dispatched ref actually carries. This catches a stale input.
94+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${INPUT_VERSION}" != "${SRC}" ]; then
95+
echo "::error::version input '${INPUT_VERSION}' does not match GOLDENS_VERSION in source '${SRC}'"
96+
exit 1
97+
fi
98+
99+
echo "version=${SRC}" >> "$GITHUB_OUTPUT"
100+
101+
if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${FORCE}" = "true" ]; then
102+
echo "force requested: will publish ${SRC} even if it already exists"
103+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
104+
exit 0
105+
fi
106+
107+
# A version is frozen once its completion marker exists on
108+
# GHCR. The marker is pushed only after every matrix job has
109+
# uploaded its tag, so a partial push (some jobs failed)
110+
# leaves no marker and the next run republishes the missing
111+
# combinations. Publishing only when the marker is absent makes the
112+
# workflow idempotent and never clobbers a complete baseline.
113+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
114+
if oras repo tags "${GHCR_IMAGE}" 2>/dev/null | grep -qxF "${SRC}-complete"; then
115+
echo "${SRC} already published (marker ${SRC}-complete present). Nothing to do."
116+
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
117+
else
118+
echo "${SRC} not fully published yet. Will publish."
119+
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
120+
fi
121+
122+
build-guests:
123+
needs: check-published
124+
if: needs.check-published.outputs.needs_publish == 'true'
125+
strategy:
126+
matrix:
127+
config: [debug, release]
128+
uses: ./.github/workflows/dep_build_guests.yml
129+
with:
130+
config: ${{ matrix.config }}
131+
secrets: inherit
132+
133+
generate-snapshots:
134+
needs: [check-published, build-guests]
135+
if: needs.check-published.outputs.needs_publish == 'true'
136+
strategy:
137+
fail-fast: false
138+
matrix:
139+
hypervisor: [kvm, mshv3, hyperv-ws2025]
140+
cpu: [amd, intel]
141+
config: [debug, release]
142+
runs-on: ${{ fromJson(
143+
format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=regen-goldens-{3}-{4}-{5}-{6}"]',
144+
matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
145+
matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
146+
matrix.cpu,
147+
matrix.config,
148+
github.run_id,
149+
github.run_number,
150+
github.run_attempt)) }}
151+
steps:
152+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
153+
154+
- uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4 # v1.9.0
155+
with:
156+
rust-toolchain: "1.94"
157+
env:
158+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
159+
160+
- name: Fix cargo home permissions
161+
if: runner.os == 'Linux'
162+
run: sudo chown -R $(id -u):$(id -g) /opt/cargo || true
163+
164+
- name: Download Rust guests
165+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
166+
with:
167+
name: rust-guests-${{ matrix.config }}
168+
path: src/tests/rust_guests/bin/${{ matrix.config }}/
169+
170+
- name: Confirm source matches resolved version
171+
env:
172+
RESOLVED_VERSION: ${{ needs.check-published.outputs.version }}
173+
run: |
174+
set -euo pipefail
175+
SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
176+
if [ "${SRC}" != "${RESOLVED_VERSION}" ]; then
177+
echo "::error::source GOLDENS_VERSION '${SRC}' does not match resolved '${RESOLVED_VERSION}'"
178+
exit 1
179+
fi
180+
181+
- name: Generate snapshots
182+
run: just snapshot-goldens-generate ${{ matrix.config }}
183+
184+
- name: Resolve produced tag
185+
id: tag
186+
env:
187+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
188+
run: |
189+
set -euo pipefail
190+
layout=$(echo "target/snapshot-goldens/${GOLDENS_VERSION}"/*/)
191+
echo "tag=$(basename "${layout%/}")" >> "$GITHUB_OUTPUT"
192+
193+
- name: Upload golden layout
194+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
195+
with:
196+
name: golden-${{ steps.tag.outputs.tag }}
197+
path: target/snapshot-goldens/${{ needs.check-published.outputs.version }}/${{ steps.tag.outputs.tag }}/
198+
if-no-files-found: error
199+
retention-days: 1
200+
201+
# Push every matrix job's snapshot from this single job, so the published set is
202+
# whole or absent. `generate-snapshots` runs `fail-fast: false` and uploads each
203+
# snapshot as an artifact, so this job's `needs` succeeds only when
204+
# all matrix jobs did. It downloads every artifact, pushes each tag, then
205+
# pushes the `{version}-complete` marker that `check-published` gates on. A
206+
# push that dies partway leaves no marker, so the next run republishes.
207+
publish:
208+
needs: [check-published, generate-snapshots]
209+
if: needs.check-published.outputs.needs_publish == 'true'
210+
runs-on: ubuntu-latest
211+
steps:
212+
- name: Install oras
213+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
214+
with:
215+
version: 1.3.2
216+
217+
- name: Download all golden layouts
218+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
219+
with:
220+
pattern: golden-*
221+
path: layouts
222+
223+
- name: Push goldens and completion marker
224+
env:
225+
GHCR_USER: ${{ github.actor }}
226+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
227+
GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
228+
run: |
229+
set -euo pipefail
230+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
231+
for layout in layouts/golden-*/; do
232+
tag=$(basename "${layout%/}")
233+
tag=${tag#golden-}
234+
echo "::group::push ${tag}"
235+
oras cp --from-oci-layout "${layout%/}:${tag}" "${GHCR_IMAGE}:${tag}"
236+
echo "::endgroup::"
237+
done
238+
printf '%s' "${GOLDENS_VERSION}" > complete.txt
239+
oras push "${GHCR_IMAGE}:${GOLDENS_VERSION}-complete" \
240+
--artifact-type application/vnd.hyperlight.goldens.complete.v1 \
241+
complete.txt:text/plain

‎.github/workflows/ValidatePullRequest.yml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,17 +79,33 @@ jobs:
7979
with:
8080
docs_only: ${{ needs.docs-pr.outputs.docs-only }}
8181

82+
# Pick the goldens mode. The `regen-goldens` label means regenerate. No label means pull.
83+
goldens-mode:
84+
runs-on: ubuntu-latest
85+
outputs:
86+
regen: ${{ steps.check.outputs.regen }}
87+
steps:
88+
- id: check
89+
if: github.event_name == 'pull_request'
90+
env:
91+
GH_TOKEN: ${{ github.token }}
92+
run: |
93+
gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} \
94+
--json labels -q '.labels[].name' | grep -qx regen-goldens \
95+
&& echo "regen=true" >> "$GITHUB_OUTPUT" || echo "regen=false" >> "$GITHUB_OUTPUT"
96+
8297
# Build and test - needs guest artifacts
8398
build-test:
8499
needs:
85100
- docs-pr
86101
- build-guests
102+
- goldens-mode
87103
# Required because update-guest-locks is skipped on non-dependabot PRs,
88104
# and a skipped dependency transitively skips all downstream jobs.
89105
# See: https://github.com/actions/runner/issues/2205
90106
if: ${{ !cancelled() && !failure() }}
91107
strategy:
92-
fail-fast: true
108+
fail-fast: false
93109
matrix:
94110
hypervisor: ['hyperv-ws2025', mshv3, kvm]
95111
cpu: [amd, intel]
@@ -101,6 +117,7 @@ jobs:
101117
hypervisor: ${{ matrix.hypervisor }}
102118
cpu: ${{ matrix.cpu }}
103119
config: ${{ matrix.config }}
120+
regen_goldens: ${{ needs.goldens-mode.outputs.regen }}
104121

105122
# Run examples - needs guest artifacts, runs in parallel with build-test
106123
run-examples:

‎.github/workflows/dep_build_test.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,19 @@ on:
2222
description: CPU architecture for the build (passed from caller matrix)
2323
required: true
2424
type: string
25+
regen_goldens:
26+
description: Regenerate snapshot goldens from the branch and skip pulling published ones
27+
required: false
28+
type: string
29+
default: "false"
2530

2631
env:
2732
CARGO_TERM_COLOR: always
2833
RUST_BACKTRACE: full
2934

3035
permissions:
3136
contents: read
37+
packages: read
3238

3339
defaults:
3440
run:
@@ -138,3 +144,29 @@ jobs:
138144
env:
139145
RUST_LOG: debug
140146
run: just test-rust-tracing ${{ inputs.config }}
147+
148+
- name: Install oras
149+
if: ${{ inputs.regen_goldens != 'true' }}
150+
uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3 # v2.0.0
151+
with:
152+
version: 1.3.2
153+
154+
# Pull the published goldens for this cell and load them with the
155+
# branch. A missing tag fails the job and flags a format break.
156+
- name: Snapshot goldens (pull and verify)
157+
if: ${{ inputs.regen_goldens != 'true' }}
158+
env:
159+
GHCR_USER: ${{ github.actor }}
160+
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
161+
run: |
162+
echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
163+
just snapshot-goldens-pull ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens ${{ inputs.config }}
164+
just snapshot-goldens-verify ${{ inputs.config }}
165+
166+
# Label path: generate the goldens from the branch and load them
167+
# back. Used when no published tag set exists yet.
168+
- name: Snapshot goldens (regenerate and verify)
169+
if: ${{ inputs.regen_goldens == 'true' }}
170+
run: |
171+
just snapshot-goldens-generate ${{ inputs.config }}
172+
just snapshot-goldens-verify ${{ inputs.config }}

0 commit comments

Comments
 (0)