forked from kunobi-ninja/kache
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathJustfile
More file actions
364 lines (328 loc) · 15.8 KB
/
Copy pathJustfile
File metadata and controls
364 lines (328 loc) · 15.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
# Don't use kache to build kache (bootstrapping problem).
export RUSTC_WRAPPER := ""
# On Windows, `just` runs each recipe line via `sh`, which Git for Windows
# provides but does not put on PATH — so recipes (e.g. `just bench`) fail with
# "could not find the shell". Point it at Git bash at its default install path.
# Unix is unaffected: `windows-shell` only applies on Windows. If Git is
# installed elsewhere, override this path (or put Git's `usr\bin` on PATH).
set windows-shell := ["C:/Program Files/Git/usr/bin/sh.exe", "-cu"]
default:
@just --list
# Run all local quality checks.
[group('dev')]
check: fmt-check lint test
# Mirror the repo CI verification flow.
[group('dev')]
ci: fmt-check lint image-service-print helm-lint coverage
# Auto-fix formatting and clippy warnings.
[group('dev')]
fix:
cargo fmt --all
cargo clippy --fix --allow-dirty --allow-staged --workspace --all-targets -- -D warnings
# Install kache to ~/.cargo/bin and register the daemon service.
[group('dev')]
install:
cargo install --path .
kache daemon install
# Build the release binary.
[group('build')]
build:
cargo build --release
# Build the remote service binary.
[group('build')]
build-service:
cargo build --release -p kache-service
# Build the service container image locally.
[group('docker')]
image-service:
docker buildx bake -f docker-bake.hcl service
# Print the resolved service image bake plan.
[group('docker')]
image-service-print:
docker buildx bake -f docker-bake.hcl --print service
# Build and push the release service image.
[group('docker')]
image-service-release:
docker buildx bake -f docker-bake.hcl release
# Run the full workspace test suite.
[group('dev')]
test:
cargo test --workspace
# Mutation-test the complete hermetic planner crate. Install the pinned local
# tool with: cargo install --locked cargo-mutants --version 27.1.0
[group('dev')]
mutants-core *ARGS:
cargo mutants --package kache-core --all-features --baseline run --timeout 300 --build-timeout 600 --output tmp/mutants/core {{ARGS}}
# Mutation-test changed Rust lines outside kache-core (which mutants-core
# already covers completely). DIFF must describe the current working tree.
[group('dev')]
mutants-diff DIFF *ARGS:
cargo mutants --workspace --all-features --in-diff "{{DIFF}}" --exclude 'crates/kache-core/**' --baseline run --timeout 300 --build-timeout 600 --output tmp/mutants/diff {{ARGS}}
# Audit dependencies with cargo-deny (advisories + licenses + bans +
# sources; config and documented exceptions in `deny.toml`). Runs once
# per workspace MEMBER on purpose: cargo-deny graphs from the current
# package, so a root-only run would cover only the `kache` bin (+
# kache-core) and silently skip everything reachable solely through
# `kache-service` — e.g. the rsa RUSTSEC-2023-0071 advisory and
# surrealdb's BUSL-1.1 license.
[group('dev')]
audit:
#!/usr/bin/env bash
set -euo pipefail
for member in . crates/kache-core crates/kache-service crates/kache-e2e; do
echo "── cargo deny check ($member) ──"
( cd "{{justfile_directory()}}/$member" \
&& cargo deny check --config "{{justfile_directory()}}/deny.toml" )
done
# Pin every GitHub Actions `uses:` to a full 40-char commit SHA (supply-chain
# hardening — a mutable `@vN` tag can be repointed at malicious code). Runs
# pinact, which rewrites `uses: owner/repo@vN` to `@<sha> # vN` in place across
# .github/. Run it after adding or bumping any action, then commit the result.
# Everything is SHA-pinned, including `dtolnay/rust-toolchain` (hand-managed —
# it has no semver tags, see `.pinact.yaml`). Needs a GitHub token to resolve
# tags → SHAs; falls back to `gh auth token` if GITHUB_TOKEN is unset.
[group('dev')]
pin-actions:
GITHUB_TOKEN="${GITHUB_TOKEN:-$(gh auth token)}" pinact run
# Verify every action is already SHA-pinned (and the `# vN` comment matches the
# SHA) WITHOUT editing files — exits non-zero if anything is unpinned or drifted.
# Use in CI or before a release to guard the pinning.
[group('dev')]
pin-actions-check:
GITHUB_TOKEN="${GITHUB_TOKEN:-$(gh auth token)}" pinact run --check --verify
# Run the end-to-end harness against every e2e scenario in scenarios/.
# Builds kache + harness in release mode, drives each fixture through
# cold → warm → noop, asserts per-fixture contracts against
# `kache report --format json`. `suite:e2e` picks fixture scenarios;
# `tier:gate` selects the fast checked-in fixture scenarios.
# Writes tmp/e2e/results.json.
[group('dev')]
e2e:
cargo build --release -p kache
cargo build --release -p kache-e2e
./target/release/kache-scenario \
--kache ./target/release/kache \
--scenarios ./scenarios \
--select suite:e2e \
--select tier:gate \
--out tmp/e2e/results.json
# Run the same gate e2e harness inside a Linux container — CI-equivalent
# results from a non-Linux host (e.g. macOS), where the host clang behaves
# differently from CI's Linux clang. Useful to validate quick compile-cache
# "cases" (e.g. the issue-#411 Firefox-flag fixture) on both OSes without
# the slow full benchmarks.
#
# Runs against a copy of the tree on a CONTAINER-NATIVE named volume, not a
# bind mount: kache restores hits via hardlink and the harness relocates
# source trees, both of which break on Docker Desktop's macOS bind-mount fs
# (cross-device hardlinks; virtiofs copy perms). The volume also keeps the
# multi-GB cargo target OFF the host — only results.json is copied back to
# tmp/e2e/. The work volume persists, so rebuilds are incremental; reset it
# with `docker volume rm kache-e2e-work`. Extra ARGS pass through to
# kache-scenario (e.g. `just e2e-docker --select name:e2e-cc-cl-xclang-deps`).
[group('dev')]
e2e-docker *ARGS:
docker build -f docker/e2e.Dockerfile -t kache-e2e:local .
mkdir -p {{justfile_directory()}}/tmp/e2e
docker run --rm \
-v {{justfile_directory()}}:/src:ro \
-v kache-e2e-work:/work \
-v {{justfile_directory()}}/tmp/e2e:/out \
-v kache-e2e-cargo-registry:/usr/local/cargo/registry \
kache-e2e:local \
bash -euo pipefail -c '\
rsync -a --delete --exclude=.git --exclude=/target --exclude=/tmp /src/ /work/ && \
cd /work && \
cargo build --release -p kache && \
cargo build --release -p kache-e2e && \
./target/release/kache-scenario \
--kache ./target/release/kache \
--scenarios ./scenarios \
--select suite:e2e \
--select tier:gate \
--out /out/results.json {{ARGS}}'
# Verify the `KACHE_FALLBACK` wrapper delegates to — and is cached by —
# a real sccache. Builds an excluded rlib through kache twice and
# asserts the rebuild is an sccache cache hit. Skips if sccache is not
# installed.
[group('dev')]
sccache-check:
cargo build --release -p kache
./scripts/sccache-fallback-check.sh ./target/release/kache
# Builds a benchmark scenario (see scenarios/) twice against one shared kache
# cache — cold (empty cache) then warm (cache populated by cold) — and
# reports cold/warm wall-clock, speedup, hit rate, and a correctness verdict.
# Tens of minutes to hours, tens of GB of disk; NOT run in CI. Flags pass through
# (`just bench firefox --skip-clone`). See scenarios/README.md.
# Omit PROFILE to list the matching benchmark profiles.
# PROFILE is a name filter — e.g. `firefox` matches `bench-firefox`.
# Scratch lives under ./tmp/bench/<scenario> (per-scenario; override with --work-dir).
[group('bench')]
bench PROFILE="" *ARGS:
@if [ -z "{{PROFILE}}" ]; then \
cargo build -q --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --list --select suite:bench --select backend:kache; \
else \
cargo build --release -p kache; \
cargo build --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --kache ./target/release/kache --select suite:bench --select backend:kache --profile "{{PROFILE}}" {{ARGS}}; \
fi
# Retry the warm phase only — restores the cold-state cache snapshot
# saved by the previous full run and re-measures warm against it. Skips
# the cold rebuild. Requires a prior successful run for the same scenario.
[group('bench')]
bench-retry PROFILE="" *ARGS:
@if [ -z "{{PROFILE}}" ]; then \
cargo build -q --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --list --select suite:bench --select backend:kache; \
else \
cargo build --release -p kache; \
cargo build --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --kache ./target/release/kache --select suite:bench --select backend:kache --profile "{{PROFILE}}" --retry {{ARGS}}; \
fi
# Full bench with `kache::cache_key=trace` enabled in both phases. After
# warm, the bench diffs the two phases' key-input traces per crate and
# writes `key-diff.{json,md}` listing what diverged across clones — the
# actionable signal when key stability drops below 100%. Trace logs grow
# by ~50–100 MB per phase.
[group('bench')]
bench-trace PROFILE="" *ARGS:
@if [ -z "{{PROFILE}}" ]; then \
cargo build -q --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --list --select suite:bench --select backend:kache; \
else \
cargo build --release -p kache; \
cargo build --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --kache ./target/release/kache --select suite:bench --select backend:kache --profile "{{PROFILE}}" --trace-keys {{ARGS}}; \
fi
# Same cold/warm clone benchmark, but with sccache as the compiler cache.
# Omit PROFILE to list sccache-backed profiles.
# Use `just bench-sccache firefox` for the Firefox comparison.
[group('bench')]
bench-sccache PROFILE="" *ARGS:
@if [ -z "{{PROFILE}}" ]; then \
cargo build -q --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --list --cache-backend sccache --select suite:bench --select backend:sccache; \
else \
cargo build --release -p kache-e2e --bin kache-scenario; \
./target/release/kache-scenario --cache-backend sccache --select suite:bench --select backend:sccache --profile "{{PROFILE}}" {{ARGS}}; \
fi
# Run clippy with deny warnings.
[group('dev')]
lint:
cargo clippy --workspace --all-targets -- -D warnings
# Format the workspace.
[group('dev')]
fmt:
cargo fmt --all
# Check formatting without changing files.
[group('dev')]
fmt-check:
cargo fmt --all -- --check
# Format flake.nix and nix/ with nixfmt. Deliberately not part of `check`
# or `ci`: nix isn't in mise.toml, so requiring it would break contributors
# who only have the mise toolchain. CI gates it in the `nix-package` job.
[group('dev')]
fmt-nix:
nix fmt
# Check Nix formatting without changing files. Deliberately NOT
# `nix fmt -- --fail-on-change`: treefmt formats in place and only then
# reports a nonzero exit, so it rewrites the working tree. Building the
# flake check runs nixfmt --check in the sandbox and touches nothing.
[group('dev')]
fmt-nix-check:
nix build --no-link ".#checks.$(nix eval --impure --raw --expr builtins.currentSystem).formatting"
# Lint the deployable Helm chart.
[group('deploy')]
helm-lint:
helm lint charts/kache-service
# Run cargo-llvm-cov and emit JSON + HTML reports under tmp/llvm-cov/.
# JSON drives the CI threshold check; HTML is uploaded as a CI artifact
# (and opened locally by `coverage-open`). `--no-report` collects
# coverage once; the two `report` invocations then emit the formats
# from that single test run.
[group('coverage')]
coverage:
cargo llvm-cov --all-features --workspace --no-report
cargo llvm-cov report --html --output-dir tmp/llvm-cov
cargo llvm-cov report --json --output-path tmp/llvm-cov/coverage.json
# Run cargo-llvm-cov and open the HTML report locally.
[group('coverage')]
coverage-open:
cargo llvm-cov --all-features --workspace --html --output-dir tmp/llvm-cov
open tmp/llvm-cov/html/index.html || \
xdg-open tmp/llvm-cov/html/index.html || true
# Show kache CI cache metrics from GitHub Actions.
[group('ops')]
monitor *ARGS:
./scripts/ci-monitor.sh {{ARGS}}
# Bump the workspace version everywhere in one shot — all 4 member manifests,
# the kache-core dep-pin, and Cargo.lock — via `cargo set-version` (NOT a broad
# `cargo update`, so the pinned kunobi-* git deps and the hand-maintained nix
# `outputHashes` stay valid; the flake derives `version` from Cargo.toml, so no
# hash change is needed). The VERSION is the full version, prerelease included:
# `just bump 0.5.0` for a final, `just bump 0.5.0-rc.4` for a candidate — both
# publish to crates.io (a prerelease is only served on an explicit --version).
# Then commit, open a PR, and merge; cut the tag from the merged commit with
# `just release` (never re-typed). Auto-installs `cargo-edit` on first use — it
# is not pinned in mise.toml because that compiles it in every CI run.
#
# Use a DOTTED-numeric prerelease (`-rc.4`, not `-rc4`): crates.io is permanent
# and semver orders the no-dot form lexically (`rc.2` would sort after `rc.10`).
# And don't bump back into an `-rc` for a version whose final already shipped
# (e.g. a `0.5.0-rc.5` after `0.5.0` is published) — the version gate checks
# tag==manifest, not crates.io monotonicity, so that would publish a permanent
# "prerelease of an already-released version".
# Usage: `just bump 0.5.0` / `just bump 0.5.0-rc.4`
[group('release')]
bump VERSION:
#!/usr/bin/env bash
set -euo pipefail
# Reject the no-dot prerelease form (-rc4): it sorts lexically on crates.io,
# which is permanent. Require -rc.4 / -alpha.2 / -beta.1 (dotted numeric).
case "{{VERSION}}" in
*-rc[0-9]*|*-alpha[0-9]*|*-beta[0-9]*)
echo "use a dotted prerelease (e.g. 0.5.0-rc.4), not the no-dot form — semver sorts no-dot lexically on crates.io" >&2
exit 1 ;;
esac
if ! command -v cargo-set-version >/dev/null 2>&1; then
echo "cargo-edit (cargo set-version) not found — installing it…"
if command -v cargo-binstall >/dev/null 2>&1; then
cargo binstall -y cargo-edit # prebuilt, fast
else
cargo install cargo-edit # source build (one-time)
fi
fi
cargo set-version --workspace {{VERSION}}
# NO --locked: set-version rewrites the lock's version entries, so --locked
# would error "lock file needs updating". Plain check settles the lock for the
# local crates only (it does not advance kunobi-* / registry deps).
cargo check --workspace
./scripts/check-version-consistency.sh
echo "Bumped to {{VERSION}}. Commit + open a PR; after merge, cut the tag with 'just release'."
# Refuses unless the tree is releasable (clean, on `main`, in sync with
# origin/main, so a tag is never cut from a dirty / off-main / un-pulled
# commit), runs the consistency gate, then pushes the tag → gated pipeline →
# crates.io. The version (final OR prerelease, e.g. 0.5.0-rc.4) comes from the
# merged manifest — never re-typed.
# Cut the release tag for the merged manifest version. Usage: `just release`
[group('release')]
release:
#!/usr/bin/env bash
set -euo pipefail
[ -z "$(git status --porcelain)" ] || { echo "working tree is dirty — commit or stash first" >&2; exit 1; }
branch="$(git rev-parse --abbrev-ref HEAD)"
[ "$branch" = "main" ] || { echo "not on main (on '$branch') — releases are cut from main" >&2; exit 1; }
git fetch --quiet origin main
[ "$(git rev-parse HEAD)" = "$(git rev-parse origin/main)" ] || { echo "local main is not in sync with origin/main — pull/push first" >&2; exit 1; }
version="$(cargo metadata --no-deps --format-version 1 \
| python3 -c 'import json,sys; print(next(p["version"] for p in json.load(sys.stdin)["packages"] if p["name"]=="kache"))')"
tag="v${version}"
./scripts/check-version-consistency.sh "$tag"
git rev-parse -q --verify "refs/tags/$tag" >/dev/null && { echo "tag $tag already exists" >&2; exit 1; } || true
git tag -a "$tag" -m "$tag"
git push origin "$tag"
echo "pushed $tag — the gated release pipeline will run; watch CI."
# Remove build artifacts.
clean:
cargo clean