|
| 1 | +name: minimum-versions-publish |
| 2 | + |
| 3 | +# Publish the audit minimum version table |
| 4 | +# (cmax/scripts/1-audit/minimum-versions.json) to the public website. |
| 5 | +# A "minimum" is the lowest version of a component that has no known applicable |
| 6 | +# vulnerability. |
| 7 | +# |
| 8 | +# The website repository is SemiAnalysisAI/clustermax-web. This job writes the |
| 9 | +# table to `data/minimum-versions.json` there. The site serves those bytes at |
| 10 | +# https://www.clustermax.ai/minimum-versions.json, and its /minimum-versions |
| 11 | +# page refetches that address on load. |
| 12 | +# |
| 13 | +# Why this job exists: |
| 14 | +# - `cmax audit security` fetches that public address on startup. A pip |
| 15 | +# installation carries a read-only table inside the installed package, so |
| 16 | +# the public copy is the one way it gets current minimums. |
| 17 | +# - The table in this repository stays the single source of truth. This job |
| 18 | +# copies it. It never edits a minimum and it never runs the generator. |
| 19 | +# - The publish happens after a merge, never before review. The public copy |
| 20 | +# is therefore always equal to a reviewed state of master. |
| 21 | +# |
| 22 | +# What the published copy adds, and why: |
| 23 | +# - `refreshed`: when the daily `minimum-versions-refresh` job last completed |
| 24 | +# successfully, so the website can say when the upstream feeds were last |
| 25 | +# checked. The table's own `generated` stamp moves only when a minimum |
| 26 | +# changes or the table nears its `maxAgeDays` limit, so it cannot say this. |
| 27 | +# - `updated`: when a minimum value last changed in the published copy. The |
| 28 | +# job compares the minimum values (not the metadata) against the copy the |
| 29 | +# website already carries and moves this stamp only on a real change. |
| 30 | +# Both are top-level metadata. Every component block is copied byte for byte, |
| 31 | +# and the command line ignores keys it does not read. |
| 32 | +# |
| 33 | +# What this job does not do: |
| 34 | +# - It does not open a pull request on the website repository. The table is |
| 35 | +# generated data with its own review gate in this repository, and a daily |
| 36 | +# manual merge on the website would hold the public copy behind. |
| 37 | +# - It does not touch any other file in the website repository. |
| 38 | + |
| 39 | +on: |
| 40 | + push: |
| 41 | + branches: |
| 42 | + - master |
| 43 | + paths: |
| 44 | + # Only a change to the table can change the published copy. |
| 45 | + - cmax/scripts/1-audit/minimum-versions.json |
| 46 | + # A change to this job must be able to publish with the new logic. |
| 47 | + - .github/workflows/minimum-versions-publish.yml |
| 48 | + schedule: |
| 49 | + # 08:43 UTC each day, after the 07:17 UTC refresh. |
| 50 | + # - The push event above publishes each merged table on the same day. |
| 51 | + # - The daily run carries the `refreshed` stamp forward after a refresh |
| 52 | + # that moved no minimum, and repairs a missed publish: a failed website |
| 53 | + # deploy, a revoked token, or a job cancelled mid-push each leave the |
| 54 | + # public copy behind master. |
| 55 | + - cron: '43 8 * * *' |
| 56 | + workflow_dispatch: |
| 57 | + |
| 58 | +# Least privilege. This job reads this repository and its workflow runs, and |
| 59 | +# writes to the website repository with a separate token. |
| 60 | +permissions: |
| 61 | + contents: read |
| 62 | + actions: read |
| 63 | + |
| 64 | +# One publish at a time. A scheduled run must not race a push run, because both |
| 65 | +# push the same branch of the website repository. |
| 66 | +concurrency: |
| 67 | + group: minimum-versions-publish |
| 68 | + cancel-in-progress: false |
| 69 | + |
| 70 | +env: |
| 71 | + MINIMUMS_PATH: cmax/scripts/1-audit/minimum-versions.json |
| 72 | + REFRESH_WORKFLOW: minimum-versions-refresh.yml |
| 73 | + WEB_REPO: SemiAnalysisAI/clustermax-web |
| 74 | + WEB_PATH: data/minimum-versions.json |
| 75 | + # The route file that serves the public path. The publish stops when it is |
| 76 | + # absent, because the address would answer 404 and every fetch would fail. |
| 77 | + WEB_ROUTE: app/minimum-versions.json/route.ts |
| 78 | + PUBLIC_URL: https://www.clustermax.ai/minimum-versions.json |
| 79 | + |
| 80 | +jobs: |
| 81 | + publish: |
| 82 | + runs-on: ubuntu-latest |
| 83 | + # The work is one shallow clone, one file write, and one push. 10 minutes |
| 84 | + # covers the deployment check and still kills a hung connection quickly. |
| 85 | + timeout-minutes: 10 |
| 86 | + steps: |
| 87 | + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 |
| 88 | + with: |
| 89 | + # This job never pushes to this repository. Keep no token in Git |
| 90 | + # configuration. |
| 91 | + persist-credentials: false |
| 92 | + |
| 93 | + - name: Check that the publish token is present |
| 94 | + env: |
| 95 | + WEB_TOKEN: ${{ secrets.CLUSTERMAX_WEB_TOKEN }} |
| 96 | + # Fail loudly. A silent skip lets the public table age out, and the |
| 97 | + # command line then reads minimums that no longer match this repository. |
| 98 | + run: | |
| 99 | + set -uo pipefail |
| 100 | + if [ -z "$WEB_TOKEN" ]; then |
| 101 | + echo "::error::the repository secret CLUSTERMAX_WEB_TOKEN is empty. Add a token with write access to $WEB_REPO, then run this job again." |
| 102 | + exit 1 |
| 103 | + fi |
| 104 | + echo "the publish token is present" |
| 105 | +
|
| 106 | + - name: Check that the table is complete |
| 107 | + # The public copy feeds an audit that grades a provider. An empty or |
| 108 | + # partial table must never reach the website. This repeats the gate in |
| 109 | + # `minimum-versions-refresh`, because a merge can also arrive by hand. |
| 110 | + run: | |
| 111 | + python3 - <<'PY' |
| 112 | + import json |
| 113 | + import os |
| 114 | + import sys |
| 115 | + from datetime import datetime |
| 116 | +
|
| 117 | + path = os.environ["MINIMUMS_PATH"] |
| 118 | + with open(path, encoding="utf-8") as handle: |
| 119 | + data = json.load(handle) |
| 120 | +
|
| 121 | + if data.get("schemaVersion") != 1: |
| 122 | + sys.exit(f"::error::{path} declares schema version {data.get('schemaVersion')}. The reader understands 1.") |
| 123 | +
|
| 124 | + components = data.get("components") or {} |
| 125 | + if not components: |
| 126 | + sys.exit(f"::error::{path} has no components. Refuse to publish it.") |
| 127 | +
|
| 128 | + stamp = str(data.get("generated") or "") |
| 129 | + try: |
| 130 | + datetime.fromisoformat(stamp.replace("Z", "+00:00")) |
| 131 | + except ValueError: |
| 132 | + sys.exit(f"::error::{path} carries no usable generated timestamp: {stamp!r}") |
| 133 | +
|
| 134 | + # Each kind keeps its minimums under its own key. Name the metadata keys |
| 135 | + # instead, so a new kind with a new value key needs no edit here. This |
| 136 | + # list matches the gate in `minimum-versions-refresh`. |
| 137 | + metadata_keys = { |
| 138 | + "advisory", "advisories", "advisoryRanges", "branchSources", |
| 139 | + "cves", "floorSources", "fixAvailability", "floorAvailability", |
| 140 | + "kind", "policy", "programCves", "programSources", "release", |
| 141 | + "source", "sources", |
| 142 | + } |
| 143 | + empty = sorted( |
| 144 | + name for name, component in components.items() |
| 145 | + if not any( |
| 146 | + value for key, value in component.items() |
| 147 | + if key not in metadata_keys |
| 148 | + ) |
| 149 | + ) |
| 150 | + if empty: |
| 151 | + sys.exit(f"::error::these components carry no minimum value: {', '.join(empty)}") |
| 152 | +
|
| 153 | + print(f"{len(components)} components carry minimum values, generated {stamp}") |
| 154 | + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as out: |
| 155 | + out.write(f"MINIMUMS_GENERATED={stamp}\n") |
| 156 | + PY |
| 157 | +
|
| 158 | + - name: Find when the upstream feeds were last checked |
| 159 | + id: refreshed |
| 160 | + env: |
| 161 | + GH_TOKEN: ${{ github.token }} |
| 162 | + # The latest successful `minimum-versions-refresh` run is the last time |
| 163 | + # the feeds were read end to end. A run that opened a pull request and a |
| 164 | + # run that found nothing both count; a failed run does not, so a broken |
| 165 | + # refresh shows on the website as an aging stamp rather than a fresh one. |
| 166 | + # This step never fails the job: the stamp is informational, and the |
| 167 | + # publish below keeps the previous value when this lookup fails. |
| 168 | + continue-on-error: true |
| 169 | + run: | |
| 170 | + set -uo pipefail |
| 171 | + stamp="$(gh api \ |
| 172 | + "repos/${GITHUB_REPOSITORY}/actions/workflows/${REFRESH_WORKFLOW}/runs?status=success&per_page=1" \ |
| 173 | + --jq '.workflow_runs[0].updated_at // empty')" |
| 174 | + if [ -z "$stamp" ]; then |
| 175 | + echo "::warning::no successful ${REFRESH_WORKFLOW} run found. The published refreshed stamp keeps its previous value." |
| 176 | + else |
| 177 | + echo "the upstream feeds were last checked at $stamp" |
| 178 | + fi |
| 179 | + echo "stamp=$stamp" >> "$GITHUB_OUTPUT" |
| 180 | +
|
| 181 | + - name: Publish the table to the website repository |
| 182 | + id: publish |
| 183 | + env: |
| 184 | + WEB_TOKEN: ${{ secrets.CLUSTERMAX_WEB_TOKEN }} |
| 185 | + SOURCE_REPO: ${{ github.repository }} |
| 186 | + SOURCE_SHA: ${{ github.sha }} |
| 187 | + REFRESHED: ${{ steps.refreshed.outputs.stamp }} |
| 188 | + run: | |
| 189 | + set -uo pipefail |
| 190 | + # Keep the token out of the log and out of the remote URL that Git |
| 191 | + # writes into the clone configuration. |
| 192 | + checkout="$RUNNER_TEMP/clustermax-web" |
| 193 | + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "$WEB_TOKEN" | base64 -w0)" \ |
| 194 | + clone --depth 1 --no-tags "https://github.com/${WEB_REPO}.git" "$checkout" || { |
| 195 | + echo "::error::the clone of $WEB_REPO failed. Check that CLUSTERMAX_WEB_TOKEN has read access." |
| 196 | + exit 1 |
| 197 | + } |
| 198 | +
|
| 199 | + if [ ! -f "$checkout/$WEB_ROUTE" ]; then |
| 200 | + echo "::error::$WEB_REPO carries no $WEB_ROUTE, so $PUBLIC_URL would answer 404. Merge the website pull request that adds the route, then run this job again." |
| 201 | + exit 1 |
| 202 | + fi |
| 203 | +
|
| 204 | + # Write the published copy: this repository's table plus the two |
| 205 | + # website-only stamps. The component blocks are copied unchanged. |
| 206 | + WEB_FILE="$checkout/$WEB_PATH" python3 - <<'PY' |
| 207 | + import json |
| 208 | + import os |
| 209 | +
|
| 210 | + metadata_keys = { |
| 211 | + "advisory", "advisories", "advisoryRanges", "branchSources", |
| 212 | + "cves", "floorSources", "fixAvailability", "floorAvailability", |
| 213 | + "kind", "policy", "programCves", "programSources", "release", |
| 214 | + "source", "sources", |
| 215 | + } |
| 216 | +
|
| 217 | + def minimums(table): |
| 218 | + # The minimum values alone, without bulletin metadata and without |
| 219 | + # the timestamps, so `updated` moves only when a version moves. |
| 220 | + return { |
| 221 | + name: {key: value for key, value in block.items() if key not in metadata_keys} |
| 222 | + for name, block in (table.get("components") or {}).items() |
| 223 | + } |
| 224 | +
|
| 225 | + with open(os.environ["MINIMUMS_PATH"], encoding="utf-8") as handle: |
| 226 | + upstream = json.load(handle) |
| 227 | +
|
| 228 | + web_file = os.environ["WEB_FILE"] |
| 229 | + previous = {} |
| 230 | + if os.path.exists(web_file): |
| 231 | + with open(web_file, encoding="utf-8") as handle: |
| 232 | + try: |
| 233 | + previous = json.load(handle) |
| 234 | + except ValueError: |
| 235 | + previous = {} |
| 236 | +
|
| 237 | + published = dict(upstream) |
| 238 | +
|
| 239 | + if minimums(previous) != minimums(upstream) or not previous.get("updated"): |
| 240 | + # A minimum moved, or the website carries no `updated` stamp yet. |
| 241 | + # The table's own timestamp is the moment the generator recorded |
| 242 | + # the change, so use it rather than the publish time. |
| 243 | + published["updated"] = upstream["generated"] |
| 244 | + changed = True |
| 245 | + else: |
| 246 | + published["updated"] = previous["updated"] |
| 247 | + changed = False |
| 248 | +
|
| 249 | + refreshed = os.environ.get("REFRESHED", "") or previous.get("refreshed") or upstream["generated"] |
| 250 | + published["refreshed"] = refreshed |
| 251 | +
|
| 252 | + os.makedirs(os.path.dirname(web_file), exist_ok=True) |
| 253 | + with open(web_file, "w", encoding="utf-8") as handle: |
| 254 | + # Same rendering as the generator: sorted keys, two-space indent, |
| 255 | + # trailing newline, so the diff on the website is only what moved. |
| 256 | + handle.write(json.dumps(published, indent=2, sort_keys=True) + "\n") |
| 257 | +
|
| 258 | + summary = ( |
| 259 | + f"minimums changed, updated={published['updated']}" if changed |
| 260 | + else f"no minimum moved, updated stays {published['updated']}" |
| 261 | + ) |
| 262 | + print(f"{summary}; refreshed={refreshed}") |
| 263 | + # GITHUB_ENV reaches later steps only, so hand the stamps to the rest |
| 264 | + # of this step through a file as well. |
| 265 | + with open(os.path.join(os.environ["RUNNER_TEMP"], "stamps.sh"), "w", encoding="utf-8") as out: |
| 266 | + out.write(f"MINIMUMS_UPDATED='{published['updated']}'\n") |
| 267 | + out.write(f"MINIMUMS_REFRESHED='{refreshed}'\n") |
| 268 | + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as out: |
| 269 | + out.write(f"MINIMUMS_UPDATED={published['updated']}\n") |
| 270 | + out.write(f"MINIMUMS_REFRESHED={refreshed}\n") |
| 271 | + PY |
| 272 | + # shellcheck disable=SC1091 |
| 273 | + . "$RUNNER_TEMP/stamps.sh" |
| 274 | +
|
| 275 | + cd "$checkout" |
| 276 | + # Stage first, then compare the index against HEAD. `git diff` alone |
| 277 | + # reports no change for a file the website repository does not carry |
| 278 | + # yet, because an untracked file is in neither the index nor HEAD, so |
| 279 | + # the first publish would skip and the address would stay a 404. |
| 280 | + git add -- "$WEB_PATH" |
| 281 | + if git diff --cached --quiet -- "$WEB_PATH"; then |
| 282 | + echo "the published table already matches this repository. Nothing to push." |
| 283 | + echo "pushed=false" >> "$GITHUB_OUTPUT" |
| 284 | + exit 0 |
| 285 | + fi |
| 286 | +
|
| 287 | + git config user.name 'clustermax-minimums[bot]' |
| 288 | + git config user.email 'clustermax-minimums@users.noreply.github.com' |
| 289 | + # Build the message in a file. A quoted multi-line argument would |
| 290 | + # carry this file's indentation into the commit message. |
| 291 | + message="$RUNNER_TEMP/commit-message.txt" |
| 292 | + { |
| 293 | + echo 'chore(security): publish the minimum version table' |
| 294 | + echo |
| 295 | + echo "Generated ${MINIMUMS_GENERATED}." |
| 296 | + echo "Minimums last changed ${MINIMUMS_UPDATED}; feeds last checked ${MINIMUMS_REFRESHED}." |
| 297 | + echo "Source: ${SOURCE_REPO}@${SOURCE_SHA}." |
| 298 | + echo |
| 299 | + echo 'This file is generated. Do not hand-edit it.' |
| 300 | + } > "$message" |
| 301 | + git commit -q -F "$message" || { |
| 302 | + echo "::error::the commit failed." |
| 303 | + exit 1 |
| 304 | + } |
| 305 | +
|
| 306 | + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "$WEB_TOKEN" | base64 -w0)" \ |
| 307 | + push origin HEAD || { |
| 308 | + echo "::error::the push to $WEB_REPO failed. A branch protection rule can reject it, and CLUSTERMAX_WEB_TOKEN needs write access to the default branch." |
| 309 | + exit 1 |
| 310 | + } |
| 311 | + echo "pushed=true" >> "$GITHUB_OUTPUT" |
| 312 | + echo "published the table generated ${MINIMUMS_GENERATED}" |
| 313 | +
|
| 314 | + - name: Check the public address |
| 315 | + # The website deploys on its own queue, so a fresh copy takes a few |
| 316 | + # minutes to serve. This step reports what the address serves now. It |
| 317 | + # never fails the job: |
| 318 | + # - A slow deployment is normal and is not a defect. |
| 319 | + # - A stale public copy cannot produce a wrong verdict. The command |
| 320 | + # line refuses a fetched table that is older than the installed |
| 321 | + # table, so a stale copy degrades to no update. |
| 322 | + # - The scheduled run checks the address again the next day. |
| 323 | + if: steps.publish.outcome == 'success' && steps.publish.outputs.pushed == 'true' |
| 324 | + run: | |
| 325 | + python3 - <<'PY' |
| 326 | + import json |
| 327 | + import os |
| 328 | + import time |
| 329 | + import urllib.error |
| 330 | + import urllib.request |
| 331 | +
|
| 332 | + url = os.environ["PUBLIC_URL"] |
| 333 | + want = os.environ.get("MINIMUMS_GENERATED", "") |
| 334 | + want_refreshed = os.environ.get("MINIMUMS_REFRESHED", "") |
| 335 | + served = None |
| 336 | + served_refreshed = None |
| 337 | + for attempt in range(1, 7): |
| 338 | + try: |
| 339 | + # Ask the CDN for a fresh copy, not the cached response. |
| 340 | + request = urllib.request.Request(url, headers={"Cache-Control": "no-cache"}) |
| 341 | + with urllib.request.urlopen(request, timeout=15) as answer: |
| 342 | + table = json.loads(answer.read(4_000_000).decode("utf-8")) |
| 343 | + served = table.get("generated") |
| 344 | + served_refreshed = table.get("refreshed") |
| 345 | + except (urllib.error.URLError, ValueError, OSError) as exc: |
| 346 | + print(f"attempt {attempt}: {url} could not be read ({exc})") |
| 347 | + else: |
| 348 | + print(f"attempt {attempt}: {url} serves the table generated {served}, refreshed {served_refreshed}") |
| 349 | + if served == want and served_refreshed == want_refreshed: |
| 350 | + break |
| 351 | + if attempt < 6: |
| 352 | + time.sleep(30) |
| 353 | +
|
| 354 | + if served == want and served_refreshed == want_refreshed: |
| 355 | + summary = f"`{url}` serves the table generated {want}, refreshed {want_refreshed}." |
| 356 | + elif served: |
| 357 | + summary = ( |
| 358 | + f"`{url}` still serves the table generated {served} (refreshed {served_refreshed}), " |
| 359 | + f"and this run published {want} (refreshed {want_refreshed}). The website " |
| 360 | + "deployment is probably still in its queue." |
| 361 | + ) |
| 362 | + print(f"::warning::{summary}") |
| 363 | + else: |
| 364 | + summary = f"`{url}` could not be read. Check the website deployment." |
| 365 | + print(f"::warning::{summary}") |
| 366 | +
|
| 367 | + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: |
| 368 | + out.write(f"### Minimum versions publish\n\n{summary}\n") |
| 369 | + PY |
0 commit comments