Generate and maintain repository metadata in tree

.gitattributes

@@ -1,2 +1,4 @@
 *.deb filter=lfs diff=lfs merge=lfs -text
 *.rpm filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -3,6 +3,10 @@ name: CI
 
 on: [push, pull_request]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   tests:
     runs-on: ubuntu-latest
@@ -17,3 +21,22 @@ jobs:
       - name: Verify the signatures of all rpm artifacts
         run: |
           ./scripts/check.py --verify --all
+
+  metadata:
+    runs-on: ubuntu-latest
+    container: debian:bookworm
+    steps:
+      - name: Install dependencies
+        run: |
+          apt-get update && apt-get install --yes python3 git git-lfs createrepo-c
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+          fetch-depth: 0
+      - name: Check repository metadata is up-to-date
+        run: |
+          git config --global --add safe.directory '*'
+          ./tools/publish-real --reproduce
+          git status
+          git diff --exit-code

public/workstation/dom0/f32/repodata/repomd.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1721160629</revision>
+  <data type="primary">
+    <checksum type="sha256">4fefc41d39ab031c5a8a8de064ec16972e1a49d93f30a8448c99d88692454549</checksum>
+    <open-checksum type="sha256">ba6710d288ec7c1b7b6d28f861edc7998fa78bccdaa783afd534dedde0b47641</open-checksum>
+    <location href="repodata/4fefc41d39ab031c5a8a8de064ec16972e1a49d93f30a8448c99d88692454549-primary.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>1846</size>
+    <open-size>14658</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">459b1903f2ab64d76e3ecbf085eb87c58a9492912a3dfc2ce2947591ad449921</checksum>
+    <open-checksum type="sha256">f0b7a1c551f36129a18fcca753ea64a6e880a843e5236b0921c8227b2aeab497</open-checksum>
+    <location href="repodata/459b1903f2ab64d76e3ecbf085eb87c58a9492912a3dfc2ce2947591ad449921-filelists.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>2316</size>
+    <open-size>42115</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">9df6ebad27017462079510ee060951acd03b166e639e64026412c3c501fac89a</checksum>
+    <open-checksum type="sha256">2cfde3d045513efda282c246168dcfd0650d4e7bf9ae68f3d09d2f72da35e501</open-checksum>
+    <location href="repodata/9df6ebad27017462079510ee060951acd03b166e639e64026412c3c501fac89a-other.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>1647</size>
+    <open-size>13444</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">2cf2fd31244e1cf392bd8cf9b50c277d1bd6e451c3fb7695a74e3a0b7c206a7f</checksum>
+    <open-checksum type="sha256">0ec3f1c7a3a2417d9ca954875e754cdab6761abcc4afc33134bbd222feedee11</open-checksum>
+    <location href="repodata/2cf2fd31244e1cf392bd8cf9b50c277d1bd6e451c3fb7695a74e3a0b7c206a7f-primary.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>4558</size>
+    <open-size>114688</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">a5e36397022236443aa92497a7bb622a3eeb90068280121d9e84a8bc5b9ed517</checksum>
+    <open-checksum type="sha256">b2001b79a2c36488da07238b6c37bc7d354b07358c73d93475e68da1d2aefb24</open-checksum>
+    <location href="repodata/a5e36397022236443aa92497a7bb622a3eeb90068280121d9e84a8bc5b9ed517-filelists.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>6225</size>
+    <open-size>57344</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">201484ffa7e3ca3bc3e55f649b0a561826f7bf0caca417bf739865379d0039ba</checksum>
+    <open-checksum type="sha256">f85208b7130bff6da33a11322668316f313f2d91898f58880f99d1dfe1340f6c</open-checksum>
+    <location href="repodata/201484ffa7e3ca3bc3e55f649b0a561826f7bf0caca417bf739865379d0039ba-other.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>3360</size>
+    <open-size>36864</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>

public/workstation/dom0/f37/repodata/repomd.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
+  <revision>1721160629</revision>
+  <data type="primary">
+    <checksum type="sha256">909a61411c890363df0d4dd0fc85161e24635003343aecd50f3cf44ef229839c</checksum>
+    <open-checksum type="sha256">50a010ce741f910cbc8f4f3126b515a0393139a728c89458d0951388f875f2c7</open-checksum>
+    <location href="repodata/909a61411c890363df0d4dd0fc85161e24635003343aecd50f3cf44ef229839c-primary.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>965</size>
+    <open-size>2409</open-size>
+  </data>
+  <data type="filelists">
+    <checksum type="sha256">167314d2ef22e2bc9099e67def782df29a0f2574b29968ae15d7bbd10ee04a49</checksum>
+    <open-checksum type="sha256">406996b650d0834a84657f1c40463e995df00df5130c20aa9b32c8fd1db55f6f</open-checksum>
+    <location href="repodata/167314d2ef22e2bc9099e67def782df29a0f2574b29968ae15d7bbd10ee04a49-filelists.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>1170</size>
+    <open-size>6331</open-size>
+  </data>
+  <data type="other">
+    <checksum type="sha256">0f32036cd64316705eafb791efd98fb6d8653b0f3ac620cc4db2e508dfb1bb4c</checksum>
+    <open-checksum type="sha256">261ce36999f11d214028beeabab126f902a3cf7d827923107f730e5d2168bec4</open-checksum>
+    <location href="repodata/0f32036cd64316705eafb791efd98fb6d8653b0f3ac620cc4db2e508dfb1bb4c-other.xml.gz"/>
+    <timestamp>1721160629</timestamp>
+    <size>549</size>
+    <open-size>1755</open-size>
+  </data>
+  <data type="primary_db">
+    <checksum type="sha256">f7ff55472e87f1f9b7af51faa7e3d69fec51758068280c55bbf1301eaf99f706</checksum>
+    <open-checksum type="sha256">d5e60bd0121cf68d71be0f11423ce21f2c2255fee18603d05d4ea7bff5a3e8a4</open-checksum>
+    <location href="repodata/f7ff55472e87f1f9b7af51faa7e3d69fec51758068280c55bbf1301eaf99f706-primary.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>2539</size>
+    <open-size>106496</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="filelists_db">
+    <checksum type="sha256">f54442555160b3c6514f2f971336a2c7aa3a383241188db9621aab4f757d8ffb</checksum>
+    <open-checksum type="sha256">91fdfa0a39f595b668241be3d55b8003a232ab1f9b5b6ca795910cd7951a603b</open-checksum>
+    <location href="repodata/f54442555160b3c6514f2f971336a2c7aa3a383241188db9621aab4f757d8ffb-filelists.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>2255</size>
+    <open-size>28672</open-size>
+    <database_version>10</database_version>
+  </data>
+  <data type="other_db">
+    <checksum type="sha256">dc89e30924a469f98825b9339b975414418e83d59d76bc9108f61a22eec1e1fd</checksum>
+    <open-checksum type="sha256">21df5ebda9462a1359017949276705a38f388bccf88f20d0be43f42bc1546e3a</open-checksum>
+    <location href="repodata/dc89e30924a469f98825b9339b975414418e83d59d76bc9108f61a22eec1e1fd-other.sqlite.bz2"/>
+    <timestamp>1721160629</timestamp>
+    <size>1140</size>
+    <open-size>24576</open-size>
+    <database_version>10</database_version>
+  </data>
+</repomd>

scripts/check.py

@@ -12,7 +12,6 @@
 
 
 def verify_sig_rpm(path):
-
     for key_path in [PROD_SIGNING_KEY_PATH, PROD_SIGNING_KEY_PATH_LEGACY]:
         try:
             subprocess.check_call(["rpmkeys", "--import", key_path])

tools/publish

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Pull the latest image
+podman pull debian:bookworm
+# Mount the git repo to /srv, install necessary packages and run the publish script
+podman run --rm -it -v $(git rev-parse --show-toplevel):/srv:Z debian:bookworm \
+    bash -c "apt-get update && apt-get install -y python3 createrepo-c && /srv/tools/publish-real $@"

tools/publish-real

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""
+Script for generating yum repository metadata in a reproducible manner.
+
+Files are copied into public/ and metadata is generated there. All RPMs
+have their mtime fixed to a specific timestamp, so the generated XML/SQLite
+files will be reproducible.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+
+def fetch_reproduce_timestamp(public: Path) -> int:
+    repomd = next(public.glob("workstation/dom0/*/repodata/repomd.xml"))
+    tree = ET.parse(repomd)
+    root = tree.getroot()
+    revision = root.find(
+        "repo:revision", {"repo": "http://linux.duke.edu/metadata/repo"}
+    )
+    print(f"Will use a timestamp of {revision.text} (from {repomd})")
+    return int(revision.text)
+
+
+def main():
+    root = Path(__file__).parent.parent
+    public = root / "public"
+    workstation = root / "workstation"
+    if "--reproduce" in sys.argv:
+        try:
+            timestamp = fetch_reproduce_timestamp(public)
+        except Exception as err:
+            raise RuntimeError("Failed to fetch timestamp from repomd.xml") from err
+    else:
+        # Use the current time
+        timestamp = int(time.time())
+    # Reset public, copy the workstation/ tree into it
+    print("Creating public/ (from scratch)")
+    if public.exists():
+        shutil.rmtree(public)
+    public.mkdir()
+    shutil.copytree(workstation, public / "workstation")
+    for rpm in public.glob("**/*.rpm"):
+        os.utime(rpm, (timestamp, timestamp))
+    # Folders are public/workstation/dom0/fXX, run createrepo_c in each one
+    for folder in public.glob("*/*/*/"):
+        if not folder.is_dir():
+            continue
+        print(f"Generating metadata for {folder}")
+        # The <revision> and <timestamp> fields are set to the current UNIX time
+        # unless we explicitly override them. Use our fixed time to ensure it's
+        # consistent regardless of how long this command takes to run.
+        subprocess.check_call(
+            [
+                "createrepo_c",
+                "--revision",
+                str(timestamp),
+                "--set-timestamp-to-revision",
+                str(folder),
+            ]
+        )
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()