Make aarch64 a first-class architecture in the scheduled pipeline

.github/workflows/test.yml

@@ -27,6 +27,7 @@ jobs:
               set -euo pipefail
               pacman -Syu --noconfirm jq
               ./bin/sync-upstream self-test
+              ./bin/sync-rebuilds --self-test
               ./bin/omarchy-pkgs self-test
               ./bin/omarchy-release self-test
             '

.gitignore

@@ -1,3 +1,4 @@
+.firecrawl/
 src/
 logs/
 build-output/

bin/advance-channel

@@ -25,6 +25,22 @@ SKIP_PROD_CHECK=false
 FAST_RING_ONLY=false
 BOOTSTRAP=false
 PACKAGES=""
+ALL_ARCHES=false
+
+# Kept for --arch all, which re-invokes this script per architecture with the
+# other arguments unchanged (minus the --arch all pair itself).
+ORIGINAL_ARGS=()
+arch_option=false
+for arg in "$@"; do
+  if [[ "$arch_option" == true ]]; then
+    [[ "$arg" != "all" ]] && ORIGINAL_ARGS+=("--arch" "$arg")
+    arch_option=false
+  elif [[ "$arg" == "--arch" ]]; then
+    arch_option=true
+  else
+    ORIGINAL_ARGS+=("$arg")
+  fi
+done
 
 usage() {
   echo "Usage: $0 --from <channel> --to <channel> [OPTIONS]"
@@ -36,7 +52,8 @@ usage() {
   echo "                              or --bootstrap (one-time initial seed)"
   echo ""
   echo "Options:"
-  echo "  --arch <arch>         Target architecture (x86_64 or aarch64, default: x86_64)"
+  echo "  --arch <arch>|all     Target architecture (x86_64 or aarch64, default: x86_64);"
+  echo "                        all = every published architecture, one after another"
   echo "  --package <names...>  Advance only the named package(s)"
   echo "  --fast-ring           Restrict to fast-ring packages (stable -> rc parity copy)"
   echo "  --bootstrap           One-time stable -> rc seed before forward-only enforcement"
@@ -65,8 +82,14 @@ while [[ $# -gt 0 ]]; do
     shift 2
     ;;
   --arch)
-    ARCH="$2"
-    update_arch_paths
+    if [[ "$2" == "all" ]]; then
+      ALL_ARCHES=true
+      ARCH=all
+    else
+      require_valid_arch "$2"
+      ARCH="$2"
+      update_arch_paths
+    fi
     shift 2
     ;;
   --package)
@@ -134,6 +157,31 @@ case "$FROM->$TO" in
   ;;
 esac
 
+if [[ "$ALL_ARCHES" == true ]]; then
+  # Hold one lock across the whole operation so a timer cannot mutate a
+  # channel between architectures. Check every source database before moving
+  # the first one; a failed sync can still require an idempotent retry, but a
+  # missing architecture never creates a knowingly partial advance.
+  if [[ "$DRY_RUN" != true ]]; then
+    acquire_release_lock || exit 1
+  fi
+
+  ARCHES=$(published_arches)
+  for arch in $ARCHES; do
+    source_db="$REPO_ROOT/$FROM/$arch/omarchy.db.tar.zst"
+    if [[ ! -f "$source_db" ]]; then
+      print_error "Source database not found: $source_db"
+      echo "Nothing has been published to the $FROM channel for $arch."
+      exit 1
+    fi
+  done
+
+  for arch in $ARCHES; do
+    "$0" --arch "$arch" "${ORIGINAL_ARGS[@]}" || exit $?
+  done
+  exit 0
+fi
+
 SOURCE_DIR="$REPO_ROOT/$FROM/$ARCH"
 TARGET_DIR="$REPO_ROOT/$TO/$ARCH"
 SOURCE_DB="$SOURCE_DIR/omarchy.db.tar.zst"

bin/auto-release

@@ -1,6 +1,10 @@
 #!/bin/bash
 # Run the release workflow for a channel when work is queued.
-# Usage: auto-release <edge|rc|stable>
+# Usage: auto-release <edge|rc|stable> [<arch>|all]
+#
+# With no architecture (what the timers pass), every published architecture
+# is processed in turn, each against its own queue and its own backoff, so a
+# failing aarch64 build never holds up x86_64 or the other way round.
 #
 # Safe to run on a tight schedule. Three guards make that true:
 #
@@ -21,16 +25,17 @@ source "$BUILD_ROOT/helpers/paths.sh"
 source "$BUILD_ROOT/helpers/lock-helpers.sh"
 
 MIRROR="${1:-}"
-STATE_DIR="${OMARCHY_STATE_DIR:-/root/.state}"
+ARCH_ARG="${2:-all}"
 
 # Backoff schedule: 10m, 20m, 40m, 80m, 160m, 320m, then hourly-ish forever
 # (capped at 6h, the cadence this system ran at before frequent timers).
 BACKOFF_BASE_SECONDS="${OMARCHY_BACKOFF_BASE:-600}"
 BACKOFF_MAX_SECONDS="${OMARCHY_BACKOFF_MAX:-21600}"
 
 if [[ -z "$MIRROR" ]]; then
-  print_error "Usage: $0 <mirror>"
+  print_error "Usage: $0 <mirror> [<arch>|all]"
   echo "  mirror: edge, rc, or stable"
+  echo "  arch:   one of $VALID_ARCHES, or all (default) for every published architecture"
   exit 1
 fi
 
@@ -39,17 +44,13 @@ if [[ "$MIRROR" != "edge" && "$MIRROR" != "rc" && "$MIRROR" != "stable" ]]; then
   exit 1
 fi
 
-STATE_FILE="$STATE_DIR/.sync-needed-$MIRROR"
-FAIL_FILE="$STATE_DIR/.build-failed-$MIRROR"
-
-# Nothing queued: stay quiet. At a 5-minute cadence this is most invocations,
-# and a header for each would bury the runs that matter in the journal.
-if [[ ! -f "$STATE_FILE" ]]; then
-  exit 0
+if [[ "$ARCH_ARG" == "all" ]]; then
+  ARCHES=$(published_arches)
+else
+  require_valid_arch "$ARCH_ARG"
+  ARCHES="$ARCH_ARG"
 fi
 
-print_header "Processing Sync for $MIRROR"
-
 # The build inputs are this checkout's contents; its HEAD identifies them.
 current_fingerprint() {
   git -C "$BUILD_ROOT" rev-parse HEAD 2>/dev/null || echo "unknown"
@@ -65,57 +66,103 @@ backoff_seconds() {
   echo "$delay"
 }
 
-FAIL_COUNT=0
-if [[ -f "$FAIL_FILE" ]]; then
-  # shellcheck disable=SC1090
-  source "$FAIL_FILE" 2>/dev/null || true
-  FAIL_COUNT="${FAILURE_COUNT:-0}"
-  failed_at="${FAILURE_AT:-0}"
-  failed_fingerprint="${FAILURE_FINGERPRINT:-}"
-
-  if [[ "$failed_fingerprint" != "$(current_fingerprint)" ]]; then
-    print_info "Repository changed since the last failure — clearing backoff and retrying"
-    rm -f "$FAIL_FILE"
-    FAIL_COUNT=0
-  else
-    delay=$(backoff_seconds "$FAIL_COUNT")
-    now=$(date +%s)
-    retry_at=$((failed_at + delay))
-    if ((now < retry_at)); then
-      print_warning "$MIRROR has failed $FAIL_COUNT time(s) on this tree — not retrying until $(date -d "@$retry_at" '+%H:%M:%S' 2>/dev/null || echo "+$((retry_at - now))s")"
-      echo "  Push a fix (any new commit clears this), or: rm $FAIL_FILE"
-      exit 0
+# Returns 0 when this architecture's queue was processed (or was empty), 1 when
+# the release failed. Backoff and "someone else holds the lock" are not
+# failures: they are this tick deciding to do nothing.
+release_arch() {
+  local arch="$1"
+  local state_file fail_file
+  state_file=$(sync_queue_file "$MIRROR" "$arch")
+  fail_file=$(sync_fail_file "$MIRROR" "$arch")
+
+  # A queue written under the pre-architecture name belongs to x86_64.
+  if [[ "$arch" == "x86_64" && ! -f "$state_file" && -f "$(legacy_sync_queue_file "$MIRROR")" ]]; then
+    state_file=$(legacy_sync_queue_file "$MIRROR")
+  fi
+  if [[ "$arch" == "x86_64" && ! -f "$fail_file" && -f "$(legacy_sync_fail_file "$MIRROR")" ]]; then
+    fail_file=$(legacy_sync_fail_file "$MIRROR")
+  fi
+
+  # Nothing queued: stay quiet. At a 5-minute cadence this is most
+  # invocations, and a header for each would bury the runs that matter in
+  # the journal.
+  [[ -f "$state_file" ]] || return 0
+
+  print_header "Processing Sync for $MIRROR ($arch)"
+
+  local fail_count=0 failed_at failed_fingerprint delay now retry_at
+  if [[ -f "$fail_file" ]]; then
+    FAILURE_COUNT=0 FAILURE_AT=0 FAILURE_FINGERPRINT=""
+    # shellcheck disable=SC1090
+    source "$fail_file" 2>/dev/null || true
+    fail_count="${FAILURE_COUNT:-0}"
+    failed_at="${FAILURE_AT:-0}"
+    failed_fingerprint="${FAILURE_FINGERPRINT:-}"
+
+    if [[ "$failed_fingerprint" != "$(current_fingerprint)" ]]; then
+      print_info "Repository changed since the last failure — clearing backoff and retrying"
+      rm -f "$fail_file"
+      fail_count=0
+    else
+      delay=$(backoff_seconds "$fail_count")
+      now=$(date +%s)
+      retry_at=$((failed_at + delay))
+      if ((now < retry_at)); then
+        print_warning "$MIRROR ($arch) has failed $fail_count time(s) on this tree — not retrying until $(date -d "@$retry_at" '+%H:%M:%S' 2>/dev/null || echo "+$((retry_at - now))s")"
+        echo "  Push a fix (any new commit clears this), or: rm $fail_file"
+        return 0
+      fi
+      print_info "Backoff elapsed — retrying $MIRROR ($arch) (failure #$((fail_count + 1)) if this fails)"
     fi
-    print_info "Backoff elapsed — retrying $MIRROR (failure #$((FAIL_COUNT + 1)) if this fails)"
   fi
-fi
 
-# Non-blocking: a build in progress means this tick has nothing to do.
-if ! try_release_lock; then
-  holder=$(release_lock_holder)
-  print_info "A release is already running (${holder:-holder unknown}) — skipping this tick"
-  exit 0
-fi
+  # Non-blocking: a build in progress means this tick has nothing to do. The
+  # lock is reentrant, so once this run holds it the remaining architectures
+  # run under the same acquisition.
+  if ! try_release_lock; then
+    local holder
+    holder=$(release_lock_holder)
+    print_info "A release is already running (${holder:-holder unknown}) — skipping this tick"
+    return 0
+  fi
 
-print_info "State file found: $STATE_FILE"
-print_info "Starting release workflow for $MIRROR..."
+  print_info "State file found: $state_file"
+  print_info "Starting release workflow for $MIRROR ($arch)..."
 
-if "$BUILD_ROOT/bin/repo" release --mirror "$MIRROR" --skip-prod-check; then
-  print_success "Release completed successfully for $MIRROR"
-  rm -f "$STATE_FILE"
-  rm -f "$FAIL_FILE"
-  print_success "State file removed: $STATE_FILE"
-else
-  status=$?
-  FAIL_COUNT=$((FAIL_COUNT + 1))
-  cat >"$FAIL_FILE" <<EOF
-FAILURE_COUNT=$FAIL_COUNT
+  if "$BUILD_ROOT/bin/repo" release --mirror "$MIRROR" --arch "$arch" --skip-prod-check; then
+    print_success "Release completed successfully for $MIRROR ($arch)"
+    local completed_state=("$state_file" "$fail_file")
+    if [[ "$arch" == "x86_64" ]]; then
+      completed_state+=("$(legacy_sync_queue_file "$MIRROR")" "$(legacy_sync_fail_file "$MIRROR")")
+    fi
+    if ! rm -f "${completed_state[@]}"; then
+      print_error "Release succeeded, but its queue state could not be cleared"
+      return 1
+    fi
+    print_success "State file removed: $state_file"
+    return 0
+  fi
+
+  fail_count=$((fail_count + 1))
+  if ! cat >"$fail_file" <<EOF
+FAILURE_COUNT=$fail_count
 FAILURE_AT=$(date +%s)
 FAILURE_FINGERPRINT=$(current_fingerprint)
 EOF
-  next=$(backoff_seconds "$FAIL_COUNT")
-  print_error "Release failed for $MIRROR (attempt $FAIL_COUNT)"
-  print_warning "State file retained for retry: $STATE_FILE"
+  then
+    print_error "Could not record failure state: $fail_file"
+    return 1
+  fi
+  local next
+  next=$(backoff_seconds "$fail_count")
+  print_error "Release failed for $MIRROR ($arch) (attempt $fail_count)"
+  print_warning "State file retained for retry: $state_file"
   print_warning "Backing off $((next / 60))m before the next attempt; a new commit retries sooner"
-  exit "$status"
-fi
+  return 1
+}
+
+status=0
+for arch in $ARCHES; do
+  release_arch "$arch" || status=1
+done
+exit "$status"