diff --git a/bin/build b/bin/build index 90808cb2..7051f8e6 100755 --- a/bin/build +++ b/bin/build @@ -13,6 +13,27 @@ print_header "Omarchy Package Builder" DRY_RUN=false +# Knobs for builds driven from CI or resumed by hand. Each defaults to the +# historical behaviour, so an unadorned `bin/build` is unchanged. +# +# OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output/$MIRROR/$ARCH instead of +# wiping it, so packages built by an earlier +# job (or a previous, interrupted run) seed +# the build database and resolve as +# dependencies of what builds now. +# OMARCHY_SKIP_BUILDER_IMAGE=1 use the omarchy-pkg-builder image already +# present instead of building it; a workflow +# that builds the image once with an external +# BuildKit cache can then fan out over many +# package jobs without each one rebuilding it. +# OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy/omarchy-settings pair +# with --nodeps (see build/build.sh); only +# for a pipeline that verifies the install +# transaction afterwards. +KEEP_BUILD_WORKSPACE=${OMARCHY_KEEP_BUILD_WORKSPACE:-0} +SKIP_BUILDER_IMAGE=${OMARCHY_SKIP_BUILDER_IMAGE:-0} +DEFER_RUNTIME_DEPS=${OMARCHY_DEFER_RUNTIME_DEPS:-false} + # Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in @@ -67,6 +88,11 @@ while [[ $# -gt 0 ]]; do echo " $0 --package yay" echo " $0 --package yay elephant cursor-bin" echo "" + echo "Environment (for CI and resumed builds; defaults keep today's behaviour):" + echo " OMARCHY_KEEP_BUILD_WORKSPACE=1 keep build-output and reuse packages already there" + echo " OMARCHY_SKIP_BUILDER_IMAGE=1 use the existing builder image instead of building it" + echo " OMARCHY_DEFER_RUNTIME_DEPS=true build the omarchy pair with --nodeps (transaction verified later)" + echo "" exit 0 ;; *) @@ -76,11 +102,45 @@ while [[ $# -gt 0 ]]; do esac done +if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then + print_error "OMARCHY_DEFER_RUNTIME_DEPS must be true or false" + exit 1 +fi + +# Deferring runtime dependencies is only sound for the omarchy pair, and only +# when both halves are built together: the pair depends on each other and on +# packages that a sharded pipeline builds in other jobs, and the consumer of +# this mode installs the assembled set in one verified transaction. Check the +# request here so a misuse fails before Docker starts. +if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then + deferred_runtime=0 + deferred_settings=0 + deferred_count=0 + for package in $PACKAGES; do + ((deferred_count += 1)) + case $package in + omarchy|omarchy-dev) deferred_runtime=1 ;; + omarchy-settings|omarchy-settings-dev) deferred_settings=1 ;; + *) + print_error "OMARCHY_DEFER_RUNTIME_DEPS only applies to the omarchy/omarchy-settings pair, not $package" + exit 1 + ;; + esac + done + if (( deferred_runtime != 1 || deferred_settings != 1 || deferred_count != 2 )); then + print_error "OMARCHY_DEFER_RUNTIME_DEPS requires --package with exactly the omarchy pair" + exit 1 + fi +fi + # Show target architecture and mirror after parsing args print_info "Target architecture: $ARCH" print_info "Mirror: $MIRROR" print_info "Build workspace: $BUILD_OUTPUT_DIR" print_info "Final output: $REPO_DIR" +if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then + print_info "Runtime dependency checks: deferred to the install transaction" +fi if [[ "$DRY_RUN" == true ]]; then print_warning "DRY RUN MODE - build plan only; no Docker or makepkg will run" @@ -88,6 +148,7 @@ if [[ "$DRY_RUN" == true ]]; then MIRROR="$MIRROR" \ PACKAGES="$PACKAGES" \ DRY_RUN=true \ + DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS" \ PKGBUILDS_DIR="$PKGBUILDS_DIR" \ BUILD_OUTPUT_DIR="$BUILD_OUTPUT_DIR" \ FINAL_OUTPUT_DIR="$REPO_DIR" \ @@ -112,9 +173,14 @@ if [[ "$(uname -m)" == "x86_64" && "$ARCH" == "aarch64" ]]; then fi fi -# Clean build-output directory to start fresh -print_info "Cleaning build workspace..." -rm -rf "$BUILD_OUTPUT_DIR"/* +# Clean build-output directory to start fresh, unless the caller seeded it +# with packages from an earlier job or is resuming an interrupted run. +if [[ $KEEP_BUILD_WORKSPACE == "1" ]]; then + print_info "Keeping existing build workspace..." +else + print_info "Cleaning build workspace..." + rm -rf "$BUILD_OUTPUT_DIR"/* +fi mkdir -p "$BUILD_OUTPUT_DIR" # Show package info @@ -124,8 +190,20 @@ else print_info "Building unscoped packages for $MIRROR mirror" fi -# Build/update the Docker image -build_docker_image "$BUILD_DIR" "$ARCH" "$MIRROR" +# Build/update the Docker image, unless the caller prepared the exact image +# already (a workflow building it once with an external BuildKit cache). A +# missing image is an error rather than a silent rebuild: the point of the +# flag is that every job runs the same bytes. +IMAGE_TAG="omarchy-pkg-builder:latest-$ARCH-$MIRROR" +if [[ $SKIP_BUILDER_IMAGE == "1" ]]; then + if ! docker image inspect "$IMAGE_TAG" >/dev/null 2>&1; then + print_error "Prepared builder image is unavailable: $IMAGE_TAG" + exit 1 + fi + print_info "Using prepared builder image: $IMAGE_TAG" +else + build_docker_image "$BUILD_DIR" "$ARCH" "$MIRROR" +fi print_info "Running package build..." @@ -144,6 +222,7 @@ DOCKER_ARGS=( -e MIRROR="$MIRROR" -e PACKAGES="$PACKAGES" -e OMARCHY_RC_PINS="${OMARCHY_RC_PINS:-}" + -e DEFER_RUNTIME_DEPS="$DEFER_RUNTIME_DEPS" -v "$BUILD_ROOT/build-output:/build-output" -v "$REPO_ROOT:/pkgs.omarchy.org" -v "$BUILD_DIR:/build:ro" @@ -152,7 +231,6 @@ DOCKER_ARGS=( ) # Run the builder with assembled args -IMAGE_TAG="omarchy-pkg-builder:latest-$ARCH-$MIRROR" PLATFORM_ARG=$(get_platform_arg "$ARCH") docker run $PLATFORM_ARG "${DOCKER_ARGS[@]}" "$IMAGE_TAG" /build/build.sh diff --git a/build/build.sh b/build/build.sh index 42f9f2ea..e0f69acb 100755 --- a/build/build.sh +++ b/build/build.sh @@ -12,9 +12,37 @@ BUILD_OUTPUT_DIR=${BUILD_OUTPUT_DIR:-/build-output/$MIRROR/$ARCH} FINAL_OUTPUT_DIR=${FINAL_OUTPUT_DIR:-/pkgs.omarchy.org/$MIRROR/$ARCH} HELPERS_DIR=${HELPERS_DIR:-/helpers} SRC_DIR=${SRC_DIR:-/src} +# Set by bin/build from OMARCHY_DEFER_RUNTIME_DEPS after it has checked the +# request; re-checked here so the container never trusts a stray value. +DEFER_RUNTIME_DEPS=${DEFER_RUNTIME_DEPS:-false} source "$HELPERS_DIR/package-metadata.sh" +if [[ $DEFER_RUNTIME_DEPS != "false" && $DEFER_RUNTIME_DEPS != "true" ]]; then + echo "DEFER_RUNTIME_DEPS must be true or false" >&2 + exit 1 +fi +if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then + deferred_runtime=0 + deferred_settings=0 + deferred_count=0 + for package in $PACKAGES; do + ((deferred_count += 1)) + case $package in + omarchy|omarchy-dev) deferred_runtime=1 ;; + omarchy-settings|omarchy-settings-dev) deferred_settings=1 ;; + *) + echo "Runtime dependency deferral only applies to the omarchy pair, not $package" >&2 + exit 1 + ;; + esac + done + if (( deferred_runtime != 1 || deferred_settings != 1 || deferred_count != 2 )); then + echo "Runtime dependency deferral requires exactly the omarchy pair" >&2 + exit 1 + fi +fi + if [[ "$DRY_RUN" != true ]]; then # Import GPG keys /build/import-gpg-keys.sh || exit 1 @@ -48,13 +76,16 @@ EOF # Create an empty database repo-add omarchy-build.db.tar.zst >/dev/null 2>&1 ln -sf omarchy-build.db.tar.zst omarchy-build.db - else - # Database exists, check if we need to rebuild it from packages - if ls *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' | grep -v 'omarchy-build\.db' | grep -q .; then - echo "==> Rebuilding build database from existing packages..." - ls *.pkg.tar.* | grep -v '\.sig$' | grep -v 'omarchy-build\.db' | xargs -r repo-add omarchy-build.db.tar.zst >/dev/null 2>&1 - ln -sf omarchy-build.db.tar.zst omarchy-build.db - fi + fi + # Fold any packages already in the workspace into the database, whether + # they came with an existing database or were dropped in by an earlier + # workflow job (OMARCHY_KEEP_BUILD_WORKSPACE). Without this a seeded + # workspace with no database would leave those packages invisible to + # dependency resolution. + if ls *.pkg.tar.* 2>/dev/null | grep -v '\.sig$' | grep -v 'omarchy-build\.db' | grep -q .; then + echo "==> Rebuilding build database from existing packages..." + ls *.pkg.tar.* | grep -v '\.sig$' | grep -v 'omarchy-build\.db' | xargs -r repo-add omarchy-build.db.tar.zst >/dev/null 2>&1 + ln -sf omarchy-build.db.tar.zst omarchy-build.db fi # Add omarchy repo if it has a database (stable packages) @@ -224,6 +255,31 @@ refresh_vcs_pkgver_preserving_local_pkgrel() { fi } +# With runtime dependency checks deferred, makepkg runs --nodeps, so the +# build-time dependencies it would normally install with -s have to be +# installed explicitly: makedepends and checkdepends, including the +# architecture-suffixed variants for the current CARCH. +install_deferred_build_dependencies() { + local pkg="$1" + local -a build_deps=() + + mapfile -t build_deps < <( + CARCH="$ARCH" bash -c ' + source PKGBUILD + arch_makedepends="makedepends_${CARCH}[@]" + arch_checkdepends="checkdepends_${CARCH}[@]" + printf "%s\n" \ + "${makedepends[@]}" "${!arch_makedepends}" \ + "${checkdepends[@]}" "${!arch_checkdepends}" + ' | awk 'NF && !seen[$0]++' + ) + + if (( ${#build_deps[@]} )); then + echo " Installing build-only dependencies for $pkg..." + sudo pacman -S --needed --noconfirm -- "${build_deps[@]}" + fi +} + # Build a package build_package() { local pkg="$1" @@ -280,9 +336,20 @@ build_package() { # Build package without signing (signing is done separately) # PACMAN override uses a wrapper that adds --ask 4 to auto-resolve conflicts # (e.g. rustup replacing rust) since --noconfirm defaults to 'N' on those prompts - MAKEPKG_FLAGS="-scf --noconfirm" + local -a makepkg_flags=(-scf --noconfirm) + if [[ $DEFER_RUNTIME_DEPS == "true" ]]; then + # The pair's runtime dependencies (each other, and packages other jobs + # of the same pipeline build) are not resolvable here; the assembled set + # is installed in one verified transaction downstream. Only the + # build-time dependencies are installed, then makepkg skips the check. + install_deferred_build_dependencies "$pkg" || { + FAILED_PACKAGES="$FAILED_PACKAGES $pkg" + return 1 + } + makepkg_flags=(-cf --noconfirm --nodeps) + fi - if PACMAN=/usr/local/bin/pacman-for-makepkg makepkg $MAKEPKG_FLAGS; then + if PACMAN=/usr/local/bin/pacman-for-makepkg makepkg "${makepkg_flags[@]}"; then # Ensure output directory exists mkdir -p "$BUILD_OUTPUT_DIR" diff --git a/helpers/docker-helpers.sh b/helpers/docker-helpers.sh index 53f63065..4a269041 100644 --- a/helpers/docker-helpers.sh +++ b/helpers/docker-helpers.sh @@ -61,9 +61,14 @@ get_platform_arg() { make_dir_writable() { local dir="$1" - if [ "$(id -u)" -eq 0 ]; then + if (( EUID == 0 )); then chmod -R 777 "$dir" else - sudo chown -R $(id -u):$(id -g) "$dir" 2>/dev/null || chmod -R 777 "$dir" + # chown can succeed on part of the tree and fail on the rest (files a + # previous container left behind as another uid); the old `|| chmod` + # fallback only ran when chown failed outright, leaving those files + # unwritable. Always follow with chmod so the whole tree is usable. + sudo chown -R "$(id -u):$(id -g)" "$dir" 2>/dev/null || true + chmod -R 777 "$dir" fi }