build_lib.sh

#!/bin/bash

# Library of useful functions for building Crowbar.
# This is sourced by build_crowbar.sh and any other interested parties.

# If we have already been sourced in the current env, don't source us again.
[[ $CROWBAR_BUILD_SOURCED = true ]] && exit 0

[[ $DEBUG ]] && {
    set -x
    export PS4='${BASH_SOURCE}@${LINENO}(${FUNCNAME[0]}): '
}

# Make sure that our git commands do not pop up with spurious editors,
export GIT_MERGE_AUTOEDIT=false
[[ $CROWBAR_TMP ]] || CROWBAR_TMP=$(mktemp -d /tmp/.crowbar-tmp-XXXXXXX)

# Location for caches that should not be erased between runs
[[ $CACHE_DIR ]] || CACHE_DIR="$HOME/.crowbar-build-cache"

export CROWBAR_TMP CACHE_DIR
# We might use lots and lots of open files.  Bump our open FD limits.
ulimit -Sn unlimited

# Hashes to hold our "interesting" information.
# Key = barclamp name
# Value = whatever interesting thing we are looking for.
[[ ${BC_QUERY_STRINGS[*]} ]] || declare -A BC_QUERY_STRINGS

# Build OS independent query strings.
BC_QUERY_STRINGS["deps"]="barclamp requires"
BC_QUERY_STRINGS["groups"]="barclamp member"
BC_QUERY_STRINGS["extra_files"]="extra_files"
BC_QUERY_STRINGS["build_cmd"]="build_cmd"
BC_QUERY_STRINGS["os_support"]="barclamp os_support"
BC_QUERY_STRINGS["gems"]="gems pkgs"
BC_QUERY_STRINGS["test_deps"]="smoketest requires"
BC_QUERY_STRINGS["test_timeouts"]="smoketest timeout"
BC_QUERY_STRINGS["supercedes"]="barclamp supercedes"
BC_QUERY_STRINGS["git_repos"]="git_repo"

# By default, do not try to update the cache or the metadata.
# These will be unset if --update-cache is passed to the build.
ALLOW_CACHE_UPDATE=false
ALLOW_CACHE_METADATA_UPDATE=false
declare -A BC_DEPS BC_GROUPS BC_PKGS BC_EXTRA_FILES BC_OS_SUPPORT BC_GEMS
declare -A BC_REPOS BC_PPAS BC_RAW_PKGS BC_BUILD_PKGS
declare -A BC_SMOKETEST_DEPS BC_SMOKETEST_TIMEOUTS BC_BUILD_CMDS
declare -A BC_SUPERCEDES BC_SRC_PKGS BC_GIT_REPOS
declare -A CACHED_PACKAGES

GEM_EXT_RE='^(.*)-\((.*)\)$'

# Extract the contents of a single git file from a repository
git_cat_file() (
    # $1 = path to the repository.
    # $2 = commit-ish to extract metadata from
    # $3 = Full path in the repository to extract file info from.
    local mode type sha name
    [[ -d $1/.git || -f $1/.git ]] || \
        die "$1 is not a git repository"
    cd "$1"
    branch_exists "$2" || die "$2 is not a branch in $1"
    read mode type sha name < <(git ls-tree --full-tree "$2" "$3")
    [[ $sha ]] || die "$3 does not exist in branch $2 in $1"
    [[ $type = "blob" ]] || die "$3 is not a file"
    git cat-file blob "$sha"
)

extract_barclamp_metadata() {
    # $1 = path to barclamp
    # $2 = git commit-ish to extract metadata from
    # Returns path to extracted crowbar.yml.
    if [[ $2 ]]; then
        git_cat_file "$1" "$2" "crowbar.yml" "$CROWBAR_TMP/$sha.yml"
        echo "$CROWBAR_TMP/$sha.yml"
    else
        echo "$CROWBAR_DIR/barclamps/$1/crowbar.yml"
    fi
}

read_barclamp_metadata() {
    # $1 = path to the .yml with the metadata.
    # $@ = args to pass to parse_yml.rb.
    local yml_file
    [[ -f $1 && $1 = *.yml ]] || die "$1 is not a YML file."
    yml_file="$1"
    shift
    "$CROWBAR_DIR/parse_yml.rb" "$yml_file" "$@" 2>/dev/null
}

get_one_barclamp_info() {
    # $1 = barclamp name
    # $2 = git commit-ish
    # Gets all the info that BC_QUERY_STRINGS wants.
    [[ -d $CROWBAR_DIR/barclamps/$1 ]] || die "$1 is not a barclamp!"
    local mdfile="$CROWBAR_DIR/barclamps/$1/crowbar.yml"
    local query line
    [[ $mdfile ]] || return
    [[ $1 = crowbar ]] || BC_DEPS["$1"]+="crowbar "
    for query in "${!BC_QUERY_STRINGS[@]}"; do
        while read line; do
            [[ $line = nil ]] && continue
            case $query in
                deps) is_in "$line "${BC_DEPS["$1"]} || \
                    BC_DEPS["$1"]+="$line ";;
                groups) is_in "$line" ${BC_GROUPS["$1"]} ||
                    BC_GROUPS["$line"]+="$1 ";;
                pkgs|os_pkgs) is_in "$line" ${BC_PKGS["$1"]} || \
                    BC_PKGS["$1"]+="$line ";;
                src_pkgs) is_in "$line" ${BC_SRC_PKGS["$1"]} || \
                    BC_SRC_PKGS["$1"]+="$line ";;
                extra_files) BC_EXTRA_FILES["$1"]+="$line\n";;
                os_support) BC_OS_SUPPORT["$1"]+="$line ";;
                gems) BC_GEMS["$1"]+="$line ";;
                repos|os_repos) BC_REPOS["$1"]+="$line\n";;
                ppas|os_ppas) [[ $PKG_TYPE = debs ]] || \
                    die "Cannot declare a PPA for $PKG_TYPE!"
                    BC_REPOS["$1"]+="ppa $line\n";;
                build_pkgs|os_build_pkgs) BC_BUILD_PKGS["$1"]+="$line ";;
                raw_pkgs|os_raw_pkgs|pkg_sources|os_pkg_sources) BC_RAW_PKGS["$1"]+="$line ";;
                test_deps) BC_SMOKETEST_DEPS["$1"]+="$line ";;
                os_build_cmd|build_cmd) [[ ${BC_BUILD_CMDS["$1"]} ]] || \
                    BC_BUILD_CMDS["$1"]="$line";;
                test_timeouts) BC_SMOKETEST_TIMEOUTS["$1"]+="$line ";;
                supercedes)
                    [[ ${BC_SUPERCEDES[$line]} ]] || \
                    BC_SUPERCEDES["$line"]="$1";;
                git_repos) BC_GIT_REPOS["$1"]+="$line\n";;
                *) die "Cannot handle query for $query."
            esac
        done < <(read_barclamp_metadata "$mdfile" ${BC_QUERY_STRINGS["$query"]})
    done
}

get_barclamp_info() {
    local bc yml_file line query newdeps dep d i
    local new_barclamps=()
    # Pull in interesting information from all our barclamps
    cd "$CROWBAR_DIR"
    for bc in barclamps/*; do
        [[ -d "$bc" ]] || continue
        bc=${bc##*/}
        is_barclamp "$bc" || {
            debug "$bc is not part of this build."
            continue
        }
        debug "Reading metadata for $bc barclamp."
        get_one_barclamp_info "$bc"
    done
    cd -

    debug "Analyzing barclamp group membership"
    # If any barclamps need group expansion, do it.
    for bc in "${!BC_DEPS[@]}"; do
        newdeps=''
        for dep in ${BC_DEPS["$bc"]}; do
            if [[ $dep = @* ]]; then
                [[ ${BC_GROUPS["${dep#@}"]} ]] || \
                    die "$bc depends on group ${dep#@}, but that group does not exist!"
                for d in ${BC_GROUPS["${dep#@}"]}; do
                    newdeps+="$d "
                done
            else
                newdeps+="$dep "
            fi
        done
        BC_DEPS["$bc"]="$newdeps"
    done

    # Group-expand barclamps if needed, and unset groups after they are expanded
    for i in "${!BARCLAMPS[@]}"; do
        bc="${BARCLAMPS[$i]}"
        if [[ $bc = @* ]]; then
            [[ ${BC_GROUPS["${bc#@}"]} ]] || \
                die "No such group ${bc#@}!"
            BARCLAMPS+=(${BC_GROUPS["${bc#@}"]})
            unset BARCLAMPS[$i]
        else
            is_barclamp "$bc" || die "$bc is not a barclamp!"
        fi
    done
    BARCLAMPS=("${BARCLAMPS[@]//@*}")

    # Pull in dependencies for the barclamps.
    # Everything depends on the crowbar barclamp, so include it first.
    for bc in "${BARCLAMPS[@]}"; do
        if [[ ${BC_SUPERCEDES[$bc]} ]]; then
            debug "$bc is superceded by ${BC_SUPERCEDES[$bc]}. Skipping."
            continue
        fi
        new_barclamps+=("$bc")
        is_barclamp "$bc" || die "$bc is not a barclamp!"
    done
    BARCLAMPS=($(all_deps "${new_barclamps[@]}"))
}

[[ $CROWBAR_BUILD_PID ]] || export CROWBAR_BUILD_PID=$$
export CLEANUP_LOCK="$CROWBAR_DIR/.cleanup.lock"
cleanup_cmds=()

git_managed_cache() [[ -d $CACHE_DIR/.git ]]

with_build_lock() {
    flock -n 65 || die "Could not grab build lock!"
    "$@"
} 65>/tmp/.build_crowbar.lock

# Test to see if Crowbar metadata for this release is tracked in
# a seperate git repository, by convention checked out
# in the releases directory.
git_tracked_checkout() [[ -d $CROWBAR_DIR/.releases/.git ]]

# Test to see if we are using flat metadata in the releases directory.
flat_checkout() { ! git_tracked_checkout && [[ -d $CROWBAR_DIR/releases/development/master ]]; }

# Our general cleanup function.  It is called as a trap whenever the
# build script exits, and it's job is to make sure we leave the local
# system in the same state we cound it, modulo a few calories of wasted heat
# and a shiny new .iso.
cleanup() {
    flock -n 70 || exit 1
    # Clean up any stray mounts we may have left behind.
    # The paranoia with the grepping is to ensure that we do not
    # inadvertently umount everything.
    if [[ $BASH_SUBSHELL -gt 0 || $BASHPID != $CROWBAR_BUILD_PID ]]; then
        kill -INT $CROWBAR_BUILD_PID
        exit 1
    fi
    for c in "${cleanup_cmds[@]}"; do
        $c || res=1
    done

    GREPOPTS=()
    [[ $CACHE_DIR ]] && GREPOPTS=(-e "$CACHE_DIR")
    [[ $IMAGE_DIR && $CACHE_DIR =~ $IMAGE_DIR ]] && GREPOPTS+=(-e "$IMAGE_DIR")
    [[ $BUILD_DIR && $CACHE_DIR =~ $BUILD_DIR ]] && GREPOPTS+=(-e "$BUILD_DIR")
    [[ $CROWBAR_DIR && -d $CROWBAR_DIR/testing ]] && GREPOPTS+=(-e "$CROWBAR_DIR/testing")
    [[ $CHROOT && $CACHE_DIR =~ $CHROOT ]] && GREPOPTS+=(-e "$CHROOT")
    if [[ $GREPOPTS ]]; then
        while read dev fs type opts rest; do
            sudo umount -d -l "$fs"
        done < <(tac /proc/self/mounts |grep "${GREPOPTS[@]}")
    fi
    # If the build process spawned a copy of webrick, make sure it is dead.
    [[ $webrick_pid && -d /proc/$webrick_pid ]] && kill -9 $webrick_pid
    # clean up after outselves from merging branches, if needed.
    [[ $CI_BARCLAMP ]] &&  {
        if ! in_repo git submodule update -N "barclamps/$CI_BARCLAMP"; then
            in_ci_barclamp git checkout -f master
        fi
        if ! in_ci_barclamp git branch -D ci-throwaway-branch; then
            in_ci_barclamp git checkout -f master
            in_ci_barclamp git branch -D ci-throwaway-branch
        fi

    }
    cd "$CROWBAR_DIR"
    if [[ $THROWAWAY_BRANCH ]]; then
        # Check out the branch we started the build process, and then
        # nuke whatever throwaway branch we may have created.
        git checkout -f "${CURRENT_BRANCH#refs/heads/}" &>/dev/null
        git branch -D "$THROWAWAY_BRANCH" &>/dev/null
    fi
    # If we saved unadded changes, resurrect them.
    [[ $THROWAWAY_STASH ]] && git stash apply "$THROWAWAY_STASH" &>/dev/null
    # Nuke any wild caches.
    if [[ $WILD_CACHE = true ]]; then
        for f in "${CACHE_DIR%/*}/.crowbar_temp_cache"*; do
            [[ -d $f ]] || continue
            sudo rm -rf "$f"
        done
    fi
    if [[ -d $CACHE_DIR && -d $CACHE_DIR/.git && $MAYBE_UPDATE_GIT_CACHE ]]; then
        cd "$CACHE_DIR"
        if ! in_cache git diff-index --cached --quiet HEAD; then
            in_cache git commit -m "Updated by build_crowbar.sh @ $(date) for ${OS_TOKEN}"
            echo "The crowbar build cache has been updated, and the updates have"
            echo "been comitted back to the cache.  Please push any changes."
        fi
    fi
    wait
    flock -u 70
    rm "$CROWBAR_DIR/".*.lock
    [[ -d $CROWBAR_TMP ]] && rm -rf "$CROWBAR_TMP"
    exit $res
} 70> "$CLEANUP_LOCK"

# Arrange for cleanup to be called at the most common exit points.
trap cleanup EXIT INT QUIT TERM
[[ $DEBUG ]] && trap -p

# Test to see if $1 is in the rest of the args.
is_in() {
    local t="(^| )$1( |\$)"
    shift
    [[ $* =~ $t ]]
}

# Run a command in our chroot environment.
in_chroot() {
    sudo -H chroot "$CHROOT" \
        /bin/bash -l -c "export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin; $*"
}

# A little helper function for doing bind mounts.
bind_mount() {
    [[ -d $2 ]] || mkdir -p "$2"
    grep -q "$2" /proc/self/mounts || sudo mount --bind "$1" "$2"
}

# Write a string with \n translated to a real line break.
write_lines() { [[ $1 ]] && printf "%b" "$1"; }

# Read base repository information from the *.list files.
read_base_repos() {
    for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
        [[ -f $pkgfile ]] || continue
        while read line; do
            line=${line%%#*}
            [[ $line ]] || continue
            if [[ $line = repository* ]]; then
                REPOS+=("${line#* }")
            else
                echo "$line in $pkgfile has been superceded by crowbar.yml metadata." >&2
                die "Please migrate $line into the appropriate barclamp config file."
            fi
        done <"$pkgfile"
    done
}

# Worker function for all_deps
__all_deps() {
    local dep
    [[ ${deps[$1]} = "unordered" ]] && die "Circular dependency of $1 depending on $1 detected!"
    [[ ${deps[$1]} ]] && return 0
    deps[$1]="unordered"
    if [[ ${BC_DEPS["$1"]} ]]; then
        for dep in ${BC_DEPS["$1"]}; do
            dep="${BC_SUPERCEDES[$dep]:-$dep}"
            [[ ${deps[$dep]} ]] && continue
            is_barclamp "$dep" || die "$1 depends on $dep, but $dep is not a barclamp!"
            __all_deps "$dep"
        done
    fi
    deps[$1]=$count
    count=$((count + 1))
    return 0
}

# Given a set of barclamps, echo all of the dependencies of that barclamp, including itself.
all_deps() {
    local -A deps
    local count=1 __deps=() d
    # Everything depends on the crowbar barclamp. Always.
    deps['crowbar']=0
    # Find our dependencies and sort them based on the values o
    for d in "$@"; do __all_deps "$d"; done
    for d in "${!deps[@]}"; do  __deps[${deps[$d]}]=$d; done
    echo "${__deps[*]}"
}

# A couple of utility functions for comparing version numbers.
num_re='^[0-9]+$'
__cmp() {
    [[ $1 || $2 ]] || return 255 # neither 1 nor 2 or set, we are done
    [[ $1 && ! $2 ]] && return 2 # 1 is set and 2 is not, 1 > 2
    [[ ! $1 && $2 ]] && return 0 # 2 is set and 1 is not, 1 < 2
    local a="$1" b="$2"
    if [[ $a =~ $num_re && $b =~ $num_re ]]; then #both numbers, numeric cmp.
        # make sure leading zeros do not confuse us
        a=${a##0} b=${b##0}
        ((${a:=0} > ${b:=0})) && return 2
        (($a < $b)) && return 0
        return 1
    else # string compare
        [[ $a > $b ]] && return 2
        [[ $a < $b ]] && return 0
        return 1
    fi
}

# Compare a version string, handling mixed numeric and alpha sequences
# as appropriate.
vercmp(){
    # $1 = version string of first package
    # $2 = version string of second package
    # Returns 0 if $1 > $2, 1 otherwise.
    # urldecode the version strings.
    local a="$(printf "%b" "${1//%/\\x}")"
    local b="$(printf "%b" "${2//%/\\x}")"
    # prepend a version epoch if one is not already there.
    [[ $a =~ ^[0-9]+: ]] || a="0:$a"
    [[ $b =~ ^[0-9]+: ]] || b="0:$b"
    local ver1=()
    local ver2=()
    local i=0
    # split the version strings into arrays using the usual field splitters
    IFS=':.-_ +' read -rs -a ver1 <<< "$a"
    IFS=':.-_ +' read -rs -a ver2 <<< "$b"
    for ((i=0;;i++)); do
        __cmp "${ver1[$i]}" "${ver2[$i]}"
        case $? in
            2) return 0;;
            0) return 1;;
            255) return 1;;
        esac
    done

}

# Index the pool of packages in the CD.
index_cd_pool() {
    # Scan through our pool to find pkgs we can easily omit.
    local pkgname='' pkg='' cache="$CACHE_DIR/$OS_TOKEN/packages-$ISO"
    if [[ $ISO_LIBRARY/$ISO -nt $cache ]]; then
        mkdir -p "${cache%/*}"
        echo 'CD_POOL_VERSION=2' > "$cache"
        while read pkg; do
            [[ -f $pkg ]] && is_pkg "$pkg" || continue
            pkgname="$(pkg_name "$pkg")"
            CD_POOL["$pkgname"]="${pkg}"
            echo "CD_POOL[\"$pkgname\"]=\"${pkg}\"" >> "$cache"
        done < <(find "$(find_cd_pool)" -type f)
        return
    fi
    . "$cache"
    [[ $CD_POOL_VERSION = 2 ]] && return
    rm "$cache"
    index_cd_pool
}

# Make a chroot environment for package-fetching purposes.
make_chroot() {
    [[ -f $CHROOT/etc/resolv.conf ]] && return 0
    local bc repo
    debug "Making utility chroot"
    if [[ $CHROOT_USE_TMPFS = true ]]; then
        sudo mount -t tmpfs -osize=3G cb_chroot "$CHROOT"
    fi
    sudo mkdir -p "$CHROOT/$CHROOT_PKGDIR"
    sudo mkdir -p "$CHROOT/$CHROOT_GEMDIR"
    sudo mkdir -p "$CHROOT/etc/profile.d"
    __make_chroot
    if [[ $http_proxy || $https_proxy || $no_proxy ]]; then
        local __f
        __f=$(mktemp /tmp/proxy-XXXXX.sh) || \
            die "Canot make utility chroot -- error adding proxies."
        [[ $http_proxy ]] && echo "http_proxy=\"$http_proxy\"" >> "$__f"
        [[ $https_proxy ]] && echo "https_proxy=\"$https_proxy\"" >> "$__f"
        [[ $no_proxy ]] && echo "no_proxy=\"$no_proxy\"" >> "$__f"
        sudo cp "$__f" "$CHROOT/etc/environment"
        [[ $http_proxy ]] && echo "export http_proxy" >> "$__f"
        [[ $https_proxy ]] && echo "export https_proxy" >> "$__f"
        [[ $no_proxy ]] && echo "export no_proxy" >> "$__f"
        sudo cp "$__f" "$CHROOT/etc/profile.d/proxy.sh"
    fi
    in_chroot ln -s /proc/self/mounts /etc/mtab

    if [[ $ALLOW_CACHE_UPDATE = true ]]; then
        read_base_repos
        # Add our basic repositories
        add_repos "${REPOS[@]}" || \
            die "Could not add base repositories for $OS_TOKEN chroot!"

        # Add the repos from the barclamps.
        # We do it here because importing the metadata takes forever
        # if we refresh on every barclamp.
        for bc in "${BARCLAMPS[@]}"; do
            while read repo; do
                add_repos "$repo"
            done < <(write_lines "${BC_REPOS[$bc]}")
        done
    fi
    add_offline_repos
    chroot_update
}

cache_add() {
    # $1 = file to add.
    # $2 = location to store it in the cache
    sudo cp "$1" "$2" || \
        die "Cannot save $1 in $2!"
    if [[ $CURRENT_CACHE_BRANCH ]]; then
        CACHE_NEEDS_COMMIT=true
        in_cache git add "${2#${CACHE_DIR}/}"
    fi
}

cache_rm() {
    if [[ $CURRENT_CACHE_BRANCH ]]; then
        CACHE_NEEDS_COMMIT=true
        in_cache git rm -f "${1#${CACHE_DIR}/}"
    fi
    sudo rm -f "$1"
}

make_barclamp_pkg_metadata() {
    [[ $ALLOW_CACHE_UPDATE != true && \
        $ALLOW_CACHE_METADATA_UPDATE != true ]] && return 0
    [[ -d $CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs ]] || return 0
    if [[ $force_update != true ]]; then
        __barclamp_pkg_metadata_needs_update "$1" || return 0
    fi
    [[ $ALLOW_CACHE_METADATA_UPDATE = false ]] && \
        die "Need to update cache metadata for $1, but --no-metadata-update passed."
    debug "Updating package cache metadata for $1"
    make_chroot
    [[ $OS_METADATA_PKGS ]] && {
        chroot_install $OS_METADATA_PKGS
        unset OS_METADATA_PKGS
    }
    sudo mount --bind "$(readlink -f "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs")" \
        "$CHROOT/mnt"
    __make_barclamp_pkg_metadata "$1"
    sudo umount "$CHROOT/mnt"
    unset CACHED_PACKAGES[$1]
    barclamp_pkg_cache_needs_update "$1"
}

install_build_packages() {
    chroot_install ${BC_BUILD_PKGS["$1"]}
}

update_barclamp_src_pkg_cache() {
    [[ ${BC_SRC_PKGS["$1"]} ]] || return 0
    local CHROOT_PKGDIR="tmp/$1"
    in_chroot mkdir -p "/$CHROOT_PKGDIR"
    chroot_fetch_source ${BC_SRC_PKGS["$1"]}
    local p
    for p in "$CHROOT/$CHROOT_PKGDIR/"*; do
        [[ -f $p ]] || continue
        cache_add "$p" "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/${p##*/}"
    done
}

# This function manages the target pkg directory structure
copy_bc_packages() {
    is_pkg "$CHROOT/$CHROOT_PKGDIR/$pkg" || continue
    [[ ${pkgs["$pkg"]} = true ]] && continue
    if [[ ${pkg%/*} != '.' ]]; then
        [[ -d $bc_cache/${pkg%/*} ]] || sudo mkdir -p "$bc_cache/${pkg%/*}"
        [[ -f $bc_cache/${pkg##*/} ]] && cache_rm "$bc_cache/${pkg##*/}"
    fi
    cache_add "$CHROOT/$CHROOT_PKGDIR/$pkg" "$bc_cache/${pkg//%3a/:}"
}


# Update the package cache for a barclamp.
update_barclamp_pkg_cache() {
    # $1 = barclamp we are working with
    local bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" pkg dest bc
    local -A pkgs
    # Wipe out the packages already in the chroot package directory.
    while read pkg; do
        is_pkg "$pkg" && sudo rm -f "$pkg"
    done < <(find "$CHROOT/$CHROOT_PKGDIR" -type f)
    for bc in $(all_deps "$1"); do
        [[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] || continue
        sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
            "$CHROOT/$CHROOT_PKGDIR"
    done
    # Remember what packages we already have.
    while read pkg; do
        is_pkg "$pkg" || continue
        pkgs["$pkg"]="true"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    [[ ${BC_BUILD_PKGS["$1"]} ]] && install_build_packages "$1"
    chroot_fetch ${BC_PKGS["$1"]} || \
        die "Could not fetch packages required by barclamp $1"
    mkdir -p "$bc_cache"
    while read pkg; do
        copy_bc_packages
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    local force_update=true
    update_barclamp_src_pkg_cache "$1"
    make_barclamp_pkg_metadata "$1"
}

# Fetch all of a gem's dependencies, followed by the gem itself.
__fetch_gem() {
    # $1 = gem to fetch.
    # $2 = (optional) version string
    if [[ ${fetched_gems["$1 $2"]} ]]; then
        debug "Gem ${fetched_gems["$1 $2"]} already fetched to satisfy $1 $2"
        return
    fi
    local gemver_re='(.+) \(([^)]*)\)'
    local our_versions=() v=()
    local -A gem_versions
    local fetch_gems=()
    local in_our_gem=false line gemname i our_gemname
    IFS=',' read -a v <<< "${2}"
    # Translate into something we can pass to our gem wrapper.
    for i in "${v[@]}"; do
        our_versions+=(--version "$i")
    done
    # Download the gem that best matches the constraints
    our_gemname=$("$CROWBAR_DIR/fetch_all_gems.rb" fetch "$1" "${our_versions[@]}")
    our_gemname=${our_gemname##*Downloaded }
    # Figure out the version we downloaded
    [[ $our_gemname =~ $GEM_RE ]] || die "Could not find version of downloaded gem!"
    fetched_gems["$1 $2"]="$our_gemname"
    # and get the dependencies for gems that match these constraints.
    our_versions=(--version "${BASH_REMATCH[2]}")
    while read line; do
        case $line in
            Gem*)
                in_our_gem=false
                gemname=${line#Gem }
                [[ $gemname = $our_gemname ]] || continue
                in_our_gem=true;;
            '') [[ $in_our_gem = true ]] && break
                gemname='';;
            *) [[ $in_our_gem = true && \
                $line =~ $gemver_re && \
                ${BASH_REMATCH[2]} != *development* ]] || continue
                fetch_gems+=("${BASH_REMATCH[1]}")
                gem_versions["${BASH_REMATCH[1]}"]="${BASH_REMATCH[2]}";;
        esac
    done < <("$CROWBAR_DIR/fetch_all_gems.rb" dependency "$1" --remote "${our_versions[@]}")
    for gemname in "${fetch_gems[@]}"; do
        [[ ${fetched_gems["$gemname ${gem_versions[$gemname]}"]} ]] && continue
        debug "Fetching $gemname ${gem_versions[$gemname]} as a dependency of $1"
        __fetch_gem "$gemname" "${gem_versions[$gemname]}"
    done
}

# Update the gem cache for a barclamp
update_barclamp_gem_cache() {
    local -A gems
    local gemname gemver gemopts bc gem
    local bc_cache="$CACHE_DIR/barclamps/$1/gems"
    which gem &>/dev/null || die "Please install rubygems before updating the gem cache!"
    local gemdir="$CROWBAR_TMP/gems/$1"
    mkdir -p "$gemdir"

    # Stage prerequisite gems first
    for bc in $(all_deps "$1"); do
        [[ -d "$CACHE_DIR/barclamps/$bc/gems/." ]] && \
            cp -a "$CACHE_DIR/barclamps/$bc/gems/." \
            "$gemdir"
    done

    # Remember what we already have.
    while read gem; do
        gems["${gem##*/}"]="true"
    done < <(find "$gemdir" -type f -name '*.gem')

    (   cd "$gemdir"
        local -A fetched_gems
        for gem in ${BC_GEMS["$1"]}; do
            debug "Fetching top-level gem $gem"
            if [[ $gem =~ $GEM_RE ]]; then
                __fetch_gem "${BASH_REMATCH[1]}" "= ${BASH_REMATCH[2]}"
            elif [[ $gem =~ $GEM_EXT_RE ]]; then
                __fetch_gem "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
            else
                __fetch_gem "$gem" ">= 0"
            fi
        done
    ) || exit 1
    mkdir -p "$bc_cache" || die "Failed to make gem cache ${bc_cache}"
    while read gem; do
        [[ ${gems["${gem##*/}"]} = "true" ]] && continue
        cache_add "$gem" "$bc_cache"
    done < <(find "$gemdir" -type f -name '*.gem')
}

# Fetch any raw packages we do not already have.
update_barclamp_raw_pkg_cache() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    mkdir -p "$bc_cache"
    # Fetch any raw_pkgs we were asked to.
    for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
        [[ -f $bc_cache/${pkg##*/} ]] && continue
        echo "Caching $pkg:"
        curl -L -o "$bc_cache/${pkg##*/}" "$pkg"
        [[ $CURRENT_CACHE_BRANCH ]] && in_cache git add "$bc_cache/${pkg##*/}"
    done
    touch "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    make_barclamp_pkg_metadata "$1"
}

# Fetch any bare files that we do not already have.
update_barclamp_file_cache() {
    local dest pkg bc_cache="$CACHE_DIR/barclamps/$1/files"
    # Fetch any extra_pkgs we need.
    mkdir -p "$bc_cache"
    while read pkg; do
        dest=${pkg#* }
        [[ $dest = $pkg ]] && dest=''
        pkg=${pkg%% *}
        [[ -f $bc_cache/files/$dest/${pkg##*/} ]] && continue
        mkdir -p "$bc_cache/$dest"
        echo "Caching $pkg:"
        curl -L -o "$bc_cache/$dest/${pkg##*/}" "$pkg"
        [[ $CURRENT_CACHE_BRANCH ]] && \
            in_cache git add "$bc_cache/$dest/${pkg##*/}"
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
}

# See if there are any package caches that might need updating.
any_pkg_cache() { [[ ${BC_PKGS[*]} ]]; }

# Check to see if the barclamp package cache needs update.
barclamp_pkg_cache_needs_update() {
    local pkg pkgname arch bc ret=1
    [[ ${BC_PKGS["$1"]} || ${BC_BUILD_PKGS["$1"]} ]] || return 1
    [[ $need_update = true || ${FORCE_BARCLAMP_UPDATE["$1"]} = true ]] && return 0
    local -A parent_pkgs present_pkgs
    # Build a list of all the packages the barclamps we depend on should provide.
    for bc in $(all_deps "$1"); do
        [[ ${CACHED_PACKAGES[$bc]} ]] || continue
        for pkg in ${CACHED_PACKAGES[$bc]}; do
            [[ ${parent_pkgs[$pkg]} ]] && continue
            parent_pkgs[$pkg]="$bc"
        done
    done
    # Get the list of packages we have.
    while read pkg; do
        is_pkg "$pkg" || continue
        present_pkgs[$(pkg_name "$pkg")]="$1"
    done < <(find "$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs")
    for pkg in ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]}; do
        [[ $pkg ]] || continue
        for arch in "${PKG_ALLOWED_ARCHES[@]}"; do
            [[ ${parent_pkgs["$pkg.$arch"]} || \
                ${present_pkgs["$pkg.$arch"]} || \
                ${CD_POOL["$pkg.$arch"]} ]] && continue 2
        done
        debug "Package $pkg is not cached, and $1 needs it."
        ret=0
    done
    CACHED_PACKAGES["$1"]="${!present_pkgs[*]}"
    return $ret
}

any_gem_cache() { [[ ${BC_GEMS[*]} ]]; }

# Check to see if the barclamp gem cache needs an update.
barclamp_gem_cache_needs_update() {
    local pkg pkgname bc ret=1
    local -A pkgs
    # Second, check to see if we have all the gems we need.
    for pkg in ${BC_GEMS["$1"]}; do
        if [[ $pkg =~ $GEM_RE ]]; then
            local finder="$pkg.gem"
        elif [[ $pkg =~ $GEM_EXT_RE ]]; then
            local finder="${BASH_REMATCH[1]}*.gem"
        else
            local finder="$pkg*.gem"
        fi
        for bc in $(all_deps "$1"); do
            local bc_cache="$CACHE_DIR/barclamps/$bc/gems"
            mkdir -p "$bc_cache"
            [[ $(find "$bc_cache" \
                -name "$finder" -type f) = *.gem ]] && continue 2
        done
        debug "Gem $pkg is not cached, and $1 needs it."
        ret=0
    done
    return $ret
}

any_raw_pkg_cache() { [[ ${BC_RAW_PKGS[*]} ]]; }

# CHeck to see if we are missing any raw packages.
barclamp_raw_pkg_cache_needs_update() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" ret=1
    [[ ${BC_RAW_PKGS["$1"]} || ${BC_PKG_SOURCES["$1"]} ]] || return 1
    mkdir -p "$bc_cache"
    # Third, check to see if we have all the raw_pkgs we need.
    for pkg in ${BC_RAW_PKGS["$1"]} ${BC_PKG_SOURCES["$1"]}; do
        [[ -f $bc_cache/${pkg##*/} ]] && continue
        debug "Raw package $pkg is not cached, and $1 needs it."
        ret=0
    done
    return $ret
}

any_file_cache() { [[ ${BC_EXTRA_FILES[*]} ]]; }

# Check to see if we are missing any raw files.
barclamp_file_cache_needs_update() {
    local pkg dest bc_cache="$CACHE_DIR/barclamps/$1/files" ret=1
    [[ "${BC_EXTRA_FILES[$1]}" ]] || return 1
    mkdir -p "$bc_cache"
    # Fourth, check to make sure we have all the extra_pkgs we need.
    while read pkg; do
        dest=${pkg#* }
        [[ $dest = $pkg ]] && dest=''
        pkg=${pkg%% *}
        [[ -f $bc_cache/$dest/${pkg##*/} ]] && continue
        debug "File $pkg is not cached, and $1 needs it."
        ret=0
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
    return $ret
}

any_git_repo_cache() { [[ ${BC_GIT_REPOS[*]} && $USE_PFS = true ]]; }

barclamp_git_repo_cache_needs_update() {
    local dest bc_cache="$CACHE_DIR/barclamps/$1/git_repos"
    [[ ${BC_GIT_REPOS[$1]} && $USE_PFS = true ]] || return 1
}

update_barclamp_git_repo_cache() {
    local repo_name repo_url repo_branches bc_cache="$CACHE_DIR/barclamps/$1/git_repos"
    mkdir -p "$bc_cache"
    while read repo_name repo_url repo_branches; do
        [[ $repo_branches ]] || repo_branches=master
        if [[ -f $bc_cache/$repo_name.tar.bz2 ]]; then
            [[ $UPDATE_GIT_REPOS ]] || continue
        elif [[ ! $UPDATE_GIT_REPOS ]]; then
            debug "Git repo $repo_name is not cached, and $1 needs it."
            debug "Please retry the build with --update-pfs-caches"
            return 0
        fi
        (   cd "$bc_cache"
            if [[ -f $repo_name.tar.bz2 ]]; then
                tar xf "$repo_name.tar.bz2"
                debug "Updating git clone of $repo_name"
                (cd "$repo_name.git"; git fetch origin)
                rm -f "$repo_name.tar.bz2"
            else
                debug "Performing initial clone of ${repo_name} from ${repo_url}"
                git clone --mirror "$repo_url" "$repo_name.git" || \
                    die "Could not perform initial git clone of $repo_name!"
            fi
            tar cjf "$repo_name.tar.bz2" "$repo_name.git"
            [[ $CURRENT_CACHE_BRANCH ]] && git add "$repo_name.tar.bz2"
            rm -rf "$repo_name.git" ) || exit 1
    done < <(write_lines "${BC_GIT_REPOS[$1]}")
    return 1
}



# Some helper functions

log() { printf "$(date '+%F %T %z'): %s\n" "$@" >&2; }

warn() { log "$@"; }

# Print a message to stderr and exit.  cleanup will be called.
die() { log "$@" >&2; res=1; exit 1; }

# Print a message to stderr and keep going.
debug() { [[ $VERBOSE ]] && log "$@"; }

# Clean up any cruft that we might have left behind from the last run.
clean_dirs() {
    local d=''
    for d in "$@"; do
        (   mkdir -p "$d"
            cd "$d"
            sudo rm -rf --one-file-system * )
    done
}

# Verify that the passed name is really a branch in the git repo.
branch_exists() { git show-ref --quiet --verify --heads -- "refs/heads/$1"; }

to_empty_branch() {
    if branch_exists empty-branch; then
        git checkout -q empty-branch
        return $?
    fi
    if [[ -d .git ]]; then
        git symbolic-ref HEAD refs/heads/empty-branch
        rm -f .git/index
    elif [[ -f .git ]]; then
        git checkout --orphan empty-branch
        git rm -r --cached .
    fi
    git clean -f -x -d
    echo "This branch intentionally left blank" >README.empty-branch
    git add README.empty-branch
    git commit -m "Created empty branch"
}

# Run a git command in the crowbar repo.
in_repo() ( cd "$CROWBAR_DIR"; "$@")

# Get the head revision of a git repository.
get_rev() (
    cd "$1"
    if [[ -d .git || -f .git ]]; then
        git rev-parse HEAD
    else
        echo "Not a Git Repository"
    fi
)

# Check out a branch, but be quiet about it unless something goes wrong.
quiet_checkout() {
    local res=''
    res=$(git checkout -q "$@" 2>&1) && return
    echo "$res" >&2
    return 1
}

# Get thecurrent HEAD.  Prefer it in human-readable format
# Assumes that $PWD is in the git repo you care about.
get_current_head() {
    local head=$(git symbolic-ref HEAD)
    if [[ $head != refs/heads/* ]]; then
        head=$(git rev-parse HEAD)
        if [[ ! $head ]]; then
            debug "Barclamp ${repo##*/}: Cannot find head commit."
            return 1
        fi
    else
        head=${head#refs/heads/}
    fi
    printf "%s" "$head"
}

# Run a git command in the build cache, assuming it is a git repository.
in_cache() (
    cd "$CACHE_DIR"
    "$@"
)

# Check to see if something is a barclamp.
is_barclamp() [[ -f "$CROWBAR_DIR/barclamps/$1/crowbar.yml" ]]
in_barclamp() {
    (   cd "$CROWBAR_DIR/barclamps/$1"
        shift
        "$@")
}

# Build our ISO image.
build_iso() (
    cd "$BUILD_DIR"
    rm -f isolinux/boot.cat
    find -name '.svn' -type d -exec rm -rf '{}' ';' 2>/dev/null >/dev/null
    find . -type f -not -name isolinux.bin -not -name sha1sums \
        -not -name crowbar.json -not -path '*/.git/*' -print0 | \
        xargs -0 -- sha1sum -b >sha1sums
    mkdir -p "$ISO_DEST"
        # Save the sha1sums and the build-info files along side the iso.
    cp sha1sums build-info "$ISO_DEST"
    if ! [[ $NO_GENERATE_ISO && $NO_GENERATE_ISO = true ]]; then
        mkisofs -r -V "${VERSION:0:30}" -cache-inodes -J -l -quiet \
            -b isolinux/isolinux.bin -c isolinux/boot.cat -joliet-long \
            -no-emul-boot --boot-load-size 4 -boot-info-table \
            -o "$ISO_DEST/$BUILT_ISO" "$IMAGE_DIR" "$BUILD_DIR"
    fi
)

# Have the smoketest framework do its thing with the ISO we just made.
test_iso() {
    run_test "$@" || \
        die "$(date '+%F %T %z'): Smoketest of $ISO_DEST/$BUILT_ISO failed."
}

get_repo_cfg() { in_repo git config --get "$1"; }
git_config_has() { git config --get "$1" &>/dev/null; }
current_build() { get_repo_cfg 'crowbar.build'; }

# Get the current release we are working on, which is a function of
# the currently checked-out branch.
current_release() {
    local rel
    rel=$(current_build) || \
        die "current_release: Cannot get current build information!"
    echo "${rel%/*}"
}

# Source the appropriate metadata helper functions.
if git_tracked_checkout; then
    . "$CROWBAR_DIR/git_tracked_metadata.sh"
elif flat_checkout; then
    . "$CROWBAR_DIR/flat_metadata.sh"
fi

# Test to see if a barclamp is part of a specific build.
barclamp_exists_in_build() {
    __barclamp_exists_in_build "$1" && return 0
    local build=${1%/*} bc=${1##*/} parent
    parent=$(parent_build "$build")|| return 1
    barclamp_exists_in_build "$parent"
}

# Given a build, give us the branch the barclamps will use.
build_branch() {
    # $1 = build
    case $1 in
        development/*) echo "master" ;;
        stable/*) echo "stable/${1%/*}/master";;
        feature/*/*) echo "${1%/*}/master";;
        *) echo "release/${1%/*}/master";;
    esac
}

# Given a branch, figure out what release it is for
release_for_branch() {
    # $1 = branch name
    local r
    case $1 in
        master) echo "development";;
        feature/*) echo "${1%/*}";;
        release/*) r="${1#release/}"; echo "${r%/*}";;
    esac
}

# Get or set the proper branch for a barclamp for a build.
barclamp_branch_for_build() {
    # $1 = build
    # $2 = barclamp
    # $3 = (optional) ref to pin the barclamp at.
    local build=$1 bcfile
    while [[ true ]]; do
        __barclamp_exists_in_build "$build/$2" && break
        build=$(parent_build "$build") || break
    done
    if [[ ! $3 ]]; then
        if [[ $build ]]; then
            get_barclamp_branch_for_build "$build" "$2"
        else
            echo "empty-branch"
        fi
        return 0
    elif [[ $build ]]; then
        in_barclamp "$2" git rev-parse --verify --quiet "$3" &>/dev/null || return 1
        set_barclamp_branch_for_build "$build" "$2" "$3"
    else
        return 1
    fi
}

# Given a build, see what barclamps will go into it.
barclamps_in_build() {
    local build bc p
    build="${1:-$(current_build)}"
    p="$(parent_build "$build")"
    [[ $p ]] && barclamps_in_build "$p"
    barclamps_from_build "$build"
}

# Given a release, find all the builds that are in it.
builds_in_release() {
    local release="${1:-$(current_release)}" p build b
    local -A builds
    release_exists "$release" || return 1
    while read build; do
        build_exists "$release/$build" || continue
        p=$(parent_build "$release/$build")
        if [[ $p && ${builds[$p]} != echoed  ]]; then
            builds["$release/$build"]="$p"
        else
            echo "$build"
            builds["$release/$build"]="echoed"
        fi
    done < <(__builds_in_release "$release")
    while [[ true ]]; do
        b=true
        for build in "${!builds[@]}"; do
            p="${builds[$build]}"
            [[ $p = echoed || ${builds[$p]} != echoed ]] && continue
            echo "${build##*/}"
            builds[$build]=echoed
            b=false
        done
        [[ $b = true ]] && break
    done
}

crowbar_version() {
    local bc build br
    build=${1:-$(current_build)}
    commits=0
    build_exists "$build" || die "$build is not a build!"
    for bc in $(barclamps_in_build); do
        br="$(barclamp_branch_for_build "$build" "$bc")"
        commits=$((commits + $(in_barclamp "$bc" git rev-list --count --sparse --no-merges "$br")))
    done
    echo "${build//\//_}.$commits"
}

all_supported_oses() {
    local os
    for os in "$CROWBAR_DIR/"*-extra; do
        [[ -f $os/build_lib.sh ]] || continue
        os=${os##*/}
        echo "${os%-extra}"
    done
}

iso_boot_prep() {
    # This has been split out so that distros that use varying boot file
    # layouts can over-ride this function.
    mkdir -p "$BUILD_DIR/isolinux"
    chmod u+wr "$BUILD_DIR/isolinux"
    rsync -rl --ignore-existing --inplace \
        "$IMAGE_DIR/isolinux" "$BUILD_DIR"
    chmod -R u+wr "$BUILD_DIR/isolinux"
    sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/isolinux"
}

do_crowbar_build() {
    # Make sure only one instance of the ISO build runs at a time.
    # Otherwise you can easily end up with a corrupted image.
    # Check and see if our local build repository is a git repo. If it is,
    # we may need to do the same sort of merging in it that we might do in the
    # Crowbar repository.
    if [[ -d $CACHE_DIR/.git ]] && \
        (cd "$CACHE_DIR"; branch_exists master) then
        CURRENT_CACHE_BRANCH=master
    fi

    if [[ $BRANCHES_TO_MERGE ]]; then
        THROWAWAY_BRANCH="build-throwaway-$$-$RANDOM"
        REPO_PWD="$PWD"
        in_repo git checkout -b "$THROWAWAY_BRANCH"
        for br in "${BRANCHES_TO_MERGE[@]}"; do

            # Merge the requested branch into the throwaway branch.
            # Die if the merge failed -- there must have been a
            # conflict, and the user needs to fix it up.
            in_repo git merge "$1" || \
                die "Merge of $1 failed, fix things up and continue"
        done
        unset br
    fi
    # Finalize where we expect to find our caches and out chroot.
    # If they were set in one of the conf files, don't touch them.

    # The directory we perform a minimal install into if we need
    # to refresh our gem or pkg caches
    [[ $CHROOT ]] || CHROOT="$CACHE_DIR/$OS_TOKEN/chroot"

    # Make sure that the $OS_TOKEN directory exist.
    mkdir -p "$CACHE_DIR/$OS_TOKEN"

    # The directory we will stage the build into.
    [[ $BUILD_DIR ]] || \
        BUILD_DIR="$CACHE_DIR/$OS_TOKEN/build"
    # The directory that we will mount the OS .ISO on .
    [[ $IMAGE_DIR ]] || \
        IMAGE_DIR="$CACHE_DIR/$OS_TOKEN/image"

    # Directory where we will look for our package lists
    [[ $PACKAGE_LISTS ]] || PACKAGE_LISTS="$BUILD_DIR/extra/packages"

    # Proxy Variables
    [[ $USE_PROXY ]] || USE_PROXY=0
    [[ $PROXY_HOST ]] || PROXY_HOST=""
    [[ $PROXY_PORT ]] || PROXY_PORT=""
    [[ $PROXY_USER ]] || PROXY_USER=""
    [[ $PROXY_ESC_USER ]] || PROXY_ESC_USER=""
    [[ $PROXY_PASSWORD ]] || PROXY_PASSWORD=""

    # Version for ISO
    [[ $VERSION ]] || VERSION="$(crowbar_version).dev"

    # Name of the built iso we will build
    [[ $BUILT_ISO ]] || BUILT_ISO="crowbar-${VERSION}-${OS}-${OS_VERSION}.iso"

    # If we were not passed a list of barclamps to include,
    # pull in all of the ones declared as submodules.
    [[ $BARCLAMPS ]] || BARCLAMPS=($(barclamps_in_build))
    # Pull in barclamp information
    get_barclamp_info

    # Make any directories we don't already have
    for d in "$ISO_LIBRARY" "$ISO_DEST" "$IMAGE_DIR" "$BUILD_DIR" \
        "$SLEDGEHAMMER_PXE_DIR" "$CHROOT"; do
        mkdir -p "$d"
    done

    debug "Checking for Sledgehammer."
    # Make sure Sledgehammer has already been built and pre-staged.
    if ! [[ -f $SLEDGEHAMMER_PXE_DIR/initrd0.img ]]; then
        debug "Slegehammer TFTP image missing from ${SLEDGEHAMMER_PXE_DIR} !"
        debug "Attempting to build Sledgehammer:"
        "$CROWBAR_DIR/build_sledgehammer.sh" || \
            die "Unable to build Sledgehammer. Cannot build Crowbar."
    else
        echo "Sledgehammer initrd.0 file found in ${SLEDGEHAMMER_PXE_DIR}"
    fi

    # Fetch the OS ISO if we need to.
    [[ -f $ISO_LIBRARY/$ISO ]] || fetch_os_iso

    # Start with a clean slate.
    clean_dirs "$IMAGE_DIR" "$BUILD_DIR" "$CHROOT"

    if [[ $NO_CLEAN_DIRS = true ]]; then
        debug "Skipping clean by user request."
    else
        debug "Cleaning up any VCS cruft."
        # Clean up any cruft that the editor may have left behind.
        (for d in "$CROWBAR_DIR" "$CROWBAR_DIR/barclamps/"*; do
            cd "$d"; $VCS_CLEAN_CMD
            done)
    fi

    # Make additional directories we will need.
    for d in discovery extra/pkgs extra/files doc/framework; do
        mkdir -p "$BUILD_DIR/$d"
    done

    # Mount our ISO for the build process.
    debug "Mounting $ISO"
    sudo mount -t iso9660 -o loop "$ISO_LIBRARY/$ISO" "$IMAGE_DIR" || \
        die "Could not mount $ISO"
    debug "Indexing CD package pool."
    index_cd_pool

    # Copy over the Crowbar bits and their prerequisites
    # The ordering here is designed to always have the most specific
    # version of the file for a build wind up staged on the DVD, without
    # having to walk the whole directory tree.
    d="$(build_cfg_dir)" && [[ -d $d/extra && -d $d/change-image ]] || \
        die "Cannot find extra and change-image directories for $(current_build)!"
    cp -r "$d/extra"/* "$BUILD_DIR/extra"
    cp -r "$CROWBAR_DIR/doc"/* "$BUILD_DIR/doc/framework"
    cp -r "$d/change-image/"* "$BUILD_DIR"
    for d in "$OS-common" "$OS_TOKEN-extra"; do
        [[ -d $CROWBAR_DIR/$d ]] || continue
        cp -r "$CROWBAR_DIR/$d"/* "$BUILD_DIR/extra"
    done

    # Add critical build meta information to build-info
    echo "build-timestamp: $(date '+%F %T %z')" > "$BUILD_DIR/build-info"
    echo "build-os: $OS_TOKEN" >>"$BUILD_DIR/build-info"
    echo "build-os-iso: $ISO" >>"$BUILD_DIR/build-info"
    echo "crowbar: $(get_rev "$CROWBAR_DIR")" >>"$BUILD_DIR/build-info"

    # Make sure we have all the barclamps we need.
    for bc in "${BARCLAMPS[@]}"; do
        is_barclamp "$bc" || die "Cannot find barclamp $bc!"
    done
    # Make sure that all our barclamps are properly staged.
    for cache in pkg gem raw_pkg file git_repo; do
        skipper="any_${cache}_cache"
        checker="barclamp_${cache}_cache_needs_update"
        updater="update_barclamp_${cache}_cache"
        [[ $(type $skipper) = "$skipper is a function"* ]] || \
            die "Cannot see if we will need to skip $cache update tests!"
        [[ $(type $checker) = "$checker is a function"* ]] || \
            die "Asked to check $cache cache, but no checker function!"
        [[ $(type $updater) = "$updater is a function"* ]] || \
            die "Might need to update $cache cache, but no updater!"
        $skipper || continue
        debug "Staging $cache cache files"
        for bc in "${BARCLAMPS[@]}"; do
            printf "\e[0G\e[2K%s" "$bc"
            $checker "$bc" || continue
            echo
            [[ $ALLOW_CACHE_UPDATE = true || $cache = git_repo ]] || {
                echo "Need up update $cache cache for $bc, but updates are disabled."
                echo "Please rerun the build with the --update-cache option."
                exit 1
            } >&2
            debug "Updating $cache cache for $bc"
            [[ $cache =~ ^(pkg|gem)$ ]] && make_chroot
            MAYBE_UPDATE_GIT_CACHE=true
            $updater "$bc"
        done
    done
    # Handle building any requests if we call for a custom build
    for bc in "${BARCLAMPS[@]}"; do
        [[ ${BC_BUILD_CMDS["$bc"]} ]] || continue
        [[ -x $CROWBAR_DIR/barclamps/$bc/${BC_BUILD_CMDS["$bc"]%% *} ]] || \
            die "Asked to do a custom build for $bc, but build script ${BC_BUILD_CMDS["$bc"]%% *} not found!"
        # Make sure the actual build runs in a subshell.  This prevents
        # cross-barclamp namespace collisions.
        (   export BC_DIR=$CROWBAR_DIR/barclamps/$bc
            export BC_CACHE=$CACHE_DIR/barclamps/$bc
            res=0
            . "$BC_DIR"/${BC_BUILD_CMDS["$bc"]}
            if bc_needs_build; then
                # Make sure we have a chroot set up and that it is ready
                # to do whatever bc_build needs.
                make_chroot
                bind_mount "$CACHE_DIR/barclamps/$bc" "$CHROOT/mnt"
                install_build_packages "$bc"
                in_chroot ln -s /mnt/$OS_TOKEN /mnt/current_os
                bc_build || res=1
                in_chroot rm -f /mnt/current_os
                sudo umount "$CHROOT/mnt"
            fi
            exit $res
        ) || die "External builder for $bc failed"
        echo "barclamps/$bc: $(get_rev "$CROWBAR_DIR/barclamps/$bc")" >> "$BUILD_DIR/build-info"
    done
    # Once all our barclamps have had their packages staged, create tarballs of them.
    mkdir -p "$BUILD_DIR/dell/barclamps"
    package_opts=(--destdir "$BUILD_DIR/dell/barclamps" --os "$OS_TOKEN")
    case $BC_PKG_TYPE in
        tar) : ;;
        deb) package_opts+=("--deb");;
        rpm) package_opts+=("--rpm");;
    esac
    "$CROWBAR_DIR/package_barclamp.sh" "${package_opts[@]}" "${BARCLAMPS[@]}" || \
        die "Could not package $bc into a $BC_PKG_TYPE"

    [[ $NO_GENERATE_ISO = true ]] && return 0

    if [[ $ALLOW_CACHE_UPDATE != true && $CURRENT_CACHE_BRANCH ]]; then
        echo "build-cache: $(get_rev "$CACHE_DIR")" >> "$BUILD_DIR/build-info"
    fi

    (cd "$BUILD_DIR"
        find extra dell -type f -print | \
            sort >> "build-info")
    # Make sure we still provide the legacy ami location
    (cd "$BUILD_DIR"; ln -sf extra/files/ami)
    # Store off the version
    echo "$VERSION" >> "$BUILD_DIR/dell/Version"

    # Custom start-up in place
    BUILD_CFG_DIR="$(build_cfg_dir)" && {
        for f in "$BUILD_CFG_DIR"/*.json ; do
            [[ -f $f ]] || continue
            mkdir -p "$BUILD_DIR/extra/config"
            cp "$f" "$BUILD_DIR/extra/config"
        done
    }

    final_build_fixups

    # Copy over the bits that Sledgehammer will look for.
    debug "Copying over Sledgehammer bits"
    cp -a "$SLEDGEHAMMER_PXE_DIR"/* "$BUILD_DIR/discovery"

    # Make our image
    debug "Creating new ISO"
    # mkisofs can merge multiple directory trees into a single iso
    # file system.  However, these trees must be disjoint -- if
    # any of the filesystem trees collide, mkisofs dies.
    # To work around that, we merge any colliding top-level trees
    # and take care to prefer stuff from $BUILD_DIR, and then
    # bind-mount empty trees on top of the colliding trees in $IMAGE_DIR
    for d in $(cat <(cd "$BUILD_DIR"; find -maxdepth 1 -type d) \
        <(cd "$IMAGE_DIR"; find -maxdepth 1 -type d) | \
        sort |uniq -d); do
        [[ $d = . ]] && continue
        d=${d#./}
        # Copy contents of the found directories into $BUILD_DIR, taking care
        # to not clobber existing files.
        mkdir -p "$BUILD_DIR/$d"
        chmod u+wr "$BUILD_DIR/$d"
        # We could also use cp -n, but rhel5 and centos5 do not understand it.
        rsync -rl --ignore-existing --inplace "$IMAGE_DIR/$d" "$BUILD_DIR"
        chmod -R u+wr "$BUILD_DIR/$d"
        # Bind mount an empty directory on the $IMAGE_DIR instance.
        sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/$d"
    done

    iso_boot_prep

    [[ $SHRINK_ISO && ! $GENERATE_MINIMAL_ISO ]] && shrink_iso
    # Make a file list and a link list.
    ( cd $BUILD_DIR
        find . -type f | \
            sort > crowbar_files.list
        find . -type l | \
            xargs ls -ld | \
            awk '{ print $8 " " $10 }' | \
            sort > crowbar_links.list
    )
    ( cd $IMAGE_DIR
        find . -type f | \
            sort >> $BUILD_DIR/crowbar_files.list
        find . -type l | \
            xargs ls -ld | \
            awk '{ print $8 " " $10 }' | \
            sort >> $BUILD_DIR/crowbar_links.list
    )

    # Make an ISO
    build_iso || die "There was a problem building our ISO."
    if [[ $GENERATE_MINIMAL_INSTALL = true ]]; then
        if [[ ! -f "$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install" ]]; then
            if [[ ! -f "$HOME/admin-installed.list" ]]; then
                SMOKETEST_ISO="$ISO_DEST/$BUILT_ISO"
                test_iso admin-only
            fi
            [[ -f "$HOME/admin-installed.list" ]] || \
                die "Could not generate minimal install list!"
            mv "$HOME/admin-installed.list" \
                "$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install"
            debug "Minimal install generated and saved to $CROWBAR_DIR/$OS_TOKEN-extra/minimal-install."
            debug "Please commit it and rerun the build with --shrink."
        fi
    fi
    echo "$(date '+%F %T %z'): Image at $ISO_DEST/$BUILT_ISO"
    if [[ $NEED_TEST = true ]]; then
        echo "$(date '+%F %T %z'): Testing new iso"
        SMOKETEST_ISO="$ISO_DEST/$BUILT_ISO"
        test_iso "${test_params[@]}" && \
            echo "$(date '+%F %T %z'): Test passed" || \
            die "Test failed."
    fi
    echo "$(date '+%F %T %z'): Finished."
}

if [[ $http_proxy ]]; then
    export USE_PROXY=1
    raw_proxy="${http_proxy#*://}"
    raw_proxy="${raw_proxy%/}"
    proxy_re='^(.+):([0-9]+)$'
    hostsplit_re='(.*)@(.*)'
    userpass_re='(.*):(.*)'
    if [[ $raw_proxy =~ $proxy_re ]]; then
        export PROXY_PORT="${BASH_REMATCH[2]}"
        raw_proxy="${BASH_REMATCH[1]}"
    fi
    if [[ $raw_proxy =~ $hostsplit_re ]]; then
        raw_proxy="${BASH_REMATCH[2]}"
        export PROXY_USER="${BASH_REMATCH[1]}"
        if [[ ${BASH_REMATCH[1]} =~ $userpass_re ]]; then
            export PROXY_PASSWORD="${BASH_REMATCH[2]}"
            export PROXY_USER="${BASH_REMATCH[1]}"
        fi
    fi
    export PROXY_HOST="$raw_proxy"
    [[ $no_proxy ]] || no_proxy="localhost,localhost.localdomain,127.0.0.0/8,$PROXY_HOST"
else
    unset USE_PROXY PROXY_HOST PROXY_PORT PROXY_USER PROXY_PASS
fi

export PATH="$PATH:$CROWBAR_DIR:$CROWBAR_DIR/extra:$CROWBAR_DIR/change-image/dell:$CROWBAR_DIR/test_framework"

CROWBAR_BUILD_SOURCED=true