Merge upstream

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @heroku/languages

.github/workflows/check_changelog.yml

@@ -2,16 +2,19 @@ name: Check Changelog
 
 on:
   pull_request:
-    types: [opened, reopened, edited, synchronize]
+    types: [opened, reopened, labeled, unlabeled, synchronize]
+
+permissions:
+  contents: read
 
 jobs:
   check-changelog:
     runs-on: ubuntu-latest
-    if: |
-      !contains(github.event.pull_request.body, '[skip changelog]') &&
-      !contains(github.event.pull_request.body, '[changelog skip]') &&
-      !contains(github.event.pull_request.body, '[skip ci]')
+    if: (!contains(github.event.pull_request.labels.*.name, 'skip changelog'))
     steps:
-      - uses: actions/checkout@v1
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Check that CHANGELOG is touched
-        run: git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md
+        run: |
+          git fetch origin ${{ github.base_ref }} --depth 1 && \
+          git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  functional-test:
+    runs-on: ubuntu-22.04
+    container:
+      image: heroku/heroku:${{ matrix.stack_number }}-build
+      options: --user root
+    strategy:
+      matrix:
+        stack_number: ["20", "22", "24"]
+    env:
+      STACK: heroku-${{ matrix.stack_number }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Functional tests on heroku:${{ matrix.stack_number }}-build
+        run: test/run
+
+  shell-lint:
+    runs-on: ubuntu-22.04
+    container:
+      image: koalaman/shellcheck-alpine:v0.9.0
+    steps:
+      - uses: actions/checkout@v4
+      - name: shellcheck
+        run: shellcheck -x bin/compile bin/detect bin/release bin/report

.gitignore

@@ -1 +1,2 @@
 .anvil
+.idea

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 ## Unreleased
 
+## 2024-06-24
+
+- The cache is now correctly invalidated if the stack version of an existing cache cannot be determined. ([#133](https://github.com/heroku/heroku-buildpack-apt/pull/133))
+- When the cache is invalidated, all APT-related files are now removed, rather than only some of them. In particular, this means old APT package index files are cleaned up. ([#133](https://github.com/heroku/heroku-buildpack-apt/pull/133))
+- Exclude test/development files from the buildpack archive published to the buildpack registry. ([#135](https://github.com/heroku/heroku-buildpack-apt/pull/135))
+
+## 2024-03-28
+
+- Warn when Aptfile contains no packages ([#126](https://github.com/heroku/heroku-buildpack-apt/pull/126))
+- Support sources parts directory for Heroku-24 compatibility ([#119](https://github.com/heroku/heroku-buildpack-apt/pull/119))
+
+## 2024-03-14
+
+- Shell hardening ([#115](https://github.com/heroku/heroku-buildpack-apt/pull/115))
+- Handle multi-package lines when capturing buildpack metadata ([#112](https://github.com/heroku/heroku-buildpack-apt/pull/112))
+
+## 2024-03-01
+
+- Add `bin/report` script to capture buildpack metadata ([#110](https://github.com/heroku/heroku-buildpack-apt/pull/110))
 
 ## 2021-03-10
 

Makefile

@@ -0,0 +1,19 @@
+test: heroku-24-build heroku-22-build heroku-20-build
+
+shellcheck:
+	@shellcheck -x bin/compile bin/detect bin/release bin/report
+
+heroku-24-build:
+	@echo "Running tests in docker (heroku-24-build)..."
+	@docker run --user root -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-24" heroku/heroku:24-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
+heroku-22-build:
+	@echo "Running tests in docker (heroku-22-build)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-22" heroku/heroku:22-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""
+
+heroku-20-build:
+	@echo "Running tests in docker (heroku-20-build)..."
+	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-20" heroku/heroku:20-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
+	@echo ""

bin/compile

@@ -10,7 +10,7 @@ set -eo pipefail
 # parse and derive params
 BUILD_DIR=$1
 CACHE_DIR=$2
-LP_DIR=`cd $(dirname $0); cd ..; pwd`
+LP_DIR=$(cd "$(dirname "$0")"; cd ..; pwd)
 
 function error() {
   echo " !     $*" >&2
@@ -29,97 +29,116 @@ function indent() {
   esac
 }
 
+if ! grep --invert-match -e "^\s*#" -e "^\s*$" -e "^:repo:" -q "${BUILD_DIR}/Aptfile"; then
+  echo "
+!       You have no packages listed in your Aptfile. If you don't need custom APT packages,
+!       delete your Aptfile and remove the buildpack with:
+!
+!       $ heroku buildpacks:remove heroku-community/apt
+"
+  exit 0
+fi
+
 # Store which STACK we are running on in the cache to bust the cache if it changes
-if [ -f $CACHE_DIR/.apt/STACK ]; then
-  CACHED_STACK=$(cat "$CACHE_DIR/.apt/STACK")
+# This file really should be inside "${CACHE_DIR}/apt" not "${CACHE_DIR}/.apt", however, this
+# buildpack has used the wrong directory name for it for some time, so there are many APT-related
+# forks that use this path, so for backwards compatibility we're not changing the path, so as to
+# not affect apps that have multiple APT-using buildpacks active at the same time.
+STACK_VERSION_FILE="${CACHE_DIR}/.apt/STACK"
+if [[ -f "${STACK_VERSION_FILE}" ]]; then
+  CACHED_STACK=$(cat "${STACK_VERSION_FILE}")
 else
-  CACHED_STACK=$STACK
+  CACHED_STACK=
 fi
 
 # Ensure we store the STACK in the cache for next time.
-mkdir -p "$CACHE_DIR/.apt"
-echo "$STACK" > "$CACHE_DIR/.apt/STACK"
+mkdir -p "${CACHE_DIR}/.apt"
+echo "${STACK}" > "${STACK_VERSION_FILE}"
 
 APT_CACHE_DIR="$CACHE_DIR/apt/cache"
 APT_STATE_DIR="$CACHE_DIR/apt/state"
 APT_SOURCELIST_DIR="$CACHE_DIR/apt/sources"   # place custom sources.list here
+APT_SOURCEPARTS_DIR="$APT_SOURCELIST_DIR/sources.list.d"
 
 APT_SOURCES="$APT_SOURCELIST_DIR/sources.list"
 
 APT_VERSION=$(apt-get -v | awk 'NR == 1{ print $2 }')
 
 case "$APT_VERSION" in
-  0* | 1.0*) APT_FORCE_YES="--force-yes";;
-  *)         APT_FORCE_YES="--allow-downgrades --allow-remove-essential --allow-change-held-packages";;
+  0* | 1.0*) APT_FORCE_YES=("--force-yes");;
+  *)         APT_FORCE_YES=("--allow-downgrades" "--allow-remove-essential" "--allow-change-held-packages");;
 esac
 
-if [ -f $APT_CACHE_DIR/Aptfile ] && cmp -s $BUILD_DIR/Aptfile $APT_CACHE_DIR/Aptfile && [[ $CACHED_STACK == $STACK ]] ; then
+if [[ -f "$APT_CACHE_DIR/Aptfile" ]] && cmp -s "$BUILD_DIR/Aptfile" "$APT_CACHE_DIR/Aptfile" && [[ "$CACHED_STACK" == "$STACK" ]] ; then
   # Old Aptfile is the same as new and STACK has not changed
   topic "Reusing cache"
 else
   # Aptfile changed or does not exist or STACK changed
   topic "Detected Aptfile or Stack changes, flushing cache"
-  rm -rf $APT_CACHE_DIR
+  rm -rf "${CACHE_DIR}/apt"
   mkdir -p "$APT_CACHE_DIR/archives/partial"
   mkdir -p "$APT_STATE_DIR/lists/partial"
   mkdir -p "$APT_SOURCELIST_DIR"   # make dir for sources
   cp -f "$BUILD_DIR/Aptfile" "$APT_CACHE_DIR/Aptfile"
   cat "/etc/apt/sources.list" > "$APT_SOURCES"    # no cp here
+  cp -R "/etc/apt/sources.list.d" "$APT_SOURCEPARTS_DIR"
   # add custom repositories from Aptfile to sources.list
   # like>>    :repo:deb http://cz.archive.ubuntu.com/ubuntu artful main universe
-  if grep -q -e "^:repo:" $BUILD_DIR/Aptfile; then
+  if grep -q -e "^:repo:" "$BUILD_DIR/Aptfile"; then
     topic "Adding custom repositories"
-    cat $BUILD_DIR/Aptfile | grep -s -e "^:repo:" | sed 's/^:repo:\(.*\)\s*$/\1/g' >> $APT_SOURCES
+    grep -s -e "^:repo:" "$BUILD_DIR/Aptfile" | sed 's/^:repo:\(.*\)\s*$/\1/g' >> "$APT_SOURCES"
   fi
 fi
 
-APT_OPTIONS="-o debug::nolocking=true -o dir::cache=$APT_CACHE_DIR -o dir::state=$APT_STATE_DIR"
+APT_OPTIONS=("-o" "debug::nolocking=true" "-o" "dir::cache=$APT_CACHE_DIR" "-o" "dir::state=$APT_STATE_DIR")
 # Override the use of /etc/apt/sources.list (sourcelist) and /etc/apt/sources.list.d/* (sourceparts).
-APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=$APT_SOURCES -o dir::etc::sourceparts=/dev/null"
+APT_OPTIONS+=("-o" "dir::etc::sourcelist=$APT_SOURCES" "-o" "dir::etc::sourceparts=$APT_SOURCEPARTS_DIR")
 
-topic "Updating apt caches"
-apt-get $APT_OPTIONS update | indent
+topic "Updating APT package index"
+apt-get "${APT_OPTIONS[@]}" update 2>&1 | indent
 
-# BEGIN: installing packages
-for PACKAGE in $(cat $BUILD_DIR/Aptfile | grep -v -s -e '^#' | grep -v -s -e "^:repo:" | grep -v -s -e '^blackhole '); do
+while IFS= read -r PACKAGE; do
   if [[ $PACKAGE == *deb ]]; then
-    PACKAGE_NAME=$(basename $PACKAGE .deb)
+    PACKAGE_NAME=$(basename "$PACKAGE" .deb)
     PACKAGE_FILE=$APT_CACHE_DIR/archives/$PACKAGE_NAME.deb
 
     topic "Fetching $PACKAGE"
-    curl --silent --show-error --fail -L -z $PACKAGE_FILE -o $PACKAGE_FILE $PACKAGE 2>&1 | indent
+    curl --silent --show-error --fail -L -z "$PACKAGE_FILE" -o "$PACKAGE_FILE" "$PACKAGE" 2>&1 | indent
   else
     topic "Fetching .debs for $PACKAGE"
-    apt-get $APT_OPTIONS -y $APT_FORCE_YES -d install --reinstall $PACKAGE | indent
+    # while this is not documented behavior, the Aptfile format technically
+    # did allow for multiple packages separated by spaces to be specified
+    # on a single line due to how the download command was implemented so we
+    # should respect that behavior since users are doing this
+    IFS=$' \t' read -ra PACKAGE_NAMES <<< "$PACKAGE"
+    apt-get "${APT_OPTIONS[@]}" -y "${APT_FORCE_YES[@]}" -d install --reinstall "${PACKAGE_NAMES[@]}" | indent
   fi
-done
+done < <(grep --invert-match -e "^\s*#" -e "^\s*$" -e "^:repo:" -e "^blackhole " "${BUILD_DIR}/Aptfile")
 
-mkdir -p $BUILD_DIR/.apt
+mkdir -p "$BUILD_DIR/.apt"
 
-for DEB in $(ls -1 $APT_CACHE_DIR/archives/*.deb); do
-  topic "Installing $(basename $DEB)"
-  dpkg -x $DEB $BUILD_DIR/.apt/
+for DEB in "$APT_CACHE_DIR/archives/"*.deb; do
+  topic "Installing $(basename "$DEB")"
+  dpkg -x "$DEB" "$BUILD_DIR/.apt/"
 done
 # END: installing packages
 
 # BEGIN: blackhole binaries
 # We wanted to `apt-get remove` preinstalled packages, but the root filesystem is read-only.
 # The best we can do is basically make a verbose version of /bin/false, name it the same as the
 # target binary, and have it come before the target binary in the PATH.
-while read BLACKHOLE_LINE; do
-  BINARY_NAME=$(echo "$BLACKHOLE_LINE" | sed -e 's/^blackhole //')
+while read -r BLACKHOLE_LINE; do
+  BINARY_NAME=${BLACKHOLE_LINE/#blackhole /}
   topic "Blackholing $BINARY_NAME"
-  mkdir -p $BUILD_DIR/.apt/usr/bin
+  mkdir -p "$BUILD_DIR"/.apt/usr/bin
   echo 'echo "THIS BINARY HAS BEEN BLACKHOLED"; exit 1' > "$BUILD_DIR/.apt/usr/bin/$BINARY_NAME"
   chmod ugo=rx "$BUILD_DIR/.apt/usr/bin/$BINARY_NAME"
-done < <(
-  cat $BUILD_DIR/Aptfile | grep -s -e '^blackhole '
-)
+done < <(grep -s -e '^blackhole ' "$BUILD_DIR"/Aptfile)
 # END: blackhole packages
 
 topic "Writing profile script"
-mkdir -p $BUILD_DIR/.profile.d
-cat <<EOF >$BUILD_DIR/.profile.d/000_apt.sh
+mkdir -p "$BUILD_DIR/.profile.d"
+cat <<EOF >"$BUILD_DIR/.profile.d/000_apt.sh"
 export PATH="\$HOME/.apt/usr/bin:\$PATH"
 export LD_LIBRARY_PATH="\$HOME/.apt/usr/lib/x86_64-linux-gnu:\$HOME/.apt/usr/lib/i386-linux-gnu:\$HOME/.apt/usr/lib:\$LD_LIBRARY_PATH"
 export LIBRARY_PATH="\$HOME/.apt/usr/lib/x86_64-linux-gnu:\$HOME/.apt/usr/lib/i386-linux-gnu:\$HOME/.apt/usr/lib:\$LIBRARY_PATH"
@@ -141,4 +160,4 @@ export PKG_CONFIG_PATH="$BUILD_DIR/.apt/usr/lib/x86_64-linux-gnu/pkgconfig:$BUIL
 export | grep -E -e ' (PATH|LD_LIBRARY_PATH|LIBRARY_PATH|INCLUDE_PATH|CPATH|CPPPATH|PKG_CONFIG_PATH)='  > "$LP_DIR/export"
 
 topic "Rewrite package-config files"
-find $BUILD_DIR/.apt -type f -ipath '*/pkgconfig/*.pc' | xargs --no-run-if-empty -n 1 sed -i -e 's!^prefix=\(.*\)$!prefix='"$BUILD_DIR"'/.apt\1!g'
+find "$BUILD_DIR/.apt" -type f -ipath '*/pkgconfig/*.pc' -print0 | xargs -0 --no-run-if-empty -n 1 sed -i -e 's!^prefix=\(.*\)$!prefix='"$BUILD_DIR"'/.apt\1!g'

bin/detect

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # bin/detect <build-dir>
 
-if [ -f $1/Aptfile ]; then
+if [[ -f "$1/Aptfile" ]]; then
   echo "Apt"
   exit 0
 else

bin/report

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# bin/report <build-dir> <cache-dir> <env-dir>
+
+### Configure environment
+
+set -o errexit    # always exit on error
+set -o pipefail   # don't ignore exit codes when piping output
+
+BUILD_DIR=${1:-}
+
+packages=()
+custom_packages=()
+custom_repositories=()
+
+while IFS= read -r line; do
+  if grep --silent -e "^:repo:" <<< "${line}"; then
+    custom_repositories+=("${line//:repo:/}")
+  elif [[ $line == *deb ]]; then
+    custom_packages+=("${line}")
+  else
+    IFS=$' \t' read -ra package_names <<< "${line}"
+    for package_name in "${package_names[@]}"; do
+      packages+=("${package_name}")
+    done
+  fi
+done < <(grep --invert-match -e "^\s*#" -e "^\s*$" "${BUILD_DIR}/Aptfile")
+
+output_key_value() {
+  local key value
+  key="$1"
+  shift
+  # sort & join the array values with a ',' then escape both '\' and '"' characters
+  value=$(printf '%s\n' "$@" | sort | tr '\n' ',' | sed 's/,$/\n/' | sed 's/\\/\\\\/g' | sed 's/"/\\"/g')
+  if [[ -n "${value}" ]]; then
+    echo "$key: \"$value\""
+  fi
+}
+
+output_key_value "packages" "${packages[@]}"
+output_key_value "custom_packages" "${custom_packages[@]}"
+output_key_value "custom_repositories" "${custom_repositories[@]}"

buildpack.toml

@@ -0,0 +1,10 @@
+[buildpack]
+name = "Apt"
+
+[publish.Ignore]
+files = [
+  ".github/",
+  "test/",
+  ".gitignore",
+  "Makefile",
+]