From 2fcdb40b65cbf36cd926801e1878743211de6a43 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:37:42 -0400 Subject: [PATCH 01/23] feat: install software updates on macOS --- nodebuilder | 88 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/nodebuilder b/nodebuilder index 866332091..41b32e6bf 100755 --- a/nodebuilder +++ b/nodebuilder @@ -113,6 +113,37 @@ install_runtime_dependencies_dnf() { fi } +install_dependencies_macos() { + if ! command -v git > /dev/null 2>&1; then + printf '%s' "Installing git via the Xcode Command Line Tools... " + # These steps were taken from: https github com/Homebrew/install/blob/aceed88a4a062e2b41dc40a7428c71309fce14c9/install.sh#L831 + # TODO: find a way to install only the git binary, not all the Xcode tools. Unattended git installation shouldn't take 20+ lines of code. + sudo touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + clt_label_command="/usr/sbin/softwareupdate -l | grep -B 1 -E 'Command Line Tools' | awk -F'*' '/^ *\\*/ {print \$2}' | sed -e 's/^ *Label: //' -e 's/^ *//' | sort -V | tail -n1 | tr -d '\n'" + clt_label="$(/bin/sh -c "${clt_label_command}")" + if [ -n "${clt_label}" ]; then + sudo /usr/sbin/softwareupdate -i "${clt_label}" > /dev/null + else + handle_error "Failed to install git, cannot find a compatible Xcode Command Line Tools package." + fi + sudo rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress + + if command -v git > /dev/null 2>&1; then + printf '%s\n' "ok." + elif [ "${unattended}" = true ]; then + handle_error "Failed to install Xcode. Try re-running without -u/--unattended or manually install git." + else + printf '%s\n%s\n' "failed." "Installing the Command Line Tools (expect a popup window)." + /usr/bin/xcode-select --install + printf '%s\n' "PRESS ENTER after you've completed the installation via the popup window." + read -r _ + if ! command -v git > /dev/null 2>&1; then + handle_error "Failed to install git via the Xcode Command Line Tools." + fi + fi + fi +} + install_runtime_dependencies_pacman() { dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_pacman.txt" dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") @@ -123,6 +154,7 @@ install_runtime_dependencies_pacman() { fi } + install_runtime_dependencies_zypper() { dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_zypper.txt" dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") @@ -133,6 +165,10 @@ install_runtime_dependencies_zypper() { fi } +install_updates_apk() { + apk update --quiet && apk upgrade --quiet +} + install_system_updates_aptget() { check_dpkg_lock stderr_install_log_file="${temp_directory}/stderr_install.log" @@ -145,8 +181,8 @@ install_system_updates_dnf() { sudo dnf --assumeyes --quiet upgrade > /dev/null } -install_system_updates_apk() { - apk update --quiet && apk upgrade --quiet +install_updates_macos() { + sudo softwareupdate --install --all } install_system_updates_pacman() { @@ -391,9 +427,10 @@ Linux) sudo reboot exit 0 fi - printf '%s\n' "ok." - - printf '%s' "Ensuring runtime dependencies... " + printf '%s\n%s' "ok." "Checking for dependencies... " + dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies.txt" + dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") + [ -z "$dependencies" ] && handle_error "The list of dependencies is empty." case "$(get_os_release_type)" in alpine) install_runtime_dependencies_apk @@ -445,37 +482,24 @@ Darwin) handle_error "Check for active internet failed." printf '%s\n' "ok." display_macos_warning - if ! command -v git > /dev/null 2>&1; then - printf '%s' "Installing git via the Xcode Command Line Tools... " - # These steps were taken from https://github.com/Homebrew/install/blob/aceed88a4a062e2b41dc40a7428c71309fce14c9/install.sh#L831 - # TODO: find a way to install only the git binary, not all the Xcode tools. Unattended xcode tools installation shouldn't take 20+ lines of code - sudo touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - clt_label_command="/usr/sbin/softwareupdate -l | grep -B 1 -E 'Command Line Tools' | awk -F'*' '/^ *\\*/ {print \$2}' | sed -e 's/^ *Label: //' -e 's/^ *//' | sort -V | tail -n1 | tr -d '\n'" - clt_label="$(/bin/sh -c "${clt_label_command}")" - if [ -n "${clt_label}" ]; then - sudo /usr/sbin/softwareupdate -i "${clt_label}" > /dev/null - else - handle_error "Failed to install git, cannot find a compatible Xcode Command Line Tools package." - fi - sudo rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress - if command -v git > /dev/null 2>&1; then - printf '%s\n' "ok." - else - if [ "${unattended}" = true ]; then - printf '\n%s\n' "ERROR: Failed to install Xcode. Try re-running without -u/--unattended." - exit 1 - fi - - printf '%s\n%s\n' "failed." "Installing the Command Line Tools (expect a GUI popup)." - /usr/bin/xcode-select --install - printf '%s' "PRESS ENTER when the installation has completed." + printf '%s' "Performing a system upgrade... " + reboot_required_list_macos="$(softwareupdate --list | grep "[restart]")" + install_updates_macos + if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then + printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" + if [ "${unattended}" = false ]; then + printf '\n%s' "PRESS ENTER to reboot or press Ctrl+C to exit... " read -r _ - if ! command -v git > /dev/null 2>&1; then - handle_error "Failed to install git via the Xcode Command Line Tools." - fi + printf '\n' fi + printf '%s\n' "Rebooting." + sudo reboot + exit 0 fi + printf '%s\n%s' "ok." "Checking for git... " + install_dependencies_macos + printf '%s\n' "ok." ;; MINGW*) handle_error "Windows is not supported. Instead, use WSL (Windows Subsystem for Linux)." From c3da70759053081b0c9df9ded348aa7cd2fd3417 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:41:57 -0400 Subject: [PATCH 02/23] fix: formatting and UI --- nodebuilder | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nodebuilder b/nodebuilder index 41b32e6bf..734bb84f1 100755 --- a/nodebuilder +++ b/nodebuilder @@ -124,21 +124,21 @@ install_dependencies_macos() { if [ -n "${clt_label}" ]; then sudo /usr/sbin/softwareupdate -i "${clt_label}" > /dev/null else - handle_error "Failed to install git, cannot find a compatible Xcode Command Line Tools package." + handle_error "Failed to install git. Cannot find a compatible Xcode Command Line Tools package." fi sudo rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress if command -v git > /dev/null 2>&1; then printf '%s\n' "ok." elif [ "${unattended}" = true ]; then - handle_error "Failed to install Xcode. Try re-running without -u/--unattended or manually install git." + handle_error "Failed to install Xcode Command Line Tools. Try re-running without -u/--unattended or manually installing git." else printf '%s\n%s\n' "failed." "Installing the Command Line Tools (expect a popup window)." /usr/bin/xcode-select --install printf '%s\n' "PRESS ENTER after you've completed the installation via the popup window." read -r _ if ! command -v git > /dev/null 2>&1; then - handle_error "Failed to install git via the Xcode Command Line Tools." + handle_error "Failed to install git via the Xcode Command Line Tools. Try manually installing git" fi fi fi From f8d9472e4fcf34dc9362debe9776abc584088e04 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:56:31 -0400 Subject: [PATCH 03/23] Update nodebuilder --- nodebuilder | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nodebuilder b/nodebuilder index 734bb84f1..14b3dd719 100755 --- a/nodebuilder +++ b/nodebuilder @@ -182,7 +182,7 @@ install_system_updates_dnf() { } install_updates_macos() { - sudo softwareupdate --install --all + sudo softwareupdate --install --recommended #--all } install_system_updates_pacman() { @@ -484,7 +484,8 @@ Darwin) display_macos_warning printf '%s' "Performing a system upgrade... " - reboot_required_list_macos="$(softwareupdate --list | grep "[restart]")" + softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING + reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart")" install_updates_macos if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" From f7a2a7b321152dbc8108cc3745b5053462a80e7b Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:06:38 -0400 Subject: [PATCH 04/23] feat(ci): install macOS updates and cache directory --- .github/workflows/validation.yaml | 40 ++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 3a50bc8eb..14765a3fd 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -136,12 +136,41 @@ jobs: sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) && sudo apt-get update --option 'Acquire::Retries=5' && sudo NEEDRESTART_MODE=a apt-get dist-upgrade --assume-yes --option 'Acquire::Retries=5' + cache-path: | + /var/cache/apt/archives/*.deb + /var/lib/apt/lists/*.ubuntu.com_ubuntu_dists_* + /var/lib/apt/lists/download.docker.com_linux_ubuntu_dists_* + /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* + /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* + /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* + - os: ubuntu-20.04 + os-friendly-name: Ubuntu 20.04 + check-version-command: grep "VERSION\|ID" /etc/os-release + install-upgrades-command: >- + sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) + && sudo apt-get update --option 'Acquire::Retries=5' + && sudo NEEDRESTART_MODE=a apt-get dist-upgrade --assume-yes --option 'Acquire::Retries=5' + cache-path: | + /var/cache/apt/archives/*.deb + /var/lib/apt/lists/*.ubuntu.com_ubuntu_dists_* + /var/lib/apt/lists/download.docker.com_linux_ubuntu_dists_* + /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* + /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* + /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* + - os: macos-14 + os-friendly-name: macOS 14 arm64 + check-version-command: sw_vers | grep Version | awk '{print $2}' + install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended + cache-path: /Library/Archives - os: macos-13 os-friendly-name: macOS 13 [x86_64] check-version-command: sw_vers | grep Version | awk '{print $2}' - os: macos-14 os-friendly-name: macOS 14 [arm64] check-version-command: sw_vers | grep Version | awk '{print $2}' + install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended + cache-path: /Library/Archives + needs: [shell-lint, yaml-lint, dockerfile-lint, markdown-spellcheck, pause-if-master-commit] steps: - name: Check the current OS version run: ${{ matrix.check-version-command }} && uname -a @@ -149,17 +178,10 @@ jobs: - name: Change owner of apt archives if: contains(matrix.os, 'ubuntu') run: sudo chown -R "$(whoami)" /var/cache/apt/archives - - name: Cache apt archives and lists - if: contains(matrix.os, 'ubuntu') + - name: Cache system archives uses: actions/cache@v4 with: - path: | - /var/cache/apt/archives/*.deb - /var/lib/apt/lists/*.ubuntu.com_ubuntu_dists_* - /var/lib/apt/lists/download.docker.com_linux_ubuntu_dists_* - /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* - /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* - /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* + path: ${{ matrix.cache-path }} key: cache-apt-${{ matrix.os }}-${{ github.run_id }} restore-keys: cache-apt-${{ matrix.os }} - name: Remove the snap package From d00ac9763235e0953a660e746bb847746318238e Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:10:34 -0400 Subject: [PATCH 05/23] Update validation.yaml --- .github/workflows/validation.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 14765a3fd..c09268c86 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -105,7 +105,7 @@ jobs: name: Pause on master commits runs-on: ubuntu-latest steps: - - name: Sleep for the cache-control max-age duration + - name: Sleep for the GitHub cache-control max-age duration run: if [ "${{ github.ref }}" = "refs/heads/master" ]; then sleep 5m; fi run-nodebuilder-baremetal: @@ -182,8 +182,8 @@ jobs: uses: actions/cache@v4 with: path: ${{ matrix.cache-path }} - key: cache-apt-${{ matrix.os }}-${{ github.run_id }} - restore-keys: cache-apt-${{ matrix.os }} + key: cache-updates-${{ matrix.os }}-${{ github.run_id }} + restore-keys: cache-updates-${{ matrix.os }} - name: Remove the snap package run: ${{ matrix.remove-snap-command }} - name: Install system upgrades From 7e20fd28e4e2a7bca9f0c4cec94cf695c31f7751 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:18:49 -0400 Subject: [PATCH 06/23] Update validation.yaml --- .github/workflows/validation.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index c09268c86..6b09f439c 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -144,7 +144,7 @@ jobs: /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* - os: ubuntu-20.04 - os-friendly-name: Ubuntu 20.04 + os-friendly-name: Ubuntu 20 check-version-command: grep "VERSION\|ID" /etc/os-release install-upgrades-command: >- sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) @@ -158,16 +158,13 @@ jobs: /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* - os: macos-14 - os-friendly-name: macOS 14 arm64 + os-friendly-name: macOS 14 [arm64] check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended cache-path: /Library/Archives - os: macos-13 os-friendly-name: macOS 13 [x86_64] check-version-command: sw_vers | grep Version | awk '{print $2}' - - os: macos-14 - os-friendly-name: macOS 14 [arm64] - check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended cache-path: /Library/Archives needs: [shell-lint, yaml-lint, dockerfile-lint, markdown-spellcheck, pause-if-master-commit] From ac265dd25f51012e7342025464460e0355682dfd Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:40:27 -0400 Subject: [PATCH 07/23] Update validation.yaml --- .github/workflows/validation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 6b09f439c..a9b105160 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -161,12 +161,12 @@ jobs: os-friendly-name: macOS 14 [arm64] check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended - cache-path: /Library/Archives + cache-path: /Library/Updates - os: macos-13 os-friendly-name: macOS 13 [x86_64] check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended - cache-path: /Library/Archives + cache-path: /Library/Updates needs: [shell-lint, yaml-lint, dockerfile-lint, markdown-spellcheck, pause-if-master-commit] steps: - name: Check the current OS version From 80d3d14df081eb66391b977264d7fef03f8c4818 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:38:52 -0400 Subject: [PATCH 08/23] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 14b3dd719..f3275dff8 100755 --- a/nodebuilder +++ b/nodebuilder @@ -182,7 +182,7 @@ install_system_updates_dnf() { } install_updates_macos() { - sudo softwareupdate --install --recommended #--all + sudo softwareupdate --install --os-only --restart } install_system_updates_pacman() { From 291ad8b66bd1bad84f183ee3bcf7cac2c8d903ce Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:39:33 -0400 Subject: [PATCH 09/23] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index f3275dff8..e88423cc3 100755 --- a/nodebuilder +++ b/nodebuilder @@ -182,7 +182,7 @@ install_system_updates_dnf() { } install_updates_macos() { - sudo softwareupdate --install --os-only --restart + sudo softwareupdate --install --os-only --recommended --restart } install_system_updates_pacman() { From 2c489df324c4f6df523f0b3d9b92f27dd8fa7f4e Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:35:50 -0400 Subject: [PATCH 10/23] Update nodebuilder --- nodebuilder | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/nodebuilder b/nodebuilder index e88423cc3..9b9a55e9f 100755 --- a/nodebuilder +++ b/nodebuilder @@ -103,17 +103,7 @@ install_runtime_dependencies_aptget() { fi } -install_runtime_dependencies_dnf() { - dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_dnf.txt" - dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") - if [ -n "${dependencies}" ]; then - printf '%s\n' "${dependencies}" | xargs sudo dnf --assumeyes --quiet install > /dev/null - else - handle_error "The list of dependencies is empty." - fi -} - -install_dependencies_macos() { +install_runtime_dependencies_darwin() { if ! command -v git > /dev/null 2>&1; then printf '%s' "Installing git via the Xcode Command Line Tools... " # These steps were taken from: https github com/Homebrew/install/blob/aceed88a4a062e2b41dc40a7428c71309fce14c9/install.sh#L831 @@ -144,6 +134,16 @@ install_dependencies_macos() { fi } +install_runtime_dependencies_dnf() { + dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_dnf.txt" + dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") + if [ -n "${dependencies}" ]; then + printf '%s\n' "${dependencies}" | xargs sudo dnf --assumeyes --quiet install > /dev/null + else + handle_error "The list of dependencies is empty." + fi +} + install_runtime_dependencies_pacman() { dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_pacman.txt" dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") @@ -154,7 +154,6 @@ install_runtime_dependencies_pacman() { fi } - install_runtime_dependencies_zypper() { dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies/runtime_dependencies_zypper.txt" dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") @@ -177,12 +176,12 @@ install_system_updates_aptget() { rm "${stderr_install_log_file}" } -install_system_updates_dnf() { - sudo dnf --assumeyes --quiet upgrade > /dev/null +install_updates_darwin() { + sudo softwareupdate --install --os-only --recommended --restart } -install_updates_macos() { - sudo softwareupdate --install --os-only --recommended --restart +install_system_updates_dnf() { + sudo dnf --assumeyes --quiet upgrade > /dev/null } install_system_updates_pacman() { @@ -499,7 +498,7 @@ Darwin) exit 0 fi printf '%s\n%s' "ok." "Checking for git... " - install_dependencies_macos + install_runtime_dependencies_darwin printf '%s\n' "ok." ;; MINGW*) From 5e846358fff6d2cef2dab881d37f7bef8f184348 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 11:38:11 -0400 Subject: [PATCH 11/23] Update nodebuilder --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 9b9a55e9f..5071be419 100755 --- a/nodebuilder +++ b/nodebuilder @@ -485,7 +485,7 @@ Darwin) printf '%s' "Performing a system upgrade... " softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart")" - install_updates_macos + install_updates_darwin if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" if [ "${unattended}" = false ]; then From 2e57ab029496789dcd26d67cbe31e08e68c9099d Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:59:21 -0400 Subject: [PATCH 12/23] Update nodebuilder --- nodebuilder | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 5071be419..ab5b2da91 100755 --- a/nodebuilder +++ b/nodebuilder @@ -177,7 +177,29 @@ install_system_updates_aptget() { } install_updates_darwin() { - sudo softwareupdate --install --os-only --recommended --restart + softwareupdate --install --os-only --recommended & + softwareupdate_pid="$!" + + # Continuously check for the presence of the restart dialog while softwareupdate is running + while ps -p "${softwareupdate_pid}" > /dev/null; do + # If the restart dialog is present, handle it + if ps aux | grep --quiet "[i]nstallassistant_springboard"; then + if is_running_in_ci || is_running_in_container; then + installassistant_pid=$(ps aux | grep "[i]nstallassistant_springboard" | awk '{print $2}') + + # DEBUG TESTING HERE DELETE BEFORE MERGING + # MIGHT ALSO NEED TO SEND A kill -TERM "${installassistant_pid}" + # MIGHT ALSO NEED TO TRY kill "${softwareupdate_pid}" + kill -INT "${installassistant_pid}" + else + echo "The system needs to restart to finish installing updates. Please click the restart button." + exit 0 + fi + fi + + # Wait a bit before checking again + sleep 1 + done } install_system_updates_dnf() { From 5f5848f9b630668ed5acf13af2e9d502cdad78ef Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:02:18 -0400 Subject: [PATCH 13/23] Update nodebuilder --- nodebuilder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodebuilder b/nodebuilder index ab5b2da91..21105ec06 100755 --- a/nodebuilder +++ b/nodebuilder @@ -1,6 +1,8 @@ #!/bin/sh # # A minimally-interactive script for launching a Bitcoin Core node +# shellcheck disable=SC2009 + set -o errexit set -o nounset From 8a2270f30dc999bc52d916121c1aa972984168fb Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:22:59 -0400 Subject: [PATCH 14/23] Update nodebuilder --- nodebuilder | 1 + 1 file changed, 1 insertion(+) diff --git a/nodebuilder b/nodebuilder index 21105ec06..eca1fc834 100755 --- a/nodebuilder +++ b/nodebuilder @@ -184,6 +184,7 @@ install_updates_darwin() { # Continuously check for the presence of the restart dialog while softwareupdate is running while ps -p "${softwareupdate_pid}" > /dev/null; do + ps aux #debug # If the restart dialog is present, handle it if ps aux | grep --quiet "[i]nstallassistant_springboard"; then if is_running_in_ci || is_running_in_container; then From 25cd856eb72605284e86e6d2ae1afdcffad82c75 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 06:12:44 -0400 Subject: [PATCH 15/23] fix(ci): remove duplicate needs field in bare metal jobs --- .github/workflows/validation.yaml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index a9b105160..0c27fe754 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -143,31 +143,16 @@ jobs: /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* - - os: ubuntu-20.04 - os-friendly-name: Ubuntu 20 - check-version-command: grep "VERSION\|ID" /etc/os-release - install-upgrades-command: >- - sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) - && sudo apt-get update --option 'Acquire::Retries=5' - && sudo NEEDRESTART_MODE=a apt-get dist-upgrade --assume-yes --option 'Acquire::Retries=5' - cache-path: | - /var/cache/apt/archives/*.deb - /var/lib/apt/lists/*.ubuntu.com_ubuntu_dists_* - /var/lib/apt/lists/download.docker.com_linux_ubuntu_dists_* - /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* - /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* - /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* - - os: macos-14 - os-friendly-name: macOS 14 [arm64] + - os: macos-13 + os-friendly-name: macOS 13 [x86_64] check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended cache-path: /Library/Updates - - os: macos-13 - os-friendly-name: macOS 13 [x86_64] + - os: macos-14 + os-friendly-name: macOS 14 [arm64] check-version-command: sw_vers | grep Version | awk '{print $2}' install-upgrades-command: softwareupdate --list && softwareupdate --install --recommended cache-path: /Library/Updates - needs: [shell-lint, yaml-lint, dockerfile-lint, markdown-spellcheck, pause-if-master-commit] steps: - name: Check the current OS version run: ${{ matrix.check-version-command }} && uname -a From 38abf023c50656491c699115606c82f0412b0b8c Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 06:26:47 -0400 Subject: [PATCH 16/23] Update nodebuilder --- nodebuilder | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder b/nodebuilder index eca1fc834..150fcb7ff 100755 --- a/nodebuilder +++ b/nodebuilder @@ -166,7 +166,7 @@ install_runtime_dependencies_zypper() { fi } -install_updates_apk() { +install_system_updates_apk() { apk update --quiet && apk upgrade --quiet } @@ -178,7 +178,7 @@ install_system_updates_aptget() { rm "${stderr_install_log_file}" } -install_updates_darwin() { +install_system_updates_darwin() { softwareupdate --install --os-only --recommended & softwareupdate_pid="$!" From d7669e510959ad1baff521f7c30e43d4cea46a74 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 06:59:47 -0400 Subject: [PATCH 17/23] fix: use correct function name --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 150fcb7ff..51226de90 100755 --- a/nodebuilder +++ b/nodebuilder @@ -510,7 +510,7 @@ Darwin) printf '%s' "Performing a system upgrade... " softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart")" - install_updates_darwin + install_system_updates_darwin if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" if [ "${unattended}" = false ]; then From 9aba6b35a37a35d0424b6c038f51aee6a6cd0cc1 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:07:04 -0400 Subject: [PATCH 18/23] add cache-path to Ubuntu 22 and other good stuff --- .github/workflows/validation.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 0c27fe754..ccbd9e0b0 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -129,9 +129,17 @@ jobs: sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) && sudo apt-get update --option 'Acquire::Retries=5' && sudo NEEDRESTART_MODE=a apt-get dist-upgrade --assume-yes --option 'Acquire::Retries=5' + cache-path: | + /var/cache/apt/archives/*.deb + /var/lib/apt/lists/*.ubuntu.com_ubuntu_dists_* + /var/lib/apt/lists/download.docker.com_linux_ubuntu_dists_* + /var/lib/apt/lists/dl.google.com_linux_chrome_deb_dists_* + /var/lib/apt/lists/packages.microsoft.com_repos_code_dists_* + /var/lib/apt/lists/pkg.cloudflareclient.com_dists_* - os: ubuntu-latest os-friendly-name: Ubuntu LTS [x86_64] check-version-command: grep "VERSION\|ID" /etc/os-release + remove-snap-command: sudo apt remove --assume-yes snap install-upgrades-command: >- sleep $(echo "scale=3; $RANDOM / 32768 * 10" | bc) && sudo apt-get update --option 'Acquire::Retries=5' @@ -168,6 +176,7 @@ jobs: restore-keys: cache-updates-${{ matrix.os }} - name: Remove the snap package run: ${{ matrix.remove-snap-command }} + if: matrix.install-upgrades-command != null - name: Install system upgrades run: ${{ matrix.install-upgrades-command }} if: matrix.install-upgrades-command != null From 875bbef6308783a40815dea7b8e1ea489e7d0022 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:19:28 -0400 Subject: [PATCH 19/23] handle case where no reboot is required --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index 51226de90..a2a3953db 100755 --- a/nodebuilder +++ b/nodebuilder @@ -509,7 +509,7 @@ Darwin) printf '%s' "Performing a system upgrade... " softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING - reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart")" + reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart" || '')" install_system_updates_darwin if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" From 5208d3d5df2cf4e82738d9f9e616aea00899dbc7 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 07:25:42 -0400 Subject: [PATCH 20/23] fix syntax bug --- nodebuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder b/nodebuilder index a2a3953db..e2c762c06 100755 --- a/nodebuilder +++ b/nodebuilder @@ -509,7 +509,7 @@ Darwin) printf '%s' "Performing a system upgrade... " softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING - reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart" || '')" + reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart" || true)" install_system_updates_darwin if [ -n "${reboot_required_list_macos}" ] && ! is_running_in_ci; then printf '\n%s\n%s\n' "REBOOT REQUIRED to upgrade the following:" "${reboot_required_list_macos}" From bf0968f13d587da86f21d62dc48a459b111d4938 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:54:58 -0400 Subject: [PATCH 21/23] fix: avoid repeated code --- nodebuilder | 3 --- 1 file changed, 3 deletions(-) diff --git a/nodebuilder b/nodebuilder index 1bc9098e0..9fb99d979 100755 --- a/nodebuilder +++ b/nodebuilder @@ -547,9 +547,6 @@ Darwin) handle_error "Check for active internet failed." printf '%s\n' "ok." display_macos_warning - printf '%s\n%s' "ok." "Checking for git... " - install_runtime_dependencies_darwin - printf '%s\n' "ok." printf '%s' "Performing a system upgrade... " softwareupdate --list #DEBUG PURPOSES DELETE THIS LINE BEFORE MERGING reboot_required_list_macos="$(softwareupdate --list 2> /dev/null | grep "restart" || true)" From ab21c7cd9020b8d4ed686a065d8b05687f5a25c7 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:51:17 -0400 Subject: [PATCH 22/23] fix: remove bad if condition --- .github/workflows/validation.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/validation.yaml b/.github/workflows/validation.yaml index 9bcb55bb5..883e3f4a2 100644 --- a/.github/workflows/validation.yaml +++ b/.github/workflows/validation.yaml @@ -177,7 +177,6 @@ jobs: - name: Remove the snap package if: matrix.remove-snap-command != null run: ${{ matrix.remove-snap-command }} - if: matrix.install-upgrades-command != null - name: Install system upgrades if: matrix.install-upgrades-command != null run: ${{ matrix.install-upgrades-command }} From 608d55083983107901357347a9a189a027a5ab98 Mon Sep 17 00:00:00 2001 From: Bitcoin Tools <156422466+bitcoin-tools@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:44:09 -0400 Subject: [PATCH 23/23] use an improved version of master's code --- nodebuilder | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nodebuilder b/nodebuilder index 629d9eed0..54f4667e9 100755 --- a/nodebuilder +++ b/nodebuilder @@ -504,10 +504,7 @@ Linux) sudo reboot exit 0 fi - printf '%s\n%s' "ok." "Checking for dependencies... " - dependencies_url="https://github.com/bitcoin-tools/nodebuilder/raw/master/dependencies.txt" - dependencies=$(curl --fail --silent --show-error --location --retry 5 --retry-delay 10 "${dependencies_url}") - [ -z "$dependencies" ] && handle_error "The list of dependencies is empty." + printf '%s\n%s' "ok." "Ensuring runtime dependencies... " case "$(get_os_release_type)" in alpine) install_runtime_dependencies_apk