From e6817d861d04bcf2a106217329c66fbcd2ebe469 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 23 Sep 2025 09:36:08 +0000 Subject: [PATCH 1/8] remove sudo when installing pipx packages --- .../tools/scripts/lib_install/pipx-packages/install-all.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh index 226425faff..31356f385a 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh @@ -4,6 +4,12 @@ set -o errexit set -o nounset set -o pipefail +is_sudo_from_non_root=$(( $EUID == 0 && ${SUDO_UID:-0} != 0 )) +if (( is_sudo_from_non_root )); then + echo "Installing pipx packages to the user environment (sudo lifted)." + exec sudo --set-home --user="$SUDO_USER" "$0" "$@" +fi + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-cmake.sh" From 1701f05b8f6682f3e35c2460639da1067dbb7bf7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 23 Sep 2025 17:48:06 +0800 Subject: [PATCH 2/8] Apply suggestions from code review Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../tools/scripts/lib_install/pipx-packages/install-all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh index 31356f385a..abe7943943 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh @@ -4,10 +4,10 @@ set -o errexit set -o nounset set -o pipefail -is_sudo_from_non_root=$(( $EUID == 0 && ${SUDO_UID:-0} != 0 )) +is_sudo_from_non_root=$(( EUID == 0 && ${SUDO_UID:-0} != 0 )) if (( is_sudo_from_non_root )); then echo "Installing pipx packages to the user environment (sudo lifted)." - exec sudo --set-home --user="$SUDO_USER" "$0" "$@" + exec sudo --preserve-env=PATH --set-home --user="$SUDO_USER" /usr/bin/env bash "${BASH_SOURCE[0]}" "$@" fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" From 896a9c47dd4bd14aa289f5f2670ddacf252400cf Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 23 Sep 2025 09:48:54 +0000 Subject: [PATCH 3/8] line break for lint --- .../tools/scripts/lib_install/pipx-packages/install-all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh index abe7943943..6500db9891 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh @@ -7,7 +7,8 @@ set -o pipefail is_sudo_from_non_root=$(( EUID == 0 && ${SUDO_UID:-0} != 0 )) if (( is_sudo_from_non_root )); then echo "Installing pipx packages to the user environment (sudo lifted)." - exec sudo --preserve-env=PATH --set-home --user="$SUDO_USER" /usr/bin/env bash "${BASH_SOURCE[0]}" "$@" + exec sudo --preserve-env=PATH --set-home --user="$SUDO_USER" \ + /usr/bin/env bash "${BASH_SOURCE[0]}" "$@" fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" From 12fc0723a12a1851696b7909f05a6c8c26582bcb Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 23 Sep 2025 14:35:02 +0000 Subject: [PATCH 4/8] Preserve all env vars when running pipx install as user --- .../core/tools/scripts/lib_install/pipx-packages/install-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh index 6500db9891..a133f6a44f 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh @@ -7,7 +7,7 @@ set -o pipefail is_sudo_from_non_root=$(( EUID == 0 && ${SUDO_UID:-0} != 0 )) if (( is_sudo_from_non_root )); then echo "Installing pipx packages to the user environment (sudo lifted)." - exec sudo --preserve-env=PATH --set-home --user="$SUDO_USER" \ + exec sudo --preserve-env --set-home --user="$SUDO_USER" \ /usr/bin/env bash "${BASH_SOURCE[0]}" "$@" fi From 24adfec1efac9c81ac5cc53bd3e3ef5a2a3439b8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 23 Sep 2025 15:10:25 +0000 Subject: [PATCH 5/8] Add pipx-packages to trigger various build CI pipelines --- .github/workflows/clp-artifact-build.yaml | 4 ++++ .github/workflows/clp-core-build-macos.yaml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index fac645c8fa..f531cd3891 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -62,24 +62,28 @@ jobs: - ".github/actions/**" - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" manylinux_2_28_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" - "components/core/tools/scripts/lib_install/manylinux_2_28/**" musllinux_1_2_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/**" - "components/core/tools/scripts/lib_install/musllinux_1_2/**" ubuntu_jammy_image: - ".github/actions/**" - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" clp: diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 12f2023024..352041a118 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -9,6 +9,7 @@ on: - "components/core/CMakeLists.txt" - "components/core/src/**" - "components/core/tests/**" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/scripts/lib_install/macos/**" - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" @@ -23,6 +24,7 @@ on: - "components/core/CMakeLists.txt" - "components/core/src/**" - "components/core/tests/**" + - "components/core/tools/scripts/lib_install/pipx-packages/**" - "components/core/tools/scripts/lib_install/macos/**" - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" From 22ec79eb4c52165933b885f45efe3b514379273a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 25 Sep 2025 19:14:34 +0000 Subject: [PATCH 6/8] fix issue where the installed binary is not immediately available for version checking --- .../lib_install/pipx-packages/install-cmake.sh | 17 ++++++++++++----- .../pipx-packages/install-go-task.sh | 17 ++++++++++++----- .../lib_install/pipx-packages/install-uv.sh | 11 +++++++++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh index 855a79b731..1497bd9282 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -15,18 +15,23 @@ readonly required_version_min="${required_version_major_min}.${required_version_ readonly required_version_major_max=3 readonly required_version_major_max_plus_1=$((required_version_major_max + 1)) -package_preinstalled=0 -if ! command -v cmake >/dev/null 2>&1; then +cmake_bin="$(command -v cmake 2>/dev/null || true)" +if [ -n "${cmake_bin}" ]; then + package_preinstalled=0 + echo "Preinstalled CMake found at: ${cmake_bin}" +else package_preinstalled=1 # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). pipx install --force "cmake>=${required_version_min},<${required_version_major_max_plus_1}" pipx ensurepath + cmake_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/cmake" + echo "Pipx CMake installed at: ${cmake_bin}" fi -installed_version=$(cmake -E capabilities | jq --raw-output ".version.string") -installed_version_major=$(cmake -E capabilities | jq --raw-output ".version.major") -installed_version_minor=$(cmake -E capabilities | jq --raw-output ".version.minor") +installed_version=$("${cmake_bin}" -E capabilities | jq --raw-output ".version.string") +installed_version_major=$("${cmake_bin}" -E capabilities | jq --raw-output ".version.major") +installed_version_minor=$("${cmake_bin}" -E capabilities | jq --raw-output ".version.minor") # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). @@ -47,3 +52,5 @@ if (("${installed_version_major}" < "${required_version_major_min}")) \ exit 1 fi + +echo "CMake version ${installed_version} installed at ${cmake_bin} satisfies version requirements." diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh index 5923a0a4cd..7a353d3918 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -12,17 +12,22 @@ fi # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 readonly required_version="3.44.0" -package_preinstalled=0 -if ! command -v task >/dev/null 2>&1; then +go_task_bin="$(command -v task 2>/dev/null || true)" +if [ -n "${go_task_bin}" ]; then + package_preinstalled=0 + echo "Preinstalled Task found at: ${go_task_bin}" +else package_preinstalled=1 pipx install --force "go-task-bin==${required_version}" pipx ensurepath + go_task_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/task" + echo "Pipx Task installed at: ${go_task_bin}" fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") -if [[ "${task_version}" != "${required_version}" ]]; then - echo "Error: Task version ${task_version} is currently unsupported (must be" \ +installed_version=$(${go_task_bin} --silent --taskfile "${script_dir}/print-go-task-version.yaml") +if [[ "${installed_version}" != "${required_version}" ]]; then + echo "Error: Task version ${installed_version} is currently unsupported (must be" \ "${required_version})." if ((0 == "${package_preinstalled}")); then @@ -35,3 +40,5 @@ if [[ "${task_version}" != "${required_version}" ]]; then exit 1 fi + +echo "Task version ${installed_version} installed at ${go_task_bin} satisfies version requirements." diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index a368dd7192..f14fa795f5 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -13,11 +13,16 @@ readonly required_version_major_min=0 readonly required_version_minor_min=8 readonly required_version_min="${required_version_major_min}.${required_version_minor_min}" -package_preinstalled=0 -if ! command -v uv >/dev/null 2>&1; then +uv_bin="$(command -v uv 2>/dev/null || true)" +if [ -n "${uv_bin}" ]; then + package_preinstalled=0 + echo "Preinstalled uv found at: ${uv_bin}" +else package_preinstalled=1 pipx install --force "uv>=${required_version_min}" pipx ensurepath + uv_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/uv" + echo "Pipx uv installed at: ${uv_bin}" fi installed_version=$(uv self version --output-format json | jq --raw-output ".version") @@ -38,3 +43,5 @@ if (("${installd_version_major}" == "${required_version_major_min}" && \ exit 1 fi + +echo "uv version ${installed_version} installed at ${uv_bin} satisfies version requirements." From 59de3b0b5c97a64a94b52ef0c5174f7dc05459d1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 25 Sep 2025 20:39:31 +0000 Subject: [PATCH 7/8] Find installed pipx binary in the more reliable way --- .../pipx-packages/find-pipx-bin.sh | 28 +++++++++++++++++++ .../pipx-packages/install-cmake.sh | 3 +- .../pipx-packages/install-go-task.sh | 5 ++-- .../lib_install/pipx-packages/install-uv.sh | 3 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100755 components/core/tools/scripts/lib_install/pipx-packages/find-pipx-bin.sh diff --git a/components/core/tools/scripts/lib_install/pipx-packages/find-pipx-bin.sh b/components/core/tools/scripts/lib_install/pipx-packages/find-pipx-bin.sh new file mode 100755 index 0000000000..cb37f09240 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx-packages/find-pipx-bin.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx not found." + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq not found." + exit 1 +fi + +if [ "$#" -ne 2 ] || [ -z "${1:-}" ] || [ -z "${2:-}" ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +pkg="$1" +app="$2" + +pipx list --json | jq --raw-output --arg pkg "$pkg" --arg app "$app" ' + .venvs[$pkg].metadata.main_package.app_paths[]?.__Path__ + | select((split("/") | last) == $app) +' | head -n1 diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh index 1497bd9282..add0047491 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -25,7 +25,8 @@ else # (see https://github.com/y-scope/clp/issues/795). pipx install --force "cmake>=${required_version_min},<${required_version_major_max_plus_1}" pipx ensurepath - cmake_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/cmake" + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + cmake_bin=$("${script_dir}/find-pipx-bin.sh" cmake cmake) echo "Pipx CMake installed at: ${cmake_bin}" fi diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh index 7a353d3918..3caa36c00c 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -12,6 +12,8 @@ fi # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 readonly required_version="3.44.0" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + go_task_bin="$(command -v task 2>/dev/null || true)" if [ -n "${go_task_bin}" ]; then package_preinstalled=0 @@ -20,11 +22,10 @@ else package_preinstalled=1 pipx install --force "go-task-bin==${required_version}" pipx ensurepath - go_task_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/task" + go_task_bin=$("${script_dir}/find-pipx-bin.sh" go-task-bin task) echo "Pipx Task installed at: ${go_task_bin}" fi -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" installed_version=$(${go_task_bin} --silent --taskfile "${script_dir}/print-go-task-version.yaml") if [[ "${installed_version}" != "${required_version}" ]]; then echo "Error: Task version ${installed_version} is currently unsupported (must be" \ diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index f14fa795f5..dce6c9f1da 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -21,7 +21,8 @@ else package_preinstalled=1 pipx install --force "uv>=${required_version_min}" pipx ensurepath - uv_bin="${PIPX_BIN_DIR:-$HOME/.local/bin}/uv" + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + uv_bin=$("${script_dir}/find-pipx-bin.sh" uv uv) echo "Pipx uv installed at: ${uv_bin}" fi From ac5ec9373ebeefbccbd81c6e7690746a19660bc4 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 25 Sep 2025 20:40:27 +0000 Subject: [PATCH 8/8] fix bug --- .../core/tools/scripts/lib_install/pipx-packages/install-uv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index dce6c9f1da..b45dcc2041 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -26,7 +26,7 @@ else echo "Pipx uv installed at: ${uv_bin}" fi -installed_version=$(uv self version --output-format json | jq --raw-output ".version") +installed_version=$(${uv_bin} self version --output-format json | jq --raw-output ".version") IFS=. read -r installd_version_major installed_version_minor _ <<<"${installed_version}" if (("${installd_version_major}" == "${required_version_major_min}" && \