From 6019361c812c8e77e38c6f1d9613a46f97f366ed Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Tue, 28 Nov 2023 13:02:28 +0400 Subject: [PATCH 1/9] Add info about `packages` and `additionalVersions` inputs --- src/python/devcontainer-feature.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7ebdad38a..3dcdf7180 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -22,6 +22,11 @@ "default": "os-provided", "description": "Select a Python version to install." }, + "additionalVersions": { + "type": "string", + "default": "", + "description": "Enter additional Python versions, separated by commas. Use 'X.Y' or 'X.Y.Z' for a specific version" + }, "installTools": { "type": "boolean", "default": true, @@ -47,6 +52,11 @@ "default": "", "description": "Configure JupyterLab to accept HTTP requests from the specified origin" }, + "packages": { + "type": "string", + "default": "", + "description": "Optional comma separated list of Python packages to install with pip" + }, "httpProxy": { "type": "string", "default": "", From c53818c5ff2e6029a1c2ee4cba7a2f6f4a78e199 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Tue, 28 Nov 2023 13:06:45 +0400 Subject: [PATCH 2/9] Introduce `packages` option - Create `utils.sh` files; - Move `sudo_if` to utils.sh`; - Replace `install_user_package` with `install_python_package` and move function to utils.sh`; - Add logic to handle `packages` input; --- src/python/install.sh | 45 ++++++++++++++++++++++++++++++++----------- src/python/utils.sh | 33 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 src/python/utils.sh diff --git a/src/python/install.sh b/src/python/install.sh index e6eefd055..5919396eb 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -26,6 +26,13 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" +# Comma-separated list of packages to be installed +# to Python version specified in PYTHON_VERSION +PYTHON_PACKAGES="${PACKAGES:-""}" + +# Import common utils +. ./utils.sh + DEFAULT_UTILS=("pylint" "flake8" "autopep8" "black" "yapf" "mypy" "pydocstyle" "pycodestyle" "bandit" "pipenv" "virtualenv" "pytest") PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5 A821E680E5FA6305" GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com @@ -300,15 +307,6 @@ install_using_oryx() { add_symlink } -sudo_if() { - COMMAND="$*" - if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then - su - "$USERNAME" -c "$COMMAND" - else - $COMMAND - fi -} - install_user_package() { INSTALL_UNDER_ROOT="$1" PACKAGE="$2" @@ -317,6 +315,8 @@ install_user_package() { sudo_if "${PYTHON_SRC}" -m pip install --upgrade --no-cache-dir "$PACKAGE" else sudo_if "${PYTHON_SRC}" -m pip install --user --upgrade --no-cache-dir "$PACKAGE" + + sudo_if "${PYTHON_SRC}" -m pip show "$PACKAGE" fi } @@ -473,8 +473,8 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then INSTALL_UNDER_ROOT=false fi - install_user_package $INSTALL_UNDER_ROOT jupyterlab - install_user_package $INSTALL_UNDER_ROOT jupyterlab-git + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab-git # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then @@ -491,6 +491,29 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then fi fi +# Install pacakages if needed +if [ ! -z "${PYTHON_PACKAGES}" ]; then + if [ -z "${PYTHON_SRC}" ]; then + echo "(!) Could not install packages. Python not found." + exit 1 + fi + + INSTALL_UNDER_ROOT=true + if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then + INSTALL_UNDER_ROOT=false + fi + + OLDIFS=$IFS + IFS="," + read -a python_packages <<< "$PYTHON_PACKAGES" + for package in "${python_packages[@]}"; do + name=${test%==*} + version=${test#*==} + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC $name $version + done + IFS=$OLDIFS +fi + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/src/python/utils.sh b/src/python/utils.sh new file mode 100644 index 000000000..6ede933d2 --- /dev/null +++ b/src/python/utils.sh @@ -0,0 +1,33 @@ +sudo_if() { + COMMAND="$*" + if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then + su - "$USERNAME" -c "$COMMAND" + else + $COMMAND + fi +} + +install_python_package() { + INSTALL_UNDER_ROOT="$1" + PYTHON_PATH="$2" + PACKAGE="$3" + VERSION="${4:-""}" + + sudo_if "${PYTHON_PATH}" -m pip uninstall --yes "${PACKAGE}" + + install_command="-m pip install --upgrade --no-cache-dir " + + if [ "$INSTALL_UNDER_ROOT" = false ]; then + install_command+="--user " + fi + + install_command+="${PACKAGE}" + + if [ ! -z "${VERSION}" ]; then + install_command+="==${VERSION}" + fi + + sudo_if "${PYTHON_PATH} ${install_command}" + + sudo_if "${PYTHON_PATH}" -m pip --no-python-version-warning show "${PACKAGE}" +} From e0dd23b8c1fc026c8173e0211832998be8418aa8 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Tue, 28 Nov 2023 17:21:34 +0400 Subject: [PATCH 3/9] Rework function to avoid issues with tokenization --- src/python/utils.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/utils.sh b/src/python/utils.sh index 6ede933d2..f5d06e8a9 100644 --- a/src/python/utils.sh +++ b/src/python/utils.sh @@ -13,9 +13,9 @@ install_python_package() { PACKAGE="$3" VERSION="${4:-""}" - sudo_if "${PYTHON_PATH}" -m pip uninstall --yes "${PACKAGE}" + sudo_if "$PYTHON_PATH -m pip uninstall --yes $PACKAGE" - install_command="-m pip install --upgrade --no-cache-dir " + install_command=" -m pip install --upgrade --no-cache-dir " if [ "$INSTALL_UNDER_ROOT" = false ]; then install_command+="--user " @@ -27,7 +27,7 @@ install_python_package() { install_command+="==${VERSION}" fi - sudo_if "${PYTHON_PATH} ${install_command}" + sudo_if "$PYTHON_PATH$install_command" - sudo_if "${PYTHON_PATH}" -m pip --no-python-version-warning show "${PACKAGE}" + sudo_if "$PYTHON_PATH -m pip --no-python-version-warning show $PACKAGE" } From b50f5a3bdd5bf2ea98f141820e26c5557719a07e Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Wed, 29 Nov 2023 13:54:13 +0400 Subject: [PATCH 4/9] Add tests --- test/python/install_packages_mixed.sh | 23 ++++++++++++ test/python/install_packages_non_root.sh | 23 ++++++++++++ test/python/install_packages_root.sh | 19 ++++++++++ .../install_packages_versions_locked.sh | 23 ++++++++++++ test/python/scenarios.json | 36 +++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100644 test/python/install_packages_mixed.sh create mode 100644 test/python/install_packages_non_root.sh create mode 100644 test/python/install_packages_root.sh create mode 100644 test/python/install_packages_versions_locked.sh diff --git a/test/python/install_packages_mixed.sh b/test/python/install_packages_mixed.sh new file mode 100644 index 000000000..594e68b99 --- /dev/null +++ b/test/python/install_packages_mixed.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults diff --git a/test/python/install_packages_non_root.sh b/test/python/install_packages_non_root.sh new file mode 100644 index 000000000..594e68b99 --- /dev/null +++ b/test/python/install_packages_non_root.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults diff --git a/test/python/install_packages_root.sh b/test/python/install_packages_root.sh new file mode 100644 index 000000000..7d55cb44d --- /dev/null +++ b/test/python/install_packages_root.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults diff --git a/test/python/install_packages_versions_locked.sh b/test/python/install_packages_versions_locked.sh new file mode 100644 index 000000000..594e68b99 --- /dev/null +++ b/test/python/install_packages_versions_locked.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 23b6cfc92..6d598a580 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -81,5 +81,41 @@ "version": "3.12" } } + }, + "install_packages_root": { + "image": "debian:bullseye-slim", + "features": { + "python": { + "version": "3.11", + "packages": "jupyterlab,jupyterlab-git" + } + } + }, + "install_packages_non_root": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.11", + "packages": "jupyterlab,jupyterlab-git" + } + } + }, + "install_packages_versions_locked": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.11", + "packages": "cryptography==41.0.5,jupyterlab==4.0.8,jupyterlab-git==0.43.0" + } + } + }, + "install_packages_mixed": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.11", + "packages": "cryptography==41.0.5,jupyterlab,jupyterlab-git==0.43.0" + } + } } } From 91ffd5b70d3ac6094291525c49b443b9344e48ea Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 30 Nov 2023 13:45:16 +0400 Subject: [PATCH 5/9] [test] Update logic for `packages` input --- src/python/install.sh | 4 ++-- src/python/utils.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index 5919396eb..7923d8ebe 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -507,8 +507,8 @@ if [ ! -z "${PYTHON_PACKAGES}" ]; then IFS="," read -a python_packages <<< "$PYTHON_PACKAGES" for package in "${python_packages[@]}"; do - name=${test%==*} - version=${test#*==} + name=$(echo ${package} | awk -F == '{ print $1 }') + version=$(echo ${package} | awk -F == '{ print $2 }') install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC $name $version done IFS=$OLDIFS diff --git a/src/python/utils.sh b/src/python/utils.sh index f5d06e8a9..fcd6e2968 100644 --- a/src/python/utils.sh +++ b/src/python/utils.sh @@ -13,6 +13,8 @@ install_python_package() { PACKAGE="$3" VERSION="${4:-""}" + set -e + sudo_if "$PYTHON_PATH -m pip uninstall --yes $PACKAGE" install_command=" -m pip install --upgrade --no-cache-dir " From 5ef0e17ad864f14cbc83406701a2b8ab64b4aebf Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 30 Nov 2023 13:46:14 +0400 Subject: [PATCH 6/9] Remove `install_user_package` function --- src/python/install.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index 7923d8ebe..1366e8f37 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -307,19 +307,6 @@ install_using_oryx() { add_symlink } -install_user_package() { - INSTALL_UNDER_ROOT="$1" - PACKAGE="$2" - - if [ "$INSTALL_UNDER_ROOT" = true ]; then - sudo_if "${PYTHON_SRC}" -m pip install --upgrade --no-cache-dir "$PACKAGE" - else - sudo_if "${PYTHON_SRC}" -m pip install --user --upgrade --no-cache-dir "$PACKAGE" - - sudo_if "${PYTHON_SRC}" -m pip show "$PACKAGE" - fi -} - add_user_jupyter_config() { CONFIG_DIR="$1" CONFIG_FILE="$2" From e8fb1f1b0bb635c96913ed3173d748807dc804c5 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 30 Nov 2023 16:13:57 +0400 Subject: [PATCH 7/9] [test] Rework `install_python_package` function --- src/python/utils.sh | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/python/utils.sh b/src/python/utils.sh index fcd6e2968..2ca00d0de 100644 --- a/src/python/utils.sh +++ b/src/python/utils.sh @@ -13,23 +13,19 @@ install_python_package() { PACKAGE="$3" VERSION="${4:-""}" - set -e + sudo_if "$PYTHON_PATH" -m pip uninstall --yes "$PACKAGE" - sudo_if "$PYTHON_PATH -m pip uninstall --yes $PACKAGE" - - install_command=" -m pip install --upgrade --no-cache-dir " - - if [ "$INSTALL_UNDER_ROOT" = false ]; then - install_command+="--user " - fi - - install_command+="${PACKAGE}" + install_package="${PACKAGE}" if [ ! -z "${VERSION}" ]; then - install_command+="==${VERSION}" + install_package+="==${VERSION}" fi - sudo_if "$PYTHON_PATH$install_command" + if [ "$INSTALL_UNDER_ROOT" = true ]; then + sudo_if "$PYTHON_PATH" -m pip install --upgrade --no-cache-dir "$install_package" + else + sudo_if "$PYTHON_PATH" -m pip install --upgrade --user --no-cache-dir "$install_package" + fi - sudo_if "$PYTHON_PATH -m pip --no-python-version-warning show $PACKAGE" + sudo_if "$PYTHON_PATH" -m pip --no-python-version-warning show "$PACKAGE" } From 0e06679cac9aa4b1efd265d9849e77407a0c886e Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 30 Nov 2023 16:56:45 +0400 Subject: [PATCH 8/9] Update scenarios.json --- test/python/scenarios.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 6d598a580..1e53ae7d4 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -92,7 +92,8 @@ } }, "install_packages_non_root": { - "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { "version": "3.11", @@ -101,7 +102,8 @@ } }, "install_packages_versions_locked": { - "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { "version": "3.11", @@ -110,7 +112,8 @@ } }, "install_packages_mixed": { - "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { "version": "3.11", From d1f43d335407cdf1c42c2c63dd7eef846894adb0 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Thu, 30 Nov 2023 16:57:52 +0400 Subject: [PATCH 9/9] Bump minor version --- src/python/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 3dcdf7180..247808ab0 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.3.1", + "version": "1.4.1", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.",