From d9b5b52eff1b31037095e7a0ed182bbe20526d18 Mon Sep 17 00:00:00 2001 From: Brad Armstrong <46848390+bradarm@users.noreply.github.com> Date: Fri, 26 Jul 2024 15:04:07 -0700 Subject: [PATCH] stage wheels into coresvc-registry (#88) * adds pyi helper scripts that will later be moved to documentation * stage the spacefx wheel into coresvc-registry's pypi server * removes pypiserver directory * move the wheels instead of copying them * adds wheel/.gitkeep --------- Co-authored-by: Brad Armstrong --- .vscode/copy_to_spacedev.sh | 2 +- chart/templates/coresvc_registry.yaml | 8 +-- pypiserver/scripts/stage_python_packages.sh | 68 ------------------- scripts/coresvc_registry.sh | 2 +- scripts/stage_spacefx.sh | 31 ++++----- pypiserver/requirements.txt => wheel/.gitkeep | 0 6 files changed, 19 insertions(+), 92 deletions(-) delete mode 100644 pypiserver/scripts/stage_python_packages.sh rename pypiserver/requirements.txt => wheel/.gitkeep (100%) diff --git a/.vscode/copy_to_spacedev.sh b/.vscode/copy_to_spacedev.sh index 6322074..c3574e5 100755 --- a/.vscode/copy_to_spacedev.sh +++ b/.vscode/copy_to_spacedev.sh @@ -86,7 +86,7 @@ function main() { copy_directory_to_dest --directory "env" copy_directory_to_dest --directory "modules" copy_directory_to_dest --directory "protos" - copy_directory_to_dest --directory "pypiserver" + copy_directory_to_dest --directory "wheel" copy_directory_to_dest --directory "scripts" while read -r shellFile; do diff --git a/chart/templates/coresvc_registry.yaml b/chart/templates/coresvc_registry.yaml index 71d41c9..925ce29 100644 --- a/chart/templates/coresvc_registry.yaml +++ b/chart/templates/coresvc_registry.yaml @@ -63,12 +63,12 @@ spec: mountPath: /var/lib/registry - name: registry-cert-mount mountPath: /certs - - name: pypiserver-data-mount - mountPath: /data + - name: pypiserver-packages-mount + mountPath: /data/packages volumes: - - name: pypiserver-data-mount + - name: pypiserver-packages-mount hostPath: - path: {{ $globalValues.spacefxDirectories.base }}/registry/pypiserver + path: {{ $globalValues.spacefxDirectories.base }}/registry/pypiserver/packages type: DirectoryOrCreate - name: registry-data-mount hostPath: diff --git a/pypiserver/scripts/stage_python_packages.sh b/pypiserver/scripts/stage_python_packages.sh deleted file mode 100644 index f56165d..0000000 --- a/pypiserver/scripts/stage_python_packages.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# -# Downloads the python packages specified in pypiserver/requirements.txt via pip -# and uploads them to coresvc-registry's pypi server via twine. -# -# Example Usage: -# -# bash ./pypiserver/scripts/stage_python_packages.sh - -############################################################ -# Script variables -############################################################ - -# The directory where this script is located -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# The directory where the python packages will be downloaded to -PACKAGE_STAGING_DIR="/data/staging" - -# The endpoint of the coresvc-registry's pypi server -SPACEFX_PYPI_SERVER="https://localhost:8080" - -############################################################ -# Help -############################################################ - -function show_help() { - echo "Usage: $0" - echo "" - echo "Downloads the python packages specified in pypiserver/requirements.txt via pip" - echo "and uploads them to coresvc-registry's pypi server via twine." - echo "" - echo "Example Usage:" - echo "" - echo "bash ./pypiserver/scripts/stage_python_packages.sh" - echo "" - exit 1 -} - -############################################################ -# Main -############################################################ - -# Check if requirements.txt exists -if [ ! -f "${SCRIPT_DIR}/../requirements.txt" ]; then - echo "ERROR: pypiserver/requirements.txt does not exist." - show_help -elif [ ! -s "${SCRIPT_DIR}/../requirements.txt" ]; then - echo "WARNING: pypiserver/requirements.txt is empty. Skipping staging." - exit 0 -fi - -# Empty and recreate the package staging directory -rm -rf "${PACKAGE_STAGING_DIR}" -mkdir -p "${PACKAGE_STAGING_DIR}" - -# Install the python packages specified in requirements.txt -echo "Downloading python packages to ${PACKAGE_STAGING_DIR}" -pip download -r "${SCRIPT_DIR}/../requirements.txt" -d "${PACKAGE_STAGING_DIR}" - -# Upload the python packages to coresvc-registry's pypi server -for package in $(ls ${PACKAGE_STAGING_DIR}); do - echo "Uploading ${package} to ${SPACEFX_PYPI_SERVER}" - /pypi-server/bin/twine upload --repository-url "${SPACEFX_PYPI_SERVER}" --username anonymous --password none "${PACKAGE_STAGING_DIR}/${package}" -done - -# Clean up the package staging directory -rm -rf "${PACKAGE_STAGING_DIR}" diff --git a/scripts/coresvc_registry.sh b/scripts/coresvc_registry.sh index 70eb56e..153d96b 100755 --- a/scripts/coresvc_registry.sh +++ b/scripts/coresvc_registry.sh @@ -326,7 +326,7 @@ function start_registry_docker(){ -v ${SPACEFX_DIR}/registry/data:/var/lib/registry \ -v ${SPACEFX_DIR}/certs/registry:/certs \ -v ${SPACEFX_DIR}/certs/ca/ca.spacefx.local.pem:/etc/pki/ca-trust/source/anchors/ca.spacefx.local.pem:ro \ - -v ${SPACEFX_DIR}/pypiserver:/data \ + -v ${SPACEFX_DIR}/registry/pypiserver/packages:/data/packages \ -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.spacefx.local.crt \ -e REGISTRY_HTTP_TLS_KEY=/certs/registry.spacefx.local.key \ --name=${REGISTRY_REPO} ${coresvc_registry_parent}/${_repo_name}:${spacefx_version_tag}" diff --git a/scripts/stage_spacefx.sh b/scripts/stage_spacefx.sh index 6295709..9c72c27 100755 --- a/scripts/stage_spacefx.sh +++ b/scripts/stage_spacefx.sh @@ -506,21 +506,17 @@ function stage_build_artifacts(){ function stage_python_wheels(){ info_log "START: ${FUNCNAME[0]}" - if [[ ! -f "${SPACEFX_DIR}/pypiserver/requirements.txt" ]]; then - info_log "...no requirements file found at '${SPACEFX_DIR}/pypiserver/requirements.txt'. Nothing to do." - info_log "END: ${FUNCNAME[0]}" - return - fi + # Stage the microsoft-azure-orbital-sdk wheel to the wheel directory + run_a_script "${SPACEFX_DIR}/scripts/stage/stage_build_artifact.sh --architecture ${ARCHITECTURE} --artifact microsoftazurespacefx-${SPACEFX_VERSION}-py3-none-any.whl" - if [[ ! -s "${SPACEFX_DIR}/pypiserver/requirements.txt" ]]; then - info_log "...the requirements file '${SPACEFX_DIR}/pypiserver/requirements.txt' is empty. Nothing to do. Nothing to do." - info_log "END: ${FUNCNAME[0]}" - return - fi + # Find all wheels in the ${SPACEFX_DIR}/wheel directory and move them to the ${SPACEFX_DIR}/registry/pypiserver/packages directory + # Move only the wheel itself to the root level of the ${SPACEFX_DIR}/registry/pypiserver/packages directory and not the entire directory structure + # The pypiserver will detect these wheels and serve the python packages when it boots up + run_a_script "mkdir -p ${SPACEFX_DIR}/registry/pypiserver/packages" + run_a_script "find ${SPACEFX_DIR}/wheel -type f -name '*.whl' -exec mv {} ${SPACEFX_DIR}/registry/pypiserver/packages \;" - info_log "Staging python wheels..." - run_a_script "docker exec coresvc-registry /data/scripts/stage_python_packages.sh" - info_log "...successfully staged python wheels." + # Remove the wheel directory + run_a_script "rm -rf ${SPACEFX_DIR}/wheel" info_log "FINISHED: ${FUNCNAME[0]}" } @@ -594,6 +590,10 @@ function main() { enable_fileserver _generate_spacefx_config_json + info_log "Staging python wheels..." + stage_python_wheels + info_log "...successfully staged python wheels" + info_log "Staging coresvc-registry..." stage_coresvc_registry info_log "...successfully staged coresvc-registry" @@ -626,13 +626,8 @@ function main() { stage_spacefx_service_images --service_group host info_log "...service images successfully staged." - - stage_python_wheels - - remove_channels_from_extra_containers - info_log "Stopping coresvc-registry..." run_a_script "${SPACEFX_DIR}/scripts/coresvc_registry.sh --stop" info_log "...successfully stopped coresvc-registry" diff --git a/pypiserver/requirements.txt b/wheel/.gitkeep similarity index 100% rename from pypiserver/requirements.txt rename to wheel/.gitkeep