uenv builder required changes

pipeline/templates/pipeline.yml

@@ -11,9 +11,9 @@ stages:
   variables:
     SLURM_TIMELIMIT: 180
   script:
-  - ./uenv-pipeline/stage-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/jenkssl $SPACK_DEVELOP -m $STACK_MOUNT -u $STACK_UARCH $NO_BWRAP
+  - ./uenv-pipeline/stage-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/uenv-build $SPACK_DEVELOP -m $STACK_MOUNT -u $STACK_UARCH $NO_BWRAP
   after_script:
-  - rm -Rf /dev/shm/jenkssl
+  - rm -Rf /dev/shm/uenv-build
 
 .base-test:
   stage: test
@@ -76,5 +76,5 @@ test-{{job.uenv}}/{{job.version}}-{{job.system}}/{{job.uarch}}:
 #@@cmd@@STACK_MOUNT="{{job.mount}}"
 #@@cmd@@STACK_UARCH="{{job.uarch}}"
 #@@cmd@@NO_BWRAP="{{job.no_bwrap}}"
-#@@cmd@@./uenv-pipeline/stage-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/jenkssl $SPACK_DEVELOP -m $STACK_MOUNT -u $STACK_UARCH $NO_BWRAP ${TESTRUN}
+#@@cmd@@./uenv-pipeline/stage-build -n $STACK_NAME -s $STACK_SYSTEM -r $STACK_RECIPE -b /dev/shm/uenv-build $SPACK_DEVELOP -m $STACK_MOUNT -u $STACK_UARCH $NO_BWRAP ${TESTRUN}
 {% endfor %}

stage-build

@@ -180,33 +180,47 @@ echo "env --ignore-environment PATH=/usr/bin:/bin:${PWD}/spack/bin make store.sq
 env --ignore-environment PATH=/usr/bin:/bin:${PWD}/spack/bin  HOME=$HOME https_proxy=$https_proxy http_proxy=$http_proxy no_proxy="$no_proxy" CSCS_REGISTRY_USERNAME=$jfrog_u CSCS_REGISTRY_PASSWORD=$jfrog_p make store.squashfs -kj64
 [[ $? -eq 0  ]] || env --ignore-environment PATH=/usr/bin:/bin:${PWD}/spack/bin  HOME=$HOME https_proxy=$https_proxy http_proxy=$http_proxy no_proxy="$no_proxy" CSCS_REGISTRY_USERNAME=$jfrog_u CSCS_REGISTRY_PASSWORD=$jfrog_p make store.squashfs -kj8
 
-if [ "$test_run" == "yes" ]; then
-    exit 0;
-fi
-
 if [ ! $? -eq 0 ]; then
     log "TODO: save WIP of build an store it for later inspection"
     log "pushing to build cache"
     env --ignore-environment PATH=/usr/bin:/bin:${PWD}/spack/bin  HOME=$HOME https_proxy=$https_proxy http_proxy=$http_proxy no_proxy="$no_proxy" CSCS_REGISTRY_USERNAME=$jfrog_u CSCS_REGISTRY_PASSWORD=$jfrog_p make cache-force -j32
-    cp -r "${build_path}/tmp/jenkssl/spack-stage/" $CI_PROJECT_DIR
+    cp -r "${build_path}/tmp/uenv-build/spack-stage/" $CI_PROJECT_DIR
     err "error building image"
 fi
 
+if [ "$test_run" == "yes" ]; then
+    exit 0;
+fi
+
 ##
 # Push build artifacts to JFrog
 ##
 
 rego=jfrog.svc.cscs.ch/uenv
 repo_base=build/${system}/${uarch}/${name}
 repo=${repo_base}:${build_id}
+outpath_repo=${rego}/${repo}
+if [[ -n "$REPOSITORY_OVERRIDE_NAME" ]] ; then
+    # Allow overriding path, where the data is stored.
+    # This can be a remote path, or a local path
+    outpath_repo="${REPOSITORY_OVERRIDE_NAME}"
+fi
 
-log "pushing store.squashfs to ${rego}/${repo}"
-oras push --registry-config ${jfrog_creds_path} ${rego}/${repo} --artifact-type application/x-squashfs store.squashfs
-[[ $? -eq 0  ]] || err "failed to push image"
+if [[ "${outpath_repo}" == "/"* ]] ; then
+    # local path
+    log "Copying store.squashfs and meta to ${outpath_repo}"
+    mkdir -p "$(dirname "${outpath_repo}")"
+    cp store.squashfs "${outpath_repo}"
+    cp -r store/meta "${outpath_repo}.meta"
+else
+    log "pushing store.squashfs to ${outpath_repo}"
+    oras push --registry-config ${jfrog_creds_path} ${outpath_repo} --artifact-type application/x-squashfs store.squashfs
+    [[ $? -eq 0  ]] || err "failed to push image"
 
-# push the metadata to jfrog
-log "pushing meta data to ${rego}/${repo}"
-(cd ./store; oras attach --registry-config ${jfrog_creds_path} --artifact-type uenv/meta ${rego}/${repo} ./meta)
+    # push the metadata to jfrog
+    log "pushing meta data to ${outpath_repo}"
+    (cd ./store; oras attach --registry-config ${jfrog_creds_path} --artifact-type uenv/meta ${outpath_repo} ./meta)
+fi
 
 log "clean up build path '${build_path}'"
 (rm -rf ${build_path})

util/setup-oras

@@ -25,12 +25,18 @@ oras_path=`mktemp -d`
 export PATH="$oras_path:$PATH"
 log "oras v${oras_version} installed in ${oras_path}"
 
-# Obtain credentials for JFrog - required to push the image and meta-data
-creds_json=$(curl --retry 5 --retry-connrefused --fail --silent "$CSCS_CI_MW_URL/credentials?token=$CI_JOB_TOKEN&job_id=$CI_JOB_ID&creds=container_registry")
-oras_creds="$(echo ${creds_json} | jq --join-output '"--username " + .container_registry.username + " --password " +.container_registry.password')"
-
-jfrog_u="$(echo ${creds_json} | jq -r '.container_registry.username')"
-jfrog_p="$(echo ${creds_json} | jq -r '.container_registry.password')"
+if [[ -z "$CSCS_REGISTRY_USERNAME" || -z "$CSCS_REGISTRY_PASSWORD" ]] ; then
+    # Obtain credentials for JFrog - required to push the image and meta-data
+    creds_json=$(curl --retry 5 --retry-connrefused --fail --silent "$CSCS_CI_MW_URL/credentials?token=$CI_JOB_TOKEN&job_id=$CI_JOB_ID&creds=container_registry")
+    oras_creds="$(echo ${creds_json} | jq --join-output '"--username " + .container_registry.username + " --password " +.container_registry.password')"
+
+    jfrog_u="$(echo ${creds_json} | jq -r '.container_registry.username')"
+    jfrog_p="$(echo ${creds_json} | jq -r '.container_registry.password')"
+else
+    log "using credentials provided in the environment"
+    jfrog_u="$CSCS_REGISTRY_USERNAME"
+    jfrog_p="$CSCS_REGISTRY_PASSWORD"
+fi
 
 log "log in to jfrog.svc.cscs.ch with oras"