JFrog integration in the CLI

.github/workflows/artifacts.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: setup-environment
         run: |
-          zypper install -y libmount-devel gcc make rpm-build git sudo python3
+          zypper install -y libmount-devel gcc make rpm-build git sudo python3 wget
       - name: clone
         uses: actions/checkout@v2
       - name: build-srpm

.gitignore

@@ -6,3 +6,5 @@ test/tmp
 
 # site generated by mkdocs
 site
+
+__pycache__

VERSION

@@ -1 +1 @@
-3.0.1
+4.0.0

activate

@@ -1,8 +1,10 @@
 # only run if bash or zsh
-[ -n "${BASH_VERSION:-}" -o -n "${ZSH_VERSION:-}" ] || return 0
+[ -n "${BASH_VERSION:-}" ] || return 0
 
 export UENV_CMD=@@impl@@
+export UENV_IMG_CMD=@@image_impl@@
 export UENV_VERSION=@@version@@
+export UENV_PREFIX=@@prefix@@
 
 function uenv {
     local _last_exitcode=$?
@@ -12,23 +14,46 @@ function uenv {
         echo ""
         echo "Usage:   uenv [--version] [--help] <command> [<args>]"
         echo ""
-        echo "the following commands are available"
+        echo "The following commands are available:"
         echo "  run             run a command in an environment"
         echo "  start           start a new shell with an environment loaded"
         echo "  stop            stop a shell with an environment loaded"
         echo "  status          print information about each running environment"
         echo "  modules         view module status and activate with --use"
         echo "  view            activate a view"
+        echo "  image           query and pull uenv images"
+        echo ""
+        echo "Type 'uenv command --help' for more information and examples for a specific command, e.g."
+        echo "  uenv start --help"
+        echo "  uenv image find --help"
     }
 
     if [ "$1" = "--version" ]; then
         echo "@@version@@";
-    elif [[ $# -eq 0 || "$1" = "--help" || "$1" = "-h" ]]; then
-        uenv_usage;
-    elif [[ " $* " =~ [[:space:]](-h|--help)[[:space:]] ]]; then
-        echo "$($UENV_CMD "$@")"
     else
-        eval "$($UENV_CMD "$@")"
+        flags=""
+        while [[ $# -gt 0 ]]; do
+            case $1 in
+                -?*|--*)
+                    flags="$flags $1"
+                    shift
+                    ;;
+                *)
+                    break
+                    ;;
+            esac
+        done
+
+        if [[ $# -eq 0 ]]; then
+            uenv_usage;
+        elif [ "$1" = "image" ]; then
+            shift;
+            $UENV_IMG_CMD $flags "$@"
+        elif [[ " $* " =~ [[:space:]](-h|--help)[[:space:]] ]]; then
+            echo "$($UENV_CMD $flags "$@")"
+        else
+            eval "$($UENV_CMD $flags "$@")"
+        fi
     fi
 
     unset -f uenv_usage

docs/images.md

@@ -0,0 +1,24 @@
+
+the `image` 
+```
+# list images available locally
+uenv image pull gromacs/2023
+# pull directly from: requires credentials
+uenv image pull --full <https://jfrog...> --name test/2023
+
+# list images available locally
+uenv image find
+# list images available on JFrog
+uenv image avail
+uenv image avail --cluster=eiger
+```
+
+updates to other commands
+```
+# this should return information about the current cluster?
+> uenv status
+no uenv loaded
+on cluster eiger
+```
+
+

install

@@ -5,6 +5,20 @@ set -euo pipefail
 script_path=$(dirname "$(readlink -f "$BASH_SOURCE")")
 version=$(cat "$script_path/VERSION")
 
+# default installation destination, only required for building rpm packages
+destdir=""
+
+get_oras_arch() {
+  case "$(uname -m)" in
+    aarch64)
+      echo "arm64"
+      ;;
+    *)
+      echo "amd64"
+      ;;
+  esac
+}
+
 function usage {
     echo "uenv installer"
     echo "Usage: install [--prefix=] [--yes] [--help]"
@@ -117,29 +131,57 @@ run () {
     fi
 }
 
-echo "installing uenv version $version in $prefix"
+echo "installing uenv version $version in $prefix from $script_path"
 echo "local install: $local_install"
 
+libexec=${prefix}/libexec
+impl_path="${libexec}/uenv-impl"
+img_impl_path="${libexec}/uenv-image"
+lib_path="${libexec}/lib"
 if [ "$local_install" == "no" ]
 then
     init_path="${destdir}/etc/profile.d/uenv.sh"
-    impl_path="${prefix}/libexec/uenv-impl"
-    impl_path_install="${destdir}/${impl_path}"
-
 else
     init_path="${prefix}/bin/activate-uenv"
-    impl_path="${prefix}/libexec/uenv-impl"
-    impl_path_install="${impl_path}"
 fi
 
+echo "installing $lib_path"
+run install -v -m755 -d "$lib_path"
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/alps.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/datastore.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/flock.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/jfrog.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/names.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/oras.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/record.py
+run install -v -m644 -D -t "${destdir}/${lib_path}" ${script_path}/lib/terminal.py
+
+echo "installing ${img_impl_path}"
+run install -v -m755 -D "$script_path/uenv-image" "${destdir}/${img_impl_path}"
+
+## ALL GOOD BELOW HERE
+
 echo "installing $impl_path"
-run install -D -m755 <(sed "s|@@version@@|$version|g" $script_path/uenv-impl) $impl_path_install
+run install -D -m755 <(sed "s|@@version@@|$version|g" $script_path/uenv-impl) ${destdir}/$impl_path
+
+# install the oras client
+echo "installing oras client"
+oras_arch=$(get_oras_arch)
+oras_version=1.1.0
+oras_file=oras_${oras_version}_linux_${oras_arch}.tar.gz
+oras_url=https://github.com/oras-project/oras/releases/download/v${oras_version}/${oras_file}
+oras_path=`mktemp -d`
+(cd "$oras_path"; run wget --quiet "$oras_url"; run tar -xzf "$oras_file"; run rm *.tar.gz)
+oras_exe_path="${libexec}/uenv-oras"
+run install -v -m755 -D "${oras_path}/oras" "${destdir}/${oras_exe_path}"
 
 echo "installing $init_path"
-run install -D -m644 <(
+run install -v -m644 -D <(
     sed \
-        -e "s|@@impl@@|$impl_path|g" \
+        -e "s|@@impl@@|$impl_path|g"  \
         -e "s|@@version@@|$version|g" \
+        -e "s|@@prefix@@|$prefix|g"   \
+        -e "s|@@image_impl@@|$img_impl_path|g" \
         $script_path/activate) \
       $init_path
 

lib/alps.py

@@ -0,0 +1,22 @@
+# arbitrary alps-specific gubbins
+
+import os
+
+# the path used to store a users cached images and meta data
+def uenv_repo_path(path: str=None) -> str:
+    if path is not None:
+        return path
+
+    # check whether the image path has been explicitly set:
+    path = os.environ.get("UENV_REPO_PATH")
+    if path is not None:
+        return path
+
+    # if not, try to use the path $SCRATCH/.uenv-images/, if SCRATCH exists
+    path = os.environ.get("SCRATCH")
+    if path is not None:
+        return path + "/.uenv-images"
+
+    terminal.error("No repository path available: set UENV_REPO_PATH or use the --repo flag")
+
+

lib/datastore.py

@@ -0,0 +1,159 @@
+import json
+import os
+
+from record import Record
+import terminal
+import names
+
+UENV_CLI_API_VERSION=1
+
+class DataStore:
+
+    def __init__(self):
+        # all images store with (key,value) = (sha256,Record)
+        self._images = {}
+        self._short_sha = {}
+
+        self._store = {"system": {}, "uarch": {}, "name": {}, "version": {}, "tag": {}}
+
+    def add_record(self, r: Record):
+        sha = r.sha256
+        short_sha = r.sha256[:16]
+        self._images.setdefault(sha, []).append(r)
+        # check for (exceedingly unlikely) collision
+        if short_sha in self._short_sha:
+            old_sha = self._short_sha[short_sha]
+            if sha != old_sha:
+                terminal.error('image hash collision:\n  {sha}\n  {old_sha}')
+        self._short_sha[sha[:16]] = sha
+        self._store["system"] .setdefault(r.system, []).append(sha)
+        self._store["uarch"]  .setdefault(r.uarch, []).append(sha)
+        self._store["name"]   .setdefault(r.name, []).append(sha)
+        self._store["version"].setdefault(r.version, []).append(sha)
+        self._store["tag"]    .setdefault(r.tag, []).append(sha)
+
+    def find_records(self, **constraints):
+        if not constraints:
+            raise ValueError("At least one constraint must be provided")
+
+        for field in constraints:
+            if (field != "sha") and (field not in self._store):
+                raise ValueError(f"Invalid field: {field}. Must be one of 'system', 'uarch', 'name', 'version', 'tag', 'sha'")
+
+        if "sha" in constraints:
+            sha = constraints["sha"]
+            matching_records_sets = [set()]
+            if len(sha)<64:
+                if sha in self._short_sha:
+                    matching_records_sets = [set([self._short_sha[sha]])]
+            else:
+                if sha in self._images:
+                    matching_records_sets = [set([sha])]
+        else:
+            # Find matching records for each constraint
+            matching_records_sets = [
+                set(self._store[field].get(value, [])) for field, value in constraints.items()
+            ]
+
+        # Intersect all sets of matching records
+        if matching_records_sets:
+            unique = set.intersection(*matching_records_sets)
+        else:
+            unique = set()
+
+        results = []
+        for sha in unique:
+            results += (self._images[sha])
+        results.sort(reverse=True)
+        return results
+
+    @property
+    def images(self):
+        return self._images
+
+    # return a list of records that match a sha
+    def get_record(self, sha: str) -> Record:
+        if names.is_full_sha256(sha):
+            return self._images.get(sha, [])
+        elif names.is_short_sha256(sha):
+            return self._images.get(self._short_sha[sha], [])
+        raise ValueError(f"{sha} is not a valid sha256 or short (16 character) sha")
+
+    # Convert to a dictionary that can be written to file as JSON
+    # The serialisation and deserialisation are central: able to represent
+    # uenv that are available in both JFrog and filesystem directory tree.
+    def serialise(self, version: int=UENV_CLI_API_VERSION):
+        image_list = []
+        for x in self._images.values():
+            image_list += x
+        terminal.info(f"serialized image list in datastore: {image_list}")
+        return {
+                "API_VERSION": version,
+                "images": [img.dictionary for img in image_list]
+        }
+
+    # Convert to a dictionary that can be written to file as JSON
+    # The serialisation and deserialisation are central: able to represent
+    # uenv that are available in both JFrog and filesystem directory tree.
+    @classmethod
+    def deserialise(cls, datastore):
+        result = cls()
+        for img in datastore["images"]:
+            result.add_record(Record.from_dictionary(img))
+
+class FileSystemCache():
+    def __init__(self, path: str):
+        self._path = path
+        self._index = path + "/index.json"
+
+        if not os.path.exists(self._index):
+            # error: cache does not exists
+            raise FileNotFoundError(f"filesystem cache not found {self._path}")
+
+        with open(self._index, "r") as fid:
+            raw = json.loads(fid.read())
+            self._database = DataStore()
+            for img in raw["images"]:
+                self._database.add_record(Record.fromjson(img))
+
+    @staticmethod
+    def create(path: str, exists_ok: bool=False):
+        if not os.path.exists(path):
+            terminal.info(f"FileSyStemCache: creating path {path}")
+            os.makedirs(path)
+        index_file = f"{path}/index.json"
+        if not os.path.exists(index_file):
+            terminal.info(f"FileSyStemCache: creating empty index {index_file}")
+            empty_config = { "API_VERSION": UENV_CLI_API_VERSION, "images": [] }
+            with open(index_file, "w") as f:
+                # default serialisation is str to serialise the pathlib.PosixPath
+                f.write(json.dumps(empty_config, sort_keys=True, indent=2, default=str))
+                f.write("\n")
+
+        terminal.info(f"FileSyStemCache: available {index_file}")
+
+    @property
+    def database(self):
+        return self._database
+
+    def add_record(self, record: Record):
+        self._database.add_record(record)
+
+    # The path where an image would be stored
+    # will return a path even for images that are not stored
+    def image_path(self, r: Record) -> str:
+        return self._path + "/images/" + r.sha256
+
+    # Return the full record for a given hash
+    # Returns None if no image with that hash is stored in the repo.
+    def get_record(self, sha256: str):
+        if not names.is_valid_sha(sha256):
+            raise ValueError(f"{sha256} is not a valid image sha256 (neither full 64 or short 16 character form)")
+        return self._database.get_record(sha256)
+
+    def publish(self):
+        with open(self._index, "w") as f:
+            # default serialisation is str to serialise the pathlib.PosixPath
+            f.write(json.dumps(self._database.serialise(), sort_keys=True, indent=2, default=str))
+            f.write("\n")
+