From af29a4a87f29d8b4f882466456ab5adcd461cff0 Mon Sep 17 00:00:00 2001 From: bcumming Date: Tue, 2 Jul 2024 17:32:22 +0200 Subject: [PATCH] knock rough edges off the user interface, move all repo commands to "uenv repo" sub command --- activate | 7 +-- lib/datastore.py | 5 ++ uenv-image | 114 +++++++++------------------------------------ uenv-impl | 117 ++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 136 insertions(+), 107 deletions(-) diff --git a/activate b/activate index caa246b..9ad2dfd 100644 --- a/activate +++ b/activate @@ -13,15 +13,16 @@ function uenv { function uenv_usage { echo "uenv - for using user environments [version @@version@@]" echo "" - echo "Usage: uenv [--version] [--help] []" + echo "Usage: uenv [--version] [--help] [--no-color] [--verbose] []" echo "" echo "The following commands are available:" + echo " image query and pull uenv images" + echo " repo query and interact with repositories" echo " run run a command in an environment" echo " start start a new shell with an environment loaded" - echo " stop stop a shell with an environment loaded" echo " status print information about each running environment" + echo " stop stop a shell with an environment loaded" echo " view activate a view" - echo " image query and pull uenv images" echo "" echo "Type 'uenv command --help' for more information and examples for a specific command, e.g." echo " uenv start --help" diff --git a/lib/datastore.py b/lib/datastore.py index a1e5273..b350353 100644 --- a/lib/datastore.py +++ b/lib/datastore.py @@ -171,10 +171,15 @@ def repo_version(repo_path: str): terminal.info(f"exception opening {db_path}: {str(err)}") return -1 +# return 2: repo does not exist # return 1: repo is fully up to date # return 0: repo needs upgrading # return -1: unrecoverable error def repo_status(repo_path: str): + index_path = repo_path + "/index.db" + if not os.path.exists(index_path): + return 2 + version = repo_version(repo_path) if version==db_version: diff --git a/uenv-image b/uenv-image index f38f071..fe165fa 100755 --- a/uenv-image +++ b/uenv-image @@ -10,6 +10,7 @@ import copy import os import pathlib import re +import shutil import sys import textwrap import time @@ -220,28 +221,6 @@ List uenv that are available. list_parser.add_argument("-a", "--uarch", required=False, type=str) list_parser.add_argument("uenv", nargs="?", default=None, type=str) - repo_parser = subparsers.add_parser("repo", - help="Create a local file system repository.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=f"""\ -Create a local repository for managing pulled uenv images. - -{colorize("Note", "cyan")}: a local repository must be created before using {colorize("uenv image", "white")}. - -{colorize("Example", "blue")} - create a repository in the default location: - {colorize("uenv image repo", "white")} - -{colorize("Example", "blue")} - create a repository in a custom location: - {colorize("uenv image repo $SCRATCH/my-custom-repo", "white")} - -{colorize("Example", "blue")} - create a repository in a custom location, with no error if the repo already exists: - {colorize("uenv image repo --exists-ok $SCRATCH/my-custom-repo", "white")} -""") - repo_parser.add_argument("--exists-ok", action="store_true", required=False, - help="No error if the local registry exists.") - repo_parser.add_argument("repo", nargs="?", default=None, type=str, - help="The path in which to create the repository. If unset, the default location will be selected.") - deploy_parser = subparsers.add_parser("deploy", help="Deploy a uenv to the 'deploy' namespace, accessible to all users.", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -275,24 +254,6 @@ Then check that it has been deployed: deploy_parser.add_argument("source", nargs=1, type=str, metavar="SOURCE", help="The full name/version:tag, id or sha256 of the uenv to deploy.") - upgrade_parser = subparsers.add_parser("upgrade", - help="Upgrade an image repository for use with this version of uenv.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=f"""\ -Upgrade an image repository for use with this version of uenv. - -{colorize("Example", "blue")} - query the status of the repository, and whether it needs upgrading. - {colorize("uenv image upgrade --status", "white")} - -{colorize("Example", "blue")} - upgrade - {colorize("uenv image upgrade", "white")} - -{colorize("Example", "blue")} - create a repository in a custom location, with no error if the repo already exists: - {colorize("uenv --repo= image upgrade", "white")} -""") - upgrade_parser.add_argument("--status", action="store_true", required=False, - help="print the status of the repo, and whether it needs upgrading. No upgrade action will be performed.") - return parser def get_options(args): @@ -378,18 +339,25 @@ def safe_repo_open(path: str) -> datastore.FileSystemRepo: Use this for all calls to open an existing FileSystemRepo in order to provide consistent and useful error messages. """ - # TODO: add a check for the db version, and print error if incompatible try: - cache = datastore.FileSystemRepo(path) - except datastore.RepoNotFoundError as err: - terminal.error(f"""The local repository {path} does not exist. -If this is your first time using uenv, a repo in the default location can be created with this command: - {colorize(f"uenv image repo", "white")} -If you want to create a repo in a custom location , provide the path as follows: - {colorize("uenv image repo ", "white")} -where {colorize("", "white")} is the desired location. + # check that the database is up to date + status = datastore.repo_status(repo_path) + # handle a non-existant repo + if status==2: + terminal.error(f"""The local repository {repo_path} does not exist. +Use the following command + {colorize(f"uenv repo --help", "white")} +for more information. """) + # handle a repo that needs to be updated + if status==0: + terminal.error(f"""The local repository {repo_path} needs to be upgraded. Run: + {colorize(f"uenv repo status", "white")} +for more information. +""") + + cache = datastore.FileSystemRepo(path) except datastore.RepoDBError as err: terminal.error(f"""The local repository {path} had a database error. Please open a CSCS ticket, or contact the uenv dev team, with the command that created the error, and this full error message. @@ -465,6 +433,10 @@ if __name__ == "__main__": # if the record isn't already in the filesystem repo download it if cache.database.get_record(t.sha256).is_empty: terminal.stdout(f"downloading image {t.sha256} {image_size_string(t.size)}") + # clean up the path if it already exists: sometimes this causes an oras error. + if os.path.exists(image_path): + terminal.info(f"removing existing path {image_path}") + shutil.rmtree(image_path) oras.pull_uenv(source_address, image_path, t) else: terminal.stdout(f"image {t.sha256} is already available locally") @@ -517,16 +489,6 @@ if __name__ == "__main__": sys.exit(0) - elif args.command == "repo": - terminal.info(f"repo path: {repo_path}") - - try: - datastore.FileSystemRepo.create(repo_path, exists_ok=args.exists_ok) - except Exception as err: - terminal.error(f"unable to find or initialise the local registry: {str(err)}") - - sys.exit(0) - elif args.command == "deploy": source = args.source[0] terminal.info(f"request to deploy '{source}' with tags '{args.tags}'") @@ -573,37 +535,3 @@ if __name__ == "__main__": sys.exit(0) - elif args.command == "upgrade": - terminal.info(f"repo path: {repo_path}") - - status = datastore.repo_status(repo_path) - terminal.info(f"{repo_path} status: {status}") - - # print a message with no errors if the user requested status with the --status flag - if args.status: - if status==1: - terminal.stdout(f"The repository {repo_path} is up to date. No action required.") - elif status==0: - terminal.stdout(f"The repository {repo_path} needs to be upgraded using the following command:") - if args.repo is None: - terminal.stdout(f" {colorize('uenv image upgrade', 'white')}") - else: - cli_string = f"uenv --repo={repo_path} image upgrade" - terminal.stdout(f" {colorize(cli_string, 'white')}") - elif status==-1: - terminal.stdout(f"The repository {repo_path} is corrupt - nothing can be done.") - sys.exit(0) - - if status==1: - terminal.stdout(f"The repository {repo_path} is up to date. No action performed.") - sys.exit(0) - elif status==-1: - terminal.error(f"The repository {repo_path} is corrupt - nothing can be done.") - - terminal.stdout(f"The repository {repo_path} will be upgraded.") - - datastore.repo_upgrade(repo_path) - - sys.exit(0) - - diff --git a/uenv-impl b/uenv-impl index 10b5fc5..ed8cc3a 100755 --- a/uenv-impl +++ b/uenv-impl @@ -186,6 +186,36 @@ set without loading the view: help="Output the environment variable changes as json.", action="store_true") + #### repo + repo_parser = subparsers.add_parser("repo", + help="interact with a uenv repo", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( +f"""\ +{colorize("Example", "blue")} - get status information of the repo: + {colorize("uenv repo status", "white")} + {colorize("uenv --repo=$custom_repo_path repo status", "white")} + +{colorize("Example", "blue")} - upgrade the repo to be compatible with this version of uenv: + {colorize("uenv repo upgrade", "white")} + {colorize("uenv --repo=$custom_repo_path repo upgrade", "white")} +if the repository is already compatible, no changes will be applied + +{colorize("Example", "blue")} - create a repository. + {colorize("uenv repo create", "white")} +This version wil create a repository in the default location. +To create a repository in a custom location: + {colorize("uenv --repo=$custom_repo_path repo create", "white")} +""" + )) + repo_subparsers = repo_parser.add_subparsers(dest="repo_command") + repo_create_parser = repo_subparsers.add_parser("create", + help="create a new repository.") + repo_upgrade_parser = repo_subparsers.add_parser("upgrade", + help="upgrade a repository to the latest repository layout supported by the installed version of uenv.") + repo_status_parser = repo_subparsers.add_parser("status", + help="print repository status.") + #### image # Dummy sub-parser to print out the correct help when wrong keyword is used # This is never used since "uenv image" commands are forwarded to "uenv-image" @@ -469,7 +499,7 @@ def generate_command(args, env_vars): elif args.command == "view": return generate_view_command(args, env, env_vars) elif args.command == "repo": - return generate_repo_command(args, env, env_vars) + return generate_repo_command(args, env) terminal.error(f"unknown command '{args.command}'", abort=False) return shell_error @@ -596,22 +626,33 @@ def parse_image_descriptions(mnt_list, repo, uarch): terminal.info(f"search filter {img_filter}") try: - fscache = datastore.FileSystemRepo(repo_path) - except datastore.RepoNotFoundError as err: - terminal.error(f"""The local repository {repo_path} does not exist. -If this is your first time using uenv, a repo in the default location can be -created with this command: - {colorize(f"uenv image repo", "white")} -If you want to create a repo in a custom location , provide the path as follows: - {colorize("uenv image repo ", "white")} -where {colorize("", "white")} is the desired location. + # check that the database is up to date + status = datastore.repo_status(repo_path) + # handle a non-existant repo + if status==2: + terminal.error(f"""The local repository {repo_path} does not exist. +Use the following command + {colorize(f"uenv repo --help", "white")} +for more information. """, abort=False) - return [] + return [] + # handle a repo that needs to be updated + if status==0: + terminal.error(f"""The local repository {repo_path} needs to be upgraded. Run: + {colorize(f"uenv repo status", "white")} +for more information. +""", abort=False) + return [] + + fscache = datastore.FileSystemRepo(repo_path) + + #except datastore.RepoNotFoundError as err: except datastore.RepoDBError as err: terminal.error(f"""The local repository {repo_path} had a database error. Please open a CSCS ticket, or contact the uenv dev team, with the command that created the error, and this full error message. {str(err)}""", abort=False) return [] + result = fscache.database.find_records(**img_filter) if result.is_empty: @@ -893,6 +934,60 @@ def generate_stop_command(args, env): return "exit $_last_exitcode" +def generate_repo_command(args, env): + + repo_path = alps.uenv_repo_path(args.repo) + + status = datastore.repo_status(repo_path) + terminal.info(f"{repo_path} status: {status}") + + commands = [] + if args.repo_command=="create": + terminal.info(f"trying to create {repo_path}") + try: + datastore.FileSystemRepo.create(repo_path, exists_ok=True) + commands.append(f"echo 'The repository {repo_path} has been created.'") + except Exception as err: + terminal.error(f"unable to find or initialise the local registry: {str(err)}", abort=False) + return shell_noop + + elif args.repo_command=="upgrade": + # TODO: fix these magic numbers... if only Python had proper support for enums + if status==2: + terminal.error(f"The repository {repo_path} does not exist.", abort=False) + return shell_error + if status==1: + commands.append(f"echo 'The repository {repo_path} is up to date. No action performed.'") + elif status==-1: + terminal.error(f"The repository {repo_path} is corrupt or does not exist - nothing can be done.", abort=False) + return shell_error + try: + datastore.repo_upgrade(repo_path) + commands.append(f"echo 'The repository {repo_path} has been upgraded.'") + except Exception as err: + terminal.error(f"unable to upgrade {str(err)}", abort=False) + return shell_error + + # this always goes last + # print status information with or without the status flag + else: + if status==2: + commands.append(f"echo 'The repository {repo_path} does not exist.'") + if status==1: + commands.append(f"echo 'The repository {repo_path} is up to date. No action required.'") + elif status==0: + commands.append(f"echo 'The repository {repo_path} needs to be upgraded using the following command:'") + if args.repo is None: + cli_string = "uenv repo upgrade" + else: + cli_string = f"uenv --repo={repo_path} repo upgrade" + commands.append(f"echo ' {colorize(cli_string, 'white')}'") + elif status==-1: + commands.append(f"echo 'The repository {repo_path} is corrupt or does not exist.'") + + commands.append(shell_noop) + return commands + if __name__ == "__main__": parser = make_argparser()