From b865552fe67d87d360b02efa992f414fbcc67744 Mon Sep 17 00:00:00 2001 From: bcumming Date: Fri, 19 Jul 2024 17:35:38 +0200 Subject: [PATCH] reasonably robust --only-meta support, --no-header flag, --force pull flag --- lib/oras.py | 106 +++++++++++++++++++++++++++------------------------ todo-meta.md | 9 +++++ todo.md | 7 ++-- uenv-image | 82 ++++++++++++++++++++++++++++++--------- 4 files changed, 133 insertions(+), 71 deletions(-) create mode 100644 todo-meta.md diff --git a/lib/oras.py b/lib/oras.py index 146dbd9..f87ee28 100644 --- a/lib/oras.py +++ b/lib/oras.py @@ -68,63 +68,69 @@ def run_command(args): terminal.error(f"oras command failed with unknown exception: {e}") -def pull_uenv(source_address, image_path, target): +def pull_uenv(source_address, image_path, target, pull_meta, pull_sqfs): # download the image using oras try: - # remove the old path if it exists - if os.path.exists(image_path): - shutil.rmtree(image_path) - time.sleep(0.1) - terms = source_address.rsplit(":", 1) source_address = terms[0] tag = terms[1] - # step 1: download the meta data if there is any - #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1 - terminal.info(f"discovering meta data") - proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"]) - stdout, stderr = proc.communicate() - if proc.returncode == 0: - terminal.info(f"successfully downloaded meta data info: {stdout}") - else: - msg = error_message_from_stderr(stderr) - terminal.error(f"failed to find meta data: {stderr}\n{msg}") - - manifests = json.loads(stdout)["manifests"] - if len(manifests)==0: - terminal.error(f"no meta data is available") - - digest = manifests[0]["digest"] - terminal.info(f"meta data digest: {digest}") + if pull_meta: + if os.path.exists(image_path+"/meta"): + shutil.rmtree(image_path+"/meta") + time.sleep(0.05) + + # step 1: download the meta data if there is any + #oras discover -o json --artifact-type 'uenv/meta' jfrog.svc.cscs.ch/uenv/deploy/eiger/zen2/cp2k/2023:v1 + terminal.info(f"discovering meta data") + proc = run_command_internal(["discover", "-o", "json", "--artifact-type", "uenv/meta", f"{source_address}:{tag}"]) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + terminal.info(f"successfully downloaded meta data info: {stdout}") + else: + msg = error_message_from_stderr(stderr) + terminal.error(f"failed to find meta data: {stderr}\n{msg}") + + manifests = json.loads(stdout)["manifests"] + if len(manifests)==0: + terminal.error(f"no meta data is available") + + digest = manifests[0]["digest"] + terminal.info(f"meta data digest: {digest}") + + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"]) + stdout, stderr = proc.communicate() + + if pull_sqfs: + sqfs_path = image_path + "/store.squashfs" + if os.path.exists(sqfs_path): + os.remove(sqfs_path) + time.sleep(0.05) + + # step 2: download the image itself + # run the oras command in a separate process so that this process can + # draw a progress bar. + proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"]) + + total_mb = target.size/(1024*1024) + while proc.poll() is None: + time.sleep(0.25) + if os.path.exists(sqfs_path) and terminal.is_tty(): + current_size = os.path.getsize(sqfs_path) + current_mb = current_size / (1024*1024) + p = current_mb/total_mb + msg = f"{int(current_mb)}/{int(total_mb)} MB" + progress.progress_bar(p, width=50, msg=msg) + stdout, stderr = proc.communicate() + if proc.returncode == 0: + # draw a final complete progress bar + progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB") + terminal.stdout("") + terminal.info(f"oras command successful: {stdout}") + else: + msg = error_message_from_stderr(stderr) + terminal.error(f"image pull failed: {stderr}\n{msg}") - proc = run_command_internal(["pull", "-o", image_path, f"{source_address}@{digest}"]) - stdout, stderr = proc.communicate() - - # step 2: download the image itself - # run the oras command in a separate process so that this process can - # draw a progress bar. - proc = run_command_internal(["pull", "-o", image_path, f"{source_address}:{tag}"]) - - sqfs_path = image_path + "/store.squashfs" - total_mb = target.size/(1024*1024) - while proc.poll() is None: - time.sleep(1.0) - if os.path.exists(sqfs_path) and terminal.is_tty(): - current_size = os.path.getsize(sqfs_path) - current_mb = current_size / (1024*1024) - p = current_mb/total_mb - msg = f"{int(current_mb)}/{int(total_mb)} MB" - progress.progress_bar(p, width=50, msg=msg) - stdout, stderr = proc.communicate() - if proc.returncode == 0: - # draw a final complete progress bar - progress.progress_bar(1.0, width=50, msg=f"{int(total_mb)}/{int(total_mb)} MB") - terminal.stdout("") - terminal.info(f"oras command successful: {stdout}") - else: - msg = error_message_from_stderr(stderr) - terminal.error(f"image pull failed: {stderr}\n{msg}") except KeyboardInterrupt: proc.terminate() terminal.stdout("") diff --git a/todo-meta.md b/todo-meta.md new file mode 100644 index 0000000..81fc0e7 --- /dev/null +++ b/todo-meta.md @@ -0,0 +1,9 @@ +TODO list for meta data operations + ++ pull meta data alongside squashfs images ++ optionally download meta data ++ image inspect + + add {meta} format option that will print the meta data path ++ uenv image pull: pull missing meta data of already downloaded images +- image ls: annotate meta-data only pulls +- image start / run : handle meta-data only pulls diff --git a/todo.md b/todo.md index 27df4fc..1309d6e 100644 --- a/todo.md +++ b/todo.md @@ -49,12 +49,11 @@ uenv start --view=develop icon setenv("PATH", ...) setenv("LD_LIBRARY_PATH", ...) | + execve(squashfs-mount) + | execve(bash) | - |_fork__ - | - | - bash + bash ``` ## Features diff --git a/uenv-image b/uenv-image index fe165fa..cbfe13b 100755 --- a/uenv-image +++ b/uenv-image @@ -109,7 +109,8 @@ sha256: {{sha256}} system: {{system}} date: {{date}} path: {{path}} -sqfs: {{sqfs}}\ +sqfs: {{sqfs}} +meta: {{meta}}\ """ , help="optional format string") path_parser.add_argument("uenv", type=str) @@ -153,6 +154,8 @@ downloaded uenv with the list command. For more information: find_parser.add_argument("-a", "--uarch", required=False, type=str) find_parser.add_argument("--build", action="store_true", help="Search undeployed builds.", required=False) + find_parser.add_argument("--no-header", action="store_true", + help="Do not print header in output.", required=False) find_parser.add_argument("uenv", nargs="?", default=None, type=str) pull_parser = subparsers.add_parser("pull", @@ -188,6 +191,10 @@ with appropriate JFrog access and with the JFrog token in their oras keychain. pull_parser.add_argument("-a", "--uarch", required=False, type=str) pull_parser.add_argument("--build", action="store_true", required=False, help="enable undeployed builds") + pull_parser.add_argument("--only-meta", action="store_true", required=False, + help="only download meta data, if it is available") + pull_parser.add_argument("--force", action="store_true", required=False, + help="force download if the image has already been downloaded") pull_parser.add_argument("uenv", nargs="?", default=None, type=str) list_parser = subparsers.add_parser("ls", @@ -219,6 +226,8 @@ List uenv that are available. """) list_parser.add_argument("-s", "--system", required=False, type=str) list_parser.add_argument("-a", "--uarch", required=False, type=str) + list_parser.add_argument("--no-header", action="store_true", + help="Do not print header in output.", required=False) list_parser.add_argument("uenv", nargs="?", default=None, type=str) deploy_parser = subparsers.add_parser("deploy", @@ -292,6 +301,8 @@ def get_filter(args): def inspect_string(record: record.Record, image_path, format_string: str) -> str: try: + meta = image_path+"/meta" if os.path.exists(image_path+"/meta") else "none" + sqfs = image_path+"/store.squashfs" if os.path.exists(image_path+"/store.squashfs") else "none" return format_string.format( system = record.system, uarch = record.uarch, @@ -302,7 +313,8 @@ def inspect_string(record: record.Record, image_path, format_string: str) -> str id = record.id, sha256 = record.sha256, path = image_path, - sqfs = image_path + "/store.squashfs", + sqfs = sqfs, + meta = meta, ) except Exception as err: terminal.error(f"unable to format {str(err)}") @@ -317,11 +329,12 @@ def image_size_string(size): return f"{(size/(1024*1024*1024)):<.1f}GB" # pretty print a list of Record -def print_records(recordset): +def print_records(recordset, no_header=False): records = recordset.records if not recordset.is_empty: - terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow")) + if not args.no_header: + terminal.stdout(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'id':16} {'size':<10}", "yellow")) for r in recordset.records: namestr = f"{r.name}/{r.version}" tagstr = f"{r.tag}" @@ -406,7 +419,7 @@ if __name__ == "__main__": terminal.error(f"no uenv matches the spec: {colorize(results.request, 'white')}") if args.command == "find": - print_records(results) + print_records(results, no_header=args.no_header) elif args.command == "pull": if not results.is_unique_sha: @@ -423,23 +436,60 @@ if __name__ == "__main__": t = records.records[0] source_address = jfrog.address(t, 'build' if args.build else 'deploy') - terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB") + terminal.info(f"pulling {t} from {source_address} {t.size/(1024*1024):.0f} MB with only-meta={args.only_meta}") terminal.info(f"repo path: {repo_path}") cache = safe_repo_open(repo_path) image_path = cache.image_path(t) + terminal.info(f"image path: {image_path}") - # if the record isn't already in the filesystem repo download it + # at this point the request is for an sha that is in the remote repository + do_download=False + + meta_path=image_path+"/meta" + sqfs_path=image_path+"/store.squashfs" + meta_exists=os.path.exists(meta_path) + sqfs_exists=os.path.exists(sqfs_path) + + only_meta=meta_exists and not sqfs_exists + + # if there is no entry in the local database do a full clean download if cache.database.get_record(t.sha256).is_empty: - terminal.stdout(f"downloading image {t.sha256} {image_size_string(t.size)}") - # clean up the path if it already exists: sometimes this causes an oras error. - if os.path.exists(image_path): - terminal.info(f"removing existing path {image_path}") - shutil.rmtree(image_path) - oras.pull_uenv(source_address, image_path, t) + terminal.info("===== is_empty") + do_download=True + pull_meta=True + pull_sqfs=not args.only_meta + elif args.force: + terminal.info("===== force") + do_download=True + pull_meta=True + pull_sqfs=not args.only_meta + # a record exists, so check whether any components are missing else: - terminal.stdout(f"image {t.sha256} is already available locally") + terminal.info("===== else") + pull_meta=not meta_exists + pull_sqfs=not sqfs_exists and (not args.only_meta) + do_download=pull_meta or pull_sqfs + + terminal.info(f"pull {t.sha256} exists: meta={meta_exists} sqfs={sqfs_exists}") + terminal.info(f"pull {t.sha256} pulling: meta={pull_meta} sqfs={pull_sqfs}") + if do_download: + terminal.info(f"downloading") + else: + terminal.info(f"nothing to pull: use --force to force the download") + + # determine whether to perform download + # check whether the image is in the database, or when only meta-data has been downloaded + if do_download: + terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} matches remote image {t.sha256}") + if pull_meta: + terminal.stdout(f"{t.id} pulling meta data") + if pull_sqfs: + terminal.stdout(f"{t.id} pulling squashfs") + oras.pull_uenv(source_address, image_path, t, pull_meta, pull_sqfs) + else: + terminal.stdout(f"{t.name}/{t.version}:{t.tag} meta data and image with id {t.id} have already been pulled") # update all the tags associated with the image. terminal.info(f"updating the local repository database") @@ -447,8 +497,6 @@ if __name__ == "__main__": terminal.stdout(f"updating local reference {r.name}/{r.version}:{r.tag}") cache.add_record(r) - terminal.stdout(f"uenv {t.name}/{t.version}:{t.tag} downloaded") - sys.exit(0) elif args.command == "ls": @@ -459,7 +507,7 @@ if __name__ == "__main__": fscache = safe_repo_open(repo_path) records = fscache.database.find_records(**img_filter) - print_records(records) + print_records(records, no_header=args.no_header) sys.exit(0)