diff --git a/img b/img index c811040..3a2c2e4 100755 --- a/img +++ b/img @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse import copy import os @@ -94,22 +94,23 @@ def relative_path_from_record(record): # pretty print a list of Record def print_records(records): if len(records)>0: - print(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'sha256':64} {'size':<10}", "yellow")) - for r in records: - namestr = f"{r.name}/{r.version}" - tagstr = f"{r.tag}" - label = namestr + ":" + tagstr - datestr = r.date.strftime("%Y-%m-%d") - S = r.size - if S<1024: - size_str = f"{S:<} bytes" - elif S<1024*1024: - size_str = f"{(S/1024):<.0f} kB" - elif S<1024*1024*1024: - size_str = f"{(S/(1024*1024)):<.0f} MB" - else: - size_str = f"{(S/(1024*1024*1024)):<.1f} GB" - print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256:64} {size_str:<10}") + print(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'sha256':16} {'size':<10}", "yellow")) + for rset in records: + for r in rset: + namestr = f"{r.name}/{r.version}" + tagstr = f"{r.tag}" + label = namestr + ":" + tagstr + datestr = r.date.strftime("%Y-%m-%d") + S = r.size + if S<1024: + size_str = f"{S:<} bytes" + elif S<1024*1024: + size_str = f"{(S/1024):<.0f} kB" + elif S<1024*1024*1024: + size_str = f"{(S/(1024*1024)):<.0f} MB" + else: + size_str = f"{(S/(1024*1024*1024)):<.1f} GB" + print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256[:16]:16} {size_str:<10}") # return dictionary {"name", "version", "tag"} from a uenv description string # "prgenv_gnu" -> ("prgenv_gnu", None, None) @@ -160,6 +161,10 @@ def is_valid_sha256(s: str): pattern = re.compile(r'^[a-fA-F-0-9]{64}$') return True if pattern.match(s) else False +def is_short_sha256(s: str): + pattern = re.compile(r'^[a-fA-F-0-9]{16}$') + return True if pattern.match(s) else False + if __name__ == "__main__": parser = make_argparser() @@ -278,11 +283,13 @@ if __name__ == "__main__": terminal.info(f"downloaded jfrog build meta data: {len(build_database.images)} images") # after this block, record is the record of the requested source - if is_valid_sha256(source): + if is_valid_sha256(source) or is_short_sha256(source): # lookup using sha256 source_record = build_database.get_record(source) - if record is None: + terminal.info(f"image to deploy: {source_record}") + if source_record is None: terminal.error(f"no record in the build repository matches the hash {source}") + source_record = source_record[0] else: img_filter = {} for key, value in parse_uenv_string(source).items(): @@ -293,10 +300,10 @@ if __name__ == "__main__": # expect that src has [name, version, tag] keys records = build_database.find_records(**img_filter) - + if not (len(records)==1): terminal.error(f"source {source} is not an image in the build repository") - source_record = records[0] + source_record = records[0][0] target_record = copy.deepcopy(source_record) target_record.tag = ','.join(tags) diff --git a/lib/datastore.py b/lib/datastore.py index 9b04187..8b4e083 100644 --- a/lib/datastore.py +++ b/lib/datastore.py @@ -1,25 +1,38 @@ +import json +import re import os -import json from record import Record import terminal UENV_CLI_API_VERSION=1 +def is_sha256(s: str): + pattern = re.compile(r'^[a-fA-F-0-9]{64}$') + return True if pattern.match(s) else False + +def is_short_sha256(s: str): + pattern = re.compile(r'^[a-fA-F-0-9]{16}$') + return True if pattern.match(s) else False + class DataStore: def __init__(self): # all images store with (key,value) = (sha256,Record) self._images = {} + self._short_sha = {} self._store = {"system": {}, "uarch": {}, "name": {}, "version": {}, "tag": {}} - def add_record(self, r: Record, overwrite: bool = False): - # test for collisions - if (not overwrite) and (self._images.get(r.sha256, None) is not None): - raise ValueError(f"an image with the hash {r.sha256} already exists") - + def add_record(self, r: Record): sha = r.sha256 - self._images[sha] = r + short_sha = r.sha256[:16] + self._images.setdefault(sha, []).append(r) + # check for (exceedingly unlikely) collision + if short_sha in self._short_sha: + old_sha = self._short_sha[short_sha] + if sha != old_sha: + terminal.error('image hash collision:\n {sha}\n {old_sha}') + self._short_sha[sha[:16]] = sha self._store["system"] .setdefault(r.system, []).append(sha) self._store["uarch"] .setdefault(r.uarch, []).append(sha) self._store["name"] .setdefault(r.name, []).append(sha) @@ -53,8 +66,12 @@ def find_records(self, **constraints): def images(self): return self._images - def get_record(self, sha256: str) -> Record: - return self._images.get(sha256, None) + def get_record(self, sha: str) -> Record: + if is_sha256(sha): + return self._images.get(sha, None) + elif is_short_sha256(sha): + return self._images.get(self._short_sha[sha], None) + raise ValueError(f"{sha} is not a valid sha256 or short (16 character) sha") # Convert to a dictionary that can be written to file as JSON # The serialisation and deserialisation are central: able to represent diff --git a/lib/jfrog.py b/lib/jfrog.py index c14a696..3167da6 100644 --- a/lib/jfrog.py +++ b/lib/jfrog.py @@ -70,7 +70,7 @@ def query() -> tuple: return (deploy_database, build_database) except Exception as error: - raise RuntimeError("unable to access the JFrog uenv API.") + raise RuntimeError(f"downloading image data from jfrog.svs.cscs.ch ({str(error)})") def relative_from_record(record): return f"{record.system}/{record.uarch}/{record.name}/{record.version}:{record.tag}"