deployment works

img

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import argparse
 import copy
 import os
@@ -94,22 +94,23 @@ def relative_path_from_record(record):
 # pretty print a list of Record
 def print_records(records):
     if len(records)>0:
-        print(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'sha256':64} {'size':<10}", "yellow"))
-        for r in records:
-            namestr = f"{r.name}/{r.version}"
-            tagstr = f"{r.tag}"
-            label = namestr + ":" + tagstr
-            datestr = r.date.strftime("%Y-%m-%d")
-            S = r.size
-            if S<1024:
-                size_str = f"{S:<} bytes"
-            elif S<1024*1024:
-                size_str = f"{(S/1024):<.0f} kB"
-            elif S<1024*1024*1024:
-                size_str = f"{(S/(1024*1024)):<.0f} MB"
-            else:
-                size_str = f"{(S/(1024*1024*1024)):<.1f} GB"
-            print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256:64} {size_str:<10}")
+        print(terminal.colorize(f"{'uenv/version:tag':40}{'uarch':6}{'date':10} {'sha256':16} {'size':<10}", "yellow"))
+        for rset in records:
+            for r in rset:
+                namestr = f"{r.name}/{r.version}"
+                tagstr = f"{r.tag}"
+                label = namestr + ":" + tagstr
+                datestr = r.date.strftime("%Y-%m-%d")
+                S = r.size
+                if S<1024:
+                    size_str = f"{S:<} bytes"
+                elif S<1024*1024:
+                    size_str = f"{(S/1024):<.0f} kB"
+                elif S<1024*1024*1024:
+                    size_str = f"{(S/(1024*1024)):<.0f} MB"
+                else:
+                    size_str = f"{(S/(1024*1024*1024)):<.1f} GB"
+                print(f"{label:<40}{r.uarch:6}{datestr:10} {r.sha256[:16]:16} {size_str:<10}")
 
 # return dictionary {"name", "version", "tag"} from a uenv description string
 #       "prgenv_gnu"              -> ("prgenv_gnu", None, None)
@@ -160,6 +161,10 @@ def is_valid_sha256(s: str):
     pattern = re.compile(r'^[a-fA-F-0-9]{64}$')
     return True if pattern.match(s) else False
 
+def is_short_sha256(s: str):
+    pattern = re.compile(r'^[a-fA-F-0-9]{16}$')
+    return True if pattern.match(s) else False
+
 if __name__ == "__main__":
 
     parser = make_argparser()
@@ -278,11 +283,13 @@ if __name__ == "__main__":
         terminal.info(f"downloaded jfrog build meta data: {len(build_database.images)} images")
 
         # after this block, record is the record of the requested source
-        if is_valid_sha256(source):
+        if is_valid_sha256(source) or is_short_sha256(source):
             # lookup using sha256
             source_record = build_database.get_record(source)
-            if record is None:
+            terminal.info(f"image to deploy: {source_record}")
+            if source_record is None:
                 terminal.error(f"no record in the build repository matches the hash {source}")
+            source_record = source_record[0]
         else:
             img_filter = {}
             for key, value in parse_uenv_string(source).items():
@@ -293,10 +300,10 @@ if __name__ == "__main__":
 
             # expect that src has [name, version, tag] keys
             records = build_database.find_records(**img_filter)
-            
+
             if not (len(records)==1):
                 terminal.error(f"source {source} is not an image in the build repository")
-            source_record = records[0]
+            source_record = records[0][0]
 
         target_record = copy.deepcopy(source_record)
         target_record.tag = ','.join(tags)

lib/datastore.py

@@ -1,25 +1,38 @@
+import json
+import re
 import os
 
-import json
 from record import Record
 import terminal
 
 UENV_CLI_API_VERSION=1
 
+def is_sha256(s: str):
+    pattern = re.compile(r'^[a-fA-F-0-9]{64}$')
+    return True if pattern.match(s) else False
+
+def is_short_sha256(s: str):
+    pattern = re.compile(r'^[a-fA-F-0-9]{16}$')
+    return True if pattern.match(s) else False
+
 class DataStore:
     def __init__(self):
         # all images store with (key,value) = (sha256,Record)
         self._images = {}
+        self._short_sha = {}
 
         self._store = {"system": {}, "uarch": {}, "name": {}, "version": {}, "tag": {}}
 
-    def add_record(self, r: Record, overwrite: bool = False):
-        # test for collisions
-        if (not overwrite) and (self._images.get(r.sha256, None) is not None):
-            raise ValueError(f"an image with the hash {r.sha256} already exists")
-
+    def add_record(self, r: Record):
         sha = r.sha256
-        self._images[sha] = r
+        short_sha = r.sha256[:16]
+        self._images.setdefault(sha, []).append(r)
+        # check for (exceedingly unlikely) collision
+        if short_sha in self._short_sha:
+            old_sha = self._short_sha[short_sha]
+            if sha != old_sha:
+                terminal.error('image hash collision:\n  {sha}\n  {old_sha}')
+        self._short_sha[sha[:16]] = sha
         self._store["system"] .setdefault(r.system, []).append(sha)
         self._store["uarch"]  .setdefault(r.uarch, []).append(sha)
         self._store["name"]   .setdefault(r.name, []).append(sha)
@@ -53,8 +66,12 @@ def find_records(self, **constraints):
     def images(self):
         return self._images
 
-    def get_record(self, sha256: str) -> Record:
-        return self._images.get(sha256, None)
+    def get_record(self, sha: str) -> Record:
+        if is_sha256(sha):
+            return self._images.get(sha, None)
+        elif is_short_sha256(sha):
+            return self._images.get(self._short_sha[sha], None)
+        raise ValueError(f"{sha} is not a valid sha256 or short (16 character) sha")
 
     # Convert to a dictionary that can be written to file as JSON
     # The serialisation and deserialisation are central: able to represent

lib/jfrog.py

@@ -70,7 +70,7 @@ def query() -> tuple:
         return (deploy_database, build_database)
 
     except Exception as error:
-        raise RuntimeError("unable to access the JFrog uenv API.")
+        raise RuntimeError(f"downloading image data from jfrog.svs.cscs.ch ({str(error)})")
 
 def relative_from_record(record):
     return f"{record.system}/{record.uarch}/{record.name}/{record.version}:{record.tag}"