Merge pull request #553 from DUNE-DAQ/plasorak/jsonifiy

Add a `jsonify-xml` utility

python/daqconf/consolidate.py

@@ -2,54 +2,55 @@
 import conffwk
 import sys
 import os
+from logging import getLogger
+log = getLogger('daqconf.consolidate')
 
 
 def get_all_includes(db, file):
     includes = db.get_includes(file)
     for include in includes:
-        if "data.xml" in include:
-            includes += get_all_includes(db, include)
+        if "data.xml" not in include:
+            continue
+        includes += get_all_includes(db, include)
 
     return list(set(includes))
 
-
 def consolidate_db(oksfile, output_file):
+    log.info(f"Consolidating database into output database \'{output_file}\'. Input database: \'{oksfile}\'.")
+
     sys.setrecursionlimit(10000)  # for example
-    print("Reading database")
+    log.debug("Reading database")
     db = conffwk.Configuration("oksconflibs:" + oksfile)
 
     schemafiles = []
     includes = get_all_includes(db, None)
     schemafiles += [i for i in includes if "schema.xml" in i]
-    print(f"Included schemas: {schemafiles}")
+    log.debug(f"Included schemas: {schemafiles}")
 
-    print("Creating new database")
+    log.debug("Creating new database")
     new_db = conffwk.Configuration("oksconflibs")
     new_db.create_db(output_file, schemafiles)
 
     new_db.commit()
 
-    print("Reading dal objects from old db")
+    log.debug("Reading dal objects from old db")
     dals = db.get_all_dals()
 
-    print(f"Copying objects to new db")
-    for dal in dals:
+    log.debug(f"Copying objects to new db")
 
-        # print(f"Loading object {dal} into cache")
-        db.get_dal(dals[dal].className(), dals[dal].id)
-
-        # print(f"Copying object: {dal}")
-        new_db.add_dal(dals[dal])
+    for dal in dals:
+        this_dal = db.get_dal(dals[dal].className(), dals[dal].id)
+        new_db.add_dal(this_dal)
 
-    print("Saving database")
+    log.debug("Saving database")
     new_db.commit()
-    print("DONE")
+
 
 def copy_configuration(dest_dir : Path, input_files: list):
     if len(input_files) == 0:
         return []
 
-    print(f"Copying configuration represented by databases: {input_files} to {dest_dir}")
+    log.info(f"Copying configuration represented by databases: \'{input_files}\' to \'{dest_dir}\'")
     dest_dir = dest_dir.resolve() # Always include by absolute path when copying
     sys.setrecursionlimit(10000)  # for example
 
@@ -62,67 +63,59 @@ def copy_configuration(dest_dir : Path, input_files: list):
         dbs = [i for i in includes if "data.xml" in i]
         newdbs = copy_configuration(dest_dir, dbs)
 
-        #print("Creating new database")
         output_file = dest_dir / os.path.basename(input_file)
 
         new_db = conffwk.Configuration("oksconflibs")
         new_db.create_db(str(output_file), schemas + newdbs)
         new_db.commit()
 
-        #print("Reading dal objects from old db")
         dals = db.get_all_dals()
 
-        #print(f"Copying objects to new db")
         for dal in dals:
 
-            # print(f"Loading object {dal} into cache")
             db.get_dal(dals[dal].className(), dals[dal].id)
 
-            # print(f"Copying object: {dal}")
             new_db.add_dal(dals[dal])
 
-        #print("Saving database")
         new_db.commit()
         output_dbs.append(str(output_file))
-    print("DONE")
-        
+    log.debug("DONE")
+
     return output_dbs
 
 
 def consolidate_files(oksfile, *input_files):
     includes = []
     dbs = []
-
-    print(f"Consolidating {len(input_files)} databases into output database {oksfile}. Input databases: {input_files}")
+    str_in_files = '\n'.join(input_files)
+    log.info(f"Consolidating {len(input_files)} databases into output database \'{oksfile}\'. Input databases: {str_in_files}")
     sys.setrecursionlimit(10000)  # for example
 
     for input_file in input_files:
         dbs.append(conffwk.Configuration("oksconflibs:" + input_file))
         includes += get_all_includes(dbs[len(dbs) - 1], None)
-        
+
     includes = list(set(includes))
-    includes = [i for i in includes if i not in input_files]    
-    print(f"Included files: {includes}")
+    includes = [i for i in includes if i not in input_files]
+    log.debug(f"Included files: {includes}")
 
     new_db = conffwk.Configuration("oksconflibs")
     new_db.create_db(oksfile, includes)
 
     new_db.commit()
 
     for db in dbs:
-        print(f"Reading dal objects from old db {db}")
+        log.debug(f"Reading dal objects from old db {db}")
         dals = db.get_all_dals()
 
-        print(f"Copying objects to new db {new_db}")
+        log.debug(f"Copying objects to new db {new_db}")
         for dal in dals:
 
-            try: 
+            try:
                 new_db.get_dal(dals[dal].className(), dals[dal].id)
-                #print(f"ERROR: Database already contains object {dal}")  
-            except:            
-                # print(f"Copying object: {dal}")
+            except:
                 new_db.add_dal(dals[dal])
             new_db.commit()
 
-    print(f"Saving database {new_db}")
+    log.debug(f"Saving database {new_db}")
     new_db.commit()

python/daqconf/jsonify.py

@@ -0,0 +1,74 @@
+import conffwk
+import json
+from logging import getLogger
+import sys
+from rich import print
+log = getLogger('daqconf.jsonify')
+
+
+def hash_function(obj):
+    # I guess we could get ObjectId from MongoDB
+    return hash(f'{obj.id}@{obj.className()}')
+
+
+def convert_to_dict(db, obj):
+    dal_dict = {
+        "__type": obj.className(),
+        "_id": {
+            "$oid": hash_function(obj), # Borrowing from MongoDB
+        }
+    }
+
+    for attribute_name, attribute_value in db.attributes(obj.className(), all=True).items():
+        dal_dict[attribute_name] = getattr(obj, attribute_name)
+
+    for relation_name, relation_value in db.relations(obj.className(), all=True).items():
+        relation_object = getattr(obj, relation_name, None)
+
+        if relation_object is None:
+            dal_dict[relation_name] = None
+
+        elif not relation_value.get('multivalue', False):
+            dal_dict[relation_name] = {
+                # "$ref": "run-registry"
+                '$id': hash_function(relation_object),
+            }
+
+        else:
+            dal_dict[relation_name] = [
+                {
+                    "$id": hash_function(one_relation_object)
+                    # "$ref": "run-registry"
+                }
+                for one_relation_object in relation_object
+            ]
+
+    return dict(sorted(dal_dict.items()))
+
+
+def jsonify_xml_data(oksfile, output):
+
+    sys.setrecursionlimit(10000)
+
+    log.info(f"JSonifying database \'{oksfile}\' to \'{output}\'.")
+
+    log.debug("Reading database")
+    db = conffwk.Configuration("oksconflibs:" + oksfile)
+
+    dals = db.get_all_dals()
+    the_big_dict = {}
+
+    for dal_str in dals:
+        dal = dals[dal_str]
+        key_name = f"{dal.id}@{dal.className()}"
+        log.debug(f"Processing DAL {key_name}")
+        if key_name in the_big_dict:
+            log.error(f"Duplicate DAL id {key_name}")
+            continue
+
+        dal_dict = convert_to_dict(db, dal)
+        the_big_dict[key_name] = dal_dict
+
+    with open(output, 'w') as f:
+        json.dump(dict(sorted(the_big_dict.items())), f, indent=4)
+

python/daqconf/utils.py

@@ -1,5 +1,37 @@
-import os
 import glob
+import logging
+import os
+from rich.logging import RichHandler
+
+
+log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+
+
+def setup_logging(level:str="INFO"):
+    level = level.upper()
+
+    loglevel = logging.INFO
+
+    match level:
+        case "DEBUG":
+            loglevel = logging.DEBUG
+        case "INFO":
+            loglevel = logging.INFO
+        case "WARNING":
+            loglevel = logging.WARNING
+        case "ERROR":
+            loglevel = logging.ERROR
+        case "CRITICAL":
+            loglevel = logging.CRITICAL
+        case _:
+            loglevel = logging.INFO
+
+    FORMAT = "%(message)s"
+    logging.basicConfig(
+        level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]
+    )
+    logging.getLogger().setLevel(loglevel)
+
 
 def find_oksincludes(includes:list[str], extra_dirs:list[str] = []):
     includefiles = []

scripts/consolidate

@@ -1,12 +1,15 @@
 #!/bin/env python3
 import click
 from daqconf.consolidate import consolidate_db
+from daqconf.utils import log_levels, setup_logging
 
 @click.command()
 @click.option('--oksfile', '-i', help='Input database to read')
+@click.option('--log-level', '-l', help='Log level', default='INFO', type=click.Choice(log_levels, case_sensitive=False))
 @click.argument('output_file')
-def consolidate(oksfile, output_file):
-    consolidate_db(oksfile, output_file)  
+def consolidate(oksfile, output_file, log_level):
+    setup_logging(log_level)
+    consolidate_db(oksfile, output_file)
 
 if __name__ == '__main__':
     consolidate()

scripts/consolidate_files

@@ -1,12 +1,15 @@
 #!/bin/env python3
 import click
 from daqconf.consolidate import consolidate_files
+from daqconf.utils import log_levels, setup_logging
 
 @click.command()
 @click.option('--oksfile', '-i', help='Input database(s) to read', multiple=True)
+@click.option('--log-level', '-l', help='Log level', default='INFO', type=click.Choice(log_levels, case_sensitive=False))
 @click.argument('output_file')
-def consolidate(oksfile, output_file):
-    consolidate_files(output_file, *oksfile)  
+def consolidate(oksfile, output_file, log_level):
+    setup_logging(log_level)
+    consolidate_files(output_file, *oksfile)
 
 if __name__ == '__main__':
     consolidate()

scripts/copy_configuration

@@ -2,13 +2,18 @@
 import click
 import pathlib
 from daqconf.consolidate import copy_configuration
+from daqconf.utils import log_levels, setup_logging
 
 @click.command()
 @click.argument('output_directory', type=click.Path(exists=True), nargs=1)
 @click.argument('databases', nargs=-1)
-def copy_config(output_directory, databases):
-    """Copy to OUTPUT_DIRECTORY configuration represented by DATABASES"""
-    copy_configuration(pathlib.Path(output_directory), databases)  
+@click.option('--log-level', '-l', help='Log level', default='INFO', type=click.Choice(log_levels, case_sensitive=False))
+def copy_config(output_directory, databases, log_level):
+    """
+    Copy to OUTPUT_DIRECTORY configuration represented by DATABASES
+    """
+    setup_logging(log_level)
+    copy_configuration(pathlib.Path(output_directory), databases)
 
 if __name__ == '__main__':
     copy_config()

scripts/jsonify-xml

@@ -0,0 +1,15 @@
+#!/bin/env python3
+import click
+from daqconf.jsonify import jsonify_xml_data
+from daqconf.utils import log_levels, setup_logging
+
+@click.command()
+@click.option('--oksfile', '-i', help='Input database to read')
+@click.option('--log-level', '-l', help='Log level', default='INFO', type=click.Choice(log_levels, case_sensitive=False))
+@click.argument('output_file')
+def jsonify_xml(oksfile, output_file, log_level):
+    setup_logging(log_level)
+    jsonify_xml_data(oksfile, output_file)
+
+if __name__ == '__main__':
+    jsonify_xml()