Resolutions for #13 and #21

- Resolve #21 with a new driver program `aipsip` that generates both the AIP and uses it to make the SIP as well, leaving all in the current working directory (along with two—count 'em, *two*—PDS labels for the price of one!).
    - Updates the Python `setuptools` metadata to generate the new `aipsip` (helps with #21).
    - Refactors logging and command-line argument setup (also for #21).
- Unifies logging between `aipgen` and `sipgen` with the new `aipsip` so that there are `--debug` and `--quiet` options; without either you get a nominal amount of "hand-holding" of output.
- Resolve #13 so that instead of billions of redundant XML parsing and XPath lookups we use a local `sqlite3` database and LRU caching.
    - Factor out XML parsing from `aipgen` and `sipgen` so we can apply caching.
    - Clear up logging messages so we can know what's calling what.
    - Create a temp DB in `sipgen` and populate it with mappings from lidvids to XML files for rapid lookups
        - But see also #25 for other uses of that DB.
- Add standardized `--version` arguments for all three programs.

With these changes, running `sipgen` on my Mac¹ can process a 272GiB `insight_cameras` export in 1:03. On `pdsimg-int1`, it handles the 1.5TiB`insight_cameras` dataset in under 4 hours.

Footnotes:

- ¹2.4 GHz 8-core Intel Core i9, SSD
- ²2.3 GHz 8-core Intel Xeon Gold 6140, unknown drive

Author: nutjob4life

setup.py

@@ -64,7 +64,8 @@
     entry_points={
         'console_scripts': [
             'sipgen=pds.aipgen.sip:main',
-            'aipgen=pds.aipgen.aip:main'
+            'aipgen=pds.aipgen.aip:main',
+            'aipsip=pds.aipgen.main:main'
         ]
     },
     namespace_packages=['pds'],

src/pds/aipgen/aip.py

@@ -31,7 +31,7 @@
 
 
 from .constants import PDS_NS_URI, XML_SCHEMA_INSTANCE_NS_URI, PDS_SCHEMA_URL, XML_MODEL_PI, INFORMATION_MODEL_VERSION
-from .utils import getPrimariesAndOtherInfo, getMD5
+from .utils import getPrimariesAndOtherInfo, getMD5, parseXML, addLoggingArguments
 from datetime import datetime
 from lxml import etree
 import argparse, logging, sys, os, os.path, hashlib
@@ -41,6 +41,7 @@
 # ---------
 
 # For ``--help``:
+_version = '0.0.0'
 _description = '''Generate an Archive Information Package or AIP. An AIP consists of three files:
  ➀ a "checksum manifest" which contains MD5 hashes of *all* files in a product;
  ➁ a "transfer manifest" which lists the "lidvids" for files within each XML label mentioned in a product; and
@@ -53,12 +54,8 @@
 # Comment to insert near the top of an AIP XML label
 _iaComment = 'Parse name from bundle logical_identifier, e.g. urn:nasa:pds:ladee_mission_bundle would be ladee_mission_bundle'
 
-
-# Logging
-# -------
-
+# Logging:
 _logger = logging.getLogger(__name__)
-logging.basicConfig(format='%(levelname)s %(message)s', level=logging.INFO)
 
 
 # Functions
@@ -94,7 +91,7 @@ def _getLIDVIDandFileInventory(xmlFile):
     identifier, return None and None.
     '''
     _logger.debug('📜 Analyzing XML in %s', xmlFile)
-    tree = etree.parse(xmlFile)
+    tree = parseXML(xmlFile)
     root = tree.getroot()
     matches = root.findall(f'./{{{PDS_NS_URI}}}Identification_Area/{{{PDS_NS_URI}}}logical_identifier')
     if not matches:
@@ -291,10 +288,10 @@ def _writeLabel(
     tree.write(labelOutputFile, encoding='utf-8', xml_declaration=True, pretty_print=True)
 
 
-def _process(bundle):
+def process(bundle):
     '''Generate a "checksum manifest", a "transfer manifest", and a PDS label from the given
     ``bundle``, which is an open file stream (with a ``name`` atribute) on the local
-    filesystem.
+    filesystem. Return the name of the generated checksum manifest file.
     '''
     _logger.info('🏃‍♀️ Starting AIP generation for %s', bundle.name)
     d = os.path.dirname(os.path.abspath(bundle.name))
@@ -304,7 +301,7 @@ def _process(bundle):
     strippedLogicalID = bundleLID.split(':')[-1]
 
     # Easy one: the checksum† manifest
-    # †It's actually an MD5 hash, not a checksum 😅
+    # †It's actually an MD5 *hash*, not a checksum 😅
     chksumFN = strippedLogicalID + '_checksum_manifest_v' + bundleVID + '.tab'
     chksumMD5, chksumSize, chksumNum = _writeChecksumManifest(chksumFN, d)
 
@@ -328,27 +325,25 @@ def _process(bundle):
         xferSize,
         xferNum
     )
-    _logger.info('🎉  Success! All done, files generated:')
+    _logger.info('🎉  Success! AIP done, files generated:')
     _logger.info('• Checksum manifest: %s', chksumFN)
     _logger.info('• Transfer manifest: %s', xferFN)
-    _logger.info('• XML label: %s', labelFN)
+    _logger.info('• XML label for them both: %s', labelFN)
+    return chksumFN
 
 
 def main():
-    '''Check the command-line for options and create a SIP from the given bundle XML'''
+    '''Check the command-line for options and create an AIP from the given bundle XML'''
     parser = argparse.ArgumentParser(description=_description)
+    parser.add_argument('--version', action='version', version=f'%(prog)s {_version}')
+    addLoggingArguments(parser)
     parser.add_argument(
         'bundle', type=argparse.FileType('rb'), metavar='IN-BUNDLE.XML', help='Root bundle XML file to read'
     )
-    parser.add_argument(
-        '-v', '--verbose', default=False, action='store_true',
-        help='Verbose logging; defaults %(default)s'
-    )
     args = parser.parse_args()
-    if args.verbose:
-        _logger.setLevel(logging.DEBUG)
+    logging.basicConfig(level=args.loglevel, format='%(levelname)s %(message)s')
     _logger.debug('⚙️ command line args = %r', args)
-    _process(args.bundle)
+    process(args.bundle)
     _logger.info('👋 Thanks for using this program! Bye!')
     sys.exit(0)
 

src/pds/aipgen/constants.py

@@ -52,3 +52,13 @@
 
 # Filename extension to use with PDS labels
 PDS_LABEL_FILENAME_EXTENSION = '.xml'
+
+# Command-line names for hash algorithms mapped to Python *implementation*
+# name which are standardized (as lower case, no less) in the ``hashlib``.
+# There are a lot more possible message digest algorithms, but we choose
+# to support just three.
+HASH_ALGORITHMS = {
+    'MD5':     'md5',
+    'SHA-1':   'sha1',
+    'SHA-256': 'sha256',
+}

src/pds/aipgen/main.py

@@ -0,0 +1,97 @@
+# encoding: utf-8
+#
+# Copyright © 2020 California Institute of Technology ("Caltech").
+# ALL RIGHTS RESERVED. U.S. Government sponsorship acknowledged.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# • Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# • Redistributions must reproduce the above copyright notice, this list of
+#   conditions and the following disclaimer in the documentation and/or other
+#   materials provided with the distribution.
+# • Neither the name of Caltech nor its operating division, the Jet Propulsion
+#   Laboratory, nor the names of its contributors may be used to endorse or
+#   promote products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+
+u'''AIP and SIP generation'''
+
+from .aip import process as aipProcess
+from .constants import HASH_ALGORITHMS
+from .sip import addSIParguments
+from .sip import produce as sipProcess
+from .utils import addLoggingArguments
+import argparse, sys, logging
+
+
+# Constants
+# ---------
+
+# For ``--help``:
+_version = '0.0.0'
+_description = '''
+Generate an Archive Information Package (AIP) and a Submission Information
+Package (SIP). This creates three files for the AIP in the current directory
+(overwriting them if they already exist): ➀ a "checksum manifest" which
+contains MD5 hashes of *all* files in a product; ➁ a "transfer manifest" which
+lists the "lidvids" for files within each XML label mentioned in a product;
+and ➂ an XML label for these two files. It also creates two files for the SIP
+(also overwriting them if they exist): ➀ A "SIP manifest" file; and an XML
+label of that file too. The names of the generated files are based on the
+logical identifier found in the bundle file, and any existing files are
+overwritten. The names of the generated files are printed upon successful
+completion.
+'''
+
+# Logging:
+_logger = logging.getLogger(__name__)
+
+
+# Functions
+# ---------
+
+def main():
+    '''Make an AIP and a SIP'''
+    parser = argparse.ArgumentParser(description=_description)
+    parser.add_argument('--version', action='version', version=f'%(prog)s {_version}')
+    addSIParguments(parser)
+    addLoggingArguments(parser)
+    parser.add_argument(
+        'bundle', type=argparse.FileType('rb'), metavar='IN-BUNDLE.XML', help='Bundle XML file to read'
+    )
+    args = parser.parse_args()
+    logging.basicConfig(level=args.loglevel, format='%(levelname)s %(message)s')
+    _logger.debug('⚙️ command line args = %r', args)
+    chksumFN = aipProcess(args.bundle)
+    with open(chksumFN, 'rb') as chksumStream:
+        sipProcess(
+            args.bundle,
+            HASH_ALGORITHMS[args.algorithm],
+            args.url,
+            args.insecure,
+            args.site,
+            args.offline,
+            args.bundle_base_url,
+            chksumStream
+        )
+    _logger.info("👋 That's it! Thanks for making an AIP and SIP with us today. Bye!")
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()