uenv repo feature, and database repair/upgrade support (#39)

activate

@@ -13,15 +13,16 @@ function uenv {
     function uenv_usage {
         echo "uenv - for using user environments [version @@version@@]"
         echo ""
-        echo "Usage:   uenv [--version] [--help] <command> [<args>]"
+        echo "Usage:   uenv [--version] [--help] [--no-color] [--verbose] <command> [<args>]"
         echo ""
         echo "The following commands are available:"
+        echo "  image           query and pull uenv images"
+        echo "  repo            query and interact with repositories"
         echo "  run             run a command in an environment"
         echo "  start           start a new shell with an environment loaded"
-        echo "  stop            stop a shell with an environment loaded"
         echo "  status          print information about each running environment"
+        echo "  stop            stop a shell with an environment loaded"
         echo "  view            activate a view"
-        echo "  image           query and pull uenv images"
         echo ""
         echo "Type 'uenv command --help' for more information and examples for a specific command, e.g."
         echo "  uenv start --help"

lib/datastore.py

@@ -1,5 +1,6 @@
 import json
 import os
+import shutil
 import sqlite3
 
 from record import Record
@@ -29,7 +30,7 @@ def __str__(self):
         return self.message
 
 create_db_commands = {
-        "v1": """
+        1: """
 BEGIN;
 
 PRAGMA foreign_keys=on;
@@ -83,7 +84,7 @@ def __str__(self):
 
 COMMIT;
 """,
-        "v2": """
+        2: """
 BEGIN;
 
 PRAGMA foreign_keys=on;
@@ -138,9 +139,89 @@ def __str__(self):
 COMMIT;
 """}
 
-db_version = "v2"
+# the version schema is an integer, bumped every time it is modified
+db_version = 2
 create_db_command = create_db_commands[db_version]
 
+# returns the version of the database in repo_path
+#  returns -1 if there was an error or unknown database format
+def repo_version(repo_path: str):
+    try:
+        # open the suspect database read only
+        db_path = f"{repo_path}/index.db"
+        db = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+        result = db.execute("SELECT * FROM images")
+        columns = [description[0] for description in result.description]
+        columns.sort()
+        db.close()
+
+        # the image tables in v1 and v2 have the respective columns:
+        # v1: images (date, id, sha256, size, system, uarch);
+        # v2: images (date, id, sha256, size);
+
+        version = -1
+        if columns == ["date", "id", "sha256", "size"]:
+            version = 2
+        elif columns == ["date", "id", "sha256", "size", "system", "uarch"]:
+            version = 1
+        terminal.info(f"database {db_path} matches schema version {version}")
+
+        return version
+    except Exception as err:
+        terminal.info(f"exception opening {db_path}: {str(err)}")
+        return -1
+
+# return 2: repo does not exist
+# return 1: repo is fully up to date
+# return 0: repo needs upgrading
+# return -1: unrecoverable error
+def repo_status(repo_path: str):
+    index_path = repo_path + "/index.db"
+    if not os.path.exists(index_path):
+        return 2
+
+    version = repo_version(repo_path)
+
+    if version==db_version:
+        return 1
+    elif version==1:
+        return 0
+
+    return -1
+
+def repo_upgrade(repo_path: str):
+    version = repo_version(repo_path)
+    if version==2:
+        return
+    elif version==-1:
+        raise RepoDBError("unable to upgrade database due to fatal error.")
+
+    if version==1:
+        # load the existing database using the v1 schema
+        db1_path = repo_path + "/index.db"
+        db1 = DataStore(db1_path)
+
+        # create a new database with the v2 schema
+        db2_path = repo_path + "/index-v1.db"
+        if os.path.exists(db2_path):
+            os.remove(db2_path)
+        FileSystemRepo.create(repo_path, exists_ok=False, db_name="index-v1.db")
+
+        # apply the records from the old database to the new db
+        db2 = DataStore(db2_path, create_command=create_db_commands[2])
+        for r in db1.images.records:
+            db2.add_record(r)
+
+        # close the databases
+        db1.close()
+        db2.close()
+
+        dbswp_path = repo_path + "/index-back.db"
+        shutil.move(db1_path, dbswp_path)
+        shutil.copy(db2_path, db1_path)
+        shutil.move(dbswp_path, db2_path)
+
+
 class RecordSet():
     def __init__(self, records, request):
         self._shas = set([r.sha256 for r in records])
@@ -192,8 +273,7 @@ def ambiguous_request_message(self):
         return lines
 
 class DataStore:
-
-    def __init__(self, path=None):
+    def __init__(self, path=None, create_command=create_db_command):
         """
         If path is a string, attempt to open the database at that location,
 
@@ -324,6 +404,9 @@ def find_records(self, **constraints):
         results.sort(reverse=True)
         return RecordSet(results, request)
 
+    def close(self):
+        self._store.close()
+
     @property
     def images(self):
         items = self._store.execute(f"SELECT * FROM records")
@@ -361,13 +444,13 @@ def __init__(self, path: str):
         self._database = DataStore(self._index)
 
     @staticmethod
-    def create(path: str, exists_ok: bool=False):
+    def create(path: str, exists_ok: bool=False, db_name: str="index.db"):
         terminal.info(f"FileSystemRepo: create new repo in {path}")
         if not os.path.exists(path):
             terminal.info(f"FileSystemRepo: creating path {path}")
             os.makedirs(path)
 
-        index_file = f"{path}/index.db"
+        index_file = f"{path}/{db_name}"
         if not os.path.exists(index_file):
             terminal.info(f"FileSystemRepo: creating new index {index_file}")
             store = sqlite3.connect(index_file)

test/create_v1_db.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python3
+
+"""
+Script for creating a v1 index.db used to test the `uenv image upgrade` functionality.
+Run with uenv v4 or v5 - not guaranteed to continue working indefinately.
+We keep a copy of a v1 index.db in the repo for running the tests, so hopefully
+you never have to read or run this.
+"""
+
+import pathlib
+import sys
+import sqlite3
+
+prefix = pathlib.Path(__file__).parent.resolve()
+lib =  prefix.parent / "lib"
+sys.path = [lib.as_posix()] + sys.path
+print(sys.path)
+
+import record
+
+prgenvgnu_records = [
+    record.Record("santis", "gh200", "prgenv-gnu", "23.11", "default", "monday", 1024, "a"*64),
+    record.Record("santis", "gh200", "prgenv-gnu", "23.11", "v2",     "monday", 1024, "a"*64),
+    record.Record("santis", "gh200", "prgenv-gnu", "23.11", "v1",     "monday", 1024, "c"*64),
+    record.Record("santis", "gh200", "prgenv-gnu", "24.2",  "default", "monday", 1024, "b"*64),
+    record.Record("santis", "gh200", "prgenv-gnu", "24.2",  "v1",     "monday", 1024, "b"*64),
+]
+
+
+create_db_command = """
+BEGIN;
+
+PRAGMA foreign_keys=on;
+
+CREATE TABLE images (
+    sha256 TEXT PRIMARY KEY CHECK(length(sha256)==64),
+    id TEXT UNIQUE CHECK(length(id)==16),
+    date TEXT NOT NULL,
+    size INTEGER NOT NULL,
+    uarch TEXT NOT NULL,
+    system TEXT NOT NULL
+);
+
+CREATE TABLE uenv (
+    version_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    version TEXT NOT NULL,
+    UNIQUE (name, version)
+);
+
+CREATE TABLE tags (
+    version_id INTEGER,
+    tag TEXT NOT NULL,
+    sha256 TEXT NOT NULL,
+    PRIMARY KEY (version_id, tag),
+    FOREIGN KEY (version_id)
+        REFERENCES uenv (version_id)
+            ON DELETE CASCADE
+            ON UPDATE CASCADE,
+    FOREIGN KEY (sha256)
+        REFERENCES images (sha256)
+            ON DELETE CASCADE
+            ON UPDATE CASCADE
+);
+
+-- for convenient generation of the Record type used internally by uenv-image
+CREATE VIEW records AS
+SELECT
+    images.system  AS system,
+    images.uarch   AS uarch,
+    uenv.name      AS name,
+    uenv.version   AS version,
+    tags.tag       AS tag,
+    images.date    AS date,
+    images.size    AS size,
+    tags.sha256    AS sha256,
+    images.id      AS id
+FROM tags
+    INNER JOIN uenv   ON uenv.version_id = tags.version_id
+    INNER JOIN images ON images.sha256   = tags.sha256;
+
+COMMIT;
+"""
+
+def create_db(path):
+    store = sqlite3.connect(path)
+    store.executescript(create_db_command)
+
+    return store
+
+def add_record(store, r):
+    cursor = store.cursor()
+
+    cursor.execute("BEGIN;")
+    cursor.execute("PRAGMA foreign_keys=on;")
+    cursor.execute("INSERT OR IGNORE INTO images (sha256, id, date, size, uarch, system) VALUES (?, ?, ?, ?, ?, ?)",
+                   (r.sha256, r.id, r.date, r.size, r.uarch, r.system))
+    # Insert a new name/version to the uenv table if no existing images with that pair exist
+    cursor.execute("INSERT OR IGNORE INTO uenv (name, version) VALUES (?, ?)",
+                   (r.name, r.version))
+    # Retrieve the version_id of the name/version pair
+    # This requires a SELECT query to get the correct version_id whether or not
+    # a new row was added in the last INSERT
+    cursor.execute("SELECT version_id FROM uenv WHERE name = ? AND version = ?",
+                   (r.name, r.version))
+    version_id = cursor.fetchone()[0]
+    # Check whether an image with the same tag already exists in the repos
+    cursor.execute("SELECT version_id, tag, sha256 FROM tags WHERE version_id = ? AND tag = ?",
+                   (version_id, r.tag))
+    existing_tag = cursor.fetchone()
+    # If the tag exists, update the entry in the table with the new sha256
+    if existing_tag:
+        cursor.execute("UPDATE tags SET sha256 = ? WHERE version_id = ? AND tag = ?",
+                       (r.sha256, version_id, r.tag))
+    # Else add a new row
+    else:
+        cursor.execute("INSERT INTO tags (version_id, tag, sha256) VALUES (?, ?, ?)",
+                       (version_id, r.tag, r.sha256))
+
+    # Commit the transaction
+    store.commit()
+
+
+if __name__ == "__main__":
+    store = create_db("index1.db")
+    for r in prgenvgnu_records:
+        add_record(store, r)
+    store.close()
+

test/input/versioned_db/readme.md

@@ -0,0 +1,3 @@
+`index.db` files for each database version.
+
+used to test the `uenv image upgrade` functionality.

test/unit/inputs.py

@@ -9,3 +9,9 @@
 
 def activate():
     return input_path / "activate"
+
+def database(version):
+    """
+    returns the full path of a index.db created with "version" of the database schema
+    """
+    return input_path / "versioned_db" / f"v{version}" / "index.db"

test/unit/test_datastore.py

@@ -1,9 +1,11 @@
 import shutil
+import sqlite3
 import unittest
 
 import datastore
 import record
 import scratch
+import inputs
 
 prgenvgnu_records = [
     record.Record("santis", "gh200", "prgenv-gnu", "23.11", "default", "monday", 1024, "a"*64),
@@ -245,11 +247,6 @@ class TestRepositoryCreate(unittest.TestCase):
     def setUp(self):
         self.path = scratch.make_scratch_path("repository_create")
 
-    def test_create_new_repo(self):
-        # expect an exception when an invalid field is passed (sustem is a typo for system)
-        with self.assertRaises(ValueError):
-            result = store.find_records(sustem="santis")
-
     def test_create_new_repo(self):
         # create a new repository with database on disk
         datastore.FileSystemRepo.create(self.path.as_posix())
@@ -267,5 +264,43 @@ def test_create_new_repo(self):
     def tearDown(self):
         shutil.rmtree(self.path, ignore_errors=True)
 
+class TestUpgrade(unittest.TestCase):
+
+    def setUp(self):
+        self.path = scratch.make_scratch_path("upgrade_from_v1")
+        self.db_path = self.path / "index.db"
+
+        source_path = inputs.database(1)
+        shutil.copy(source_path, self.db_path)
+
+        # TODO: add checks that a v2 database is correctly identified and no action is taken
+
+    def test_v1_to_v2(self):
+
+        repo_path = self.path.as_posix()
+
+        # test that the database is correctly identified as v1
+        self.assertEqual(datastore.repo_version(repo_path), 1) 
+        # test that the database is correctly identified as out of date, but fixable
+        self.assertEqual(datastore.repo_status(repo_path), 0)
+
+        # perform upgrade
+        datastore.repo_upgrade(repo_path)
+
+        # open the upgraded database and check contents
+        self.assertEqual(datastore.repo_version(repo_path), 2) 
+
+        store = datastore.FileSystemRepo(self.path.as_posix())
+        db = store.database
+        self.assertEqual(len(db.find_records(version="23.11").records), 3)
+        self.assertEqual(len(db.find_records(version="24.2").records),  2)
+        self.assertEqual(len(db.find_records(sha="b"*64).records),  2)
+        self.assertEqual(len(db.find_records(sha="c"*64).records),  1)
+        self.assertEqual(len(db.find_records(name="prgenv-gnu").records),  5)
+        self.assertEqual(len(db.find_records(tag="v1").records),  2)
+
+    def tearDown(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+
 if __name__ == '__main__':
     unittest.main()