Improved View API

This extends the view API to allow for the registration of multiple views using custom names.
It also introduces an option to overwrite the default storage model used by the storage
abstraction. Together this allows to fully customize and extend the storage interface.

src/machinable/core/mixin.py

@@ -4,7 +4,7 @@
 
 
 class MixinInstance:
-    def __init__(self, controller, mixin_class, attribute=False):
+    def __init__(self, controller, mixin_class, attribute=None):
         self._binding = {
             "controller": controller,
             "class": mixin_class,
@@ -36,7 +36,7 @@ def __getattr__(self, item):
 
         def bound_method(*args, **kwargs):
             # bind mixin instance to controller for mixin self reference
-            if self._binding["attribute"] is not False:
+            if self._binding["attribute"] is not None:
                 self._binding["controller"].__mixin__ = getattr(
                     self._binding["controller"], self._binding["attribute"]
                 )

src/machinable/storage/__init__.py

@@ -1,5 +1,4 @@
 from .storage import Storage
-from .view import View
 
 
 def get_component(url):

src/machinable/storage/component.py

@@ -8,17 +8,14 @@
 from ..storage.models import StorageComponentModel
 from ..utils.utils import sentinel
 from .collections import RecordCollection
-from .models.filesystem import StorageComponentFileSystemModel
-from .view import View
+from .views.views import get as get_view
 
 
 class StorageComponent:
     def __init__(
         self, url: Union[str, dict, StorageComponentModel], experiment=None, cache=None
     ):
-        self._model = StorageComponentModel.create(
-            url, template=StorageComponentFileSystemModel
-        )
+        self._model = StorageComponentModel.create(url)
         self._cache = cache or {}
         self._cache["experiment"] = experiment
 
@@ -48,16 +45,15 @@ def file(self, filepath, default=sentinel, reload=None):
                 reload = True
 
         if filepath not in self._cache or reload:
-            with open_fs(self.url) as filesystem:
-                try:
-                    self._cache[filepath] = filesystem.load_file(filepath)
-                    if filepath not in self._cache["_files"]:
-                        self._cache["_files"][filepath] = {}
-                    self._cache["_files"][filepath]["loaded_at"] = pendulum.now()
-                except FileNotFoundError:
-                    if default is not sentinel:
-                        return default
-                    raise
+            try:
+                self._cache[filepath] = self._model.file(filepath)
+                if filepath not in self._cache["_files"]:
+                    self._cache["_files"][filepath] = {}
+                self._cache["_files"][filepath]["loaded_at"] = pendulum.now()
+            except FileNotFoundError:
+                if default is not sentinel:
+                    return default
+                raise
 
         return self._cache[filepath]
 
@@ -290,7 +286,17 @@ def is_incomplete(self):
     @property
     def view(self):
         """Returns the registered view"""
-        return View.bind("component", self)
+        return get_view("component", self)
+
+    def __getattr__(self, item):
+        if item.startswith("_") and item.endswith("_"):
+            view = get_view("component", self, name=item)
+            if view is not None:
+                return view
+
+        raise AttributeError(
+            f"{self.__class__.__name__} object has no attribute {item}"
+        )
 
     def serialize(self):
         return {

src/machinable/storage/experiment.py

@@ -8,15 +8,12 @@
 from .collections import ComponentStorageCollection, ExperimentStorageCollection
 from .component import StorageComponent
 from .models import StorageExperimentModel
-from .models.filesystem import StorageExperimentFileSystemModel
-from .view import View
+from .views.views import get as get_view
 
 
 class StorageExperiment:
     def __init__(self, url: Union[str, dict, StorageExperimentModel], cache=None):
-        self._model = StorageExperimentModel.create(
-            url, template=StorageExperimentFileSystemModel
-        )
+        self._model = StorageExperimentModel.create(url)
         self._cache = cache or {}
 
     @classmethod
@@ -198,7 +195,17 @@ def ancestor(self) -> Optional["StorageExperiment"]:
     @property
     def view(self):
         """Returns the registered view"""
-        return View.bind("experiment", self)
+        return get_view("experiment", self)
+
+    def __getattr__(self, item):
+        if item.startswith("_") and item.endswith("_"):
+            view = get_view("experiment", self, name=item)
+            if view is not None:
+                return view
+
+        raise AttributeError(
+            f"{self.__class__.__name__} object has no attribute {item}"
+        )
 
     def serialize(self):
         return {

src/machinable/storage/models/__init__.py

@@ -1 +1 @@
-from .model import StorageComponentModel, StorageExperimentModel, StorageModel
+from .models import StorageComponentModel, StorageExperimentModel, StorageModel

src/machinable/storage/models/filesystem.py

@@ -1,23 +1,20 @@
 import os
 
-import pendulum
-
 from ...filesystem import open_fs
 from ...utils.identifiers import decode_experiment_id
-from ...utils.utils import sentinel
-from .model import StorageComponentModel, StorageExperimentModel
+from .models import StorageComponentModel, StorageExperimentModel
 
 
 class StorageFileSystemModel:
     def file(self, filepath):
         with open_fs(self.url) as filesystem:
             return filesystem.load_file(filepath)
 
-    def experiment_model(self, data):
-        return StorageExperimentFileSystemModel(data)
+    def experiment_model(self, url):
+        return StorageExperimentFileSystemModel(url)
 
-    def component_model(self, data):
-        return StorageComponentFileSystemModel(data)
+    def component_model(self, url):
+        return StorageComponentFileSystemModel(url)
 
 
 class StorageExperimentFileSystemModel(StorageFileSystemModel, StorageExperimentModel):

src/machinable/storage/models/model.py → src/machinable/storage/models/models.py

@@ -1,4 +1,10 @@
 from ...filesystem import parse_storage_url
+from ...utils.importing import ModuleClass
+
+_register = {
+    "experiment": None,
+    "component": None,
+}
 
 
 class StorageModel:
@@ -15,14 +21,25 @@ def __init__(self, url: str):
         self.component_id = parsed["component_id"]
 
     @classmethod
-    def create(cls, args, template=None):
-        if isinstance(args, StorageModel):
-            return args
+    def clear(cls, types=None):
+        if types is None:
+            types = ["experiment", "component"]
+        if isinstance(types, str):
+            types = [types]
+        for k in types:
+            _register[k] = None
 
-        if template is not None:
-            return template(args)
+    @classmethod
+    def component(cls, model=None):
+        if isinstance(model, str):
+            model = ModuleClass(model, baseclass=StorageComponentModel)
+        _register["component"] = model
 
-        return cls(args)
+    @classmethod
+    def experiment(cls, model=None):
+        if isinstance(model, str):
+            model = ModuleClass(model, baseclass=StorageExperimentModel)
+        _register["component"] = model
 
     def submit(self, key, value):
         pass
@@ -33,10 +50,10 @@ def prefetch(self):
     def file(self, filepath):
         raise NotImplementedError
 
-    def experiment_model(self, data):
+    def experiment_model(self, url):
         raise NotImplementedError
 
-    def component_model(self, data):
+    def component_model(self, url):
         raise NotImplementedError
 
 
@@ -46,6 +63,20 @@ def __init__(self, data):
         if self.component_id is not None:
             raise ValueError("The provided URL is not a valid experiment storage URL/")
 
+    @classmethod
+    def create(cls, args):
+        if isinstance(args, StorageExperimentModel):
+            return args
+
+        # find registered default
+        if _register["experiment"] is not None:
+            return _register["experiment"](args)
+
+        # use global default
+        from .filesystem import StorageExperimentFileSystemModel
+
+        return StorageExperimentFileSystemModel(args)
+
     def exists(self):
         try:
             return self.file("execution.json")["experiment_id"] == self.experiment_id
@@ -74,6 +105,20 @@ def __init__(self, data):
         if self.component_id is None:
             raise ValueError("The provided URL is not a valid component storage URL")
 
+    @classmethod
+    def create(cls, args):
+        if isinstance(args, StorageComponentModel):
+            return args
+
+        # find registered default
+        if _register["component"] is not None:
+            return _register["component"](args)
+
+        # use global default
+        from .filesystem import StorageComponentFileSystemModel
+
+        return StorageComponentFileSystemModel(args)
+
     def exists(self):
         try:
             return bool(self.file("status.json")["started_at"])

src/machinable/storage/views/__init__.py

@@ -0,0 +1 @@
+from .views import StorageExperimentView, StorageComponentView, StorageView

src/machinable/storage/views/views.py

@@ -0,0 +1,72 @@
+import inspect
+
+from ...core.mixin import Mixin, MixinInstance
+from ...utils.importing import ModuleClass
+
+_register = {
+    "experiment": {},
+    "component": {},
+}
+
+
+def get(view_type, instance, name=None):
+    try:
+        return MixinInstance(instance, _register[view_type][name], attribute=name)
+    except KeyError:
+        return None
+
+
+class StorageView:
+    @classmethod
+    def clear(cls, types=None):
+        if types is None:
+            types = ["experiment", "component"]
+        if isinstance(types, str):
+            types = [types]
+        for k in types:
+            _register[k] = {}
+
+    @classmethod
+    def component(cls, view=None, *, name=None):
+        if name is not None:
+            name = "_" + name + "_"
+
+        def _decorate(f):
+            if isinstance(f, str):
+                f = ModuleClass(f, baseclass=StorageComponentView)
+            else:
+                if not inspect.isclass(f):
+                    raise ValueError(f"View has to be a class")
+            _register["component"][name] = f
+
+        if view:
+            return _decorate(view)
+
+        return _decorate
+
+    @classmethod
+    def experiment(cls, view=None, *, name=None):
+        if name is not None:
+            name = "_" + name + "_"
+
+        def _decorate(f):
+            if isinstance(f, str):
+                f = ModuleClass(f, baseclass=StorageExperimentView)
+            else:
+                if not inspect.isclass(f):
+                    raise ValueError(f"View has to be a class")
+
+            _register["experiment"][name] = f
+
+        if view:
+            return _decorate(view)
+
+        return _decorate
+
+
+class StorageExperimentView(Mixin):
+    pass
+
+
+class StorageComponentView(Mixin):
+    pass