Merge pull request #928 from OpenSourceBrain/feature/biomodels-ankur

Biomodels integration

applications/osb-portal/src/components/repository/resources.ts

@@ -3,4 +3,5 @@ import { RepositoryType } from "../../apiclient/workspaces";
 export default {
   [RepositoryType.Dandi]: "DANDI Archive",
   [RepositoryType.Github]: "GitHub",
+  [RepositoryType.Biomodels]: "Biomodels",
 } as any;

applications/osb-portal/src/pages/RepositoryPage.tsx

@@ -257,6 +257,13 @@ export const RepositoryPage = (props: any) => {
       case "figshare":
         window.open(`${repository.uri}`, "_blank");
         break;
+      // Biomodels: repo.version
+      case "biomodels":
+        window.open(
+          `${repository.uri + "." + repository.defaultContext}`,
+          "_blank"
+        );
+        break;
       default:
         window.open(`#`, "_blank");
     }

applications/workspaces/server/workspaces/models/__init__.py

@@ -45,4 +45,4 @@
 from workspaces.models.workspace_resource_entity import WorkspaceResourceEntity
 from workspaces.models.workspace_resource_entity_all_of import WorkspaceResourceEntityAllOf
 from workspaces.models.repository_info import RepositoryInfo
-from workspaces.models.biomodels_repository_resource import BioModelsRepositoryResource
+from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource

applications/workspaces/server/workspaces/models/biomodels_repository_resource.py

@@ -17,7 +17,7 @@ class BiomodelsRepositoryResource(Model):
     Do not edit the class manually.
     """
 
-    def __init__(self, name=None, path=None, osbrepository_id=None, size=None, timestamp_modified=None):  # noqa: E501
+    def __init__(self, name=None, path=None, osbrepository_id=None, size=None, timestamp_modified=None, ref=None, sha=None):  # noqa: E501
         """BiomodelsRepositoryResource - a model defined in OpenAPI
 
         :param name: The name of this BiomodelsRepositoryResource.  # noqa: E501
@@ -30,28 +30,38 @@ def __init__(self, name=None, path=None, osbrepository_id=None, size=None, times
         :type size: int
         :param timestamp_modified: The timestamp_modified of this BiomodelsRepositoryResource.  # noqa: E501
         :type timestamp_modified: datetime
+        :param ref: The ref of this BiomodelsRepositoryResource.  # noqa: E501
+        :type ref: str
+        :param sha: The sha of this BiomodelsRepositoryResource.  # noqa: E501
+        :type sha: str
         """
         self.openapi_types = {
             'name': str,
             'path': str,
             'osbrepository_id': int,
             'size': int,
-            'timestamp_modified': datetime
+            'timestamp_modified': datetime,
+            'ref': str,
+            'sha': str
         }
 
         self.attribute_map = {
             'name': 'name',
             'path': 'path',
             'osbrepository_id': 'osbrepository_id',
             'size': 'size',
-            'timestamp_modified': 'timestamp_modified'
+            'timestamp_modified': 'timestamp_modified',
+            'ref': 'ref',
+            'sha': 'sha'
         }
 
         self._name = name
         self._path = path
         self._osbrepository_id = osbrepository_id
         self._size = size
         self._timestamp_modified = timestamp_modified
+        self._ref = ref
+        self._sha = sha
 
     @classmethod
     def from_dict(cls, dikt) -> 'BiomodelsRepositoryResource':
@@ -178,3 +188,49 @@ def timestamp_modified(self, timestamp_modified):
         """
 
         self._timestamp_modified = timestamp_modified
+
+    @property
+    def ref(self):
+        """Gets the ref of this BiomodelsRepositoryResource.
+
+        The GIT ref  # noqa: E501
+
+        :return: The ref of this BiomodelsRepositoryResource.
+        :rtype: str
+        """
+        return self._ref
+
+    @ref.setter
+    def ref(self, ref):
+        """Sets the ref of this BiomodelsRepositoryResource.
+
+        The GIT ref  # noqa: E501
+
+        :param ref: The ref of this BiomodelsRepositoryResource.
+        :type ref: str
+        """
+
+        self._ref = ref
+
+    @property
+    def sha(self):
+        """Gets the sha of this BiomodelsRepositoryResource.
+
+        The GIT sha of the resource  # noqa: E501
+
+        :return: The sha of this BiomodelsRepositoryResource.
+        :rtype: str
+        """
+        return self._sha
+
+    @sha.setter
+    def sha(self, sha):
+        """Sets the sha of this BiomodelsRepositoryResource.
+
+        The GIT sha of the resource  # noqa: E501
+
+        :param sha: The sha of this BiomodelsRepositoryResource.
+        :type sha: str
+        """
+
+        self._sha = sha

applications/workspaces/server/workspaces/models/repository_resource.py

@@ -6,14 +6,12 @@
 from typing import List, Dict  # noqa: F401
 
 from workspaces.models.base_model_ import Model
-from workspaces.models.dandi_repository_resource import DandiRepositoryResource
-from workspaces.models.figshare_repository_resource import FigshareRepositoryResource
-from workspaces.models.git_repository_resource import GITRepositoryResource
-from workspaces import util
-
 from workspaces.models.dandi_repository_resource import DandiRepositoryResource  # noqa: E501
 from workspaces.models.figshare_repository_resource import FigshareRepositoryResource  # noqa: E501
 from workspaces.models.git_repository_resource import GITRepositoryResource  # noqa: E501
+from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource  # noqa: E501
+from workspaces import util
+
 
 class RepositoryResource(Model):
     """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

applications/workspaces/server/workspaces/service/osbrepository/__init__.py

@@ -7,7 +7,7 @@
 
 
 
-from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter
+from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter
 
 
 def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=None, uri=None, *args, **kwargs):
@@ -20,6 +20,8 @@ def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=No
         return DandiAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
     elif repository_type == "figshare":
         return FigShareAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
+    elif repository_type == "biomodels":
+        return BiomodelsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
     return None
 
 

applications/workspaces/server/workspaces/service/osbrepository/adapters/__init__.py

@@ -1,4 +1,4 @@
 from workspaces.service.osbrepository.adapters.dandiadapter import DandiAdapter
 from workspaces.service.osbrepository.adapters.figshareadapter import FigShareAdapter
 from workspaces.service.osbrepository.adapters.githubadapter import GitHubAdapter
-from workspaces.service.osbrepository.adapters.biomodelsadapter import BioModelsAdapter
+from workspaces.service.osbrepository.adapters.biomodelsadapter import BiomodelsAdapter

applications/workspaces/server/workspaces/service/osbrepository/adapters/biomodelsadapter.py

@@ -6,59 +6,142 @@
 from cloudharness import log as logger
 from workspaces.models import RepositoryResourceNode, RepositoryInfo
 from workspaces.models.resource_origin import ResourceOrigin
-from workspaces.models.biomodels_repository_resource import BioModelsRepositoryResource
+from workspaces.models.biomodels_repository_resource import BiomodelsRepositoryResource
 
 from .utils import add_to_tree
 
 
+class BiomodelsException(Exception):
+    pass
 
 
-class BioModelsAdapter:
+class BiomodelsAdapter:
     """
-    Adapter for FigShare
+    Adapter for Biomodels
 
-    https://docs.figshare.com/
+    https://www.ebi.ac.uk/biomodels/
     """
 
     def __init__(self, osbrepository, uri=None):
         self.osbrepository = osbrepository
         self.uri = uri if uri else osbrepository.uri
-        # even for different figshare "instances", the IDs remain the same, and
-        # there's only one API end point
-        self.api_url = ...
-
-
+        self.api_url = "https://www.ebi.ac.uk/biomodels"
+
+        try:
+            self.model_id = re.search(
+                f"{self.api_url}/(\\w+)",
+                self.uri.strip("/")).group(1)
+
+        except AttributeError:
+            raise BiomodelsException(f"{uri} is not a valid Biomodels URL")
+
+    def get_json(self, uri):
+        logger.debug(f"Getting: {uri}")
+        try:
+            r = requests.get(
+                uri,
+                params={"format": "json"}
+            )
+            if r.status_code == 200:
+                return r.json()
+            else:
+                raise BiomodelsException(
+                    f"Unexpected requests status code: {r.status_code}")
+        except Exception as e:
+            raise BiomodelsException("Unexpected error:", sys.exc_info()[0])
+
+    def get_base_uri(self):
+        return self.uri
+
+    def get_info(self) -> RepositoryInfo:
+        info = self.get_json(
+            f"{self.api_url}/{self.model_id}")
+        return RepositoryInfo(name=info["name"], contexts=self.get_contexts(), tags=info["format"]["name"], summary=info.get("description", ""))
 
     def get_contexts(self):
-        ...
+        result = self.get_json(f"{self.api_url}/{self.model_id}")
+        revisions = result["history"]["revisions"]
+        return [str(v["version"]) for v in revisions]
 
-    def get_resources(self, context):
-        ...
+    def _get_filelist(self, context):
+        logger.debug(f"Getting filelist: {context}")
+        contents = self.get_json(f"{self.api_url}/model/files/{self.model_id}.{context}")
+        files = (contents.get("additional", []) + contents.get("main", []))
+        return files
 
+    def get_resources(self, context):
+        logger.debug(f"Getting resources: {context}")
+        files = self._get_filelist(context)
+
+        tree = RepositoryResourceNode(
+            resource=BiomodelsRepositoryResource(
+                name="/",
+                path="/",
+                osbrepository_id=self.osbrepository.id,
+                ref=context,
+            ),
+            children=[],
+        )
+
+        for afile in files:
+            download_url = f"{self.api_url}/model/download/{self.model_id}.{context}?filename={afile['name']}"
+            add_to_tree(
+                tree=tree,
+                tree_path=[afile["name"]],
+                path=download_url,
+                size=int(afile["fileSize"]),
+                osbrepository_id=self.osbrepository.id,
+            )
+
+        return tree
 
     def get_description(self, context):
-        ...
+        logger.debug(f"Getting description: {context}")
+        try:
+            result = self.get_json(f"{self.api_url}/{self.model_id}.{context}")
+            return result["description"]
+        except Exception as e:
+            logger.debug(
+                "unable to get the description from biomodels, %", str(e))
+            return ""
 
     def get_tags(self, context):
-        ...
+        # using the format name for the moment, since they don't do explict
+        # tags/keywords
+        logger.debug(f"Getting tags: {context}")
+        result = self.get_json(f"{self.api_url}/{self.model_id}.{context}")
+        return result["format"]["name"]
 
+    # biomodels files are usually small, so one task is enough
     def create_copy_task(self, workspace_id, origins: List[ResourceOrigin]):
-        tasks = []
         import workspaces.service.workflow as workflow
-        for origin in origins:
-            path = origin.path
-            # no file tree in FigShare
-            folder = self.osbrepository.name
-
-
-            # username / password are optional and future usage,
-            # e.g. for accessing non public repos
-            tasks.append(workflow.create_copy_task(
-                image_name="workspaces-biomodels-copy",
-                workspace_id=workspace_id,
-                folder=folder,
-                url=path,
-                username="",
-                password="",
-            ))
-        return tasks
+
+        # no file tree in Biomodels from the looks of it
+        folder = self.osbrepository.name
+
+        # if nothing is selected, origins has one entry with path "/"
+        # we get the file list and download individual files
+        # Biomodels does allow downloading the archive, but that is generated
+        # on the fly and can require us to wait for an unspecified amount of
+        # time
+        if len(origins) == 1 and origins[0].path == "/":
+            """
+            # to use the archive method, just set paths to ""
+            paths = ""
+            """
+            files = self._get_filelist(self.osbrepository.default_context)
+            download_url_prefix = f"{self.api_url}/model/download/{self.model_id}.{self.osbrepository.default_context}?filename="
+            paths = "\\".join(f"{download_url_prefix}{file['name']}" for file in files)
+        else:
+            paths = "\\".join(o.path for o in origins)
+
+        # username / password are not currently used
+        return workflow.create_copy_task(
+            image_name="workspaces-biomodels-copy",
+            workspace_id=workspace_id,
+            folder=folder,
+            url=f"{self.model_id}.{self.osbrepository.default_context}",
+            paths=paths,
+            username="",
+            password="",
+        )

applications/workspaces/server/workspaces/service/osbrepository/osbrepository_service.py

@@ -5,7 +5,7 @@
 
 
 
-from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter
+from workspaces.service.osbrepository.adapters import DandiAdapter, FigShareAdapter, GitHubAdapter, BiomodelsAdapter
 
 
 def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=None, uri=None, *args, **kwargs):
@@ -19,7 +19,7 @@ def get_repository_adapter(osbrepository: OSBRepository=None, repository_type=No
     elif repository_type == "figshare":
         return FigShareAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
     elif repository_type == "biomodels":
-        return BioModelsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
+        return BiomodelsAdapter(*args, osbrepository=osbrepository, uri=uri, **kwargs)
     return None
 
 

applications/workspaces/tasks/biomodels-copy/Dockerfile

@@ -1,6 +1,10 @@
-FROM python:3.9
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
+
+# much faster than curl/wget
+# https://pkgs.alpinelinux.org/packages?name=aria2&branch=edge
+RUN apk add aria2 unzip
 
-RUN pip install --no-cache-dir #TODO
 
 ADD . /
 

applications/workspaces/tasks/biomodels-copy/README.md

@@ -0,0 +1,16 @@
+# Biomodels copy task
+
+
+How to test
+
+```
+shared_directory=/tmp folder=osbv2/develop url=BIOMD0000000998.9 ./run.sh
+```
+
+The above should checkout the file README.md and the full directory applications/workspaces inside /tmp/osbv2/develop
+
+
+```
+shared_directory=/tmp folder=osbv2/develop url=https://github.com/OpenSourceBrain/OSBv2 branch=develop paths= ./run.sh
+```
+This should checkout the whole repo