From 189571960dc9f01e5a389d69abf6b42f6f9e98ca Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 9 Oct 2024 09:55:01 +0530 Subject: [PATCH 01/59] add missing dependencies for serverless Update pyproject.toml --- pyproject.toml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a37d07d..946c00a3 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,19 +36,25 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "build>=0.8.0", - "django~=4.2", - "djangorestframework~=3.14", "filelock>=3.7.1", - "numpy>=1.18.0,<2", - "packaging>=21.0", "docker>=7.1.0", "pypng>=0.20220715.0", - "python-dateutil>=2.8.0", - "pytz>=2021.3", "requests>=2.32", "urllib3<3.0.0", "Pillow>=9.3.0", + "python-dateutil>=2.8.0", + "pytz>=2021.3", + "packaging>=21.0", + "build>=0.8.0", + # core ADR dependencies + "django~=4.2", + "djangorestframework~=3.14", + "django-guardian~=2.4", + "tzlocal~=5.0", + "numpy>=1.18.0,<2", + "python-pptx==0.6.19", + "pandas~=2.0", + "statsmodels~=0.14", ] [tool.setuptools.packages.find] From aaa3e9ad28bcd2e5d5e3b10edb757cf55e061ec2 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 9 Oct 2024 10:22:23 +0530 Subject: [PATCH 02/59] reset flag before save --- src/ansys/dynamicreporting/core/serverless/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 0cc2b7bb..e9b62fcc 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -286,6 +286,8 @@ def from_db(cls, orm_instance, parent=None): @handle_field_errors def save(self, **kwargs): + self._saved = False # reset + cls_fields = self._get_all_field_names() model_fields = self._get_orm_field_names(self._orm_instance) for field_ in cls_fields: From 51f72c5e450711a043ff1253b675b00f0658e639 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 9 Oct 2024 11:13:39 +0530 Subject: [PATCH 03/59] fix some type hints Update template.py --- .../dynamicreporting/core/serverless/adr.py | 31 ++++++++++--------- .../core/serverless/template.py | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index ae9d384f..afb43713 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable +# from collections.abc import Iterable import os from pathlib import Path import platform @@ -8,7 +8,7 @@ import django from django.core import management -from django.db import IntegrityError, connection, connections +# from django.db import IntegrityError, connection, connections from django.http import HttpRequest from .. import DEFAULT_ANSYS_VERSION @@ -31,14 +31,14 @@ def __init__( self, ansys_installation: str, *, - db_directory: str = None, - databases: dict = None, - media_directory: str = None, - static_directory: str = None, - debug: bool = None, - opts: dict = None, - request: HttpRequest = None, - logfile: str = None, + db_directory: Optional[str] = None, + databases: Optional[dict] = None, + media_directory: Optional[str] = None, + static_directory: Optional[str] = None, + debug: Optional[bool] = None, + opts: Optional[dict] = None, + request: Optional[HttpRequest] = None, + logfile: Optional[str] = None, ) -> None: self._db_directory = None self._databases = databases or {} @@ -87,7 +87,7 @@ def __init__( elif "CEI_NEXUS_LOCAL_STATIC_DIR" in os.environ: self._static_directory = self._check_dir(os.environ["CEI_NEXUS_LOCAL_STATIC_DIR"]) - def _get_install_directory(self, ansys_installation: Optional[str]) -> Path: + def _get_install_directory(self, ansys_installation: str) -> Path: dirs_to_check = [] if ansys_installation: # User passed directory @@ -273,7 +273,7 @@ def get_report(self, **kwargs) -> Template: self._logger.error(f"{e}") raise e - def get_reports(self, fields: list = None, flat: bool = False) -> Union[ObjectSet, list]: + def get_reports(self, fields: Optional[list] = None, flat: bool = False) -> Union[ObjectSet, list]: # return list of reports by default. # if fields are mentioned, return value list try: @@ -286,7 +286,7 @@ def get_reports(self, fields: list = None, flat: bool = False) -> Union[ObjectSe return out - def get_list_reports(self, r_type: Optional[str] = "name") -> Union[ObjectSet, list]: + def get_list_reports(self, r_type: str = "name") -> Union[ObjectSet, list]: supported_types = ["name", "report"] if r_type not in supported_types: raise ADRException(f"r_type must be one of {supported_types}") @@ -295,7 +295,7 @@ def get_list_reports(self, r_type: Optional[str] = "name") -> Union[ObjectSet, l else: return self.get_reports() - def render_report(self, context: dict = None, query: str = None, **kwargs: Any) -> str: + def render_report(self, context: Optional[dict] = None, query: str = "", **kwargs: Any) -> str: try: return Template.get(**kwargs).render( request=self._request, context=context, query=query @@ -307,9 +307,10 @@ def render_report(self, context: dict = None, query: str = None, **kwargs: Any) def query( self, query_type: Union[Session, Dataset, Type[Item], Type[Template]], - query: Optional[str] = "", + query: str = "", ) -> ObjectSet: if not issubclass(query_type, (Item, Template, Session, Dataset)): self._logger.error(f"{query_type} is not valid") raise TypeError(f"{query_type} is not valid") return query_type.find(query=query) + diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index df1f1312..445fc104 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -172,7 +172,7 @@ def find(cls, **kwargs): new_kwargs = {**kwargs, "query": f"A|t_types|cont|{cls.report_type};{query}"} return super().find(**new_kwargs) - def render(self, context=None, request=None, query=None) -> Optional[str]: + def render(self, context=None, request=None, query="") -> str: if context is None: context = {} ctx = {**context, "request": request, "ansys_version": None} From d3abc60bfd45b1257f37218f5a2e931ca1406613 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 9 Oct 2024 14:16:15 +0530 Subject: [PATCH 04/59] fix formatting --- .../dynamicreporting/core/serverless/adr.py | 8 +- tests/test_serverless.py | 12 + top.html | 396 ++++++++++++++++++ 3 files changed, 413 insertions(+), 3 deletions(-) create mode 100644 tests/test_serverless.py create mode 100644 top.html diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index afb43713..65666731 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -8,6 +8,7 @@ import django from django.core import management + # from django.db import IntegrityError, connection, connections from django.http import HttpRequest @@ -151,7 +152,7 @@ def setup(self, collect_static: bool = False) -> None: if self._databases: if "default" not in self._databases: raise ImproperlyConfiguredError( - """ The database configuration must be a dictionary of the following format with + """ The 'databases' option must be a dictionary of the following format with a "default" database specified. { "default": { @@ -273,7 +274,9 @@ def get_report(self, **kwargs) -> Template: self._logger.error(f"{e}") raise e - def get_reports(self, fields: Optional[list] = None, flat: bool = False) -> Union[ObjectSet, list]: + def get_reports( + self, fields: Optional[list] = None, flat: bool = False + ) -> Union[ObjectSet, list]: # return list of reports by default. # if fields are mentioned, return value list try: @@ -313,4 +316,3 @@ def query( self._logger.error(f"{query_type} is not valid") raise TypeError(f"{query_type} is not valid") return query_type.find(query=query) - diff --git a/tests/test_serverless.py b/tests/test_serverless.py new file mode 100644 index 00000000..9bc70e5b --- /dev/null +++ b/tests/test_serverless.py @@ -0,0 +1,12 @@ +from os.path import join + +import numpy as np +import pytest + +from ansys.dynamicreporting.core import ADR +from ansys.dynamicreporting.core.item import Image + + +@pytest.mark.ado_test +def test_create_img(adr_serverless_create, request) -> bool: + ... diff --git a/top.html b/top.html new file mode 100644 index 00000000..791e7812 --- /dev/null +++ b/top.html @@ -0,0 +1,396 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

Serverless Simulation Report

+ +
+

Table of Content

+
+
+
+ +
+
+ + + +
+
+ + + +

This section describes the settings for the simulation: initial conditions, solver settings, and such.

+ + +
+ +
+ +
+
+ + + +
+ + + + + + + + +
SolverMy Solver
Number cells10000000.00
Mesh Size1.0 mm^3
Mesh TypeHex8
+
+ + +
+ +
+ + + +
+ +
+
+ + + + + + +
+ +
+ + + + + + +
+
+ +
+ + + + + + + + + + + + + + +
From 597aac5a3a01dff78f1f09e8c8562164d931fb8e Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 9 Oct 2024 17:55:32 +0530 Subject: [PATCH 05/59] Delete top.html --- top.html | 396 ------------------------------------------------------- 1 file changed, 396 deletions(-) delete mode 100644 top.html diff --git a/top.html b/top.html deleted file mode 100644 index 791e7812..00000000 --- a/top.html +++ /dev/null @@ -1,396 +0,0 @@ -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

Serverless Simulation Report

- -
-

Table of Content

-
-
-
- -
-
- - - -
-
- - - -

This section describes the settings for the simulation: initial conditions, solver settings, and such.

- - -
- -
- -
-
- - - -
- - - - - - - - -
SolverMy Solver
Number cells10000000.00
Mesh Size1.0 mm^3
Mesh TypeHex8
-
- - -
- -
- - - -
- -
-
- - - - - - -
- -
- - - - - - -
-
- -
- - - - - - - - - - - - - - -
From 77566b145c8095e0d7e44fcccb2a27d1f6cd1642 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 10 Oct 2024 18:22:53 +0530 Subject: [PATCH 06/59] Delete test_serverless.py --- tests/test_serverless.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 tests/test_serverless.py diff --git a/tests/test_serverless.py b/tests/test_serverless.py deleted file mode 100644 index 9bc70e5b..00000000 --- a/tests/test_serverless.py +++ /dev/null @@ -1,12 +0,0 @@ -from os.path import join - -import numpy as np -import pytest - -from ansys.dynamicreporting.core import ADR -from ansys.dynamicreporting.core.item import Image - - -@pytest.mark.ado_test -def test_create_img(adr_serverless_create, request) -> bool: - ... From 598a75ff1def28d3d58d7da31756c66570bf1667 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 10 Oct 2024 18:33:33 +0530 Subject: [PATCH 07/59] formatting --- .github/workflows/ci_cd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index d21e9d01..6bd0d493 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -26,7 +26,6 @@ env: jobs: style: - name: Code style runs-on: ubuntu-latest steps: From 98d3d0f87b4707bd8c04bfc9ce04cfb5a65bd7c9 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 10 Oct 2024 18:40:39 +0530 Subject: [PATCH 08/59] disable docs-style Revert "disable docs-style" This reverts commit 3b285ab88a9ec752f61840e22ba13fceea7affbb. Reapply "disable docs-style" This reverts commit 4a0b25d8a248ddf2ff07413eaf47084538d38ff2. --- .github/workflows/ci_cd.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 6bd0d493..fb8c9fe1 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -35,14 +35,14 @@ jobs: python-version: ${{ env.MAIN_PYTHON_VERSION }} show-diff-on-failure: false - docs-style: - name: Documentation style check - runs-on: ubuntu-latest - steps: - - name: PyAnsys documentation style checks - uses: ansys/actions/doc-style@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} +# docs-style: +# name: Documentation style check +# runs-on: ubuntu-latest +# steps: +# - name: PyAnsys documentation style checks +# uses: ansys/actions/doc-style@v4 +# with: +# token: ${{ secrets.GITHUB_TOKEN }} smoke-tests: name: Build and smoke tests @@ -114,7 +114,7 @@ jobs: docs: name: Build docs runs-on: ubuntu-latest - needs: [docs-style] +# needs: [docs-style] steps: - name: Run Ansys documentation building action uses: ansys/actions/doc-build@v4 @@ -183,7 +183,7 @@ jobs: build-failure: name: Teams notify on failure if: failure() && (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.ref_type == 'tag') - needs: [ style, test, docs-style ] + needs: [ style, test] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From cfe846c57cc3c67ec508c0e5b87350ab825f77e5 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 11 Oct 2024 19:04:23 +0530 Subject: [PATCH 09/59] handle database errors with save() Update base.py --- src/ansys/dynamicreporting/core/exceptions.py | 6 ++ .../dynamicreporting/core/serverless/base.py | 68 +++++++++++-------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/ansys/dynamicreporting/core/exceptions.py b/src/ansys/dynamicreporting/core/exceptions.py index 78634313..074a6834 100644 --- a/src/ansys/dynamicreporting/core/exceptions.py +++ b/src/ansys/dynamicreporting/core/exceptions.py @@ -120,3 +120,9 @@ class MultipleObjectsReturnedError(ADRException): """Exception raised if only one object was expected, but multiple were returned.""" detail = "get() returned more than one object." + + +class IntegrityError(ADRException): + """Exception raised if there is a constraint violation while saving an object in the database.""" + + detail = "A database integrity check failed." diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index e9b62fcc..0fa29f64 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -17,13 +17,14 @@ ObjectDoesNotExist, ValidationError, ) -from django.db import DatabaseError +from django.db import DataError from django.db.models import Model, QuerySet from django.db.models.base import subclass_exception from django.db.models.manager import Manager from ..exceptions import ( ADRException, + IntegrityError, MultipleObjectsReturnedError, ObjectDoesNotExistError, ObjectNotSavedError, @@ -42,7 +43,7 @@ def handle_field_errors(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except (FieldError, FieldDoesNotExist, ValidationError, DatabaseError) as e: + except (FieldError, FieldDoesNotExist, ValidationError, DataError) as e: raise ADRException(extra_detail=f"One or more fields set or accessed are invalid: {e}") return wrapper @@ -97,6 +98,13 @@ def __new__( parents, namespace.get("__module__"), ) + add_exception_to_cls( + "IntegrityError", + IntegrityError, + new_cls, + parents, + namespace.get("__module__"), + ) # all classes must be dataclasses new_cls = dataclass(eq=False, order=False, repr=False)(new_cls) return new_cls @@ -284,36 +292,47 @@ def from_db(cls, orm_instance, parent=None): obj._saved = True return obj - @handle_field_errors - def save(self, **kwargs): - self._saved = False # reset - - cls_fields = self._get_all_field_names() - model_fields = self._get_orm_field_names(self._orm_instance) + def _prepare_for_save(self, **kwargs): + obj = self + obj._saved = False # reset + cls_fields = obj._get_all_field_names() + model_fields = obj._get_orm_field_names(obj._orm_instance) for field_ in cls_fields: if field_ not in model_fields: continue - value = getattr(self, field_, None) - if value is None: + value = getattr(obj, field_, None) + if value is None: # skip and use defaults continue if isinstance(value, list): - obj_list = [] - for obj in value: - obj_list.append(obj._orm_instance) - getattr(self._orm_instance, field_).add(*obj_list) + objs = [o._orm_instance for o in value] + getattr(obj._orm_instance, field_).add(*objs) else: if isinstance(value, BaseModel): # relations try: value = value._orm_instance.__class__.objects.using( kwargs.get("using", "default") ).get(guid=value.guid) - except ObjectDoesNotExist: - raise value.__class__.DoesNotExist + except ObjectDoesNotExist as e: + raise value.__class__.DoesNotExist( + extra_detail=f"Object with guid '{value.guid}'" f" does not exist: {e}" + ) # for all others - setattr(self._orm_instance, field_, value) + setattr(obj._orm_instance, field_, value) + return obj - self._orm_instance.save(**kwargs) - self._saved = True + @handle_field_errors + def save(self, **kwargs): + try: + obj = self._prepare_for_save(**kwargs) + obj._orm_instance.save(**kwargs) + except IntegrityError as e: + raise self.__class__.IntegrityError( + extra_detail=f"Save failed for object with guid '{self.guid}': {e}" + ) + except Exception as e: + raise e + else: + obj._saved = True @classmethod @handle_field_errors @@ -324,7 +343,9 @@ def create(cls, **kwargs): def delete(self, **kwargs): if not self._saved: - raise self.__class__.NotSaved(extra_detail="Delete failed") + raise self.__class__.NotSaved( + extra_detail=f"Delete failed for object with guid '{self.guid}'." + ) count, _ = self._orm_instance.delete(**kwargs) self._saved = False return count @@ -347,13 +368,6 @@ def filter(cls, **kwargs): qs = cls._orm_model_cls.objects.filter(**kwargs) return ObjectSet(_model=cls, _orm_model=cls._orm_model_cls, _orm_queryset=qs) - @classmethod - @handle_field_errors - def bulk_create(cls, **kwargs): - objs = cls._orm_model_cls.objects.bulk_create(**kwargs) - qs = cls._orm_model_cls.objects.filter(pk__in=[obj.pk for obj in objs]) - return ObjectSet(_model=cls, _orm_model=cls._orm_model_cls, _orm_queryset=qs) - @classmethod @handle_field_errors def find(cls, query="", reverse=False, sort_tag="date"): From ba3c552c55693c20a599a262c5652c7af44a9d64 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 18 Oct 2024 15:39:22 -0400 Subject: [PATCH 10/59] make sure to close file handles properly --- .../dynamicreporting/core/serverless/item.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 7dbe669c..58c11825 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -131,9 +131,8 @@ def process(self, value, obj): f"Expected content to be a file path. " f"'{file_path.name}' does not exist or is not a file." ) - - file = DjangoFile(file_path.open(mode="rb")) - + with file_path.open(mode="rb") as f: + file = DjangoFile(f) # check file type file_ext = Path(file.name).suffix.lower() if self.ALLOWED_EXT is not None and file_ext.replace(".", "") not in self.ALLOWED_EXT: @@ -141,7 +140,6 @@ def process(self, value, obj): # check for empty files if file.size == 0: raise ValueError("The file specified is empty") - # save a ref on the object. setattr(obj, "_file", file) return file_str @@ -152,9 +150,10 @@ class ImageContent(FileValidator): def process(self, value, obj): file_str = super().process(value, obj) - file_ext = Path(obj._file.name).suffix.lower() - img_bytes = obj._file.read() + with obj._file.open(mode="rb") as f: + img_bytes = f.read() image = PILImage.open(io.BytesIO(img_bytes)) + file_ext = Path(obj._file.name).suffix.lower() if file_ext in (".tif", ".tiff"): metadata = report_utils.is_enhanced(image) if not metadata: @@ -192,6 +191,10 @@ def save(self, **kwargs): class FilePayloadMixin: _file: DjangoFile = field(init=False, compare=False, default=None) + @property + def has_file(self): + return self._file is not None + @classmethod def from_db(cls, orm_instance, **kwargs): obj = super().from_db(orm_instance, **kwargs) @@ -203,8 +206,9 @@ def save(self, **kwargs): self._orm_instance.payloadfile = f"{str(self.guid)}_{file_name}" # more general path, save the file into the media directory with open(self._orm_instance.get_payload_server_pathname(), "wb") as out_file: - for chunk in self._file.chunks(): - out_file.write(chunk) # chunk -> bytes + with self._file.open(mode="rb") as f: + for chunk in self._file.chunks(): + out_file.write(chunk) # chunk -> bytes super().save(**kwargs) From 79c1d7fef68e15fd2de1b2eb8386cfa9041bda82 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 18 Oct 2024 15:41:26 -0400 Subject: [PATCH 11/59] pass generic kwargs to find() --- src/ansys/dynamicreporting/core/serverless/adr.py | 9 ++++----- src/ansys/dynamicreporting/core/serverless/base.py | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 65666731..53d11101 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,4 +1,4 @@ -# from collections.abc import Iterable +from collections.abc import Iterable import os from pathlib import Path import platform @@ -8,8 +8,6 @@ import django from django.core import management - -# from django.db import IntegrityError, connection, connections from django.http import HttpRequest from .. import DEFAULT_ANSYS_VERSION @@ -120,7 +118,7 @@ def _get_install_directory(self, ansys_installation: str) -> Path: raise InvalidAnsysPath(f"Unable to detect an installation in: {','.join(dirs_to_check)}") def _check_dir(self, dir_): - dir_path = Path(dir_) + dir_path = Path(dir_) if not isinstance(dir_, Path) else dir_ if not dir_path.is_dir(): self._logger.error(f"Invalid directory path: {dir_}") raise InvalidPath(extra_detail=dir_) @@ -311,8 +309,9 @@ def query( self, query_type: Union[Session, Dataset, Type[Item], Type[Template]], query: str = "", + **kwargs: Any, ) -> ObjectSet: if not issubclass(query_type, (Item, Template, Session, Dataset)): self._logger.error(f"{query_type} is not valid") raise TypeError(f"{query_type} is not valid") - return query_type.find(query=query) + return query_type.find(query=query, **kwargs) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 0fa29f64..dbe6094c 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -370,8 +370,8 @@ def filter(cls, **kwargs): @classmethod @handle_field_errors - def find(cls, query="", reverse=False, sort_tag="date"): - qs = cls._orm_model_cls.find(query=query, reverse=reverse, sort_tag=sort_tag) + def find(cls, query="", **kwargs): + qs = cls._orm_model_cls.find(query=query, **kwargs) return ObjectSet(_model=cls, _orm_model=cls._orm_model_cls, _orm_queryset=qs) def get_tags(self): From 73be3d62d1b3fd5476906ccd05b2418ed26bf4da Mon Sep 17 00:00:00 2001 From: viseshrp Date: Mon, 21 Oct 2024 14:40:08 -0400 Subject: [PATCH 12/59] use the file handle properly for file payload saves --- src/ansys/dynamicreporting/core/serverless/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 58c11825..4851beaf 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -207,7 +207,7 @@ def save(self, **kwargs): # more general path, save the file into the media directory with open(self._orm_instance.get_payload_server_pathname(), "wb") as out_file: with self._file.open(mode="rb") as f: - for chunk in self._file.chunks(): + for chunk in f.chunks(): out_file.write(chunk) # chunk -> bytes super().save(**kwargs) From 9714097c251c9a42c1090f47fa2a116b1a8b945d Mon Sep 17 00:00:00 2001 From: viseshrp Date: Mon, 21 Oct 2024 15:13:54 -0400 Subject: [PATCH 13/59] add create_objects and copy_objects stubs --- .../dynamicreporting/core/serverless/adr.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 53d11101..912d4a30 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -315,3 +315,95 @@ def query( self._logger.error(f"{query_type} is not valid") raise TypeError(f"{query_type} is not valid") return query_type.find(query=query, **kwargs) + + @staticmethod + def create_objects( + objects: Union[list, ObjectSet], + **kwargs: Any, + ) -> int: + if not isinstance(objects, Iterable): + raise ADRException("objects must be an iterable") + count = 0 + for obj in objects: + try: + obj.save(**kwargs) + except obj.__class__.IntegrityError: # skip if exists + continue + else: + count += 1 + return count + + def _is_sqlite(self, database: str) -> bool: + return "sqlite" in self._databases[database]["ENGINE"] + + def _get_db_dir(self, database: str) -> str: + return self._databases[database]["NAME"] + + def copy_objects( + self, + object_type: Union[Session, Dataset, Type[Item], Type[Template]], + target_database: str, + source_database: str = "default", + query: str = "", + target_media_dir: str = "", + test: bool = False, + ) -> int: + """ + This copies a selected collection of objects from one database to another. + + GUIDs are preserved and any referenced session and dataset objects are copied as + well. + """ + if not issubclass(object_type, (Item, Template, Session, Dataset)): + self._logger.error(f"{object_type} is not valid") + raise TypeError(f"{object_type} is not valid") + + if target_database not in self._databases or source_database not in self._databases: + raise ADRException( + f"'{source_database}' and '{target_database}' must be configured first" + ) + + objects = self.query(object_type, query=query) + copy_list = [] + media_dir = None + if issubclass(object_type, Item): + for item in objects: + if getattr( + item, "has_file", False + ): # check for media dir if item has a physical file + if not media_dir: + if target_media_dir: + media_dir = target_media_dir + elif self._is_sqlite(target_database): + media_dir = self._check_dir( + Path(self._get_db_dir(target_database)).parent / "media" + ) + else: + raise ADRException( + "'target_media_dir' must be specified because one of the objects" + " contains media to copy.'" + ) + # save the sessions, datasets + # assign to the item to create the new relation + # and then add to the copy list + item.session.save(using=target_database) + item.dataset.save(using=target_database) + copy_list.append(item) + elif issubclass(object_type, Template): + ... + else: # sessions, datasets + copy_list = list(objects) + + if test: + self._logger.info(f"Copying {len(copy_list)} objects...") + return + + try: + count = self.create_objects(copy_list, using=target_database) + except Exception as e: + raise ADRException(f"Some objects could not be copied: {e}") + + # copy media + print(media_dir) + + return count From 8a2f9410d8b3a550e1669bdb1f628e2df7fd52b3 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 23 Oct 2024 11:52:01 -0400 Subject: [PATCH 14/59] migrate multiple databases at once --- .../dynamicreporting/core/serverless/adr.py | 127 +++++------------- .../dynamicreporting/core/serverless/base.py | 66 +++++---- 2 files changed, 72 insertions(+), 121 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 912d4a30..e6faaa8a 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,15 +1,18 @@ -from collections.abc import Iterable import os -from pathlib import Path import platform import sys -from typing import Any, Optional, Type, Union import uuid +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Optional, Type, Union import django from django.core import management from django.http import HttpRequest +from .base import ObjectSet +from .item import Dataset, Item, Session +from .template import Template from .. import DEFAULT_ANSYS_VERSION from ..adr_utils import get_logger from ..exceptions import ( @@ -20,9 +23,6 @@ InvalidPath, StaticFilesCollectionError, ) -from .base import ObjectSet -from .item import Dataset, Item, Session -from .template import Template class ADR: @@ -124,6 +124,23 @@ def _check_dir(self, dir_): raise InvalidPath(extra_detail=dir_) return dir_path + def _migrate_db(self, db): + try: # upgrade databases + management.call_command("migrate", "--no-input", "--database", db, verbosity=0) + except Exception as e: + self._logger.error(f"{e}") + raise DatabaseMigrationError(extra_detail=str(e)) + else: + from django.contrib.auth.models import Group, Permission, User + + if not User.objects.using(db).filter(is_superuser=True).exists(): + user = User.objects.using(db).create_superuser("nexus", "", "cei") + # include the nexus group (with all permissions) + nexus_group, created = Group.objects.using(db).get_or_create(name="nexus") + if created: + nexus_group.permissions.set(Permission.objects.using(db).all()) + nexus_group.user_set.add(user) + def setup(self, collect_static: bool = False) -> None: from django.conf import settings @@ -193,22 +210,11 @@ def setup(self, collect_static: bool = False) -> None: self._dataset = Dataset.create() # migrations - if self._db_directory is not None: - try: # upgrades all databases - management.call_command("migrate", "--no-input", verbosity=0) - except Exception as e: - self._logger.error(f"{e}") - raise DatabaseMigrationError(extra_detail=str(e)) - else: - from django.contrib.auth.models import Group, Permission, User - - if not User.objects.filter(is_superuser=True).exists(): - user = User.objects.create_superuser("nexus", "", "cei") - # include the nexus group (with all permissions) - nexus_group, created = Group.objects.get_or_create(name="nexus") - if created: - nexus_group.permissions.set(Permission.objects.all()) - nexus_group.user_set.add(user) + if self._databases: + for db in self._databases: + self._migrate_db(db) + elif self._db_directory is not None: + self._migrate_db("default") # collectstatic if collect_static and self._static_directory is not None: @@ -325,12 +331,8 @@ def create_objects( raise ADRException("objects must be an iterable") count = 0 for obj in objects: - try: - obj.save(**kwargs) - except obj.__class__.IntegrityError: # skip if exists - continue - else: - count += 1 + obj.save(**kwargs) + count += 1 return count def _is_sqlite(self, database: str) -> bool: @@ -338,72 +340,3 @@ def _is_sqlite(self, database: str) -> bool: def _get_db_dir(self, database: str) -> str: return self._databases[database]["NAME"] - - def copy_objects( - self, - object_type: Union[Session, Dataset, Type[Item], Type[Template]], - target_database: str, - source_database: str = "default", - query: str = "", - target_media_dir: str = "", - test: bool = False, - ) -> int: - """ - This copies a selected collection of objects from one database to another. - - GUIDs are preserved and any referenced session and dataset objects are copied as - well. - """ - if not issubclass(object_type, (Item, Template, Session, Dataset)): - self._logger.error(f"{object_type} is not valid") - raise TypeError(f"{object_type} is not valid") - - if target_database not in self._databases or source_database not in self._databases: - raise ADRException( - f"'{source_database}' and '{target_database}' must be configured first" - ) - - objects = self.query(object_type, query=query) - copy_list = [] - media_dir = None - if issubclass(object_type, Item): - for item in objects: - if getattr( - item, "has_file", False - ): # check for media dir if item has a physical file - if not media_dir: - if target_media_dir: - media_dir = target_media_dir - elif self._is_sqlite(target_database): - media_dir = self._check_dir( - Path(self._get_db_dir(target_database)).parent / "media" - ) - else: - raise ADRException( - "'target_media_dir' must be specified because one of the objects" - " contains media to copy.'" - ) - # save the sessions, datasets - # assign to the item to create the new relation - # and then add to the copy list - item.session.save(using=target_database) - item.dataset.save(using=target_database) - copy_list.append(item) - elif issubclass(object_type, Template): - ... - else: # sessions, datasets - copy_list = list(objects) - - if test: - self._logger.info(f"Copying {len(copy_list)} objects...") - return - - try: - count = self.create_objects(copy_list, using=target_database) - except Exception as e: - raise ADRException(f"Some objects could not be copied: {e}") - - # copy media - print(media_dir) - - return count diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index dbe6094c..b3abe097 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -1,13 +1,13 @@ +import importlib +import inspect +import shlex +import uuid from abc import ABC, ABCMeta, abstractmethod from collections.abc import Iterable from dataclasses import dataclass, field from dataclasses import fields as dataclass_fields -import importlib -import inspect from itertools import chain -import shlex from typing import Any, get_args, get_origin -import uuid from uuid import UUID from django.core.exceptions import ( @@ -21,6 +21,7 @@ from django.db.models import Model, QuerySet from django.db.models.base import subclass_exception from django.db.models.manager import Manager +from django.db.utils import IntegrityError as DBIntegrityError from ..exceptions import ( ADRException, @@ -58,11 +59,11 @@ class BaseMeta(ABCMeta): _model_cls_registry: dict[str, type[Model]] = {} def __new__( - mcs, - cls_name: str, - bases: tuple[type[Any], ...], - namespace: dict[str, Any], - **kwargs: Any, + mcs, + cls_name: str, + bases: tuple[type[Any], ...], + namespace: dict[str, Any], + **kwargs: Any, ) -> type: super_new = super().__new__ # ensure initialization is only performed for subclasses of BaseModel @@ -227,9 +228,17 @@ def _get_all_field_names(cls): return tuple(property_fields) + cls._get_field_names() @property - def saved(self): + def saved(self) -> bool: return self._saved + @property + def _orm_saved(self) -> bool: + return not self._orm_instance._state.adding + + @property + def _orm_db(self) -> str: + return self._orm_instance._state.db + @classmethod def from_db(cls, orm_instance, parent=None): cls_fields = dict(cls._get_field_names(with_types=True, include_private=True)) @@ -293,39 +302,44 @@ def from_db(cls, orm_instance, parent=None): return obj def _prepare_for_save(self, **kwargs): - obj = self - obj._saved = False # reset - cls_fields = obj._get_all_field_names() - model_fields = obj._get_orm_field_names(obj._orm_instance) + target_db = kwargs.pop("using", "default") + # reset + if self._orm_saved: + self._saved = False + # handle cross-database saves + if target_db != self._orm_db: + self._orm_instance = self.__class__._orm_model_cls() + + cls_fields = self._get_all_field_names() + model_fields = self._get_orm_field_names(self._orm_instance) for field_ in cls_fields: if field_ not in model_fields: continue - value = getattr(obj, field_, None) + value = getattr(self, field_, None) if value is None: # skip and use defaults continue if isinstance(value, list): objs = [o._orm_instance for o in value] - getattr(obj._orm_instance, field_).add(*objs) + getattr(self._orm_instance, field_).add(*objs) else: if isinstance(value, BaseModel): # relations try: - value = value._orm_instance.__class__.objects.using( - kwargs.get("using", "default") - ).get(guid=value.guid) + value = value._orm_instance.__class__.objects.using(target_db).get(guid=value.guid) except ObjectDoesNotExist as e: raise value.__class__.DoesNotExist( extra_detail=f"Object with guid '{value.guid}'" f" does not exist: {e}" ) # for all others - setattr(obj._orm_instance, field_, value) - return obj + setattr(self._orm_instance, field_, value) + + return self @handle_field_errors def save(self, **kwargs): try: obj = self._prepare_for_save(**kwargs) obj._orm_instance.save(**kwargs) - except IntegrityError as e: + except DBIntegrityError as e: raise self.__class__.IntegrityError( extra_detail=f"Save failed for object with guid '{self.guid}': {e}" ) @@ -354,7 +368,9 @@ def delete(self, **kwargs): @handle_field_errors def get(cls, **kwargs): try: - orm_instance = cls._orm_model_cls.objects.get(**kwargs) + orm_instance = cls._orm_model_cls.objects.using( + kwargs.pop("using", "default") + ).get(**kwargs) except ObjectDoesNotExist: raise cls.DoesNotExist except MultipleObjectsReturned: @@ -365,7 +381,9 @@ def get(cls, **kwargs): @classmethod @handle_field_errors def filter(cls, **kwargs): - qs = cls._orm_model_cls.objects.filter(**kwargs) + qs = cls._orm_model_cls.objects.using( + kwargs.pop("using", "default") + ).filter(**kwargs) return ObjectSet(_model=cls, _orm_model=cls._orm_model_cls, _orm_queryset=qs) @classmethod From 86ea3191717d2de848ec2fc6742129923d7d190c Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 23 Oct 2024 15:32:20 -0400 Subject: [PATCH 15/59] add as_dict and get_or_create --- .../dynamicreporting/core/serverless/base.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index b3abe097..1c6f8a84 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -301,6 +301,19 @@ def from_db(cls, orm_instance, parent=None): obj._saved = True return obj + def as_dict(self, **kwargs): + out_dict = {} + cls_fields = self._get_all_field_names() + model_fields = self._get_orm_field_names(self._orm_instance) + for field_ in cls_fields: + if field_ not in model_fields: + continue + value = getattr(self, field_, None) + if value is None: # skip and use defaults + continue + out_dict[field_] = value + return out_dict + def _prepare_for_save(self, **kwargs): target_db = kwargs.pop("using", "default") # reset @@ -378,6 +391,22 @@ def get(cls, **kwargs): return cls.from_db(orm_instance) + @classmethod + @handle_field_errors + def get_or_create(cls, **kwargs): + try: + return cls.get(**kwargs), False + except ObjectDoesNotExist: + # Try to create an object using passed params. + try: + return cls.create(**kwargs), True + except cls.IntegrityError: + try: + return cls.get(**kwargs), False + except cls.DoesNotExist: + pass + raise + @classmethod @handle_field_errors def filter(cls, **kwargs): From efbf622cdc082d3bf78c5f25ec9839e39c8dfde9 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 23 Oct 2024 15:49:28 -0400 Subject: [PATCH 16/59] fix as_dict --- .../dynamicreporting/core/serverless/base.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 1c6f8a84..6d739282 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -303,10 +303,9 @@ def from_db(cls, orm_instance, parent=None): def as_dict(self, **kwargs): out_dict = {} - cls_fields = self._get_all_field_names() - model_fields = self._get_orm_field_names(self._orm_instance) + cls_fields = self._get_field_names() for field_ in cls_fields: - if field_ not in model_fields: + if field_.startswith("_"): continue value = getattr(self, field_, None) if value is None: # skip and use defaults @@ -368,15 +367,6 @@ def create(cls, **kwargs): obj.save(force_insert=True) return obj - def delete(self, **kwargs): - if not self._saved: - raise self.__class__.NotSaved( - extra_detail=f"Delete failed for object with guid '{self.guid}'." - ) - count, _ = self._orm_instance.delete(**kwargs) - self._saved = False - return count - @classmethod @handle_field_errors def get(cls, **kwargs): @@ -407,6 +397,15 @@ def get_or_create(cls, **kwargs): pass raise + def delete(self, **kwargs): + if not self._saved: + raise self.__class__.NotSaved( + extra_detail=f"Delete failed for object with guid '{self.guid}'." + ) + count, _ = self._orm_instance.delete(**kwargs) + self._saved = False + return count + @classmethod @handle_field_errors def filter(cls, **kwargs): From b3988cb6cdc3b8d36d39990dc11ae6b8247eb7c9 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 23 Oct 2024 16:20:46 -0400 Subject: [PATCH 17/59] fix Session attrs --- src/ansys/dynamicreporting/core/serverless/item.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 4851beaf..d02e8fb1 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -23,8 +23,8 @@ class Session(BaseModel): date: datetime = field(compare=False, kw_only=True, default_factory=timezone.now) - hostname: str = field(compare=False, kw_only=True, default=str(platform.node)) - platform: str = field(compare=False, kw_only=True, default=str(report_utils.enve_arch)) + hostname: str = field(compare=False, kw_only=True, default=str(platform.node())) + platform: str = field(compare=False, kw_only=True, default=str(report_utils.enve_arch())) application: str = field(compare=False, kw_only=True, default="Serverless ADR Python API") version: str = field(compare=False, kw_only=True, default="1.0") _orm_model: str = "data.models.Session" From c061a07224527e1e67a1160c0e74830246a981ff Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 23 Oct 2024 16:44:05 -0400 Subject: [PATCH 18/59] allow setting guid --- src/ansys/dynamicreporting/core/serverless/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 6d739282..8303f5ba 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -130,7 +130,7 @@ def __getattribute__(cls, name): class BaseModel(metaclass=BaseMeta): - guid: UUID = field(init=False, compare=False, kw_only=True, default_factory=uuid.uuid1) + guid: UUID = field(compare=False, kw_only=True, default_factory=uuid.uuid1) tags: str = field(compare=False, kw_only=True, default="") _saved: bool = field( init=False, compare=False, default=False @@ -301,7 +301,7 @@ def from_db(cls, orm_instance, parent=None): obj._saved = True return obj - def as_dict(self, **kwargs): + def as_dict(self): out_dict = {} cls_fields = self._get_field_names() for field_ in cls_fields: @@ -363,8 +363,9 @@ def save(self, **kwargs): @classmethod @handle_field_errors def create(cls, **kwargs): + target_db = kwargs.pop("using", "default") obj = cls(**kwargs) - obj.save(force_insert=True) + obj.save(force_insert=True, using=target_db) return obj @classmethod @@ -386,7 +387,7 @@ def get(cls, **kwargs): def get_or_create(cls, **kwargs): try: return cls.get(**kwargs), False - except ObjectDoesNotExist: + except cls.DoesNotExist: # Try to create an object using passed params. try: return cls.create(**kwargs), True From dedf8ee3ee934a6034680c4b99e05781263b1df2 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 25 Oct 2024 12:04:04 -0400 Subject: [PATCH 19/59] don't reset the orm object before save --- src/ansys/dynamicreporting/core/serverless/base.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 8303f5ba..1182486b 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -314,14 +314,9 @@ def as_dict(self): return out_dict def _prepare_for_save(self, **kwargs): - target_db = kwargs.pop("using", "default") - # reset - if self._orm_saved: - self._saved = False - # handle cross-database saves - if target_db != self._orm_db: - self._orm_instance = self.__class__._orm_model_cls() + self._saved = False + target_db = kwargs.pop("using", "default") cls_fields = self._get_all_field_names() model_fields = self._get_orm_field_names(self._orm_instance) for field_ in cls_fields: From 0b8869bc587d4bde48f9bf36587e3e3215a48fe5 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sat, 26 Oct 2024 09:17:20 -0400 Subject: [PATCH 20/59] add a few helper methods --- .../dynamicreporting/core/serverless/base.py | 139 ++++++++++-------- .../dynamicreporting/core/serverless/item.py | 63 +++++--- .../core/serverless/template.py | 129 +++++++++------- 3 files changed, 195 insertions(+), 136 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 1182486b..d906bd5c 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -218,14 +218,22 @@ def _get_field_names(cls, with_types=False, include_private=False): fields_.append((f.name, f.type) if with_types else f.name) return tuple(fields_) + def _get_var_field_names(self, include_private=False): + fields_ = [] + for f in vars(self).keys(): + if not include_private and f.startswith("_"): + continue + fields_.append(f) + return tuple(fields_) + @classmethod - def _get_all_field_names(cls): + def _get_prop_field_names(cls): """Returns a list of all field names from a dataclass, including properties.""" property_fields = [] for name, value in inspect.getmembers(cls): if isinstance(value, property): property_fields.append(name) - return tuple(property_fields) + cls._get_field_names() + return tuple(property_fields) @property def saved(self) -> bool: @@ -239,6 +247,70 @@ def _orm_saved(self) -> bool: def _orm_db(self) -> str: return self._orm_instance._state.db + def as_dict(self): + out_dict = {} + # use a combination of vars and fields + cls_fields = set(self._get_field_names() + self._get_var_field_names()) + for field_ in cls_fields: + if field_.startswith("_"): + continue + value = getattr(self, field_, None) + if value is None: # skip and use defaults + continue + out_dict[field_] = value + return out_dict + + def _prepare_for_save(self, **kwargs): + self._saved = False + + target_db = kwargs.pop("using", "default") + cls_fields = self._get_field_names() + self._get_prop_field_names() + model_fields = self._get_orm_field_names(self._orm_instance) + for field_ in cls_fields: + if field_ not in model_fields: + continue + value = getattr(self, field_, None) + if value is None: # skip and use defaults + continue + if isinstance(value, list): + objs = [o._orm_instance for o in value] + getattr(self._orm_instance, field_).add(*objs) + else: + if isinstance(value, BaseModel): # relations + try: + value = value._orm_instance.__class__.objects.using(target_db).get(guid=value.guid) + except ObjectDoesNotExist as e: + raise value.__class__.DoesNotExist( + extra_detail=f"Object with guid '{value.guid}'" f" does not exist: {e}" + ) + # for all others + setattr(self._orm_instance, field_, value) + + return self + + @handle_field_errors + def save(self, **kwargs): + try: + obj = self._prepare_for_save(**kwargs) + obj._orm_instance.save(**kwargs) + except DBIntegrityError as e: + raise self.__class__.IntegrityError( + extra_detail=f"Save failed for object with guid '{self.guid}': {e}" + ) + except Exception as e: + raise e + else: + obj._saved = True + + def delete(self, **kwargs): + if not self._saved: + raise self.__class__.NotSaved( + extra_detail=f"Delete failed for object with guid '{self.guid}'." + ) + count, _ = self._orm_instance.delete(**kwargs) + self._saved = False + return count + @classmethod def from_db(cls, orm_instance, parent=None): cls_fields = dict(cls._get_field_names(with_types=True, include_private=True)) @@ -301,60 +373,6 @@ def from_db(cls, orm_instance, parent=None): obj._saved = True return obj - def as_dict(self): - out_dict = {} - cls_fields = self._get_field_names() - for field_ in cls_fields: - if field_.startswith("_"): - continue - value = getattr(self, field_, None) - if value is None: # skip and use defaults - continue - out_dict[field_] = value - return out_dict - - def _prepare_for_save(self, **kwargs): - self._saved = False - - target_db = kwargs.pop("using", "default") - cls_fields = self._get_all_field_names() - model_fields = self._get_orm_field_names(self._orm_instance) - for field_ in cls_fields: - if field_ not in model_fields: - continue - value = getattr(self, field_, None) - if value is None: # skip and use defaults - continue - if isinstance(value, list): - objs = [o._orm_instance for o in value] - getattr(self._orm_instance, field_).add(*objs) - else: - if isinstance(value, BaseModel): # relations - try: - value = value._orm_instance.__class__.objects.using(target_db).get(guid=value.guid) - except ObjectDoesNotExist as e: - raise value.__class__.DoesNotExist( - extra_detail=f"Object with guid '{value.guid}'" f" does not exist: {e}" - ) - # for all others - setattr(self._orm_instance, field_, value) - - return self - - @handle_field_errors - def save(self, **kwargs): - try: - obj = self._prepare_for_save(**kwargs) - obj._orm_instance.save(**kwargs) - except DBIntegrityError as e: - raise self.__class__.IntegrityError( - extra_detail=f"Save failed for object with guid '{self.guid}': {e}" - ) - except Exception as e: - raise e - else: - obj._saved = True - @classmethod @handle_field_errors def create(cls, **kwargs): @@ -393,15 +411,6 @@ def get_or_create(cls, **kwargs): pass raise - def delete(self, **kwargs): - if not self._saved: - raise self.__class__.NotSaved( - extra_detail=f"Delete failed for object with guid '{self.guid}'." - ) - count, _ = self._orm_instance.delete(**kwargs) - self._saved = False - return count - @classmethod @handle_field_errors def filter(cls, **kwargs): diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index d02e8fb1..4453aeed 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -59,7 +59,12 @@ def reset(self): self._start_tags = [] -class StringContent(Validator): +class ItemContent(Validator): + def process(self, value, obj): + return None + + +class StringContent(ItemContent): def process(self, value, obj): if not isinstance(value, str): raise TypeError("Expected content to be a string") @@ -74,7 +79,7 @@ def process(self, value, obj): return html_str -class TableContent(Validator): +class TableContent(ItemContent): def process(self, value, obj): if not isinstance(value, numpy.ndarray): raise TypeError("Expected content to be a numpy array") @@ -85,7 +90,7 @@ def process(self, value, obj): return value -class TreeContent(Validator): +class TreeContent(ItemContent): ALLOWED_VALUE_TYPES = (float, int, datetime, str, bool, uuid.UUID, type(None)) def _validate_tree_value(self, value): @@ -219,6 +224,7 @@ class Item(BaseModel): sequence: int = field(compare=False, kw_only=True, default=0) session: Session = field(compare=False, kw_only=True, default=None) dataset: Dataset = field(compare=False, kw_only=True, default=None) + content: ItemContent = ItemContent() type: str = "none" _orm_model: str = "data.models.Item" # Class-level registry of subclasses keyed by type @@ -229,15 +235,11 @@ def __init_subclass__(cls, **kwargs): # Automatically register the subclass based on its type attribute Item._type_registry[cls.type] = cls - @classmethod - def from_db(cls, orm_instance, **kwargs): - # Create a new instance of the correct subclass - if cls is Item: - # Get the class based on the type attribute - item_cls = cls._type_registry[orm_instance.type] - return item_cls.from_db(orm_instance, **kwargs) - - return super().from_db(orm_instance, **kwargs) + def __post_init__(self): + # todo: can be bypassed by setting type at instantiation + if self.type == "none": + raise TypeError("Cannot instantiate Item directly. Use Item.create()") + super().__post_init__() def save(self, **kwargs): if self.session is None or self.dataset is None: @@ -260,27 +262,54 @@ def delete(self, **kwargs): delete_item_media(self._orm_instance.guid) return super().delete(**kwargs) + @classmethod + def from_db(cls, orm_instance, **kwargs): + # Create a new instance of the correct subclass + if cls is Item: + # Get the class based on the type attribute + item_cls = cls._type_registry[orm_instance.type] + return item_cls.from_db(orm_instance, **kwargs) + + return super().from_db(orm_instance, **kwargs) + + @classmethod + def create(cls, **kwargs): + # Create a new instance of the correct subclass + if cls is Item: + # Get the class based on the type attribute + try: + item_cls = cls._type_registry[kwargs.pop("type")] + except KeyError: + raise ADRException("The 'type' must be passed when using the Item class") + return item_cls.create(**kwargs) + + new_kwargs = {"type": cls.type, **kwargs} + return super().create(**new_kwargs) + @classmethod def get(cls, **kwargs): new_kwargs = {"type": cls.type, **kwargs} if cls.type != "none" else kwargs return super().get(**new_kwargs) + @classmethod + def get_or_create(cls, **kwargs): + new_kwargs = {"type": cls.type, **kwargs} if cls.type != "none" else kwargs + return super().get_or_create(**new_kwargs) + @classmethod def filter(cls, **kwargs): new_kwargs = {"type": cls.type, **kwargs} if cls.type != "none" else kwargs return super().filter(**new_kwargs) @classmethod - def find(cls, **kwargs): + def find(cls, query="", **kwargs): if cls.type == "none": - return super().find(**kwargs) - query = kwargs.pop("query", "") + return super().find(query=query, **kwargs) if "i_type|cont" in query: raise ADRException( extra_detail="The 'i_type' filter is not required if using a subclass of Item" ) - new_kwargs = {**kwargs, "query": f"A|i_type|cont|{cls.type};{query}"} - return super().find(**new_kwargs) + return super().find(query=f"A|i_type|cont|{cls.type};{query}", **kwargs) def render(self, context=None, request=None) -> Optional[str]: if context is None: diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index 445fc104..72d26562 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -1,13 +1,12 @@ +import json from dataclasses import field from datetime import datetime -import json -from typing import Optional from django.template.loader import render_to_string from django.utils import timezone -from ..exceptions import ADRException from .base import BaseModel +from ..exceptions import ADRException class Template(BaseModel): @@ -32,21 +31,28 @@ def __init_subclass__(cls, **kwargs): # Automatically register the subclass based on its type attribute Template._type_registry[cls.report_type] = cls - @classmethod - def from_db(cls, orm_instance, **kwargs): - # Create a new instance of the correct subclass - if cls is Template: - # Get the class based on the type attribute - templ_cls = cls._type_registry[orm_instance.report_type] - obj = templ_cls.from_db(orm_instance, **kwargs) - else: - obj = super().from_db(orm_instance, **kwargs) - # add relevant props from property dict. - props = obj.get_property() - for prop in cls._properties: - if prop in props: - setattr(obj, prop, props[prop]) - return obj + def __post_init__(self): + if self.report_type == "": + raise TypeError("Cannot instantiate Template directly. Use Template.create()") + super().__post_init__() + + @property + def type(self): + return self.report_type + + @type.setter + def type(self, value): + if not isinstance(value, str): + raise ValueError(f"{value} must be a string") + self.report_type = value + + @property + def children_order(self): + return ",".join([str(child.guid) for child in self.children]) + + @property + def master(self): + return self.parent is None def save(self, **kwargs): if self.parent is not None and not self.parent._saved: @@ -68,23 +74,60 @@ def save(self, **kwargs): self.add_property(prop_dict) super().save(**kwargs) - @property - def type(self): - return self.report_type + @classmethod + def from_db(cls, orm_instance, **kwargs): + # Create a new instance of the correct subclass + if cls is Template: + # Get the class based on the type attribute + templ_cls = cls._type_registry[orm_instance.report_type] + obj = templ_cls.from_db(orm_instance, **kwargs) + else: + obj = super().from_db(orm_instance, **kwargs) + # add relevant props from property dict. + props = obj.get_property() + for prop in cls._properties: + if prop in props: + setattr(obj, prop, props[prop]) + return obj - @type.setter - def type(self, value): - if not isinstance(value, str): - raise ValueError(f"{value} must be a string") - self.report_type = value + @classmethod + def create(cls, **kwargs): + # Create a new instance of the correct subclass + if cls is Template: + # Get the class based on the type attribute + try: + templ_cls = cls._type_registry[kwargs.pop("report_type")] + except KeyError: + raise ADRException("The 'report_type' must be passed when using the Template class") + return templ_cls.create(**kwargs) - @property - def children_order(self): - return ",".join([str(child.guid) for child in self.children]) + new_kwargs = {"report_type": cls.report_type, **kwargs} + return super().create(**new_kwargs) - @property - def master(self): - return self.parent is None + @classmethod + def get(cls, **kwargs): + new_kwargs = {"report_type": cls.report_type, **kwargs} if cls.report_type else kwargs + return super().get(**new_kwargs) + + @classmethod + def get_or_create(cls, **kwargs): + new_kwargs = {"report_type": cls.report_type, **kwargs} if cls.report_type else kwargs + return super().get_or_create(**new_kwargs) + + @classmethod + def filter(cls, **kwargs): + new_kwargs = {"report_type": cls.report_type, **kwargs} if cls.report_type else kwargs + return super().filter(**new_kwargs) + + @classmethod + def find(cls, query="", **kwargs): + if not cls.report_type: + return super().find(query=query, **kwargs) + if "t_types|cont" in query: + raise ADRException( + extra_detail="The 't_types' filter is not required if using a subclass of Template" + ) + return super().find(query=f"A|t_types|cont|{cls.report_type};{query}", **kwargs) def reorder_children(self) -> None: guid_to_child = {str(child.guid): child for child in self.children} @@ -150,28 +193,6 @@ def add_property(self, new_props: dict): params["properties"] = curr_props | new_props self.params = json.dumps(params) - @classmethod - def get(cls, **kwargs): - new_kwargs = {"report_type": cls.report_type, **kwargs} if cls.report_type else kwargs - return super().get(**new_kwargs) - - @classmethod - def filter(cls, **kwargs): - new_kwargs = {"report_type": cls.report_type, **kwargs} if cls.report_type else kwargs - return super().filter(**new_kwargs) - - @classmethod - def find(cls, **kwargs): - if not cls.report_type: - return super().find(**kwargs) - query = kwargs.pop("query", "") - if "t_types|cont" in query: - raise ADRException( - extra_detail="The 't_types' filter is not required if using a subclass of Template" - ) - new_kwargs = {**kwargs, "query": f"A|t_types|cont|{cls.report_type};{query}"} - return super().find(**new_kwargs) - def render(self, context=None, request=None, query="") -> str: if context is None: context = {} From 82c80f470d8a052edf9e98a5af28971cb4b92658 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sun, 27 Oct 2024 09:51:59 -0400 Subject: [PATCH 21/59] fix some exceptions in copy_objects --- .../dynamicreporting/core/serverless/adr.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index e6faaa8a..9084347d 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -340,3 +340,75 @@ def _is_sqlite(self, database: str) -> bool: def _get_db_dir(self, database: str) -> str: return self._databases[database]["NAME"] + + def copy_objects( + self, + object_type: Union[Session, Dataset, Type[Item], Type[Template]], + target_database: str, + source_database: str = "default", + query: str = "", + target_media_dir: str = "", + test: bool = False, + ) -> int: + """ + This copies a selected collection of objects from one database to another. + + GUIDs are preserved and any referenced session and dataset objects are copied as + well. + """ + if not issubclass(object_type, (Item, Template, Session, Dataset)): + self._logger.error(f"{object_type} is not valid") + raise TypeError(f"{object_type} is not valid") + + if target_database not in self._databases or source_database not in self._databases: + raise ADRException( + f"'{source_database}' and '{target_database}' must be configured first" + ) + + objects = self.query(object_type, query=query) + copy_list = [] + media_dir = None + + if issubclass(object_type, Item): + for item in objects: + # check for media dir if item has a physical file + if getattr( + item, "has_file", False + ) and not media_dir: + if target_media_dir: + media_dir = target_media_dir + elif self._is_sqlite(target_database): + media_dir = self._check_dir( + Path(self._get_db_dir(target_database)).parent / "media" + ) + else: + raise ADRException( + "'target_media_dir' argument must be specified because one of the objects" + " contains media to copy.'" + ) + # save the sessions, datasets + # assign to the item to create the new relation + # and then add to the copy list + session, _ = Session.get_or_create(**item.session.as_dict(), using=target_database) + item.session = session + dataset, _ = Dataset.get_or_create(**item.dataset.as_dict(), using=target_database) + item.dataset = dataset + copy_list.append(item) + elif issubclass(object_type, Template): + ... + else: # sessions, datasets + copy_list = list(objects) + + if test: + self._logger.info(f"Copying {len(copy_list)} objects...") + return len(copy_list) + + try: + count = self.create_objects(copy_list, using=target_database) + except Exception as e: + raise ADRException(f"Some objects could not be copied: {e}") + + # copy media + print(media_dir) + + return count From 447ca14e6990c7c3db1727562ce2b80b4afbe8b6 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sun, 27 Oct 2024 20:59:52 -0400 Subject: [PATCH 22/59] fix style --- .../dynamicreporting/core/serverless/adr.py | 16 ++++------ .../dynamicreporting/core/serverless/base.py | 32 +++++++++---------- .../core/serverless/template.py | 4 +-- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 9084347d..c4ad4a01 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,18 +1,15 @@ +from collections.abc import Iterable import os +from pathlib import Path import platform import sys -import uuid -from collections.abc import Iterable -from pathlib import Path from typing import Any, Optional, Type, Union +import uuid import django from django.core import management from django.http import HttpRequest -from .base import ObjectSet -from .item import Dataset, Item, Session -from .template import Template from .. import DEFAULT_ANSYS_VERSION from ..adr_utils import get_logger from ..exceptions import ( @@ -23,6 +20,9 @@ InvalidPath, StaticFilesCollectionError, ) +from .base import ObjectSet +from .item import Dataset, Item, Session +from .template import Template class ADR: @@ -372,9 +372,7 @@ def copy_objects( if issubclass(object_type, Item): for item in objects: # check for media dir if item has a physical file - if getattr( - item, "has_file", False - ) and not media_dir: + if getattr(item, "has_file", False) and not media_dir: if target_media_dir: media_dir = target_media_dir elif self._is_sqlite(target_database): diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index d906bd5c..fd31b482 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -1,13 +1,13 @@ -import importlib -import inspect -import shlex -import uuid from abc import ABC, ABCMeta, abstractmethod from collections.abc import Iterable from dataclasses import dataclass, field from dataclasses import fields as dataclass_fields +import importlib +import inspect from itertools import chain +import shlex from typing import Any, get_args, get_origin +import uuid from uuid import UUID from django.core.exceptions import ( @@ -59,11 +59,11 @@ class BaseMeta(ABCMeta): _model_cls_registry: dict[str, type[Model]] = {} def __new__( - mcs, - cls_name: str, - bases: tuple[type[Any], ...], - namespace: dict[str, Any], - **kwargs: Any, + mcs, + cls_name: str, + bases: tuple[type[Any], ...], + namespace: dict[str, Any], + **kwargs: Any, ) -> type: super_new = super().__new__ # ensure initialization is only performed for subclasses of BaseModel @@ -278,7 +278,9 @@ def _prepare_for_save(self, **kwargs): else: if isinstance(value, BaseModel): # relations try: - value = value._orm_instance.__class__.objects.using(target_db).get(guid=value.guid) + value = value._orm_instance.__class__.objects.using(target_db).get( + guid=value.guid + ) except ObjectDoesNotExist as e: raise value.__class__.DoesNotExist( extra_detail=f"Object with guid '{value.guid}'" f" does not exist: {e}" @@ -385,9 +387,9 @@ def create(cls, **kwargs): @handle_field_errors def get(cls, **kwargs): try: - orm_instance = cls._orm_model_cls.objects.using( - kwargs.pop("using", "default") - ).get(**kwargs) + orm_instance = cls._orm_model_cls.objects.using(kwargs.pop("using", "default")).get( + **kwargs + ) except ObjectDoesNotExist: raise cls.DoesNotExist except MultipleObjectsReturned: @@ -414,9 +416,7 @@ def get_or_create(cls, **kwargs): @classmethod @handle_field_errors def filter(cls, **kwargs): - qs = cls._orm_model_cls.objects.using( - kwargs.pop("using", "default") - ).filter(**kwargs) + qs = cls._orm_model_cls.objects.using(kwargs.pop("using", "default")).filter(**kwargs) return ObjectSet(_model=cls, _orm_model=cls._orm_model_cls, _orm_queryset=qs) @classmethod diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index 72d26562..359dd934 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -1,12 +1,12 @@ -import json from dataclasses import field from datetime import datetime +import json from django.template.loader import render_to_string from django.utils import timezone -from .base import BaseModel from ..exceptions import ADRException +from .base import BaseModel class Template(BaseModel): From 2367c740900f95a346a45a982786614a20473293 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 31 Oct 2024 20:07:26 -0400 Subject: [PATCH 23/59] add a reinit method for integrity Update adr.py --- src/ansys/dynamicreporting/core/serverless/adr.py | 8 ++++---- src/ansys/dynamicreporting/core/serverless/base.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index c4ad4a01..8f6a95c1 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -384,12 +384,12 @@ def copy_objects( "'target_media_dir' argument must be specified because one of the objects" " contains media to copy.'" ) - # save the sessions, datasets - # assign to the item to create the new relation - # and then add to the copy list + # save or load sessions, datasets session, _ = Session.get_or_create(**item.session.as_dict(), using=target_database) - item.session = session dataset, _ = Dataset.get_or_create(**item.dataset.as_dict(), using=target_database) + # required if copying across databases + item.reinit() + item.session = session item.dataset = dataset copy_list.append(item) elif issubclass(object_type, Template): diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index fd31b482..bdbada59 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -290,6 +290,9 @@ def _prepare_for_save(self, **kwargs): return self + def reinit(self): + self._orm_instance = self.__class__._orm_model_cls() + @handle_field_errors def save(self, **kwargs): try: From f1d275e11958522ded7e42affbb2fa9d977817a3 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 1 Nov 2024 17:24:55 -0400 Subject: [PATCH 24/59] fix save in file payloads --- .../dynamicreporting/core/serverless/item.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 4453aeed..130216a1 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -208,14 +208,23 @@ def from_db(cls, orm_instance, **kwargs): def save(self, **kwargs): file_name = Path(self._file.name).name - self._orm_instance.payloadfile = f"{str(self.guid)}_{file_name}" - # more general path, save the file into the media directory - with open(self._orm_instance.get_payload_server_pathname(), "wb") as out_file: - with self._file.open(mode="rb") as f: - for chunk in f.chunks(): - out_file.write(chunk) # chunk -> bytes + self._orm_instance.payloadfile = file_name + # if it does not exist in the target location, create one + if not Path(self._orm_instance.get_payload_server_pathname()).is_file(): + self._orm_instance.payloadfile = f"{str(self.guid)}_{file_name}" + # more general path, save the file into the media directory + with open(self._orm_instance.get_payload_server_pathname(), "wb") as out_file: + with self._file.open(mode="rb") as f: + for chunk in f.chunks(): + out_file.write(chunk) # chunk -> bytes super().save(**kwargs) + def delete(self, **kwargs): + from data.utils import delete_item_media + + delete_item_media(self._orm_instance.guid) + return super().delete(**kwargs) + class Item(BaseModel): name: str = field(compare=False, kw_only=True, default="") @@ -256,12 +265,6 @@ def save(self, **kwargs): raise ADRException(extra_detail=f"The item {self.guid} must have some content to save") super().save(**kwargs) - def delete(self, **kwargs): - from data.utils import delete_item_media - - delete_item_media(self._orm_instance.guid) - return super().delete(**kwargs) - @classmethod def from_db(cls, orm_instance, **kwargs): # Create a new instance of the correct subclass From 484c0c11bdc785bf69506a4d990e6ac13f6373ba Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 1 Nov 2024 17:26:14 -0400 Subject: [PATCH 25/59] fix ObjectSet.delete --- src/ansys/dynamicreporting/core/serverless/base.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index bdbada59..63e21795 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -247,6 +247,10 @@ def _orm_saved(self) -> bool: def _orm_db(self) -> str: return self._orm_instance._state.db + @property + def db(self): + return self._orm_db + def as_dict(self): out_dict = {} # use a combination of vars and fields @@ -494,9 +498,13 @@ def saved(self): return self._saved def delete(self): + count = 0 + for obj in self._obj_set: + obj.delete() + count += 1 + self._orm_queryset.delete() self._obj_set = [] self._saved = False - count, _ = self._orm_queryset.delete() return count def values_list(self, *fields, flat=False): From a8b4dc6ebfc62bd72e99526afd3457fe224d1389 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 1 Nov 2024 17:26:39 -0400 Subject: [PATCH 26/59] fix copying of items --- .../dynamicreporting/core/serverless/adr.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 8f6a95c1..e8c2993e 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,15 +1,19 @@ -from collections.abc import Iterable import os -from pathlib import Path import platform +import shutil import sys -from typing import Any, Optional, Type, Union import uuid +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Optional, Type, Union import django from django.core import management from django.http import HttpRequest +from .base import ObjectSet +from .item import Dataset, Item, Session +from .template import Template from .. import DEFAULT_ANSYS_VERSION from ..adr_utils import get_logger from ..exceptions import ( @@ -20,9 +24,6 @@ InvalidPath, StaticFilesCollectionError, ) -from .base import ObjectSet -from .item import Dataset, Item, Session -from .template import Template class ADR: @@ -331,6 +332,9 @@ def create_objects( raise ADRException("objects must be an iterable") count = 0 for obj in objects: + if kwargs.get("using", "default") != obj.db: + # required if copying across databases + obj.reinit() obj.save(**kwargs) count += 1 return count @@ -372,7 +376,7 @@ def copy_objects( if issubclass(object_type, Item): for item in objects: # check for media dir if item has a physical file - if getattr(item, "has_file", False) and not media_dir: + if getattr(item, "has_file", False) and media_dir is None: if target_media_dir: media_dir = target_media_dir elif self._is_sqlite(target_database): @@ -384,11 +388,10 @@ def copy_objects( "'target_media_dir' argument must be specified because one of the objects" " contains media to copy.'" ) - # save or load sessions, datasets + # save or load sessions, datasets - since it is possible they are shared + # and were saved already. session, _ = Session.get_or_create(**item.session.as_dict(), using=target_database) dataset, _ = Dataset.get_or_create(**item.dataset.as_dict(), using=target_database) - # required if copying across databases - item.reinit() item.session = session item.dataset = dataset copy_list.append(item) @@ -407,6 +410,9 @@ def copy_objects( raise ADRException(f"Some objects could not be copied: {e}") # copy media - print(media_dir) + if issubclass(object_type, Item) and media_dir is not None: + for item in objects: + if getattr(item, "has_file", False): + shutil.copy(Path(item.content), media_dir) return count From b5a28dde333ae88b2d15cdb3e57f7436dfea494d Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 6 Nov 2024 17:20:15 -0500 Subject: [PATCH 27/59] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a32f298b..c8e1ffd3 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ doc/_build # TIFF files *.tiff _test_enhanced_images.py +test_adr.py From e49ae63e2c0ca34a251d4a746cada0708a29e6cb Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 12 Nov 2024 11:07:55 -0500 Subject: [PATCH 28/59] fix imports --- src/ansys/dynamicreporting/core/serverless/adr.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index edf032b7..6f26506f 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,20 +1,17 @@ +from collections.abc import Iterable import os +from pathlib import Path import platform import shutil import sys -import uuid -from collections.abc import Iterable -from pathlib import Path from typing import Any, Optional, Type, Union +import uuid import django from django.core import management from django.core.management.utils import get_random_secret_key from django.http import HttpRequest -from .base import ObjectSet -from .item import Dataset, Item, Session -from .template import Template from .. import DEFAULT_ANSYS_VERSION from ..adr_utils import get_logger from ..exceptions import ( From 03d0f50b4921e29031519fccd45ed9c7bb2216e0 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 12 Nov 2024 11:10:21 -0500 Subject: [PATCH 29/59] revert copy_objects --- .../dynamicreporting/core/serverless/adr.py | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 6f26506f..de2af26c 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -375,75 +375,3 @@ def _is_sqlite(self, database: str) -> bool: def _get_db_dir(self, database: str) -> str: return self._databases[database]["NAME"] - - def copy_objects( - self, - object_type: Union[Session, Dataset, Type[Item], Type[Template]], - target_database: str, - source_database: str = "default", - query: str = "", - target_media_dir: str = "", - test: bool = False, - ) -> int: - """ - This copies a selected collection of objects from one database to another. - - GUIDs are preserved and any referenced session and dataset objects are copied as - well. - """ - if not issubclass(object_type, (Item, Template, Session, Dataset)): - self._logger.error(f"{object_type} is not valid") - raise TypeError(f"{object_type} is not valid") - - if target_database not in self._databases or source_database not in self._databases: - raise ADRException( - f"'{source_database}' and '{target_database}' must be configured first" - ) - - objects = self.query(object_type, query=query) - copy_list = [] - media_dir = None - - if issubclass(object_type, Item): - for item in objects: - # check for media dir if item has a physical file - if getattr(item, "has_file", False) and media_dir is None: - if target_media_dir: - media_dir = target_media_dir - elif self._is_sqlite(target_database): - media_dir = self._check_dir( - Path(self._get_db_dir(target_database)).parent / "media" - ) - else: - raise ADRException( - "'target_media_dir' argument must be specified because one of the objects" - " contains media to copy.'" - ) - # save or load sessions, datasets - since it is possible they are shared - # and were saved already. - session, _ = Session.get_or_create(**item.session.as_dict(), using=target_database) - dataset, _ = Dataset.get_or_create(**item.dataset.as_dict(), using=target_database) - item.session = session - item.dataset = dataset - copy_list.append(item) - elif issubclass(object_type, Template): - ... - else: # sessions, datasets - copy_list = list(objects) - - if test: - self._logger.info(f"Copying {len(copy_list)} objects...") - return len(copy_list) - - try: - count = self.create_objects(copy_list, using=target_database) - except Exception as e: - raise ADRException(f"Some objects could not be copied: {e}") - - # copy media - if issubclass(object_type, Item) and media_dir is not None: - for item in objects: - if getattr(item, "has_file", False): - shutil.copy(Path(item.content), media_dir) - - return count From 860f336146e9a619fa44b868a7f07fe9701f187e Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 12 Nov 2024 11:16:07 -0500 Subject: [PATCH 30/59] fix _get_db_dir --- src/ansys/dynamicreporting/core/serverless/adr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index de2af26c..39850674 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -374,4 +374,6 @@ def _is_sqlite(self, database: str) -> bool: return "sqlite" in self._databases[database]["ENGINE"] def _get_db_dir(self, database: str) -> str: - return self._databases[database]["NAME"] + if self._is_sqlite(database): + return self._databases[database]["NAME"] + return "" From 1d0ae290b1f3d590a647e9b8e2433945f103a654 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 12 Nov 2024 15:27:00 -0500 Subject: [PATCH 31/59] add backward compat support for report_type --- src/ansys/dynamicreporting/core/serverless/template.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index 359dd934..14e65707 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -78,8 +78,14 @@ def save(self, **kwargs): def from_db(cls, orm_instance, **kwargs): # Create a new instance of the correct subclass if cls is Template: + # the typename should be: Class:Classname where Class can be 'Layout' or 'Generator' + # originally, there were no Class values, so for backward compatibility, we prefix + # with 'Layout'... + type_name = orm_instance.report_type + if ':' not in type_name: + type_name = 'Layout:' + type_name # Get the class based on the type attribute - templ_cls = cls._type_registry[orm_instance.report_type] + templ_cls = cls._type_registry[type_name] obj = templ_cls.from_db(orm_instance, **kwargs) else: obj = super().from_db(orm_instance, **kwargs) From ffeb3f1cd317f04575633870c293e0bafc0f9e7c Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 12 Nov 2024 15:35:07 -0500 Subject: [PATCH 32/59] fix style --- src/ansys/dynamicreporting/core/serverless/template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index 14e65707..f02cb965 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -82,8 +82,8 @@ def from_db(cls, orm_instance, **kwargs): # originally, there were no Class values, so for backward compatibility, we prefix # with 'Layout'... type_name = orm_instance.report_type - if ':' not in type_name: - type_name = 'Layout:' + type_name + if ":" not in type_name: + type_name = "Layout:" + type_name # Get the class based on the type attribute templ_cls = cls._type_registry[type_name] obj = templ_cls.from_db(orm_instance, **kwargs) From 84e8ba2a4c6ac59f137fb6615d4835df9f899fe0 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 13 Nov 2024 16:17:49 -0500 Subject: [PATCH 33/59] use str for guid type instead of UUID --- src/ansys/dynamicreporting/core/serverless/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 63e21795..24e3622e 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -54,6 +54,9 @@ def is_generic_class(cls): return not isinstance(cls, type) or get_origin(cls) is not None +def get_uuid(): + return str(uuid.uuid1()) + class BaseMeta(ABCMeta): _cls_registry: dict[str, type["BaseModel"]] = {} _model_cls_registry: dict[str, type[Model]] = {} @@ -130,7 +133,7 @@ def __getattribute__(cls, name): class BaseModel(metaclass=BaseMeta): - guid: UUID = field(compare=False, kw_only=True, default_factory=uuid.uuid1) + guid: str = field(compare=False, kw_only=True, default_factory=get_uuid) tags: str = field(compare=False, kw_only=True, default="") _saved: bool = field( init=False, compare=False, default=False From 717a73300f3c14666229c7d18820c21763b0098b Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 13 Nov 2024 16:20:44 -0500 Subject: [PATCH 34/59] Update base.py --- src/ansys/dynamicreporting/core/serverless/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 24e3622e..c8575ebc 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -57,6 +57,7 @@ def is_generic_class(cls): def get_uuid(): return str(uuid.uuid1()) + class BaseMeta(ABCMeta): _cls_registry: dict[str, type["BaseModel"]] = {} _model_cls_registry: dict[str, type[Model]] = {} From 206f078fd7a6a2ef1b32a158b06df22ceba6e994 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 14 Nov 2024 18:28:34 -0500 Subject: [PATCH 35/59] add casting for some values --- src/ansys/dynamicreporting/core/serverless/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index c8575ebc..46c1afb5 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -378,6 +378,9 @@ def from_db(cls, orm_instance, parent=None): value = type_(obj_set) else: value = type_() + else: + if value is not None and not isinstance(value, field_type): + value = field_type(value) # set the orm value on the proxy object setattr(obj, attr, value) From 5c5180c39956b981aa2a01e35512546629480543 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Mon, 18 Nov 2024 10:24:08 -0500 Subject: [PATCH 36/59] create users/groups only for the default database --- src/ansys/dynamicreporting/core/serverless/adr.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 39850674..b8d59ca4 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -151,14 +151,18 @@ def _migrate_db(self, db): self._logger.error(f"{e}") raise DatabaseMigrationError(extra_detail=str(e)) else: + # create users/groups only for the default database + if db != "default": + return + from django.contrib.auth.models import Group, Permission, User - if not User.objects.using(db).filter(is_superuser=True).exists(): - user = User.objects.using(db).create_superuser("nexus", "", "cei") + if not User.objects.filter(is_superuser=True).exists(): + user = User.objects.create_superuser("nexus", "", "cei") # include the nexus group (with all permissions) - nexus_group, created = Group.objects.using(db).get_or_create(name="nexus") + nexus_group, created = Group.objects.get_or_create(name="nexus") if created: - nexus_group.permissions.set(Permission.objects.using(db).all()) + nexus_group.permissions.set(Permission.objects.all()) nexus_group.user_set.add(user) def setup(self, collect_static: bool = False) -> None: From 895454a5691b287ee179aa997fcec6f9e6d3877d Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 19 Nov 2024 17:18:43 -0500 Subject: [PATCH 37/59] add missing generators --- src/ansys/dynamicreporting/core/serverless/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/__init__.py b/src/ansys/dynamicreporting/core/serverless/__init__.py index a4f7b98c..b62af9df 100644 --- a/src/ansys/dynamicreporting/core/serverless/__init__.py +++ b/src/ansys/dynamicreporting/core/serverless/__init__.py @@ -8,6 +8,8 @@ DataFilterLayout, FooterLayout, HeaderLayout, + ItemsComparisonGenerator, + IteratorGenerator, IteratorLayout, PanelLayout, PPTXLayout, @@ -15,6 +17,7 @@ ReportLinkLayout, SliderLayout, SQLQueryGenerator, + StatisticalGenerator, TabLayout, TableMergeGenerator, TableMergeRCFilterGenerator, From b2e51c9422dfd7f58f0cc4709deb089811774b9b Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 21 Nov 2024 17:42:20 -0500 Subject: [PATCH 38/59] make unique_id optional in rebuild_3d_geometry --- src/ansys/dynamicreporting/core/utils/geofile_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/utils/geofile_processing.py b/src/ansys/dynamicreporting/core/utils/geofile_processing.py index ef00c14e..042562ae 100644 --- a/src/ansys/dynamicreporting/core/utils/geofile_processing.py +++ b/src/ansys/dynamicreporting/core/utils/geofile_processing.py @@ -106,7 +106,7 @@ def file_is_3d_geometry(filename: str, file_item_only: bool = True) -> bool: return extension in (".csf", ".stl", ".ply", ".avz", ".evsn", ".ens", ".scdoc") -def rebuild_3d_geometry(csf_file: str, unique_id: str, exec_basis: str = None): +def rebuild_3d_geometry(csf_file: str, unique_id: str = "", exec_basis: str = None): """Rebuild the media directory representation of the file (udrw format, avz, scdoc or evsn)""" # We are looking to convert the .csf or other udrw file to .avz with this command: From c75a6fc22cafc2649d5e0601a3a36253aac06065 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 21 Nov 2024 17:43:13 -0500 Subject: [PATCH 39/59] improve handling of images + support for deep imgs --- .../dynamicreporting/core/serverless/item.py | 92 +++++++++++++++---- 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 130216a1..786eaefc 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -18,6 +18,7 @@ from ..exceptions import ADRException from ..utils import report_utils from ..utils.geofile_processing import file_is_3d_geometry, rebuild_3d_geometry +from ..utils.report_utils import is_enhanced from .base import BaseModel, Validator @@ -133,37 +134,40 @@ def process(self, value, obj): file_path = Path(file_str) if not file_path.is_file(): raise ValueError( - f"Expected content to be a file path. " + f"Expected content to be a file path: " f"'{file_path.name}' does not exist or is not a file." ) with file_path.open(mode="rb") as f: file = DjangoFile(f) # check file type - file_ext = Path(file.name).suffix.lower() - if self.ALLOWED_EXT is not None and file_ext.replace(".", "") not in self.ALLOWED_EXT: + file_ext = Path(file.name).suffix.lower().lstrip(".") + if self.ALLOWED_EXT is not None and file_ext not in self.ALLOWED_EXT: raise ValueError(f"File type {file_ext} is not supported by {obj.__class__}") # check for empty files if file.size == 0: raise ValueError("The file specified is empty") # save a ref on the object. setattr(obj, "_file", file) + setattr(obj, "_file_ext", file_ext) return file_str class ImageContent(FileValidator): - ALLOWED_EXT = ("png", "jpg", "tif", "tiff") + ENHANCED_EXT = ("tif", "tiff") + ALLOWED_EXT = ("png", "jpg") + ENHANCED_EXT def process(self, value, obj): file_str = super().process(value, obj) with obj._file.open(mode="rb") as f: img_bytes = f.read() image = PILImage.open(io.BytesIO(img_bytes)) - file_ext = Path(obj._file.name).suffix.lower() - if file_ext in (".tif", ".tiff"): + if obj._file_ext in self.ENHANCED_EXT: metadata = report_utils.is_enhanced(image) if not metadata: raise ADRException("The enhanced image is empty") + obj._enhanced = True obj._width, obj._height = image.size + image.close() return file_str @@ -195,28 +199,46 @@ def save(self, **kwargs): class FilePayloadMixin: _file: DjangoFile = field(init=False, compare=False, default=None) + _file_ext: str = field(init=False, compare=False, default="") + + @property + def file_ext(self): + return self._file_ext @property def has_file(self): return self._file is not None + def get_file_path(self): + return self._orm_instance.payloadfile.path + @classmethod def from_db(cls, orm_instance, **kwargs): obj = super().from_db(orm_instance, **kwargs) obj.content = obj._orm_instance.payloadfile.path return obj - def save(self, **kwargs): - file_name = Path(self._file.name).name - self._orm_instance.payloadfile = file_name - # if it does not exist in the target location, create one - if not Path(self._orm_instance.get_payload_server_pathname()).is_file(): - self._orm_instance.payloadfile = f"{str(self.guid)}_{file_name}" - # more general path, save the file into the media directory - with open(self._orm_instance.get_payload_server_pathname(), "wb") as out_file: - with self._file.open(mode="rb") as f: + @staticmethod + def _save_file(target_path, content): + if Path(target_path).is_file(): + return + with open(target_path, "wb") as out_file: + if isinstance(content, bytes): + out_file.write(content) + else: + with content.open(mode="rb") as f: for chunk in f.chunks(): - out_file.write(chunk) # chunk -> bytes + out_file.write(chunk) + + def save(self, **kwargs): + # todo: check backward compatibility: _movie is now _anim. + self._orm_instance.payloadfile = f"{self.guid}_{self.type}.{self._file_ext}" + # Save file to the target path + target_path = self.get_file_path() + self._save_file(target_path, self._file) + # Update content and save ORM instance + self.content = target_path + super().save(**kwargs) def delete(self, **kwargs): @@ -375,6 +397,7 @@ class Tree(SimplePayloadMixin, Item): class Image(FilePayloadMixin, Item): _width: int = field(compare=False, init=False, default=0) _height: int = field(compare=False, init=False, default=0) + _enhanced: bool = field(compare=False, init=False, default=False) content: ImageContent = ImageContent() type: str = "image" @@ -386,6 +409,35 @@ def width(self): def height(self): return self._height + @property + def enhanced(self): + return self._enhanced + + def save(self, **kwargs): + with self._file.open(mode="rb") as f: + img_bytes = f.read() + image = PILImage.open(io.BytesIO(img_bytes)) + + # Determine final file name and format + target_ext = "png" if not self._enhanced else self._file_ext + self._orm_instance.payloadfile = f"{self.guid}_image.{target_ext}" + + # Save the image + target_path = self.get_file_path() + if target_ext == "png" and self._file_ext != target_ext: + try: + image.save(target_path, format="PNG") + except OSError as e: + print(f"Error converting image to PNG: {e}") + else: # save image as is (if enhanced or already PNG) + self._save_file(target_path, img_bytes) + + # Close image and update content + image.close() + self.content = target_path + + super().save(**kwargs) + class Animation(FilePayloadMixin, Item): content: AnimContent = AnimContent() @@ -399,8 +451,8 @@ class Scene(FilePayloadMixin, Item): def save(self, **kwargs): super().save(**kwargs) rebuild_3d_geometry( - self._orm_instance.get_payload_server_pathname(), - self._orm_instance.get_unique_id(), + self.get_file_path(), + unique_id="", exec_basis="", ) @@ -414,7 +466,7 @@ def save(self, **kwargs): file_name = Path(self._file.name).name if file_is_3d_geometry(file_name): rebuild_3d_geometry( - self._orm_instance.get_payload_server_pathname(), - self._orm_instance.get_unique_id(), + self.get_file_path(), + unique_id="", exec_basis="", ) From c4dbabf75e27aeb22dbd0a3c7f916b0583357d0c Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 21 Nov 2024 17:51:51 -0500 Subject: [PATCH 40/59] improve ADR.setup() --- src/ansys/dynamicreporting/core/serverless/adr.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index b8d59ca4..389426a2 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -89,9 +89,12 @@ def __init__( if media_directory is not None: self._media_directory = self._check_dir(media_directory) - os.environ["CEI_NEXUS_LOCAL_MEDIA_DIR"] = media_directory + os.environ["CEI_NEXUS_LOCAL_MEDIA_DIR"] = str(self._media_directory.parent) + # the env var here is actually the parent directory that contains the media directory elif "CEI_NEXUS_LOCAL_MEDIA_DIR" in os.environ: - self._media_directory = self._check_dir(os.environ["CEI_NEXUS_LOCAL_MEDIA_DIR"]) + self._media_directory = ( + self._check_dir(os.environ["CEI_NEXUS_LOCAL_MEDIA_DIR"]) / "media" + ) elif self._db_directory is not None: # fallback to the db dir self._media_directory = self._check_dir(self._db_directory / "media") else: @@ -188,6 +191,12 @@ def setup(self, collect_static: bool = False) -> None: if self._debug is not None: overrides["DEBUG"] = self._debug + if self._media_directory is not None: + overrides["MEDIA_ROOT"] = str(self._media_directory) + + if self._static_directory is not None: + overrides["STATIC_ROOT"] = str(self._static_directory) + if self._databases: if "default" not in self._databases: raise ImproperlyConfiguredError( From 4f26c1bf3c519be3f0b5e359dcf3ab53054eafdd Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 21 Nov 2024 17:55:53 -0500 Subject: [PATCH 41/59] remove double content set --- src/ansys/dynamicreporting/core/serverless/item.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 786eaefc..f6a7c8db 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -238,7 +238,6 @@ def save(self, **kwargs): self._save_file(target_path, self._file) # Update content and save ORM instance self.content = target_path - super().save(**kwargs) def delete(self, **kwargs): @@ -417,11 +416,9 @@ def save(self, **kwargs): with self._file.open(mode="rb") as f: img_bytes = f.read() image = PILImage.open(io.BytesIO(img_bytes)) - # Determine final file name and format target_ext = "png" if not self._enhanced else self._file_ext self._orm_instance.payloadfile = f"{self.guid}_image.{target_ext}" - # Save the image target_path = self.get_file_path() if target_ext == "png" and self._file_ext != target_ext: @@ -431,11 +428,7 @@ def save(self, **kwargs): print(f"Error converting image to PNG: {e}") else: # save image as is (if enhanced or already PNG) self._save_file(target_path, img_bytes) - - # Close image and update content image.close() - self.content = target_path - super().save(**kwargs) From f136c5625915a9ef299cf3a5a530230262093576 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 22 Nov 2024 15:35:59 -0500 Subject: [PATCH 42/59] bump ansys version & change install path error handling --- src/ansys/dynamicreporting/core/__init__.py | 4 ++-- src/ansys/dynamicreporting/core/serverless/adr.py | 9 ++++----- src/ansys/dynamicreporting/core/serverless/base.py | 1 - src/ansys/dynamicreporting/core/serverless/item.py | 1 - 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ansys/dynamicreporting/core/__init__.py b/src/ansys/dynamicreporting/core/__init__.py index ce86a8c7..210a4847 100644 --- a/src/ansys/dynamicreporting/core/__init__.py +++ b/src/ansys/dynamicreporting/core/__init__.py @@ -8,9 +8,9 @@ __version__ = importlib_metadata.version(__name__.replace(".", "-")) VERSION = __version__ -DEFAULT_ANSYS_VERSION = "251" +DEFAULT_ANSYS_VERSION = "252" -ansys_version = "2025R1" +ansys_version = "2025R2" # Ansys version number that this release is associated with __ansys_version__ = DEFAULT_ANSYS_VERSION diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 389426a2..ff29d014 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -174,13 +174,12 @@ def setup(self, collect_static: bool = False) -> None: if settings.configured: raise RuntimeError("ADR has already been configured. setup() can be called only once.") + # import hack try: - # import hack - sys.path.append( - str(self._ansys_installation / f"nexus{self._ansys_version}" / "django") - ) + adr_path = (self._ansys_installation / f"nexus{self._ansys_version}" / "django").resolve(strict=True) + sys.path.append(str(adr_path)) from ceireports import settings_serverless - except ImportError as e: + except (ImportError, OSError) as e: raise ImportError(f"Failed to import from the Ansys installation: {e}") overrides = {} diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 46c1afb5..3619efac 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -8,7 +8,6 @@ import shlex from typing import Any, get_args, get_origin import uuid -from uuid import UUID from django.core.exceptions import ( FieldDoesNotExist, diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index f6a7c8db..627d617b 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -18,7 +18,6 @@ from ..exceptions import ADRException from ..utils import report_utils from ..utils.geofile_processing import file_is_3d_geometry, rebuild_3d_geometry -from ..utils.report_utils import is_enhanced from .base import BaseModel, Validator From d809fe5996229c7492f7bdad2cb2d318865a5d5a Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 22 Nov 2024 15:42:27 -0500 Subject: [PATCH 43/59] fix style --- src/ansys/dynamicreporting/core/serverless/adr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index ff29d014..2d3a4cf3 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -176,7 +176,9 @@ def setup(self, collect_static: bool = False) -> None: # import hack try: - adr_path = (self._ansys_installation / f"nexus{self._ansys_version}" / "django").resolve(strict=True) + adr_path = ( + self._ansys_installation / f"nexus{self._ansys_version}" / "django" + ).resolve(strict=True) sys.path.append(str(adr_path)) from ceireports import settings_serverless except (ImportError, OSError) as e: From 04e766b0e8009e1622ce60b6bd7dbb1de10dcdb3 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 22 Nov 2024 16:31:09 -0500 Subject: [PATCH 44/59] fix ansys path exception Update adr.py --- src/ansys/dynamicreporting/core/serverless/adr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 2d3a4cf3..24637bad 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -138,7 +138,9 @@ def _get_install_directory(self, ansys_installation: str) -> Path: if launch_file.exists(): return install_dir - raise InvalidAnsysPath(f"Unable to detect an installation in: {','.join(dirs_to_check)}") + raise InvalidAnsysPath( + f"Unable to detect an installation in: {[str(d) for d in dirs_to_check]}" + ) def _check_dir(self, dir_): dir_path = Path(dir_) if not isinstance(dir_, Path) else dir_ From 96c90660c7e2b6def78e3f5f84d8c774a7d6d33d Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 22 Nov 2024 16:41:59 -0500 Subject: [PATCH 45/59] change nightly to run before ADR builds --- .github/workflows/nightly-docs.yml | 4 ++-- .github/workflows/nightly.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-docs.yml b/.github/workflows/nightly-docs.yml index 23a29c3d..f4cf4af7 100644 --- a/.github/workflows/nightly-docs.yml +++ b/.github/workflows/nightly-docs.yml @@ -1,8 +1,8 @@ name: Nightly Documentation Build on: - schedule: # UTC at 0400 - 12am EDT - - cron: '0 4 * * *' + schedule: # UTC at 23:00 = 7pm EDT + - cron: '0 23 * * *' workflow_dispatch: env: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e57f98c8..6a04b8bb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,8 +1,8 @@ name: Nightly Build and Test on: - schedule: # UTC at 0300 = 11pm EDT - - cron: '0 3 * * *' + schedule: # UTC at 22:00 = 6pm EDT + - cron: '0 22 * * *' workflow_dispatch: env: From f925d3eac2fc4f2d67962151b33d9590abb3fc02 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 22 Nov 2024 16:51:50 -0500 Subject: [PATCH 46/59] fix import --- src/ansys/dynamicreporting/core/serverless/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 34cfaa55..55297579 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -162,7 +162,7 @@ def process(self, value, obj): img_bytes = f.read() image = PILImage.open(io.BytesIO(img_bytes)) if obj._file_ext in self.ENHANCED_EXT: - metadata = report_utils.is_enhanced(image) + metadata = is_enhanced(image) if not metadata: raise ADRException("The enhanced image is empty") obj._enhanced = True From 90f1434a47bbc0713dbe4ad52da6ccc727da8041 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sun, 24 Nov 2024 14:20:19 -0500 Subject: [PATCH 47/59] rebuild 3d only if necessary add get_avz_directory --- .../dynamicreporting/core/serverless/item.py | 20 +++++++------------ .../core/utils/geofile_processing.py | 11 ++++++---- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/item.py b/src/ansys/dynamicreporting/core/serverless/item.py index 55297579..5a4c6d4e 100644 --- a/src/ansys/dynamicreporting/core/serverless/item.py +++ b/src/ansys/dynamicreporting/core/serverless/item.py @@ -17,7 +17,7 @@ from ..adr_utils import table_attr from ..exceptions import ADRException from ..utils import report_utils -from ..utils.geofile_processing import file_is_3d_geometry, rebuild_3d_geometry +from ..utils.geofile_processing import file_is_3d_geometry, get_avz_directory, rebuild_3d_geometry from ..utils.report_utils import is_enhanced from .base import BaseModel, Validator @@ -443,11 +443,9 @@ class Scene(FilePayloadMixin, Item): def save(self, **kwargs): super().save(**kwargs) - rebuild_3d_geometry( - self.get_file_path(), - unique_id="", - exec_basis="", - ) + file_name = self.get_file_path() + if not Path(get_avz_directory(file_name)).exists(): + rebuild_3d_geometry(file_name) class File(FilePayloadMixin, Item): @@ -456,10 +454,6 @@ class File(FilePayloadMixin, Item): def save(self, **kwargs): super().save(**kwargs) - file_name = Path(self._file.name).name - if file_is_3d_geometry(file_name): - rebuild_3d_geometry( - self.get_file_path(), - unique_id="", - exec_basis="", - ) + file_name = self.get_file_path() + if file_is_3d_geometry(file_name) and not Path(get_avz_directory(file_name)).exists(): + rebuild_3d_geometry(file_name) diff --git a/src/ansys/dynamicreporting/core/utils/geofile_processing.py b/src/ansys/dynamicreporting/core/utils/geofile_processing.py index 3a7a1fdf..ca6ee07c 100644 --- a/src/ansys/dynamicreporting/core/utils/geofile_processing.py +++ b/src/ansys/dynamicreporting/core/utils/geofile_processing.py @@ -5,11 +5,9 @@ These functions will convert files supported by the UDRW interface into AVZ files and extract proxy data from AVZ and EVSN files to simplify their display. """ -import glob import io import os import platform -import shutil import subprocess import typing import zipfile @@ -106,6 +104,11 @@ def file_is_3d_geometry(filename: str, file_item_only: bool = True) -> bool: return extension in (".csf", ".stl", ".ply", ".avz", ".evsn", ".ens", ".scdoc", ".scdocx") +def get_avz_directory(csf_file: str) -> str: + """Return the directory name for the AVZ file associated with the input CSF file.""" + return os.path.splitext(csf_file)[0] + + def rebuild_3d_geometry(csf_file: str, unique_id: str = "", exec_basis: str = None): """Rebuild the media directory representation of the file (udrw format, avz, scdoc or evsn)""" @@ -128,8 +131,8 @@ def rebuild_3d_geometry(csf_file: str, unique_id: str = "", exec_basis: str = No # make the associated directory in all cases try: os.mkdir(avz_dir) - except OSError: - print(f"Warning: unable to create 3D geometry directory: {avz_dir}") + except OSError as e: + print(f"Warning: unable to create 3D geometry directory: {e}") return avz_filename = csf_file # Easiest case, handle SCDOC and SCDOCX files From 30d3a7c67a48ed19d8c1d36ca69c91e8dcad6e35 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sun, 24 Nov 2024 14:22:53 -0500 Subject: [PATCH 48/59] set saved as false during reinit --- src/ansys/dynamicreporting/core/serverless/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 3619efac..3a6551fa 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -298,6 +298,7 @@ def _prepare_for_save(self, **kwargs): return self def reinit(self): + self._saved = False self._orm_instance = self.__class__._orm_model_cls() @handle_field_errors From e46c4f1b97c78c227b3d3bd81710e2d79e428882 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Sun, 24 Nov 2024 14:26:37 -0500 Subject: [PATCH 49/59] fix copy_objects with support for items only --- .../dynamicreporting/core/serverless/adr.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 24637bad..2ace0758 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -24,6 +24,7 @@ StaticFilesCollectionError, ) from ..utils import report_utils +from ..utils.geofile_processing import file_is_3d_geometry, rebuild_3d_geometry from .base import ObjectSet from .item import Dataset, Item, Session from .template import Template @@ -393,3 +394,79 @@ def _get_db_dir(self, database: str) -> str: if self._is_sqlite(database): return self._databases[database]["NAME"] return "" + + def copy_objects( + self, + object_type: Union[Session, Dataset, Type[Item], Type[Template]], + target_database: str, + source_database: str = "default", + query: str = "", + target_media_dir: str = "", + test: bool = False, + ) -> int: + """ + This copies a selected collection of objects from one database to another. + + GUIDs are preserved and any referenced session and dataset objects are copied as + well. + """ + if not issubclass(object_type, (Item, Template, Session, Dataset)): + self._logger.error(f"{object_type} is not valid") + raise TypeError(f"{object_type} is not valid") + + if target_database not in self._databases or source_database not in self._databases: + raise ADRException( + f"'{source_database}' and '{target_database}' must be configured first" + ) + + objects = self.query(object_type, query=query) + copy_list = [] + media_dir = None + + if issubclass(object_type, Item): + for item in objects: + # check for media dir if item has a physical file + if getattr(item, "has_file", False) and media_dir is None: + if target_media_dir: + media_dir = target_media_dir + elif self._is_sqlite(target_database): + media_dir = self._check_dir( + Path(self._get_db_dir(target_database)).parent / "media" + ) + else: + raise ADRException( + "'target_media_dir' argument must be specified because one of the objects" + " contains media to copy.'" + ) + # save or load sessions, datasets - since it is possible they are shared + # and were saved already. + session, _ = Session.get_or_create(**item.session.as_dict(), using=target_database) + dataset, _ = Dataset.get_or_create(**item.dataset.as_dict(), using=target_database) + item.session = session + item.dataset = dataset + copy_list.append(item) + elif issubclass(object_type, Template): + raise NotImplementedError("Copying templates is not supported at the moment") + else: # sessions, datasets + copy_list = list(objects) + + if test: + self._logger.info(f"Copying {len(copy_list)} objects...") + return len(copy_list) + + try: + count = self.create_objects(copy_list, using=target_database) + except Exception as e: + raise ADRException(f"Some objects could not be copied: {e}") + + # copy media + if issubclass(object_type, Item) and media_dir is not None: + for item in objects: + if not getattr(item, "has_file", False): + continue + shutil.copy(Path(item.content), media_dir) + file_name = str((media_dir / Path(item.content).name).resolve(strict=True)) + if file_is_3d_geometry(file_name, file_item_only=(item.type == "file")): + rebuild_3d_geometry(file_name) + + return count From 33a295afd7e188c50d6e94c6dd190d981d887ed9 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Mon, 2 Dec 2024 11:58:49 -0500 Subject: [PATCH 50/59] fix some Template attributes fix Template.master Update template.py fix report filtering Update template.py --- src/ansys/dynamicreporting/core/serverless/adr.py | 6 +++--- src/ansys/dynamicreporting/core/serverless/template.py | 10 ++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 2ace0758..8f3a0250 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -322,7 +322,7 @@ def create_template(self, template_type: Type[Template], **kwargs: Any) -> Templ def get_report(self, **kwargs) -> Template: try: - return Template.get(master=True, **kwargs) + return Template.get(parent=None, **kwargs) except Exception as e: self._logger.error(f"{e}") raise e @@ -333,7 +333,7 @@ def get_reports( # return list of reports by default. # if fields are mentioned, return value list try: - out = Template.filter(master=True) + out = Template.filter(parent=None) if fields: out = out.values_list(*fields, flat=flat) except Exception as e: @@ -343,7 +343,7 @@ def get_reports( return out def get_list_reports(self, r_type: str = "name") -> Union[ObjectSet, list]: - supported_types = ["name", "report"] + supported_types = ("name", "report") if r_type not in supported_types: raise ADRException(f"r_type must be one of {supported_types}") if r_type == "name": diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index f02cb965..e82f5f8b 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -12,14 +12,14 @@ class Template(BaseModel): date: datetime = field(compare=False, kw_only=True, default_factory=timezone.now) name: str = field(compare=False, kw_only=True, default="") - params: str = field(compare=False, kw_only=True, default="") + params: str = field(compare=False, kw_only=True, default="{}") item_filter: str = field(compare=False, kw_only=True, default="") parent: "Template" = field(compare=False, kw_only=True, default=None) children: list["Template"] = field(compare=False, kw_only=True, default_factory=list) _children_order: str = field( compare=False, init=False, default="" ) # computed from self.children - _master: bool = field(compare=False, init=False, default=None) # computed from self.parent + _master: bool = field(compare=False, init=False, default=True) report_type: str = "" _properties: tuple = tuple() # todo: add properties of each type ref: report_objects _orm_model: str = "reports.models.Template" @@ -48,11 +48,11 @@ def type(self, value): @property def children_order(self): - return ",".join([str(child.guid) for child in self.children]) + return self._children_order @property def master(self): - return self.parent is None + return self._master def save(self, **kwargs): if self.parent is not None and not self.parent._saved: @@ -64,6 +64,8 @@ def save(self, **kwargs): raise Template.NotSaved( extra_detail="Failed to save template because its children are not saved" ) + self._children_order = ",".join([str(child.guid) for child in self.children]) + self._master = self.parent is None # set properties prop_dict = {} for prop in self._properties: From 44e0797e9013e56e8b51794bc5cd2c38b67297d0 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 3 Dec 2024 19:17:15 -0500 Subject: [PATCH 51/59] fix parent-child load from_db --- src/ansys/dynamicreporting/core/serverless/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/base.py b/src/ansys/dynamicreporting/core/serverless/base.py index 3a6551fa..353de211 100644 --- a/src/ansys/dynamicreporting/core/serverless/base.py +++ b/src/ansys/dynamicreporting/core/serverless/base.py @@ -280,7 +280,7 @@ def _prepare_for_save(self, **kwargs): if value is None: # skip and use defaults continue if isinstance(value, list): - objs = [o._orm_instance for o in value] + objs = [obj._orm_instance for obj in value] getattr(self._orm_instance, field_).add(*objs) else: if isinstance(value, BaseModel): # relations @@ -348,7 +348,10 @@ def from_db(cls, orm_instance, parent=None): type_ = cls._cls_registry[field_type] else: type_ = field_type - if issubclass(cls, type_): # same hierarchy means there is a parent-child relation + if issubclass(cls, type_) and parent is not None: + # Same hierarchy means there is a parent-child relation. + # We avoid loading the parent object again and use the one passed + # from the previous 'from_db' load to prevent infinite recursion. value = parent else: value = type_.from_db(value) From 9df05e1f08daf9cc9dc44f97f2628b36fecb54bd Mon Sep 17 00:00:00 2001 From: viseshrp Date: Tue, 3 Dec 2024 19:20:22 -0500 Subject: [PATCH 52/59] fix copy_objects support for templates --- .../dynamicreporting/core/serverless/adr.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 8f3a0250..a28f09a0 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -1,4 +1,5 @@ from collections.abc import Iterable +import copy import os from pathlib import Path import platform @@ -146,7 +147,6 @@ def _get_install_directory(self, ansys_installation: str) -> Path: def _check_dir(self, dir_): dir_path = Path(dir_) if not isinstance(dir_, Path) else dir_ if not dir_path.exists() or not dir_path.is_dir(): - self._logger.error(f"Invalid directory path: {dir_}") raise InvalidPath(extra_detail=dir_) return dir_path @@ -154,7 +154,6 @@ def _migrate_db(self, db): try: # upgrade databases management.call_command("migrate", "--no-input", "--database", db, verbosity=0) except Exception as e: - self._logger.error(f"{e}") raise DatabaseMigrationError(extra_detail=str(e)) else: # create users/groups only for the default database @@ -239,7 +238,6 @@ def setup(self, collect_static: bool = False) -> None: settings.configure(**overrides) django.setup() except Exception as e: - self._logger.error(f"{e}") raise ImproperlyConfiguredError(extra_detail=str(e)) # migrations @@ -255,7 +253,6 @@ def setup(self, collect_static: bool = False) -> None: do_geometry_update_check(self._logger) except Exception as e: - self._logger.error(f"Unable to migrate geometry definition: {e}") raise GeometryMigrationError(extra_detail=str(e)) # collect static files @@ -263,7 +260,6 @@ def setup(self, collect_static: bool = False) -> None: try: management.call_command("collectstatic", "--no-input", verbosity=0) except Exception as e: - self._logger.error(f"{e}") raise StaticFilesCollectionError(extra_detail=str(e)) # create session and dataset w/ defaults if not provided. @@ -311,7 +307,6 @@ def create_item(self, item_type: Type[Item], **kwargs: Any) -> Item: def create_template(self, template_type: Type[Template], **kwargs: Any) -> Template: if not issubclass(template_type, Template): - self._logger.error(f"{template_type} is not valid") raise TypeError(f"{template_type} is not valid") template = template_type.create(**kwargs) parent = kwargs.get("parent") @@ -324,7 +319,6 @@ def get_report(self, **kwargs) -> Template: try: return Template.get(parent=None, **kwargs) except Exception as e: - self._logger.error(f"{e}") raise e def get_reports( @@ -337,7 +331,6 @@ def get_reports( if fields: out = out.values_list(*fields, flat=flat) except Exception as e: - self._logger.error(f"{e}") raise e return out @@ -357,7 +350,6 @@ def render_report(self, context: Optional[dict] = None, query: str = "", **kwarg request=self._request, context=context, query=query ) except Exception as e: - self._logger.error(f"{e}") raise e def query( @@ -367,7 +359,6 @@ def query( **kwargs: Any, ) -> ObjectSet: if not issubclass(query_type, (Item, Template, Session, Dataset)): - self._logger.error(f"{query_type} is not valid") raise TypeError(f"{query_type} is not valid") return query_type.find(query=query, **kwargs) @@ -395,6 +386,28 @@ def _get_db_dir(self, database: str) -> str: return self._databases[database]["NAME"] return "" + def _copy_template(self, template: Template, **kwargs) -> Template: + # depth-first walk down from the root, which is 'template', + # and copy the children along the way. + out_template = copy.deepcopy(template) + if out_template.parent is not None: + parent = out_template.parent + out_template.parent = Template.get( + guid=parent.guid, using=kwargs.get("using", "default") + ) + out_template.reorder_children() # preserves legacy code from Server.copy_items + children = out_template.children + out_template.children = [] + out_template.reinit() + out_template.save(**kwargs) + new_children = [] + for child in children: + child.parent = out_template + new_child = self._copy_template(child, **kwargs) + new_children.append(new_child) + out_template.children = new_children + return out_template + def copy_objects( self, object_type: Union[Session, Dataset, Type[Item], Type[Template]], @@ -411,7 +424,6 @@ def copy_objects( well. """ if not issubclass(object_type, (Item, Template, Session, Dataset)): - self._logger.error(f"{object_type} is not valid") raise TypeError(f"{object_type} is not valid") if target_database not in self._databases or source_database not in self._databases: @@ -446,7 +458,12 @@ def copy_objects( item.dataset = dataset copy_list.append(item) elif issubclass(object_type, Template): - raise NotImplementedError("Copying templates is not supported at the moment") + for template in objects: + # only copy top-level templates + if template.parent is not None: + raise ADRException("Only top-level templates can be copied.") + new_template = self._copy_template(template, using=target_database) + copy_list.append(new_template) else: # sessions, datasets copy_list = list(objects) From 563d9935f5719eac07c9697fd517ebc3c60e7ded Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 4 Dec 2024 14:26:14 -0500 Subject: [PATCH 53/59] refactor children_order save --- src/ansys/dynamicreporting/core/serverless/adr.py | 1 + src/ansys/dynamicreporting/core/serverless/template.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index a28f09a0..3f1a95b2 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -392,6 +392,7 @@ def _copy_template(self, template: Template, **kwargs) -> Template: out_template = copy.deepcopy(template) if out_template.parent is not None: parent = out_template.parent + # parents are always copied first, so they should exist out_template.parent = Template.get( guid=parent.guid, using=kwargs.get("using", "default") ) diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index e82f5f8b..80fdab74 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -59,12 +59,14 @@ def save(self, **kwargs): raise Template.NotSaved( extra_detail="Failed to save template because its parent is not saved" ) + children_order = [] for child in self.children: if not child._saved: raise Template.NotSaved( extra_detail="Failed to save template because its children are not saved" ) - self._children_order = ",".join([str(child.guid) for child in self.children]) + children_order.append(str(child.guid)) + self._children_order = ",".join(children_order) self._master = self.parent is None # set properties prop_dict = {} From 5754d7d1003182d12c929c6e5cddedcb5e33cec4 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Wed, 4 Dec 2024 19:35:24 -0500 Subject: [PATCH 54/59] fix setting session, dataset via create_item --- src/ansys/dynamicreporting/core/serverless/adr.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 3f1a95b2..db5dce89 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -303,7 +303,11 @@ def session_guid(self) -> uuid.UUID: def create_item(self, item_type: Type[Item], **kwargs: Any) -> Item: if not issubclass(item_type, Item): raise TypeError(f"{item_type} is not valid") - return item_type.create(session=self._session, dataset=self._dataset, **kwargs) + return item_type.create( + session=kwargs.pop("session", self._session), + dataset=kwargs.pop("dataset", self._dataset), + **kwargs, + ) def create_template(self, template_type: Type[Template], **kwargs: Any) -> Template: if not issubclass(template_type, Template): @@ -362,8 +366,8 @@ def query( raise TypeError(f"{query_type} is not valid") return query_type.find(query=query, **kwargs) - @staticmethod def create_objects( + self, objects: Union[list, ObjectSet], **kwargs: Any, ) -> int: From 4340654782498ad712c31087c799ec57734b8448 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Thu, 5 Dec 2024 17:28:50 -0500 Subject: [PATCH 55/59] use item_filter for template render --- src/ansys/dynamicreporting/core/serverless/adr.py | 4 ++-- src/ansys/dynamicreporting/core/serverless/template.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index db5dce89..b9a5bf62 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -348,10 +348,10 @@ def get_list_reports(self, r_type: str = "name") -> Union[ObjectSet, list]: else: return self.get_reports() - def render_report(self, context: Optional[dict] = None, query: str = "", **kwargs: Any) -> str: + def render_report(self, context: Optional[dict] = None, item_filter: str = "", **kwargs: Any) -> str: try: return Template.get(**kwargs).render( - request=self._request, context=context, query=query + request=self._request, context=context, item_filter=item_filter ) except Exception as e: raise e diff --git a/src/ansys/dynamicreporting/core/serverless/template.py b/src/ansys/dynamicreporting/core/serverless/template.py index 80fdab74..f3880ab6 100644 --- a/src/ansys/dynamicreporting/core/serverless/template.py +++ b/src/ansys/dynamicreporting/core/serverless/template.py @@ -203,7 +203,7 @@ def add_property(self, new_props: dict): params["properties"] = curr_props | new_props self.params = json.dumps(params) - def render(self, context=None, request=None, query="") -> str: + def render(self, context=None, request=None, item_filter="") -> str: if context is None: context = {} ctx = {**context, "request": request, "ansys_version": None} @@ -213,7 +213,7 @@ def render(self, context=None, request=None, query="") -> str: template_obj = self._orm_instance engine = template_obj.get_engine() - items = Item.find(query=query) + items = Item.find(query=item_filter) # properties that can change during iteration need to go on the class as well as globals TemplateEngine.set_global_context({"page_number": 1, "root_template": template_obj}) TemplateEngine.start_toc_session() From f5cd70ce53002d4a1d321011991652a2f8ee2930 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 6 Dec 2024 10:07:51 -0500 Subject: [PATCH 56/59] specify r_type in get_list_reports as kwarg --- src/ansys/dynamicreporting/core/serverless/adr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index b9a5bf62..029cefcd 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -344,7 +344,7 @@ def get_list_reports(self, r_type: str = "name") -> Union[ObjectSet, list]: if r_type not in supported_types: raise ADRException(f"r_type must be one of {supported_types}") if r_type == "name": - return self.get_reports([r_type], flat=True) + return self.get_reports(fields=[r_type, ], flat=True) else: return self.get_reports() From 7fe4eb8c2c77197e88b457a0fd8e12d2a2d143b2 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 6 Dec 2024 10:29:35 -0500 Subject: [PATCH 57/59] handle empty db dirs --- .../dynamicreporting/core/serverless/adr.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index 029cefcd..e14300f9 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -79,6 +79,14 @@ def __init__( # And make a target file (.nexdb) for auto launching of the report viewer... with open(self._db_directory / "view_report.nexdb", "w") as f: f.write(secret_key) + else: + # check if there is a sqlite db in the directory + db_files = list(self._db_directory.glob("*.sqlite3")) + if not db_files: + raise InvalidPath( + extra_detail="No sqlite3 database found in the directory. Remove the existing directory if" + "you would like to create a new database." + ) os.environ["CEI_NEXUS_LOCAL_DB_DIR"] = db_directory elif "CEI_NEXUS_LOCAL_DB_DIR" in os.environ: @@ -344,11 +352,18 @@ def get_list_reports(self, r_type: str = "name") -> Union[ObjectSet, list]: if r_type not in supported_types: raise ADRException(f"r_type must be one of {supported_types}") if r_type == "name": - return self.get_reports(fields=[r_type, ], flat=True) + return self.get_reports( + fields=[ + r_type, + ], + flat=True, + ) else: return self.get_reports() - def render_report(self, context: Optional[dict] = None, item_filter: str = "", **kwargs: Any) -> str: + def render_report( + self, context: Optional[dict] = None, item_filter: str = "", **kwargs: Any + ) -> str: try: return Template.get(**kwargs).render( request=self._request, context=context, item_filter=item_filter From af724afd97aaccb0d7fbbde5c4bc0619f6c88fb3 Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 6 Dec 2024 10:32:27 -0500 Subject: [PATCH 58/59] improve copy_objects error message --- src/ansys/dynamicreporting/core/serverless/adr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index e14300f9..c676dcc5 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -448,7 +448,7 @@ def copy_objects( if target_database not in self._databases or source_database not in self._databases: raise ADRException( - f"'{source_database}' and '{target_database}' must be configured first" + f"'{source_database}' and '{target_database}' must be configured first using the 'databases' option." ) objects = self.query(object_type, query=query) From eb4a1a09f4fbb8b42db64e86186b5617b2537c1b Mon Sep 17 00:00:00 2001 From: viseshrp Date: Fri, 6 Dec 2024 10:37:46 -0500 Subject: [PATCH 59/59] fix error message Update adr.py --- src/ansys/dynamicreporting/core/serverless/adr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ansys/dynamicreporting/core/serverless/adr.py b/src/ansys/dynamicreporting/core/serverless/adr.py index c676dcc5..39abaebc 100644 --- a/src/ansys/dynamicreporting/core/serverless/adr.py +++ b/src/ansys/dynamicreporting/core/serverless/adr.py @@ -85,7 +85,7 @@ def __init__( if not db_files: raise InvalidPath( extra_detail="No sqlite3 database found in the directory. Remove the existing directory if" - "you would like to create a new database." + " you would like to create a new database." ) os.environ["CEI_NEXUS_LOCAL_DB_DIR"] = db_directory