Refs #203. Implemented base for hard delete - working for assets and …

…devices so far.

teraserver/python/opentera/db/Base.py

@@ -196,22 +196,35 @@ def insert(cls, db_object):
             cls.commit()
             return db_object
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         return None  # Can delete by default
 
     @classmethod
-    def delete(cls, id_todel, autocommit: bool = True):
-        delete_obj = cls.db().session.query(cls).filter(getattr(cls, cls.get_primary_key_name()) == id_todel).first()
+    def delete(cls, id_todel, autocommit: bool = True, hard_delete: bool = False):
+        delete_obj = cls.db().session.query(cls).filter(getattr(cls, cls.get_primary_key_name()) == id_todel)\
+            .execution_options(include_deleted=hard_delete).first()
 
         if delete_obj:
             cannot_be_deleted_exception = delete_obj.delete_check_integrity()
             if cannot_be_deleted_exception:
                 raise cannot_be_deleted_exception
 
-            if getattr(delete_obj, 'soft_delete', None):
+            has_soft_delete = getattr(delete_obj, 'soft_delete', None) is not None
+            has_hard_delete = getattr(delete_obj, 'hard_delete', None) is not None
+            if has_soft_delete and not hard_delete:
                 delete_obj.soft_delete()
             else:
-                cls.db().session.delete(delete_obj)
+                # if has_soft_delete:
+                #     # Check that object was soft deleted before doing a hard delete
+                #     if not delete_obj.deleted_at:
+                #         # Object must be soft deleted first before being hard deleted!
+                #         raise SQLAlchemyError(cls.__name__ + ' with id ' + str(id_todel) +
+                #                               ' cannot be hard deleted: not soft deleted beforehand!')
+                if has_hard_delete and hard_delete:
+                    delete_obj.hard_delete()
+                    return
+                else:
+                    cls.db().session.delete(delete_obj)
             if autocommit:
                 cls.commit()
         else:
@@ -231,21 +244,16 @@ def undelete(cls, id_to_undelete):
 
 
     # @classmethod
-    # def handle_include_deleted_flag(cls, include_deleted=False):
-    #     if 'include_deleted' not in cls.db().session.info:
-    #         cls.db().session.info['include_deleted'] = list()
-    #
-    #     if include_deleted:
-    #         cls.db().session.info['include_deleted'].push(cls.__name__)
-    #     else:
-    #         cls.db().session.info['include_deleted'].pop(-1)
-
-    @classmethod
-    def hard_delete(cls, id_todel):
-        delete_obj = cls.db().session.query(cls).filter(getattr(cls, cls.get_primary_key_name()) == id_todel).first()
-        if delete_obj:
-            cls.db().session.delete(delete_obj)
-            cls.commit()
+    # def hard_delete(cls, id_todel):
+    #     delete_obj = cls.db().session.query(cls).execution_options(include_deleted=True)\
+    #         .filter(getattr(cls, cls.get_primary_key_name()) == id_todel).first()
+    #     if delete_obj:
+    #         if not delete_obj.deleted_at:
+    #             # Object must be soft deleted first before being hard deleted!
+    #             raise SQLAlchemyError(cls.__name__ + ' with id ' + str(id_todel) +
+    #                                   ' cannot be hard deleted: not soft deleted beforehand!')
+    #         cls.db().session.delete(delete_obj)
+    #         cls.commit()
 
     @classmethod
     def query_with_filters(cls, filters=None, with_deleted: bool = False):

teraserver/python/opentera/db/SoftDeleteMixin.py

@@ -25,31 +25,35 @@
 def activate_soft_delete_hook(deleted_field_name: str, disable_soft_delete_option_name: str):
     """Activate an event hook to rewrite the queries."""
     # Enable Soft Delete on all Relationship Loads which implement SoftDeleteMixin
-    # @listens_for(Session, "do_orm_execute")
-    # def soft_delete_execute(state: ORMExecuteState):
-    #     if not state.is_select:
-    #         return
-    #     if 'include_deleted' in state.session.info and len(state.session.info['include_deleted']) > 0:
-    #         print('test_include_deleted')
-    #         return
-    #
-    #     adapted = SoftDeleteQueryRewriter(deleted_field_name, disable_soft_delete_option_name).rewrite_statement(
-    #         state.statement
-    #     )
-    #     state.statement = adapted
-    @listens_for(Engine, "before_execute", retval=True)
-    def soft_delete_execute(conn: Connection, clauseelement, multiparams, params, execution_options):
-        if not isinstance(clauseelement, Select):
-            return clauseelement, multiparams, params
+    @listens_for(Session, "do_orm_execute")
+    def soft_delete_execute(state: ORMExecuteState):
+        if not state.is_select:
+            return
+
+        if disable_soft_delete_option_name in state.execution_options \
+                and state.execution_options[disable_soft_delete_option_name]:
+            return
 
-        if disable_soft_delete_option_name in execution_options and execution_options[disable_soft_delete_option_name]:
-            # print('test_include_deleted')
-            return clauseelement, multiparams, params
+        if 'include_deleted' in state.session.info and len(state.session.info['include_deleted']) > 0:
+            return
 
         adapted = SoftDeleteQueryRewriter(deleted_field_name, disable_soft_delete_option_name).rewrite_statement(
-            clauseelement
+            state.statement
         )
-        return adapted, multiparams, params
+        state.statement = adapted
+    # @listens_for(Engine, "before_execute", retval=True)
+    # def soft_delete_execute(conn: Connection, clauseelement, multiparams, params, execution_options):
+    #     if not isinstance(clauseelement, Select):
+    #         return clauseelement, multiparams, params
+    #
+    #     if disable_soft_delete_option_name in execution_options and execution_options[disable_soft_delete_option_name]:
+    #         # print('test_include_deleted')
+    #         return clauseelement, multiparams, params
+    #
+    #     adapted = SoftDeleteQueryRewriter(deleted_field_name, disable_soft_delete_option_name).rewrite_statement(
+    #         clauseelement
+    #     )
+    #     return adapted, multiparams, params
 
 
 def generate_soft_delete_mixin_class(
@@ -62,7 +66,9 @@ def generate_soft_delete_mixin_class(
     delete_method_default_value: Callable[[], Any] = lambda: datetime.utcnow(),
     generate_undelete_method: bool = True,
     undelete_method_name: str = "undelete",
-    handle_cascade_delete: bool = True
+    handle_cascade_delete: bool = True,
+    generate_hard_delete_method: bool = True,
+    hard_delete_method_name: str = "hard_delete"
 ) -> Type:
     """Generate the actual soft-delete Mixin class."""
     class_attributes = {deleted_field_name: Column(deleted_field_name, deleted_field_type)}
@@ -76,7 +82,6 @@ def get_class_from_tablename(_self, tablename: str) -> DeclarativeMeta | None:
     class_attributes['get_class_from_tablename'] = get_class_from_tablename
 
     if generate_delete_method:
-
         def delete_method(_self, v: Optional[Any] = None):
             setattr(_self, deleted_field_name, v or delete_method_default_value())
             if handle_cascade_delete:
@@ -104,8 +109,43 @@ def delete_method(_self, v: Optional[Any] = None):
 
         class_attributes[delete_method_name] = delete_method
 
-    if generate_undelete_method:
+    if generate_hard_delete_method:
+        def hard_delete_method(_self):
+            _self.handle_include_deleted_flag(True)
+            # Callback actions before doing hard delete, if required
+            if getattr(_self, 'hard_delete_before', None):
+                _self.hard_delete_before()
+            if handle_cascade_delete:
+                primary_key_name = inspect(_self.__class__).primary_key[0].name
+                for relation in inspect(_self.__class__).relationships.items():
+                    # Relationship has a cascade delete or a secondary table
+                    if relation[1].cascade.delete:
+                        for item in getattr(_self, relation[0]):
+                            # print("Cascade deleting " + str(item))
+                            hard_item_deleter = getattr(item, hard_delete_method_name)
+                            hard_item_deleter()
 
+                    if relation[1].secondary is not None and relation[1].passive_deletes:
+                        if deleted_field_name in relation[1].entity.columns.keys():
+                            model_class = _self.get_class_from_tablename(relation[1].secondary.name)
+                            if model_class:
+                                related_items = model_class.query.filter(text(primary_key_name + "=" +
+                                                                              str(getattr(_self, primary_key_name)))
+                                                                         ).execution_options(include_deleted=True).all()
+                                for item in related_items:
+                                    # print("Cascade deleting " + str(model_class) + ": " + primary_key_name + " = " +
+                                    #       str(getattr(_self, primary_key_name)))
+                                    item_hard_deleter = getattr(item, hard_delete_method_name)
+                                    item_hard_deleter()
+
+            if _self not in _self.db().session.deleted:
+                _self.db().session.delete(_self)
+                _self.commit()
+            _self.handle_include_deleted_flag(False)
+
+        class_attributes[hard_delete_method_name] = hard_delete_method
+
+    if generate_undelete_method:
         def undelete_method(_self):
             if handle_cascade_delete:
                 primary_key_name = inspect(_self.__class__).primary_key[0].name
@@ -137,6 +177,17 @@ def undelete_method(_self):
 
     activate_soft_delete_hook(deleted_field_name, disable_soft_delete_filtering_option_name)
 
+    def handle_include_deleted_flag(_self, include_deleted=False):
+        if 'include_deleted' not in _self.db().session.info:
+            _self.db().session.info['include_deleted'] = list()
+
+        if include_deleted:
+            _self.db().session.info['include_deleted'].append(_self.get_model_name())
+        else:
+            _self.db().session.info['include_deleted'].pop(-1)
+
+    class_attributes['handle_include_deleted_flag'] = handle_include_deleted_flag
+
     generated_class = type(class_name, tuple(), class_attributes)
 
     return generated_class

teraserver/python/opentera/db/models/TeraDevice.py

@@ -246,21 +246,26 @@ def update(cls, update_id: int, values: dict):
 
         super().update(update_id=update_id, values=values)
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         # Safety check - can't delete participants with sessions
-        if TeraDeviceParticipant.get_count(filters={'id_device': self.id_device}) > 0:
+        if TeraDeviceParticipant.get_count(filters={'id_device': self.id_device}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device still associated to participant(s)', self.id_device, 't_devices_participants')
 
-        if TeraSessionDevices.get_count(filters={'id_device': self.id_device}) > 0:
+        if TeraSessionDevices.get_count(filters={'id_device': self.id_device}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device still has sessions', self.id_device, 't_sessions_devices')
 
-        if TeraSession.get_count(filters={'id_creator_device': self.id_device}) > 0:
+        if TeraSession.get_count(filters={'id_creator_device': self.id_device}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device still has created sessions', self.id_device, 't_sessions')
 
-        if TeraAsset.get_count(filters={'id_device': self.id_device}) > 0:
+        if TeraAsset.get_count(filters={'id_device': self.id_device}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device still has created assets', self.id_device, 't_assets')
 
-        if TeraTest.get_count(filters={'id_device': self.id_device}) > 0:
+        if TeraTest.get_count(filters={'id_device': self.id_device}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device still has created tests', self.id_device, 't_tests')
 
         return None
+
+    def hard_delete_before(self):
+        # Delete sessions that we are part of since they will not be deleted otherwise
+        for ses in self.device_sessions:
+            ses.hard_delete()

teraserver/python/opentera/db/models/TeraDeviceProject.py

@@ -81,19 +81,19 @@ def delete_with_ids(device_id: int, project_id: int, autocommit: bool = True):
         if delete_obj:
             TeraDeviceProject.delete(delete_obj.id_device_project, autocommit=autocommit)
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         from opentera.db.models.TeraDeviceParticipant import TeraDeviceParticipant
         from opentera.db.models.TeraParticipant import TeraParticipant
         from opentera.db.models.TeraSession import TeraSession
 
-        if TeraDeviceParticipant.query.join(TeraParticipant).\
+        if TeraDeviceParticipant.query.execution_options(include_deleted=with_deleted).join(TeraParticipant).\
             filter(TeraParticipant.id_project == self.id_project).\
                 filter(TeraDeviceParticipant.id_device == self.id_device).count():
             return IntegrityError('Project still has participant associated to the device',
                                   self.id_device_project, 't_participants')
 
         # Find sessions with matching device and project
-        device_sessions = TeraSession.get_sessions_for_device(self.id_device)
+        device_sessions = TeraSession.get_sessions_for_device(self.id_device, with_deleted=with_deleted)
         device_project_sessions = [ses.id_session for ses in device_sessions
                                    if ses.get_associated_project_id() == self.id_project]
 

teraserver/python/opentera/db/models/TeraDeviceSite.py

@@ -116,17 +116,18 @@ def delete(cls, id_todel, autocommit: bool = True):
             for device_project in specific_device_projects:
                 TeraDeviceProject.delete(device_project.id_device_project, autocommit=autocommit)
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         from opentera.db.models.TeraDeviceProject import TeraDeviceProject
         from opentera.db.models.TeraProject import TeraProject
 
         # Will check if device is part of a project in the site
         specific_device_projects = TeraDeviceProject.query.join(TeraProject).\
-            filter(TeraDeviceProject.id_device == self.id_device).filter(TeraProject.id_site == self.id_site).all()
+            filter(TeraDeviceProject.id_device == self.id_device).filter(TeraProject.id_site == self.id_site).\
+            execution_options(include_deleted=with_deleted).all()
 
         # Check integrity of device_projects
         for device_project in specific_device_projects:
-            integrity_check = device_project.delete_check_integrity()
+            integrity_check = device_project.delete_check_integrity(with_deleted)
             if integrity_check is not None:
                 return integrity_check
 

teraserver/python/opentera/db/models/TeraDeviceSubType.py

@@ -60,8 +60,8 @@ def get_device_subtype_by_id(dev_subtype: int):
     def get_device_subtypes_for_type(dev_type: int):
         return TeraDeviceSubType.query.filter_by(id_device_type=dev_type).all()
 
-    def delete_check_integrity(self) -> IntegrityError | None:
-        if (TeraDevice.get_count(filters={'id_device_subtype': self.id_device_subtype})) > 0:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
+        if (TeraDevice.get_count(filters={'id_device_subtype': self.id_device_subtype}, with_deleted=with_deleted)) > 0:
             return IntegrityError('Device subtype still have devices with that subtype', self.id_device_subtype,
                                   't_devices')
         return None

teraserver/python/opentera/db/models/TeraDeviceType.py

@@ -69,10 +69,10 @@ def get_device_type_by_name(dev_name: str):
     def get_device_type_by_key(dev_key: str):
         return TeraDeviceType.query.filter_by(device_type_key=dev_key).first()
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         # More efficient that relationships
         from opentera.db.models.TeraDevice import TeraDevice  # Here to prevent circular import
-        if TeraDevice.get_count(filters={'id_device_type': self.id_device_type}) > 0:
+        if TeraDevice.get_count(filters={'id_device_type': self.id_device_type}, with_deleted=with_deleted) > 0:
             return IntegrityError('Device Type still has associated devices', self.id_device_type, 't_devices')
         return None
 

teraserver/python/opentera/db/models/TeraParticipant.py

@@ -390,18 +390,20 @@ def insert(cls, participant):
             raise IntegrityError('Participant project disabled - no insert allowed', -1, 't_projects')
         TeraParticipant.db().session.commit()
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         # Safety check - can't delete participants with sessions
-        if TeraSessionParticipants.get_session_count_for_participant(self.id_participant) > 0:
+        if TeraSessionParticipants.get_session_count_for_participant(self.id_participant,
+                                                                     with_deleted=with_deleted) > 0:
             return IntegrityError('Participant still has sessions', self.id_participant, 't_sessions_participants')
 
-        if TeraSession.get_count(filters={'id_creator_participant': self.id_participant}) > 0:
+        if TeraSession.get_count(filters={'id_creator_participant': self.id_participant},
+                                 with_deleted=with_deleted) > 0:
             return IntegrityError('Participant still has created sessions', self.id_participant, 't_sessions')
 
-        if TeraAsset.get_count(filters={'id_participant': self.id_participant}) > 0:
+        if TeraAsset.get_count(filters={'id_participant': self.id_participant}, with_deleted=with_deleted) > 0:
             return IntegrityError('Participant still has created assets', self.id_participant, 't_assets')
 
-        if TeraTest.get_count(filters={'id_participant': self.id_participant}) > 0:
+        if TeraTest.get_count(filters={'id_participant': self.id_participant}, with_deleted=with_deleted) > 0:
             return IntegrityError('Participant still has created tests', self.id_participant, 't_tests')
 
         return None

teraserver/python/opentera/db/models/TeraParticipantGroup.py

@@ -71,15 +71,15 @@ def create_defaults(test=False):
     def update(cls, update_id: int, values: dict):
         # If group project changed, also changed project from all participants in that group
         if 'id_project' in values:
-            updated_group:TeraParticipantGroup = TeraParticipantGroup.get_participant_group_by_id(update_id)
+            updated_group: TeraParticipantGroup = TeraParticipantGroup.get_participant_group_by_id(update_id)
             if updated_group:
                 for participant in updated_group.participant_group_participants:
                     participant.id_project = values['id_project']
         super().update(update_id=update_id, values=values)
 
-    def delete_check_integrity(self) -> IntegrityError | None:
+    def delete_check_integrity(self, with_deleted: bool = False) -> IntegrityError | None:
         for participant in self.participant_group_participants:
-            cannot_be_deleted_exception = participant.delete_check_integrity()
+            cannot_be_deleted_exception = participant.delete_check_integrity(with_deleted=with_deleted)
             if cannot_be_deleted_exception:
                 return IntegrityError('Participant group still has participant(s)', self.id_participant_group,
                                       't_participants')