diff --git a/.gitignore b/.gitignore index 03e325756..6f87aafc1 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ local/ *.njsproj.user *.sln *.sw? +.editorconfig # ignore custom templates src/presenters/templates/custom/* diff --git a/src/core/model/attribute.py b/src/core/model/attribute.py index 3c52d8ef5..3d827ee19 100644 --- a/src/core/model/attribute.py +++ b/src/core/model/attribute.py @@ -7,7 +7,8 @@ import os from xml.etree.ElementTree import iterparse from marshmallow import fields, post_load -from sqlalchemy import orm, func, or_ +from sqlalchemy import orm, func, or_, and_ +from sqlalchemy.orm import backref from managers import log_manager from managers.db_manager import db @@ -48,7 +49,7 @@ class AttributeEnum(db.Model): imported = db.Column(db.Boolean, default=False) attribute_id = db.Column(db.Integer, db.ForeignKey("attribute.id")) - attribute = db.relationship("Attribute") + attribute = db.relationship("Attribute", back_populates="attribute_enums") def __init__(self, id, index, value, description): """Initialize the attribute enum. @@ -306,7 +307,6 @@ class Attribute(db.Model): find_by_type: Finds an attribute by type. get: Retrieves attributes based on search criteria. get_all_json: Retrieves all attributes in JSON format. - get_enums: Retrieves attribute enums for a given attribute. create_attribute: Creates a new attribute. add_attribute: Adds a new attribute. update: Updates an attribute. @@ -325,6 +325,15 @@ class Attribute(db.Model): validator = db.Column(db.Enum(AttributeValidator)) validator_parameter = db.Column(db.String()) + attribute_enums = db.relationship("AttributeEnum", + primaryjoin=and_( + id == AttributeEnum.attribute_id, + or_( + type == AttributeType.RADIO, type == AttributeType.ENUM + ) + ), + back_populates="attribute", lazy="subquery") + def __init__(self, id, name, description, type, default_value, validator, validator_parameter, attribute_enums): """Initialize an Attribute object. @@ -448,21 +457,6 @@ def get_all_json(cls, search): attribute_schema = AttributePresentationSchema(many=True) return {"total_count": total_count, "items": attribute_schema.dump(attributes)} - @classmethod - def get_enums(cls, attribute): - """Retrieve attribute enums for a given attribute. - - Args: - attribute (Attribute): The attribute object. - - Returns: - list: A list of attribute enums. - """ - if attribute.type == AttributeType.RADIO or attribute.type == AttributeType.ENUM: - return AttributeEnum.get_all_for_attribute(attribute.id) - else: - return [] - @classmethod def create_attribute(cls, attribute): """Create a new attribute. diff --git a/src/core/model/news_item.py b/src/core/model/news_item.py index 21378a87e..3535ec7c4 100644 --- a/src/core/model/news_item.py +++ b/src/core/model/news_item.py @@ -45,7 +45,7 @@ class NewsItemData(db.Model): published = db.Column(db.String()) updated = db.Column(db.DateTime, default=datetime.now()) - attributes = db.relationship('NewsItemAttribute', secondary="news_item_data_news_item_attribute") + attributes = db.relationship('NewsItemAttribute', secondary="news_item_data_news_item_attribute", lazy='selectin') osint_source_id = db.Column(db.String, db.ForeignKey('osint_source.id'), nullable=True) osint_source = db.relationship('OSINTSource') @@ -172,7 +172,7 @@ class NewsItem(db.Model): relevance = db.Column(db.Integer, default=0) news_item_data_id = db.Column(db.String, db.ForeignKey('news_item_data.id')) - news_item_data = db.relationship('NewsItemData') + news_item_data = db.relationship('NewsItemData', lazy='selectin') news_item_aggregate_id = db.Column(db.Integer, db.ForeignKey('news_item_aggregate.id')) @@ -398,9 +398,9 @@ class NewsItemAggregate(db.Model): osint_source_group_id = db.Column(db.String, db.ForeignKey('osint_source_group.id')) - news_items = db.relationship("NewsItem") + news_items = db.relationship("NewsItem", lazy='joined') - news_item_attributes = db.relationship("NewsItemAttribute", secondary='news_item_aggregate_news_item_attribute') + news_item_attributes = db.relationship("NewsItemAttribute", secondary='news_item_aggregate_news_item_attribute', lazy='selectin') @classmethod def find(cls, news_item_aggregate_id): diff --git a/src/core/model/report_item.py b/src/core/model/report_item.py index 1dd47fd08..073df5bd8 100644 --- a/src/core/model/report_item.py +++ b/src/core/model/report_item.py @@ -115,11 +115,11 @@ class ReportItemAttribute(db.Model): current = db.Column(db.Boolean, default=True) attribute_group_item_id = db.Column(db.Integer, db.ForeignKey("attribute_group_item.id")) - attribute_group_item = db.relationship("AttributeGroupItem", viewonly=True) + attribute_group_item = db.relationship("AttributeGroupItem", viewonly=True, lazy="joined", order_by=AttributeGroupItem.index) attribute_group_item_title = db.Column(db.String) report_item_id = db.Column(db.Integer, db.ForeignKey("report_item.id"), nullable=True) - report_item = db.relationship("ReportItem") + report_item = db.relationship("ReportItem", back_populates="attributes") user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=True) user = db.relationship("User") @@ -261,7 +261,7 @@ class ReportItem(db.Model): secondaryjoin=ReportItemRemoteReportItem.remote_report_item_id == id, ) - attributes = db.relationship("ReportItemAttribute", back_populates="report_item", cascade="all, delete-orphan") + attributes = db.relationship("ReportItemAttribute", back_populates="report_item", cascade="all, delete-orphan", lazy="joined") report_item_cpes = db.relationship("ReportItemCpe", cascade="all, delete-orphan") @@ -697,7 +697,7 @@ def update_report_item(cls, id, data, user): if "attribute_id" in data: for attribute in report_item.attributes: - # Compare attribute IDs + # convert ID to string, we compare types: int & int or int & str if str(attribute.id) == str(data["attribute_id"]): if attribute.value != data["attribute_value"]: modified = True @@ -737,7 +737,7 @@ def update_report_item(cls, id, data, user): if "attribute_id" in data: attribute_to_delete = None for attribute in report_item.attributes: - # sometime we compare: int & int or int & str + # convert ID to string, we compare types: int & int or int & str if str(attribute.id) == str(data["attribute_id"]): attribute_to_delete = attribute break @@ -945,8 +945,8 @@ def update_cpes(self): self.report_item_cpes = [] if self.completed is True: for attribute in self.attributes: - attribute_group = AttributeGroupItem.find(attribute.attribute_group_item_id) - if attribute_group.attribute.type == AttributeType.CPE: + item = AttributeGroupItem.find(attribute.attribute_group_item_id) + if item.attribute.type == AttributeType.CPE: self.report_item_cpes.append(ReportItemCpe(attribute.value)) diff --git a/src/core/model/report_item_type.py b/src/core/model/report_item_type.py index 3e190c940..21a8c2e1d 100644 --- a/src/core/model/report_item_type.py +++ b/src/core/model/report_item_type.py @@ -28,10 +28,10 @@ class AttributeGroupItem(db.Model): max_occurrence = db.Column(db.Integer) attribute_group_id = db.Column(db.Integer, db.ForeignKey('attribute_group.id')) - attribute_group = db.relationship("AttributeGroup", viewonly=True) + attribute_group = db.relationship("AttributeGroup", back_populates="attribute_group_items", viewonly=True, lazy="joined") attribute_id = db.Column(db.Integer, db.ForeignKey('attribute.id')) - attribute = db.relationship("Attribute") + attribute = db.relationship("Attribute", lazy="joined") def __init__(self, id, title, description, index, min_occurrence, max_occurrence, attribute_id): if id is not None and id != -1: @@ -50,10 +50,6 @@ def __init__(self, id, title, description, index, min_occurrence, max_occurrence def find(cls, id): return cls.query.get(id) - @staticmethod - def sort(attribute_group_item): - return attribute_group_item.index - class NewAttributeGroupSchema(AttributeGroupBaseSchema): attribute_group_items = fields.Nested('NewAttributeGroupItemSchema', many=True) @@ -76,7 +72,7 @@ class AttributeGroup(db.Model): report_item_type = db.relationship("ReportItemType") attribute_group_items = db.relationship('AttributeGroupItem', back_populates="attribute_group", - cascade="all, delete-orphan") + cascade="all, delete-orphan", lazy="joined", order_by=AttributeGroupItem.index) def __init__(self, id, title, description, section, section_title, index, attribute_group_items): if id is not None and id != -1: @@ -91,14 +87,6 @@ def __init__(self, id, title, description, section, section_title, index, attrib self.index = index self.attribute_group_items = attribute_group_items - @orm.reconstructor - def reconstruct(self): - self.attribute_group_items.sort(key=AttributeGroupItem.sort) - - @staticmethod - def sort(attribute_group): - return attribute_group.index - def update(self, updated_attribute_group): self.title = updated_attribute_group.title self.description = updated_attribute_group.description @@ -148,7 +136,7 @@ class ReportItemType(db.Model): description = db.Column(db.String()) attribute_groups = db.relationship('AttributeGroup', back_populates="report_item_type", - cascade="all, delete-orphan") + cascade="all, delete-orphan", lazy="joined", order_by=AttributeGroup.index) def __init__(self, id, title, description, attribute_groups): self.id = None @@ -162,7 +150,6 @@ def __init__(self, id, title, description, attribute_groups): def reconstruct(self): self.subtitle = self.description self.tag = "mdi-file-table-outline" - self.attribute_groups.sort(key=AttributeGroup.sort) @classmethod def find(cls, id): @@ -200,16 +187,11 @@ def get(cls, search, user, acl_check): func.lower(ReportItemType.title).like(search_string), func.lower(ReportItemType.description).like(search_string))) - return query.order_by(db.asc(ReportItemType.title)).all(), query.count() + return query.order_by(ReportItemType.title).all(), query.count() @classmethod def get_all_json(cls, search, user, acl_check): report_item_types, count = cls.get(search, user, acl_check) - for report_item_type in report_item_types: - for attribute_group in report_item_type.attribute_groups: - for attribute_group_item in attribute_group.attribute_group_items: - attribute_group_item.attribute.attribute_enums = Attribute.get_enums( - attribute_group_item.attribute) report_item_type_schema = ReportItemTypePresentationSchema(many=True) return {'total_count': count, 'items': report_item_type_schema.dump(report_item_types)} diff --git a/src/core/model/user.py b/src/core/model/user.py index 10fdeb7d2..28da67adc 100644 --- a/src/core/model/user.py +++ b/src/core/model/user.py @@ -38,10 +38,10 @@ class User(db.Model): organizations = db.relationship("Organization", secondary="user_organization") roles = db.relationship(Role, secondary='user_role') - permissions = db.relationship(Permission, secondary='user_permission') + permissions = db.relationship(Permission, secondary='user_permission', lazy='joined') profile_id = db.Column(db.Integer, db.ForeignKey('user_profile.id')) - profile = db.relationship("UserProfile", cascade="all") + profile = db.relationship("UserProfile", cascade="all", lazy='joined') def __init__(self, id, username, name, password, organizations, roles, permissions): self.id = None @@ -273,8 +273,8 @@ class UserProfile(db.Model): spellcheck = db.Column(db.Boolean, default=True) dark_theme = db.Column(db.Boolean, default=False) language = db.Column(db.String(2)) - hotkeys = db.relationship("Hotkey", cascade="all, delete-orphan") - word_lists = db.relationship('WordList', secondary='user_profile_word_list') + hotkeys = db.relationship("Hotkey", cascade="all, delete-orphan", lazy='joined') + word_lists = db.relationship('WordList', secondary='user_profile_word_list', lazy='joined') def __init__(self, spellcheck, dark_theme, language, hotkeys, word_lists): self.id = None diff --git a/src/core/model/word_list.py b/src/core/model/word_list.py index 75cdc4dd0..5e45f43eb 100644 --- a/src/core/model/word_list.py +++ b/src/core/model/word_list.py @@ -38,7 +38,7 @@ class WordList(db.Model): description = db.Column(db.String(), nullable=False) use_for_stop_words = db.Column(db.Boolean, default=False) - categories = db.relationship("WordListCategory", cascade="all, delete-orphan") + categories = db.relationship("WordListCategory", cascade="all, delete-orphan", lazy='joined') def __init__(self, id, name, description, categories, use_for_stop_words): self.id = None @@ -148,7 +148,7 @@ class WordListCategory(db.Model): word_list_id = db.Column(db.Integer, db.ForeignKey('word_list.id')) - entries = db.relationship("WordListEntry", cascade="all, delete-orphan") + entries = db.relationship("WordListEntry", cascade="all, delete-orphan", lazy='joined') def __init__(self, name = None, description = None, link = None, entries = None): self.id = None diff --git a/src/presenters/api/presenters.py b/src/presenters/api/presenters.py index b0ed53b50..7ca007336 100644 --- a/src/presenters/api/presenters.py +++ b/src/presenters/api/presenters.py @@ -13,6 +13,7 @@ def get(self): @api_key_required def post(self): + # print("=== GENERATE FROM THE FOLLOWING JSON ===", request.json, flush=True) return presenters_manager.generate(request.json) diff --git a/src/shared/shared/schema/attribute.py b/src/shared/shared/schema/attribute.py index 63ab74c7d..f5114a04e 100644 --- a/src/shared/shared/schema/attribute.py +++ b/src/shared/shared/schema/attribute.py @@ -69,7 +69,7 @@ def make_attribute_enum(self, data, **kwargs): class AttributeEnum: """Class representing an attribute enum.""" - def __init__(self, index, value, description): + def __init__(self, id, index, value, description): """ Initialize an Attribute object. @@ -78,6 +78,7 @@ def __init__(self, index, value, description): value (str): The value of the attribute. description (str): The description of the attribute. """ + self.id = id self.index = index self.value = value self.description = description @@ -120,7 +121,7 @@ class AttributePresentationSchema(AttributeSchema, PresentationSchema): class Attribute: """Class representing an attribute.""" - def __init__(self, id, name, description, type, default_value, validator, validator_parameter): + def __init__(self, id, name, description, type, default_value, validator, validator_parameter, attribute_enums): """ Initialize an Attribute object. @@ -132,6 +133,7 @@ def __init__(self, id, name, description, type, default_value, validator, valida default_value: The default value of the attribute. validator: The validator function for the attribute. validator_parameter: The parameter for the validator function. + attribute_enums: Attribute enum values. Returns: None @@ -143,3 +145,4 @@ def __init__(self, id, name, description, type, default_value, validator, valida self.default_value = default_value self.validator = validator self.validator_parameter = validator_parameter + self.attribute_enums = attribute_enums \ No newline at end of file