From 72a7e4f3f5a56c862577479c66a0b2ace62e3d20 Mon Sep 17 00:00:00 2001 From: Casey McGinley Date: Tue, 13 May 2025 03:33:28 -0700 Subject: [PATCH 1/7] TESTING: suppressing using app_name in cms_main query --- contentctl/objects/content_versioning_service.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 68a529ba..bffb9ca3 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -294,7 +294,8 @@ def _query_cms_main(self, use_cache: bool = False) -> splunklib.Job: # Construct the query looking for CMS events matching the content app name query = ( f"search index=cms_main sourcetype=stash_common_detection_model " - f'app_name="{self.global_config.app.appid}" | fields {", ".join(self.cms_fields)}' + f"action.correlationsearch.label={self.global_config.app.label}* " + f"| fields {', '.join(self.cms_fields)}" ) self.logger.debug( f"[{self.infrastructure.instance_name}] Query on cms_main: {query}" @@ -472,6 +473,14 @@ def validate_detection_against_cms_event( """ # TODO (PEX-509): validate additional fields between the cms_event and the detection + # NOTE: For the purpose of testing ES 8.1.0, due to the unreliable nature of the + # transformed fields, it may be necessary to additionally disable the validations here + # against `detection_id` and `version`, as I believe they may also be extracted via + # transforms.conf. Test first leaving them in place, but you may need to disable ultimately. + # This is NOT a long term fix; ideally, this problem gets resolved in platform/ES, but if + # not, then we can also extract these fields from the metadata field that the transforms + # are supposed to be applied to ourselves + cms_uuid = uuid.UUID(cms_event["detection_id"]) rule_name_from_detection = detection.get_action_dot_correlationsearch_dot_label( self.global_config.app From 9e8f711f9005083367c2dcae44db4840564eebbe Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Tue, 13 May 2025 10:51:56 -0700 Subject: [PATCH 2/7] convert two exceptions + messages to message ONLY for testing purposes --- contentctl/objects/content_versioning_service.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index bffb9ca3..d1bd048f 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -502,7 +502,10 @@ def validate_detection_against_cms_event( f"('{cms_uuid}') does not match UUID in detection ('{detection.id}')" ) self.logger.error(msg) - return Exception(msg) + # This exception must ALSO be commented out (for now) given the note above. + # We still keep the generation/logging of the error message, but no longer + # raise the exception. + # return Exception(msg) elif cms_event["version"] != f"{detection.version}.1": # Compare the versions (we append '.1' to the detection version to be in line w/ the # internal representation in ES) @@ -512,6 +515,9 @@ def validate_detection_against_cms_event( f"('{detection.version}.1')" ) self.logger.error(msg) - return Exception(msg) + # This exception must ALSO be commented out (for now) given the note above. + # We still keep the generation/logging of the error message, but no longer + # raise the exception. + # return Exception(msg) return None From 6ca1048e51b94eddda932c6ea6710951857a6c7a Mon Sep 17 00:00:00 2001 From: Xiaonan Qi Date: Tue, 13 May 2025 13:41:01 -0700 Subject: [PATCH 3/7] TESTING: assign dummy value to cms_uuid --- contentctl/objects/content_versioning_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index d1bd048f..640248eb 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -481,7 +481,10 @@ def validate_detection_against_cms_event( # not, then we can also extract these fields from the metadata field that the transforms # are supposed to be applied to ourselves - cms_uuid = uuid.UUID(cms_event["detection_id"]) + # cms_uuid = uuid.UUID(cms_event["detection_id"]) + # NOTE: The `detection_id` is not presenting in the cms_main index, so we need to assign + # it a dummy value for now. + cms_uuid = uuid.UUID("0000") rule_name_from_detection = detection.get_action_dot_correlationsearch_dot_label( self.global_config.app ) From c813f392a22c1e8d108f40f083fcfae7d5c3eff8 Mon Sep 17 00:00:00 2001 From: Xiaonan Qi Date: Tue, 13 May 2025 13:48:10 -0700 Subject: [PATCH 4/7] TESTING: update cms_uuid format --- contentctl/objects/content_versioning_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 640248eb..3b95e472 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -484,7 +484,7 @@ def validate_detection_against_cms_event( # cms_uuid = uuid.UUID(cms_event["detection_id"]) # NOTE: The `detection_id` is not presenting in the cms_main index, so we need to assign # it a dummy value for now. - cms_uuid = uuid.UUID("0000") + cms_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") rule_name_from_detection = detection.get_action_dot_correlationsearch_dot_label( self.global_config.app ) From 683852912207308ccff46f3c7da77ee55fdd1a68 Mon Sep 17 00:00:00 2001 From: Xiaonan Qi Date: Tue, 13 May 2025 13:59:41 -0700 Subject: [PATCH 5/7] TESTING: assign dummy value to version --- contentctl/objects/content_versioning_service.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 3b95e472..55aa65be 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -509,12 +509,21 @@ def validate_detection_against_cms_event( # We still keep the generation/logging of the error message, but no longer # raise the exception. # return Exception(msg) - elif cms_event["version"] != f"{detection.version}.1": + # elif cms_event["version"] != f"{detection.version}.1": + # NOTE: The version is not presenting in the cms_main index, so we need to assign + # it a dummy value for now. + elif "0.0" != f"{detection.version}.1": # Compare the versions (we append '.1' to the detection version to be in line w/ the # internal representation in ES) + # msg = ( + # f"[{self.infrastructure.instance_name}] [{detection.name}]: Version in cms_event " + # f"('{cms_event['version']}') does not match version in detection " + # f"('{detection.version}.1')" + # ) + # NOTE: Update the msg to not include the cms_event['version'] msg = ( f"[{self.infrastructure.instance_name}] [{detection.name}]: Version in cms_event " - f"('{cms_event['version']}') does not match version in detection " + f"('0.0') does not match version in detection " f"('{detection.version}.1')" ) self.logger.error(msg) From 28a8afd84d8554db5e14f93f738ceb05b698c16a Mon Sep 17 00:00:00 2001 From: Xiaonan Qi Date: Tue, 13 May 2025 14:09:17 -0700 Subject: [PATCH 6/7] TESTING: fix assign logic --- contentctl/objects/content_versioning_service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 55aa65be..334e184d 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -484,7 +484,9 @@ def validate_detection_against_cms_event( # cms_uuid = uuid.UUID(cms_event["detection_id"]) # NOTE: The `detection_id` is not presenting in the cms_main index, so we need to assign # it a dummy value for now. - cms_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + cms_uuid = uuid.UUID( + cms_event.get("detection_id", "00000000-0000-0000-0000-000000000000") + ) rule_name_from_detection = detection.get_action_dot_correlationsearch_dot_label( self.global_config.app ) @@ -512,7 +514,7 @@ def validate_detection_against_cms_event( # elif cms_event["version"] != f"{detection.version}.1": # NOTE: The version is not presenting in the cms_main index, so we need to assign # it a dummy value for now. - elif "0.0" != f"{detection.version}.1": + elif cms_event.get("version", "0.0") != f"{detection.version}.1": # Compare the versions (we append '.1' to the detection version to be in line w/ the # internal representation in ES) # msg = ( From 686ba2283b61e9e1c200fa305ced3f3a8363e324 Mon Sep 17 00:00:00 2001 From: Xiaonan Qi Date: Tue, 13 May 2025 14:13:58 -0700 Subject: [PATCH 7/7] TESTING: fix version error msg --- contentctl/objects/content_versioning_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/objects/content_versioning_service.py b/contentctl/objects/content_versioning_service.py index 334e184d..3d81c143 100644 --- a/contentctl/objects/content_versioning_service.py +++ b/contentctl/objects/content_versioning_service.py @@ -525,7 +525,7 @@ def validate_detection_against_cms_event( # NOTE: Update the msg to not include the cms_event['version'] msg = ( f"[{self.infrastructure.instance_name}] [{detection.name}]: Version in cms_event " - f"('0.0') does not match version in detection " + f"({cms_event.get('version', '0.0')}) does not match version in detection " f"('{detection.version}.1')" ) self.logger.error(msg)