test: add analysis report JSON schema validation (#824)

Signed-off-by: Trong Nhan Mai <[email protected]>

Author: Trong Nhan Mai

pyproject.toml

@@ -100,6 +100,7 @@ test = [
 ]
 
 test-docker = [
+    "jsonschema >= 4.22.0,<5.0.0",
     "cfgv >=3.4.0,<4.0.0",
     "ruamel.yaml >=0.18.6,<1.0.0",
 ]

tests/integration/README.md

@@ -186,11 +186,12 @@ In case you want to debug the utility script itself, there is the verbose mode f
 ### Step Schema
 
 * `name` (`string`, required): The name of the step.
-* `kind` (`"analyze" | "verify" | "compare" | "shell"`, required): The kind of the step. There are 4 kinds of steps:
+* `kind` (`"analyze" | "verify" | "compare" | "shell" | "validate_schema`, required): The kind of the step. There are 5 kinds of steps:
   * `"analyze"`: runs the `macaron analyze` command.
   * `"verify"`: runs the `macaron verify-policy` command.
   * `"compare"`: compares an output file with an expected output file.
   * `"shell"`: runs an arbitrary shell command.
+  * `"validate_schema"`: validates an output file with a schema.
 * `options`: Configuration options for the step. These options are specific to the step kind. See their schema below.
 * `env` (`dict[string, string | null]`, optional): Key value pairs of environment variables being modified during the step after inheriting the environment in which the utility is executed within. Each value can be a string if you want to set a value to the environment variable, or null if you want to "unset" the variable.
 * `expect_fail` (`bool`, optional, default is `false`): If `true`, assert that the step must exit with non-zero code. This should be used for cases where we expect a command to fail.
@@ -218,6 +219,13 @@ In case you want to debug the utility script itself, there is the verbose mode f
 * `result` (`string`, required): The output file (a relative path from test case directory).
 * `expected` (`string`, required): The expected output file (a relative path from test case directory).
 
+### Validate Schema step options Schema
+
+* `kind` (`"json_schema"`, required): The kind of schema validation to perform. For now, only json-schema is supported.
+* `result` (`string`, required): The output file (a relative path from test case directory).
+* `schema` (`output_json_report`, required): The name of the schema for the validation. These are the default schemas available for the integration test.
+* `custom_schema_path` (`string`, optional): The path to the custom schema (a relative path from test case directory). If it is provided, the validation will use this schema and ignore the schema corresponding with `schema`.
+
 ### Shell step options Schema
 
 * `cmd` (`string`, required): The shell command to run.

tests/integration/cases/micronaut-projects_micronaut-test/test.yaml

@@ -15,6 +15,12 @@ steps:
     - -c
     - micronaut_test_config.yaml
     - --skip-deps
+- name: Validate JSON report schema
+  kind: validate_schema
+  options:
+    kind: json_schema
+    schema: output_json_report
+    result: output/reports/github_com/micronaut-projects/micronaut-test/micronaut-test.json
 - name: Compare dependency report
   kind: compare
   options:

tests/integration/run.py

@@ -81,6 +81,14 @@ def configure_logging(verbose: bool) -> None:
     "vsa": ["tests", "vsa", "compare_vsa.py"],
 }
 
+VALIDATE_SCHEMA_SCRIPTS: dict[str, Sequence[str]] = {
+    "json_schema": ["tests", "schema_validation", "json_schema_validate.py"],
+}
+
+DEFAULT_SCHEMAS: dict[str, Sequence[str]] = {
+    "output_json_report": ["tests", "schema_validation", "report_schema.json"],
+}
+
 
 def check_required_file(cwd: str) -> Callable[[str], None]:
     """Check for a required file of a test case."""
@@ -208,6 +216,72 @@ def cmd(self, macaron_cmd: str) -> list[str]:
         return self.options["cmd"].strip().split()
 
 
+class ValidateSchemaStepOptions(TypedDict):
+    """The configuration options of a schema validation step."""
+
+    kind: str
+    result: str
+    schema: str
+    custom_schema_path: str | None
+
+
+@dataclass
+class ValidateSchemaStep(Step[ValidateSchemaStepOptions]):
+    """A schema validation step in a test case, which allows for validating a file against a schema."""
+
+    @staticmethod
+    def options_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
+        """Generate the schema of a schema validation step."""
+        if check_expected_result_files:
+            check_file = check_required_file(cwd)
+        else:
+            check_file = cfgv.check_string
+
+        return cfgv.Map(
+            "schema options",
+            None,
+            *[
+                cfgv.Required(
+                    key="kind",
+                    check_fn=cfgv.check_one_of(tuple(VALIDATE_SCHEMA_SCRIPTS.keys())),
+                ),
+                cfgv.Required(
+                    key="result",
+                    check_fn=cfgv.check_string,
+                ),
+                cfgv.Required(
+                    key="schema",
+                    check_fn=cfgv.check_one_of(tuple(DEFAULT_SCHEMAS.keys())),
+                ),
+                cfgv.Optional(
+                    key="custom_schema_path",
+                    default=None,
+                    check_fn=check_file,
+                ),
+            ],
+        )
+
+    def cmd(self, macaron_cmd: str) -> list[str]:
+        kind = self.options["kind"]
+        result_file = self.options["result"]
+        schema = self.options["schema"]
+        custom_schema_path = self.options["custom_schema_path"]
+
+        if custom_schema_path is None:
+            return [
+                "python",
+                os.path.abspath(os.path.join(*VALIDATE_SCHEMA_SCRIPTS[kind])),
+                *[result_file, os.path.abspath(os.path.join(*DEFAULT_SCHEMAS[schema]))],
+            ]
+
+        logger.info("A custom schema path at %s is given, using that instead.", custom_schema_path)
+        return [
+            "python",
+            os.path.abspath(os.path.join(*VALIDATE_SCHEMA_SCRIPTS[kind])),
+            *[result_file, custom_schema_path],
+        ]
+
+
 class CompareStepOptions(TypedDict):
     """Configuration of a compare step."""
 
@@ -473,6 +547,7 @@ def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
                         "compare",
                         "analyze",
                         "verify",
+                        "validate_schema",
                     ),
                 ),
             ),
@@ -482,6 +557,15 @@ def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
                 key="options",
                 schema=ShellStep.options_schema(),
             ),
+            cfgv.ConditionalRecurse(
+                condition_key="kind",
+                condition_value="validate_schema",
+                key="options",
+                schema=ValidateSchemaStep.options_schema(
+                    cwd=cwd,
+                    check_expected_result_files=check_expected_result_files,
+                ),
+            ),
             cfgv.ConditionalRecurse(
                 condition_key="kind",
                 condition_value="compare",
@@ -699,6 +783,7 @@ def parse_step_config(step_id: int, step_config: Mapping) -> Step:
         "verify": VerifyStep,
         "shell": ShellStep,
         "compare": CompareStep,
+        "validate_schema": ValidateSchemaStep,
     }[kind]
     return step_cls(  # type: ignore  # https://github.com/python/mypy/issues/3115
         step_id=step_id,

tests/schema_validation/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

tests/schema_validation/json_schema_validate.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module validates the result JSON files against a JSON schema."""
+
+import json
+import os
+import sys
+from collections.abc import Sequence
+
+import jsonschema
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run main logic."""
+    if not argv or not len(argv) == 3:
+        print("Usage: python3 schema_validate.py <json_path> <schema_path>")
+        return os.EX_USAGE
+
+    data_path = sys.argv[1]
+    schema_path = sys.argv[2]
+
+    schema = None
+    with open(schema_path, encoding="utf-8") as file:
+        try:
+            schema = json.load(file)
+        except json.JSONDecodeError as error:
+            print(f"Failed to load schema at {schema_path}, err:\n{error}\n")
+            return os.EX_DATAERR
+
+    data = None
+    with open(data_path, encoding="utf-8") as file:
+        try:
+            data = json.load(file)
+        except json.JSONDecodeError as error:
+            print(f"Failed to load JSON data at {data_path}, err:\n{error}\n")
+            return os.EX_DATAERR
+
+    try:
+        jsonschema.validate(
+            schema=schema,
+            instance=data,
+        )
+        print(f"JSON data at {data_path} PASSED schema {schema_path}.")
+        return os.EX_OK
+    except jsonschema.ValidationError as error:
+        print(f"JSON data at {data_path} FAILED schema {schema_path}, err:\n{error}\n")
+        return os.EX_DATAERR
+    except jsonschema.SchemaError as error:
+        print(f"The schema at {schema_path} is not valid, err:\n{error}\n")
+        return os.EX_DATAERR
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))