diff --git a/hermetic_build/common/model/gapic_config.py b/hermetic_build/common/model/gapic_config.py index 3bbba39454..2f416d9785 100644 --- a/hermetic_build/common/model/gapic_config.py +++ b/hermetic_build/common/model/gapic_config.py @@ -39,6 +39,9 @@ def is_stable(self): def get_version(self): return self.version + def to_dict(self): + return {"proto_path": self.proto_path} + def __parse_version(self) -> Optional[str]: version_regex = re.compile(r"^v[1-9]+(p[1-9]+)*(alpha|beta)?.*") for directory in self.proto_path.split("/"): diff --git a/hermetic_build/common/model/generation_config.py b/hermetic_build/common/model/generation_config.py index fabbde41cc..d19c3442c9 100644 --- a/hermetic_build/common/model/generation_config.py +++ b/hermetic_build/common/model/generation_config.py @@ -18,6 +18,11 @@ from typing import Optional from common.model.library_config import LibraryConfig from common.model.gapic_config import GapicConfig +from common.model.owlbot_yaml_config import ( + OwlbotYamlConfig, + OwlbotYamlAdditionRemoval, + DeepCopyRegexItem, +) REPO_LEVEL_PARAMETER = "Repo level parameter" LIBRARY_LEVEL_PARAMETER = "Library level parameter" @@ -77,6 +82,23 @@ def contains_common_protos(self) -> bool: break return self.__contains_common_protos + def to_dict(self): + return { + "gapic_generator_version": self.gapic_generator_version, + "googleapis_commitish": self.googleapis_commitish, + "libraries_bom_version": self.libraries_bom_version, + "libraries": [library.to_dict() for library in self.libraries], + } + + def write_object_to_yaml(self, file_path): + """Writes a Python object to a YAML file.""" + try: + with open(file_path, "w") as file: + yaml.dump(self.to_dict(), file, indent=2, sort_keys=False) + print(f"Object written to {file_path}") + except Exception as e: + print(f"Error writing to YAML file: {e}") + @staticmethod def __set_generator_version(gapic_generator_version: Optional[str]) -> str: if gapic_generator_version is not None: @@ -131,6 +153,8 @@ def from_yaml(path_to_yaml: str) -> "GenerationConfig": new_gapic = GapicConfig(proto_path) parsed_gapics.append(new_gapic) + owlbot_yaml = _owlbot_yaml_config_from_yaml(library) + new_library = LibraryConfig( api_shortname=_required(library, "api_shortname"), api_description=_required(library, "api_description"), @@ -160,6 +184,7 @@ def from_yaml(path_to_yaml: str) -> "GenerationConfig": recommended_package=_optional(library, "recommended_package", None), min_java_version=_optional(library, "min_java_version", None), transport=_optional(library, "transport", None), + owlbot_yaml=owlbot_yaml, ) parsed_libraries.append(new_library) @@ -190,3 +215,59 @@ def _optional(config: dict, key: str, default: any): if key not in config: return default return config[key] + + +def _owlbot_yaml_addition_remove_from_yaml(data: dict) -> OwlbotYamlAdditionRemoval: + """ + Parses the addition or remove section from owlbot_yaml data. + """ + deep_copy_regex = _optional(data, "deep_copy_regex", None) + deep_remove_regex = _optional(data, "deep_remove_regex", None) + deep_preserve_regex = _optional(data, "deep_preserve_regex", None) + + parsed_deep_copy_regex = None + if deep_copy_regex: + parsed_deep_copy_regex = [ + _deep_copy_regex_item_from_yaml(item) for item in deep_copy_regex + ] + + return OwlbotYamlAdditionRemoval( + deep_copy_regex=parsed_deep_copy_regex, + deep_remove_regex=deep_remove_regex, + deep_preserve_regex=deep_preserve_regex, + ) + + +def _deep_copy_regex_item_from_yaml(data: dict) -> DeepCopyRegexItem: + """ + Parses a DeepCopyRegexItem from a dictionary. + """ + source = _required(data, "source") + dest = _required(data, "dest") + return DeepCopyRegexItem(source=source, dest=dest) + + +def _owlbot_yaml_config_from_yaml( + library: LibraryConfig, +) -> Optional["OwlbotYamlConfig"]: + """ + Parses the owlbot_yaml section from a library's data. + """ + owlbot_yaml_data = _optional(library, "owlbot_yaml", None) + + if not owlbot_yaml_data: + return None + + addition_data = _optional(owlbot_yaml_data, "addition", None) + removal_data = _optional(owlbot_yaml_data, "remove", None) + + additions = None + removals = None + + if addition_data: + additions = _owlbot_yaml_addition_remove_from_yaml(addition_data) + + if removal_data: + removals = _owlbot_yaml_addition_remove_from_yaml(removal_data) + + return OwlbotYamlConfig(additions=additions, removals=removals) diff --git a/hermetic_build/common/model/library_config.py b/hermetic_build/common/model/library_config.py index 9b9d31b810..421a9a2480 100644 --- a/hermetic_build/common/model/library_config.py +++ b/hermetic_build/common/model/library_config.py @@ -16,6 +16,8 @@ from typing import Optional from common.model.gapic_config import GapicConfig from common.model.gapic_inputs import GapicInputs +from common.model.owlbot_yaml_config import OwlbotYamlConfig +from collections import OrderedDict MAVEN_COORDINATE_SEPARATOR = ":" @@ -54,6 +56,7 @@ def __init__( recommended_package: Optional[str] = None, min_java_version: Optional[int] = None, transport: Optional[str] = None, + owlbot_yaml: Optional[OwlbotYamlConfig] = None, ): self.api_shortname = api_shortname self.api_description = api_description @@ -81,6 +84,7 @@ def __init__( self.min_java_version = min_java_version self.distribution_name = self.__get_distribution_name(distribution_name) self.transport = self.__validate_transport(transport) + self.owlbot_yaml = owlbot_yaml def set_gapic_configs(self, gapic_configs: list[GapicConfig]) -> None: """ @@ -122,6 +126,61 @@ def get_transport(self, gapic_inputs: GapicInputs) -> str: """ return self.transport if self.transport is not None else gapic_inputs.transport + def to_dict(self): + """Converts the LibraryConfig object to a dictionary with ordered keys.""" + data = {} + data["api_shortname"] = self.api_shortname + data["name_pretty"] = self.name_pretty + data["product_documentation"] = self.product_documentation + data["api_description"] = self.api_description + if self.library_type and self.library_type != "GAPIC_AUTO": + data["library_type"] = self.library_type + if self.release_level: + data["release_level"] = self.release_level + if self.api_id: + data["api_id"] = self.api_id + if self.api_reference: + data["api_reference"] = self.api_reference + if self.codeowner_team: + data["codeowner_team"] = self.codeowner_team + if self.client_documentation: + data["client_documentation"] = self.client_documentation + if self.distribution_name: + data["distribution_name"] = self.distribution_name + if self.excluded_dependencies: + data["excluded_dependencies"] = self.excluded_dependencies + if self.excluded_poms: + data["excluded_poms"] = self.excluded_poms + if self.googleapis_commitish: + data["googleapis_commitish"] = self.googleapis_commitish + if self.group_id and self.group_id != "com.google.cloud": + data["group_id"] = self.group_id + if self.issue_tracker: + data["issue_tracker"] = self.issue_tracker + if self.library_name: + data["library_name"] = self.library_name + if self.rest_documentation: + data["rest_documentation"] = self.rest_documentation + if self.rpc_documentation: + data["rpc_documentation"] = self.rpc_documentation + if self.cloud_api is False: # Only spell out when false + data["cloud_api"] = self.cloud_api + if self.requires_billing is False: # Only spell out when false + data["requires_billing"] = self.requires_billing + if self.extra_versioned_modules: + data["extra_versioned_modules"] = self.extra_versioned_modules + if self.recommended_package: + data["recommended_package"] = self.recommended_package + if self.min_java_version: + data["min_java_version"] = self.min_java_version + if self.transport: + data["transport"] = self.transport + if self.gapic_configs: + data["GAPICs"] = [gc.to_dict() for gc in self.gapic_configs] + if self.owlbot_yaml: + data["owlbot_yaml"] = self.owlbot_yaml.to_dict() + return data + def __get_distribution_name(self, distribution_name: Optional[str]) -> str: LibraryConfig.__check_distribution_name(distribution_name) if distribution_name: diff --git a/hermetic_build/common/model/owlbot_yaml_config.py b/hermetic_build/common/model/owlbot_yaml_config.py new file mode 100644 index 0000000000..a311902b43 --- /dev/null +++ b/hermetic_build/common/model/owlbot_yaml_config.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Optional, Dict + + +class DeepCopyRegexItem: + def __init__(self, source: str, dest: str): + self.source = source + self.dest = dest + + def to_dict(self): + return { + "source": self.source, + "dest": self.dest, + } + + +class OwlbotYamlAdditionRemoval: + def __init__( + self, + deep_copy_regex: Optional[List[DeepCopyRegexItem]] = None, + deep_remove_regex: Optional[List[str]] = None, + deep_preserve_regex: Optional[List[str]] = None, + ): + self.deep_copy_regex = deep_copy_regex + self.deep_remove_regex = deep_remove_regex + self.deep_preserve_regex = deep_preserve_regex + + def to_dict(self): + data = {} + if self.deep_copy_regex: + data["deep_copy_regex"] = [item.to_dict() for item in self.deep_copy_regex] + if self.deep_remove_regex: + data["deep_remove_regex"] = self.deep_remove_regex + if self.deep_preserve_regex: + data["deep_preserve_regex"] = self.deep_preserve_regex + return data + + +class OwlbotYamlConfig: + def __init__( + self, + additions: Optional[OwlbotYamlAdditionRemoval] = None, + removals: Optional[OwlbotYamlAdditionRemoval] = None, + ): + self.additions = additions + self.removals = removals + + def to_dict(self): + data = {} + if self.additions: + data["additions"] = self.additions.to_dict() + if self.removals: + data["removals"] = self.removals.to_dict() + return data diff --git a/hermetic_build/common/tests/model/gapic_config_unit_tests.py b/hermetic_build/common/tests/model/gapic_config_unit_tests.py index 864b2556e4..35c6afb907 100644 --- a/hermetic_build/common/tests/model/gapic_config_unit_tests.py +++ b/hermetic_build/common/tests/model/gapic_config_unit_tests.py @@ -54,6 +54,13 @@ def test_is_stable_with_stable_version_returns_true(self): GapicConfig(proto_path="example/dir1/dir2/v30").is_stable(), ) + def test_to_dict_returns_proto_path_as_dict(self): + expected_config_as_dict = {"proto_path": "example/dir1/dir2"} + self.assertEqual( + expected_config_as_dict, + GapicConfig(proto_path="example/dir1/dir2").to_dict(), + ) + def test_compare_configs_without_a_version(self): config_len_3 = GapicConfig(proto_path="example/dir1/dir2") config_len_4 = GapicConfig(proto_path="example/dir1/dir2/dir3") diff --git a/hermetic_build/common/tests/model/generation_config_unit_tests.py b/hermetic_build/common/tests/model/generation_config_unit_tests.py index 2b89a1acc8..ea4a75f172 100644 --- a/hermetic_build/common/tests/model/generation_config_unit_tests.py +++ b/hermetic_build/common/tests/model/generation_config_unit_tests.py @@ -11,9 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from io import StringIO import os -import unittest from pathlib import Path +import unittest +from unittest.mock import patch, mock_open +import yaml + from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig @@ -104,6 +109,21 @@ def test_from_yaml_succeeds(self): self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path) self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path) + owlbot_yaml_addition = library.owlbot_yaml.additions + owlbot_yaml_removal = library.owlbot_yaml.removals + self.assertEqual( + "/java-asset/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java", + owlbot_yaml_removal.deep_preserve_regex[0], + ) + self.assertEqual( + "/owl-bot-staging/java-accesscontextmanager/type/proto-google-identity-accesscontextmanager-type/src", + owlbot_yaml_addition.deep_copy_regex[0].dest, + ) + self.assertEqual( + "/google/identity/accesscontextmanager/type/.*-java/proto-google-.*/src", + owlbot_yaml_addition.deep_copy_regex[0].source, + ) + def test_get_proto_path_to_library_name_success(self): paths = GenerationConfig.from_yaml( f"{test_config_dir}/generation_config.yaml" @@ -256,3 +276,65 @@ def test_from_yaml_with_zero_proto_path_raise_exception(self): GenerationConfig.from_yaml, f"{test_config_dir}/config_without_gapics_value.yaml", ) + + def test_to_dict_return_correctly(self): + config = GenerationConfig( + gapic_generator_version="x.y.z", + libraries_bom_version="a.b.c", + googleapis_commitish="foo", + libraries=[library_1], + ) + expect_config_as_dict = { + "gapic_generator_version": "x.y.z", + "libraries_bom_version": "a.b.c", + "googleapis_commitish": "foo", + "libraries": [library_1.to_dict()], + } + self.assertEqual(expect_config_as_dict, config.to_dict()) + + @patch("builtins.open", new_callable=mock_open) + def test_write_object_to_yaml_success(self, mock_open_file): + config = GenerationConfig( + gapic_generator_version="x.y.z", + libraries_bom_version="a.b.c", + googleapis_commitish="foo", + libraries=[], + ) + + file_path = "test_output.yaml" + config.write_object_to_yaml(file_path) + + # Assert that open was called with the correct arguments + mock_open_file.assert_called_once_with(file_path, "w") + + # Get the handle that was used to write to the file + handle = mock_open_file() + + # Get the written YAML data + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + + # Load the written data using yaml to verify + loaded_data = yaml.safe_load(written_data) + + expected_data = { + "gapic_generator_version": "x.y.z", + "libraries_bom_version": "a.b.c", + "googleapis_commitish": "foo", + "libraries": [], + } + + self.assertEqual(loaded_data, expected_data) + + @patch("builtins.open", side_effect=Exception("File system error")) + @patch("sys.stdout", new_callable=StringIO) + def test_write_object_to_yaml_error(self, mock_stdout, mock_open_file): + config = GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + libraries=[library_1], + ) + file_path = "test_output.yaml" + config.write_object_to_yaml(file_path) + + # Assert that the error message was printed to stdout + self.assertIn("Error writing to YAML file:", mock_stdout.getvalue()) diff --git a/hermetic_build/common/tests/model/library_config_unit_tests.py b/hermetic_build/common/tests/model/library_config_unit_tests.py index 4935979ffa..6c0be58a2c 100644 --- a/hermetic_build/common/tests/model/library_config_unit_tests.py +++ b/hermetic_build/common/tests/model/library_config_unit_tests.py @@ -120,3 +120,26 @@ def test_get_distribution_name_with_distribution_name(self): "com.example:baremetalsolution", library.get_maven_coordinate() ) self.assertEqual("baremetalsolution", library.get_artifact_id()) + + def test_to_dict_some_values(self): + library = LibraryConfig( + api_shortname="secret", + name_pretty="", + product_documentation="", + api_description="", + gapic_configs=[GapicConfig("test/proto/path")], + library_name="secretmanager", + ) + + expected_dict = { + "api_shortname": "secret", + "name_pretty": "", + "product_documentation": "", + "api_description": "", + "library_name": "secretmanager", + "GAPICs": [{"proto_path": "test/proto/path"}], + # calculated at init + "distribution_name": "com.google.cloud:google-cloud-secretmanager", + "release_level": "preview", + } + self.assertEqual(library.to_dict(), expected_dict) diff --git a/hermetic_build/common/tests/model/owlbot_yaml_confit_unit_tests.py b/hermetic_build/common/tests/model/owlbot_yaml_confit_unit_tests.py new file mode 100644 index 0000000000..4c26e7fab4 --- /dev/null +++ b/hermetic_build/common/tests/model/owlbot_yaml_confit_unit_tests.py @@ -0,0 +1,95 @@ +import unittest +from typing import List, Optional, Dict + +from common.model.owlbot_yaml_config import ( + DeepCopyRegexItem, + OwlbotYamlAdditionRemoval, + OwlbotYamlConfig, +) + + +class TestDeepCopyRegexItem(unittest.TestCase): + + def test_to_dict(self): + item = DeepCopyRegexItem(source="src/path", dest="dest/path") + expected_dict = {"source": "src/path", "dest": "dest/path"} + self.assertEqual(item.to_dict(), expected_dict) + + +class TestOwlbotYamlAdditionRemoval(unittest.TestCase): + + def test_to_dict_all_values(self): + item1 = DeepCopyRegexItem(source="src1", dest="dest1") + item2 = DeepCopyRegexItem(source="src2", dest="dest2") + obj = OwlbotYamlAdditionRemoval( + deep_copy_regex=[item1, item2], + deep_remove_regex=["remove1", "remove2"], + deep_preserve_regex=["preserve1", "preserve2"], + ) + expected_dict = { + "deep_copy_regex": [ + {"source": "src1", "dest": "dest1"}, + {"source": "src2", "dest": "dest2"}, + ], + "deep_remove_regex": ["remove1", "remove2"], + "deep_preserve_regex": ["preserve1", "preserve2"], + } + self.assertEqual(obj.to_dict(), expected_dict) + + def test_to_dict_some_values_none(self): + obj = OwlbotYamlAdditionRemoval(deep_remove_regex=["remove1"]) + expected_dict = {"deep_remove_regex": ["remove1"]} + self.assertEqual(obj.to_dict(), expected_dict) + + def test_to_dict_empty(self): + obj = OwlbotYamlAdditionRemoval() + expected_dict = {} + self.assertEqual(obj.to_dict(), expected_dict) + + +class TestOwlbotYamlConfig(unittest.TestCase): + + def test_to_dict_all_values(self): + item1 = DeepCopyRegexItem(source="src1", dest="dest1") + addition_obj = OwlbotYamlAdditionRemoval( + deep_copy_regex=[item1], deep_remove_regex=["remove1"] + ) + remove_obj = OwlbotYamlAdditionRemoval(deep_preserve_regex=["preserve1"]) + config = OwlbotYamlConfig(additions=addition_obj, removals=remove_obj) + expected_dict = { + "additions": { + "deep_copy_regex": [{"source": "src1", "dest": "dest1"}], + "deep_remove_regex": ["remove1"], + }, + "removals": {"deep_preserve_regex": ["preserve1"]}, + } + self.assertEqual(config.to_dict(), expected_dict) + + def test_to_dict_addition_none(self): + remove_obj = OwlbotYamlAdditionRemoval(deep_preserve_regex=["preserve1"]) + config = OwlbotYamlConfig(removals=remove_obj) + expected_dict = {"removals": {"deep_preserve_regex": ["preserve1"]}} + self.assertEqual(config.to_dict(), expected_dict) + + def test_to_dict_remove_none(self): + item1 = DeepCopyRegexItem(source="src1", dest="dest1") + addition_obj = OwlbotYamlAdditionRemoval( + deep_copy_regex=[item1], deep_remove_regex=["remove1"] + ) + config = OwlbotYamlConfig(additions=addition_obj) + expected_dict = { + "additions": { + "deep_copy_regex": [{"source": "src1", "dest": "dest1"}], + "deep_remove_regex": ["remove1"], + } + } + self.assertEqual(config.to_dict(), expected_dict) + + def test_to_dict_empty(self): + config = OwlbotYamlConfig() + expected_dict = {} + self.assertEqual(config.to_dict(), expected_dict) + + +if __name__ == "__main__": + unittest.main() diff --git a/hermetic_build/common/tests/resources/test-config/generation_config.yaml b/hermetic_build/common/tests/resources/test-config/generation_config.yaml index 94fb1c8337..065a979231 100644 --- a/hermetic_build/common/tests/resources/test-config/generation_config.yaml +++ b/hermetic_build/common/tests/resources/test-config/generation_config.yaml @@ -21,3 +21,15 @@ libraries: - proto_path: google/cloud/asset/v1p2beta1 - proto_path: google/cloud/asset/v1p5beta1 - proto_path: google/cloud/asset/v1p7beta1 + owlbot_yaml: + additions: + deep_preserve_regex: + - /java-asset/google-cloud-.*/src/test/java/com/google/cloud/.*/it + - /java-asset/google-cloud-asset/src/test/java/com/google/cloud/asset/v1/VPCServiceControlTest.java + - /java-asset/proto-google-cloud-asset-v1/src/main/java/com/google/cloud/asset/v1/ProjectName.java + deep_copy_regex: + - source: /google/identity/accesscontextmanager/type/.*-java/proto-google-.*/src + dest: /owl-bot-staging/java-accesscontextmanager/type/proto-google-identity-accesscontextmanager-type/src + removals: + deep_preserve_regex: + - /java-asset/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java \ No newline at end of file diff --git a/hermetic_build/library_generation/tests/utilities_unit_tests.py b/hermetic_build/library_generation/tests/utilities_unit_tests.py index a6796b706d..c066fd72fb 100644 --- a/hermetic_build/library_generation/tests/utilities_unit_tests.py +++ b/hermetic_build/library_generation/tests/utilities_unit_tests.py @@ -26,6 +26,11 @@ from common.model.gapic_inputs import GapicInputs from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig +from common.model.owlbot_yaml_config import ( + OwlbotYamlConfig, + OwlbotYamlAdditionRemoval, + DeepCopyRegexItem, +) from library_generation.tests.test_utils import FileComparator from library_generation.tests.test_utils import cleanup @@ -73,6 +78,18 @@ class UtilitiesTest(unittest.TestCase): Unit tests for utilities.py """ + content = """ +deep-remove-regex: +- "/java-connectgateway/proto-google-.*/src" +- "/java-connectgateway/google-.*/src" +deep-preserve-regex: +- "/java-connectgateway/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" +deep-copy-regex: +- source: "/google/cloud/gkeconnect/gateway/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-connectgateway/$1/proto-google-cloud-connectgateway-$1/src" +api-name: connectgateway +""" + CONFIGURATION_YAML_PATH = os.path.join( script_dir, "resources", @@ -302,6 +319,104 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual(["misc"], library_path) shutil.rmtree(repo_config.output_folder) + def test_apply_owlbot_config_remove_deep_remove_and_preserve(self): + config = OwlbotYamlConfig( + removals=OwlbotYamlAdditionRemoval( + deep_remove_regex=["/java-connectgateway/proto-google-.*/src"], + deep_preserve_regex=[ + "/java-connectgateway/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + ], + ) + ) + expected_content = """ +deep-remove-regex: +- "/java-connectgateway/google-.*/src" +deep-preserve-regex: +deep-copy-regex: +- source: "/google/cloud/gkeconnect/gateway/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-connectgateway/$1/proto-google-cloud-connectgateway-$1/src" +api-name: connectgateway +""" + self.assertEqual( + util.apply_owlbot_config(self.content, config), expected_content + ) + + def test_apply_owlbot_config_remove_deep_copy_regex(self): + item1 = DeepCopyRegexItem( + source="/google/cloud/gkeconnect/gateway/(v.*)/.*-java/proto-google-.*/src", + dest="/owl-bot-staging/java-connectgateway/$1/proto-google-cloud-connectgateway-$1/src", + ) + + config = OwlbotYamlConfig( + removals=OwlbotYamlAdditionRemoval(deep_copy_regex=[item1]) + ) + expected_content = """ +deep-remove-regex: +- "/java-connectgateway/proto-google-.*/src" +- "/java-connectgateway/google-.*/src" +deep-preserve-regex: +- "/java-connectgateway/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" +deep-copy-regex: +api-name: connectgateway +""" + self.assertEqual( + util.apply_owlbot_config(self.content, config), expected_content + ) + + def test_apply_owlbot_config_add_deep_remove_and_preserve(self): + config = OwlbotYamlConfig( + additions=OwlbotYamlAdditionRemoval( + deep_remove_regex=["/new/path"], + deep_preserve_regex=["/new/path/to/preserve"], + ) + ) + expected_content = """ +deep-remove-regex: +- "/new/path" +- "/java-connectgateway/proto-google-.*/src" +- "/java-connectgateway/google-.*/src" +deep-preserve-regex: +- "/new/path/to/preserve" +- "/java-connectgateway/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" +deep-copy-regex: +- source: "/google/cloud/gkeconnect/gateway/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-connectgateway/$1/proto-google-cloud-connectgateway-$1/src" +api-name: connectgateway +""" + self.assertEqual( + util.apply_owlbot_config(self.content, config), expected_content + ) + + def test_apply_owlbot_config_add_deep_copy_regex(self): + + item1 = DeepCopyRegexItem(source="/path/to/copy", dest="/dest/to/copy") + config = OwlbotYamlConfig( + additions=OwlbotYamlAdditionRemoval(deep_copy_regex=[item1]) + ) + expected_content = """ +deep-remove-regex: +- "/java-connectgateway/proto-google-.*/src" +- "/java-connectgateway/google-.*/src" +deep-preserve-regex: +- "/java-connectgateway/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" +deep-copy-regex: +- source: "/path/to/copy" + dest: "/dest/to/copy" +- source: "/google/cloud/gkeconnect/gateway/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-connectgateway/$1/proto-google-cloud-connectgateway-$1/src" +api-name: connectgateway +""" + self.assertEqual( + util.apply_owlbot_config(self.content, config), expected_content + ) + + def test_apply_owlbot_config_no_config(self): + config = None + expected_content = self.content + self.assertEqual( + util.apply_owlbot_config(self.content, config), expected_content + ) + def __setup_postprocessing_prerequisite_files( self, combination: int, diff --git a/hermetic_build/library_generation/utils/file_render.py b/hermetic_build/library_generation/utils/file_render.py index 5c68445753..9ebe9c8d9b 100644 --- a/hermetic_build/library_generation/utils/file_render.py +++ b/hermetic_build/library_generation/utils/file_render.py @@ -19,6 +19,21 @@ def render(template_name: str, output_name: str, **kwargs): template = jinja_env.get_template(template_name) t = template.stream(kwargs) directory = os.path.dirname(output_name) - if not os.path.isdir(directory): - os.makedirs(directory) + os.makedirs(directory, exist_ok=True) t.dump(str(output_name)) + + +def render_to_str(template_name: str, **kwargs) -> str: + """ + Renders a Jinja2 template and returns the output as a string. + + Args: + template_name: The name of the Jinja2 template file. + **kwargs: Keyword arguments containing the data to pass to the template. + + Returns: + The rendered template content as a string. + """ + template = jinja_env.get_template(template_name) + rendered_content = template.render(**kwargs) + return rendered_content diff --git a/hermetic_build/library_generation/utils/utilities.py b/hermetic_build/library_generation/utils/utilities.py index ec5c03d069..bc548b667a 100755 --- a/hermetic_build/library_generation/utils/utilities.py +++ b/hermetic_build/library_generation/utils/utilities.py @@ -12,24 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +import os +import re import sys import subprocess -import os from pathlib import Path -from typing import Any +from typing import Any, Optional from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig -from typing import List from library_generation.model.repo_config import RepoConfig -from library_generation.utils.file_render import render +from library_generation.utils.file_render import render, render_to_str from library_generation.utils.proto_path_utils import remove_version_from script_dir = os.path.dirname(os.path.realpath(__file__)) SDK_PLATFORM_JAVA = "googleapis/sdk-platform-java" -def create_argument(arg_key: str, arg_container: object) -> List[str]: +def create_argument(arg_key: str, arg_container: object) -> list[str]: """ Generates a list of two elements [argument, value], or returns an empty array if arg_val is None @@ -41,7 +41,7 @@ def create_argument(arg_key: str, arg_container: object) -> List[str]: def run_process_and_print_output( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs + arguments: list[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs ) -> Any: """ Runs a process with the given "arguments" list and prints its output. @@ -68,7 +68,7 @@ def run_process_and_print_output( def run_process_and_get_output_string( - arguments: List[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs + arguments: list[str] | str, job_name: str = "Job", exit_on_fail=True, **kwargs ) -> Any: """ Wrapper of run_process_and_print_output() that returns the merged @@ -123,7 +123,7 @@ def eprint(*args, **kwargs): def prepare_repo( gen_config: GenerationConfig, - library_config: List[LibraryConfig], + library_config: list[LibraryConfig], repo_path: str, language: str = "java", ) -> RepoConfig: @@ -276,14 +276,23 @@ def generate_postprocessing_prerequisite_files( else f"{library_path}/.github/{owlbot_yaml_file}" ) if not os.path.exists(path_to_owlbot_yaml_file): - render( + # 1. Render the base content + generated_content = render_to_str( template_name="owlbot.yaml.monorepo.j2", - output_name=path_to_owlbot_yaml_file, artifact_id=artifact_id, proto_path=remove_version_from(proto_path), module_name=repo_metadata["repo_short"], api_shortname=library.api_shortname, + owlbot_yaml=library.owlbot_yaml, ) + # 2. Apply additions and removals + modified_content = apply_owlbot_config(generated_content, library.owlbot_yaml) + + # 3. Write the modified content back to the file + directory = os.path.dirname(path_to_owlbot_yaml_file) + os.makedirs(directory, exist_ok=True) + with open(path_to_owlbot_yaml_file, "w") as f: + f.write(modified_content) # generate owlbot.py py_file = "owlbot.py" @@ -307,3 +316,127 @@ def generate_postprocessing_prerequisite_files( should_include_templates=True, template_excludes=template_excludes, ) + + +def apply_owlbot_config( + content: str, owlbot_config: Optional["OwlbotYamlConfig"] +) -> str: + """ + Applies the addition and removal configurations to the generated owlbot.yaml content. + + Args: + content: The generated owlbot.yaml content as a string. + owlbot_config: The OwlbotYamlConfig object containing the addition and removal rules. + + Returns: + The modified owlbot.yaml content. + """ + if not owlbot_config: + return content + + modified_content = content + + if owlbot_config.removals: + if owlbot_config.removals.deep_remove_regex: + for regex_pattern in owlbot_config.removals.deep_remove_regex: + modified_content = re.sub( + r'- "' + regex_pattern + r'"[^\S\n]*\n', "", modified_content + ) + if owlbot_config.removals.deep_preserve_regex: + for regex_pattern in owlbot_config.removals.deep_preserve_regex: + modified_content = re.sub( + r'- "' + regex_pattern + r'"[^\S\n]*\n', "", modified_content + ) + if owlbot_config.removals.deep_copy_regex: + for item in owlbot_config.removals.deep_copy_regex: + # Construct the regex to match both source and dest lines + source_regex = r'- source: "' + re.escape(item.source) + r'"\n' + dest_regex = r'\s+dest: "' + re.escape(item.dest) + r'"\n' + removal_regex = rf"{source_regex}{dest_regex}" + modified_content = re.sub( + removal_regex, "", modified_content, flags=re.MULTILINE + ) + if owlbot_config.additions: + if owlbot_config.additions.deep_remove_regex: + modified_content = _add_items_after_key( + content=modified_content, + key="deep-remove-regex:", + items=owlbot_config.additions.deep_remove_regex, + quote=True, + ) + if owlbot_config.additions.deep_preserve_regex: + modified_content = _add_items_after_key( + content=modified_content, + key="deep-preserve-regex:", + items=owlbot_config.additions.deep_preserve_regex, + quote=True, + ) + if owlbot_config.additions.deep_copy_regex: + modified_content = _add_deep_copy_regex_items( + content=modified_content, + key="deep-copy-regex:", + items=owlbot_config.additions.deep_copy_regex, + ) + + return modified_content + + +def _add_items_after_key( + content: str, key: str, items: list[str], quote: bool = False +) -> str: + """ + Adds items after a specified key in the content, following the template pattern. + + Args: + content: The generated owlbot.yaml content. + key: The key after which to add the items (e.g., "deep-remove-regex:"). + items: The list of items to add. + quote: Whether to enclose the item in double quotes. + + Returns: + The modified content. + """ + + def format_item(item: str) -> str: + if quote: + return f'- "{item}"\n' + else: + return f"- {item}\n" + + pattern = re.compile(rf"^{re.escape(key)}", re.MULTILINE) + match = pattern.search(content) + + if match: + insert_position = match.end() + 1 # Insert after the newline + new_items = "".join(format_item(item) for item in items) + return content[:insert_position] + new_items + content[insert_position:] + else: + return content # Key not found, return original content + + +def _add_deep_copy_regex_items( + content: str, key: str, items: list["DeepCopyRegexItem"] +) -> str: + """ + Adds deep_copy_regex items after a specified key in the content. + + Args: + content: The generated owlbot.yaml content. + key: The key after which to add the items (e.g., "deep-copy-regex:"). + items: The list of DeepCopyRegexItem objects to add. + + Returns: + The modified content. + """ + + pattern = re.compile(rf"^{re.escape(key)}", re.MULTILINE) + match = pattern.search(content) + + if match: + insert_position = match.end() + 1 # Insert after the newline + new_items = "".join( + f'- source: "{item.source}"\n dest: "{item.dest}"\n' for item in items + ) + return content[:insert_position] + new_items + content[insert_position:] + else: + return content