feat: generate proto-only repository (#2720)

JoeWang1127 · web-flow · commit f7a5161b637c · 2024-05-16T17:52:15.000-04:00
In this PR:
- Enable hermetic build script to generate `java-common-protos` and
`java-iam` in this repo.
- Change `libraries_bom_version` to optional.
- Refactor unit tests.
- Remove `java-common-protos/codecov.yaml` and `java-iam/codecov.yaml`
as they are no longer needed.
diff --git a/java-common-protos/codecov.yaml b/java-common-protos/codecov.yaml
diff --git a/java-common-protos/owlbot.py b/java-common-protos/owlbot.py
@@ -44,6 +44,7 @@
     "SECURITY.md",
     "java.header",
     "license-checks.xml",
+    "README.md",
     "renovate.json",
     ".gitignore"
 ])
diff --git a/java-iam/codecov.yaml b/java-iam/codecov.yaml
diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py
@@ -48,7 +48,6 @@ def generate_from_yaml(
 
     for library_path, library in repo_config.libraries.items():
         print(f"generating library {library.get_library_name()}")
-
         generate_composed_library(
             config_path=config_path,
             config=config,
@@ -58,8 +57,7 @@ def generate_from_yaml(
             versions_file=repo_config.versions_file,
         )
 
-    # we skip monorepo_postprocessing if not in a monorepo
-    if not config.is_monorepo():
+    if not config.is_monorepo() or config.contains_common_protos():
         return
 
     monorepo_postprocessing(
diff --git a/library_generation/model/gapic_inputs.py b/library_generation/model/gapic_inputs.py
@@ -56,7 +56,7 @@ def __init__(
         self,
         proto_only="true",
         additional_protos="google/cloud/common_resources.proto",
-        transport="",
+        transport="grpc",
         rest_numeric_enum="",
         gapic_yaml="",
         service_config="",
diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py
@@ -20,6 +20,7 @@
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"
 GAPIC_LEVEL_PARAMETER = "GAPIC level parameter"
+COMMON_PROTOS_LIBRARY_NAME = "common-protos"
 
 
 class GenerationConfig:
@@ -31,9 +32,9 @@ def __init__(
         self,
         gapic_generator_version: str,
         googleapis_commitish: str,
-        libraries_bom_version: str,
         template_excludes: list[str],
         libraries: list[LibraryConfig],
+        libraries_bom_version: Optional[str] = None,
         grpc_version: Optional[str] = None,
         protoc_version: Optional[str] = None,
     ):
@@ -44,6 +45,9 @@ def __init__(
         self.libraries = libraries
         self.grpc_version = grpc_version
         self.protoc_version = protoc_version
+        # explicit set to None so that we can compute the
+        # value in getter.
+        self.__contains_common_protos = None
         self.__validate()
 
     def get_proto_path_to_library_name(self) -> dict[str, str]:
@@ -61,6 +65,15 @@ def get_proto_path_to_library_name(self) -> dict[str, str]:
     def is_monorepo(self) -> bool:
         return len(self.libraries) > 1
 
+    def contains_common_protos(self) -> bool:
+        if self.__contains_common_protos is None:
+            self.__contains_common_protos = False
+            for library in self.libraries:
+                if library.get_library_name() == COMMON_PROTOS_LIBRARY_NAME:
+                    self.__contains_common_protos = True
+                    break
+        return self.__contains_common_protos
+
     def __validate(self) -> None:
         seen_library_names = dict()
         for library in self.libraries:
@@ -133,15 +146,13 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig:
         gapic_generator_version=__required(
             config, "gapic_generator_version", REPO_LEVEL_PARAMETER
         ),
-        grpc_version=__optional(config, "grpc_version", None),
-        protoc_version=__optional(config, "protoc_version", None),
         googleapis_commitish=__required(
             config, "googleapis_commitish", REPO_LEVEL_PARAMETER
         ),
-        libraries_bom_version=__required(
-            config, "libraries_bom_version", REPO_LEVEL_PARAMETER
-        ),
         template_excludes=__required(config, "template_excludes", REPO_LEVEL_PARAMETER),
+        grpc_version=__optional(config, "grpc_version", None),
+        protoc_version=__optional(config, "protoc_version", None),
+        libraries_bom_version=__optional(config, "libraries_bom_version", None),
         libraries=parsed_libraries,
     )
 
diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py
@@ -22,6 +22,7 @@
 
 class EntryPointTest(unittest.TestCase):
     def test_entry_point_without_config_raise_file_exception(self):
+        os.chdir(script_dir)
         runner = CliRunner()
         # noinspection PyTypeChecker
         result = runner.invoke(generate, ["--repository-path=."])
diff --git a/library_generation/test/model/gapic_inputs_unit_tests.py b/library_generation/test/model/gapic_inputs_unit_tests.py
@@ -0,0 +1,132 @@
+import os
+import unittest
+from pathlib import Path
+
+from parameterized import parameterized
+from library_generation.model.gapic_inputs import parse
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+resources_dir = os.path.join(script_dir, "..", "resources")
+build_file = Path(os.path.join(resources_dir, "misc")).resolve()
+
+
+class UtilitiesTest(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ("BUILD_no_additional_protos.bazel", " "),
+            ("BUILD_common_resources.bazel", "  google/cloud/common_resources.proto"),
+            ("BUILD_comment_common_resources.bazel", " "),
+            ("BUILD_locations.bazel", "  google/cloud/location/locations.proto"),
+            ("BUILD_comment_locations.bazel", " "),
+            ("BUILD_iam_policy.bazel", "  google/iam/v1/iam_policy.proto"),
+            ("BUILD_comment_iam_policy.bazel", " "),
+            (
+                "BUILD_iam_locations.bazel",
+                "  google/cloud/location/locations.proto google/iam/v1/iam_policy.proto",
+            ),
+        ]
+    )
+    def test_gapic_inputs_parse_additional_protos(self, build_name, expected):
+        parsed = parse(build_file, "", build_name)
+        self.assertEqual(
+            expected,
+            parsed.additional_protos,
+        )
+
+    def test_gapic_inputs_parse_grpc_only_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_grpc.bazel")
+        self.assertEqual("grpc", parsed.transport)
+
+    def test_gapic_inputs_parse_grpc_rest_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_grpc_rest.bazel")
+        self.assertEqual("grpc+rest", parsed.transport)
+
+    def test_gapic_inputs_parse_rest_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest.bazel")
+        self.assertEqual("rest", parsed.transport)
+
+    def test_gapic_inputs_parse_empty_include_samples_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
+        self.assertEqual("false", parsed.include_samples)
+
+    def test_gapic_inputs_parse_include_samples_false_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_false.bazel")
+        self.assertEqual("false", parsed.include_samples)
+
+    def test_gapic_inputs_parse_include_samples_true_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_include_samples_true.bazel")
+        self.assertEqual("true", parsed.include_samples)
+
+    def test_gapic_inputs_parse_empty_rest_numeric_enums_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_empty.bazel")
+        self.assertEqual("false", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_rest_numeric_enums_false_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_false.bazel")
+        self.assertEqual("false", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_rest_numeric_enums_true_succeeds(self):
+        parsed = parse(build_file, "", "BUILD_rest_numeric_enums_true.bazel")
+        self.assertEqual("true", parsed.rest_numeric_enum)
+
+    def test_gapic_inputs_parse_no_gapic_library_returns_proto_only_true(self):
+        # include_samples_empty only has a gradle assembly rule
+        parsed = parse(build_file, "", "BUILD_include_samples_empty.bazel")
+        self.assertEqual("true", parsed.proto_only)
+
+    def test_gapic_inputs_parse_with_gapic_library_returns_proto_only_false(self):
+        # rest.bazel has a java_gapic_library rule
+        parsed = parse(build_file, "", "BUILD_rest.bazel")
+        self.assertEqual("false", parsed.proto_only)
+
+    def test_gapic_inputs_parse_gapic_yaml_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_gapic_yaml.bazel")
+        self.assertEqual("test/versioned/path/test_gapic_yaml.yaml", parsed.gapic_yaml)
+
+    def test_gapic_inputs_parse_no_gapic_yaml_returns_empty_string(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_no_gapic_yaml.bazel")
+        self.assertEqual("", parsed.gapic_yaml)
+
+    def test_gapic_inputs_parse_service_config_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_service_config.bazel")
+        self.assertEqual(
+            "test/versioned/path/test_service_config.json", parsed.service_config
+        )
+
+    def test_gapic_inputs_parse_service_yaml_relative_target(self):
+        parsed = parse(
+            build_file,
+            "google/cloud/compute/v1",
+            "BUILD_service_config_relative_target.bazel",
+        )
+        self.assertEqual(
+            "google/cloud/compute/v1/compute_grpc_service_config.json",
+            parsed.service_config,
+        )
+
+    def test_gapic_inputs_parse_no_service_config_returns_empty_string(self):
+        parsed = parse(
+            build_file, "test/versioned/path", "BUILD_no_service_config.bazel"
+        )
+        self.assertEqual("", parsed.service_config)
+
+    def test_gapic_inputs_parse_service_yaml_succeeds(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_service_yaml.bazel")
+        self.assertEqual(
+            "test/versioned/path/test_service_yaml.yaml", parsed.service_yaml
+        )
+
+    def test_gapic_inputs_parse_service_yaml_absolute_target(self):
+        parsed = parse(build_file, "", "BUILD_service_yaml_absolute_target.bazel")
+        self.assertEqual(
+            "google/cloud/videointelligence/videointelligence_v1p3beta1.yaml",
+            parsed.service_yaml,
+        )
+
+    def test_gapic_inputs_parse_no_service_yaml_returns_empty_string(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_no_service_yaml.bazel")
+        self.assertEqual("", parsed.service_yaml)
+
+    def test_gapic_inputs_parse_proto_only_returns_grpc(self):
+        parsed = parse(build_file, "test/versioned/path", "BUILD_proto_only.bazel")
+        self.assertEqual("grpc", parsed.transport)
diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py
@@ -35,6 +35,13 @@
     product_documentation="",
     gapic_configs=[],
 )
+common_protos_library = LibraryConfig(
+    api_shortname="common-protos",
+    api_description="",
+    name_pretty="",
+    product_documentation="",
+    gapic_configs=[],
+)
 
 
 class GenerationConfigTest(unittest.TestCase):
@@ -123,6 +130,26 @@ def test_is_monorepo_with_two_libraries_returns_true(self):
         )
         self.assertTrue(config.is_monorepo())
 
+    def test_contains_common_protos_with_common_protos_returns_true(self):
+        config = GenerationConfig(
+            gapic_generator_version="",
+            googleapis_commitish="",
+            libraries_bom_version="",
+            template_excludes=[],
+            libraries=[library_1, library_2, common_protos_library],
+        )
+        self.assertTrue(config.contains_common_protos())
+
+    def test_contains_common_protos_without_common_protos_returns_false(self):
+        config = GenerationConfig(
+            gapic_generator_version="",
+            googleapis_commitish="",
+            libraries_bom_version="",
+            template_excludes=[],
+            libraries=[library_1, library_2],
+        )
+        self.assertFalse(config.contains_common_protos())
+
     def test_validate_with_duplicate_library_name_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
@@ -131,8 +158,6 @@ def test_validate_with_duplicate_library_name_raise_exception(self):
             gapic_generator_version="",
             googleapis_commitish="",
             libraries_bom_version="",
-            owlbot_cli_image="",
-            synthtool_commitish="",
             template_excludes=[],
             libraries=[
                 LibraryConfig(
@@ -169,30 +194,6 @@ def test_from_yaml_without_googleapis_commitish_raise_exception(self):
             f"{test_config_dir}/config_without_googleapis.yaml",
         )
 
-    def test_from_yaml_without_libraries_bom_version_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, libraries_bom_version",
-            from_yaml,
-            f"{test_config_dir}/config_without_libraries_bom_version.yaml",
-        )
-
-    def test_from_yaml_without_owlbot_cli_image_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, owlbot_cli_image",
-            from_yaml,
-            f"{test_config_dir}/config_without_owlbot.yaml",
-        )
-
-    def test_from_yaml_without_synthtool_commitish_raise_exception(self):
-        self.assertRaisesRegex(
-            ValueError,
-            "Repo level parameter, synthtool_commitish",
-            from_yaml,
-            f"{test_config_dir}/config_without_synthtool.yaml",
-        )
-
     def test_from_yaml_without_template_excludes_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
diff --git a/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json b/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json
@@ -0,0 +1,17 @@
+{
+  "api_shortname": "baremetalsolution",
+  "name_pretty": "Bare Metal Solution",
+  "product_documentation": "https://cloud.google.com/bare-metal/docs",
+  "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.",
+  "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview",
+  "release_level": "preview",
+  "transport": "grpc",
+  "language": "java",
+  "repo": "googleapis/sdk-platform-java",
+  "repo_short": "java-bare-metal-solution",
+  "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution",
+  "library_type": "OTHER",
+  "requires_billing": true,
+  "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest",
+  "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc"
+}
diff --git a/library_generation/test/resources/misc/BUILD_proto_only.bazel b/library_generation/test/resources/misc/BUILD_proto_only.bazel
@@ -0,0 +1,16 @@
+java_gapic_assembly_gradle_pkg(
+    name = "google-api-java",
+    transport = "grpc+rest",
+    deps = [
+        "annotations_proto",
+        "auth_proto",
+        "backend_proto",
+        "billing_proto",
+        "client_proto",
+        "config_change_proto",
+        "consumer_proto",
+        "context_proto",
+        "control_proto",
+        "distribution_proto",
+    ],
+)
diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py
diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py
