chore: improve build info extraction for Python packages (#1230)

Improve the extraction and handling of build information for Python packages for build spec generation. This update refines how metadata is gathered and fixes the default command for pure Python packages.

Signed-off-by: behnazh-w <[email protected]>

Author: behnazh-w

src/macaron/build_spec_generator/build_command_patcher.py

@@ -26,6 +26,45 @@
 
 PatchValueType = GradleOptionPatchValueType | MavenOptionPatchValueType
 
+CLI_COMMAND_PATCHES: dict[
+    PatchCommandBuildTool,
+    Mapping[str, PatchValueType | None],
+] = {
+    PatchCommandBuildTool.MAVEN: {
+        "goals": ["clean", "package"],
+        "--batch-mode": False,
+        "--quiet": False,
+        "--no-transfer-progress": False,
+        # Example pkg:maven/io.liftwizard/[email protected]
+        # https://github.com/liftwizard/liftwizard/blob/
+        # 4ea841ffc9335b22a28a7a19f9156e8ba5820027/.github/workflows/build-and-test.yml#L23
+        "--threads": None,
+        # For cases such as
+        # pkg:maven/org.apache.isis.valuetypes/[email protected]
+        "--version": False,
+        "--define": {
+            # pkg:maven/org.owasp/[email protected]
+            # To remove "-Dgpg.passphrase=$MACARON_UNKNOWN"
+            "gpg.passphrase": None,
+            "skipTests": "true",
+            "maven.test.skip": "true",
+            "maven.site.skip": "true",
+            "rat.skip": "true",
+            "maven.javadoc.skip": "true",
+        },
+    },
+    PatchCommandBuildTool.GRADLE: {
+        "tasks": ["clean", "assemble"],
+        "--console": "plain",
+        "--exclude-task": ["test"],
+        "--project-prop": {
+            "skip.signing": "",
+            "skipSigning": "",
+            "gnupg.skip": "",
+        },
+    },
+}
+
 
 def _patch_commands(
     cmds_sequence: Sequence[list[str]],

src/macaron/build_spec_generator/build_spec_generator.py

@@ -6,14 +6,12 @@
 import json
 import logging
 import os
-from collections.abc import Mapping
 from enum import Enum
 
 from packageurl import PackageURL
 from sqlalchemy import create_engine
 from sqlalchemy.orm import Session
 
-from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType
 from macaron.build_spec_generator.common_spec.core import gen_generic_build_spec
 from macaron.build_spec_generator.reproducible_central.reproducible_central import gen_reproducible_central_build_spec
 from macaron.console import access_handler
@@ -31,46 +29,6 @@ class BuildSpecFormat(str, Enum):
     DEFAULT = "default-buildspec"
 
 
-CLI_COMMAND_PATCHES: dict[
-    PatchCommandBuildTool,
-    Mapping[str, PatchValueType | None],
-] = {
-    PatchCommandBuildTool.MAVEN: {
-        "goals": ["clean", "package"],
-        "--batch-mode": False,
-        "--quiet": False,
-        "--no-transfer-progress": False,
-        # Example pkg:maven/io.liftwizard/[email protected]
-        # https://github.com/liftwizard/liftwizard/blob/
-        # 4ea841ffc9335b22a28a7a19f9156e8ba5820027/.github/workflows/build-and-test.yml#L23
-        "--threads": None,
-        # For cases such as
-        # pkg:maven/org.apache.isis.valuetypes/[email protected]
-        "--version": False,
-        "--define": {
-            # pkg:maven/org.owasp/[email protected]
-            # To remove "-Dgpg.passphrase=$MACARON_UNKNOWN"
-            "gpg.passphrase": None,
-            "skipTests": "true",
-            "maven.test.skip": "true",
-            "maven.site.skip": "true",
-            "rat.skip": "true",
-            "maven.javadoc.skip": "true",
-        },
-    },
-    PatchCommandBuildTool.GRADLE: {
-        "tasks": ["clean", "assemble"],
-        "--console": "plain",
-        "--exclude-task": ["test"],
-        "--project-prop": {
-            "skip.signing": "",
-            "skipSigning": "",
-            "gnupg.skip": "",
-        },
-    },
-}
-
-
 def gen_build_spec_for_purl(
     purl: PackageURL,
     database_path: str,
@@ -113,7 +71,7 @@ def gen_build_spec_for_purl(
 
     with Session(db_engine) as session, session.begin():
         try:
-            build_spec = gen_generic_build_spec(purl=purl, session=session, patches=CLI_COMMAND_PATCHES)
+            build_spec = gen_generic_build_spec(purl=purl, session=session)
         except GenerateBuildSpecError as error:
             logger.error("Error while generating the build spec: %s.", error)
             return os.EX_DATAERR
@@ -128,7 +86,7 @@ def gen_build_spec_for_purl(
             # Default build spec.
             case BuildSpecFormat.DEFAULT:
                 try:
-                    build_spec_content = json.dumps(build_spec)
+                    build_spec_content = json.dumps(build_spec, indent=4)
                 except ValueError as error:
                     logger.error("Error while serializing the build spec: %s.", error)
                     return os.EX_DATAERR

src/macaron/build_spec_generator/common_spec/base_spec.py

@@ -92,3 +92,26 @@ def resolve_fields(self, purl: PackageURL) -> None:
         This method should be implemented by subclasses to handle
         logic specific to a given package ecosystem, such as Maven or PyPI.
         """
+
+    @abstractmethod
+    def get_default_build_command(
+        self,
+        build_tool_name: str,
+    ) -> list[str]:
+        """Return a default build command for the build tool.
+
+        Parameters
+        ----------
+        build_tool_name: str
+            The build tool to get the default build command.
+
+        Returns
+        -------
+        list[str]
+            The build command as a list[str].
+
+        Raises
+        ------
+        GenerateBuildSpecError
+            If there is no default build command available for the specified build tool.
+        """

src/macaron/build_spec_generator/common_spec/core.py

@@ -6,15 +6,13 @@
 import logging
 import pprint
 import shlex
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from enum import Enum
 from importlib import metadata as importlib_metadata
-from pprint import pformat
 
 import sqlalchemy.orm
 from packageurl import PackageURL
 
-from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType, patch_commands
 from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict
 from macaron.build_spec_generator.common_spec.maven_spec import MavenBuildSpec
 from macaron.build_spec_generator.common_spec.pypi_spec import PyPIBuildSpec
@@ -119,45 +117,6 @@ def compose_shell_commands(cmds_sequence: list[list[str]]) -> str:
     return result
 
 
-def get_default_build_command(
-    build_tool_name: MacaronBuildToolName,
-) -> list[str] | None:
-    """Return a default build command for the build tool.
-
-    Parameters
-    ----------
-    build_tool_name: MacaronBuildToolName
-        The type of build tool to get the default build command.
-
-    Returns
-    -------
-    list[str] | None
-        The build command as a list[str] or None if we cannot get one for this tool.
-    """
-    default_build_command = None
-
-    match build_tool_name:
-        case MacaronBuildToolName.MAVEN:
-            default_build_command = "mvn clean package".split()
-        case MacaronBuildToolName.GRADLE:
-            default_build_command = "./gradlew clean assemble publishToMavenLocal".split()
-        case MacaronBuildToolName.PIP:
-            default_build_command = "python -m build".split()
-        case MacaronBuildToolName.POETRY:
-            default_build_command = "poetry build".split()
-        case _:
-            pass
-
-    if not default_build_command:
-        logger.critical(
-            "There is no default build command available for the build tool %s.",
-            build_tool_name,
-        )
-        return None
-
-    return default_build_command
-
-
 def get_macaron_build_tool_name(
     build_tool_facts: Sequence[BuildToolFacts], target_language: str
 ) -> MacaronBuildToolName | None:
@@ -322,10 +281,6 @@ def get_language_version(
 def gen_generic_build_spec(
     purl: PackageURL,
     session: sqlalchemy.orm.Session,
-    patches: Mapping[
-        PatchCommandBuildTool,
-        Mapping[str, PatchValueType | None],
-    ],
 ) -> BaseBuildSpecDict:
     """
     Generate and return the Buildspec file.
@@ -336,9 +291,6 @@ def gen_generic_build_spec(
         The PackageURL to generate build spec for.
     session : sqlalchemy.orm.Session
         The SQLAlchemy Session opened for the database to extract build information.
-    patches : Mapping[PatchCommandBuildTool, Mapping[str, PatchValueType | None]]
-        The patches to apply to the build commands in ``build_info`` before being populated in
-        the output Buildspec.
 
     Returns
     -------
@@ -360,12 +312,6 @@ def gen_generic_build_spec(
             f"PURL type '{purl.type}' is not supported. Supported: {[e.name.lower() for e in ECOSYSTEMS]}"
         )
 
-    logger.debug(
-        "Generating build spec for %s with command patches:\n%s",
-        purl,
-        pformat(patches),
-    )
-
     target_language = LANGUAGES[purl.type.upper()].value
     group = purl.namespace
     artifact = purl.name
@@ -414,22 +360,7 @@ def gen_generic_build_spec(
         build_command_info or "Cannot find any.",
     )
 
-    selected_build_command = (
-        build_command_info.command
-        if build_command_info
-        else get_default_build_command(
-            build_tool_name,
-        )
-    )
-    if not selected_build_command:
-        raise GenerateBuildSpecError(f"Failed to get a build command for {purl}.")
-
-    patched_build_commands = patch_commands(
-        cmds_sequence=[selected_build_command],
-        patches=patches,
-    )
-    if not patched_build_commands:
-        raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_command}.")
+    selected_build_command = build_command_info.command if build_command_info else []
 
     lang_version = get_language_version(build_command_info) if build_command_info else ""
 
@@ -447,7 +378,7 @@ def gen_generic_build_spec(
             "purl": str(purl),
             "language": target_language,
             "build_tool": build_tool_name.value,
-            "build_commands": patched_build_commands,
+            "build_commands": [selected_build_command],
         }
     )
     ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl)

src/macaron/build_spec_generator/common_spec/maven_spec.py

@@ -8,9 +8,11 @@
 
 from packageurl import PackageURL
 
+from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands
 from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict
 from macaron.build_spec_generator.common_spec.jdk_finder import find_jdk_version_from_central_maven_repo
 from macaron.build_spec_generator.common_spec.jdk_version_normalizer import normalize_jdk_version
+from macaron.errors import GenerateBuildSpecError
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -29,6 +31,46 @@ def __init__(self, data: BaseBuildSpecDict):
         """
         self.data = data
 
+    def get_default_build_command(
+        self,
+        build_tool_name: str,
+    ) -> list[str]:
+        """Return a default build command for the build tool.
+
+        Parameters
+        ----------
+        build_tool_name: str
+            The build tool to get the default build command.
+
+        Returns
+        -------
+        list[str]
+            The build command as a list[str].
+
+        Raises
+        ------
+        GenerateBuildSpecError
+            If there is no default build command available for the specified build tool.
+        """
+        default_build_command = None
+
+        match build_tool_name:
+            case "maven":
+                default_build_command = "mvn clean package".split()
+            case "gradle":
+                default_build_command = "./gradlew clean assemble publishToMavenLocal".split()
+            case _:
+                pass
+
+        if not default_build_command:
+            logger.critical(
+                "There is no default build command available for the build tool %s.",
+                build_tool_name,
+            )
+            raise GenerateBuildSpecError("Unable to find a default build command.")
+
+        return default_build_command
+
     def resolve_fields(self, purl: PackageURL) -> None:
         """
         Resolve Maven-specific fields in the build specification.
@@ -69,3 +111,17 @@ def resolve_fields(self, purl: PackageURL) -> None:
             return
 
         self.data["language_version"] = [major_jdk_version]
+
+        # Resolve and patch build commands.
+        selected_build_commands = self.data["build_commands"] or [
+            self.get_default_build_command(self.data["build_tool"])
+        ]
+
+        patched_build_commands = patch_commands(
+            cmds_sequence=selected_build_commands,
+            patches=CLI_COMMAND_PATCHES,
+        )
+        if not patched_build_commands:
+            raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.")
+
+        self.data["build_commands"] = patched_build_commands