Add option to bundle external libraries

Signed-off-by: Cristian Le <[email protected]>

Author: LecrisUT

README.md

@@ -229,6 +229,9 @@ wheel.repair.in-wheel = true
 # Patch the dynamic links to libraries in other wheels.
 wheel.repair.cross-wheel = false
 
+# A list of external library files that will be bundled in the wheel.
+wheel.repair.bundle-external = []
+
 # If CMake is less than this value, backport a copy of FindPython.
 backport.find-python = "3.26.1"
 

docs/reference/configs.md

@@ -576,6 +576,17 @@ print(mk_skbuild_docs())
 
 ## wheel.repair
 
+```{eval-rst}
+.. confval:: wheel.repair.bundle-external
+  :type: ``list[str]``
+
+  A list of external library files that will be bundled in the wheel.
+
+  Each entry is treated as a regex pattern, and only the filenames are considered
+  for the match. The libraries are taken from the CMake dependency during the CMake
+  build. The bundled libraries are installed under ``site-packages/${name}.libs``
+```
+
 ```{eval-rst}
 .. confval:: wheel.repair.cross-wheel
   :type: ``bool``

src/scikit_build_core/repair_wheel/__init__.py

@@ -8,6 +8,8 @@
 import functools
 import os
 import platform
+import re
+import shutil
 import sysconfig
 from abc import ABC, abstractmethod
 from pathlib import Path
@@ -99,6 +101,28 @@ def __init_subclass__(cls) -> None:
         if cls._platform:
             WheelRepairer._platform_repairers[cls._platform] = cls
 
+    @functools.cached_property
+    def bundled_libs_path(self) -> Path:
+        """Staging path for the bundled library directory."""
+        return Path(self.wheel_dirs["platlib"]) / f"{self.name}.libs"
+
+    @functools.cached_property
+    def bundle_external(self) -> list[re.Pattern[str]]:
+        """List of compiled regex patterns of the library files to bundle."""
+        patterns = []
+        for pattern_str in self.settings.wheel.repair.bundle_external:
+            try:
+                pattern = re.compile(pattern_str)
+            except re.error as exc:
+                logger.warning(
+                    'Skipping "{pattern}" as an invalid pattern',
+                    pattern=pattern_str,
+                )
+                logger.debug(str(exc))
+                continue
+            patterns.append(pattern)
+        return patterns
+
     @functools.cached_property
     def configuration(self) -> Configuration:
         """Current file-api configuration."""
@@ -204,8 +228,91 @@ def get_library_dependencies(self, target: Target) -> list[Target]:
             dependencies.append(dep_target)
         return dependencies
 
+    def try_bundle(self, external_lib: Path) -> Path | None:
+        """
+        Try to bundle an external library file.
+
+        :param external_lib: path to actual external library to bundle
+        :returns: ``None`` if the library is not bundled, otherwise the path
+          to the bundled file
+        """
+        assert external_lib.is_absolute()
+        if not external_lib.exists():
+            logger.warning(
+                "External library file does not exist: {external_lib}",
+                external_lib=external_lib,
+            )
+            return None
+        if external_lib.is_dir():
+            logger.debug(
+                "Skip bundling directory: {external_lib}",
+                external_lib=external_lib,
+            )
+            return None
+        libname = external_lib.name
+        bundled_lib = self.bundled_libs_path / libname
+        if bundled_lib.exists():
+            # If we have already bundled the library no need to do it again
+            return bundled_lib
+        for pattern in self.bundle_external:
+            if pattern.match(libname):
+                logger.debug(
+                    'Bundling library matching "{pattern}": {external_lib}',
+                    external_lib=external_lib,
+                    pattern=pattern.pattern,
+                )
+                shutil.copy(external_lib, bundled_lib)
+                return bundled_lib
+        logger.debug(
+            "Skip bundling: {external_lib}",
+            external_lib=external_lib,
+        )
+        return None
+
+    def get_package_lib_path(
+        self, original_lib: Path, relative_to: Path | None = None
+    ) -> Path | None:
+        """
+        Get the file path of a library to be used.
+
+        This checks for the settings in ``settings.wheel.repair`` returning either:
+         - If the dependency should be skipped: ``None``
+         - If ``original_lib`` is a library in another wheel: a relative path to the original library file
+         - If ``original_lib`` is a library to be bundled: a relative path to the bundled library file
+
+        The relative paths are relative to ``relative_to`` or the ``platlib`` wheel path if not passed.
+        """
+        if not original_lib.is_absolute() or not original_lib.exists():
+            logger.debug(
+                "Could not handle {original_lib} because it is either relative or does not exist.",
+                original_lib=original_lib,
+            )
+            return None
+        if self.path_is_in_site_packages(original_lib):
+            # The other library is in another wheel
+            if not self.settings.wheel.repair.cross_wheel:
+                logger.debug(
+                    "Skipping {original_lib} because it is in another wheel.",
+                    original_lib=original_lib,
+                )
+                return None
+            final_lib = original_lib
+        # Otherwise, check if we need to bundle the external library
+        elif not self.bundle_external or not (
+            final_lib := self.try_bundle(original_lib)  # type: ignore[assignment]
+        ):
+            logger.debug(
+                "Skipping {original_lib} because it is not being bundled.",
+                original_lib=original_lib,
+            )
+            return None
+        return self.path_relative_site_packages(final_lib, relative_to=relative_to)
+
     def repair_wheel(self) -> None:
         """Repair the current wheel."""
+        if self.bundle_external:
+            self.bundled_libs_path.mkdir(exist_ok=True)
+
         for target in self.targets:
             if self._filter_targets:
                 if target.type == "STATIC_LIBRARY":

src/scikit_build_core/repair_wheel/rpath.py

@@ -91,31 +91,40 @@ def get_package_rpaths(self, target: Target, install_path: Path) -> list[str]:
                             # Skip empty rpaths. Most likely will have on at the end
                             continue
                         rpath = Path(rpath_str)
-                        if not self.path_is_in_site_packages(rpath):
-                            # Skip any paths that cannot be handled. We do not check for paths in
-                            # the build directory, it should be covered by `get_dependency_rpaths`
+                        # Relative paths should be covered by `get_dependency_rpaths` so we skip them.
+                        if not rpath.is_absolute():
+                            continue
+                        # Get the relative rpath to either the cross-wheel or bundled file
+                        if not (
+                            rpath := self.get_package_lib_path(  # type: ignore[assignment]
+                                rpath, relative_to=install_path
+                            )
+                        ):
                             continue
-                        rpath = self.path_relative_site_packages(rpath, install_path)
                         new_rpath_str = f"{self._origin_symbol}/{rpath}"
                         rpaths.append(new_rpath_str)
                     continue
                 # The remaining case should be a path
                 try:
                     # TODO: how to best catch if a string is a valid path?
                     rpath = Path(link_part)
-                    if not rpath.is_absolute():
-                        # Relative paths should be handled by `get_dependency_rpaths`
-                        continue
-                    rpath = self.path_relative_site_packages(rpath, install_path)
-                    new_rpath_str = f"{self._origin_symbol}/{rpath.parent}"
-                    rpaths.append(new_rpath_str)
                 except Exception as exc:
                     logger.warning(
                         "Could not parse link-library as a path: {fragment}\nexc = {exc}",
                         fragment=link_command.fragment,
                         exc=exc,
                     )
                     continue
+                if not rpath.is_absolute():
+                    # Relative paths should be covered by `get_dependency_rpaths` so we skip them.
+                    continue
+                # Get the relative rpath to either the cross-wheel or bundled file
+                if not (
+                    rpath := self.get_package_lib_path(rpath, relative_to=install_path)  # type: ignore[assignment]
+                ):
+                    continue
+                new_rpath_str = f"{self._origin_symbol}/{rpath.parent}"
+                rpaths.append(new_rpath_str)
         return rpaths
 
     def get_existing_rpaths(self, artifact: Path) -> list[str]:
@@ -168,10 +177,7 @@ def patch_target(self, target: Target) -> None:
                 dependency_rpaths = self.get_dependency_rpaths(target, install_path)
             else:
                 dependency_rpaths = []
-            if self.settings.wheel.repair.cross_wheel:
-                package_rpaths = self.get_package_rpaths(target, install_path)
-            else:
-                package_rpaths = []
+            package_rpaths = self.get_package_rpaths(target, install_path)
             existing_rpaths = self.get_existing_rpaths(artifact_path)
             logger.debug(
                 "Patching rpaths for artifact {artifact}\n"

src/scikit_build_core/repair_wheel/windows.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 import dataclasses
+import functools
 import os.path
 import textwrap
 from pathlib import Path
@@ -14,6 +15,8 @@
 from . import WheelRepairer, _get_buildenv_platlib
 
 if TYPE_CHECKING:
+    import re
+
     from ..file_api.model.codemodel import Target
 
 __all__ = ["WindowsWheelRepairer"]
@@ -52,8 +55,20 @@ def _skbuild_patch_dll_dir():
     dll_dirs: set[Path] = dataclasses.field(default_factory=set, init=False)
     """All dll paths used relative to ``platlib``."""
 
+    @functools.cached_property
+    def bundle_external(self) -> list[re.Pattern[str]]:
+        if self.settings.wheel.repair.bundle_external:
+            logger.warning("Bundling Windows dll files is not supported yet.")
+        return []
+
+    def try_bundle(self, external_lib: Path) -> Path | None:
+        # Everything should be gated by `bundle_external` so this should not be called
+        # TODO: figure out a better way to find the corresponding dll file of the linked lib file
+        raise NotImplementedError
+
     def get_dll_path_from_lib(self, lib_path: Path) -> Path | None:
         """Guess the dll path from lib path."""
+        # TODO: rework the logic of this to work with `try_bundle`
         dll_path = None
         platlib = Path(_get_buildenv_platlib())
         lib_path = lib_path.relative_to(platlib)
@@ -180,32 +195,27 @@ def get_package_dll(self, target: Target) -> list[Path]:
             try:
                 # TODO: how to best catch if a string is a valid path?
                 lib_path = Path(link_command.fragment)
-                if not lib_path.is_absolute():
-                    # If the link_command is a space-separated list of libraries, this should be skipped
-                    logger.debug(
-                        "Skipping non-absolute-path library: {fragment}",
-                        fragment=link_command.fragment,
-                    )
-                    continue
-                try:
-                    self.path_relative_site_packages(lib_path)
-                except ValueError:
-                    logger.debug(
-                        "Skipping library outside site-package path: {lib_path}",
-                        lib_path=lib_path,
-                    )
-                    continue
-                dll_path = self.get_dll_path_from_lib(lib_path)
-                if not dll_path:
-                    continue
-                dll_paths.append(dll_path.parent)
             except Exception as exc:
                 logger.warning(
                     "Could not parse link-library as a path: {fragment}\nexc = {exc}",
                     fragment=link_command.fragment,
                     exc=exc,
                 )
                 continue
+            if not lib_path.is_absolute():
+                # If the link_command is a space-separated list of libraries, this should be skipped
+                logger.debug(
+                    "Skipping non-absolute-path library: {fragment}",
+                    fragment=link_command.fragment,
+                )
+                continue
+            # TODO: Handle this better when revisiting `try_bundle`
+            if not self.get_package_lib_path(lib_path):
+                continue
+            dll_path = self.get_dll_path_from_lib(lib_path)
+            if not dll_path:
+                continue
+            dll_paths.append(dll_path.parent)
         return dll_paths
 
     def patch_target(self, target: Target) -> None:
@@ -214,10 +224,7 @@ def patch_target(self, target: Target) -> None:
             dependency_dlls = self.get_dependency_dll(target)
         else:
             dependency_dlls = []
-        if self.settings.wheel.repair.cross_wheel:
-            package_dlls = self.get_package_dll(target)
-        else:
-            package_dlls = []
+        package_dlls = self.get_package_dll(target)
 
         if not package_dlls and not dependency_dlls:
             logger.warning(

src/scikit_build_core/resources/scikit-build.schema.json

@@ -258,6 +258,13 @@
               "type": "boolean",
               "default": false,
               "description": "Patch the dynamic links to libraries in other wheels."
+            },
+            "bundle-external": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "A list of external library files that will be bundled in the wheel."
             }
           },
           "description": "Wheel repair options"

src/scikit_build_core/settings/skbuild_model.py

@@ -247,6 +247,15 @@ class WheelRepair:
        not required.
     """
 
+    bundle_external: List[str] = dataclasses.field(default_factory=list)
+    """
+    A list of external library files that will be bundled in the wheel.
+
+    Each entry is treated as a regex pattern, and only the filenames are considered
+    for the match. The libraries are taken from the CMake dependency during the CMake
+    build. The bundled libraries are installed under ``site-packages/${name}.libs``
+    """
+
 
 @dataclasses.dataclass
 class WheelSettings: