stack: complete pre-compile stack

.github/workflows/ci.yml

@@ -74,6 +74,31 @@ jobs:
       python-version: ${{ matrix.python-version }}
       cmake-args: ${{ matrix.cmake-args }}
 
+  # Build the test suite against the precompiled pybind11 static library; a
+  # definition missing from the -inl.h split shows up here as a link error.
+  precompile:
+    if: github.event.pull_request.draft == false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runs-on: ubuntu-latest
+            python-version: '3.13'
+          # free-threaded: compiles the Py_GIL_DISABLED-only code in the -inl.h files
+          - runs-on: ubuntu-latest
+            python-version: '3.14t'
+          - runs-on: macos-latest
+            python-version: '3.13'
+          - runs-on: windows-2022
+            python-version: '3.13'
+
+    name: ⚡
+    uses: ./.github/workflows/reusable-standard.yml
+    with:
+      runs-on: ${{ matrix.runs-on }}
+      python-version: ${{ matrix.python-version }}
+      cmake-args: -DPYBIND11_TEST_PRECOMPILE=ON
+
   standard-large:
     if: github.event.pull_request.draft == false
     strategy:

.pre-commit-config.yaml

@@ -141,6 +141,11 @@ repos:
     language: pygrep
     entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest
     exclude: ^\.pre-commit-config.yaml$
+  - id: check-inl-headers
+    name: Check -inl.h files for the precompiled mode
+    language: python
+    entry: python tools/check_inl_headers.py
+    files: -inl\.h$
 
 # PyLint has native support - not always usable, but works for us
 - repo: https://github.com/PyCQA/pylint

CMakeLists.txt

@@ -190,20 +190,25 @@ endif()
 
 set(PYBIND11_HEADERS
     include/pybind11/detail/argument_vector.h
+    include/pybind11/detail/class-inl.h
     include/pybind11/detail/class.h
+    include/pybind11/detail/common-inl.h
     include/pybind11/detail/common.h
     include/pybind11/detail/cpp_conduit.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
+    include/pybind11/detail/exception_translation-inl.h
     include/pybind11/detail/exception_translation.h
     include/pybind11/detail/function_record_pyobject.h
     include/pybind11/detail/function_ref.h
     include/pybind11/detail/holder_caster_foreign_helpers.h
     include/pybind11/detail/init.h
+    include/pybind11/detail/internals-inl.h
     include/pybind11/detail/internals.h
     include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/pybind11_namespace_macros.h
     include/pybind11/detail/struct_smart_holder.h
+    include/pybind11/detail/type_caster_base-inl.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
     include/pybind11/detail/using_smart_holder.h
@@ -233,7 +238,9 @@ set(PYBIND11_HEADERS
     include/pybind11/native_enum.h
     include/pybind11/numpy.h
     include/pybind11/operators.h
+    include/pybind11/pybind11-inl.h
     include/pybind11/pybind11.h
+    include/pybind11/pytypes-inl.h
     include/pybind11/pytypes.h
     include/pybind11/subinterpreter.h
     include/pybind11/stl.h
@@ -265,6 +272,11 @@ endif()
 
 list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
+# Library sources for the opt-in precompiled mode (pybind11_precompile()).
+set(pybind11_SRC_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}/src"
+    CACHE INTERNAL "Directory containing the pybind11 library sources")
+
 # Cache variable so this can be used in parent projects
 set(pybind11_INCLUDE_DIR
     "${CMAKE_CURRENT_LIST_DIR}/include"
@@ -331,6 +343,8 @@ if(PYBIND11_INSTALL)
     install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}")
   endif()
   install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  set(pybind11_install_srcdir "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
+  install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/ DESTINATION "${pybind11_install_srcdir}")
   set(PYBIND11_CMAKECONFIG_INSTALL_DIR
       "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}"
       CACHE STRING "install path for pybind11Config.cmake")
@@ -340,6 +354,11 @@ if(PYBIND11_INSTALL)
   else()
     set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}")
   endif()
+  if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}")
+    set(pybind11_SRCDIR "${pybind11_install_srcdir}")
+  else()
+    set(pybind11_SRCDIR "\$\{PACKAGE_PREFIX_DIR\}/${pybind11_install_srcdir}")
+  endif()
 
   configure_package_config_file(
     tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
@@ -391,6 +410,7 @@ if(PYBIND11_INSTALL)
     endif()
   endif()
   join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
+  join_paths(srcdir_for_pc_file "\${prefix}" "${pybind11_install_srcdir}")
   configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"
                  "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"

docs/Doxyfile

@@ -1,6 +1,7 @@
 PROJECT_NAME           = pybind11
 INPUT                  = ../include/pybind11/
 RECURSIVE              = YES
+EXCLUDE_PATTERNS       = *-inl.h
 
 GENERATE_HTML          = NO
 GENERATE_LATEX         = NO

docs/compiling.rst

@@ -348,7 +348,8 @@ function with the following signature:
 .. code-block:: cmake
 
     pybind11_add_module(<name> [MODULE | SHARED] [EXCLUDE_FROM_ALL]
-                        [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] source1 [source2 ...])
+                        [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] [PRECOMPILE | NO_PRECOMPILE]
+                        source1 [source2 ...])
 
 This function behaves very much like CMake's builtin ``add_library`` (in fact,
 it's a wrapper function around that command). It will add a library target
@@ -404,6 +405,104 @@ optimizations remain disabled.
 
 .. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html
 
+.. _precompile-mode:
+
+Pre-compiling part of pybind11
+------------------------------
+
+pybind11 is header-only by default: every translation unit compiles its own
+copy of the non-template implementation. The opt-in *precompiled* mode
+compiles that implementation once, into a static library built inside your
+own project with your own flags. This reduces the build time, most of all for
+projects with many translation units or many modules in one build.
+
+.. code-block:: cmake
+
+    pybind11_add_module(example PRECOMPILE example.cpp)
+
+The first ``PRECOMPILE`` target creates the library target
+``pybind11::precompiled``; further targets reuse it. Set the CMake variable
+``PYBIND11_PRECOMPILE`` to make it the default for all
+``pybind11_add_module`` calls; use ``NO_PRECOMPILE`` on a target to opt back
+out. For targets you create yourself, call the ``pybind11_precompile()``
+function and link ``pybind11::precompiled`` PRIVATE; the target carries the
+required ``PYBIND11_PRECOMPILED`` compile definition PUBLIC, so your sources
+also get it.
+
+Requirements and caveats:
+
+* The library and every module linking it must agree on the configuration
+  macros ``PYBIND11_INTERNALS_VERSION``, ``Py_GIL_DISABLED``,
+  ``PYBIND11_SIMPLE_GIL_MANAGEMENT``,
+  ``PYBIND11_DETAILED_ERROR_MESSAGES`` (defaults on in debug builds),
+  ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, and
+  ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``. A mismatch produces one
+  readable undefined symbol at link time referencing
+  ``pybind11_precompiled_config``.
+* Configuration macros that only change code inside the library (for example
+  ``PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING``) must be defined when the
+  library is compiled; a definition only on your module has no effect. This
+  also applies to a custom ``PYBIND11_NAMESPACE`` visibility attribute: set
+  it at the directory level (``add_compile_definitions``) so that the library
+  gets it too.
+* The library picks up your directory-level flags and C++ standard when it is
+  first created, so set those before the first ``PRECOMPILE`` target. A
+  status message reports the directory that created the library.
+* The library is not compiled with link-time optimization, and the per-target
+  ``THIN_LTO`` and ``OPT_SIZE`` options of ``pybind11_add_module`` do not
+  apply to it. In a Release build this adds a few nanoseconds to each call of
+  a bound function (up to about 10% for a function that does nothing,
+  depending on the compiler; a few percent with LTO on the library). To change this, call ``pybind11_precompile()``
+  yourself and
+  set the properties on the created target, ``pybind11_precompiled`` (the
+  real target behind the ``pybind11::precompiled`` alias; CMake does not let
+  you set properties through an alias):
+
+  .. code-block:: cmake
+
+      pybind11_precompile()
+      set_target_properties(pybind11_precompiled PROPERTIES
+                            INTERPROCEDURAL_OPTIMIZATION ON)
+
+* The library is static and per-build-tree; it is never installed or shared
+  between projects. Each extension module links its own copy, which keeps
+  pybind11's per-module state the same as in header-only mode.
+* Not available with ``PYBIND11_NOPYTHON`` (the library needs Python
+  headers).
+
+For build systems other than CMake, the same sources ship with the pybind11
+package: compile ``pybind11_combined.cpp`` from the directory reported by
+``python -m pybind11 --srcdir`` (also available as
+``pybind11.get_source_dir()`` and the ``srcdir`` pkg-config variable) into
+a static library or into your extension, and define
+``PYBIND11_PRECOMPILED`` for every translation unit.
+
+With Meson, build the library once per build tree and link it into each
+extension module, the same as the CMake path:
+
+.. code-block:: meson
+
+    pybind11_dep = dependency('pybind11')
+    pybind11_src = run_command(py, ['-m', 'pybind11', '--srcdir'],
+                               check : true).stdout().strip()
+
+    pybind11_precompiled = static_library('pybind11_precompiled',
+        pybind11_src / 'pybind11_combined.cpp',
+        cpp_args : ['-DPYBIND11_PRECOMPILED'],
+        gnu_symbol_visibility : 'hidden',
+        dependencies : [pybind11_dep, py.dependency()])
+
+    py.extension_module('example', 'example.cpp',
+        cpp_args : ['-DPYBIND11_PRECOMPILED'],
+        link_with : pybind11_precompiled,
+        dependencies : [pybind11_dep])
+
+The configuration-macro rules above apply here too: the static library and
+every module that links it must be compiled with the same configuration
+macros, and ``-DPYBIND11_PRECOMPILED`` must appear in both ``cpp_args``
+lists. (``pybind11_dep.get_variable('srcdir')`` also reports the source
+directory when Meson finds pybind11 through pkg-config.)
+
 Configuration variables
 -----------------------
 

docs/faq.rst

@@ -79,7 +79,12 @@ and the binding code
 How can I reduce the build time?
 ================================
 
-It's good practice to split binding code over multiple files, as in the
+First, consider the opt-in precompiled mode: it compiles the non-template
+part of pybind11 once per project instead of once for each translation unit.
+In CMake, this is one keyword on ``pybind11_add_module``. See
+:ref:`precompile-mode`.
+
+It's also good practice to split binding code over multiple files, as in the
 following example:
 
 :file:`example.cpp`: