Give _tkinter $ORIGIN-relative dependencies on glibc and an rpath on musl #745
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -265,24 +265,32 @@ static ELF_ALLOWED_LIBRARIES_BY_TRIPLE: Lazy<HashMap<&'static str, Vec<&'static | |
| .collect() | ||
| }); | ||
|
|
||
| #[derive(Copy, Clone, PartialEq)] | ||
| enum DepSource { | ||
| SystemRequired, | ||
| SystemOptional, | ||
| Vendored, | ||
| } | ||
| use DepSource::*; | ||
|
|
||
| static ELF_ALLOWED_LIBRARIES_BY_MODULE: Lazy< | ||
| HashMap<&'static str, Vec<(&'static str, DepSource)>>, | ||
| > = Lazy::new(|| { | ||
| [ | ||
| ( | ||
| // libcrypt is provided by the system, but only on older distros. | ||
| "_crypt", | ||
| vec![("libcrypt.so.1", SystemOptional)], | ||
| ), | ||
| ( | ||
| "_tkinter", | ||
| vec![("libtcl8.6.so", Vendored), ("libtk8.6.so", Vendored)], | ||
| ), | ||
| ] | ||
| .iter() | ||
| .cloned() | ||
| .collect() | ||
| }); | ||
|
|
||
| static DARWIN_ALLOWED_DYLIBS: Lazy<Vec<MachOAllowedDylib>> = Lazy::new(|| { | ||
| [ | ||
|
|
@@ -1022,7 +1030,7 @@ fn validate_elf<Elf: FileHeader<Endian = Endianness>>( | |
| if let Some(filename) = path.file_name() { | ||
| if let Some((module, _)) = filename.to_string_lossy().split_once(".cpython-") { | ||
| if let Some(extra) = ELF_ALLOWED_LIBRARIES_BY_MODULE.get(module) { | ||
| allowed_libraries.extend(extra.iter().map(|x| x.0.to_string())); | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -2186,6 +2194,85 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result<Vec<String>> { | |
| errors.push("errors running interpreter tests".to_string()); | ||
| } | ||
|
|
||
| // Explicitly test ldd directly on the extension modules, which PyInstaller | ||
| // relies on. This is not strictly needed for a working distribution (e.g. | ||
| // you can set an rpath on just python+libpython), so we test here for | ||
| // compatibility with tools that run ldd. | ||
| // that fails this check (e.g. by setting an rpath on just python+libpython). | ||
| // https://github.com/pyinstaller/pyinstaller/issues/9204#issuecomment-3171050891 | ||
| // TODO(geofft): musl doesn't do lazy binding for the argument to | ||
| // ldd, so we will get complaints about missing Py_* symbols. Need | ||
| // to handle this somehow, skip testing for now. | ||
|
Comment on lines
+2203
to
+2205
There was a problem hiding this comment. You can just filter out those lines?: $ /lib/ld-musl-x86_64.so.1 --list /root/.local/share/uv/python/cpython-3.13.9-linux-x86_64-musl/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-musl.so 2>&1 | grep -v relocating
/lib/ld-musl-x86_64.so.1 (0x7453c8bed000)
Error loading shared library libtcl8.6.so: No such file or directory (needed by /root/.local/share/uv/python/cpython-3.13.9-linux-x86_64-musl/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-musl.so)
Error loading shared library libtk8.6.so: No such file or directory (needed by /root/.local/share/uv/python/cpython-3.13.9-linux-x86_64-musl/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-musl.so)
libc.so => /lib/ld-musl-x86_64.so.1 (0x7453c8bed000)$ patchelf --print-needed /root/.local/share/uv/python/cpython-3.13.9-linux-x86_64-musl/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-musl.so
libtcl8.6.so
libtk8.6.so
libc.so |
||
| if cfg!(target_os = "linux") && !python_json.target_triple.contains("-musl") { | ||
| // musl's ldd is packaged in the "musl-tools" Debian package. | ||
| let ldd = if python_json.target_triple.contains("-musl") && cfg!(not(target_env = "musl")) { | ||
| "musl-ldd" | ||
| } else { | ||
| "ldd" | ||
| }; | ||
| for (name, variants) in python_json.build_info.extensions.iter() { | ||
| for ext in variants { | ||
| let Some(shared_lib) = &ext.shared_lib else { | ||
| continue; | ||
| }; | ||
| let shared_lib_path = temp_dir.path().join("python").join(shared_lib); | ||
| let output = duct::cmd(ldd, [shared_lib_path]) | ||
| .unchecked() | ||
| .stdout_capture() | ||
| .run() | ||
| .context(format!("Failed to run `{ldd} {shared_lib}`"))?; | ||
| let stdout = String::from_utf8_lossy(&output.stdout); | ||
| // Format of ldd output, for both glibc and musl: | ||
| // - Everything starts with a tab. | ||
| // - Most things are "libxyz.so.1 => /usr/lib/libxyz.so.1 (0xabcde000)". | ||
| // - The ELF interpreter is displayed as just "/lib/ld.so (0xabcde000)". | ||
| // - glibc, but not musl, shows the vDSO as "linux-vdso.so.1 (0xfffff000)". | ||
|
Comment on lines
+2227
to
+2229
There was a problem hiding this comment. This is not true regarding glibc vs musl for vDSO, just look at the output I showed from Ubuntu and Fedora containers with Realistically, you should only be interested in direct dependencies? As per Anything transitive is resolved at runtime, thus |
||
| // - If a library is listed in DT_NEEDED with an absolute path, or (currently only | ||
| // supported on glibc) with an $ORIGIN-relative path, it displays as just | ||
| // "/path/to/libxyz.so (0xabcde000)". | ||
| // - On glibc, if a library cannot be found ldd returns zero and shows "=> not | ||
| // found" as the resolution (even if it wouldn't use the => form if found). | ||
| // - On musl, if a library cannot be found, ldd returns nonzero and shows "Error | ||
| // loading shared library ...:" on stderr. | ||
|
Comment on lines
+2233
to
+2236
There was a problem hiding this comment. In glibc you'll often find # `/usr/bin/ldd` is a bash script wrapper around this:
$ /lib64/ld-linux-x86-64.so.2 --list /lib64/libc.so.6
/lib64/ld-linux-x86-64.so.2 (0x00007f8ead016000)
linux-vdso.so.1 (0x00007fff355f6000)
# Failure example:
$ /lib64/ld-linux-x86-64.so.2 --list /root/.local/share/uv/python/cpython-3.13.6-linux-x86_64-gnu/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-gnu.so
/root/.local/share/uv/python/cpython-3.13.6-linux-x86_64-gnu/lib/python3.13/lib-dynload/_tkinter.cpython-313-x86_64-linux-gnu.so: error while loading shared libraries: libtcl8.6.so: cannot open shared object file: No such file or directoryAs you can see the error message differs from the Alpine has a very simple one for musl: $ docker run --rm -it alpine
$ cat /usr/bin/ldd
#!/bin/sh
exec /lib/ld-musl-x86_64.so.1 --list "$@"For testing that validates direct deps of a library/executable are found, you should be able to use |
||
| if !output.status.success() { | ||
| // TODO: If we ever have any optional dependencies besides libcrypt (which is | ||
| // glibc-only), we will need to capture musl ldd's stderr and parse it. | ||
| errors.push(format!( | ||
| "`{ldd} {shared_lib}` exited with {}:\n{stdout}", | ||
| output.status | ||
| )); | ||
|
Comment on lines
+2237
to
+2243
There was a problem hiding this comment. Perhaps instead of relying on Might be more consistent via a crate library to implement this logic for, without the caveats of handling different program outputs? There was a problem hiding this comment. There's a related crate for |
||
| } else { | ||
| let mut ldd_errors = vec![]; | ||
| let deps = ELF_ALLOWED_LIBRARIES_BY_MODULE.get(&name[..]); | ||
| let temp_dir_lossy = temp_dir.path().to_string_lossy().into_owned(); | ||
| for line in stdout.lines() { | ||
| let Some((needed, resolution)) = line.trim().split_once(" => ") else { | ||
| continue; | ||
| }; | ||
| let dep_source = deps | ||
| .and_then(|deps| { | ||
| deps.iter().find(|dep| dep.0 == needed).map(|dep| dep.1) | ||
| }) | ||
| .unwrap_or(SystemRequired); | ||
| if resolution.starts_with("not found") && dep_source != SystemOptional { | ||
| ldd_errors.push(format!("{needed} was expected to be found")); | ||
| } else if !resolution.contains(&temp_dir_lossy) && dep_source == Vendored { | ||
| ldd_errors.push(format!( | ||
| "{needed} should not come from the OS (missing rpath/$ORIGIN?)" | ||
| )); | ||
| } | ||
| } | ||
| if !ldd_errors.is_empty() { | ||
| errors.push(format!( | ||
| "In `{ldd} {shared_lib}`:\n - {}\n{stdout}", | ||
| ldd_errors.join("\n - ") | ||
| )); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Ok(errors) | ||
| } | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this actually necessary? (Fedora 42 container)
You can see that just setting the RPath was sufficient (no need to be more strict with path prefixes to deps /
DT_NEEDED). SettingDT_RUNPATHprovides additional search path(s) that will be used in addition to whateverldconfig --print-cache(for glibc; musl equivalent differs) +LD_LIBRARY_PATHprovide on the system (at runtime resolution).Both of these runtime configs can also be used to satisfy PyInstaller's discovery of libraries (and PyInstaller's executable it distributes will set
LD_LIBRARY_PATHfor it's process accordingly at runtime to where it's bundled libs are), although the relative RPath is a better approach (and remains compatible).There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For reference (Ubuntu 24.04 container), this is what PyTorch does as well (only adding extra search paths via RPath):