Upgrade xz from 5.2.12 -> 5.8.1

LICENSE.liblzma.txt

@@ -1 +1,13 @@
-liblzma is in the public domain.
+Copyright (C) The XZ Utils authors and contributors
+
+Permission to use, copy, modify, and/or distribute this
+software for any purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

cpython-unix/build-xz.sh

@@ -13,6 +13,15 @@ tar -xf xz-${XZ_VERSION}.tar.gz
 
 pushd xz-${XZ_VERSION}
 
+EXTRA_CONFIGURE_FLAGS=
+
+# musl-clang injects flags that are not used during compilation,
+# e.g. -fuse-ld=musl-clang. These raise warnings that can be ignored but
+# cause the -Werror check to fail. Skip the check.
+if [ "${CC}" = "musl-clang" ]; then
+    EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} SKIP_WERROR_CHECK=yes"
+fi
+
 CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CCASFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \
     --build=${BUILD_TRIPLE} \
     --host=${TARGET_TRIPLE} \
@@ -23,7 +32,8 @@ CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CC
     --disable-lzmadec \
     --disable-lzmainfo \
     --disable-lzma-links \
-    --disable-scripts
+    --disable-scripts \
+    ${EXTRA_CONFIGURE_FLAGS}
 
 make -j ${NUM_CPUS}
 make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out

cpython-windows/build.py

@@ -598,23 +598,40 @@ def hack_project_files(
     with sqlite3_path.open("wb") as fh:
         fh.write(data)
 
-    # Our version of the xz sources is newer than what's in cpython-source-deps
-    # and the xz sources changed the path to config.h. Hack the project file
+    # Our version of the xz sources may be newer than what's in cpython-source-deps.
+    # The source files and locations may have changed. Hack the project file
     # accordingly.
     #
-    # ... but CPython finally upgraded liblzma in 2022, so newer CPython releases
-    # already have this patch. So we're phasing it out.
+    # CPython updates xz occasionally. When these changes make it into a release
+    # these modification to the project file are not needed.
+    # The most recent change was an update to version 5.8.1:
+    # https://github.com/python/cpython/pull/141022
     try:
         liblzma_path = pcbuild_path / "liblzma.vcxproj"
         static_replace_in_file(
             liblzma_path,
+            rb"$(lzmaDir)windows/vs2019;$(lzmaDir)src/liblzma/common;",
             rb"$(lzmaDir)windows;$(lzmaDir)src/liblzma/common;",
-            rb"$(lzmaDir)windows\vs2019;$(lzmaDir)src/liblzma/common;",
         )
         static_replace_in_file(
             liblzma_path,
-            rb'<ClInclude Include="$(lzmaDir)windows\config.h" />',
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc32_fast.c" />\r\n    <ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc32_table.c" />\r\n',
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc32_fast.c" />\r\n ',
+        )
+        static_replace_in_file(
+            liblzma_path,
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc64_fast.c" />\r\n    <ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc64_table.c" />\r\n',
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\check\\crc64_fast.c" />\r\n ',
+        )
+        static_replace_in_file(
+            liblzma_path,
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\simple\\arm.c" />',
+            b'<ClCompile Include="$(lzmaDir)src\\liblzma\\simple\\arm.c" />\r\n    <ClCompile Include="$(lzmaDir)src\\liblzma\\simple\\arm64.c" />',
+        )
+        static_replace_in_file(
+            liblzma_path,
             rb'<ClInclude Include="$(lzmaDir)windows\vs2019\config.h" />',
+            rb'<ClInclude Include="$(lzmaDir)windows\config.h" />',
         )
     except NoSearchStringError:
         pass
@@ -1412,6 +1429,15 @@ def build_cpython(
             for f in fs:
                 f.result()
 
+        # Copy the config.h file used by upstream CPython for xz 5.8.1
+        # https://github.com/python/cpython-source-deps/blob/665d407bd6bc941944db2152e4b5dca388ea586e/windows/config.h
+        xz_version = DOWNLOADS["xz"]["version"]
+        xz_path = td / ("xz-%s" % xz_version)
+        config_src = SUPPORT / "xz-support" / "config.h"
+        config_dest = xz_path / "windows" / "config.h"
+        log(f"copying {config_src} to {config_dest}")
+        shutil.copyfile(config_src, config_dest)
+
         extract_tar_to_directory(libffi_archive, td)
 
         # We need all the OpenSSL library files in the same directory to appease

cpython-windows/xz-support/README

@@ -0,0 +1,8 @@
+The upstream xz sources requires cmake to build on windows.
+This can be avoided by extracting a config.h file extracted from the CMake's 
+results, as is done by CPython.
+This file may need to be updated when upgrading the xz version.
+The file in this directory is taken from the xz branch of 
+https://github.com/python/cpython-source-deps.
+Specifically:
+https://github.com/python/cpython-source-deps/blob/665d407bd6bc941944db2152e4b5dca388ea586e/windows/config.h

cpython-windows/xz-support/config.h

@@ -0,0 +1,67 @@
+/* Configuration extracted from CMake'd project files.
+
+This is used by CPython, and is not part of the regular xz release.
+*/
+
+#define HAVE_CHECK_CRC32 1
+#define HAVE_CHECK_CRC64 1
+#define HAVE_CHECK_SHA256 1
+
+#define HAVE_DECODERS 1
+#define HAVE_DECODER_ARM 1
+#define HAVE_DECODER_ARM64 1
+#define HAVE_DECODER_ARMTHUMB 1
+#define HAVE_DECODER_DELTA 1
+#define HAVE_DECODER_IA64 1
+#define HAVE_DECODER_POWERPC 1
+#define HAVE_DECODER_LZMA1 1
+#define HAVE_DECODER_LZMA2 1
+#define HAVE_DECODER_SPARC 1
+#define HAVE_DECODER_X86 1
+
+#define HAVE_ENCODERS 1
+#define HAVE_ENCODER_ARM 1
+#define HAVE_ENCODER_ARM64 1
+#define HAVE_ENCODER_ARMTHUMB 1
+#define HAVE_ENCODER_DELTA 1
+#define HAVE_ENCODER_IA64 1
+#define HAVE_ENCODER_POWERPC 1
+#define HAVE_ENCODER_LZMA1 1
+#define HAVE_ENCODER_LZMA2 1
+#define HAVE_ENCODER_SPARC 1
+#define HAVE_ENCODER_X86 1
+
+#if defined(_M_ARM64)
+
+#undef HAVE_IMMINTRIN_H
+#undef HAVE_USABLE_CLMUL
+
+#else
+
+#define HAVE_IMMINTRIN_H 1
+#define HAVE_USABLE_CLMUL 1
+#define HAVE__MM_MOVEMASK_EPI8 1
+#define TUKLIB_FAST_UNALIGNED_ACCESS 1
+
+#endif
+
+#define HAVE___BUILTIN_ASSUME_ALIGNED 1
+#define HAVE__BOOL 1
+
+#define HAVE_INTTYPES_H 1
+#define HAVE_MF_BT2 1
+#define HAVE_MF_BT3 1
+#define HAVE_MF_BT4 1
+#define HAVE_MF_HC3 1
+#define HAVE_MF_HC4 1
+#define HAVE_STDBOOL_H 1
+#define HAVE_STDINT_H 1
+#define HAVE_VISIBILITY 0
+
+#define MYTHREAD_VISTA 1
+
+#define PACKAGE_BUGREPORT "[email protected]"
+#define PACKAGE_NAME "XZ Utils"
+#define PACKAGE_URL "https://tukaani.org/xz/"
+
+#define TUKLIB_SYMBOL_PREFIX lzma_

pythonbuild/downloads.py

@@ -414,19 +414,18 @@
         "sha256": "936b74c60b19c317c3f3cb1b114575032528dbdaf428740483200ea874c2ca0a",
         "version": "1.6.0",
     },
-    # IMPORTANT: xz 5.6 has a backdoor. Be extremely cautious before taking any xz
-    # upgrade since it isn't clear which versions are safe.
+    # IMPORTANT: xz 5.6.0 was released with a backdoor (CVE-2024-3094). This has been resolved.
+    # Be cautious before taking any xz upgrades given this past behavior.
     "xz": {
-        "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20240224/xz-5.2.12.tar.gz",
-        "size": 2190541,
-        "sha256": "61bda930767dcb170a5328a895ec74cab0f5aac4558cdda561c83559db582a13",
-        "version": "5.2.12",
+        "url": "https://github.com/tukaani-project/xz/releases/download/v5.8.1/xz-5.8.1.tar.gz",
+        "size": 2587189,
+        "sha256": "507825b599356c10dca1cd720c9d0d0c9d5400b9de300af00e4d1ea150795543",
+        "version": "5.8.1",
         "library_names": ["lzma"],
-        # liblzma is in the public domain. Other parts of code have licenses. But
-        # we only use liblzma.
-        "licenses": [],
+        # liblzma is licensed as 0BSD. Other parts of code have different licenses.
+        # But we only use liblzma.
+        "licenses": ["0BSD"],
         "license_file": "LICENSE.liblzma.txt",
-        "license_public_domain": True,
     },
     "zlib": {
         "url": "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz",