From de56164b210cd702716e474ebc253cef08df0f05 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 00:56:45 +0000 Subject: [PATCH 01/10] gh-91349: Replace zlib with zlib-ng in Windows build --- Misc/externals.spdx.json | 12 +- Modules/zlibmodule.c | 6 + PC/zbuild.h | 343 ++++++++++++++++++++++++++++++++ PC/zconf-ng.h | 174 ++++++++++++++++ PC/zconf.h | 204 +++++++++++++++++++ PCbuild/get_externals.bat | 2 +- PCbuild/pcbuild.proj | 1 + PCbuild/pcbuild.sln | 34 ++++ PCbuild/python.props | 1 + PCbuild/pythoncore.vcxproj | 41 +--- PCbuild/zlib-ng.vcxproj | 201 +++++++++++++++++++ PCbuild/zlib-ng.vcxproj.filters | 249 +++++++++++++++++++++++ 12 files changed, 1231 insertions(+), 37 deletions(-) create mode 100644 PC/zbuild.h create mode 100644 PC/zconf-ng.h create mode 100644 PC/zconf.h create mode 100644 PCbuild/zlib-ng.vcxproj create mode 100644 PCbuild/zlib-ng.vcxproj.filters diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index d54b1fbe251378..5bd5177b613924 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -171,25 +171,25 @@ "versionInfo": "5.2.5" }, { - "SPDXID": "SPDXRef-PACKAGE-zlib", + "SPDXID": "SPDXRef-PACKAGE-zlib-ng", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e3f3fb32564952006eb18b091ca8464740e5eca29d328cfb0b2da22768e0b638" + "checksumValue": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-1.3.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:zlib:zlib:1.3.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:zlib-ng:zlib-ng:2.2.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", - "name": "zlib", + "name": "zlib-ng", "primaryPackagePurpose": "SOURCE", - "versionInfo": "1.3.1" + "versionInfo": "2.2.4" } ], "spdxVersion": "SPDX-2.3" diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index b90665ae7ef64a..94575025b16941 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2101,6 +2101,12 @@ zlib_exec(PyObject *mod) PyUnicode_FromString(zlibVersion())) < 0) { return -1; } +#ifdef ZLIBNG_VERSION + if (PyModule_Add(mod, "ZLIBNG_VERSION", + PyUnicode_FromString(ZLIBNG_VERSION)) < 0) { + return -1; + } +#endif if (PyModule_AddStringConstant(mod, "__version__", "1.0") < 0) { return -1; } diff --git a/PC/zbuild.h b/PC/zbuild.h new file mode 100644 index 00000000000000..157ab6ffedc1ae --- /dev/null +++ b/PC/zbuild.h @@ -0,0 +1,343 @@ +#ifndef _ZBUILD_H +#define _ZBUILD_H + +#define _POSIX_SOURCE 1 /* fileno */ +#ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200809L /* snprintf, posix_memalign, strdup */ +#endif +#ifndef _ISOC11_SOURCE +# define _ISOC11_SOURCE 1 /* aligned_alloc */ +#endif +#ifdef __OpenBSD__ +# define _BSD_SOURCE 1 +#endif + +#include +#include +#include +#include + +/* Determine compiler version of C Standard */ +#ifdef __STDC_VERSION__ +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +# if __STDC_VERSION__ >= 201112L +# ifndef STDC11 +# define STDC11 +# endif +# endif +#endif + +#ifndef Z_HAS_ATTRIBUTE +# if defined(__has_attribute) +# define Z_HAS_ATTRIBUTE(a) __has_attribute(a) +# else +# define Z_HAS_ATTRIBUTE(a) 0 +# endif +#endif + +#ifndef Z_FALLTHROUGH +# if Z_HAS_ATTRIBUTE(__fallthrough__) || (defined(__GNUC__) && (__GNUC__ >= 7)) +# define Z_FALLTHROUGH __attribute__((__fallthrough__)) +# else +# define Z_FALLTHROUGH do {} while(0) /* fallthrough */ +# endif +#endif + +#ifndef Z_TARGET +# if Z_HAS_ATTRIBUTE(__target__) +# define Z_TARGET(x) __attribute__((__target__(x))) +# else +# define Z_TARGET(x) +# endif +#endif + +/* This has to be first include that defines any types */ +#if defined(_MSC_VER) +# if defined(_WIN64) + typedef __int64 ssize_t; +# else + typedef long ssize_t; +# endif + +# if defined(_WIN64) + #define SSIZE_MAX _I64_MAX +# else + #define SSIZE_MAX LONG_MAX +# endif +#endif + +/* MS Visual Studio does not allow inline in C, only C++. + But it provides __inline instead, so use that. */ +#if defined(_MSC_VER) && !defined(inline) && !defined(__cplusplus) +# define inline __inline +#endif + +#if defined(ZLIB_COMPAT) +# define PREFIX(x) x +# define PREFIX2(x) ZLIB_ ## x +# define PREFIX3(x) z_ ## x +# define PREFIX4(x) x ## 64 +# define zVersion zlibVersion +#else +# define PREFIX(x) zng_ ## x +# define PREFIX2(x) ZLIBNG_ ## x +# define PREFIX3(x) zng_ ## x +# define PREFIX4(x) zng_ ## x +# define zVersion zlibng_version +# define z_size_t size_t +#endif + +/* In zlib-compat some functions and types use unsigned long, but zlib-ng use size_t */ +#if defined(ZLIB_COMPAT) +# define z_uintmax_t unsigned long +#else +# define z_uintmax_t size_t +#endif + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) +/* Maximum of a and b. */ +#define MAX(a, b) ((a) < (b) ? (b) : (a)) +/* Ignore unused variable warning */ +#define Z_UNUSED(var) (void)(var) + +#if defined(HAVE_VISIBILITY_INTERNAL) +# define Z_INTERNAL __attribute__((visibility ("internal"))) +#elif defined(HAVE_VISIBILITY_HIDDEN) +# define Z_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define Z_INTERNAL +#endif + +/* Symbol versioning helpers, allowing multiple versions of a function to exist. + * Functions using this must also be added to zlib-ng.map for each version. + * Double @@ means this is the default for newly compiled applications to link against. + * Single @ means this is kept for backwards compatibility. + * This is only used for Zlib-ng native API, and only on platforms supporting this. + */ +#if defined(HAVE_SYMVER) +# define ZSYMVER(func,alias,ver) __asm__(".symver " func ", " alias "@ZLIB_NG_" ver); +# define ZSYMVER_DEF(func,alias,ver) __asm__(".symver " func ", " alias "@@ZLIB_NG_" ver); +#else +# define ZSYMVER(func,alias,ver) +# define ZSYMVER_DEF(func,alias,ver) +#endif + +#ifndef __cplusplus +# define Z_REGISTER register +#else +# define Z_REGISTER +#endif + +/* Reverse the bytes in a value. Use compiler intrinsics when + possible to take advantage of hardware implementations. */ +#if defined(_MSC_VER) && (_MSC_VER >= 1300) +# include +# pragma intrinsic(_byteswap_ulong) +# define ZSWAP16(q) _byteswap_ushort(q) +# define ZSWAP32(q) _byteswap_ulong(q) +# define ZSWAP64(q) _byteswap_uint64(q) + +#elif defined(__clang__) || (defined(__GNUC__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) +# define ZSWAP16(q) __builtin_bswap16(q) +# define ZSWAP32(q) __builtin_bswap32(q) +# define ZSWAP64(q) __builtin_bswap64(q) + +#elif defined(__GNUC__) && (__GNUC__ >= 2) && defined(__linux__) +# include +# define ZSWAP16(q) bswap_16(q) +# define ZSWAP32(q) bswap_32(q) +# define ZSWAP64(q) bswap_64(q) + +#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) +# include +# define ZSWAP16(q) bswap16(q) +# define ZSWAP32(q) bswap32(q) +# define ZSWAP64(q) bswap64(q) +#elif defined(__OpenBSD__) +# include +# define ZSWAP16(q) swap16(q) +# define ZSWAP32(q) swap32(q) +# define ZSWAP64(q) swap64(q) +#elif defined(__INTEL_COMPILER) +/* ICC does not provide a two byte swap. */ +# define ZSWAP16(q) ((((q) & 0xff) << 8) | (((q) & 0xff00) >> 8)) +# define ZSWAP32(q) _bswap(q) +# define ZSWAP64(q) _bswap64(q) + +#else +# define ZSWAP16(q) ((((q) & 0xff) << 8) | (((q) & 0xff00) >> 8)) +# define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) +# define ZSWAP64(q) \ + (((q & 0xFF00000000000000u) >> 56u) | \ + ((q & 0x00FF000000000000u) >> 40u) | \ + ((q & 0x0000FF0000000000u) >> 24u) | \ + ((q & 0x000000FF00000000u) >> 8u) | \ + ((q & 0x00000000FF000000u) << 8u) | \ + ((q & 0x0000000000FF0000u) << 24u) | \ + ((q & 0x000000000000FF00u) << 40u) | \ + ((q & 0x00000000000000FFu) << 56u)) +#endif + +/* Only enable likely/unlikely if the compiler is known to support it */ +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__INTEL_COMPILER) || defined(__clang__) +# define LIKELY_NULL(x) __builtin_expect((x) != 0, 0) +# define LIKELY(x) __builtin_expect(!!(x), 1) +# define UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +# define LIKELY_NULL(x) x +# define LIKELY(x) x +# define UNLIKELY(x) x +#endif /* (un)likely */ + +#if defined(HAVE_ATTRIBUTE_ALIGNED) +# define ALIGNED_(x) __attribute__ ((aligned(x))) +#elif defined(_MSC_VER) +# define ALIGNED_(x) __declspec(align(x)) +#else +/* TODO: Define ALIGNED_ for your compiler */ +# define ALIGNED_(x) +#endif + +#ifdef HAVE_BUILTIN_ASSUME_ALIGNED +# define HINT_ALIGNED(p,n) __builtin_assume_aligned((void *)(p),(n)) +#else +# define HINT_ALIGNED(p,n) (p) +#endif +#define HINT_ALIGNED_16(p) HINT_ALIGNED((p),16) +#define HINT_ALIGNED_64(p) HINT_ALIGNED((p),64) +#define HINT_ALIGNED_4096(p) HINT_ALIGNED((p),4096) + +/* PADSZ returns needed bytes to pad bpos to pad size + * PAD_NN calculates pad size and adds it to bpos, returning the result. + * All take an integer or a pointer as bpos input. + */ +#define PADSZ(bpos, pad) (((pad) - ((uintptr_t)(bpos) % (pad))) % (pad)) +#define PAD_16(bpos) ((bpos) + PADSZ((bpos),16)) +#define PAD_64(bpos) ((bpos) + PADSZ((bpos),64)) +#define PAD_4096(bpos) ((bpos) + PADSZ((bpos),4096)) + +/* Diagnostic functions */ +#ifdef ZLIB_DEBUG +# include + extern int Z_INTERNAL z_verbose; + extern void Z_INTERNAL z_error(const char *m); +# define Assert(cond, msg) {int _cond = (cond); if (!_cond) z_error(msg);} +# define Trace(x) {if (z_verbose >= 0) fprintf x;} +# define Tracev(x) {if (z_verbose > 0) fprintf x;} +# define Tracevv(x) {if (z_verbose > 1) fprintf x;} +# define Tracec(c, x) {if (z_verbose > 0 && (c)) fprintf x;} +# define Tracecv(c, x) {if (z_verbose > 1 && (c)) fprintf x;} +#else +# define Assert(cond, msg) +# define Trace(x) +# define Tracev(x) +# define Tracevv(x) +# define Tracec(c, x) +# define Tracecv(c, x) +#endif + +/* OPTIMAL_CMP values determine the comparison width: + * 64: Best for 64-bit architectures with unaligned access + * 32: Best for 32-bit architectures with unaligned access + * 16: Safe default for unknown architectures + * 8: Safe fallback for architectures without unaligned access + * Note: The unaligned access mentioned is cpu-support, this allows compiler or + * separate unaligned intrinsics to utilize safe unaligned access, without + * utilizing unaligned C pointers that are known to have undefined behavior. + */ +#if !defined(OPTIMAL_CMP) +# if defined(__x86_64__) || defined(_M_X64) || defined(__amd64__) || defined(_M_AMD64) +# define OPTIMAL_CMP 64 +# elif defined(__i386__) || defined(__i486__) || defined(__i586__) || \ + defined(__i686__) || defined(_X86_) || defined(_M_IX86) +# define OPTIMAL_CMP 32 +# elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +# if defined(__ARM_FEATURE_UNALIGNED) || defined(_WIN32) +# define OPTIMAL_CMP 64 +# else +# define OPTIMAL_CMP 8 +# endif +# elif defined(__arm__) || defined(_M_ARM) +# if defined(__ARM_FEATURE_UNALIGNED) || defined(_WIN32) +# define OPTIMAL_CMP 32 +# else +# define OPTIMAL_CMP 8 +# endif +# elif defined(__powerpc64__) || defined(__ppc64__) +# define OPTIMAL_CMP 64 +# elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) +# define OPTIMAL_CMP 32 +# endif +#endif +#if !defined(OPTIMAL_CMP) +# define OPTIMAL_CMP 16 +#endif + +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +# define Z_ADDRESS_SANITIZER 1 +# endif +#elif defined(__SANITIZE_ADDRESS__) +# define Z_ADDRESS_SANITIZER 1 +#endif + +/* + * __asan_loadN() and __asan_storeN() calls are inserted by compilers in order to check memory accesses. + * They can be called manually too, with the following caveats: + * gcc says: "warning: implicit declaration of function '...'" + * g++ says: "error: new declaration '...' ambiguates built-in declaration '...'" + * Accommodate both. + */ +#ifdef Z_ADDRESS_SANITIZER +#ifndef __cplusplus +void __asan_loadN(void *, long); +void __asan_storeN(void *, long); +#endif +#else +# define __asan_loadN(a, size) do { Z_UNUSED(a); Z_UNUSED(size); } while (0) +# define __asan_storeN(a, size) do { Z_UNUSED(a); Z_UNUSED(size); } while (0) +#endif + +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# define Z_MEMORY_SANITIZER 1 +# include +# endif +#endif + +#ifndef Z_MEMORY_SANITIZER +# define __msan_check_mem_is_initialized(a, size) do { Z_UNUSED(a); Z_UNUSED(size); } while (0) +# define __msan_unpoison(a, size) do { Z_UNUSED(a); Z_UNUSED(size); } while (0) +#endif + +/* Notify sanitizer runtime about an upcoming read access. */ +#define instrument_read(a, size) do { \ + void *__a = (void *)(a); \ + long __size = size; \ + __asan_loadN(__a, __size); \ + __msan_check_mem_is_initialized(__a, __size); \ +} while (0) + +/* Notify sanitizer runtime about an upcoming write access. */ +#define instrument_write(a, size) do { \ + void *__a = (void *)(a); \ + long __size = size; \ + __asan_storeN(__a, __size); \ +} while (0) + +/* Notify sanitizer runtime about an upcoming read/write access. */ +#define instrument_read_write(a, size) do { \ + void *__a = (void *)(a); \ + long __size = size; \ + __asan_storeN(__a, __size); \ + __msan_check_mem_is_initialized(__a, __size); \ +} while (0) + +#endif diff --git a/PC/zconf-ng.h b/PC/zconf-ng.h new file mode 100644 index 00000000000000..dc62edbef7bbc5 --- /dev/null +++ b/PC/zconf-ng.h @@ -0,0 +1,174 @@ +/* zconf-ng.h -- configuration of the zlib-ng compression library + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifndef ZCONFNG_H +#define ZCONFNG_H + +#if !defined(_WIN32) && defined(__WIN32__) +# define _WIN32 +#endif + +/* Clang macro for detecting declspec support + * https://clang.llvm.org/docs/LanguageExtensions.html#has-declspec-attribute + */ +#ifndef __has_declspec_attribute +# define __has_declspec_attribute(x) 0 +#endif + +/* Always define z_const as const */ +#define z_const const + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# define MAX_MEM_LEVEL 9 +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MIN_WBITS +# define MIN_WBITS 8 /* 256 LZ77 window */ +#endif +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + +/* Type declarations */ + +#ifdef ZLIB_INTERNAL +# define Z_INTERNAL ZLIB_INTERNAL +#endif + +/* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +#if defined(ZLIB_DLL) && (defined(_WIN32) || (__has_declspec_attribute(dllexport) && __has_declspec_attribute(dllimport))) +# ifdef Z_INTERNAL +# define Z_EXTERN extern __declspec(dllexport) +# else +# define Z_EXTERN extern __declspec(dllimport) +# endif +#endif + +/* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +#if defined(ZLIB_WINAPI) && defined(_WIN32) +# include + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define Z_EXPORT WINAPI +# define Z_EXPORTVA WINAPIV +#endif + +#ifndef Z_EXTERN +# define Z_EXTERN extern +#endif +#ifndef Z_EXPORT +# define Z_EXPORT +#endif +#ifndef Z_EXPORTVA +# define Z_EXPORTVA +#endif + +/* Conditional exports */ +#define ZNG_CONDEXPORT Z_EXPORT + +/* Fallback for something that includes us. */ +typedef unsigned char Byte; +typedef Byte Bytef; + +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +typedef char charf; +typedef int intf; +typedef uInt uIntf; +typedef uLong uLongf; + +typedef void const *voidpc; +typedef void *voidpf; +typedef void *voidp; + +#if 0 /* was set to #if 0 by configure/cmake/etc */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef NEED_PTRDIFF_T /* may be set to #if 1 by configure/cmake/etc */ +typedef PTRDIFF_TYPE ptrdiff_t; +#endif + +#include /* for off_t */ + +#include /* for wchar_t and NULL */ + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifndef z_off_t +# define z_off_t off_t +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && defined(WITH_GZFILEOP) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(__MSYS__) +# define z_off64_t _off64_t +# elif defined(_WIN32) && !defined(__GNUC__) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +#endif /* ZCONFNG_H */ diff --git a/PC/zconf.h b/PC/zconf.h new file mode 100644 index 00000000000000..cd1307a334f298 --- /dev/null +++ b/PC/zconf.h @@ -0,0 +1,204 @@ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifndef ZCONF_H +#define ZCONF_H + +#if !defined(_WIN32) && defined(__WIN32__) +# define _WIN32 +#endif + +/* Clang macro for detecting declspec support + * https://clang.llvm.org/docs/LanguageExtensions.html#has-declspec-attribute + */ +#ifndef __has_declspec_attribute +# define __has_declspec_attribute(x) 0 +#endif + +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# define MAX_MEM_LEVEL 9 +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MIN_WBITS +# define MIN_WBITS 8 /* 256 LZ77 window */ +#endif +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + +/* Type declarations */ + + +#ifndef OF /* function prototypes */ +# define OF(args) args +#endif + +#ifdef ZLIB_INTERNAL +# define Z_INTERNAL ZLIB_INTERNAL +#endif + +/* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +#if defined(ZLIB_DLL) && (defined(_WIN32) || (__has_declspec_attribute(dllexport) && __has_declspec_attribute(dllimport))) +# ifdef Z_INTERNAL +# define Z_EXTERN extern __declspec(dllexport) +# else +# define Z_EXTERN extern __declspec(dllimport) +# endif +#endif + +/* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +#if defined(ZLIB_WINAPI) && defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define Z_EXPORT WINAPI +# define Z_EXPORTVA WINAPIV +#endif + +#ifndef Z_EXTERN +# define Z_EXTERN extern +#endif +#ifndef Z_EXPORT +# define Z_EXPORT +#endif +#ifndef Z_EXPORTVA +# define Z_EXPORTVA +#endif + +/* Conditional exports */ +#define ZNG_CONDEXPORT Z_INTERNAL + +/* For backwards compatibility */ + +#ifndef ZEXTERN +# define ZEXTERN Z_EXTERN +#endif +#ifndef ZEXPORT +# define ZEXPORT Z_EXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA Z_EXPORTVA +#endif +#ifndef FAR +# define FAR +#endif + +/* Legacy zlib typedefs for backwards compatibility. Don't assume stdint.h is defined. */ +typedef unsigned char Byte; +typedef Byte Bytef; + +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +typedef char charf; +typedef int intf; +typedef uInt uIntf; +typedef uLong uLongf; + +typedef void const *voidpc; +typedef void *voidpf; +typedef void *voidp; + +typedef unsigned int z_crc_t; + +#if 0 /* was set to #if 0 by configure/cmake/etc */ +# define Z_HAVE_UNISTD_H +#endif + +#ifdef NEED_PTRDIFF_T /* may be set to #if 1 by configure/cmake/etc */ +typedef PTRDIFF_TYPE ptrdiff_t; +#endif + +#include /* for off_t */ + +#include /* for wchar_t and NULL */ + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifndef z_off_t +# define z_off_t off_t +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(__MSYS__) +# define z_off64_t _off64_t +# elif defined(_WIN32) && !defined(__GNUC__) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +typedef size_t z_size_t; + +#endif /* ZCONF_H */ diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index dfacd1d1e788d4..d7cb3f0a4fedbd 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -59,7 +59,7 @@ set libraries=%libraries% sqlite-3.45.3.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 -set libraries=%libraries% zlib-1.3.1 +set libraries=%libraries% zlib-ng-2.2.4 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index c7ddc1d23b301c..a4152fb6f689f9 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -48,6 +48,7 @@ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(ProjectName) $(TargetName)$(PyDebugExt) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8afd69d605aae2..563b45ffd828f6 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -101,7 +101,7 @@ /Zm200 %(AdditionalOptions) $(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories) - $(zlibNgDir);$(IntDir);%(AdditionalIncludeDirectories) + $(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) From f9197492cb75e9e660fca77c08a896686c709ec1 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 01:10:11 +0000 Subject: [PATCH 04/10] Use correct include dirs --- PCbuild/pcbuild.proj | 3 ++- PCbuild/pyproject.props | 2 +- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/zlib-ng.vcxproj | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index a4152fb6f689f9..0ae87a0f21f039 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -47,8 +47,9 @@ - + + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ $(ProjectName) $(TargetName)$(PyDebugExt) false diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 563b45ffd828f6..0ee0766991030a 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -111,7 +111,7 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) - zlib-ng$(PyDebugExt).lib;%(AdditionalDependencies) + zlib-ng.lib;%(AdditionalDependencies) diff --git a/PCbuild/zlib-ng.vcxproj b/PCbuild/zlib-ng.vcxproj index 6d6a7ed1ba6676..69ac63aabcf998 100644 --- a/PCbuild/zlib-ng.vcxproj +++ b/PCbuild/zlib-ng.vcxproj @@ -70,6 +70,7 @@ {FB91C8B2-6FBC-3A01-B644-1637111F902D} zlib-ng Win32Proj + zlib-ng @@ -93,7 +94,7 @@ 4206;4054;4324 stdc11 NotUsing - $(zlibNgDir);$(PySourceDir)PC;%(AdditionalIncludeDirectories) + $(zlibNgDir);$(PySourceDir)PC;$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories) %(PreprocessorDefinitions);ZLIB_COMPAT;WITH_GZFILEOP;NO_FSEEKO;HAVE_BUILTIN_ASSUME_ALIGNED;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;X86_FEATURES;X86_HAVE_XSAVE_INTRIN;X86_SSE2;X86_SSSE3;X86_SSE42;X86_PCLMULQDQ_CRC;X86_AVX2;X86_AVX512;X86_AVX512VNNI;X86_VPCLMULQDQ_CRC %(PreprocessorDefinitions);ZLIB_DEBUG AdvancedVectorExtensions2 From fb6500d2b72050c06a8d95cdab41cc05b8a3cd40 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 01:12:01 +0000 Subject: [PATCH 05/10] Add NEWS --- .../next/Windows/2025-03-19-01-11-56.gh-issue-91349.pmtp-J.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2025-03-19-01-11-56.gh-issue-91349.pmtp-J.rst diff --git a/Misc/NEWS.d/next/Windows/2025-03-19-01-11-56.gh-issue-91349.pmtp-J.rst b/Misc/NEWS.d/next/Windows/2025-03-19-01-11-56.gh-issue-91349.pmtp-J.rst new file mode 100644 index 00000000000000..4e33b8c57896cc --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-03-19-01-11-56.gh-issue-91349.pmtp-J.rst @@ -0,0 +1,2 @@ +Replaces our copy of ``zlib`` with ``zlib-ng``, for performance improvements +in :mod:`zlib`. From 89d44a16c3908d1b085d3bf59c71cb0c8703cb4b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 01:25:28 +0000 Subject: [PATCH 06/10] Fix debug reference and some warnings --- PCbuild/pyproject.props | 1 + PCbuild/pythoncore.vcxproj | 2 +- PCbuild/zlib-ng.vcxproj | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 324f8339b7f700..b5a236ee3e37d5 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -108,6 +108,7 @@ -d2:-pattern-opt-disable:-932189325 %(AdditionalOptions) + false true true true diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 0ee0766991030a..563b45ffd828f6 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -111,7 +111,7 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) - zlib-ng.lib;%(AdditionalDependencies) + zlib-ng$(PyDebugExt).lib;%(AdditionalDependencies) diff --git a/PCbuild/zlib-ng.vcxproj b/PCbuild/zlib-ng.vcxproj index 69ac63aabcf998..cb539f0af3ef7b 100644 --- a/PCbuild/zlib-ng.vcxproj +++ b/PCbuild/zlib-ng.vcxproj @@ -70,7 +70,6 @@ {FB91C8B2-6FBC-3A01-B644-1637111F902D} zlib-ng Win32Proj - zlib-ng From 08eecb1540b55a8da6074c53a2f2d0e01e9844bd Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 01:34:55 +0000 Subject: [PATCH 07/10] Exclude Intel optimizations better on ARM64 --- PCbuild/zlib-ng.vcxproj | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/PCbuild/zlib-ng.vcxproj b/PCbuild/zlib-ng.vcxproj index cb539f0af3ef7b..12a87d890d92ae 100644 --- a/PCbuild/zlib-ng.vcxproj +++ b/PCbuild/zlib-ng.vcxproj @@ -94,9 +94,10 @@ stdc11 NotUsing $(zlibNgDir);$(PySourceDir)PC;$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories) - %(PreprocessorDefinitions);ZLIB_COMPAT;WITH_GZFILEOP;NO_FSEEKO;HAVE_BUILTIN_ASSUME_ALIGNED;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE;X86_FEATURES;X86_HAVE_XSAVE_INTRIN;X86_SSE2;X86_SSSE3;X86_SSE42;X86_PCLMULQDQ_CRC;X86_AVX2;X86_AVX512;X86_AVX512VNNI;X86_VPCLMULQDQ_CRC + %(PreprocessorDefinitions);ZLIB_COMPAT;WITH_GZFILEOP;NO_FSEEKO;HAVE_BUILTIN_ASSUME_ALIGNED;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_DEPRECATE; + %(PreprocessorDefinitions);X86_FEATURES;X86_HAVE_XSAVE_INTRIN;X86_SSE2;X86_SSSE3;X86_SSE42;X86_PCLMULQDQ_CRC;X86_AVX2;X86_AVX512;X86_AVX512VNNI;X86_VPCLMULQDQ_CRC %(PreprocessorDefinitions);ZLIB_DEBUG - AdvancedVectorExtensions2 + AdvancedVectorExtensions2 @@ -130,9 +131,11 @@ + + + - @@ -150,6 +153,8 @@ + + From 548daa7592b4a896dd5aa07b4e9b96ff26d7145c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 13:44:20 +0000 Subject: [PATCH 08/10] Update docs and whatsnew --- Doc/library/zlib.rst | 12 ++++++++++++ Doc/whatsnew/3.14.rst | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 965b82a3daffb9..db0537f8ccc0a6 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -330,6 +330,18 @@ the following constants: .. versionadded:: 3.3 +.. data:: ZLIBNG_VERSION + + The version string of the zlib-ng library that was used for building the + module if zlib-ng was used. When present, the :data:`ZLIB_VERSION` and + :data:`ZLIB_RUNTIME_VERSION` constants reflect the version of the zlib API + provided by zlib-ng. + + If zlib-ng was not used to build the module, this constant will be absent. + + .. versionadded:: 3.14 + + .. seealso:: Module :mod:`gzip` diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 79b219dd72651c..31d99eab1da708 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1026,6 +1026,7 @@ zipinfo to produce reproducible output. (Contributed by Jiahao Li in :gh:`91279`.) + .. Add improved modules above alphabetically, not here at the end. Optimizations @@ -1078,6 +1079,18 @@ uuid (Contributed by Bénédikt Tran in :gh:`128150`.) +zlib +---- + +* On Windows, ``zlib-ng`` is now used as the implementation of the + :mod:`zlib` module. This should produce compatible and comparable + results with better performance, though it is worth noting that + :const:`~zlib.Z_BEST_SPEED` (1) may result in significantly less + compression than the previous implementation (while also significantly + reducing the time taken to compress). + (Contributed by Steve Dower in :gh:`91349`.) + + Deprecated ========== From f1080449cb9f4ab96cc4ee19b1ac5bb9f27259db Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 14:25:28 +0000 Subject: [PATCH 09/10] Remove unresolved reference from docs --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 31d99eab1da708..303d7922d1708d 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1085,7 +1085,7 @@ zlib * On Windows, ``zlib-ng`` is now used as the implementation of the :mod:`zlib` module. This should produce compatible and comparable results with better performance, though it is worth noting that - :const:`~zlib.Z_BEST_SPEED` (1) may result in significantly less + ``zlib.Z_BEST_SPEED`` (1) may result in significantly less compression than the previous implementation (while also significantly reducing the time taken to compress). (Contributed by Steve Dower in :gh:`91349`.) From 085fa10a4a7d9d942c6094dfb1f77f87c6797c0e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 19 Mar 2025 18:37:00 +0000 Subject: [PATCH 10/10] Include ZLIBNG_VERSION in pythoninfo --- Lib/test/pythoninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 38236d82376f37..195551e7422896 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -651,7 +651,7 @@ def collect_zlib(info_add): except ImportError: return - attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') + attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION', 'ZLIBNG_VERSION') copy_attributes(info_add, zlib, 'zlib.%s', attributes)