From 6d7a968bc68de42a214522ccf0643a7d0a5d2f32 Mon Sep 17 00:00:00 2001 From: Tom J Nowell Date: Mon, 22 Jun 2026 17:19:41 +0100 Subject: [PATCH 1/3] VFS: add .sdzst support (zip with zstd / method-93 entries) Adds a new .sdzst archive format: a zip container whose entries use Zstandard (PKWARE compression method 93). The vendored minizip reader is left almost untouched - the only change is whitelisting method 93 in its two method-validation guards so such an entry can be opened in raw mode. All zstd handling lives in CZipArchive::GetFileImpl, which reads the raw frame and decompresses it with libzstd, verifying the CRC manually (raw reads bypass minizip's CRC-on-close check). zstd decoding is global: existing .sdz archives that happen to contain method-93 entries now read too; .sdzst is a naming/signalling convention, registered as the same CZipArchive behind a new "sdzst" factory. The decoder reuses the already-vendored Tracy zstd sources, built as a decompress-only static lib (zstd_dec), so no new external dependency is introduced. Read-only; the zip writer is unchanged. --- rts/CMakeLists.txt | 3 +- rts/System/FileSystem/ArchiveLoader.cpp | 2 + rts/System/FileSystem/Archives/ArchiveTypes.h | 5 ++- rts/System/FileSystem/Archives/CMakeLists.txt | 8 ++-- rts/System/FileSystem/Archives/ZipArchive.cpp | 45 +++++++++++++++++++ rts/System/FileSystem/Archives/ZipArchive.h | 13 ++++++ rts/lib/CMakeLists.txt | 25 +++++++++++ rts/lib/minizip/unzip.c | 2 + rts/lib/minizip/unzip.h | 5 +++ 9 files changed, 101 insertions(+), 7 deletions(-) diff --git a/rts/CMakeLists.txt b/rts/CMakeLists.txt index 001ea7cb6ba..3cfb708f19f 100644 --- a/rts/CMakeLists.txt +++ b/rts/CMakeLists.txt @@ -42,6 +42,7 @@ include_directories(BEFORE lib) include_directories(BEFORE lib/lua/include) include_directories(${CMAKE_SOURCE_DIR}/include/AL) include_directories(${SPRING_MINIZIP_INCLUDE_DIR}) +include_directories(${SPRING_ZSTD_INCLUDE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib/cereal/include) @@ -85,7 +86,7 @@ endif (UNIX AND NOT MINGW) find_package_static(ZLIB 1.2.7 REQUIRED) list(APPEND engineCommonLibraries DevIL::IL) -list(APPEND engineCommonLibraries 7zip prd::jsoncpp ${SPRING_MINIZIP_LIBRARY} ZLIB::ZLIB Tracy::TracyClient) +list(APPEND engineCommonLibraries 7zip prd::jsoncpp ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} ZLIB::ZLIB Tracy::TracyClient) list(APPEND engineCommonLibraries lua luasocket archives assimp simdjson::simdjson fastgltf gflags_nothreads_static) if(CMAKE_SYSTEM_NAME MATCHES "OpenBSD") diff --git a/rts/System/FileSystem/ArchiveLoader.cpp b/rts/System/FileSystem/ArchiveLoader.cpp index 1b6b4e412ef..c84d128e8db 100644 --- a/rts/System/FileSystem/ArchiveLoader.cpp +++ b/rts/System/FileSystem/ArchiveLoader.cpp @@ -19,6 +19,7 @@ static CPoolArchiveFactory sdpArchiveFactory; static CDirArchiveFactory sddArchiveFactory; static CZipArchiveFactory sdzArchiveFactory; +static CZipStdArchiveFactory sdzstArchiveFactory; static CSevenZipArchiveFactory sd7ArchiveFactory; static CVirtualArchiveFactory sdvArchiveFactory; @@ -30,6 +31,7 @@ CArchiveLoader::CArchiveLoader() AddFactory(ARCHIVE_TYPE_SDP, sdpArchiveFactory); AddFactory(ARCHIVE_TYPE_SDD, sddArchiveFactory); AddFactory(ARCHIVE_TYPE_SDZ, sdzArchiveFactory); + AddFactory(ARCHIVE_TYPE_SDZST, sdzstArchiveFactory); AddFactory(ARCHIVE_TYPE_SD7, sd7ArchiveFactory); AddFactory(ARCHIVE_TYPE_SDV, sdvArchiveFactory); diff --git a/rts/System/FileSystem/Archives/ArchiveTypes.h b/rts/System/FileSystem/Archives/ArchiveTypes.h index d77a50860ec..0cba0ffcba6 100644 --- a/rts/System/FileSystem/Archives/ArchiveTypes.h +++ b/rts/System/FileSystem/Archives/ArchiveTypes.h @@ -9,8 +9,9 @@ enum { ARCHIVE_TYPE_SDZ = 2, // zip ARCHIVE_TYPE_SD7 = 3, // 7zip ARCHIVE_TYPE_SDV = 4, // virtual - ARCHIVE_TYPE_CNT = 5, - ARCHIVE_TYPE_BUF = 6, // buffered, not created directly + ARCHIVE_TYPE_SDZST = 5, // zip with zstd entries (.sdzst files) + ARCHIVE_TYPE_CNT = 6, + ARCHIVE_TYPE_BUF = 7, // buffered, not created directly }; #endif diff --git a/rts/System/FileSystem/Archives/CMakeLists.txt b/rts/System/FileSystem/Archives/CMakeLists.txt index 51e8d79f4ab..a62c8c0fa03 100644 --- a/rts/System/FileSystem/Archives/CMakeLists.txt +++ b/rts/System/FileSystem/Archives/CMakeLists.txt @@ -40,13 +40,13 @@ if (THREADPOOL) target_compile_definitions(archives PRIVATE -DTHREADPOOL) endif() target_include_directories(archives - PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR}) + PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${SPRING_ZSTD_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR}) target_link_libraries(archives - PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} Tracy::TracyClient nowide::nowide fmt::fmt) + PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} Tracy::TracyClient nowide::nowide fmt::fmt) add_library(archives_nothreadpool STATIC ${archives_sources}) target_compile_options(archives_nothreadpool PRIVATE ${PIC_FLAG}) target_include_directories(archives_nothreadpool - PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR}) + PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${SPRING_ZSTD_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR}) target_link_libraries(archives_nothreadpool - PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} nowide::nowide fmt::fmt) + PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} nowide::nowide fmt::fmt) diff --git a/rts/System/FileSystem/Archives/ZipArchive.cpp b/rts/System/FileSystem/Archives/ZipArchive.cpp index 37914269c49..ca01887b6ac 100644 --- a/rts/System/FileSystem/Archives/ZipArchive.cpp +++ b/rts/System/FileSystem/Archives/ZipArchive.cpp @@ -6,6 +6,9 @@ #include #include +#include +#include + #include "System/StringUtil.h" #include "System/Log/ILog.h" #include "System/Threading/ThreadPool.h" @@ -16,6 +19,13 @@ IArchive* CZipArchiveFactory::DoCreateArchive(const std::string& filePath) const return new CZipArchive(filePath); } +IArchive* CZipStdArchiveFactory::DoCreateArchive(const std::string& filePath) const +{ + // .sdzst is a regular zip whose entries use zstd (method 93); CZipArchive + // reads both transparently, see CZipArchive::GetFileImpl. + return new CZipArchive(filePath); +} + CZipArchive::CZipArchive(const std::string& archiveName) : CBufferedArchive(archiveName) { @@ -144,6 +154,41 @@ int CZipArchive::GetFileImpl(uint32_t fid, std::vector& buffer) unz_file_info fi; unzGetCurrentFileInfo(thisThreadZip, &fi, nullptr, 0, nullptr, 0, nullptr, 0); + // minizip cannot decode zstd entries (method 93). Read the raw compressed + // frame and decompress it here; CRC is verified manually since raw reads + // bypass minizip's CRC-on-close check (see unzip.h Z_ZSTDED note). + if (fi.compression_method == Z_ZSTDED) { + int method = 0; + int level = 0; + if (unzOpenCurrentFile2(thisThreadZip, &method, &level, 1 /*raw*/) != UNZ_OK) + return -3; + + std::vector compressed(fi.compressed_size); + + const bool readOk = compressed.empty() || + unzReadCurrentFile(thisThreadZip, compressed.data(), compressed.size()) == static_cast(compressed.size()); + unzCloseCurrentFile(thisThreadZip); + + if (!readOk) + return -1; + + buffer.clear(); + buffer.resize(fi.uncompressed_size); + + const size_t dSize = ZSTD_decompress(buffer.data(), buffer.size(), compressed.data(), compressed.size()); + if (ZSTD_isError(dSize) || dSize != buffer.size()) { + buffer.clear(); + return -1; + } + + if (crc32(0, buffer.data(), buffer.size()) != fileEntries[fid].crc) { + buffer.clear(); + return 0; + } + + return 1; + } + if (unzOpenCurrentFile(thisThreadZip) != UNZ_OK) return -3; diff --git a/rts/System/FileSystem/Archives/ZipArchive.h b/rts/System/FileSystem/Archives/ZipArchive.h index 3207338bcad..b3102a145fa 100644 --- a/rts/System/FileSystem/Archives/ZipArchive.h +++ b/rts/System/FileSystem/Archives/ZipArchive.h @@ -24,6 +24,19 @@ class CZipArchiveFactory : public IArchiveFactory { }; +/** + * Creates archives stored as a zip with zstd-compressed (method 93) entries. + * Backed by the same CZipArchive as .sdz; only the extension differs. + */ +class CZipStdArchiveFactory : public IArchiveFactory { +public: + CZipStdArchiveFactory(): IArchiveFactory("sdzst") {} + +private: + IArchive* DoCreateArchive(const std::string& filePath) const; +}; + + /** * A zip compressed, single-file archive. */ diff --git a/rts/lib/CMakeLists.txt b/rts/lib/CMakeLists.txt index 74b8b28076a..b1eae9283f0 100644 --- a/rts/lib/CMakeLists.txt +++ b/rts/lib/CMakeLists.txt @@ -45,6 +45,31 @@ endif() ADD_SUBDIRECTORY(lua) ADD_SUBDIRECTORY(luasocket) ADD_SUBDIRECTORY(minizip) + +# zstd decompress-only static library, reusing the vendored Tracy zstd sources. +# Required to read .sdzst archives (zip containers with zstd / method-93 entries). +set(ZSTD_DEC_DIR "${CMAKE_SOURCE_DIR}/rts/lib/tracy/zstd") +set(zstdDecSources + "${ZSTD_DEC_DIR}/common/debug.c" + "${ZSTD_DEC_DIR}/common/entropy_common.c" + "${ZSTD_DEC_DIR}/common/error_private.c" + "${ZSTD_DEC_DIR}/common/fse_decompress.c" + "${ZSTD_DEC_DIR}/common/pool.c" + "${ZSTD_DEC_DIR}/common/threading.c" + "${ZSTD_DEC_DIR}/common/xxhash.c" + "${ZSTD_DEC_DIR}/common/zstd_common.c" + "${ZSTD_DEC_DIR}/decompress/huf_decompress.c" + "${ZSTD_DEC_DIR}/decompress/zstd_ddict.c" + "${ZSTD_DEC_DIR}/decompress/zstd_decompress.c" + "${ZSTD_DEC_DIR}/decompress/zstd_decompress_block.c" +) +add_library(zstd_dec STATIC EXCLUDE_FROM_ALL ${zstdDecSources}) +target_include_directories(zstd_dec PUBLIC ${ZSTD_DEC_DIR}) +target_compile_definitions(zstd_dec PRIVATE ZSTD_DISABLE_ASM) +set_target_properties(zstd_dec PROPERTIES COMPILE_FLAGS "${PIC_FLAG}") +set_global(SPRING_ZSTD_INCLUDE_DIR "${ZSTD_DEC_DIR}") +set_global(SPRING_ZSTD_LIBRARY zstd_dec) + ADD_SUBDIRECTORY(nowide) ADD_SUBDIRECTORY(headlessStubs) if (ENABLE_STREFLOP) diff --git a/rts/lib/minizip/unzip.c b/rts/lib/minizip/unzip.c index ea05b7d62a0..b0cd4b89f3d 100644 --- a/rts/lib/minizip/unzip.c +++ b/rts/lib/minizip/unzip.c @@ -1297,6 +1297,7 @@ local int unz64local_CheckCurrentFileCoherencyHeader(unz64_s* s, uInt* piSizeVar /* #ifdef HAVE_BZIP2 */ (s->cur_file_info.compression_method!=Z_BZIP2ED) && /* #endif */ + (s->cur_file_info.compression_method!=Z_ZSTDED) && (s->cur_file_info.compression_method!=Z_DEFLATED)) err=UNZ_BADZIPFILE; @@ -1403,6 +1404,7 @@ extern int ZEXPORT unzOpenCurrentFile3(unzFile file, int* method, /* #ifdef HAVE_BZIP2 */ (s->cur_file_info.compression_method!=Z_BZIP2ED) && /* #endif */ + (s->cur_file_info.compression_method!=Z_ZSTDED) && (s->cur_file_info.compression_method!=Z_DEFLATED)) err=UNZ_BADZIPFILE; diff --git a/rts/lib/minizip/unzip.h b/rts/lib/minizip/unzip.h index 5cfc9c6274e..6749f922ff4 100644 --- a/rts/lib/minizip/unzip.h +++ b/rts/lib/minizip/unzip.h @@ -61,6 +61,11 @@ extern "C" { #define Z_BZIP2ED 12 +/* Zstandard (PKWARE APPNOTE compression method 93). minizip itself does not + * decode this; it is whitelisted so the entry can be opened in raw mode and + * decompressed by the caller (see CZipArchive::GetFileImpl). */ +#define Z_ZSTDED 93 + #if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) /* like the STRICT of WIN32, we define a pointer that cannot be converted from (void*) without cast */ From 8fed0b469ad993d0549589b97dd862b0c1961efc Mon Sep 17 00:00:00 2001 From: Tom J Nowell Date: Mon, 22 Jun 2026 18:08:50 +0100 Subject: [PATCH 2/3] VFS: read .sdzst entries without patching minizip The engine links a prebuilt minizip (from spring-static-libs), so the earlier source patch to rts/lib/minizip was not used in CI: ZipArchive failed to build (Z_ZSTDED undeclared) and, even fixed, the prebuilt minizip would still reject method 93 on its open path at runtime. Revert the minizip changes and instead read zstd entries self-contained: minizip still enumerates method-93 entries fine (the method is only validated on the decode-open path), so CZipArchive::GetFileZstd locates the raw frame via the zip central-directory / local-file headers (from the unz_file_pos we already store per entry) and decompresses it with libzstd, verifying the CRC manually. Zip64 local-header offsets handled. Verified by round-tripping a real Python-authored method-93 .sdzst (text, lua, empty-file) byte-identically against a pristine minizip. --- rts/System/FileSystem/Archives/ZipArchive.cpp | 142 +++++++++++++----- rts/System/FileSystem/Archives/ZipArchive.h | 2 + rts/lib/minizip/unzip.c | 2 - rts/lib/minizip/unzip.h | 5 - 4 files changed, 110 insertions(+), 41 deletions(-) diff --git a/rts/System/FileSystem/Archives/ZipArchive.cpp b/rts/System/FileSystem/Archives/ZipArchive.cpp index ca01887b6ac..9781a3ba50e 100644 --- a/rts/System/FileSystem/Archives/ZipArchive.cpp +++ b/rts/System/FileSystem/Archives/ZipArchive.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -14,6 +15,19 @@ #include "System/Threading/ThreadPool.h" #include "System/TimeUtil.h" +namespace { + // PKWARE APPNOTE compression method for Zstandard entries. + constexpr uint16_t ZIP_METHOD_ZSTD = 93; + + uint16_t rdLE16(const uint8_t* p) { return uint16_t(p[0]) | (uint16_t(p[1]) << 8); } + uint32_t rdLE32(const uint8_t* p) { + return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24); + } + uint64_t rdLE64(const uint8_t* p) { + return uint64_t(rdLE32(p)) | (uint64_t(rdLE32(p + 4)) << 32); + } +} + IArchive* CZipArchiveFactory::DoCreateArchive(const std::string& filePath) const { return new CZipArchive(filePath); @@ -124,6 +138,93 @@ IArchive::SFileInfo CZipArchive::FileInfo(uint32_t fid) const }; } +// Reads a zstd-compressed (method 93) entry. minizip cannot decode these and +// rejects them in its open path, so we locate the raw frame directly from the +// zip headers and decompress it with libzstd. The entry's central-directory +// position is the unz_file_pos minizip already gave us when scanning. +int CZipArchive::GetFileZstd(uint32_t fid, uint64_t compressedSize, uint64_t uncompressedSize, std::vector& buffer) +{ + std::ifstream f(GetArchiveFile(), std::ios::binary); + if (!f) + return -4; + + // Central directory file header (46-byte fixed part) for this entry. + const uint64_t cdOffset = fileEntries[fid].fp.pos_in_zip_directory; + uint8_t cd[46]; + f.seekg(cdOffset); + f.read(reinterpret_cast(cd), sizeof(cd)); + if (!f || rdLE32(cd) != 0x02014b50u) + return -3; + + const uint16_t cdNameLen = rdLE16(cd + 28); + const uint16_t cdExtraLen = rdLE16(cd + 30); + uint64_t lhOffset = rdLE32(cd + 42); // relative offset of local header + + // Zip64: a 0xFFFFFFFF offset lives in the Zip64 extended-info extra field. + if (lhOffset == 0xFFFFFFFFu) { + std::vector extra(cdExtraLen); + f.seekg(cdOffset + 46 + cdNameLen); + if (cdExtraLen != 0) + f.read(reinterpret_cast(extra.data()), cdExtraLen); + if (!f) + return -3; + + // Within tag 0x0001 the 8-byte fields appear only for the 32-bit fields + // that were set to 0xFFFFFFFF, in order: uncompressed, compressed, offset. + const size_t skip = (rdLE32(cd + 24) == 0xFFFFFFFFu ? 8 : 0) + + (rdLE32(cd + 20) == 0xFFFFFFFFu ? 8 : 0); + bool found = false; + for (size_t i = 0; i + 4 <= extra.size(); ) { + const uint16_t tag = rdLE16(&extra[i]); + const uint16_t sz = rdLE16(&extra[i + 2]); + if (tag == 0x0001 && i + 4 + skip + 8 <= extra.size()) { + lhOffset = rdLE64(&extra[i + 4 + skip]); + found = true; + break; + } + i += 4 + sz; + } + if (!found) + return -3; + } + + // Local file header (30-byte fixed part); name/extra lengths can differ + // from the central-directory copy, so read them here. + uint8_t lh[30]; + f.seekg(lhOffset); + f.read(reinterpret_cast(lh), sizeof(lh)); + if (!f || rdLE32(lh) != 0x04034b50u) + return -3; + + const uint64_t dataOffset = lhOffset + 30 + rdLE16(lh + 26) + rdLE16(lh + 28); + + std::vector compressed(compressedSize); + if (!compressed.empty()) { + f.seekg(dataOffset); + f.read(reinterpret_cast(compressed.data()), compressed.size()); + if (!f) + return -1; + } + + buffer.clear(); + buffer.resize(uncompressedSize); + + const size_t dSize = ZSTD_decompress( + buffer.empty() ? nullptr : buffer.data(), buffer.size(), + compressed.empty() ? nullptr : compressed.data(), compressed.size()); + if (ZSTD_isError(dSize) || dSize != buffer.size()) { + buffer.clear(); + return -1; + } + + if (crc32(0, buffer.data(), buffer.size()) != fileEntries[fid].crc) { + buffer.clear(); + return 0; + } + + return 1; +} + // To simplify things, files are always read completely into memory from // the zip-file, since zlib does not provide any way of reading more // than one file at a time @@ -154,40 +255,13 @@ int CZipArchive::GetFileImpl(uint32_t fid, std::vector& buffer) unz_file_info fi; unzGetCurrentFileInfo(thisThreadZip, &fi, nullptr, 0, nullptr, 0, nullptr, 0); - // minizip cannot decode zstd entries (method 93). Read the raw compressed - // frame and decompress it here; CRC is verified manually since raw reads - // bypass minizip's CRC-on-close check (see unzip.h Z_ZSTDED note). - if (fi.compression_method == Z_ZSTDED) { - int method = 0; - int level = 0; - if (unzOpenCurrentFile2(thisThreadZip, &method, &level, 1 /*raw*/) != UNZ_OK) - return -3; - - std::vector compressed(fi.compressed_size); - - const bool readOk = compressed.empty() || - unzReadCurrentFile(thisThreadZip, compressed.data(), compressed.size()) == static_cast(compressed.size()); - unzCloseCurrentFile(thisThreadZip); - - if (!readOk) - return -1; - - buffer.clear(); - buffer.resize(fi.uncompressed_size); - - const size_t dSize = ZSTD_decompress(buffer.data(), buffer.size(), compressed.data(), compressed.size()); - if (ZSTD_isError(dSize) || dSize != buffer.size()) { - buffer.clear(); - return -1; - } - - if (crc32(0, buffer.data(), buffer.size()) != fileEntries[fid].crc) { - buffer.clear(); - return 0; - } - - return 1; - } + // minizip can enumerate but not decode zstd entries (method 93). It also + // rejects method 93 in its open path, and the engine may link a prebuilt + // minizip we cannot patch. So for zstd entries we bypass minizip's decoder + // entirely: locate the raw frame via the zip headers and decompress it + // ourselves (see GetFileZstd). + if (fi.compression_method == ZIP_METHOD_ZSTD) + return GetFileZstd(fid, fi.compressed_size, fi.uncompressed_size, buffer); if (unzOpenCurrentFile(thisThreadZip) != UNZ_OK) return -3; diff --git a/rts/System/FileSystem/Archives/ZipArchive.h b/rts/System/FileSystem/Archives/ZipArchive.h index b3102a145fa..48ce695ad59 100644 --- a/rts/System/FileSystem/Archives/ZipArchive.h +++ b/rts/System/FileSystem/Archives/ZipArchive.h @@ -64,6 +64,8 @@ class CZipArchive : public CBufferedArchive protected: int GetFileImpl(uint32_t fid, std::vector& buffer) override; private: + int GetFileZstd(uint32_t fid, uint64_t compressedSize, uint64_t uncompressedSize, std::vector& buffer); + static constexpr int MAX_THREADS = 32; Recoil::AtomicFirstIndex afi; diff --git a/rts/lib/minizip/unzip.c b/rts/lib/minizip/unzip.c index b0cd4b89f3d..ea05b7d62a0 100644 --- a/rts/lib/minizip/unzip.c +++ b/rts/lib/minizip/unzip.c @@ -1297,7 +1297,6 @@ local int unz64local_CheckCurrentFileCoherencyHeader(unz64_s* s, uInt* piSizeVar /* #ifdef HAVE_BZIP2 */ (s->cur_file_info.compression_method!=Z_BZIP2ED) && /* #endif */ - (s->cur_file_info.compression_method!=Z_ZSTDED) && (s->cur_file_info.compression_method!=Z_DEFLATED)) err=UNZ_BADZIPFILE; @@ -1404,7 +1403,6 @@ extern int ZEXPORT unzOpenCurrentFile3(unzFile file, int* method, /* #ifdef HAVE_BZIP2 */ (s->cur_file_info.compression_method!=Z_BZIP2ED) && /* #endif */ - (s->cur_file_info.compression_method!=Z_ZSTDED) && (s->cur_file_info.compression_method!=Z_DEFLATED)) err=UNZ_BADZIPFILE; diff --git a/rts/lib/minizip/unzip.h b/rts/lib/minizip/unzip.h index 6749f922ff4..5cfc9c6274e 100644 --- a/rts/lib/minizip/unzip.h +++ b/rts/lib/minizip/unzip.h @@ -61,11 +61,6 @@ extern "C" { #define Z_BZIP2ED 12 -/* Zstandard (PKWARE APPNOTE compression method 93). minizip itself does not - * decode this; it is whitelisted so the entry can be opened in raw mode and - * decompressed by the caller (see CZipArchive::GetFileImpl). */ -#define Z_ZSTDED 93 - #if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) /* like the STRICT of WIN32, we define a pointer that cannot be converted from (void*) without cast */ From bea89a451238c84908673ad33f6b500b1e659e9d Mon Sep 17 00:00:00 2001 From: Tom J Nowell Date: Mon, 22 Jun 2026 18:39:17 +0100 Subject: [PATCH 3/3] VFS: source zstd via find_package instead of Tracy's sources The previous commit built a zstd_dec static lib out of Tracy's internal vendored zstd sources, which couples the VFS build to Tracy's layout and would break on a Tracy update. Source zstd as a first-class dependency instead: add FindZstd.cmake (modelled on FindMiniZip.cmake) and resolve it with find_package_static, the same way zlib and minizip are found from the static-libs / mingwlibs bundle. SPRING_ZSTD_INCLUDE_DIR / SPRING_ZSTD_LIBRARY now point at the found package; the consuming targets are unchanged. Note: this requires libzstd to be present in the static-libs / mingwlibs bundles. Until that is added CI cannot link, which is expected. --- rts/build/cmake/FindZstd.cmake | 32 ++++++++++++++++++++++++++++++++ rts/lib/CMakeLists.txt | 29 ++++++----------------------- 2 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 rts/build/cmake/FindZstd.cmake diff --git a/rts/build/cmake/FindZstd.cmake b/rts/build/cmake/FindZstd.cmake new file mode 100644 index 00000000000..6c49e26a647 --- /dev/null +++ b/rts/build/cmake/FindZstd.cmake @@ -0,0 +1,32 @@ +# This file is part of the Spring engine (GPL v2 or later), see LICENSE.html + +# - Find the Zstandard (zstd) library +# Find the native zstd includes and library (static or shared) +# +# ZSTD_INCLUDE_DIR - where to find zstd.h +# ZSTD_LIBRARIES - List of libraries when using zstd. +# ZSTD_FOUND - True if zstd was found. + +Include(FindPackageHandleStandardArgs) + +If (ZSTD_INCLUDE_DIR) + # Already in cache, be silent + Set(ZSTD_FIND_QUIETLY TRUE) +EndIf (ZSTD_INCLUDE_DIR) + +Find_Path(ZSTD_INCLUDE_DIR zstd.h) + +Set(ZSTD_NAMES zstd) +Find_Library(ZSTD_LIBRARY NAMES ${ZSTD_NAMES}) + +# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if +# all listed variables are TRUE +Find_Package_Handle_Standard_Args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +If (ZSTD_FOUND) + Set(ZSTD_LIBRARIES ${ZSTD_LIBRARY}) +Else (ZSTD_FOUND) + Set(ZSTD_LIBRARIES) +EndIf (ZSTD_FOUND) + +Mark_As_Advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR) diff --git a/rts/lib/CMakeLists.txt b/rts/lib/CMakeLists.txt index b1eae9283f0..621a6040ab8 100644 --- a/rts/lib/CMakeLists.txt +++ b/rts/lib/CMakeLists.txt @@ -46,29 +46,12 @@ ADD_SUBDIRECTORY(lua) ADD_SUBDIRECTORY(luasocket) ADD_SUBDIRECTORY(minizip) -# zstd decompress-only static library, reusing the vendored Tracy zstd sources. -# Required to read .sdzst archives (zip containers with zstd / method-93 entries). -set(ZSTD_DEC_DIR "${CMAKE_SOURCE_DIR}/rts/lib/tracy/zstd") -set(zstdDecSources - "${ZSTD_DEC_DIR}/common/debug.c" - "${ZSTD_DEC_DIR}/common/entropy_common.c" - "${ZSTD_DEC_DIR}/common/error_private.c" - "${ZSTD_DEC_DIR}/common/fse_decompress.c" - "${ZSTD_DEC_DIR}/common/pool.c" - "${ZSTD_DEC_DIR}/common/threading.c" - "${ZSTD_DEC_DIR}/common/xxhash.c" - "${ZSTD_DEC_DIR}/common/zstd_common.c" - "${ZSTD_DEC_DIR}/decompress/huf_decompress.c" - "${ZSTD_DEC_DIR}/decompress/zstd_ddict.c" - "${ZSTD_DEC_DIR}/decompress/zstd_decompress.c" - "${ZSTD_DEC_DIR}/decompress/zstd_decompress_block.c" -) -add_library(zstd_dec STATIC EXCLUDE_FROM_ALL ${zstdDecSources}) -target_include_directories(zstd_dec PUBLIC ${ZSTD_DEC_DIR}) -target_compile_definitions(zstd_dec PRIVATE ZSTD_DISABLE_ASM) -set_target_properties(zstd_dec PROPERTIES COMPILE_FLAGS "${PIC_FLAG}") -set_global(SPRING_ZSTD_INCLUDE_DIR "${ZSTD_DEC_DIR}") -set_global(SPRING_ZSTD_LIBRARY zstd_dec) +# zstd: required to read .sdzst archives (zip containers with zstd / method-93 +# entries). Sourced like the other third-party libs from the prebuilt +# static-libs / mingwlibs bundle. +find_package_static(Zstd REQUIRED) +set_global(SPRING_ZSTD_INCLUDE_DIR "${ZSTD_INCLUDE_DIR}") +set_global(SPRING_ZSTD_LIBRARY "${ZSTD_LIBRARY}") ADD_SUBDIRECTORY(nowide) ADD_SUBDIRECTORY(headlessStubs)