VFS: add .sdzst support (zip with zstd / method-93 entries)

rts/CMakeLists.txt

@@ -42,6 +42,7 @@ include_directories(BEFORE lib)
 include_directories(BEFORE lib/lua/include)
 include_directories(${CMAKE_SOURCE_DIR}/include/AL)
 include_directories(${SPRING_MINIZIP_INCLUDE_DIR})
+include_directories(${SPRING_ZSTD_INCLUDE_DIR})
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib/cereal/include)
 
@@ -85,7 +86,7 @@ endif (UNIX AND NOT MINGW)
 
 find_package_static(ZLIB 1.2.7 REQUIRED)
 list(APPEND engineCommonLibraries DevIL::IL)
-list(APPEND engineCommonLibraries 7zip prd::jsoncpp ${SPRING_MINIZIP_LIBRARY} ZLIB::ZLIB Tracy::TracyClient)
+list(APPEND engineCommonLibraries 7zip prd::jsoncpp ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} ZLIB::ZLIB Tracy::TracyClient)
 list(APPEND engineCommonLibraries lua luasocket archives assimp simdjson::simdjson fastgltf
 	gflags_nothreads_static)
 if(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")

rts/System/FileSystem/ArchiveLoader.cpp

@@ -19,6 +19,7 @@
 static CPoolArchiveFactory sdpArchiveFactory;
 static CDirArchiveFactory sddArchiveFactory;
 static CZipArchiveFactory sdzArchiveFactory;
+static CZipStdArchiveFactory sdzstArchiveFactory;
 static CSevenZipArchiveFactory sd7ArchiveFactory;
 static CVirtualArchiveFactory sdvArchiveFactory;
 
@@ -30,6 +31,7 @@ CArchiveLoader::CArchiveLoader()
 	AddFactory(ARCHIVE_TYPE_SDP, sdpArchiveFactory);
 	AddFactory(ARCHIVE_TYPE_SDD, sddArchiveFactory);
 	AddFactory(ARCHIVE_TYPE_SDZ, sdzArchiveFactory);
+	AddFactory(ARCHIVE_TYPE_SDZST, sdzstArchiveFactory);
 	AddFactory(ARCHIVE_TYPE_SD7, sd7ArchiveFactory);
 	AddFactory(ARCHIVE_TYPE_SDV, sdvArchiveFactory);
 

rts/System/FileSystem/Archives/ArchiveTypes.h

@@ -9,8 +9,9 @@ enum {
 	ARCHIVE_TYPE_SDZ = 2, // zip
 	ARCHIVE_TYPE_SD7 = 3, // 7zip
 	ARCHIVE_TYPE_SDV = 4, // virtual
-	ARCHIVE_TYPE_CNT = 5,
-	ARCHIVE_TYPE_BUF = 6, // buffered, not created directly
+	ARCHIVE_TYPE_SDZST = 5, // zip with zstd entries (.sdzst files)
+	ARCHIVE_TYPE_CNT = 6,
+	ARCHIVE_TYPE_BUF = 7, // buffered, not created directly
 };
 
 #endif

rts/System/FileSystem/Archives/CMakeLists.txt

@@ -40,13 +40,13 @@ if (THREADPOOL)
 	target_compile_definitions(archives PRIVATE -DTHREADPOOL)
 endif()
 target_include_directories(archives
-	PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR})
+	PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${SPRING_ZSTD_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR})
 target_link_libraries(archives
-	PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} Tracy::TracyClient nowide::nowide fmt::fmt)
+	PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} Tracy::TracyClient nowide::nowide fmt::fmt)
 
 add_library(archives_nothreadpool STATIC ${archives_sources})
 target_compile_options(archives_nothreadpool PRIVATE ${PIC_FLAG})
 target_include_directories(archives_nothreadpool
-	PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR})
+	PRIVATE ${SOURCE_ROOT} ${SPRING_MINIZIP_INCLUDE_DIR} ${SPRING_ZSTD_INCLUDE_DIR} ${NOWIDE_INCLUDE_DIR})
 target_link_libraries(archives_nothreadpool
-	PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} nowide::nowide fmt::fmt)
+	PRIVATE 7zip ${SPRING_MINIZIP_LIBRARY} ${SPRING_ZSTD_LIBRARY} nowide::nowide fmt::fmt)

rts/System/FileSystem/Archives/ZipArchive.cpp

@@ -5,17 +5,41 @@
 #include <algorithm>
 #include <stdexcept>
 #include <cassert>
+#include <fstream>
+
+#include <zstd.h>
+#include <zlib.h>
 
 #include "System/StringUtil.h"
 #include "System/Log/ILog.h"
 #include "System/Threading/ThreadPool.h"
 #include "System/TimeUtil.h"
 
+namespace {
+	// PKWARE APPNOTE compression method for Zstandard entries.
+	constexpr uint16_t ZIP_METHOD_ZSTD = 93;
+
+	uint16_t rdLE16(const uint8_t* p) { return uint16_t(p[0]) | (uint16_t(p[1]) << 8); }
+	uint32_t rdLE32(const uint8_t* p) {
+		return uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24);
+	}
+	uint64_t rdLE64(const uint8_t* p) {
+		return uint64_t(rdLE32(p)) | (uint64_t(rdLE32(p + 4)) << 32);
+	}
+}
+
 IArchive* CZipArchiveFactory::DoCreateArchive(const std::string& filePath) const
 {
 	return new CZipArchive(filePath);
 }
 
+IArchive* CZipStdArchiveFactory::DoCreateArchive(const std::string& filePath) const
+{
+	// .sdzst is a regular zip whose entries use zstd (method 93); CZipArchive
+	// reads both transparently, see CZipArchive::GetFileImpl.
+	return new CZipArchive(filePath);
+}
+
 CZipArchive::CZipArchive(const std::string& archiveName)
 	: CBufferedArchive(archiveName)
 {
@@ -114,6 +138,93 @@ IArchive::SFileInfo CZipArchive::FileInfo(uint32_t fid) const
 	};
 }
 
+// Reads a zstd-compressed (method 93) entry. minizip cannot decode these and
+// rejects them in its open path, so we locate the raw frame directly from the
+// zip headers and decompress it with libzstd. The entry's central-directory
+// position is the unz_file_pos minizip already gave us when scanning.
+int CZipArchive::GetFileZstd(uint32_t fid, uint64_t compressedSize, uint64_t uncompressedSize, std::vector<std::uint8_t>& buffer)
+{
+	std::ifstream f(GetArchiveFile(), std::ios::binary);
+	if (!f)
+		return -4;
+
+	// Central directory file header (46-byte fixed part) for this entry.
+	const uint64_t cdOffset = fileEntries[fid].fp.pos_in_zip_directory;
+	uint8_t cd[46];
+	f.seekg(cdOffset);
+	f.read(reinterpret_cast<char*>(cd), sizeof(cd));
+	if (!f || rdLE32(cd) != 0x02014b50u)
+		return -3;
+
+	const uint16_t cdNameLen = rdLE16(cd + 28);
+	const uint16_t cdExtraLen = rdLE16(cd + 30);
+	uint64_t lhOffset = rdLE32(cd + 42); // relative offset of local header
+
+	// Zip64: a 0xFFFFFFFF offset lives in the Zip64 extended-info extra field.
+	if (lhOffset == 0xFFFFFFFFu) {
+		std::vector<uint8_t> extra(cdExtraLen);
+		f.seekg(cdOffset + 46 + cdNameLen);
+		if (cdExtraLen != 0)
+			f.read(reinterpret_cast<char*>(extra.data()), cdExtraLen);
+		if (!f)
+			return -3;
+
+		// Within tag 0x0001 the 8-byte fields appear only for the 32-bit fields
+		// that were set to 0xFFFFFFFF, in order: uncompressed, compressed, offset.
+		const size_t skip = (rdLE32(cd + 24) == 0xFFFFFFFFu ? 8 : 0)
+		                  + (rdLE32(cd + 20) == 0xFFFFFFFFu ? 8 : 0);
+		bool found = false;
+		for (size_t i = 0; i + 4 <= extra.size(); ) {
+			const uint16_t tag = rdLE16(&extra[i]);
+			const uint16_t sz = rdLE16(&extra[i + 2]);
+			if (tag == 0x0001 && i + 4 + skip + 8 <= extra.size()) {
+				lhOffset = rdLE64(&extra[i + 4 + skip]);
+				found = true;
+				break;
+			}
+			i += 4 + sz;
+		}
+		if (!found)
+			return -3;
+	}
+
+	// Local file header (30-byte fixed part); name/extra lengths can differ
+	// from the central-directory copy, so read them here.
+	uint8_t lh[30];
+	f.seekg(lhOffset);
+	f.read(reinterpret_cast<char*>(lh), sizeof(lh));
+	if (!f || rdLE32(lh) != 0x04034b50u)
+		return -3;
+
+	const uint64_t dataOffset = lhOffset + 30 + rdLE16(lh + 26) + rdLE16(lh + 28);
+
+	std::vector<std::uint8_t> compressed(compressedSize);
+	if (!compressed.empty()) {
+		f.seekg(dataOffset);
+		f.read(reinterpret_cast<char*>(compressed.data()), compressed.size());
+		if (!f)
+			return -1;
+	}
+
+	buffer.clear();
+	buffer.resize(uncompressedSize);
+
+	const size_t dSize = ZSTD_decompress(
+		buffer.empty() ? nullptr : buffer.data(), buffer.size(),
+		compressed.empty() ? nullptr : compressed.data(), compressed.size());
+	if (ZSTD_isError(dSize) || dSize != buffer.size()) {
+		buffer.clear();
+		return -1;
+	}
+
+	if (crc32(0, buffer.data(), buffer.size()) != fileEntries[fid].crc) {
+		buffer.clear();
+		return 0;
+	}
+
+	return 1;
+}
+
 // To simplify things, files are always read completely into memory from
 // the zip-file, since zlib does not provide any way of reading more
 // than one file at a time
@@ -144,6 +255,14 @@ int CZipArchive::GetFileImpl(uint32_t fid, std::vector<std::uint8_t>& buffer)
 	unz_file_info fi;
 	unzGetCurrentFileInfo(thisThreadZip, &fi, nullptr, 0, nullptr, 0, nullptr, 0);
 
+	// minizip can enumerate but not decode zstd entries (method 93). It also
+	// rejects method 93 in its open path, and the engine may link a prebuilt
+	// minizip we cannot patch. So for zstd entries we bypass minizip's decoder
+	// entirely: locate the raw frame via the zip headers and decompress it
+	// ourselves (see GetFileZstd).
+	if (fi.compression_method == ZIP_METHOD_ZSTD)
+		return GetFileZstd(fid, fi.compressed_size, fi.uncompressed_size, buffer);
+
 	if (unzOpenCurrentFile(thisThreadZip) != UNZ_OK)
 		return -3;
 

rts/System/FileSystem/Archives/ZipArchive.h

@@ -24,6 +24,19 @@ class CZipArchiveFactory : public IArchiveFactory {
 };
 
 
+/**
+ * Creates archives stored as a zip with zstd-compressed (method 93) entries.
+ * Backed by the same CZipArchive as .sdz; only the extension differs.
+ */
+class CZipStdArchiveFactory : public IArchiveFactory {
+public:
+	CZipStdArchiveFactory(): IArchiveFactory("sdzst") {}
+
+private:
+	IArchive* DoCreateArchive(const std::string& filePath) const;
+};
+
+
 /**
  * A zip compressed, single-file archive.
  */
@@ -51,6 +64,8 @@ class CZipArchive : public CBufferedArchive
 protected:
 	int GetFileImpl(uint32_t fid, std::vector<std::uint8_t>& buffer) override;
 private:
+	int GetFileZstd(uint32_t fid, uint64_t compressedSize, uint64_t uncompressedSize, std::vector<std::uint8_t>& buffer);
+
 	static constexpr int MAX_THREADS = 32;
 
 	Recoil::AtomicFirstIndex<uint32_t> afi;

rts/build/cmake/FindZstd.cmake

@@ -0,0 +1,32 @@
+# This file is part of the Spring engine (GPL v2 or later), see LICENSE.html
+
+# - Find the Zstandard (zstd) library
+# Find the native zstd includes and library (static or shared)
+#
+#  ZSTD_INCLUDE_DIR - where to find zstd.h
+#  ZSTD_LIBRARIES   - List of libraries when using zstd.
+#  ZSTD_FOUND       - True if zstd was found.
+
+Include(FindPackageHandleStandardArgs)
+
+If     (ZSTD_INCLUDE_DIR)
+  # Already in cache, be silent
+  Set(ZSTD_FIND_QUIETLY TRUE)
+EndIf  (ZSTD_INCLUDE_DIR)
+
+Find_Path(ZSTD_INCLUDE_DIR zstd.h)
+
+Set(ZSTD_NAMES zstd)
+Find_Library(ZSTD_LIBRARY NAMES ${ZSTD_NAMES})
+
+# handle the QUIETLY and REQUIRED arguments and set ZSTD_FOUND to TRUE if
+# all listed variables are TRUE
+Find_Package_Handle_Standard_Args(Zstd DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+If     (ZSTD_FOUND)
+  Set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+Else   (ZSTD_FOUND)
+  Set(ZSTD_LIBRARIES)
+EndIf  (ZSTD_FOUND)
+
+Mark_As_Advanced(ZSTD_LIBRARY ZSTD_INCLUDE_DIR)

rts/lib/CMakeLists.txt

@@ -45,6 +45,14 @@ endif()
 ADD_SUBDIRECTORY(lua)
 ADD_SUBDIRECTORY(luasocket)
 ADD_SUBDIRECTORY(minizip)
+
+# zstd: required to read .sdzst archives (zip containers with zstd / method-93
+# entries). Sourced like the other third-party libs from the prebuilt
+# static-libs / mingwlibs bundle.
+find_package_static(Zstd REQUIRED)
+set_global(SPRING_ZSTD_INCLUDE_DIR "${ZSTD_INCLUDE_DIR}")
+set_global(SPRING_ZSTD_LIBRARY "${ZSTD_LIBRARY}")
+
 ADD_SUBDIRECTORY(nowide)
 ADD_SUBDIRECTORY(headlessStubs)
 if (ENABLE_STREFLOP)