Skip to content

Commit 8661ce6

Browse files
mihemihe
committed
Add support for delta encoding to patch PCKs
1 parent 084d5d4 commit 8661ce6

17 files changed

+863
-91
lines changed

‎core/io/delta_encoding.cpp‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/**************************************************************************/
2+
/* delta_encoding.cpp */
3+
/**************************************************************************/
4+
/* This file is part of: */
5+
/* GODOT ENGINE */
6+
/* https://godotengine.org */
7+
/**************************************************************************/
8+
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9+
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10+
/* */
11+
/* Permission is hereby granted, free of charge, to any person obtaining */
12+
/* a copy of this software and associated documentation files (the */
13+
/* "Software"), to deal in the Software without restriction, including */
14+
/* without limitation the rights to use, copy, modify, merge, publish, */
15+
/* distribute, sublicense, and/or sell copies of the Software, and to */
16+
/* permit persons to whom the Software is furnished to do so, subject to */
17+
/* the following conditions: */
18+
/* */
19+
/* The above copyright notice and this permission notice shall be */
20+
/* included in all copies or substantial portions of the Software. */
21+
/* */
22+
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23+
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24+
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25+
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26+
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27+
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28+
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29+
/**************************************************************************/
30+
31+
#include "delta_encoding.h"
32+
33+
#include <zstd.h>
34+
35+
#define ERR_FAIL_ZSTD_V_MSG(m_result, m_retval, m_msg) \
36+
ERR_FAIL_COND_V_MSG(ZSTD_isError(m_result), m_retval, vformat("%s Zstandard reported error code %d: '%s'.", m_msg, ZSTD_getErrorCode(m_result), ZSTD_getErrorString(ZSTD_getErrorCode(m_result))))
37+
38+
struct ZstdCompressionContext {
39+
ZSTD_CCtx *context = ZSTD_createCCtx();
40+
~ZstdCompressionContext() { ZSTD_freeCCtx(context); }
41+
operator ZSTD_CCtx *() { return context; }
42+
};
43+
44+
struct ZstdDecompressionContext {
45+
ZSTD_DCtx *context = ZSTD_createDCtx();
46+
~ZstdDecompressionContext() { ZSTD_freeDCtx(context); }
47+
operator ZSTD_DCtx *() { return context; }
48+
};
49+
50+
struct DeltaHeader {
51+
uint8_t magic[4];
52+
uint8_t version;
53+
};
54+
55+
static_assert(alignof(DeltaHeader) == 1); // There shouldn't be any padding.
56+
57+
static constexpr size_t DELTA_HEADER_SIZE = sizeof(DeltaHeader);
58+
static constexpr uint8_t DELTA_MAGIC[4] = { 'G', 'D', 'D', 'L' };
59+
static constexpr int DELTA_VERSION_NUMBER = 1;
60+
61+
Error DeltaEncoding::encode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_new_data, Vector<uint8_t> &r_delta, int p_compression_level) {
62+
size_t zstd_result = ZSTD_compressBound(p_new_data.size());
63+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Calculating compression bounds failed.");
64+
65+
r_delta.reserve_exact(DELTA_HEADER_SIZE + zstd_result);
66+
r_delta.resize(DELTA_HEADER_SIZE + zstd_result);
67+
68+
DeltaHeader header;
69+
memcpy(header.magic, DELTA_MAGIC, 4);
70+
header.version = DELTA_VERSION_NUMBER;
71+
memcpy(r_delta.ptrw(), &header, DELTA_HEADER_SIZE);
72+
73+
ZstdCompressionContext zstd_context;
74+
75+
ZSTD_parameters zstd_params = ZSTD_getParams(p_compression_level, p_new_data.size(), p_old_data.size());
76+
zstd_params.fParams.contentSizeFlag = 1;
77+
zstd_params.fParams.checksumFlag = 1;
78+
79+
zstd_result = ZSTD_CCtx_setParams(zstd_context, zstd_params);
80+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Setting compression parameters failed.");
81+
82+
zstd_result = ZSTD_CCtx_refPrefix(zstd_context, p_old_data.ptr(), p_old_data.size());
83+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Setting prefix dictionary failed.");
84+
85+
zstd_result = ZSTD_compress2(zstd_context, r_delta.ptrw() + DELTA_HEADER_SIZE, r_delta.size() - DELTA_HEADER_SIZE, p_new_data.ptr(), p_new_data.size());
86+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to encode delta. Compression failed.");
87+
88+
r_delta.resize(DELTA_HEADER_SIZE + zstd_result);
89+
90+
return OK;
91+
}
92+
93+
Error DeltaEncoding::decode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_delta, Vector<uint8_t> &r_new_data) {
94+
ERR_FAIL_COND_V_MSG(p_delta.size() < DELTA_HEADER_SIZE, ERR_INVALID_DATA, vformat("Failed to decode delta. File size (%d) is too small.", p_delta.size()));
95+
96+
DeltaHeader header;
97+
memcpy(&header, p_delta.ptr(), DELTA_HEADER_SIZE);
98+
99+
ERR_FAIL_COND_V_MSG(memcmp(header.magic, DELTA_MAGIC, 4) != 0, ERR_FILE_CORRUPT, "Failed to decode delta. Header is invalid.");
100+
ERR_FAIL_COND_V_MSG(header.version != DELTA_VERSION_NUMBER, ERR_FILE_UNRECOGNIZED, vformat("Failed to decode delta. Expected version %d but found %d.", DELTA_VERSION_NUMBER, header.version));
101+
102+
size_t zstd_result = ZSTD_findDecompressedSize(p_delta.ptr() + DELTA_HEADER_SIZE, p_delta.size() - DELTA_HEADER_SIZE);
103+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Unable to find decompressed size.");
104+
105+
r_new_data.reserve_exact(zstd_result);
106+
r_new_data.resize(zstd_result);
107+
108+
ZstdDecompressionContext zstd_context;
109+
110+
zstd_result = ZSTD_DCtx_refPrefix(zstd_context, p_old_data.ptr(), p_old_data.size());
111+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Setting prefix dictionary failed.");
112+
113+
zstd_result = ZSTD_decompressDCtx(zstd_context, r_new_data.ptrw(), r_new_data.size(), p_delta.ptr() + DELTA_HEADER_SIZE, p_delta.size() - DELTA_HEADER_SIZE);
114+
ERR_FAIL_ZSTD_V_MSG(zstd_result, FAILED, "Failed to decode delta. Decompression failed.");
115+
ERR_FAIL_COND_V(zstd_result != (size_t)r_new_data.size(), ERR_FILE_CORRUPT);
116+
117+
return OK;
118+
}

‎core/io/delta_encoding.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/**************************************************************************/
2+
/* delta_encoding.h */
3+
/**************************************************************************/
4+
/* This file is part of: */
5+
/* GODOT ENGINE */
6+
/* https://godotengine.org */
7+
/**************************************************************************/
8+
/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9+
/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10+
/* */
11+
/* Permission is hereby granted, free of charge, to any person obtaining */
12+
/* a copy of this software and associated documentation files (the */
13+
/* "Software"), to deal in the Software without restriction, including */
14+
/* without limitation the rights to use, copy, modify, merge, publish, */
15+
/* distribute, sublicense, and/or sell copies of the Software, and to */
16+
/* permit persons to whom the Software is furnished to do so, subject to */
17+
/* the following conditions: */
18+
/* */
19+
/* The above copyright notice and this permission notice shall be */
20+
/* included in all copies or substantial portions of the Software. */
21+
/* */
22+
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23+
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24+
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25+
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26+
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27+
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28+
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29+
/**************************************************************************/
30+
31+
#pragma once
32+
33+
#include "core/io/file_access.h"
34+
35+
class DeltaEncoding {
36+
public:
37+
static Error encode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_new_data, Vector<uint8_t> &r_delta, int p_compression_level = 19);
38+
static Error decode_delta(Span<uint8_t> p_old_data, Span<uint8_t> p_delta, Vector<uint8_t> &r_new_data);
39+
};

‎core/io/file_access_pack.cpp‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "file_access_pack.h"
3232

3333
#include "core/io/file_access_encrypted.h"
34+
#include "core/io/file_access_patched.h"
3435
#include "core/object/script_language.h"
3536
#include "core/os/os.h"
3637
#include "core/version.h"
@@ -45,7 +46,7 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t
4546
return ERR_FILE_UNRECOGNIZED;
4647
}
4748

48-
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) {
49+
void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle, bool p_delta) {
4950
String simplified_path = p_path.simplify_path().trim_prefix("res://");
5051
PathMD5 pmd5(simplified_path.md5_buffer());
5152

@@ -54,6 +55,7 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
5455
PackedFile pf;
5556
pf.encrypted = p_encrypted;
5657
pf.bundle = p_bundle;
58+
pf.delta = p_delta;
5759
pf.pack = p_pkg_path;
5860
pf.offset = p_ofs;
5961
pf.size = p_size;
@@ -62,8 +64,11 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64
6264
}
6365
pf.src = p_src;
6466

65-
if (!exists || p_replace_files) {
67+
if (p_delta) {
68+
delta_patches[pmd5].push_back(pf);
69+
} else if (!exists || p_replace_files) {
6670
files[pmd5] = pf;
71+
delta_patches[pmd5].clear();
6772
}
6873

6974
if (!exists) {
@@ -137,6 +142,28 @@ uint8_t *PackedData::get_file_hash(const String &p_path) {
137142
return E->value.md5;
138143
}
139144

145+
Vector<PackedData::PackedFile> PackedData::get_delta_patches(const String &p_path) const {
146+
String simplified_path = p_path.simplify_path().trim_prefix("res://");
147+
PathMD5 pmd5(simplified_path.md5_buffer());
148+
HashMap<PathMD5, Vector<PackedFile>, PathMD5>::ConstIterator E = delta_patches.find(pmd5);
149+
if (!E) {
150+
return Vector<PackedFile>();
151+
}
152+
153+
return E->value;
154+
}
155+
156+
bool PackedData::has_delta_patches(const String &p_path) const {
157+
String simplified_path = p_path.simplify_path().trim_prefix("res://");
158+
PathMD5 pmd5(simplified_path.md5_buffer());
159+
HashMap<PathMD5, Vector<PackedFile>, PathMD5>::ConstIterator E = delta_patches.find(pmd5);
160+
if (!E) {
161+
return false;
162+
}
163+
164+
return !E->value.is_empty();
165+
}
166+
140167
HashSet<String> PackedData::get_file_paths() const {
141168
HashSet<String> file_paths;
142169
_get_file_paths(root, root->name, file_paths);
@@ -155,6 +182,7 @@ void PackedData::_get_file_paths(PackedDir *p_dir, const String &p_parent_dir, H
155182

156183
void PackedData::clear() {
157184
files.clear();
185+
delta_patches.clear();
158186
_free_packed_dirs(root);
159187
root = memnew(PackedDir);
160188
}
@@ -322,15 +350,25 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files,
322350
if (flags & PACK_FILE_REMOVAL) { // The file was removed.
323351
PackedData::get_singleton()->remove_path(path);
324352
} else {
325-
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle);
353+
PackedData::get_singleton()->add_path(p_path, path, file_base + ofs, size, md5, this, p_replace_files, (flags & PACK_FILE_ENCRYPTED), sparse_bundle, (flags & PACK_FILE_DELTA));
326354
}
327355
}
328356

329357
return true;
330358
}
331359

332360
Ref<FileAccess> PackedSourcePCK::get_file(const String &p_path, PackedData::PackedFile *p_file) {
333-
return memnew(FileAccessPack(p_path, *p_file));
361+
Ref<FileAccess> file(memnew(FileAccessPack(p_path, *p_file)));
362+
363+
if (PackedData::get_singleton()->has_delta_patches(p_path)) {
364+
Ref<FileAccessPatched> file_patched;
365+
file_patched.instantiate();
366+
Error err = file_patched->open_custom(file);
367+
ERR_FAIL_COND_V(err != OK, Ref<FileAccess>());
368+
file = file_patched;
369+
}
370+
371+
return file;
334372
}
335373

336374
//////////////////////////////////////////////////////////////////
@@ -362,7 +400,7 @@ void PackedSourceDirectory::add_directory(const String &p_path, bool p_replace_f
362400
for (const String &file_name : da->get_files()) {
363401
String file_path = p_path.path_join(file_name);
364402
uint8_t md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
365-
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false);
403+
PackedData::get_singleton()->add_path(p_path, file_path, 0, 0, md5, this, p_replace_files, false, false, false);
366404
}
367405

368406
for (const String &sub_dir_name : da->get_directories()) {
@@ -470,6 +508,7 @@ void FileAccessPack::close() {
470508
}
471509

472510
FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFile &p_file) {
511+
path = p_path;
473512
pf = p_file;
474513
if (pf.bundle) {
475514
String simplified_path = p_path.simplify_path();

‎core/io/file_access_pack.h‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ enum PackFlags {
5454
enum PackFileFlags {
5555
PACK_FILE_ENCRYPTED = 1 << 0,
5656
PACK_FILE_REMOVAL = 1 << 1,
57+
PACK_FILE_DELTA = 1 << 2,
5758
};
5859

5960
class PackSource;
@@ -72,6 +73,7 @@ class PackedData {
7273
PackSource *src = nullptr;
7374
bool encrypted;
7475
bool bundle;
76+
bool delta;
7577
};
7678

7779
private:
@@ -103,6 +105,7 @@ class PackedData {
103105
};
104106

105107
HashMap<PathMD5, PackedFile, PathMD5> files;
108+
HashMap<PathMD5, Vector<PackedFile>, PathMD5> delta_patches;
106109

107110
Vector<PackSource *> sources;
108111

@@ -116,9 +119,11 @@ class PackedData {
116119

117120
public:
118121
void add_pack_source(PackSource *p_source);
119-
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false); // for PackSource
122+
void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false, bool p_delta = false); // for PackSource
120123
void remove_path(const String &p_path);
121124
uint8_t *get_file_hash(const String &p_path);
125+
Vector<PackedFile> get_delta_patches(const String &p_path) const;
126+
bool has_delta_patches(const String &p_path) const;
122127
HashSet<String> get_file_paths() const;
123128

124129
void set_disabled(bool p_disabled) { disabled = p_disabled; }
@@ -166,6 +171,7 @@ class FileAccessPack : public FileAccess {
166171
GDSOFTCLASS(FileAccessPack, FileAccess);
167172
PackedData::PackedFile pf;
168173

174+
String path;
169175
mutable uint64_t pos;
170176
mutable bool eof;
171177
uint64_t off;
@@ -186,6 +192,9 @@ class FileAccessPack : public FileAccess {
186192
public:
187193
virtual bool is_open() const override;
188194

195+
virtual String get_path() const override { return path; }
196+
virtual String get_path_absolute() const override { return path; }
197+
189198
virtual void seek(uint64_t p_position) override;
190199
virtual void seek_end(int64_t p_position = 0) override;
191200
virtual uint64_t get_position() const override;

0 commit comments

Comments
 (0)