From 2d6705e27d369a02cc38c91c098feed7205dd22a Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:42:19 +0200 Subject: [PATCH 01/19] fix(file_archiver): do not recursively add dir content when using `AppenderEntries` --- .../src/tools/file_archiver/appender.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/mithril-aggregator/src/tools/file_archiver/appender.rs index 1e7b4de95ea..f84c7308cc2 100644 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ b/mithril-aggregator/src/tools/file_archiver/appender.rs @@ -132,7 +132,7 @@ impl TarAppender for AppenderEntries { for entry in &self.entries { let entry_path = self.base_directory.join(entry); if entry_path.is_dir() { - tar.append_dir_all(entry, entry_path.clone()).with_context(|| { + tar.append_dir(entry, entry_path.clone()).with_context(|| { format!( "Can not add directory: '{}' to the archive", entry_path.display() @@ -277,9 +277,17 @@ mod tests { let source = test_dir.join(create_dir(&test_dir, "source")); let directory_to_archive_path = create_dir(&source, "directory_to_archive"); + let file_in_dir_to_archive_path = + create_file(&source, "directory_to_archive/file_in_dir_to_archive.txt"); let file_to_archive_path = create_file(&source, "file_to_archive.txt"); + let empty_directory_to_archive_path = create_dir(&source, "empty_directory_to_archive"); + create_dir(&source, "directory_not_to_archive"); create_file(&source, "file_not_to_archive.txt"); + create_file( + &source, + "directory_to_archive/file_in_dir_not_to_archive.txt", + ); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -291,7 +299,12 @@ mod tests { compression_algorithm: CompressionAlgorithm::Zstandard, }, AppenderEntries::new( - vec![directory_to_archive_path.clone(), file_to_archive_path.clone()], + vec![ + directory_to_archive_path, + file_in_dir_to_archive_path, + file_to_archive_path, + empty_directory_to_archive_path, + ], source, ) .unwrap(), @@ -303,6 +316,8 @@ mod tests { assert_dir_eq!( &unpack_path, "* directory_to_archive/ + ** file_in_dir_to_archive.txt + * empty_directory_to_archive/ * file_to_archive.txt" ); } From f86fb872dafa68a42dde2103cdad4b0e627aa57f Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:57:00 +0200 Subject: [PATCH 02/19] test(aggregator): add tests harness for file archiver reproductibility --- Cargo.lock | 1 + mithril-aggregator/Cargo.toml | 1 + .../src/tools/file_archiver/mod.rs | 2 + .../src/tools/file_archiver/tests.rs | 1005 +++++++++++++++++ 4 files changed, 1009 insertions(+) create mode 100644 mithril-aggregator/src/tools/file_archiver/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 37aaa19a061..7f0ca79e39c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4259,6 +4259,7 @@ dependencies = [ "semver", "serde", "serde_json", + "sha2 0.10.9", "slog", "slog-async", "slog-bunyan", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 41c42557ba2..bf094843cb0 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -78,6 +78,7 @@ mithril-api-spec = { path = "../internal/tests/mithril-api-spec" } mithril-common = { path = "../mithril-common", features = ["allow_skip_signer_certification"] } mithril-test-http-server = { path = "../internal/tests/mithril-test-http-server" } mockall = { workspace = true } +sha2 = "0.10.9" slog-scope = "4.4.1" slog-term = { workspace = true } tempfile = "3.27.0" diff --git a/mithril-aggregator/src/tools/file_archiver/mod.rs b/mithril-aggregator/src/tools/file_archiver/mod.rs index 024f937f9a0..280754d8a43 100644 --- a/mithril-aggregator/src/tools/file_archiver/mod.rs +++ b/mithril-aggregator/src/tools/file_archiver/mod.rs @@ -1,6 +1,8 @@ mod api; pub mod appender; mod entities; +#[cfg(test)] +mod tests; pub use api::*; pub use entities::*; diff --git a/mithril-aggregator/src/tools/file_archiver/tests.rs b/mithril-aggregator/src/tools/file_archiver/tests.rs new file mode 100644 index 00000000000..879e7b28750 --- /dev/null +++ b/mithril-aggregator/src/tools/file_archiver/tests.rs @@ -0,0 +1,1005 @@ +//! Reproducibility contract for archives produced by [`FileArchiver`]. +//! +//! Given identical archive entry paths and contents, `FileArchiver` must produce a +//! byte-identical `.tar.zst` archive regardless of the host system, source base +//! directory, creation time, modification times, permissions, or input entry order. +//! +//! This contract assumes identical archive-format dependencies and zstandard +//! compression parameters. Changing either is an archive-format change and requires +//! intentionally versioning the format and updating its golden hashes. + +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Path, PathBuf}; + +use mithril_common::entities::CompressionAlgorithm; +use mithril_common::temp_dir_create; + +use crate::ZstandardCompressionParameters; +use crate::test::TestLogger; +use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::*}; + +mod helpers { + use sha2::{Digest, Sha256}; + use std::io::{Read, Write}; + + use super::*; + + /// Assert that two files have exactly the same bytes. + /// + /// SHA-256 hashes and file sizes are reported when they differ. + #[track_caller] + pub fn assert_files_are_byte_identical, A: AsRef>(expected: E, actual: A) { + let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); + let expected_bytes = std::fs::read(expected_path).unwrap_or_else(|error| { + panic!( + "Could not read expected file '{}': {error}", + expected_path.display() + ) + }); + let actual_bytes = std::fs::read(actual_path).unwrap_or_else(|error| { + panic!( + "Could not read actual file '{}': {error}", + actual_path.display() + ) + }); + + if expected_bytes != actual_bytes { + let expected_hash = hex::encode(Sha256::digest(&expected_bytes)); + let actual_hash = hex::encode(Sha256::digest(&actual_bytes)); + + panic!( + "Files are not byte-identical:\n\ + expected: '{}' ({} bytes, SHA-256: {})\n\ + actual: '{}' ({} bytes, SHA-256: {})", + expected_path.display(), + expected_bytes.len(), + expected_hash, + actual_path.display(), + actual_bytes.len(), + actual_hash, + ); + } + } + + /// **IMPORTANT** Default zstandard compression parameters are used. + pub fn file_archiver(work_dir: &Path) -> FileArchiver { + FileArchiver::new( + ZstandardCompressionParameters::default(), + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + + pub fn create_dir>(base_dir: &Path, dir_name: P) -> PathBuf { + let dir_path = base_dir.join(dir_name); + std::fs::create_dir(&dir_path).unwrap(); + dir_path + } + + pub fn create_file(root_dir: &Path, file_name: &str, size: Option) -> PathBuf { + let path = root_dir.join(file_name); + let mut file = File::create(&path).unwrap(); + + write!(file, "This is a test file named '{file_name}'").unwrap(); + writeln!(file).unwrap(); + + if let Some(file_size) = size { + file.set_len(file_size).unwrap(); + } + + path + } + + pub fn alter_file(path: &Path, alter_fn: F) { + let file = File::options().write(true).open(path).unwrap(); + alter_fn(&file); + } + + pub fn archive_parameters(filename: &str, target_dir: &Path) -> ArchiveParameters { + ArchiveParameters { + archive_name_without_extension: filename.to_string(), + target_directory: target_dir.to_path_buf(), + compression_algorithm: CompressionAlgorithm::Zstandard, + } + } + + pub fn compute_file_sha256(path: &Path) -> String { + let mut hasher = Sha256::new(); + + if path.is_file() { + hash_file(path, &mut hasher); + } else { + panic!("Path is not a file: {:?}", path.display()); + } + + hex::encode(hasher.finalize()) + } + + fn hash_file(path: &Path, hasher: &mut Sha256) { + let mut file = File::open(path).unwrap(); + let mut buffer = [0; 64 * 1024]; + + loop { + let bytes_read = file.read(&mut buffer).unwrap(); + if bytes_read == 0 { + break; + } + + hasher.update(&buffer[..bytes_read]); + } + } + + #[test] + fn assert_files_are_byte_identical_succeed_with_identical_files() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let identical_file_path = test_dir.join("altered_file"); + std::fs::copy(&reference_file_path, &identical_file_path).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &identical_file_path); + } + + #[test] + #[should_panic(expected = "Files are not byte-identical")] + fn assert_files_are_byte_identical_fails_if_one_byte_changes() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let altered_content = { + let mut altered_content = content.clone(); + altered_content[0] = !altered_content[0]; + altered_content + }; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let altered_file_path = test_dir.join("altered_file"); + let mut altered_file = File::create_new(&altered_file_path).unwrap(); + altered_file.write_all(&altered_content).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &altered_file_path); + } +} + +/// Fixed test data set for file archiver tests. +mod test_data { + use serde::{Deserialize, Serialize}; + + use super::*; + + pub const TEST_BYTES: [u8; 10] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + /// See: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks + pub(crate) const ZSTD_MAX_BLOCK_SIZE: u64 = 131_072; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + pub struct TestStruct { + field_str: String, + field_int: i32, + // Important note: using BTreeMap to ensure deterministic serialization, types like HashMap should + // not be used as the key order is not guaranteed + field_map: BTreeMap, + } + + impl Default for TestStruct { + fn default() -> Self { + Self { + field_str: "test".to_string(), + field_int: 42, + field_map: BTreeMap::from([ + ("key_3".to_string(), "value_3".to_string()), + ("key_1".to_string(), "value_1".to_string()), + ("key_2".to_string(), "value_2".to_string()), + ]), + } + } + } + + /// Create a file named `test.txt` in the given directory + pub fn create_test_txt(root_dir: &Path) -> PathBuf { + helpers::create_file(root_dir, "test.txt", Some(1048)) + } + + /// [AppenderEntries] ready list of entries created by [`create_test_dir`] + pub fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + ] + } + + /// Create a directory named `test_dir` in the given directory + /// + /// It will be filled with the following structure: + /// ```no_run + /// test_dir/ + /// ├── bar/ + /// ├── foo/ + /// │ └── bar.txt + /// ├── file_1.txt + /// ├── file_2.txt + /// ├── empty_file.txt + /// └── file_with_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt + /// ``` + pub fn create_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file(&test_dir, "file_2.txt", Some(ZSTD_MAX_BLOCK_SIZE * 2 + 1)); + + test_dir + } +} + +mod reproducibility { + use std::time::{Duration, SystemTime}; + + use super::*; + + mod repeated_archiving_produces_byte_identical_archives { + use std::time::Instant; + + use super::*; + + fn run_scenario(test_dir: PathBuf, ref_appender: T, repeated_appender: T) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || {}); + } + + fn run_scenario_in_different_unix_seconds( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + ) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || { + let seconds_since_unix_epoch = || { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + }; + + let reference_second = seconds_since_unix_epoch(); + let deadline = Instant::now() + Duration::from_millis(1500); + + while seconds_since_unix_epoch() == reference_second { + assert!( + Instant::now() < deadline, + "Unix time did not advance to the next second" + ); + std::thread::sleep(Duration::from_millis(10)); + } + }); + } + + fn run_scenario_with_hook( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + before_repeated_archive: F, + ) { + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + ref_appender, + ) + .unwrap(); + + before_repeated_archive(); + + let repeated_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("repeated", &test_dir), + repeated_appender, + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.filepath, + repeated_archive.filepath, + ); + } + + #[test] + fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ); + } + + #[test] + fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content.clone()), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderFile::append_at_archive_root(content.clone()).unwrap(), + AppenderFile::append_at_archive_root(content).unwrap(), + ); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderDirAll::new(content.clone()), + AppenderDirAll::new(content), + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ); + } + } + + mod source_metadata_does_not_affect_archive_bytes { + use std::fs::Permissions; + + use super::*; + + /// [AppenderEntries] ready list of the reference/altered dirs created by [setup_test_dirs]. + fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("empty/"), + PathBuf::from("subdir/"), + PathBuf::from("subdir/file_2.txt"), + PathBuf::from("subdir/file_3.txt"), + PathBuf::from("file_1.txt"), + ] + } + + /// Create two directories with the same structure and applies the given function to each + /// reference/altered files pair. + /// + /// To be used with Appender that can work on a group of files and directories. + /// + /// Each directory contains the following files: + /// ```no_run + /// (reference|altered)/ + /// ├── empty/ + /// ├── subdir/ + /// │ ├── file_2.txt + /// │ └── file_3.txt + /// └── file_1.txt + /// ``` + fn setup_test_dirs( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + let subdirs = vec![Path::new("empty"), Path::new("subdir")]; + + for dir_path in &subdirs { + helpers::create_dir(&reference_dir, dir_path); + helpers::create_dir(&altered_dir, dir_path); + } + + for file_path in ["file_1.txt", "subdir/file_2.txt", "subdir/file_3.txt"] { + let reference_file = helpers::create_file(&reference_dir, file_path, None); + let altered_file = helpers::create_file(&altered_dir, file_path, None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + // Alter dirs after the files to avoid permission issues + for dir_path in &subdirs { + setup_reference_and_altered_path_fn( + &reference_dir.join(dir_path), + &altered_dir.join(dir_path), + ); + } + setup_reference_and_altered_path_fn(&reference_dir, &altered_dir); + } + + /// Create two directories with a single "test.txt" file and applies the given function + /// to the reference/altered file pair. + /// + /// To be used with Appender that can work on a single file. + fn setup_test_file( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + + let reference_file = helpers::create_file(&reference_dir, "file.txt", None); + let altered_file = helpers::create_file(&altered_dir, "file.txt", None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + fn run_scenario A>( + test_dir: PathBuf, + build_tar_appender: B, + ) { + let source = test_dir.join("source"); + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + build_tar_appender(source.join("reference")), + ) + .unwrap(); + + let archive_with_different_metadata = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("altered_metadata", &test_dir), + build_tar_appender(source.join("altered")), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.filepath, + archive_with_different_metadata.filepath, + ); + } + + mod modification_time { + use super::*; + + fn setup_modification_time(reference_path: &Path, path_with_different_metadata: &Path) { + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + // can't modify dir times until + // is stabilized (currently planned for rust 1.99) + return; + } + + let base_time = SystemTime::UNIX_EPOCH; + helpers::alter_file(reference_path, |file| file.set_modified(base_time).unwrap()); + helpers::alter_file(path_with_different_metadata, |file| { + // note: in TAR, entries mtimes have a granularity of a second + file.set_modified(base_time + Duration::from_millis(5300)).unwrap() + }); + + assert_ne!( + reference_path.metadata().unwrap().modified().unwrap(), + path_with_different_metadata.metadata().unwrap().modified().unwrap() + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_modification_time); + run_scenario(test_dir, AppenderDirAll::new); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } + + mod permissions { + use super::*; + + #[cfg(unix)] + fn setup_permissions(reference_path: &Path, path_with_different_metadata: &Path) { + use std::os::unix::fs::PermissionsExt; + + let (reference_permission, altered_permission) = + // IMPORTANT: for directory the owner permission must be `7`, else this prevents + // `temp_dir_create` cleanup and make appenders fails on subdir files + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + (Permissions::from_mode(0o766), Permissions::from_mode(0o767)) + } else { + (Permissions::from_mode(0o644), Permissions::from_mode(0o646)) + }; + + std::fs::set_permissions(reference_path, reference_permission).unwrap(); + std::fs::set_permissions(path_with_different_metadata, altered_permission).unwrap(); + + assert_ne!( + reference_path.metadata().unwrap().permissions(), + path_with_different_metadata.metadata().unwrap().permissions() + ); + } + + #[cfg(unix)] + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }) + } + + #[cfg(unix)] + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_permissions); + run_scenario(test_dir, AppenderDirAll::new); + } + + #[cfg(unix)] + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } + } + + mod source_base_directory_does_not_affect_archive { + use super::*; + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_txt(&subdir_1); + let same_content_in_other_dir = test_data::create_test_txt(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderFile::append_at_archive_root(same_content_in_other_dir).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + &archive.filepath, + &archive_with_same_content_but_from_another_dir.filepath, + ); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_dir(&subdir_1); + let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderDirAll::new(content), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderDirAll::new(same_content_in_other_dir), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + &archive.filepath, + &archive_with_same_content_but_from_another_dir.filepath, + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_dir(&subdir_1); + let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), same_content_in_other_dir) + .unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + &archive.filepath, + &archive_with_same_content_but_from_another_dir.filepath, + ); + } + } + + mod appender_entry_specifics { + use super::*; + + fn to_entries(paths: [&str; N]) -> Vec { + paths.into_iter().map(PathBuf::from).collect() + } + + #[test] + fn equivalent_entry_paths_produce_identical_archives() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_entries = ["bar/", "foo/", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + let equivalent_entries = ["bar", "foo", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(to_entries(reference_entries), content.clone()).unwrap(), + ) + .unwrap(); + + let archive_with_equivalent_entries_spelling = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("equivalent_spelling", &test_dir), + AppenderEntries::new(to_entries(equivalent_entries), content).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + &reference_archive.filepath, + &archive_with_equivalent_entries_spelling.filepath, + ); + } + + #[test] + fn supplied_entry_order_does_not_affect_appender_entries_archive() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + ) + .unwrap(); + + for (label, entries) in [ + ( + "child_before_parent_dir", + ["bar/", "foo/bar.txt", "foo/", "file_1.txt", "file_2.txt"], + ), + ( + "directories_before_files", + ["foo/", "bar/", "file_2.txt", "file_1.txt", "foo/bar.txt"], + ), + ( + "files_before_directories", + ["file_2.txt", "file_1.txt", "foo/bar.txt", "foo/", "bar/"], + ), + ( + "reverse_dirs_order", + ["foo/", "bar/", "foo/bar.txt", "file_1.txt", "file_2.txt"], + ), + ( + "reverse_files_order", + ["bar/", "foo/", "foo/bar.txt", "file_2.txt", "file_1.txt"], + ), + ] { + let archive_with_same_content_but_different_entries_order = + helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters(label, &test_dir), + AppenderEntries::new(to_entries(entries), content.clone()).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + &reference_archive.filepath, + &archive_with_same_content_but_different_entries_order.filepath, + ); + } + } + } +} + +// The golden hashes are pinned to: the tar/zstd crate versions, GOLDEN_COMPRESSION_PARAMETERS +// (multi-threaded zstd), and the exact bytes produced by `test_data::create_test_*`. +// Any change to one of those invalidates them. +mod golden_master { + use super::*; + + // ** These hashes define TAR_ZSTD_V1. Update them only for an intentional archive-format change. ** + pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = + "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; + pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256: &str = + "647bba62029e63b0cbd329bccdfdb850ec934a07ab0811cc4d2d7682b72d3054"; + pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = + "565da17762fcf8f417f1ad78fda003b19507757bf56ac09ff24a9aac179bf78c"; + pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = + "fd5a178f1c717de39aef526d378e731871b74ee1ba11bdc159a4a872867d3748"; + pub const TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256: &str = + "6843b658c2c176afa156e82f7ee3d828f63cd7e9de16dc3166a5da96f0d3e15f"; + pub const TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256: &str = + "ff5447a1b80b530b3d94cfeb2ad69fdc392a4d5175f24ffc13ad10b08a7e12ba"; + + /// Create a directory named `test_dir` in the given directory + /// + /// It will be filled with the following structure: + /// ```no_run + /// test_dir/ + /// ├── bar/ + /// ├── foo/ + /// │ └── bar.txt + /// ├── empty_file.txt + /// ├── file_1.txt + /// ├── file_2.txt + /// └── file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt + /// ``` + pub fn create_golden_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + File::create(test_dir.join("empty.txt")).unwrap(); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file( + &test_dir, + "file_2.txt", + Some(test_data::ZSTD_MAX_BLOCK_SIZE * 2 + 1), + ); + helpers::create_file( + &test_dir, + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + None, + ); + + test_dir + } + + /// [AppenderEntries] ready list of entries created by [`create_golden_test_dir`] + pub fn golden_test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("empty.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from( + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + ), + ] + } + + #[track_caller] + fn assert_archive_not_empty(archive: &FileArchive) { + assert!( + archive.uncompressed_size > 0, + "Archive '{}' has no content, fix the archive creation and try again", + archive.filepath.display() + ); + } + + /// Assert an archive matches its golden hash. + #[track_caller] + fn assert_archive_matches_golden_sha256(archive: &FileArchive, expected_sha256: &str) { + let actual_sha256 = helpers::compute_file_sha256(&archive.filepath); + + assert_eq!( + expected_sha256, + actual_sha256, + "Archive bytes changed ('{}', {} bytes).\n\ + Either the archive format is no longer reproducible, or the change is intentional \ + and the TAR_ZSTD_V* constants must be recomputed and their version bumped.", + archive.filepath.display(), + archive.archive_filesize, + ); + } + + /// Compression parameters the golden hashes were computed with. + /// + /// They must mirror the production defaults, see [defaults_are_the_golden_parameters]. + const GOLDEN_COMPRESSION_PARAMETERS: ZstandardCompressionParameters = + ZstandardCompressionParameters { + level: 9, + number_of_workers: 4, + }; + + #[test] + fn defaults_are_the_golden_parameters() { + assert_eq!( + GOLDEN_COMPRESSION_PARAMETERS, + ZstandardCompressionParameters::default(), + "zstandard defaults changed: archives are no longer byte-compatible with the \ + previously published ones, the golden hashes must be recomputed and TAR_ZSTD_V1 \ + bumped to V2" + ); + } + + #[test] + fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_DATA_SHA256); + } + + #[test] + fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_FILE_SHA256); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderDirAll::new(content), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256, + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + // Should construct the same archive as `AppenderDirAll` as we include all the source directory entries + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256, + ); + } + + #[test] + fn chain_appender() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content) + .unwrap() + .chain( + AppenderData::from_json( + PathBuf::from("test.json"), + &test_data::TestStruct::default(), + ) + .unwrap(), + ), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256, + ); + } +} From 0380a6dd03225ee7ded2b0b2d9e4bffcc4854914 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:57:40 +0200 Subject: [PATCH 03/19] refactor(file_archiver): use deterministic header mode and disable symlink following in tar builder configuration --- mithril-aggregator/src/tools/file_archiver/api.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mithril-aggregator/src/tools/file_archiver/api.rs b/mithril-aggregator/src/tools/file_archiver/api.rs index bce2e595888..0e3253f9c34 100644 --- a/mithril-aggregator/src/tools/file_archiver/api.rs +++ b/mithril-aggregator/src/tools/file_archiver/api.rs @@ -6,7 +6,7 @@ use std::{ io::{Read, Seek, SeekFrom}, path::{Path, PathBuf}, }; -use tar::{Archive, Entry, EntryType}; +use tar::{Archive, Entry, EntryType, HeaderMode}; use zstd::{Decoder, Encoder}; use mithril_common::StdResult; @@ -152,6 +152,7 @@ impl FileArchiver { enc.multithread(self.zstandard_compression_parameter.number_of_workers) .with_context(|| "ZstandardEncoder can not set the number of workers")?; let mut tar = tar::Builder::new(enc); + Self::configure_tar_builder(&mut tar); appender .append(&mut tar) @@ -272,6 +273,11 @@ impl FileArchiver { Ok(()) } + + fn configure_tar_builder(builder: &mut tar::Builder) { + builder.mode(HeaderMode::Deterministic); + builder.follow_symlinks(false); + } } #[cfg(test)] From fae6bbee37fe3f2bc7862f26eee07cbbe8231ba3 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:26:24 +0200 Subject: [PATCH 04/19] fix(aggregator): use fixed mtime for AppenderData entries to ensure reproducibility --- .../src/tools/file_archiver/appender.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/mithril-aggregator/src/tools/file_archiver/appender.rs index f84c7308cc2..c5abe2adfc1 100644 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ b/mithril-aggregator/src/tools/file_archiver/appender.rs @@ -175,6 +175,10 @@ pub struct AppenderData { } impl AppenderData { + /// Timestamp arbitrarily chosen to `2026-01-01 00:00:00 UTC` + /// IMPORTANT: Do NOT change it, else the `AppenderData` archives bytes would change. + const FIXED_MTIME_ATTRIBUTE: u64 = 1767225600; + /// Create a new instance of `AppenderData` from an object that will be serialized to JSON. pub fn from_json( location_in_archive: PathBuf, @@ -204,7 +208,7 @@ impl TarAppender for AppenderData { let mut header = tar::Header::new_gnu(); header.set_size(self.bytes.len() as u64); header.set_mode(READ_WRITE_PERMISSION); - header.set_mtime(chrono::Utc::now().timestamp() as u64); + header.set_mtime(Self::FIXED_MTIME_ATTRIBUTE); header.set_cksum(); tar.append_data( @@ -551,14 +555,13 @@ mod tests { } #[test] - fn appended_entry_have_read_write_permissions_and_time_metadata() { + fn appended_entry_have_read_write_permissions_and_fixed_time_metadata() { let test_dir = temp_dir_create!(); let object = TestStruct { field1: "test".to_string(), field2: 42, }; let location_in_archive = PathBuf::from("folder").join("test.json"); - let start_time_stamp = chrono::Utc::now().timestamp() as u64; let data_appender = AppenderData::from_json(location_in_archive.clone(), &object).unwrap(); @@ -584,11 +587,7 @@ mod tests { appended_entry.header().mode().unwrap() ); let mtime = appended_entry.header().mtime().unwrap(); - assert!( - mtime >= start_time_stamp, - "entry mtime should be greater than or equal to the timestamp before the archive \ - creation:\n {mtime} < {start_time_stamp}" - ); + assert_eq!(AppenderData::FIXED_MTIME_ATTRIBUTE, mtime); } #[test] From 03f27e6b5c36fd028d36b0f21bd6a0d74f13ba97 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:21:47 +0200 Subject: [PATCH 05/19] fix(aggregator): normalize and sort entries in `AppenderEntries` for deterministic archive output --- .../src/tools/file_archiver/appender.rs | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/mithril-aggregator/src/tools/file_archiver/appender.rs index c5abe2adfc1..a24367f41a0 100644 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ b/mithril-aggregator/src/tools/file_archiver/appender.rs @@ -114,6 +114,8 @@ pub struct AppenderEntries { impl AppenderEntries { /// Create a new instance of `AppenderEntries`. /// + /// Entries are normalized and sorted to ensure deterministic archive output. + /// /// Returns an error if the `entries` are empty. pub fn new(entries: Vec, base_directory: PathBuf) -> StdResult { if entries.is_empty() { @@ -121,10 +123,19 @@ impl AppenderEntries { } Ok(Self { - entries, + entries: Self::normalize_entries(entries), base_directory, }) } + + fn normalize_entries(entries: Vec) -> Vec { + let mut normalized: Vec = entries + .into_iter() + .map(|entry| entry.components().collect()) + .collect(); + normalized.sort(); + normalized + } } impl TarAppender for AppenderEntries { @@ -132,14 +143,14 @@ impl TarAppender for AppenderEntries { for entry in &self.entries { let entry_path = self.base_directory.join(entry); if entry_path.is_dir() { - tar.append_dir(entry, entry_path.clone()).with_context(|| { + tar.append_dir(entry, &entry_path).with_context(|| { format!( "Can not add directory: '{}' to the archive", entry_path.display() ) })?; } else if entry_path.is_file() { - let mut file = File::open(entry_path.clone())?; + let mut file = File::open(&entry_path)?; tar.append_file(entry, &mut file).with_context(|| { format!( "Can not add file: '{}' to the archive", @@ -275,6 +286,36 @@ mod tests { mod appender_entries { use super::*; + #[test] + fn normalizes_directory_spelling_and_sorts_entries() { + let appender = AppenderEntries::new( + vec![ + PathBuf::from("foo/bar.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/pika/"), + PathBuf::from("foo/pika/chuu.txt"), + PathBuf::from("file_1.txt"), + ], + PathBuf::from("source"), + ) + .unwrap(); + + assert_eq!( + vec![ + PathBuf::from("bar"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from("foo"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("foo/pika"), + PathBuf::from("foo/pika/chuu.txt"), + ], + appender.entries + ); + } + #[test] fn create_archive_only_for_specified_directories_and_files() { let test_dir = temp_dir_create!(); From c2283149895265b57c45c2a27500f5d91852794e Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:39:06 +0200 Subject: [PATCH 06/19] refactor(aggregator): extract `normalize_entry` method and add Windows-specific tests --- .../src/tools/file_archiver/appender.rs | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/mithril-aggregator/src/tools/file_archiver/appender.rs index a24367f41a0..a043af76561 100644 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ b/mithril-aggregator/src/tools/file_archiver/appender.rs @@ -129,13 +129,14 @@ impl AppenderEntries { } fn normalize_entries(entries: Vec) -> Vec { - let mut normalized: Vec = entries - .into_iter() - .map(|entry| entry.components().collect()) - .collect(); + let mut normalized: Vec = entries.into_iter().map(Self::normalize_entry).collect(); normalized.sort(); normalized } + + fn normalize_entry(entry: PathBuf) -> PathBuf { + entry.components().collect() + } } impl TarAppender for AppenderEntries { @@ -316,6 +317,40 @@ mod tests { ); } + #[cfg(windows)] + #[test] + fn normalizes_windows_separators_and_sorts_entries() { + let appender = AppenderEntries::new( + vec![ + PathBuf::from(r"foo\pika\chuu.txt"), + PathBuf::from(r"foo\bar.txt"), + PathBuf::from(r"bar\\"), + PathBuf::from(r"foo\\"), + ], + PathBuf::from("source"), + ) + .unwrap(); + + assert_eq!( + vec![ + PathBuf::from("bar"), + PathBuf::from("foo"), + PathBuf::from("foo").join("bar.txt"), + PathBuf::from("foo").join("pika").join("chuu.txt"), + ], + appender.entries + ); + } + + #[cfg(windows)] + #[test] + fn forward_and_backward_separators_have_the_same_normalized_path() { + assert_eq!( + AppenderEntries::normalize_entry(PathBuf::from("foo/bar.txt")), + AppenderEntries::normalize_entry(PathBuf::from(r"foo\bar.txt")), + ); + } + #[test] fn create_archive_only_for_specified_directories_and_files() { let test_dir = temp_dir_create!(); From 895782673007a4811fec7f244eb9842d6327402f Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:02:37 +0200 Subject: [PATCH 07/19] chore: scaffold `internal/mithril-file-archiver` crate --- .github/workflows/ci.yml | 4 ++-- Cargo.lock | 18 ++++++++++++++++ Cargo.toml | 1 + README.md | 2 ++ internal/mithril-file-archiver/Cargo.toml | 25 +++++++++++++++++++++++ internal/mithril-file-archiver/Makefile | 19 +++++++++++++++++ internal/mithril-file-archiver/README.md | 10 +++++++++ internal/mithril-file-archiver/src/lib.rs | 10 +++++++++ 8 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 internal/mithril-file-archiver/Cargo.toml create mode 100644 internal/mithril-file-archiver/Makefile create mode 100644 internal/mithril-file-archiver/README.md create mode 100644 internal/mithril-file-archiver/src/lib.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53c910f7c3b..626e448c8a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -872,8 +872,8 @@ jobs: cargo doc --no-deps --lib -p mithril-stm -p mithril-common \ -p mithril-cardano-node-chain -p mithril-cardano-node-internal-database \ -p mithril-aggregator-client -p mithril-aggregator-discovery -p mithril-build-script -p mithril-cli-helper \ - -p mithril-dmq -p mithril-doc -p mithril-doc-derive \ - -p mithril-era -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ + -p mithril-dmq -p mithril-doc -p mithril-doc-derive -p mithril-era -p mithril-file-archiver \ + -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ -p mithril-ticker -p mithril-signed-entity-lock -p mithril-signed-entity-preloader \ -p mithril-aggregator -p mithril-signer -p mithril-client -p mithril-client-cli \ -p mithril-api-spec -p mithril-test-http-server \ diff --git a/Cargo.lock b/Cargo.lock index 7f0ca79e39c..d6c2b9eadbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4642,6 +4642,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "mithril-file-archiver" +version = "0.1.0" +dependencies = [ + "anyhow", + "hex", + "mithril-cardano-node-internal-database", + "mithril-common", + "serde", + "serde_json", + "sha2 0.10.9", + "slog", + "slog-async", + "slog-term", + "tar", + "zstd", +] + [[package]] name = "mithril-merkle-tree" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index b9a66922369..3f3c1580227 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "internal/mithril-doc", "internal/mithril-doc-derive", "internal/mithril-era", + "internal/mithril-file-archiver", "internal/mithril-merkle-tree", "internal/mithril-metric", "internal/mithril-persistence", diff --git a/README.md b/README.md index 50b2f3cf138..04ffacb0f96 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ This repository consists of the following parts: - [**Mithril era**](./internal/mithril-era): mechanisms to read and check **Mithril Era markers**, used by **Mithril network** nodes. + - [**Mithril file archiver**](./internal/mithril-file-archiver): An API that generates byte stables `tar.zst` archives, used by the **Mithril aggregator**. + - [**Mithril metric**](./internal/mithril-metric): materials to expose **metrics** in **Mithril network** nodes. - [**Mithril persistence**](./internal/mithril-persistence): the **persistence** library that is used by **Mithril network** nodes. diff --git a/internal/mithril-file-archiver/Cargo.toml b/internal/mithril-file-archiver/Cargo.toml new file mode 100644 index 00000000000..755697b01ed --- /dev/null +++ b/internal/mithril-file-archiver/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "mithril-file-archiver" +version = "0.1.0" +authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow = { workspace = true } +mithril-common = { path = "../../mithril-common" } +serde = { workspace = true } +serde_json = { workspace = true } +slog = { workspace = true } +tar = "0.4.46" +zstd = { version = "0.13.3", features = ["zstdmt"] } + +[dev-dependencies] +hex = { workspace = true } +mithril-cardano-node-internal-database = { path = "../cardano-node/mithril-cardano-node-internal-database" } +sha2 = "0.10.9" +slog-async = { workspace = true } +slog-term = { workspace = true } diff --git a/internal/mithril-file-archiver/Makefile b/internal/mithril-file-archiver/Makefile new file mode 100644 index 00000000000..d66d6d9fefc --- /dev/null +++ b/internal/mithril-file-archiver/Makefile @@ -0,0 +1,19 @@ +.PHONY: all build test check doc + +CARGO = cargo + +all: test build + +build: + ${CARGO} build --release + +test: + ${CARGO} test + +check: + ${CARGO} check --release --all-features --all-targets + ${CARGO} clippy --release --all-features --all-targets + ${CARGO} fmt --check + +doc: + ${CARGO} doc --no-deps --open diff --git a/internal/mithril-file-archiver/README.md b/internal/mithril-file-archiver/README.md new file mode 100644 index 00000000000..67eb54cf7ec --- /dev/null +++ b/internal/mithril-file-archiver/README.md @@ -0,0 +1,10 @@ +# Mithril-file-archiver + +**This is a work in progress** 🛠 + +An API to generate tar archives from files, directories, or serializable data (leveraging serde). + +Produced archives are byte stable across systems as long as the following invariants do not change: + +- The version of the zstandard compression library +- The parameters of the zstandard compression diff --git a/internal/mithril-file-archiver/src/lib.rs b/internal/mithril-file-archiver/src/lib.rs new file mode 100644 index 00000000000..8ef879d67b9 --- /dev/null +++ b/internal/mithril-file-archiver/src/lib.rs @@ -0,0 +1,10 @@ +#![warn(missing_docs)] + +//! # Mithril-file-archiver +//! +//! An API to generate tar archives from files, directories, or serializable data (leveraging serde). +//! +//! Produced archives are byte stable across systems as long as the following invariants do not change: +//! * The version of the zstandard compression library +//! * The parameters of the zstandard compression +//! From 3c611134ecca745dc36c25269c64a8089b499645 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:58:32 +0200 Subject: [PATCH 08/19] refactor(file_archiver): migrate and reorganize file archiver code into a dedicated crate --- Cargo.lock | 2 +- .../mithril-file-archiver/src}/api.rs | 28 +++++---- .../mithril-file-archiver/src}/appender.rs | 24 +++++--- .../mithril-file-archiver/src}/entities.rs | 59 ++++++++----------- internal/mithril-file-archiver/src/lib.rs | 9 +++ .../src/test/double/dummies.rs | 17 ++++++ .../src/test/double/mod.rs | 5 ++ .../src/test/extensions.rs | 35 +++++++++++ .../mithril-file-archiver/src/test}/mod.rs | 22 ++++--- .../src/tools/file_size.rs | 6 +- .../mithril-file-archiver/src/tools/mod.rs | 3 + .../mithril-file-archiver/tests}/tests.rs | 50 ++++++++-------- mithril-aggregator/Cargo.toml | 2 +- .../src/artifact_builder/cardano_database.rs | 11 +++- .../cardano_database_artifacts/ancillary.rs | 2 +- .../cardano_database_artifacts/digest.rs | 34 ++++++----- .../cardano_database_artifacts/immutable.rs | 21 ++++--- mithril-aggregator/src/configuration.rs | 20 +------ .../src/dependency_injection/builder/mod.rs | 2 +- .../builder/protocol/artifacts.rs | 2 +- mithril-aggregator/src/lib.rs | 2 +- .../compressed_archive_snapshotter.rs | 26 +++++--- .../src/services/snapshotter/interface.rs | 3 +- .../src/services/snapshotter/test_doubles.rs | 2 +- mithril-aggregator/src/tools/mod.rs | 2 - 25 files changed, 236 insertions(+), 153 deletions(-) rename {mithril-aggregator/src/tools/file_archiver => internal/mithril-file-archiver/src}/api.rs (95%) rename {mithril-aggregator/src/tools/file_archiver => internal/mithril-file-archiver/src}/appender.rs (97%) rename {mithril-aggregator/src/tools/file_archiver => internal/mithril-file-archiver/src}/entities.rs (72%) create mode 100644 internal/mithril-file-archiver/src/test/double/dummies.rs create mode 100644 internal/mithril-file-archiver/src/test/double/mod.rs create mode 100644 internal/mithril-file-archiver/src/test/extensions.rs rename {mithril-aggregator/src/tools/file_archiver => internal/mithril-file-archiver/src/test}/mod.rs (74%) rename {mithril-aggregator => internal/mithril-file-archiver}/src/tools/file_size.rs (97%) create mode 100644 internal/mithril-file-archiver/src/tools/mod.rs rename {mithril-aggregator/src/tools/file_archiver => internal/mithril-file-archiver/tests}/tests.rs (96%) diff --git a/Cargo.lock b/Cargo.lock index d6c2b9eadbf..7ee12d2132a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4241,6 +4241,7 @@ dependencies = [ "mithril-dmq", "mithril-doc", "mithril-era", + "mithril-file-archiver", "mithril-metric", "mithril-persistence", "mithril-protocol-config", @@ -4259,7 +4260,6 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2 0.10.9", "slog", "slog-async", "slog-bunyan", diff --git a/mithril-aggregator/src/tools/file_archiver/api.rs b/internal/mithril-file-archiver/src/api.rs similarity index 95% rename from mithril-aggregator/src/tools/file_archiver/api.rs rename to internal/mithril-file-archiver/src/api.rs index 0e3253f9c34..99583bcd511 100644 --- a/mithril-aggregator/src/tools/file_archiver/api.rs +++ b/internal/mithril-file-archiver/src/api.rs @@ -13,12 +13,10 @@ use mithril_common::StdResult; use mithril_common::entities::CompressionAlgorithm; use mithril_common::logging::LoggerExtensions; -use crate::ZstandardCompressionParameters; +use crate::appender::TarAppender; +use crate::entities::{ArchiveParameters, FileArchive, ZstandardCompressionParameters}; use crate::tools::file_size; -use super::appender::TarAppender; -use super::{ArchiveParameters, FileArchive}; - /// Tool to archive files and directories. pub struct FileArchiver { zstandard_compression_parameter: ZstandardCompressionParameters, @@ -41,14 +39,18 @@ impl FileArchiver { } } - #[cfg(test)] - pub fn new_for_test(verification_temp_dir: PathBuf) -> Self { - use crate::test::TestLogger; - Self { - zstandard_compression_parameter: ZstandardCompressionParameters::default(), + /// Constructs a new `FileArchiver` that uses the default compression parameters. + pub fn new_with_default_parameters(verification_temp_dir: PathBuf, logger: Logger) -> Self { + Self::new( + ZstandardCompressionParameters::default(), verification_temp_dir, - logger: TestLogger::stdout(), - } + logger, + ) + } + + #[cfg(test)] + pub(crate) fn new_for_test(verification_temp_dir: PathBuf) -> Self { + Self::new_with_default_parameters(verification_temp_dir, crate::test::TestLogger::stdout()) } /// Archive the content of a directory. @@ -286,8 +288,8 @@ mod tests { use mithril_common::test::assert_equivalent; - use crate::tools::file_archiver::appender::{AppenderDirAll, AppenderFile}; - use crate::tools::file_archiver::test_tools::*; + use crate::appender::{AppenderDirAll, AppenderFile}; + use crate::test::{FileArchiveTestExtension, create_dir, create_file, get_test_directory}; use super::*; diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/internal/mithril-file-archiver/src/appender.rs similarity index 97% rename from mithril-aggregator/src/tools/file_archiver/appender.rs rename to internal/mithril-file-archiver/src/appender.rs index a043af76561..e16ce0c10ec 100644 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ b/internal/mithril-file-archiver/src/appender.rs @@ -1,3 +1,5 @@ +//! Define how to append data to a [crate::FileArchiver] + use anyhow::{Context, anyhow}; use serde::Serialize; use std::fs::File; @@ -12,10 +14,13 @@ const READ_WRITE_PERMISSION: u32 = 0o666; /// Define multiple ways to append content to a tar archive. pub trait TarAppender: Send { + /// Appends the contents of the current object to the given tar archive builder. fn append(&self, tar: &mut tar::Builder) -> StdResult<()>; + /// Computes the total uncompressed size of the data that will be added to the archive. fn compute_uncompressed_data_size(&self) -> StdResult; + /// Chains this appender with another, combining their contents into a single archive. fn chain(self, appender_right: A2) -> ChainAppender where Self: Sized, @@ -24,19 +29,18 @@ pub trait TarAppender: Send { } } -#[cfg(test)] +/// An appender that add a directory and all of its content (recursively). pub struct AppenderDirAll { target_directory: PathBuf, } -#[cfg(test)] + impl AppenderDirAll { - // Note: Not used anymore outside of tests but useful tool to keep around if we ever need to archive a directory + /// [AppenderDirAll] factory pub fn new(target_directory: PathBuf) -> Self { Self { target_directory } } } -#[cfg(test)] impl TarAppender for AppenderDirAll { fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { tar.append_dir_all(".", &self.target_directory).with_context(|| { @@ -53,6 +57,7 @@ impl TarAppender for AppenderDirAll { } } +/// An appender that add one file. pub struct AppenderFile { /// Location of the file in the archive. location_in_archive: PathBuf, @@ -106,6 +111,9 @@ impl TarAppender for AppenderFile { } } +/// An appender that add a list of entries, files, or directories. +/// +/// Directory contents are not added if not specified. pub struct AppenderEntries { entries: Vec, base_directory: PathBuf, @@ -178,7 +186,7 @@ impl TarAppender for AppenderEntries { } } -/// Append data to the archive. +/// An appender that add either [serde::Serialize] serializable data or raw bytes. pub struct AppenderData { /// Location of the file in the archive where the data will be appended. location_in_archive: PathBuf, @@ -250,6 +258,7 @@ pub struct ChainAppender { } impl ChainAppender { + /// [ChainAppender] factory pub fn new(appender_left: L, appender_right: R) -> Self { Self { appender_left, @@ -279,8 +288,9 @@ mod tests { use mithril_common::entities::CompressionAlgorithm; use mithril_common::{assert_dir_eq, temp_dir_create}; - use crate::tools::file_archiver::test_tools::*; - use crate::tools::file_archiver::{ArchiveParameters, FileArchiver}; + use crate::api::FileArchiver; + use crate::entities::ArchiveParameters; + use crate::test::{FileArchiveTestExtension, create_dir, create_file}; use super::*; diff --git a/mithril-aggregator/src/tools/file_archiver/entities.rs b/internal/mithril-file-archiver/src/entities.rs similarity index 72% rename from mithril-aggregator/src/tools/file_archiver/entities.rs rename to internal/mithril-file-archiver/src/entities.rs index a4e9200688e..cefc7ba966f 100644 --- a/mithril-aggregator/src/tools/file_archiver/entities.rs +++ b/internal/mithril-file-archiver/src/entities.rs @@ -1,12 +1,36 @@ use std::path::{Path, PathBuf}; +use serde::Deserialize; + use mithril_common::entities::CompressionAlgorithm; +/// [Zstandard][CompressionAlgorithm::Zstandard] specific parameters +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +pub struct ZstandardCompressionParameters { + /// Level of compression, default to 9. + pub level: i32, + + /// Number of workers when compressing, 0 will disable multithreading, default to 4. + pub number_of_workers: u32, +} + +impl Default for ZstandardCompressionParameters { + fn default() -> Self { + Self { + level: 9, + number_of_workers: 4, + } + } +} + /// Parameters for creating an archive. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ArchiveParameters { + /// Archive name without file extension pub archive_name_without_extension: String, + /// Directory where the archive will be created pub target_directory: PathBuf, + /// Compression algorithm to use for the archive pub compression_algorithm: CompressionAlgorithm, } @@ -69,41 +93,6 @@ impl FileArchive { pub fn get_compression_algorithm(&self) -> CompressionAlgorithm { self.compression_algorithm } - - /// Unpack the archive to a directory. - /// - /// An 'unpack' directory will be created in the given parent directory. - #[cfg(test)] - pub fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf { - use super::test_tools::create_dir; - use std::fs::File; - use tar::Archive; - use zstd::stream::read::Decoder; - if self.compression_algorithm != CompressionAlgorithm::Zstandard { - panic!("Only Zstandard compression is supported"); - } - - let parent_dir = parent_dir.as_ref(); - let file_tar_zst = File::open(self.get_file_path()).unwrap(); - let file_tar_zst_decoder = Decoder::new(file_tar_zst).unwrap(); - let mut archive = Archive::new(file_tar_zst_decoder); - let unpack_path = parent_dir.join(create_dir(parent_dir, "unpack")); - archive.unpack(&unpack_path).unwrap(); - - unpack_path - } -} - -#[cfg(test)] -impl mithril_common::test::double::Dummy for FileArchive { - fn dummy() -> Self { - Self { - filepath: PathBuf::from("archive.tar.zst"), - archive_filesize: 10, - uncompressed_size: 789, - compression_algorithm: CompressionAlgorithm::Zstandard, - } - } } #[cfg(test)] diff --git a/internal/mithril-file-archiver/src/lib.rs b/internal/mithril-file-archiver/src/lib.rs index 8ef879d67b9..bac69444fbe 100644 --- a/internal/mithril-file-archiver/src/lib.rs +++ b/internal/mithril-file-archiver/src/lib.rs @@ -8,3 +8,12 @@ //! * The version of the zstandard compression library //! * The parameters of the zstandard compression //! + +mod api; +pub mod appender; +mod entities; +pub mod test; +pub mod tools; + +pub use api::*; +pub use entities::*; diff --git a/internal/mithril-file-archiver/src/test/double/dummies.rs b/internal/mithril-file-archiver/src/test/double/dummies.rs new file mode 100644 index 00000000000..a59ae70320a --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/dummies.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use mithril_common::entities::CompressionAlgorithm; +use mithril_common::test::double::Dummy; + +use crate::FileArchive; + +impl Dummy for FileArchive { + fn dummy() -> Self { + Self { + filepath: PathBuf::from("archive.tar.zst"), + archive_filesize: 10, + uncompressed_size: 789, + compression_algorithm: CompressionAlgorithm::Zstandard, + } + } +} diff --git a/internal/mithril-file-archiver/src/test/double/mod.rs b/internal/mithril-file-archiver/src/test/double/mod.rs new file mode 100644 index 00000000000..ddcde6270ff --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/mod.rs @@ -0,0 +1,5 @@ +//! Test doubles +//! +//! Enable unit testing with controlled inputs and predictable behavior. + +mod dummies; diff --git a/internal/mithril-file-archiver/src/test/extensions.rs b/internal/mithril-file-archiver/src/test/extensions.rs new file mode 100644 index 00000000000..97d3d8e6153 --- /dev/null +++ b/internal/mithril-file-archiver/src/test/extensions.rs @@ -0,0 +1,35 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; + +use tar::Archive; +use zstd::stream::read::Decoder; + +use mithril_common::entities::CompressionAlgorithm; + +use crate::FileArchive; + +/// Extension trait adding test utilities to [FileArchive] +pub trait FileArchiveTestExtension { + /// `TEST ONLY` - Unpack the archive to a directory. + /// + /// An 'unpack' directory will be created in the given parent directory. + fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf; +} + +impl FileArchiveTestExtension for FileArchive { + fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf { + if self.compression_algorithm != CompressionAlgorithm::Zstandard { + panic!("Only Zstandard compression is supported"); + } + + let parent_dir = parent_dir.as_ref(); + let file_tar_zst = File::open(self.get_file_path()).unwrap(); + let file_tar_zst_decoder = Decoder::new(file_tar_zst).unwrap(); + let mut archive = Archive::new(file_tar_zst_decoder); + let unpack_path = parent_dir.join("unpack"); + std::fs::create_dir(&unpack_path).unwrap(); + archive.unpack(&unpack_path).unwrap(); + + unpack_path + } +} diff --git a/mithril-aggregator/src/tools/file_archiver/mod.rs b/internal/mithril-file-archiver/src/test/mod.rs similarity index 74% rename from mithril-aggregator/src/tools/file_archiver/mod.rs rename to internal/mithril-file-archiver/src/test/mod.rs index 280754d8a43..7bc5278f37e 100644 --- a/mithril-aggregator/src/tools/file_archiver/mod.rs +++ b/internal/mithril-file-archiver/src/test/mod.rs @@ -1,19 +1,25 @@ -mod api; -pub mod appender; -mod entities; -#[cfg(test)] -mod tests; +//! Test utilities. +//! +//! ⚠ Do not use in production code ⚠ +//! +//! This module provides in particular test doubles for the traits defined in this crate. + +pub mod double; +mod extensions; -pub use api::*; -pub use entities::*; +pub use extensions::*; +#[cfg(test)] +pub(crate) use internal_tests_only::*; #[cfg(test)] -pub(crate) mod test_tools { +mod internal_tests_only { use std::fs::File; use std::path::{Path, PathBuf}; use mithril_common::test::TempDir; + mithril_common::define_test_logger!(); + pub fn get_test_directory(dir_name: &str) -> PathBuf { TempDir::create("file_archiver", dir_name) } diff --git a/mithril-aggregator/src/tools/file_size.rs b/internal/mithril-file-archiver/src/tools/file_size.rs similarity index 97% rename from mithril-aggregator/src/tools/file_size.rs rename to internal/mithril-file-archiver/src/tools/file_size.rs index d53be02d65b..f7ccbd9d446 100644 --- a/mithril-aggregator/src/tools/file_size.rs +++ b/internal/mithril-file-archiver/src/tools/file_size.rs @@ -1,3 +1,5 @@ +//! Tooling to compute File and Directory Sizes + use anyhow::Context; use std::{ collections::HashSet, @@ -7,7 +9,7 @@ use std::{ use mithril_common::StdResult; /// Compute the size of the given paths that could be files or folders. -pub(crate) fn compute_size(paths: Vec) -> StdResult { +pub fn compute_size(paths: Vec) -> StdResult { fn remove_duplicated_paths(paths: Vec) -> Vec { let mut result_folders = vec![]; let mut result_files = HashSet::new(); @@ -42,7 +44,7 @@ pub(crate) fn compute_size(paths: Vec) -> StdResult { /// Compute the size of one given path that could be a file or a folder. /// /// Returns 0 if the path is not a file or a folder. -pub(crate) fn compute_size_of_path(path: &Path) -> StdResult { +pub fn compute_size_of_path(path: &Path) -> StdResult { if path.is_file() { let metadata = std::fs::metadata(path) .with_context(|| format!("Failed to read metadata for file: {path:?}"))?; diff --git a/internal/mithril-file-archiver/src/tools/mod.rs b/internal/mithril-file-archiver/src/tools/mod.rs new file mode 100644 index 00000000000..ad45d2e7d2e --- /dev/null +++ b/internal/mithril-file-archiver/src/tools/mod.rs @@ -0,0 +1,3 @@ +//! Utilities used in file archiving + +pub mod file_size; diff --git a/mithril-aggregator/src/tools/file_archiver/tests.rs b/internal/mithril-file-archiver/tests/tests.rs similarity index 96% rename from mithril-aggregator/src/tools/file_archiver/tests.rs rename to internal/mithril-file-archiver/tests/tests.rs index 879e7b28750..d78be21bea8 100644 --- a/mithril-aggregator/src/tools/file_archiver/tests.rs +++ b/internal/mithril-file-archiver/tests/tests.rs @@ -15,9 +15,10 @@ use std::path::{Path, PathBuf}; use mithril_common::entities::CompressionAlgorithm; use mithril_common::temp_dir_create; -use crate::ZstandardCompressionParameters; -use crate::test::TestLogger; -use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::*}; +use mithril_file_archiver::appender::*; +use mithril_file_archiver::{ + ArchiveParameters, FileArchive, FileArchiver, ZstandardCompressionParameters, +}; mod helpers { use sha2::{Digest, Sha256}; @@ -64,10 +65,9 @@ mod helpers { /// **IMPORTANT** Default zstandard compression parameters are used. pub fn file_archiver(work_dir: &Path) -> FileArchiver { - FileArchiver::new( - ZstandardCompressionParameters::default(), + FileArchiver::new_with_default_parameters( work_dir.join("verification"), - TestLogger::stdout(), + slog::Logger::root(slog::Discard, slog::o!()), ) } @@ -309,8 +309,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - reference_archive.filepath, - repeated_archive.filepath, + reference_archive.get_file_path(), + repeated_archive.get_file_path(), ); } @@ -482,8 +482,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - reference_archive.filepath, - archive_with_different_metadata.filepath, + reference_archive.get_file_path(), + archive_with_different_metadata.get_file_path(), ); } @@ -618,8 +618,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - &archive.filepath, - &archive_with_same_content_but_from_another_dir.filepath, + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), ); } @@ -647,8 +647,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - &archive.filepath, - &archive_with_same_content_but_from_another_dir.filepath, + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), ); } @@ -677,8 +677,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - &archive.filepath, - &archive_with_same_content_but_from_another_dir.filepath, + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), ); } } @@ -714,8 +714,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - &reference_archive.filepath, - &archive_with_equivalent_entries_spelling.filepath, + reference_archive.get_file_path(), + archive_with_equivalent_entries_spelling.get_file_path(), ); } @@ -763,8 +763,8 @@ mod reproducibility { .unwrap(); helpers::assert_files_are_byte_identical( - &reference_archive.filepath, - &archive_with_same_content_but_different_entries_order.filepath, + reference_archive.get_file_path(), + archive_with_same_content_but_different_entries_order.get_file_path(), ); } } @@ -847,16 +847,16 @@ mod golden_master { #[track_caller] fn assert_archive_not_empty(archive: &FileArchive) { assert!( - archive.uncompressed_size > 0, + archive.get_uncompressed_size() > 0, "Archive '{}' has no content, fix the archive creation and try again", - archive.filepath.display() + archive.get_file_path().display() ); } /// Assert an archive matches its golden hash. #[track_caller] fn assert_archive_matches_golden_sha256(archive: &FileArchive, expected_sha256: &str) { - let actual_sha256 = helpers::compute_file_sha256(&archive.filepath); + let actual_sha256 = helpers::compute_file_sha256(archive.get_file_path()); assert_eq!( expected_sha256, @@ -864,8 +864,8 @@ mod golden_master { "Archive bytes changed ('{}', {} bytes).\n\ Either the archive format is no longer reproducible, or the change is intentional \ and the TAR_ZSTD_V* constants must be recomputed and their version bumped.", - archive.filepath.display(), - archive.archive_filesize, + archive.get_file_path().display(), + archive.get_archive_size(), ); } diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index bf094843cb0..ad7c3ec96af 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -33,6 +33,7 @@ mithril-common = { path = "../mithril-common" } mithril-dmq = { path = "../internal/mithril-dmq" } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } +mithril-file-archiver = { path = "../internal/mithril-file-archiver" } mithril-metric = { path = "../internal/mithril-metric" } mithril-persistence = { path = "../internal/mithril-persistence" } mithril-protocol-config = { path = "../internal/mithril-protocol-config" } @@ -78,7 +79,6 @@ mithril-api-spec = { path = "../internal/tests/mithril-api-spec" } mithril-common = { path = "../mithril-common", features = ["allow_skip_signer_certification"] } mithril-test-http-server = { path = "../internal/tests/mithril-test-http-server" } mockall = { workspace = true } -sha2 = "0.10.9" slog-scope = "4.4.1" slog-term = { workspace = true } tempfile = "3.27.0" diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index 33fbe71d838..86e120481f4 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -104,6 +104,7 @@ mod tests { use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest; use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutable_trio_names}; + use mithril_file_archiver::FileArchiver; use mithril_common::{ CardanoNetwork, @@ -126,7 +127,7 @@ mod tests { services::CompressedArchiveSnapshotter, services::ancillary_signer::MockAncillarySigner, test::TestLogger, - tools::{file_archiver::FileArchiver, url_sanitizer::SanitizedUrlWithTrailingSlash}, + tools::url_sanitizer::SanitizedUrlWithTrailingSlash, }; use super::*; @@ -201,7 +202,10 @@ mod tests { cardano_db.get_dir().to_path_buf(), test_dir.join("ongoing_snapshots"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(FileArchiver::new_with_default_parameters( + test_dir.join("verification"), + TestLogger::stdout(), + )), Arc::new(MockAncillarySigner::that_succeeds_with_signature( ancillary_manifest_signature, )), @@ -271,8 +275,9 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("http://aggregator_uri").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test( + file_archiver: Arc::new(FileArchiver::new_with_default_parameters( test_dir.join("verification"), + TestLogger::stdout(), )), target_location: test_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs index 03e9f6494c1..4c07c17ef97 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs @@ -9,12 +9,12 @@ use mithril_common::{ entities::{AncillaryLocation, CardanoDbBeacon, CompressionAlgorithm}, logging::LoggerExtensions, }; +use mithril_file_archiver::FileArchive; use crate::{ DumbUploader, FileUploader, file_uploaders::{CloudUploader, LocalUploader}, services::Snapshotter, - tools::file_archiver::FileArchive, }; /// The [AncillaryFileUploader] trait allows identifying uploaders that return locations for ancillary archive files. diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs index 36dd8b368d2..041f9269155 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs @@ -6,21 +6,20 @@ use std::{ use anyhow::Context; use async_trait::async_trait; +use slog::{Logger, error}; + use mithril_common::{ CardanoNetwork, StdResult, entities::{CardanoDbBeacon, CompressionAlgorithm, DigestLocation}, logging::LoggerExtensions, messages::CardanoDatabaseDigestListItemMessage, }; -use slog::{Logger, error}; +use mithril_file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::AppenderFile}; use crate::{ DumbUploader, FileUploader, ImmutableFileDigestMapper, file_uploaders::{CloudUploader, LocalUploader}, - tools::{ - file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::AppenderFile}, - url_sanitizer::SanitizedUrlWithTrailingSlash, - }, + tools::url_sanitizer::SanitizedUrlWithTrailingSlash, }; /// The [DigestFileUploader] trait allows identifying uploaders that return locations for digest files. @@ -301,11 +300,11 @@ mod tests { messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage}, test::{TempDir, assert_equivalent, double::Dummy}, }; + use mithril_file_archiver::FileArchiver; use crate::{ file_uploaders::FileUploadRetryPolicy, immutable_file_digest_mapper::MockImmutableFileDigestMapper, test::TestLogger, - tools::file_archiver::FileArchiver, }; use super::*; @@ -371,6 +370,13 @@ mod tests { Ok(()) } + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + #[tokio::test] async fn digest_artifact_builder_return_digests_route_on_aggregator() { let temp_dir = TempDir::create("digest", current_function!()); @@ -384,7 +390,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -417,7 +423,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -451,7 +457,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -476,7 +482,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -509,7 +515,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -549,7 +555,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -597,7 +603,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -639,7 +645,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(build_local_uploader(&uploader_path))], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(tmp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&tmp_dir)), target_location: digests_archive_dir.clone(), compression_algorithm, }, diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs index 7d83f5fe17a..70203f73e74 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs @@ -282,12 +282,12 @@ mod tests { entities::TemplateUri, test::{TempDir, assert_equivalent, equivalent_to}, }; + use mithril_file_archiver::FileArchiver; use semver::Version; use crate::services::ancillary_signer::MockAncillarySigner; use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter}; use crate::test::TestLogger; - use crate::tools::file_archiver::FileArchiver; use super::*; @@ -329,6 +329,13 @@ mod tests { uploader } + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + fn create_fake_file(path: &Path, content: &str) { let mut file = File::create(path).unwrap(); write!(file, "{content}").unwrap(); @@ -360,7 +367,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -454,7 +461,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -501,7 +508,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -536,7 +543,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -569,7 +576,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -626,7 +633,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 4ea033f231a..3eb89039aca 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -22,6 +22,7 @@ use mithril_common::{AggregateSignatureType, CardanoNetwork, StdResult}; use mithril_dmq::DmqNetwork; use mithril_doc::{Documenter, DocumenterDefault, StructDoc}; use mithril_era::adapters::EraReaderAdapterType; +use mithril_file_archiver::ZstandardCompressionParameters; use crate::entities::AggregatorEpochSettings; use crate::http_server::SERVER_BASE_PATH; @@ -706,25 +707,6 @@ pub enum SnapshotUploaderType { Local, } -/// [Zstandard][CompressionAlgorithm::Zstandard] specific parameters -#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] -pub struct ZstandardCompressionParameters { - /// Level of compression, default to 9. - pub level: i32, - - /// Number of workers when compressing, 0 will disable multithreading, default to 4. - pub number_of_workers: u32, -} - -impl Default for ZstandardCompressionParameters { - fn default() -> Self { - Self { - level: 9, - number_of_workers: 4, - } - } -} - /// Configuration to connect to the Blockfrost API. /// /// Currently only used to fetch the ticker and name for registered pools. diff --git a/mithril-aggregator/src/dependency_injection/builder/mod.rs b/mithril-aggregator/src/dependency_injection/builder/mod.rs index 9ddd29e84d8..e7b4c58ba7e 100644 --- a/mithril-aggregator/src/dependency_injection/builder/mod.rs +++ b/mithril-aggregator/src/dependency_injection/builder/mod.rs @@ -33,6 +33,7 @@ use mithril_common::{ signable_builder::{SignableBuilderService, SignableSeedBuilder}, }; use mithril_era::{EraChecker, EraReader, EraReaderAdapter}; +use mithril_file_archiver::FileArchiver; use mithril_persistence::sqlite::{SqliteConnection, SqliteConnectionPool}; use mithril_protocol_config::interface::{ MithrilNetworkConfigurationProvider, ProtocolConfigurationMarkersReader, @@ -61,7 +62,6 @@ use crate::{ SignedEntityService, SignerSynchronizer, Snapshotter, StakeDistributionService, UpkeepService, }, - tools::file_archiver::FileArchiver, }; use super::{ diff --git a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs index 4b70fd37c07..c579f2d2320 100644 --- a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs +++ b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use std::sync::Arc; use mithril_common::crypto_helper::ManifestSigner; +use mithril_file_archiver::FileArchiver; use crate::artifact_builder::{ AncillaryArtifactBuilder, AncillaryFileUploader, CardanoBlocksTransactionsArtifactBuilder, @@ -28,7 +29,6 @@ use crate::services::{ SignedEntityServiceArtifactsDependencies, Snapshotter, }; use crate::tools::DEFAULT_GCP_CREDENTIALS_JSON_ENV_VAR; -use crate::tools::file_archiver::FileArchiver; use crate::{DumbUploader, ExecutionEnvironment, FileUploader, SnapshotUploaderType}; impl DependenciesBuilder { diff --git a/mithril-aggregator/src/lib.rs b/mithril-aggregator/src/lib.rs index 939265118c2..0425fc455f0 100644 --- a/mithril-aggregator/src/lib.rs +++ b/mithril-aggregator/src/lib.rs @@ -34,7 +34,7 @@ mod tools; pub use crate::artifact_builder::ArtifactBuilder; pub use crate::configuration::{ ConfigurationSource, DefaultConfiguration, ExecutionEnvironment, ServeCommandConfiguration, - SnapshotUploaderType, ZstandardCompressionParameters, + SnapshotUploaderType, }; pub use crate::multi_signer::{MultiSigner, MultiSignerImpl}; pub use commands::{CommandType, MainOpts}; diff --git a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs index 2473397ab86..333e8cb5d67 100644 --- a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs +++ b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs @@ -13,11 +13,13 @@ use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutabl use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; use mithril_common::logging::LoggerExtensions; +use mithril_file_archiver::{ + ArchiveParameters, FileArchive, FileArchiver, + appender::{AppenderData, AppenderEntries, TarAppender}, + tools::file_size, +}; use crate::dependency_injection::DependenciesBuilderError; -use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender}; -use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver}; -use crate::tools::file_size; use super::{Snapshotter, ancillary_signer::AncillarySigner}; @@ -318,6 +320,7 @@ mod tests { use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_common::test::assert_equivalent; use mithril_common::{assert_dir_eq, current_function, temp_dir_create}; + use mithril_file_archiver::test::FileArchiveTestExtension; use crate::services::ancillary_signer::MockAncillarySigner; use crate::test::TestLogger; @@ -333,6 +336,13 @@ mod tests { .collect() } + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + fn snapshotter_for_test( test_directory: &Path, db_directory: &Path, @@ -343,9 +353,7 @@ mod tests { db_directory.to_path_buf(), test_directory.join("ongoing_snapshot"), compression_algorithm, - Arc::new(FileArchiver::new_for_test( - test_directory.join("verification"), - )), + Arc::new(file_archiver_for_test(test_directory)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -378,7 +386,7 @@ mod tests { db_directory, ongoing_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -402,7 +410,7 @@ mod tests { db_directory, ongoing_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -425,7 +433,7 @@ mod tests { cardano_db.get_dir().clone(), pending_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) diff --git a/mithril-aggregator/src/services/snapshotter/interface.rs b/mithril-aggregator/src/services/snapshotter/interface.rs index 08584c15c3a..03f4bda22fc 100644 --- a/mithril-aggregator/src/services/snapshotter/interface.rs +++ b/mithril-aggregator/src/services/snapshotter/interface.rs @@ -2,8 +2,7 @@ use async_trait::async_trait; use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; - -use crate::tools::file_archiver::FileArchive; +use mithril_file_archiver::FileArchive; #[cfg_attr(test, mockall::automock)] #[async_trait] diff --git a/mithril-aggregator/src/services/snapshotter/test_doubles.rs b/mithril-aggregator/src/services/snapshotter/test_doubles.rs index ab2765cb663..dded8e14b17 100644 --- a/mithril-aggregator/src/services/snapshotter/test_doubles.rs +++ b/mithril-aggregator/src/services/snapshotter/test_doubles.rs @@ -6,9 +6,9 @@ use std::sync::RwLock; use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; +use mithril_file_archiver::FileArchive; use crate::services::Snapshotter; -use crate::tools::file_archiver::FileArchive; /// Snapshotter that does nothing. It is mainly used for test purposes. pub struct DumbSnapshotter { diff --git a/mithril-aggregator/src/tools/mod.rs b/mithril-aggregator/src/tools/mod.rs index 44aeb35c791..93b3777bdde 100644 --- a/mithril-aggregator/src/tools/mod.rs +++ b/mithril-aggregator/src/tools/mod.rs @@ -1,7 +1,5 @@ mod certificates_hash_migrator; mod era; -pub mod file_archiver; -pub mod file_size; mod genesis; mod protocol_configuration; pub mod signer_importer; From fea510248d3cb4c67613d56f51a3e8d6e1ae629b Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:24:53 +0200 Subject: [PATCH 09/19] test(file_archiver): split reproducibility and golden hashes integration tests into multiple files --- .../tests/extensions/helpers.rs | 150 +++ .../tests/extensions/mod.rs | 6 + .../tests/extensions/test_data.rs | 79 ++ .../tests/golden_master.rs | 241 ++++ .../tests/reproducibility.rs | 534 +++++++++ internal/mithril-file-archiver/tests/tests.rs | 1005 ----------------- 6 files changed, 1010 insertions(+), 1005 deletions(-) create mode 100644 internal/mithril-file-archiver/tests/extensions/helpers.rs create mode 100644 internal/mithril-file-archiver/tests/extensions/mod.rs create mode 100644 internal/mithril-file-archiver/tests/extensions/test_data.rs create mode 100644 internal/mithril-file-archiver/tests/golden_master.rs create mode 100644 internal/mithril-file-archiver/tests/reproducibility.rs delete mode 100644 internal/mithril-file-archiver/tests/tests.rs diff --git a/internal/mithril-file-archiver/tests/extensions/helpers.rs b/internal/mithril-file-archiver/tests/extensions/helpers.rs new file mode 100644 index 00000000000..8ad19e8cec2 --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/helpers.rs @@ -0,0 +1,150 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use mithril_common::entities::CompressionAlgorithm; +use mithril_common::temp_dir_create; +use mithril_file_archiver::{ArchiveParameters, FileArchiver}; + +/// Assert that two files have exactly the same bytes. +/// +/// SHA-256 hashes and file sizes are reported when they differ. +#[track_caller] +pub fn assert_files_are_byte_identical, A: AsRef>(expected: E, actual: A) { + let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); + let expected_bytes = std::fs::read(expected_path).unwrap_or_else(|error| { + panic!( + "Could not read expected file '{}': {error}", + expected_path.display() + ) + }); + let actual_bytes = std::fs::read(actual_path).unwrap_or_else(|error| { + panic!( + "Could not read actual file '{}': {error}", + actual_path.display() + ) + }); + + if expected_bytes != actual_bytes { + let expected_hash = hex::encode(Sha256::digest(&expected_bytes)); + let actual_hash = hex::encode(Sha256::digest(&actual_bytes)); + + panic!( + "Files are not byte-identical:\n\ + expected: '{}' ({} bytes, SHA-256: {})\n\ + actual: '{}' ({} bytes, SHA-256: {})", + expected_path.display(), + expected_bytes.len(), + expected_hash, + actual_path.display(), + actual_bytes.len(), + actual_hash, + ); + } +} + +/// **IMPORTANT** Default zstandard compression parameters are used. +pub fn file_archiver(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + slog::Logger::root(slog::Discard, slog::o!()), + ) +} + +pub fn create_dir>(base_dir: &Path, dir_name: P) -> PathBuf { + let dir_path = base_dir.join(dir_name); + std::fs::create_dir(&dir_path).unwrap(); + dir_path +} + +pub fn create_file(root_dir: &Path, file_name: &str, size: Option) -> PathBuf { + let path = root_dir.join(file_name); + let mut file = File::create(&path).unwrap(); + + write!(file, "This is a test file named '{file_name}'").unwrap(); + writeln!(file).unwrap(); + + if let Some(file_size) = size { + file.set_len(file_size).unwrap(); + } + + path +} + +pub fn alter_file(path: &Path, alter_fn: F) { + let file = File::options().write(true).open(path).unwrap(); + alter_fn(&file); +} + +pub fn archive_parameters(filename: &str, target_dir: &Path) -> ArchiveParameters { + ArchiveParameters { + archive_name_without_extension: filename.to_string(), + target_directory: target_dir.to_path_buf(), + compression_algorithm: CompressionAlgorithm::Zstandard, + } +} + +pub fn compute_file_sha256(path: &Path) -> String { + let mut hasher = Sha256::new(); + + if path.is_file() { + hash_file(path, &mut hasher); + } else { + panic!("Path is not a file: {:?}", path.display()); + } + + hex::encode(hasher.finalize()) +} + +fn hash_file(path: &Path, hasher: &mut Sha256) { + let mut file = File::open(path).unwrap(); + let mut buffer = [0; 64 * 1024]; + + loop { + let bytes_read = file.read(&mut buffer).unwrap(); + if bytes_read == 0 { + break; + } + + hasher.update(&buffer[..bytes_read]); + } +} + +#[test] +fn assert_files_are_byte_identical_succeed_with_identical_files() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let identical_file_path = test_dir.join("altered_file"); + std::fs::copy(&reference_file_path, &identical_file_path).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &identical_file_path); +} + +#[test] +#[should_panic(expected = "Files are not byte-identical")] +fn assert_files_are_byte_identical_fails_if_one_byte_changes() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let altered_content = { + let mut altered_content = content.clone(); + altered_content[0] = !altered_content[0]; + altered_content + }; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let altered_file_path = test_dir.join("altered_file"); + let mut altered_file = File::create_new(&altered_file_path).unwrap(); + altered_file.write_all(&altered_content).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &altered_file_path); +} diff --git a/internal/mithril-file-archiver/tests/extensions/mod.rs b/internal/mithril-file-archiver/tests/extensions/mod.rs new file mode 100644 index 00000000000..128154a4ca4 --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/mod.rs @@ -0,0 +1,6 @@ +// Avoid clippy warnings generated by tests that doesn't use every tests_extensions (since each test +// is a different compilation target). +#![allow(dead_code)] + +pub mod helpers; +pub mod test_data; diff --git a/internal/mithril-file-archiver/tests/extensions/test_data.rs b/internal/mithril-file-archiver/tests/extensions/test_data.rs new file mode 100644 index 00000000000..ffe1f86ebbe --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/test_data.rs @@ -0,0 +1,79 @@ +//! Fixed test data set for file archiver tests. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::Serialize; + +use crate::extensions::helpers; + +pub const TEST_BYTES: [u8; 10] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + +/// See: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks +pub(crate) const ZSTD_MAX_BLOCK_SIZE: u64 = 131_072; + +#[derive(Debug, PartialEq, Serialize)] +pub struct TestStruct { + field_str: String, + field_int: i32, + // Important note: using BTreeMap to ensure deterministic serialization, types like HashMap should + // not be used as the key order is not guaranteed + field_map: BTreeMap, +} + +impl Default for TestStruct { + fn default() -> Self { + Self { + field_str: "test".to_string(), + field_int: 42, + field_map: BTreeMap::from([ + ("key_3".to_string(), "value_3".to_string()), + ("key_1".to_string(), "value_1".to_string()), + ("key_2".to_string(), "value_2".to_string()), + ]), + } + } +} + +/// Create a file named `test.txt` in the given directory +pub fn create_test_txt(root_dir: &Path) -> PathBuf { + helpers::create_file(root_dir, "test.txt", Some(1048)) +} + +/// [AppenderEntries] ready list of entries created by [`create_test_dir`] +pub fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + ] +} + +/// Create a directory named `test_dir` in the given directory +/// +/// It will be filled with the following structure: +/// ```no_run +/// test_dir/ +/// ├── bar/ +/// ├── foo/ +/// │ └── bar.txt +/// ├── file_1.txt +/// ├── file_2.txt +/// ├── empty_file.txt +/// └── file_with_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt +/// ``` +pub fn create_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file(&test_dir, "file_2.txt", Some(ZSTD_MAX_BLOCK_SIZE * 2 + 1)); + + test_dir +} diff --git a/internal/mithril-file-archiver/tests/golden_master.rs b/internal/mithril-file-archiver/tests/golden_master.rs new file mode 100644 index 00000000000..1713568ba1c --- /dev/null +++ b/internal/mithril-file-archiver/tests/golden_master.rs @@ -0,0 +1,241 @@ +//! The golden hashes are pinned to: the tar/zstd crate versions, GOLDEN_COMPRESSION_PARAMETERS +//! (multi-threaded zstd), and the exact bytes produced by `test_data::create_test_*`. +//! Any change to one of those invalidates them. + +mod extensions; + +use std::fs::File; +use std::path::{Path, PathBuf}; + +use mithril_common::temp_dir_create; + +use mithril_file_archiver::appender::*; +use mithril_file_archiver::{FileArchive, ZstandardCompressionParameters}; + +use extensions::*; + +// ** These hashes define TAR_ZSTD_V1. Update them only for an intentional archive-format change. ** +pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = + "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; +pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256: &str = + "647bba62029e63b0cbd329bccdfdb850ec934a07ab0811cc4d2d7682b72d3054"; +pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = + "565da17762fcf8f417f1ad78fda003b19507757bf56ac09ff24a9aac179bf78c"; +pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = + "fd5a178f1c717de39aef526d378e731871b74ee1ba11bdc159a4a872867d3748"; +pub const TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256: &str = + "6843b658c2c176afa156e82f7ee3d828f63cd7e9de16dc3166a5da96f0d3e15f"; +pub const TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256: &str = + "ff5447a1b80b530b3d94cfeb2ad69fdc392a4d5175f24ffc13ad10b08a7e12ba"; + +/// Create a directory named `test_dir` in the given directory +/// +/// It will be filled with the following structure: +/// ```no_run +/// test_dir/ +/// ├── bar/ +/// ├── foo/ +/// │ └── bar.txt +/// ├── empty_file.txt +/// ├── file_1.txt +/// ├── file_2.txt +/// └── file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt +/// ``` +pub fn create_golden_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + File::create(test_dir.join("empty.txt")).unwrap(); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file( + &test_dir, + "file_2.txt", + Some(test_data::ZSTD_MAX_BLOCK_SIZE * 2 + 1), + ); + helpers::create_file( + &test_dir, + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + None, + ); + + test_dir +} + +/// [AppenderEntries] ready list of entries created by [`create_golden_test_dir`] +pub fn golden_test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("empty.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from( + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + ), + ] +} + +#[track_caller] +fn assert_archive_not_empty(archive: &FileArchive) { + assert!( + archive.get_uncompressed_size() > 0, + "Archive '{}' has no content, fix the archive creation and try again", + archive.get_file_path().display() + ); +} + +/// Assert an archive matches its golden hash. +#[track_caller] +fn assert_archive_matches_golden_sha256(archive: &FileArchive, expected_sha256: &str) { + let actual_sha256 = helpers::compute_file_sha256(archive.get_file_path()); + + assert_eq!( + expected_sha256, + actual_sha256, + "Archive bytes changed ('{}', {} bytes).\n\ + Either the archive format is no longer reproducible, or the change is intentional \ + and the TAR_ZSTD_V* constants must be recomputed and their version bumped.", + archive.get_file_path().display(), + archive.get_archive_size(), + ); +} + +/// Compression parameters the golden hashes were computed with. +/// +/// They must mirror the production defaults, see [defaults_are_the_golden_parameters]. +const GOLDEN_COMPRESSION_PARAMETERS: ZstandardCompressionParameters = + ZstandardCompressionParameters { + level: 9, + number_of_workers: 4, + }; + +#[test] +fn defaults_are_the_golden_parameters() { + assert_eq!( + GOLDEN_COMPRESSION_PARAMETERS, + ZstandardCompressionParameters::default(), + "zstandard defaults changed: archives are no longer byte-compatible with the \ + previously published ones, the golden hashes must be recomputed and TAR_ZSTD_V1 \ + bumped to V2" + ); +} + +#[test] +fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_DATA_SHA256); +} + +#[test] +fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256); +} + +#[test] +fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_FILE_SHA256); +} + +#[test] +fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderDirAll::new(content), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256, + ); +} + +#[test] +fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + // Should construct the same archive as `AppenderDirAll` as we include all the source directory entries + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256, + ); +} + +#[test] +fn chain_appender() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content) + .unwrap() + .chain( + AppenderData::from_json( + PathBuf::from("test.json"), + &test_data::TestStruct::default(), + ) + .unwrap(), + ), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256, + ); +} diff --git a/internal/mithril-file-archiver/tests/reproducibility.rs b/internal/mithril-file-archiver/tests/reproducibility.rs new file mode 100644 index 00000000000..9be3f2a5e8f --- /dev/null +++ b/internal/mithril-file-archiver/tests/reproducibility.rs @@ -0,0 +1,534 @@ +//! Reproducibility contract for archives produced by [`FileArchiver`]. +//! +//! Given identical archive entry paths and contents, `FileArchiver` must produce a +//! byte-identical `.tar.zst` archive regardless of the host system, source base +//! directory, creation time, modification times, permissions, or input entry order. +//! +//! This contract assumes identical archive-format dependencies and zstandard +//! compression parameters. Changing either is an archive-format change and requires +//! intentionally versioning the format and updating its golden hashes. + +mod extensions; + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use mithril_common::temp_dir_create; + +use mithril_file_archiver::appender::*; + +use extensions::*; + +mod repeated_archiving_produces_byte_identical_archives { + use std::time::Instant; + + use super::*; + + fn run_scenario(test_dir: PathBuf, ref_appender: T, repeated_appender: T) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || {}); + } + + fn run_scenario_in_different_unix_seconds( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + ) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || { + let seconds_since_unix_epoch = || { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + }; + + let reference_second = seconds_since_unix_epoch(); + let deadline = Instant::now() + Duration::from_millis(1500); + + while seconds_since_unix_epoch() == reference_second { + assert!( + Instant::now() < deadline, + "Unix time did not advance to the next second" + ); + std::thread::sleep(Duration::from_millis(10)); + } + }); + } + + fn run_scenario_with_hook( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + before_repeated_archive: F, + ) { + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + ref_appender, + ) + .unwrap(); + + before_repeated_archive(); + + let repeated_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("repeated", &test_dir), + repeated_appender, + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + repeated_archive.get_file_path(), + ); + } + + #[test] + fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ); + } + + #[test] + fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content.clone()), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderFile::append_at_archive_root(content.clone()).unwrap(), + AppenderFile::append_at_archive_root(content).unwrap(), + ); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderDirAll::new(content.clone()), + AppenderDirAll::new(content), + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ); + } +} + +mod source_metadata_does_not_affect_archive_bytes { + use super::*; + + /// [AppenderEntries] ready list of the reference/altered dirs created by [setup_test_dirs]. + fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("empty/"), + PathBuf::from("subdir/"), + PathBuf::from("subdir/file_2.txt"), + PathBuf::from("subdir/file_3.txt"), + PathBuf::from("file_1.txt"), + ] + } + + /// Create two directories with the same structure and applies the given function to each + /// reference/altered files pair. + /// + /// To be used with Appender that can work on a group of files and directories. + /// + /// Each directory contains the following files: + /// ```no_run + /// (reference|altered)/ + /// ├── empty/ + /// ├── subdir/ + /// │ ├── file_2.txt + /// │ └── file_3.txt + /// └── file_1.txt + /// ``` + fn setup_test_dirs( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + let subdirs = vec![Path::new("empty"), Path::new("subdir")]; + + for dir_path in &subdirs { + helpers::create_dir(&reference_dir, dir_path); + helpers::create_dir(&altered_dir, dir_path); + } + + for file_path in ["file_1.txt", "subdir/file_2.txt", "subdir/file_3.txt"] { + let reference_file = helpers::create_file(&reference_dir, file_path, None); + let altered_file = helpers::create_file(&altered_dir, file_path, None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + // Alter dirs after the files to avoid permission issues + for dir_path in &subdirs { + setup_reference_and_altered_path_fn( + &reference_dir.join(dir_path), + &altered_dir.join(dir_path), + ); + } + setup_reference_and_altered_path_fn(&reference_dir, &altered_dir); + } + + /// Create two directories with a single "test.txt" file and applies the given function + /// to the reference/altered file pair. + /// + /// To be used with Appender that can work on a single file. + fn setup_test_file( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + + let reference_file = helpers::create_file(&reference_dir, "file.txt", None); + let altered_file = helpers::create_file(&altered_dir, "file.txt", None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + fn run_scenario A>(test_dir: PathBuf, build_tar_appender: B) { + let source = test_dir.join("source"); + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + build_tar_appender(source.join("reference")), + ) + .unwrap(); + + let archive_with_different_metadata = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("altered_metadata", &test_dir), + build_tar_appender(source.join("altered")), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_different_metadata.get_file_path(), + ); + } + + mod modification_time { + use super::*; + + fn setup_modification_time(reference_path: &Path, path_with_different_metadata: &Path) { + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + // can't modify dir times until + // is stabilized (currently planned for rust 1.99) + return; + } + + let base_time = SystemTime::UNIX_EPOCH; + helpers::alter_file(reference_path, |file| file.set_modified(base_time).unwrap()); + helpers::alter_file(path_with_different_metadata, |file| { + // note: in TAR, entries mtimes have a granularity of a second + file.set_modified(base_time + Duration::from_millis(5300)).unwrap() + }); + + assert_ne!( + reference_path.metadata().unwrap().modified().unwrap(), + path_with_different_metadata.metadata().unwrap().modified().unwrap() + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_modification_time); + run_scenario(test_dir, AppenderDirAll::new); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } + + #[cfg(unix)] + mod permissions { + use std::fs::Permissions; + + use super::*; + + fn setup_permissions(reference_path: &Path, path_with_different_metadata: &Path) { + use std::os::unix::fs::PermissionsExt; + + let (reference_permission, altered_permission) = + // IMPORTANT: for directory the owner permission must be `7`, else this prevents + // `temp_dir_create` cleanup and make appenders fails on subdir files + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + (Permissions::from_mode(0o766), Permissions::from_mode(0o767)) + } else { + (Permissions::from_mode(0o644), Permissions::from_mode(0o646)) + }; + + std::fs::set_permissions(reference_path, reference_permission).unwrap(); + std::fs::set_permissions(path_with_different_metadata, altered_permission).unwrap(); + + assert_ne!( + reference_path.metadata().unwrap().permissions(), + path_with_different_metadata.metadata().unwrap().permissions() + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }) + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_permissions); + run_scenario(test_dir, AppenderDirAll::new); + } + + #[cfg(unix)] + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } +} + +mod source_base_directory_does_not_affect_archive { + use super::*; + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_txt(&subdir_1); + let same_content_in_other_dir = test_data::create_test_txt(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderFile::append_at_archive_root(same_content_in_other_dir).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), + ); + } + + #[test] + fn appender_dir_all() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_dir(&subdir_1); + let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderDirAll::new(content), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderDirAll::new(same_content_in_other_dir), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_dir(&subdir_1); + let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), same_content_in_other_dir) + .unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), + ); + } +} + +mod appender_entry_specifics { + use super::*; + + fn to_entries(paths: [&str; N]) -> Vec { + paths.into_iter().map(PathBuf::from).collect() + } + + #[test] + fn equivalent_entry_paths_produce_identical_archives() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_entries = ["bar/", "foo/", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + let equivalent_entries = ["bar", "foo", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(to_entries(reference_entries), content.clone()).unwrap(), + ) + .unwrap(); + + let archive_with_equivalent_entries_spelling = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("equivalent_spelling", &test_dir), + AppenderEntries::new(to_entries(equivalent_entries), content).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_equivalent_entries_spelling.get_file_path(), + ); + } + + #[test] + fn supplied_entry_order_does_not_affect_appender_entries_archive() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + ) + .unwrap(); + + for (label, entries) in [ + ( + "child_before_parent_dir", + ["bar/", "foo/bar.txt", "foo/", "file_1.txt", "file_2.txt"], + ), + ( + "directories_before_files", + ["foo/", "bar/", "file_2.txt", "file_1.txt", "foo/bar.txt"], + ), + ( + "files_before_directories", + ["file_2.txt", "file_1.txt", "foo/bar.txt", "foo/", "bar/"], + ), + ( + "reverse_dirs_order", + ["foo/", "bar/", "foo/bar.txt", "file_1.txt", "file_2.txt"], + ), + ( + "reverse_files_order", + ["bar/", "foo/", "foo/bar.txt", "file_2.txt", "file_1.txt"], + ), + ] { + let archive_with_same_content_but_different_entries_order = + helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters(label, &test_dir), + AppenderEntries::new(to_entries(entries), content.clone()).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_same_content_but_different_entries_order.get_file_path(), + ); + } + } +} diff --git a/internal/mithril-file-archiver/tests/tests.rs b/internal/mithril-file-archiver/tests/tests.rs deleted file mode 100644 index d78be21bea8..00000000000 --- a/internal/mithril-file-archiver/tests/tests.rs +++ /dev/null @@ -1,1005 +0,0 @@ -//! Reproducibility contract for archives produced by [`FileArchiver`]. -//! -//! Given identical archive entry paths and contents, `FileArchiver` must produce a -//! byte-identical `.tar.zst` archive regardless of the host system, source base -//! directory, creation time, modification times, permissions, or input entry order. -//! -//! This contract assumes identical archive-format dependencies and zstandard -//! compression parameters. Changing either is an archive-format change and requires -//! intentionally versioning the format and updating its golden hashes. - -use std::collections::BTreeMap; -use std::fs::File; -use std::path::{Path, PathBuf}; - -use mithril_common::entities::CompressionAlgorithm; -use mithril_common::temp_dir_create; - -use mithril_file_archiver::appender::*; -use mithril_file_archiver::{ - ArchiveParameters, FileArchive, FileArchiver, ZstandardCompressionParameters, -}; - -mod helpers { - use sha2::{Digest, Sha256}; - use std::io::{Read, Write}; - - use super::*; - - /// Assert that two files have exactly the same bytes. - /// - /// SHA-256 hashes and file sizes are reported when they differ. - #[track_caller] - pub fn assert_files_are_byte_identical, A: AsRef>(expected: E, actual: A) { - let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); - let expected_bytes = std::fs::read(expected_path).unwrap_or_else(|error| { - panic!( - "Could not read expected file '{}': {error}", - expected_path.display() - ) - }); - let actual_bytes = std::fs::read(actual_path).unwrap_or_else(|error| { - panic!( - "Could not read actual file '{}': {error}", - actual_path.display() - ) - }); - - if expected_bytes != actual_bytes { - let expected_hash = hex::encode(Sha256::digest(&expected_bytes)); - let actual_hash = hex::encode(Sha256::digest(&actual_bytes)); - - panic!( - "Files are not byte-identical:\n\ - expected: '{}' ({} bytes, SHA-256: {})\n\ - actual: '{}' ({} bytes, SHA-256: {})", - expected_path.display(), - expected_bytes.len(), - expected_hash, - actual_path.display(), - actual_bytes.len(), - actual_hash, - ); - } - } - - /// **IMPORTANT** Default zstandard compression parameters are used. - pub fn file_archiver(work_dir: &Path) -> FileArchiver { - FileArchiver::new_with_default_parameters( - work_dir.join("verification"), - slog::Logger::root(slog::Discard, slog::o!()), - ) - } - - pub fn create_dir>(base_dir: &Path, dir_name: P) -> PathBuf { - let dir_path = base_dir.join(dir_name); - std::fs::create_dir(&dir_path).unwrap(); - dir_path - } - - pub fn create_file(root_dir: &Path, file_name: &str, size: Option) -> PathBuf { - let path = root_dir.join(file_name); - let mut file = File::create(&path).unwrap(); - - write!(file, "This is a test file named '{file_name}'").unwrap(); - writeln!(file).unwrap(); - - if let Some(file_size) = size { - file.set_len(file_size).unwrap(); - } - - path - } - - pub fn alter_file(path: &Path, alter_fn: F) { - let file = File::options().write(true).open(path).unwrap(); - alter_fn(&file); - } - - pub fn archive_parameters(filename: &str, target_dir: &Path) -> ArchiveParameters { - ArchiveParameters { - archive_name_without_extension: filename.to_string(), - target_directory: target_dir.to_path_buf(), - compression_algorithm: CompressionAlgorithm::Zstandard, - } - } - - pub fn compute_file_sha256(path: &Path) -> String { - let mut hasher = Sha256::new(); - - if path.is_file() { - hash_file(path, &mut hasher); - } else { - panic!("Path is not a file: {:?}", path.display()); - } - - hex::encode(hasher.finalize()) - } - - fn hash_file(path: &Path, hasher: &mut Sha256) { - let mut file = File::open(path).unwrap(); - let mut buffer = [0; 64 * 1024]; - - loop { - let bytes_read = file.read(&mut buffer).unwrap(); - if bytes_read == 0 { - break; - } - - hasher.update(&buffer[..bytes_read]); - } - } - - #[test] - fn assert_files_are_byte_identical_succeed_with_identical_files() { - let test_dir = temp_dir_create!(); - let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - - let reference_file_path = test_dir.join("reference_file"); - let mut reference_file = File::create_new(&reference_file_path).unwrap(); - reference_file.write_all(&content).unwrap(); - - let identical_file_path = test_dir.join("altered_file"); - std::fs::copy(&reference_file_path, &identical_file_path).unwrap(); - - assert_files_are_byte_identical(&reference_file_path, &identical_file_path); - } - - #[test] - #[should_panic(expected = "Files are not byte-identical")] - fn assert_files_are_byte_identical_fails_if_one_byte_changes() { - let test_dir = temp_dir_create!(); - let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let altered_content = { - let mut altered_content = content.clone(); - altered_content[0] = !altered_content[0]; - altered_content - }; - - let reference_file_path = test_dir.join("reference_file"); - let mut reference_file = File::create_new(&reference_file_path).unwrap(); - reference_file.write_all(&content).unwrap(); - - let altered_file_path = test_dir.join("altered_file"); - let mut altered_file = File::create_new(&altered_file_path).unwrap(); - altered_file.write_all(&altered_content).unwrap(); - - assert_files_are_byte_identical(&reference_file_path, &altered_file_path); - } -} - -/// Fixed test data set for file archiver tests. -mod test_data { - use serde::{Deserialize, Serialize}; - - use super::*; - - pub const TEST_BYTES: [u8; 10] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - - /// See: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks - pub(crate) const ZSTD_MAX_BLOCK_SIZE: u64 = 131_072; - - #[derive(Debug, PartialEq, Serialize, Deserialize)] - pub struct TestStruct { - field_str: String, - field_int: i32, - // Important note: using BTreeMap to ensure deterministic serialization, types like HashMap should - // not be used as the key order is not guaranteed - field_map: BTreeMap, - } - - impl Default for TestStruct { - fn default() -> Self { - Self { - field_str: "test".to_string(), - field_int: 42, - field_map: BTreeMap::from([ - ("key_3".to_string(), "value_3".to_string()), - ("key_1".to_string(), "value_1".to_string()), - ("key_2".to_string(), "value_2".to_string()), - ]), - } - } - } - - /// Create a file named `test.txt` in the given directory - pub fn create_test_txt(root_dir: &Path) -> PathBuf { - helpers::create_file(root_dir, "test.txt", Some(1048)) - } - - /// [AppenderEntries] ready list of entries created by [`create_test_dir`] - pub fn test_dir_entries() -> Vec { - vec![ - PathBuf::from("bar/"), - PathBuf::from("foo/"), - PathBuf::from("foo/bar.txt"), - PathBuf::from("file_1.txt"), - PathBuf::from("file_2.txt"), - ] - } - - /// Create a directory named `test_dir` in the given directory - /// - /// It will be filled with the following structure: - /// ```no_run - /// test_dir/ - /// ├── bar/ - /// ├── foo/ - /// │ └── bar.txt - /// ├── file_1.txt - /// ├── file_2.txt - /// ├── empty_file.txt - /// └── file_with_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt - /// ``` - pub fn create_test_dir(root_dir: &Path) -> PathBuf { - let test_dir = helpers::create_dir(root_dir, "test_dir"); - - helpers::create_dir(&test_dir, "bar"); - - let foo_dir = helpers::create_dir(&test_dir, "foo"); - helpers::create_file(&foo_dir, "bar.txt", None); - - helpers::create_file(&test_dir, "file_1.txt", Some(511)); - helpers::create_file(&test_dir, "file_2.txt", Some(ZSTD_MAX_BLOCK_SIZE * 2 + 1)); - - test_dir - } -} - -mod reproducibility { - use std::time::{Duration, SystemTime}; - - use super::*; - - mod repeated_archiving_produces_byte_identical_archives { - use std::time::Instant; - - use super::*; - - fn run_scenario(test_dir: PathBuf, ref_appender: T, repeated_appender: T) { - run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || {}); - } - - fn run_scenario_in_different_unix_seconds( - test_dir: PathBuf, - ref_appender: T, - repeated_appender: T, - ) { - run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || { - let seconds_since_unix_epoch = || { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - }; - - let reference_second = seconds_since_unix_epoch(); - let deadline = Instant::now() + Duration::from_millis(1500); - - while seconds_since_unix_epoch() == reference_second { - assert!( - Instant::now() < deadline, - "Unix time did not advance to the next second" - ); - std::thread::sleep(Duration::from_millis(10)); - } - }); - } - - fn run_scenario_with_hook( - test_dir: PathBuf, - ref_appender: T, - repeated_appender: T, - before_repeated_archive: F, - ) { - let reference_archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - ref_appender, - ) - .unwrap(); - - before_repeated_archive(); - - let repeated_archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("repeated", &test_dir), - repeated_appender, - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - reference_archive.get_file_path(), - repeated_archive.get_file_path(), - ); - } - - #[test] - fn appender_data_from_json() { - let test_dir = temp_dir_create!(); - let content = test_data::TestStruct::default(); - - // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. - run_scenario_in_different_unix_seconds( - test_dir, - AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), - AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), - ); - } - - #[test] - fn appender_data_from_raw_bytes() { - let test_dir = temp_dir_create!(); - let content = test_data::TEST_BYTES.to_vec(); - - // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. - run_scenario_in_different_unix_seconds( - test_dir, - AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content.clone()), - AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), - ); - } - - #[test] - fn appender_file() { - let test_dir = temp_dir_create!(); - let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); - - run_scenario( - test_dir, - AppenderFile::append_at_archive_root(content.clone()).unwrap(), - AppenderFile::append_at_archive_root(content).unwrap(), - ); - } - - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); - - run_scenario( - test_dir, - AppenderDirAll::new(content.clone()), - AppenderDirAll::new(content), - ); - } - - #[test] - fn appender_entries() { - let test_dir = temp_dir_create!(); - let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); - - run_scenario( - test_dir, - AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), - AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), - ); - } - } - - mod source_metadata_does_not_affect_archive_bytes { - use std::fs::Permissions; - - use super::*; - - /// [AppenderEntries] ready list of the reference/altered dirs created by [setup_test_dirs]. - fn test_dir_entries() -> Vec { - vec![ - PathBuf::from("empty/"), - PathBuf::from("subdir/"), - PathBuf::from("subdir/file_2.txt"), - PathBuf::from("subdir/file_3.txt"), - PathBuf::from("file_1.txt"), - ] - } - - /// Create two directories with the same structure and applies the given function to each - /// reference/altered files pair. - /// - /// To be used with Appender that can work on a group of files and directories. - /// - /// Each directory contains the following files: - /// ```no_run - /// (reference|altered)/ - /// ├── empty/ - /// ├── subdir/ - /// │ ├── file_2.txt - /// │ └── file_3.txt - /// └── file_1.txt - /// ``` - fn setup_test_dirs( - test_dir: &Path, - setup_reference_and_altered_path_fn: F, - ) { - let source = helpers::create_dir(test_dir, "source"); - let reference_dir = helpers::create_dir(&source, "reference"); - let altered_dir = helpers::create_dir(&source, "altered"); - let subdirs = vec![Path::new("empty"), Path::new("subdir")]; - - for dir_path in &subdirs { - helpers::create_dir(&reference_dir, dir_path); - helpers::create_dir(&altered_dir, dir_path); - } - - for file_path in ["file_1.txt", "subdir/file_2.txt", "subdir/file_3.txt"] { - let reference_file = helpers::create_file(&reference_dir, file_path, None); - let altered_file = helpers::create_file(&altered_dir, file_path, None); - - // Ensure the two files have the same content - helpers::assert_files_are_byte_identical(&reference_file, &altered_file); - - setup_reference_and_altered_path_fn(&reference_file, &altered_file); - } - - // Alter dirs after the files to avoid permission issues - for dir_path in &subdirs { - setup_reference_and_altered_path_fn( - &reference_dir.join(dir_path), - &altered_dir.join(dir_path), - ); - } - setup_reference_and_altered_path_fn(&reference_dir, &altered_dir); - } - - /// Create two directories with a single "test.txt" file and applies the given function - /// to the reference/altered file pair. - /// - /// To be used with Appender that can work on a single file. - fn setup_test_file( - test_dir: &Path, - setup_reference_and_altered_path_fn: F, - ) { - let source = helpers::create_dir(test_dir, "source"); - let reference_dir = helpers::create_dir(&source, "reference"); - let altered_dir = helpers::create_dir(&source, "altered"); - - let reference_file = helpers::create_file(&reference_dir, "file.txt", None); - let altered_file = helpers::create_file(&altered_dir, "file.txt", None); - - // Ensure the two files have the same content - helpers::assert_files_are_byte_identical(&reference_file, &altered_file); - - setup_reference_and_altered_path_fn(&reference_file, &altered_file); - } - - fn run_scenario A>( - test_dir: PathBuf, - build_tar_appender: B, - ) { - let source = test_dir.join("source"); - let reference_archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - build_tar_appender(source.join("reference")), - ) - .unwrap(); - - let archive_with_different_metadata = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("altered_metadata", &test_dir), - build_tar_appender(source.join("altered")), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - reference_archive.get_file_path(), - archive_with_different_metadata.get_file_path(), - ); - } - - mod modification_time { - use super::*; - - fn setup_modification_time(reference_path: &Path, path_with_different_metadata: &Path) { - if reference_path.is_dir() || path_with_different_metadata.is_dir() { - // can't modify dir times until - // is stabilized (currently planned for rust 1.99) - return; - } - - let base_time = SystemTime::UNIX_EPOCH; - helpers::alter_file(reference_path, |file| file.set_modified(base_time).unwrap()); - helpers::alter_file(path_with_different_metadata, |file| { - // note: in TAR, entries mtimes have a granularity of a second - file.set_modified(base_time + Duration::from_millis(5300)).unwrap() - }); - - assert_ne!( - reference_path.metadata().unwrap().modified().unwrap(), - path_with_different_metadata.metadata().unwrap().modified().unwrap() - ); - } - - #[test] - fn appender_file() { - let test_dir = temp_dir_create!(); - setup_test_file(&test_dir, setup_modification_time); - run_scenario(test_dir, |source| { - AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() - }); - } - - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_modification_time); - run_scenario(test_dir, AppenderDirAll::new); - } - - #[test] - fn appender_entries() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_modification_time); - run_scenario(test_dir, |source| { - AppenderEntries::new(test_dir_entries(), source).unwrap() - }); - } - } - - mod permissions { - use super::*; - - #[cfg(unix)] - fn setup_permissions(reference_path: &Path, path_with_different_metadata: &Path) { - use std::os::unix::fs::PermissionsExt; - - let (reference_permission, altered_permission) = - // IMPORTANT: for directory the owner permission must be `7`, else this prevents - // `temp_dir_create` cleanup and make appenders fails on subdir files - if reference_path.is_dir() || path_with_different_metadata.is_dir() { - (Permissions::from_mode(0o766), Permissions::from_mode(0o767)) - } else { - (Permissions::from_mode(0o644), Permissions::from_mode(0o646)) - }; - - std::fs::set_permissions(reference_path, reference_permission).unwrap(); - std::fs::set_permissions(path_with_different_metadata, altered_permission).unwrap(); - - assert_ne!( - reference_path.metadata().unwrap().permissions(), - path_with_different_metadata.metadata().unwrap().permissions() - ); - } - - #[cfg(unix)] - #[test] - fn appender_file() { - let test_dir = temp_dir_create!(); - setup_test_file(&test_dir, setup_permissions); - run_scenario(test_dir, |source| { - AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() - }) - } - - #[cfg(unix)] - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_permissions); - run_scenario(test_dir, AppenderDirAll::new); - } - - #[cfg(unix)] - #[test] - fn appender_entries() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_permissions); - run_scenario(test_dir, |source| { - AppenderEntries::new(test_dir_entries(), source).unwrap() - }); - } - } - } - - mod source_base_directory_does_not_affect_archive { - use super::*; - - #[test] - fn appender_file() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let subdir_1 = helpers::create_dir(&source, "first"); - let subdir_2 = helpers::create_dir(&source, "second"); - - let content = test_data::create_test_txt(&subdir_1); - let same_content_in_other_dir = test_data::create_test_txt(&subdir_2); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderFile::append_at_archive_root(content).unwrap(), - ) - .unwrap(); - let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("from_another_dir", &test_dir), - AppenderFile::append_at_archive_root(same_content_in_other_dir).unwrap(), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - archive.get_file_path(), - archive_with_same_content_but_from_another_dir.get_file_path(), - ); - } - - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let subdir_1 = helpers::create_dir(&source, "first"); - let subdir_2 = helpers::create_dir(&source, "second"); - - let content = test_data::create_test_dir(&subdir_1); - let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderDirAll::new(content), - ) - .unwrap(); - let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("from_another_dir", &test_dir), - AppenderDirAll::new(same_content_in_other_dir), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - archive.get_file_path(), - archive_with_same_content_but_from_another_dir.get_file_path(), - ); - } - - #[test] - fn appender_entries() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let subdir_1 = helpers::create_dir(&source, "first"); - let subdir_2 = helpers::create_dir(&source, "second"); - - let content = test_data::create_test_dir(&subdir_1); - let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), - ) - .unwrap(); - let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("from_another_dir", &test_dir), - AppenderEntries::new(test_data::test_dir_entries(), same_content_in_other_dir) - .unwrap(), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - archive.get_file_path(), - archive_with_same_content_but_from_another_dir.get_file_path(), - ); - } - } - - mod appender_entry_specifics { - use super::*; - - fn to_entries(paths: [&str; N]) -> Vec { - paths.into_iter().map(PathBuf::from).collect() - } - - #[test] - fn equivalent_entry_paths_produce_identical_archives() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let content = test_data::create_test_dir(&source); - - let reference_entries = ["bar/", "foo/", "foo/bar.txt", "file_1.txt", "file_2.txt"]; - let equivalent_entries = ["bar", "foo", "foo/bar.txt", "file_1.txt", "file_2.txt"]; - - let reference_archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderEntries::new(to_entries(reference_entries), content.clone()).unwrap(), - ) - .unwrap(); - - let archive_with_equivalent_entries_spelling = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("equivalent_spelling", &test_dir), - AppenderEntries::new(to_entries(equivalent_entries), content).unwrap(), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - reference_archive.get_file_path(), - archive_with_equivalent_entries_spelling.get_file_path(), - ); - } - - #[test] - fn supplied_entry_order_does_not_affect_appender_entries_archive() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let content = test_data::create_test_dir(&source); - - let reference_archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), - ) - .unwrap(); - - for (label, entries) in [ - ( - "child_before_parent_dir", - ["bar/", "foo/bar.txt", "foo/", "file_1.txt", "file_2.txt"], - ), - ( - "directories_before_files", - ["foo/", "bar/", "file_2.txt", "file_1.txt", "foo/bar.txt"], - ), - ( - "files_before_directories", - ["file_2.txt", "file_1.txt", "foo/bar.txt", "foo/", "bar/"], - ), - ( - "reverse_dirs_order", - ["foo/", "bar/", "foo/bar.txt", "file_1.txt", "file_2.txt"], - ), - ( - "reverse_files_order", - ["bar/", "foo/", "foo/bar.txt", "file_2.txt", "file_1.txt"], - ), - ] { - let archive_with_same_content_but_different_entries_order = - helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters(label, &test_dir), - AppenderEntries::new(to_entries(entries), content.clone()).unwrap(), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - reference_archive.get_file_path(), - archive_with_same_content_but_different_entries_order.get_file_path(), - ); - } - } - } -} - -// The golden hashes are pinned to: the tar/zstd crate versions, GOLDEN_COMPRESSION_PARAMETERS -// (multi-threaded zstd), and the exact bytes produced by `test_data::create_test_*`. -// Any change to one of those invalidates them. -mod golden_master { - use super::*; - - // ** These hashes define TAR_ZSTD_V1. Update them only for an intentional archive-format change. ** - pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = - "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; - pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256: &str = - "647bba62029e63b0cbd329bccdfdb850ec934a07ab0811cc4d2d7682b72d3054"; - pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = - "565da17762fcf8f417f1ad78fda003b19507757bf56ac09ff24a9aac179bf78c"; - pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = - "fd5a178f1c717de39aef526d378e731871b74ee1ba11bdc159a4a872867d3748"; - pub const TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256: &str = - "6843b658c2c176afa156e82f7ee3d828f63cd7e9de16dc3166a5da96f0d3e15f"; - pub const TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256: &str = - "ff5447a1b80b530b3d94cfeb2ad69fdc392a4d5175f24ffc13ad10b08a7e12ba"; - - /// Create a directory named `test_dir` in the given directory - /// - /// It will be filled with the following structure: - /// ```no_run - /// test_dir/ - /// ├── bar/ - /// ├── foo/ - /// │ └── bar.txt - /// ├── empty_file.txt - /// ├── file_1.txt - /// ├── file_2.txt - /// └── file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt - /// ``` - pub fn create_golden_test_dir(root_dir: &Path) -> PathBuf { - let test_dir = helpers::create_dir(root_dir, "test_dir"); - - helpers::create_dir(&test_dir, "bar"); - - let foo_dir = helpers::create_dir(&test_dir, "foo"); - helpers::create_file(&foo_dir, "bar.txt", None); - - File::create(test_dir.join("empty.txt")).unwrap(); - - helpers::create_file(&test_dir, "file_1.txt", Some(511)); - helpers::create_file( - &test_dir, - "file_2.txt", - Some(test_data::ZSTD_MAX_BLOCK_SIZE * 2 + 1), - ); - helpers::create_file( - &test_dir, - "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", - None, - ); - - test_dir - } - - /// [AppenderEntries] ready list of entries created by [`create_golden_test_dir`] - pub fn golden_test_dir_entries() -> Vec { - vec![ - PathBuf::from("bar/"), - PathBuf::from("foo/"), - PathBuf::from("foo/bar.txt"), - PathBuf::from("empty.txt"), - PathBuf::from("file_1.txt"), - PathBuf::from("file_2.txt"), - PathBuf::from( - "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", - ), - ] - } - - #[track_caller] - fn assert_archive_not_empty(archive: &FileArchive) { - assert!( - archive.get_uncompressed_size() > 0, - "Archive '{}' has no content, fix the archive creation and try again", - archive.get_file_path().display() - ); - } - - /// Assert an archive matches its golden hash. - #[track_caller] - fn assert_archive_matches_golden_sha256(archive: &FileArchive, expected_sha256: &str) { - let actual_sha256 = helpers::compute_file_sha256(archive.get_file_path()); - - assert_eq!( - expected_sha256, - actual_sha256, - "Archive bytes changed ('{}', {} bytes).\n\ - Either the archive format is no longer reproducible, or the change is intentional \ - and the TAR_ZSTD_V* constants must be recomputed and their version bumped.", - archive.get_file_path().display(), - archive.get_archive_size(), - ); - } - - /// Compression parameters the golden hashes were computed with. - /// - /// They must mirror the production defaults, see [defaults_are_the_golden_parameters]. - const GOLDEN_COMPRESSION_PARAMETERS: ZstandardCompressionParameters = - ZstandardCompressionParameters { - level: 9, - number_of_workers: 4, - }; - - #[test] - fn defaults_are_the_golden_parameters() { - assert_eq!( - GOLDEN_COMPRESSION_PARAMETERS, - ZstandardCompressionParameters::default(), - "zstandard defaults changed: archives are no longer byte-compatible with the \ - previously published ones, the golden hashes must be recomputed and TAR_ZSTD_V1 \ - bumped to V2" - ); - } - - #[test] - fn appender_data_from_json() { - let test_dir = temp_dir_create!(); - let content = test_data::TestStruct::default(); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_DATA_SHA256); - } - - #[test] - fn appender_data_from_raw_bytes() { - let test_dir = temp_dir_create!(); - let content = test_data::TEST_BYTES.to_vec(); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256); - } - - #[test] - fn appender_file() { - let test_dir = temp_dir_create!(); - let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderFile::append_at_archive_root(content).unwrap(), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_FILE_SHA256); - } - - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderDirAll::new(content), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256( - &archive, - TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256, - ); - } - - #[test] - fn appender_entries() { - let test_dir = temp_dir_create!(); - let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); - - // Should construct the same archive as `AppenderDirAll` as we include all the source directory entries - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderEntries::new(golden_test_dir_entries(), content).unwrap(), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256( - &archive, - TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256, - ); - } - - #[test] - fn chain_appender() { - let test_dir = temp_dir_create!(); - let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderEntries::new(golden_test_dir_entries(), content) - .unwrap() - .chain( - AppenderData::from_json( - PathBuf::from("test.json"), - &test_data::TestStruct::default(), - ) - .unwrap(), - ), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256( - &archive, - TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256, - ); - } -} From cef439fc13b53a2058c5b3ad369959f0aacd5d4d Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:38:21 +0200 Subject: [PATCH 10/19] refactor(aggregrator): remove unused `tar` and `zstd` dependencies and centralize `unpack_archive` test utility function --- Cargo.lock | 2 -- .../src/test/extensions.rs | 14 ++++--------- .../mithril-file-archiver/src/test/mod.rs | 18 +++++++++++++++++ mithril-aggregator/Cargo.toml | 2 -- .../cardano_database_artifacts/digest.rs | 20 ++----------------- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ee12d2132a..e3d42f408bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4266,7 +4266,6 @@ dependencies = [ "slog-scope", "slog-term", "sqlite", - "tar", "tempfile", "thiserror 2.0.18", "tikv-jemallocator", @@ -4274,7 +4273,6 @@ dependencies = [ "tokio-util", "uuid", "warp", - "zstd", ] [[package]] diff --git a/internal/mithril-file-archiver/src/test/extensions.rs b/internal/mithril-file-archiver/src/test/extensions.rs index 97d3d8e6153..830f551c82d 100644 --- a/internal/mithril-file-archiver/src/test/extensions.rs +++ b/internal/mithril-file-archiver/src/test/extensions.rs @@ -1,12 +1,9 @@ -use std::fs::File; use std::path::{Path, PathBuf}; -use tar::Archive; -use zstd::stream::read::Decoder; - use mithril_common::entities::CompressionAlgorithm; use crate::FileArchive; +use crate::test::unpack_archive; /// Extension trait adding test utilities to [FileArchive] pub trait FileArchiveTestExtension { @@ -22,13 +19,10 @@ impl FileArchiveTestExtension for FileArchive { panic!("Only Zstandard compression is supported"); } - let parent_dir = parent_dir.as_ref(); - let file_tar_zst = File::open(self.get_file_path()).unwrap(); - let file_tar_zst_decoder = Decoder::new(file_tar_zst).unwrap(); - let mut archive = Archive::new(file_tar_zst_decoder); - let unpack_path = parent_dir.join("unpack"); + let unpack_path = parent_dir.as_ref().join("unpack"); std::fs::create_dir(&unpack_path).unwrap(); - archive.unpack(&unpack_path).unwrap(); + + unpack_archive(self.get_file_path(), &unpack_path).unwrap(); unpack_path } diff --git a/internal/mithril-file-archiver/src/test/mod.rs b/internal/mithril-file-archiver/src/test/mod.rs index 7bc5278f37e..86a82296dc9 100644 --- a/internal/mithril-file-archiver/src/test/mod.rs +++ b/internal/mithril-file-archiver/src/test/mod.rs @@ -8,9 +8,27 @@ pub mod double; mod extensions; pub use extensions::*; + #[cfg(test)] pub(crate) use internal_tests_only::*; +/// Unpack a zstandard-compressed tar archive to a specified directory. +/// +/// Note: `unpack_dir` must exist. +pub fn unpack_archive( + archive_path: &std::path::Path, + unpack_dir: &std::path::Path, +) -> mithril_common::StdResult<()> { + let mut archive = { + let file_tar_zst = std::fs::File::open(archive_path)?; + let file_tar_zst_decoder = zstd::Decoder::new(file_tar_zst)?; + tar::Archive::new(file_tar_zst_decoder) + }; + + archive.unpack(unpack_dir)?; + Ok(()) +} + #[cfg(test)] mod internal_tests_only { use std::fs::File; diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index ad7c3ec96af..2c84fd4671d 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -54,13 +54,11 @@ slog = { workspace = true, features = ["max_level_trace", "release_max_level_deb slog-async = { workspace = true } slog-bunyan = { workspace = true } sqlite = { version = "0.37.0", features = ["bundled"] } -tar = "0.4.46" thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } tokio-util = { version = "0.7.18", features = ["codec"] } uuid = { version = "1.23.4", features = ["v4", "fast-rng", "macro-diagnostics"] } warp = { workspace = true } -zstd = { version = "0.13.3", features = ["zstdmt"] } [target.'cfg(all(not(target_os = "windows"), not(all(target_os = "linux", target_arch = "aarch64"))))'.dependencies] tikv-jemallocator = { version = "0.7.0" } diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs index 041f9269155..885d84f7b3b 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs @@ -287,12 +287,7 @@ impl DigestArtifactBuilder { #[cfg(test)] mod tests { use anyhow::anyhow; - use std::{ - collections::BTreeMap, - fs::{File, read_to_string}, - }; - use tar::Archive; - use zstd::Decoder; + use std::{collections::BTreeMap, fs::read_to_string}; use mithril_common::{ current_function, @@ -300,7 +295,7 @@ mod tests { messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage}, test::{TempDir, assert_equivalent, double::Dummy}, }; - use mithril_file_archiver::FileArchiver; + use mithril_file_archiver::{FileArchiver, test::unpack_archive}; use crate::{ file_uploaders::FileUploadRetryPolicy, @@ -359,17 +354,6 @@ mod tests { immutable_file_digest_mapper } - fn unpack_archive(archive_path: &Path, unpack_dir: &Path) -> StdResult<()> { - let mut archive = { - let file_tar_zst = File::open(archive_path)?; - let file_tar_zst_decoder = Decoder::new(file_tar_zst)?; - Archive::new(file_tar_zst_decoder) - }; - - archive.unpack(unpack_dir)?; - Ok(()) - } - fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { FileArchiver::new_with_default_parameters( work_dir.join("verification"), From 4b8d4723ce522c184ea7e75edd325c845bb2a3ae Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:41:24 +0200 Subject: [PATCH 11/19] refactor(aggregator): remove unused `tempfile``, and `mithril-test-http-server` dependencies --- Cargo.lock | 2 -- mithril-aggregator/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3d42f408bd..850cf296386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4248,7 +4248,6 @@ dependencies = [ "mithril-resource-pool", "mithril-signed-entity-lock", "mithril-signed-entity-preloader", - "mithril-test-http-server", "mithril-ticker", "mockall", "paste", @@ -4266,7 +4265,6 @@ dependencies = [ "slog-scope", "slog-term", "sqlite", - "tempfile", "thiserror 2.0.18", "tikv-jemallocator", "tokio", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 2c84fd4671d..34afc261c6b 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -75,11 +75,9 @@ http = "1.4.2" httpmock = "0.8.3" mithril-api-spec = { path = "../internal/tests/mithril-api-spec" } mithril-common = { path = "../mithril-common", features = ["allow_skip_signer_certification"] } -mithril-test-http-server = { path = "../internal/tests/mithril-test-http-server" } mockall = { workspace = true } slog-scope = "4.4.1" slog-term = { workspace = true } -tempfile = "3.27.0" warp = { workspace = true, features = ["test"] } [[bench]] From 2685982eeea5f93e08078fe1e49caa082d635d20 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:47:47 +0200 Subject: [PATCH 12/19] chore(file_archiver): pin `zstd` and `tar` versions to ensure archive byte stability across builds - `tar` pinned to `0.4.46` - `zstd` pinned to `0.13.3` --- internal/mithril-file-archiver/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/mithril-file-archiver/Cargo.toml b/internal/mithril-file-archiver/Cargo.toml index 755697b01ed..a4e79c71230 100644 --- a/internal/mithril-file-archiver/Cargo.toml +++ b/internal/mithril-file-archiver/Cargo.toml @@ -14,8 +14,8 @@ mithril-common = { path = "../../mithril-common" } serde = { workspace = true } serde_json = { workspace = true } slog = { workspace = true } -tar = "0.4.46" -zstd = { version = "0.13.3", features = ["zstdmt"] } +tar = "=0.4.46" # Pinned to ensure archive bytes stability across builds +zstd = { version = "=0.13.3", features = ["zstdmt"] } # Pinned to ensure archive bytes stability across builds [dev-dependencies] hex = { workspace = true } From 41762f7e7b06bd0c7661a25a488d43a99e742072 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:49:48 +0200 Subject: [PATCH 13/19] fix(file_archiver): handle leading `./` in paths for consistent normalization across platforms --- .../mithril-file-archiver/src/appender.rs | 24 +++++++++++++++++-- .../tests/reproducibility.rs | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/internal/mithril-file-archiver/src/appender.rs b/internal/mithril-file-archiver/src/appender.rs index e16ce0c10ec..064798ae060 100644 --- a/internal/mithril-file-archiver/src/appender.rs +++ b/internal/mithril-file-archiver/src/appender.rs @@ -4,7 +4,7 @@ use anyhow::{Context, anyhow}; use serde::Serialize; use std::fs::File; use std::io::Write; -use std::path::PathBuf; +use std::path::{Component, PathBuf}; use mithril_common::StdResult; @@ -143,7 +143,10 @@ impl AppenderEntries { } fn normalize_entry(entry: PathBuf) -> PathBuf { - entry.components().collect() + entry + .components() + .filter(|c| !matches!(c, Component::CurDir)) + .collect() } } @@ -297,6 +300,23 @@ mod tests { mod appender_entries { use super::*; + #[test] + fn removes_leading_current_directory_component() { + assert_eq!( + PathBuf::from("foo/bar.txt"), + AppenderEntries::normalize_entry(PathBuf::from("./foo/bar.txt")), + ); + } + + #[cfg(windows)] + #[test] + fn removes_windows_leading_current_directory_component() { + assert_eq!( + PathBuf::from("foo").join("bar.txt"), + AppenderEntries::normalize_entry(PathBuf::from(r".\foo\bar.txt")), + ); + } + #[test] fn normalizes_directory_spelling_and_sorts_entries() { let appender = AppenderEntries::new( diff --git a/internal/mithril-file-archiver/tests/reproducibility.rs b/internal/mithril-file-archiver/tests/reproducibility.rs index 9be3f2a5e8f..93f566d63bb 100644 --- a/internal/mithril-file-archiver/tests/reproducibility.rs +++ b/internal/mithril-file-archiver/tests/reproducibility.rs @@ -460,7 +460,7 @@ mod appender_entry_specifics { let content = test_data::create_test_dir(&source); let reference_entries = ["bar/", "foo/", "foo/bar.txt", "file_1.txt", "file_2.txt"]; - let equivalent_entries = ["bar", "foo", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + let equivalent_entries = ["bar", "foo", "foo/bar.txt", "./file_1.txt", "file_2.txt"]; let reference_archive = helpers::file_archiver(&test_dir) .archive( From a297233945b6693d2012a120481f04abf8cbd68c Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:56:01 +0200 Subject: [PATCH 14/19] fix(file_archiver): disable sparse file support in tar builder and update golden master hashes --- internal/mithril-file-archiver/src/api.rs | 3 +++ internal/mithril-file-archiver/tests/golden_master.rs | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/mithril-file-archiver/src/api.rs b/internal/mithril-file-archiver/src/api.rs index 99583bcd511..b52d8e1c61d 100644 --- a/internal/mithril-file-archiver/src/api.rs +++ b/internal/mithril-file-archiver/src/api.rs @@ -279,6 +279,9 @@ impl FileArchiver { fn configure_tar_builder(builder: &mut tar::Builder) { builder.mode(HeaderMode::Deterministic); builder.follow_symlinks(false); + // disable sparse files, as their support is not uniform across platforms and the size + // difference won't matter with zstandard compression + builder.sparse(false); } } diff --git a/internal/mithril-file-archiver/tests/golden_master.rs b/internal/mithril-file-archiver/tests/golden_master.rs index 1713568ba1c..fc1f693a117 100644 --- a/internal/mithril-file-archiver/tests/golden_master.rs +++ b/internal/mithril-file-archiver/tests/golden_master.rs @@ -18,15 +18,15 @@ use extensions::*; pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256: &str = - "647bba62029e63b0cbd329bccdfdb850ec934a07ab0811cc4d2d7682b72d3054"; + "fb55c9b984eab74e599466ea2fbd8af92f5e8fd52296439c9a29340f5f801bea"; pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = - "565da17762fcf8f417f1ad78fda003b19507757bf56ac09ff24a9aac179bf78c"; + "cf6c2755fe389e40891c5cb6e4b08e0f01e09846e58f229d4dfe138adc7d1157"; pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = "fd5a178f1c717de39aef526d378e731871b74ee1ba11bdc159a4a872867d3748"; pub const TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256: &str = "6843b658c2c176afa156e82f7ee3d828f63cd7e9de16dc3166a5da96f0d3e15f"; pub const TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256: &str = - "ff5447a1b80b530b3d94cfeb2ad69fdc392a4d5175f24ffc13ad10b08a7e12ba"; + "d2f8df8672bb6eca365d485445011bdd0adde34c978f4deded2cd548e97c24ab"; /// Create a directory named `test_dir` in the given directory /// From 45b7bf5d89ca36e1504181438f1057a4a9fa050a Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:22:02 +0200 Subject: [PATCH 15/19] refactor(file_archiver): remove `AppenderDirAll` and its related tests, update usages to `AppenderEntries` or new `FailAppender` --- internal/mithril-file-archiver/src/api.rs | 52 ++++++++--------- .../mithril-file-archiver/src/appender.rs | 57 ------------------- .../src/test/double/fail_appender.rs | 21 +++++++ .../src/test/double/mod.rs | 3 + .../mithril-file-archiver/src/test/mod.rs | 6 -- .../tests/golden_master.rs | 21 ------- .../tests/reproducibility.rs | 56 ------------------ 7 files changed, 49 insertions(+), 167 deletions(-) create mode 100644 internal/mithril-file-archiver/src/test/double/fail_appender.rs diff --git a/internal/mithril-file-archiver/src/api.rs b/internal/mithril-file-archiver/src/api.rs index b52d8e1c61d..030abe97ba2 100644 --- a/internal/mithril-file-archiver/src/api.rs +++ b/internal/mithril-file-archiver/src/api.rs @@ -289,10 +289,11 @@ impl FileArchiver { mod tests { use std::fs::File; + use mithril_common::temp_dir_create; use mithril_common::test::assert_equivalent; - use crate::appender::{AppenderDirAll, AppenderFile}; - use crate::test::{FileArchiveTestExtension, create_dir, create_file, get_test_directory}; + use crate::appender::{AppenderEntries, AppenderFile}; + use crate::test::{FileArchiveTestExtension, create_dir, create_file, double::FailAppender}; use super::*; @@ -305,18 +306,17 @@ mod tests { #[test] fn should_create_a_valid_archive_with_zstandard_compression() { - let test_dir = - get_test_directory("should_create_a_valid_archive_with_zstandard_compression"); + let test_dir = temp_dir_create!(); let target_archive = test_dir.join("archive.tar.zst"); - let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory")); - create_file(&archived_directory, "file_to_archive.txt"); + let source_dir = test_dir.join(create_dir(&test_dir, "source")); + let archived_file = source_dir.join(create_file(&source_dir, "file_to_archive.txt")); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); let archive = file_archiver .create_archive( &target_archive, - AppenderDirAll::new(archived_directory), + AppenderFile::append_at_archive_root(archived_file).unwrap(), CompressionAlgorithm::Zstandard, ) .expect("create_archive should not fail"); @@ -327,10 +327,7 @@ mod tests { #[test] fn should_delete_tmp_file_in_target_directory_if_archiving_fail() { - let test_dir = - get_test_directory("should_delete_tmp_file_in_target_directory_if_archiving_fail"); - // Note: the archived directory does not exist in order to make the archive process fail - let archived_directory = test_dir.join("db"); + let test_dir = temp_dir_create!(); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -343,7 +340,7 @@ mod tests { compression_algorithm: CompressionAlgorithm::Zstandard, }; let _ = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive(archive_params, FailAppender) .expect_err("FileArchiver::archive should fail if the target path doesn't exist."); let remaining_files: Vec = list_remaining_files(&test_dir); @@ -352,11 +349,7 @@ mod tests { #[test] fn should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail() { - let test_dir = get_test_directory( - "should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail", - ); - // Note: the archived directory does not exist in order to make the archive process fail - let archived_directory = test_dir.join("db"); + let test_dir = temp_dir_create!(); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -372,7 +365,7 @@ mod tests { compression_algorithm: CompressionAlgorithm::Zstandard, }; let _ = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive(archive_params, FailAppender) .expect_err("FileArchiver::archive should fail if the db is empty."); let remaining_files: Vec = list_remaining_files(&test_dir); @@ -384,11 +377,9 @@ mod tests { #[test] fn overwrite_already_existing_archive_when_archiving_succeed() { - let test_dir = - get_test_directory("overwrite_already_existing_archive_when_archiving_succeed"); - let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory")); - - create_file(&archived_directory, "file_to_archive.txt"); + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + let file_to_archive = create_file(&source, "file_to_archive.txt"); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -400,15 +391,22 @@ mod tests { let first_archive = file_archiver .archive( archive_params.clone(), - AppenderDirAll::new(archived_directory.clone()), + AppenderEntries::new(vec![file_to_archive.clone()], source.clone()).unwrap(), ) .unwrap(); let first_archive_size = first_archive.get_archive_size(); - create_file(&archived_directory, "another_file_to_archive.txt"); + let another_file_to_archive = create_file(&source, "another_file_to_archive.txt"); let second_archive = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive( + archive_params, + AppenderEntries::new( + vec![file_to_archive, another_file_to_archive], + source.clone(), + ) + .unwrap(), + ) .unwrap(); let second_archive_size = second_archive.get_archive_size(); @@ -420,7 +418,7 @@ mod tests { #[test] fn compute_size_of_uncompressed_data_and_archive() { - let test_dir = get_test_directory("compute_size_of_uncompressed_data_and_archive"); + let test_dir = temp_dir_create!(); let file_path = test_dir.join("file.txt"); let file = File::create(&file_path).unwrap(); diff --git a/internal/mithril-file-archiver/src/appender.rs b/internal/mithril-file-archiver/src/appender.rs index 064798ae060..5f7805df125 100644 --- a/internal/mithril-file-archiver/src/appender.rs +++ b/internal/mithril-file-archiver/src/appender.rs @@ -29,34 +29,6 @@ pub trait TarAppender: Send { } } -/// An appender that add a directory and all of its content (recursively). -pub struct AppenderDirAll { - target_directory: PathBuf, -} - -impl AppenderDirAll { - /// [AppenderDirAll] factory - pub fn new(target_directory: PathBuf) -> Self { - Self { target_directory } - } -} - -impl TarAppender for AppenderDirAll { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - tar.append_dir_all(".", &self.target_directory).with_context(|| { - format!( - "Create archive error: Can not add directory: '{}' to the archive", - self.target_directory.display() - ) - })?; - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - file_size::compute_size_of_path(&self.target_directory) - } -} - /// An appender that add one file. pub struct AppenderFile { /// Location of the file in the archive. @@ -586,35 +558,6 @@ mod tests { } } - mod appender_dir_all { - use super::*; - - #[test] - fn compute_uncompressed_size() { - let test_dir = "appender_dir_all_compute_size"; - - let immutable_trio_file_size = 777; - let ledger_file_size = 6666; - let volatile_file_size = 99; - - let cardano_db = DummyCardanoDbBuilder::new(test_dir) - .with_immutables(&[1, 2]) - .set_immutable_trio_file_size(immutable_trio_file_size) - .with_legacy_ledger_snapshots(&[437, 537, 637]) - .set_ledger_file_size(ledger_file_size) - .with_volatile_files(&["blocks-0.dat"]) - .set_volatile_file_size(volatile_file_size) - .build(); - - let appender_dir_all = AppenderDirAll::new(cardano_db.get_dir().clone()); - - let entries_size = appender_dir_all.compute_uncompressed_data_size().unwrap(); - let expected_total_size = - (immutable_trio_file_size * 2) + (3 * ledger_file_size) + volatile_file_size; - assert_eq!(expected_total_size, entries_size); - } - } - mod appender_data { use serde::Deserialize; use zstd::Decoder; diff --git a/internal/mithril-file-archiver/src/test/double/fail_appender.rs b/internal/mithril-file-archiver/src/test/double/fail_appender.rs new file mode 100644 index 00000000000..420e9ad78c8 --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/fail_appender.rs @@ -0,0 +1,21 @@ +use std::io::Write; +use tar::Builder; + +use mithril_common::StdResult; + +use crate::appender::TarAppender; + +/// A test double appender that always fails. +/// +/// Used in tests to verify error handling behavior when appending operations fail. +pub struct FailAppender; + +impl TarAppender for FailAppender { + fn append(&self, _tar: &mut Builder) -> StdResult<()> { + anyhow::bail!("FailAppender always fails (append)") + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + anyhow::bail!("FailAppender always fails (compute_uncompressed_data_size)") + } +} diff --git a/internal/mithril-file-archiver/src/test/double/mod.rs b/internal/mithril-file-archiver/src/test/double/mod.rs index ddcde6270ff..e569a50bff9 100644 --- a/internal/mithril-file-archiver/src/test/double/mod.rs +++ b/internal/mithril-file-archiver/src/test/double/mod.rs @@ -3,3 +3,6 @@ //! Enable unit testing with controlled inputs and predictable behavior. mod dummies; +mod fail_appender; + +pub use fail_appender::FailAppender; diff --git a/internal/mithril-file-archiver/src/test/mod.rs b/internal/mithril-file-archiver/src/test/mod.rs index 86a82296dc9..c782b4b0e7a 100644 --- a/internal/mithril-file-archiver/src/test/mod.rs +++ b/internal/mithril-file-archiver/src/test/mod.rs @@ -34,14 +34,8 @@ mod internal_tests_only { use std::fs::File; use std::path::{Path, PathBuf}; - use mithril_common::test::TempDir; - mithril_common::define_test_logger!(); - pub fn get_test_directory(dir_name: &str) -> PathBuf { - TempDir::create("file_archiver", dir_name) - } - /// Create a file in the root directory. /// /// Returns the relative path to the created file based on the root directory. diff --git a/internal/mithril-file-archiver/tests/golden_master.rs b/internal/mithril-file-archiver/tests/golden_master.rs index fc1f693a117..56af53c94c6 100644 --- a/internal/mithril-file-archiver/tests/golden_master.rs +++ b/internal/mithril-file-archiver/tests/golden_master.rs @@ -17,8 +17,6 @@ use extensions::*; // ** These hashes define TAR_ZSTD_V1. Update them only for an intentional archive-format change. ** pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; -pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256: &str = - "fb55c9b984eab74e599466ea2fbd8af92f5e8fd52296439c9a29340f5f801bea"; pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = "cf6c2755fe389e40891c5cb6e4b08e0f01e09846e58f229d4dfe138adc7d1157"; pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = @@ -174,25 +172,6 @@ fn appender_file() { assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_FILE_SHA256); } -#[test] -fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("test", &test_dir), - AppenderDirAll::new(content), - ) - .unwrap(); - - assert_archive_not_empty(&archive); - assert_archive_matches_golden_sha256( - &archive, - TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_DIR_ALL_SHA256, - ); -} - #[test] fn appender_entries() { let test_dir = temp_dir_create!(); diff --git a/internal/mithril-file-archiver/tests/reproducibility.rs b/internal/mithril-file-archiver/tests/reproducibility.rs index 93f566d63bb..42afaacdb95 100644 --- a/internal/mithril-file-archiver/tests/reproducibility.rs +++ b/internal/mithril-file-archiver/tests/reproducibility.rs @@ -120,18 +120,6 @@ mod repeated_archiving_produces_byte_identical_archives { ); } - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); - - run_scenario( - test_dir, - AppenderDirAll::new(content.clone()), - AppenderDirAll::new(content), - ); - } - #[test] fn appender_entries() { let test_dir = temp_dir_create!(); @@ -282,13 +270,6 @@ mod source_metadata_does_not_affect_archive_bytes { }); } - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_modification_time); - run_scenario(test_dir, AppenderDirAll::new); - } - #[test] fn appender_entries() { let test_dir = temp_dir_create!(); @@ -335,14 +316,6 @@ mod source_metadata_does_not_affect_archive_bytes { }) } - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - setup_test_dirs(&test_dir, setup_permissions); - run_scenario(test_dir, AppenderDirAll::new); - } - - #[cfg(unix)] #[test] fn appender_entries() { let test_dir = temp_dir_create!(); @@ -386,35 +359,6 @@ mod source_base_directory_does_not_affect_archive { ); } - #[test] - fn appender_dir_all() { - let test_dir = temp_dir_create!(); - let source = helpers::create_dir(&test_dir, "source"); - let subdir_1 = helpers::create_dir(&source, "first"); - let subdir_2 = helpers::create_dir(&source, "second"); - - let content = test_data::create_test_dir(&subdir_1); - let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); - - let archive = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("reference", &test_dir), - AppenderDirAll::new(content), - ) - .unwrap(); - let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) - .archive( - helpers::archive_parameters("from_another_dir", &test_dir), - AppenderDirAll::new(same_content_in_other_dir), - ) - .unwrap(); - - helpers::assert_files_are_byte_identical( - archive.get_file_path(), - archive_with_same_content_but_from_another_dir.get_file_path(), - ); - } - #[test] fn appender_entries() { let test_dir = temp_dir_create!(); From 7f060e5eb298270a4cf76ae834537c8785ac3ae1 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:30:59 +0200 Subject: [PATCH 16/19] refactor(file_archiver): introduce `ArchiveEntry` abstraction to centralize tar entry management and improve consistency It will allow making the `ChainAppender` deterministic and commutative --- Cargo.lock | 1 - internal/mithril-file-archiver/Cargo.toml | 1 - .../mithril-file-archiver/src/appender.rs | 541 ++++++++++++------ 3 files changed, 354 insertions(+), 189 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 850cf296386..d3408e24b04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4644,7 +4644,6 @@ version = "0.1.0" dependencies = [ "anyhow", "hex", - "mithril-cardano-node-internal-database", "mithril-common", "serde", "serde_json", diff --git a/internal/mithril-file-archiver/Cargo.toml b/internal/mithril-file-archiver/Cargo.toml index a4e79c71230..f2a895402a5 100644 --- a/internal/mithril-file-archiver/Cargo.toml +++ b/internal/mithril-file-archiver/Cargo.toml @@ -19,7 +19,6 @@ zstd = { version = "=0.13.3", features = ["zstdmt"] } # Pinned to ensure archive [dev-dependencies] hex = { workspace = true } -mithril-cardano-node-internal-database = { path = "../cardano-node/mithril-cardano-node-internal-database" } sha2 = "0.10.9" slog-async = { workspace = true } slog-term = { workspace = true } diff --git a/internal/mithril-file-archiver/src/appender.rs b/internal/mithril-file-archiver/src/appender.rs index 5f7805df125..e563f6cd43c 100644 --- a/internal/mithril-file-archiver/src/appender.rs +++ b/internal/mithril-file-archiver/src/appender.rs @@ -1,16 +1,23 @@ //! Define how to append data to a [crate::FileArchiver] -use anyhow::{Context, anyhow}; -use serde::Serialize; +use std::cmp::Ordering; +use std::collections::BTreeSet; use std::fs::File; use std::io::Write; -use std::path::{Component, PathBuf}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, anyhow}; +use serde::Serialize; use mithril_common::StdResult; use crate::tools::file_size; const READ_WRITE_PERMISSION: u32 = 0o666; +/// Timestamp arbitrarily chosen to `2026-01-01 00:00:00 UTC` +/// IMPORTANT: Do NOT change it, else the `AppenderData` archives bytes would change. +const FIXED_MTIME_ATTRIBUTE_FOR_DATA: u64 = 1767225600; /// Define multiple ways to append content to a tar archive. pub trait TarAppender: Send { @@ -29,12 +36,213 @@ pub trait TarAppender: Send { } } -/// An appender that add one file. +/// Represents an object that can provide a list of entries to append to a tar archive. +pub trait ArchiveEntryProvider: Send { + /// Get the list of archive entries held by this provider. + fn collect_entries(&self) -> StdResult>; +} + +impl TarAppender for T { + fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { + for entry in self + .collect_entries() + .with_context(|| "Failed to collect entries from appender")? + { + entry.append_to_archive(tar)?; + } + Ok(()) + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + let mut size: u64 = 0; + for entry in self + .collect_entries() + .with_context(|| "Failed to collect entries from appender")? + { + size = size + .checked_add(entry.compute_uncompressed_data_size()?) + .with_context(|| "Failed to compute uncompressed data size")?; + } + Ok(size) + } +} + +/// Represents an entry to be added to a tar archive. +/// +/// Archive entries are identified and ordered solely by their normalized archive path. +/// Content and source fields do not participate in equality. +#[derive(Debug, Clone)] +pub enum ArchiveEntry { + /// A file entry. + File { + /// Path of the file in the archive. + location_in_archive: PathBuf, + /// Path to the source file on disk. + target_file: PathBuf, + }, + /// A directory entry. + Directory { + /// Path of the directory in the archive. + location_in_archive: PathBuf, + /// Path to the source directory on disk. + target_dir: PathBuf, + }, + /// Raw data entry. + Data { + /// Path of the data in the archive. + location_in_archive: PathBuf, + /// The data bytes. + data: Arc>, + }, +} + +impl ArchiveEntry { + /// Creates a directory entry. + pub fn from_dir(location_in_archive: PathBuf, target_dir: PathBuf) -> Self { + ArchiveEntry::Directory { + location_in_archive: Self::normalize_entry(location_in_archive), + target_dir, + } + } + + /// Creates a file entry. + pub fn from_file(location_in_archive: PathBuf, target_file: PathBuf) -> Self { + ArchiveEntry::File { + location_in_archive: Self::normalize_entry(location_in_archive), + target_file, + } + } + + /// Creates a data entry. + pub fn from_data(location_in_archive: PathBuf, data: Vec) -> Self { + ArchiveEntry::Data { + location_in_archive: Self::normalize_entry(location_in_archive), + data: Arc::new(data), + } + } + + /// Returns the location of this entry in the archive. + pub fn location_in_archive(&self) -> &Path { + match self { + ArchiveEntry::File { + location_in_archive, + .. + } => location_in_archive, + ArchiveEntry::Directory { + location_in_archive, + .. + } => location_in_archive, + ArchiveEntry::Data { + location_in_archive, + .. + } => location_in_archive, + } + } + + /// Appends this entry to the given tar archive builder. + pub fn append_to_archive(&self, tar: &mut tar::Builder) -> StdResult<()> { + match self { + ArchiveEntry::File { + location_in_archive, + target_file, + } => { + if !target_file.is_file() { + anyhow::bail!( + "File '{}' does not exist, can not add it to the archive at '{}'", + target_file.display(), + location_in_archive.display() + ); + } + + let mut file = File::open(target_file)?; + tar.append_file(location_in_archive, &mut file).with_context(|| { + format!( + "Can not add file: '{}' to the archive", + target_file.display() + ) + })?; + } + ArchiveEntry::Directory { + location_in_archive, + target_dir, + } => { + if !target_dir.is_dir() { + anyhow::bail!( + "Directory '{}' does not exist, can not add it to the archive at '{}'", + target_dir.display(), + location_in_archive.display() + ); + } + + tar.append_dir(location_in_archive, target_dir).with_context(|| { + format!( + "Can not add directory: '{}' to the archive", + location_in_archive.display() + ) + })?; + } + ArchiveEntry::Data { + location_in_archive, + data, + } => { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(READ_WRITE_PERMISSION); + header.set_mtime(FIXED_MTIME_ATTRIBUTE_FOR_DATA); + header.set_cksum(); + + tar.append_data(&mut header, location_in_archive, data.as_slice()) + .with_context(|| { + format!( + "Can not add file: '{}' to the archive", + location_in_archive.display() + ) + })?; + } + } + + Ok(()) + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + match self { + ArchiveEntry::File { target_file, .. } => file_size::compute_size_of_path(target_file), + ArchiveEntry::Directory { .. } => Ok(0), + ArchiveEntry::Data { data, .. } => Ok(data.len() as u64), + } + } + + fn normalize_entry(entry: PathBuf) -> PathBuf { + entry + .components() + .filter(|c| !matches!(c, Component::CurDir)) + .collect() + } +} + +impl Ord for ArchiveEntry { + fn cmp(&self, other: &Self) -> Ordering { + self.location_in_archive().cmp(other.location_in_archive()) + } +} + +impl PartialOrd for ArchiveEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for ArchiveEntry { + fn eq(&self, other: &Self) -> bool { + self.location_in_archive() == other.location_in_archive() + } +} + +impl Eq for ArchiveEntry {} + +/// An appender that adds one file. pub struct AppenderFile { - /// Location of the file in the archive. - location_in_archive: PathBuf, - /// Path to the file to add to the archive. - target_file: PathBuf, + entry: ArchiveEntry, } impl AppenderFile { @@ -58,37 +266,22 @@ impl AppenderFile { .to_owned(); Ok(Self { - location_in_archive: PathBuf::from(location_in_archive), - target_file, + entry: ArchiveEntry::from_file(PathBuf::from(location_in_archive), target_file), }) } } -impl TarAppender for AppenderFile { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - let mut file = File::open(&self.target_file) - .with_context(|| format!("Can not open file: '{}'", self.target_file.display()))?; - tar.append_file(&self.location_in_archive, &mut file) - .with_context(|| { - format!( - "Can not add file: '{}' to the archive", - self.target_file.display() - ) - })?; - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - file_size::compute_size_of_path(&self.target_file) +impl ArchiveEntryProvider for AppenderFile { + fn collect_entries(&self) -> StdResult> { + Ok(BTreeSet::from([self.entry.clone()])) } } -/// An appender that add a list of entries, files, or directories. +/// An appender that adds a list of entries, files, or directories. /// /// Directory contents are not added if not specified. pub struct AppenderEntries { - entries: Vec, - base_directory: PathBuf, + entries: BTreeSet, } impl AppenderEntries { @@ -102,78 +295,37 @@ impl AppenderEntries { return Err(anyhow!("The entries can not be empty")); } - Ok(Self { - entries: Self::normalize_entries(entries), - base_directory, - }) - } - - fn normalize_entries(entries: Vec) -> Vec { - let mut normalized: Vec = entries.into_iter().map(Self::normalize_entry).collect(); - normalized.sort(); - normalized - } - - fn normalize_entry(entry: PathBuf) -> PathBuf { - entry - .components() - .filter(|c| !matches!(c, Component::CurDir)) - .collect() - } -} + let mut archive_entries: BTreeSet = BTreeSet::new(); -impl TarAppender for AppenderEntries { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - for entry in &self.entries { - let entry_path = self.base_directory.join(entry); + for entry in entries { + let entry_path = base_directory.join(&entry); if entry_path.is_dir() { - tar.append_dir(entry, &entry_path).with_context(|| { - format!( - "Can not add directory: '{}' to the archive", - entry_path.display() - ) - })?; + archive_entries.insert(ArchiveEntry::from_dir(entry, entry_path)); } else if entry_path.is_file() { - let mut file = File::open(&entry_path)?; - tar.append_file(entry, &mut file).with_context(|| { - format!( - "Can not add file: '{}' to the archive", - entry_path.display() - ) - })?; + archive_entries.insert(ArchiveEntry::from_file(entry, entry_path)); } else { - return Err(anyhow!( - "The entry: '{}' is not valid", - entry_path.display() - )); + anyhow::bail!("The entry: '{}' is not valid", entry_path.display()); } } - Ok(()) + + Ok(Self { + entries: archive_entries, + }) } +} - fn compute_uncompressed_data_size(&self) -> StdResult { - let full_entries_path = self - .entries - .iter() - .map(|entry| self.base_directory.join(entry)) - .collect(); - file_size::compute_size(full_entries_path) +impl ArchiveEntryProvider for AppenderEntries { + fn collect_entries(&self) -> StdResult> { + Ok(self.entries.clone()) } } -/// An appender that add either [serde::Serialize] serializable data or raw bytes. +/// An appender that adds either [serde::Serialize] serializable data or raw bytes. pub struct AppenderData { - /// Location of the file in the archive where the data will be appended. - location_in_archive: PathBuf, - /// Byte array of the data to append. - bytes: Vec, + entry: ArchiveEntry, } impl AppenderData { - /// Timestamp arbitrarily chosen to `2026-01-01 00:00:00 UTC` - /// IMPORTANT: Do NOT change it, else the `AppenderData` archives bytes would change. - const FIXED_MTIME_ATTRIBUTE: u64 = 1767225600; - /// Create a new instance of `AppenderData` from an object that will be serialized to JSON. pub fn from_json( location_in_archive: PathBuf, @@ -192,37 +344,14 @@ impl AppenderData { /// Create a new instance of `AppenderData` from a byte array. pub fn from_raw_bytes(location_in_archive: PathBuf, bytes: Vec) -> Self { Self { - location_in_archive, - bytes, + entry: ArchiveEntry::from_data(location_in_archive, bytes), } } } -impl TarAppender for AppenderData { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - let mut header = tar::Header::new_gnu(); - header.set_size(self.bytes.len() as u64); - header.set_mode(READ_WRITE_PERMISSION); - header.set_mtime(Self::FIXED_MTIME_ATTRIBUTE); - header.set_cksum(); - - tar.append_data( - &mut header, - &self.location_in_archive, - self.bytes.as_slice(), - ) - .with_context(|| { - format!( - "Can not add file: '{}' to the archive", - self.location_in_archive.display() - ) - })?; - - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - Ok(self.bytes.len() as u64) +impl ArchiveEntryProvider for AppenderData { + fn collect_entries(&self) -> StdResult> { + Ok(BTreeSet::from([self.entry.clone()])) } } @@ -258,8 +387,6 @@ impl TarAppender for ChainAppender { #[cfg(test)] mod tests { - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; - use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR}; use mithril_common::entities::CompressionAlgorithm; use mithril_common::{assert_dir_eq, temp_dir_create}; @@ -269,14 +396,31 @@ mod tests { use super::*; - mod appender_entries { + mod archive_entry { use super::*; + #[test] + fn removes_trailing_separator_from_directory_component() { + assert_eq!( + PathBuf::from("foo"), + ArchiveEntry::normalize_entry(PathBuf::from("foo/")), + ); + } + + #[cfg(windows)] + #[test] + fn removes_windows_trailing_separator_from_directory_component() { + assert_eq!( + PathBuf::from("foo"), + ArchiveEntry::normalize_entry(PathBuf::from("foo\\")), + ); + } + #[test] fn removes_leading_current_directory_component() { assert_eq!( PathBuf::from("foo/bar.txt"), - AppenderEntries::normalize_entry(PathBuf::from("./foo/bar.txt")), + ArchiveEntry::normalize_entry(PathBuf::from("./foo/bar.txt")), ); } @@ -285,25 +429,31 @@ mod tests { fn removes_windows_leading_current_directory_component() { assert_eq!( PathBuf::from("foo").join("bar.txt"), - AppenderEntries::normalize_entry(PathBuf::from(r".\foo\bar.txt")), + ArchiveEntry::normalize_entry(PathBuf::from(r".\foo\bar.txt")), ); } + #[cfg(windows)] #[test] - fn normalizes_directory_spelling_and_sorts_entries() { - let appender = AppenderEntries::new( - vec![ - PathBuf::from("foo/bar.txt"), - PathBuf::from("file_2.txt"), - PathBuf::from("bar/"), - PathBuf::from("foo/"), - PathBuf::from("foo/pika/"), - PathBuf::from("foo/pika/chuu.txt"), - PathBuf::from("file_1.txt"), - ], - PathBuf::from("source"), - ) - .unwrap(); + fn forward_and_backward_separators_have_the_same_normalized_path() { + assert_eq!( + ArchiveEntry::normalize_entry(PathBuf::from("foo/bar.txt")), + ArchiveEntry::normalize_entry(PathBuf::from(r"foo\bar.txt")), + ); + } + + #[test] + fn entries_are_sorted_by_path() { + let mut entries = [ + ArchiveEntry::from_file(PathBuf::from("foo/bar.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("file_2.txt"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from("bar/"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from("foo/"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("foo/pika/"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("foo/pika/chuu.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("file_1.txt"), PathBuf::new()), + ]; + entries.sort(); assert_eq!( vec![ @@ -315,23 +465,23 @@ mod tests { PathBuf::from("foo/pika"), PathBuf::from("foo/pika/chuu.txt"), ], - appender.entries + entries + .into_iter() + .map(|entry| entry.location_in_archive().to_path_buf()) + .collect::>() ); } #[cfg(windows)] #[test] - fn normalizes_windows_separators_and_sorts_entries() { - let appender = AppenderEntries::new( - vec![ - PathBuf::from(r"foo\pika\chuu.txt"), - PathBuf::from(r"foo\bar.txt"), - PathBuf::from(r"bar\\"), - PathBuf::from(r"foo\\"), - ], - PathBuf::from("source"), - ) - .unwrap(); + fn entries_with_windows_path_are_sorted_by_path() { + let mut entries = [ + ArchiveEntry::from_file(PathBuf::from(r"foo\pika\chuu.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from(r"foo\bar.txt"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from(r"bar\\"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from(r"foo\\"), PathBuf::new()), + ]; + entries.sort(); assert_eq!( vec![ @@ -340,18 +490,48 @@ mod tests { PathBuf::from("foo").join("bar.txt"), PathBuf::from("foo").join("pika").join("chuu.txt"), ], - appender.entries + entries + .into_iter() + .map(|entry| entry.location_in_archive().to_path_buf()) + .collect::>() ); } - #[cfg(windows)] #[test] - fn forward_and_backward_separators_have_the_same_normalized_path() { + fn appending_fails_if_source_file_does_not_exist() { + let entry = + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("not_exist.txt")); + let mut tar = tar::Builder::new(Vec::new()); + + let res = entry.append_to_archive(&mut tar); + assert!(res.is_err()); + } + + #[test] + fn appending_fails_if_source_dir_does_not_exist() { + let entry = ArchiveEntry::from_dir(PathBuf::from("foo/"), PathBuf::from("not_exist/")); + let mut tar = tar::Builder::new(Vec::new()); + + let res = entry.append_to_archive(&mut tar); + assert!(res.is_err()); + } + + #[test] + fn entries_with_the_same_archive_path_but_different_content_are_considered_equal() { + assert_eq!( + ArchiveEntry::from_data(PathBuf::from("foo.txt"), vec![0, 1, 2]), + ArchiveEntry::from_data(PathBuf::from("foo.txt"), vec![3, 4, 5]), + ); + assert_eq!( - AppenderEntries::normalize_entry(PathBuf::from("foo/bar.txt")), - AppenderEntries::normalize_entry(PathBuf::from(r"foo\bar.txt")), + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("file.txt")), + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("other.txt")) ); } + } + + mod appender_entries { + use super::*; #[test] fn create_archive_only_for_specified_directories_and_files() { @@ -405,24 +585,14 @@ mod tests { } #[test] - fn return_error_when_appending_file_or_directory_that_does_not_exist() { + fn creation_fails_when_entry_does_not_exist() { let test_dir = temp_dir_create!(); - let target_archive = test_dir.join("whatever.tar.zst"); - let source = test_dir.join(create_dir(&test_dir, "source")); + let res = AppenderEntries::new(vec![PathBuf::from("not_exist")], test_dir); - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - - file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - AppenderEntries::new(vec![PathBuf::from("not_exist")], source).unwrap(), - ) - .expect_err("AppenderEntries should return error when file or directory not exist"); - assert!(!target_archive.exists()); + assert!( + res.is_err(), + "AppenderEntries should return error when file or directory not exist" + ); } #[test] @@ -473,36 +643,33 @@ mod tests { #[test] fn compute_uncompressed_size_of_its_paths() { - let test_dir = "compute_uncompressed_size_of_its_paths"; - - let immutable_trio_file_size = 777; - let ledger_file_size = 6666; - let volatile_file_size = 99; + fn create_file_with_len(path: &Path, len: u64) { + let file = File::create(path) + .unwrap_or_else(|_| panic!("failed to create '{}'", path.display())); + file.set_len(len).unwrap(); + } - let cardano_db = DummyCardanoDbBuilder::new(test_dir) - .with_immutables(&[1, 2, 3]) - .set_immutable_trio_file_size(immutable_trio_file_size) - .with_legacy_ledger_snapshots(&[437, 537, 637, 737]) - .set_ledger_file_size(ledger_file_size) - .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"]) - .set_volatile_file_size(volatile_file_size) - .build(); + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + let subdir = source.join(create_dir(&source, "subdir")); + create_file_with_len(&source.join("file_1"), 100); + create_file_with_len(&source.join("file_2"), 200); + create_file_with_len(&subdir.join("file_3"), 300); + create_file_with_len(&subdir.join("file_not_to_include"), 400); let appender_entries = AppenderEntries::new( vec![ - PathBuf::from(IMMUTABLE_DIR), - PathBuf::from(LEDGER_DIR).join("437"), - PathBuf::from(LEDGER_DIR).join("537"), - PathBuf::from(VOLATILE_DIR).join("blocks-0.dat"), + PathBuf::from("file_1"), + PathBuf::from("file_2"), + PathBuf::from("subdir/"), + PathBuf::from("subdir/file_3"), ], - cardano_db.get_dir().clone(), + source, ) .unwrap(); let entries_size = appender_entries.compute_uncompressed_data_size().unwrap(); - let expected_total_size = - (immutable_trio_file_size * 3) + (2 * ledger_file_size) + volatile_file_size; - assert_eq!(expected_total_size, entries_size); + assert_eq!(600, entries_size); } } @@ -636,7 +803,7 @@ mod tests { appended_entry.header().mode().unwrap() ); let mtime = appended_entry.header().mtime().unwrap(); - assert_eq!(AppenderData::FIXED_MTIME_ATTRIBUTE, mtime); + assert_eq!(FIXED_MTIME_ATTRIBUTE_FOR_DATA, mtime); } #[test] From 19ed00dc16a1f91771ca57f2bcb053d340b9a17d Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:22:15 +0200 Subject: [PATCH 17/19] refactor(file_archiver): update `ChainAppender` logic to only work on `ArchiveEntryProvider`, and add tests for entry commutativity --- .../mithril-file-archiver/src/appender.rs | 66 +++++----- .../tests/extensions/helpers.rs | 44 +++++-- .../tests/reproducibility.rs | 121 ++++++++++++++++++ .../compressed_archive_snapshotter.rs | 2 +- 4 files changed, 190 insertions(+), 43 deletions(-) diff --git a/internal/mithril-file-archiver/src/appender.rs b/internal/mithril-file-archiver/src/appender.rs index e563f6cd43c..cc75fb5955c 100644 --- a/internal/mithril-file-archiver/src/appender.rs +++ b/internal/mithril-file-archiver/src/appender.rs @@ -1,4 +1,4 @@ -//! Define how to append data to a [crate::FileArchiver] +//! Define how to append data to a [FileArchiver][crate::FileArchiver] use std::cmp::Ordering; use std::collections::BTreeSet; @@ -26,20 +26,24 @@ pub trait TarAppender: Send { /// Computes the total uncompressed size of the data that will be added to the archive. fn compute_uncompressed_data_size(&self) -> StdResult; - - /// Chains this appender with another, combining their contents into a single archive. - fn chain(self, appender_right: A2) -> ChainAppender - where - Self: Sized, - { - ChainAppender::new(self, appender_right) - } } /// Represents an object that can provide a list of entries to append to a tar archive. pub trait ArchiveEntryProvider: Send { /// Get the list of archive entries held by this provider. fn collect_entries(&self) -> StdResult>; + + /// Chains this provider with another, combining their contents into a single archive. + /// + /// - when paths are overlapping, the rightmost provider takes precedence. + /// - if there are no overlapping paths, chaining is commutative `(A | B) = (B | A)`. + /// - chaining is always associative: `((A | B) | C) = (A | (B | C))`, even when paths are overlapping. + fn chain(self, provider_right: P2) -> ChainAppender + where + Self: Sized, + { + ChainAppender::new(self, provider_right) + } } impl TarAppender for T { @@ -355,33 +359,37 @@ impl ArchiveEntryProvider for AppenderData { } } -/// Chain multiple `TarAppender` instances together. +/// Combines multiple archive entry providers into one deterministic entry collection. +/// +/// - when paths are overlapping, the rightmost provider takes precedence. +/// - if there are no overlapping paths, chaining is commutative `(A | B) = (B | A)`. +/// - chaining is always associative: `((A | B) | C) = (A | (B | C))`, even when paths are overlapping. pub struct ChainAppender { - appender_left: L, - appender_right: R, + provider_left: L, + provider_right: R, } -impl ChainAppender { +impl ChainAppender { /// [ChainAppender] factory - pub fn new(appender_left: L, appender_right: R) -> Self { + pub fn new(provider_left: L, provider_right: R) -> Self { Self { - appender_left, - appender_right, + provider_left, + provider_right, } } -} -impl TarAppender for ChainAppender { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - self.appender_left.append(tar)?; - self.appender_right.append(tar) + fn merge_entries_with_right_precedence(&self) -> StdResult> { + let mut entries = self.provider_right.collect_entries()?; + entries.extend(self.provider_left.collect_entries()?); + Ok(entries) } +} - fn compute_uncompressed_data_size(&self) -> StdResult { - // Size is aggregated even if the data is overwritten by the right appender because we - // can't know if there is an overlap or not - Ok(self.appender_left.compute_uncompressed_data_size()? - + self.appender_right.compute_uncompressed_data_size()?) +impl ArchiveEntryProvider + for ChainAppender +{ + fn collect_entries(&self) -> StdResult> { + self.merge_entries_with_right_precedence() } } @@ -901,16 +909,14 @@ mod tests { } #[test] - fn compute_uncompressed_size_cant_discriminate_overlaps_and_return_aggregated_appenders_sizes() - { + fn compute_overlapping_uncompressed_size() { let overlapping_path = PathBuf::from("whatever.json"); let left_appender = AppenderData::from_json(overlapping_path.clone(), &"overwritten data").unwrap(); let right_appender = AppenderData::from_json(overlapping_path.clone(), &"final data").unwrap(); - let expected_size = left_appender.compute_uncompressed_data_size().unwrap() - + right_appender.compute_uncompressed_data_size().unwrap(); + let expected_size = right_appender.compute_uncompressed_data_size().unwrap(); let chain_appender = left_appender.chain(right_appender); let size = chain_appender.compute_uncompressed_data_size().unwrap(); diff --git a/internal/mithril-file-archiver/tests/extensions/helpers.rs b/internal/mithril-file-archiver/tests/extensions/helpers.rs index 8ad19e8cec2..ee4b3116de6 100644 --- a/internal/mithril-file-archiver/tests/extensions/helpers.rs +++ b/internal/mithril-file-archiver/tests/extensions/helpers.rs @@ -14,18 +14,8 @@ use mithril_file_archiver::{ArchiveParameters, FileArchiver}; #[track_caller] pub fn assert_files_are_byte_identical, A: AsRef>(expected: E, actual: A) { let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); - let expected_bytes = std::fs::read(expected_path).unwrap_or_else(|error| { - panic!( - "Could not read expected file '{}': {error}", - expected_path.display() - ) - }); - let actual_bytes = std::fs::read(actual_path).unwrap_or_else(|error| { - panic!( - "Could not read actual file '{}': {error}", - actual_path.display() - ) - }); + let expected_bytes = read_file_bytes(expected_path); + let actual_bytes = read_file_bytes(actual_path); if expected_bytes != actual_bytes { let expected_hash = hex::encode(Sha256::digest(&expected_bytes)); @@ -45,6 +35,36 @@ pub fn assert_files_are_byte_identical, A: AsRef>(expected: } } +/// Assert that two files are different. +/// +/// SHA-256 hashes and file sizes are reported when they are equal. +#[track_caller] +pub fn assert_files_are_byte_different, A: AsRef>(expected: E, actual: A) { + let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); + let expected_bytes = read_file_bytes(expected_path); + let actual_bytes = read_file_bytes(actual_path); + + if expected_bytes == actual_bytes { + let hash = hex::encode(Sha256::digest(&expected_bytes)); + + panic!( + "Files are byte-identical:\n\ + expected: '{}'\n\ + actual: '{}'\n\ + bytes: {} bytes, SHA-256: {}", + expected_path.display(), + actual_path.display(), + expected_bytes.len(), + hash, + ); + } +} + +fn read_file_bytes(path: &Path) -> Vec { + std::fs::read(path) + .unwrap_or_else(|error| panic!("Could not read file '{}': {error}", path.display())) +} + /// **IMPORTANT** Default zstandard compression parameters are used. pub fn file_archiver(work_dir: &Path) -> FileArchiver { FileArchiver::new_with_default_parameters( diff --git a/internal/mithril-file-archiver/tests/reproducibility.rs b/internal/mithril-file-archiver/tests/reproducibility.rs index 42afaacdb95..bdc66d67ad8 100644 --- a/internal/mithril-file-archiver/tests/reproducibility.rs +++ b/internal/mithril-file-archiver/tests/reproducibility.rs @@ -476,3 +476,124 @@ mod appender_entry_specifics { } } } + +mod chain_specifics { + use super::*; + + #[test] + fn chaining_non_overlapping_appenders_is_commutative() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let file_content = test_data::create_test_txt(&source); + let data_content = test_data::TEST_BYTES; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderFile::append_at_archive_root(file_content.clone()) + .unwrap() + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + data_content.to_vec(), + )), + ) + .unwrap(); + + let archive_chained_in_reverse = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("archive_chained_in_reverse", &test_dir), + AppenderData::from_raw_bytes(PathBuf::from("data.bytes"), data_content.to_vec()) + .chain(AppenderFile::append_at_archive_root(file_content.clone()).unwrap()), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_chained_in_reverse.get_file_path(), + ); + } + + #[test] + // Commutativity of chaining appenders is broken if they have overlapping paths as only the path + // from the rightmost appender is used. + fn chaining_overlapping_appenders_is_not_commutative() { + let test_dir = temp_dir_create!(); + let path_in_archive = PathBuf::from("data.bytes"); + let first_data_content = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let second_data_content = [11, 12, 13, 14, 15, 16, 17, 18, 18, 19]; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderData::from_raw_bytes(path_in_archive.clone(), first_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + second_data_content.to_vec(), + )), + ) + .unwrap(); + + let archive_chained_in_reverse = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("archive_chained_in_reverse", &test_dir), + AppenderData::from_raw_bytes(path_in_archive.clone(), second_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + first_data_content.to_vec(), + )), + ) + .unwrap(); + + helpers::assert_files_are_byte_different( + reference_archive.get_file_path(), + archive_chained_in_reverse.get_file_path(), + ); + } + + #[test] + fn chaining_non_overlapping_appenders_is_associative() { + let test_dir = temp_dir_create!(); + let a_data_content = [1, 2, 3, 4, 5]; + let b_data_content = [6, 7, 8, 9, 10]; + let c_data_content = [11, 12, 13, 14, 15]; + + let left_grouped = + AppenderData::from_raw_bytes(PathBuf::from("c.txt"), c_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("a.txt"), + a_data_content.to_vec(), + )) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("b.txt"), + b_data_content.to_vec(), + )); + + let right_grouped = + AppenderData::from_raw_bytes(PathBuf::from("c.txt"), c_data_content.to_vec()).chain( + AppenderData::from_raw_bytes(PathBuf::from("a.txt"), a_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("b.txt"), + b_data_content.to_vec(), + )), + ); + + let left_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("left-grouped", &test_dir), + left_grouped, + ) + .unwrap(); + + let right_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("right-grouped", &test_dir), + right_grouped, + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + left_archive.get_file_path(), + right_archive.get_file_path(), + ); + } +} diff --git a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs index 333e8cb5d67..8327477c949 100644 --- a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs +++ b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs @@ -15,7 +15,7 @@ use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; use mithril_common::logging::LoggerExtensions; use mithril_file_archiver::{ ArchiveParameters, FileArchive, FileArchiver, - appender::{AppenderData, AppenderEntries, TarAppender}, + appender::{AppenderData, AppenderEntries, ArchiveEntryProvider, TarAppender}, tools::file_size, }; From 0e059bbeacb71b1f91e7e27b6ce5305efd59e990 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:46:41 +0200 Subject: [PATCH 18/19] docs(file_archiver): document byte stability guarantees and archive-format invariants --- internal/mithril-file-archiver/README.md | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/internal/mithril-file-archiver/README.md b/internal/mithril-file-archiver/README.md index 67eb54cf7ec..22702997335 100644 --- a/internal/mithril-file-archiver/README.md +++ b/internal/mithril-file-archiver/README.md @@ -8,3 +8,32 @@ Produced archives are byte stable across systems as long as the following invari - The version of the zstandard compression library - The parameters of the zstandard compression + +## Byte stability guarantees + +Given identical archive entry paths and contents, `FileArchiver` produces byte-identical `.tar.zst` archives across +runs and supported host systems. + +Archive bytes are unaffected by: + +- The source base directory +- File creation and modification times +- File permissions +- The order in which entries are supplied +- Equivalent path spellings, such as `foo/` and `foo`, or `./foo.txt` and `foo.txt` +- The order in which non-overlapping appenders are chained + +Entries are normalized and sorted by their archive paths. When chained appenders provide the same archive path, the +rightmost appender takes precedence. + +This guarantee requires the following archive-format invariants to remain unchanged: + +- Archive entry paths and contents +- The versions and behavior of the TAR and Zstandard libraries +- The Zstandard compression parameters, including the compression level and number of workers +- TAR header generation and metadata normalization +- JSON serialization output when using `AppenderData::from_json` + +Changing one of these invariants can change the resulting bytes and must be treated as an intentional archive-format +change. +Such a change requires bumping the archive-format version and updating the golden hashes that pin the expected output. From aab251e39560dcdca408c97cfe54eacff92c5148 Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:41:19 +0200 Subject: [PATCH 19/19] chore: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a169509601f..30a5c4e9570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ As a minor extension, we have adopted a slightly different versioning convention | mithril-signer | `-` | | mithril-stm | `-` | +- Reworked the Mithril aggregator file archiver to output byte stable archives systems. + - **REMOVED** support for `Gzip` compression/decompression in the Mithril aggregator and client: - The aggregator no longer produces or supports `Gzip` compression for snapshot-related artifacts: immutable files and ancillaries. - The Mithril client no longer supports `Gzip` decompression when downloading snapshot artifacts.