refactor(aggregator): deterministic snapshots archives

.github/workflows/ci.yml

@@ -872,8 +872,8 @@ jobs:
           cargo doc --no-deps --lib -p mithril-stm -p mithril-common \
                     -p mithril-cardano-node-chain -p mithril-cardano-node-internal-database \
                     -p mithril-aggregator-client -p mithril-aggregator-discovery -p mithril-build-script -p mithril-cli-helper \
-                    -p mithril-dmq -p mithril-doc -p mithril-doc-derive \
-                    -p mithril-era -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \
+                    -p mithril-dmq -p mithril-doc -p mithril-doc-derive -p mithril-era -p mithril-file-archiver \
+                    -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \
                     -p mithril-ticker -p mithril-signed-entity-lock -p mithril-signed-entity-preloader \
                     -p mithril-aggregator -p mithril-signer -p mithril-client -p mithril-client-cli \
                     -p mithril-api-spec -p mithril-test-http-server \

CHANGELOG.md

@@ -23,6 +23,8 @@ As a minor extension, we have adopted a slightly different versioning convention
 | mithril-signer      | `-`     |
 | mithril-stm         | `-`     |
 
+- Reworked the Mithril aggregator file archiver to output byte stable archives systems.
+
 - **REMOVED** support for `Gzip` compression/decompression in the Mithril aggregator and client:
   - The aggregator no longer produces or supports `Gzip` compression for snapshot-related artifacts: immutable files and ancillaries.
   - The Mithril client no longer supports `Gzip` decompression when downloading snapshot artifacts.

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "internal/mithril-doc",
     "internal/mithril-doc-derive",
     "internal/mithril-era",
+    "internal/mithril-file-archiver",
     "internal/mithril-merkle-tree",
     "internal/mithril-metric",
     "internal/mithril-persistence",

README.md

@@ -100,6 +100,8 @@ This repository consists of the following parts:
 
   - [**Mithril era**](./internal/mithril-era): mechanisms to read and check **Mithril Era markers**, used by **Mithril network** nodes.
 
+  - [**Mithril file archiver**](./internal/mithril-file-archiver): An API that generates byte stables `tar.zst` archives, used by the **Mithril aggregator**.
+
   - [**Mithril metric**](./internal/mithril-metric): materials to expose **metrics** in **Mithril network** nodes.
 
   - [**Mithril persistence**](./internal/mithril-persistence): the **persistence** library that is used by **Mithril network** nodes.

internal/mithril-file-archiver/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "mithril-file-archiver"
+version = "0.1.0"
+authors.workspace = true
+documentation.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+mithril-common = { path = "../../mithril-common" }
+serde = { workspace = true }
+serde_json = { workspace = true }
+slog = { workspace = true }
+tar = "=0.4.46" # Pinned to ensure archive bytes stability across builds
+zstd = { version = "=0.13.3", features = ["zstdmt"] } # Pinned to ensure archive bytes stability across builds
+
+[dev-dependencies]
+hex = { workspace = true }
+sha2 = "0.10.9"
+slog-async = { workspace = true }
+slog-term = { workspace = true }

internal/mithril-file-archiver/Makefile

@@ -0,0 +1,19 @@
+.PHONY: all build test check doc
+
+CARGO = cargo
+
+all: test build
+
+build:
+	${CARGO} build --release
+
+test:
+	${CARGO} test
+
+check:
+	${CARGO} check --release --all-features --all-targets
+	${CARGO} clippy --release --all-features --all-targets
+	${CARGO} fmt --check
+
+doc:
+	${CARGO} doc --no-deps --open

internal/mithril-file-archiver/README.md

@@ -0,0 +1,39 @@
+# Mithril-file-archiver
+
+**This is a work in progress** 🛠
+
+An API to generate tar archives from files, directories, or serializable data (leveraging serde).
+
+Produced archives are byte stable across systems as long as the following invariants do not change:
+
+- The version of the zstandard compression library
+- The parameters of the zstandard compression
+
+## Byte stability guarantees
+
+Given identical archive entry paths and contents, `FileArchiver` produces byte-identical `.tar.zst` archives across
+runs and supported host systems.
+
+Archive bytes are unaffected by:
+
+- The source base directory
+- File creation and modification times
+- File permissions
+- The order in which entries are supplied
+- Equivalent path spellings, such as `foo/` and `foo`, or `./foo.txt` and `foo.txt`
+- The order in which non-overlapping appenders are chained
+
+Entries are normalized and sorted by their archive paths. When chained appenders provide the same archive path, the
+rightmost appender takes precedence.
+
+This guarantee requires the following archive-format invariants to remain unchanged:
+
+- Archive entry paths and contents
+- The versions and behavior of the TAR and Zstandard libraries
+- The Zstandard compression parameters, including the compression level and number of workers
+- TAR header generation and metadata normalization
+- JSON serialization output when using `AppenderData::from_json`
+
+Changing one of these invariants can change the resulting bytes and must be treated as an intentional archive-format
+change.
+Such a change requires bumping the archive-format version and updating the golden hashes that pin the expected output.

mithril-aggregator/src/tools/file_archiver/api.rs → internal/mithril-file-archiver/src/api.rs

@@ -6,19 +6,17 @@ use std::{
     io::{Read, Seek, SeekFrom},
     path::{Path, PathBuf},
 };
-use tar::{Archive, Entry, EntryType};
+use tar::{Archive, Entry, EntryType, HeaderMode};
 use zstd::{Decoder, Encoder};
 
 use mithril_common::StdResult;
 use mithril_common::entities::CompressionAlgorithm;
 use mithril_common::logging::LoggerExtensions;
 
-use crate::ZstandardCompressionParameters;
+use crate::appender::TarAppender;
+use crate::entities::{ArchiveParameters, FileArchive, ZstandardCompressionParameters};
 use crate::tools::file_size;
 
-use super::appender::TarAppender;
-use super::{ArchiveParameters, FileArchive};
-
 /// Tool to archive files and directories.
 pub struct FileArchiver {
     zstandard_compression_parameter: ZstandardCompressionParameters,
@@ -41,14 +39,18 @@ impl FileArchiver {
         }
     }
 
-    #[cfg(test)]
-    pub fn new_for_test(verification_temp_dir: PathBuf) -> Self {
-        use crate::test::TestLogger;
-        Self {
-            zstandard_compression_parameter: ZstandardCompressionParameters::default(),
+    /// Constructs a new `FileArchiver` that uses the default compression parameters.
+    pub fn new_with_default_parameters(verification_temp_dir: PathBuf, logger: Logger) -> Self {
+        Self::new(
+            ZstandardCompressionParameters::default(),
             verification_temp_dir,
-            logger: TestLogger::stdout(),
-        }
+            logger,
+        )
+    }
+
+    #[cfg(test)]
+    pub(crate) fn new_for_test(verification_temp_dir: PathBuf) -> Self {
+        Self::new_with_default_parameters(verification_temp_dir, crate::test::TestLogger::stdout())
     }
 
     /// Archive the content of a directory.
@@ -152,6 +154,7 @@ impl FileArchiver {
                 enc.multithread(self.zstandard_compression_parameter.number_of_workers)
                     .with_context(|| "ZstandardEncoder can not set the number of workers")?;
                 let mut tar = tar::Builder::new(enc);
+                Self::configure_tar_builder(&mut tar);
 
                 appender
                     .append(&mut tar)
@@ -272,16 +275,25 @@ impl FileArchiver {
 
         Ok(())
     }
+
+    fn configure_tar_builder<W: std::io::Write>(builder: &mut tar::Builder<W>) {
+        builder.mode(HeaderMode::Deterministic);
+        builder.follow_symlinks(false);
+        // disable sparse files, as their support is not uniform across platforms and the size
+        // difference won't matter with zstandard compression
+        builder.sparse(false);
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use std::fs::File;
 
+    use mithril_common::temp_dir_create;
     use mithril_common::test::assert_equivalent;
 
-    use crate::tools::file_archiver::appender::{AppenderDirAll, AppenderFile};
-    use crate::tools::file_archiver::test_tools::*;
+    use crate::appender::{AppenderEntries, AppenderFile};
+    use crate::test::{FileArchiveTestExtension, create_dir, create_file, double::FailAppender};
 
     use super::*;
 
@@ -294,18 +306,17 @@ mod tests {
 
     #[test]
     fn should_create_a_valid_archive_with_zstandard_compression() {
-        let test_dir =
-            get_test_directory("should_create_a_valid_archive_with_zstandard_compression");
+        let test_dir = temp_dir_create!();
         let target_archive = test_dir.join("archive.tar.zst");
-        let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory"));
-        create_file(&archived_directory, "file_to_archive.txt");
+        let source_dir = test_dir.join(create_dir(&test_dir, "source"));
+        let archived_file = source_dir.join(create_file(&source_dir, "file_to_archive.txt"));
 
         let file_archiver = FileArchiver::new_for_test(test_dir.join("verification"));
 
         let archive = file_archiver
             .create_archive(
                 &target_archive,
-                AppenderDirAll::new(archived_directory),
+                AppenderFile::append_at_archive_root(archived_file).unwrap(),
                 CompressionAlgorithm::Zstandard,
             )
             .expect("create_archive should not fail");
@@ -316,10 +327,7 @@ mod tests {
 
     #[test]
     fn should_delete_tmp_file_in_target_directory_if_archiving_fail() {
-        let test_dir =
-            get_test_directory("should_delete_tmp_file_in_target_directory_if_archiving_fail");
-        // Note: the archived directory does not exist in order to make the archive process fail
-        let archived_directory = test_dir.join("db");
+        let test_dir = temp_dir_create!();
 
         let file_archiver = FileArchiver::new_for_test(test_dir.join("verification"));
 
@@ -332,7 +340,7 @@ mod tests {
             compression_algorithm: CompressionAlgorithm::Zstandard,
         };
         let _ = file_archiver
-            .archive(archive_params, AppenderDirAll::new(archived_directory))
+            .archive(archive_params, FailAppender)
             .expect_err("FileArchiver::archive should fail if the target path doesn't exist.");
 
         let remaining_files: Vec<String> = list_remaining_files(&test_dir);
@@ -341,11 +349,7 @@ mod tests {
 
     #[test]
     fn should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail() {
-        let test_dir = get_test_directory(
-            "should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail",
-        );
-        // Note: the archived directory does not exist in order to make the archive process fail
-        let archived_directory = test_dir.join("db");
+        let test_dir = temp_dir_create!();
 
         let file_archiver = FileArchiver::new_for_test(test_dir.join("verification"));
 
@@ -361,7 +365,7 @@ mod tests {
             compression_algorithm: CompressionAlgorithm::Zstandard,
         };
         let _ = file_archiver
-            .archive(archive_params, AppenderDirAll::new(archived_directory))
+            .archive(archive_params, FailAppender)
             .expect_err("FileArchiver::archive should fail if the db is empty.");
         let remaining_files: Vec<String> = list_remaining_files(&test_dir);
 
@@ -373,11 +377,9 @@ mod tests {
 
     #[test]
     fn overwrite_already_existing_archive_when_archiving_succeed() {
-        let test_dir =
-            get_test_directory("overwrite_already_existing_archive_when_archiving_succeed");
-        let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory"));
-
-        create_file(&archived_directory, "file_to_archive.txt");
+        let test_dir = temp_dir_create!();
+        let source = test_dir.join(create_dir(&test_dir, "source"));
+        let file_to_archive = create_file(&source, "file_to_archive.txt");
 
         let file_archiver = FileArchiver::new_for_test(test_dir.join("verification"));
 
@@ -389,15 +391,22 @@ mod tests {
         let first_archive = file_archiver
             .archive(
                 archive_params.clone(),
-                AppenderDirAll::new(archived_directory.clone()),
+                AppenderEntries::new(vec![file_to_archive.clone()], source.clone()).unwrap(),
             )
             .unwrap();
         let first_archive_size = first_archive.get_archive_size();
 
-        create_file(&archived_directory, "another_file_to_archive.txt");
+        let another_file_to_archive = create_file(&source, "another_file_to_archive.txt");
 
         let second_archive = file_archiver
-            .archive(archive_params, AppenderDirAll::new(archived_directory))
+            .archive(
+                archive_params,
+                AppenderEntries::new(
+                    vec![file_to_archive, another_file_to_archive],
+                    source.clone(),
+                )
+                .unwrap(),
+            )
             .unwrap();
         let second_archive_size = second_archive.get_archive_size();
 
@@ -409,7 +418,7 @@ mod tests {
 
     #[test]
     fn compute_size_of_uncompressed_data_and_archive() {
-        let test_dir = get_test_directory("compute_size_of_uncompressed_data_and_archive");
+        let test_dir = temp_dir_create!();
 
         let file_path = test_dir.join("file.txt");
         let file = File::create(&file_path).unwrap();