diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53c910f7c3b..626e448c8a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -872,8 +872,8 @@ jobs: cargo doc --no-deps --lib -p mithril-stm -p mithril-common \ -p mithril-cardano-node-chain -p mithril-cardano-node-internal-database \ -p mithril-aggregator-client -p mithril-aggregator-discovery -p mithril-build-script -p mithril-cli-helper \ - -p mithril-dmq -p mithril-doc -p mithril-doc-derive \ - -p mithril-era -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ + -p mithril-dmq -p mithril-doc -p mithril-doc-derive -p mithril-era -p mithril-file-archiver \ + -p mithril-merkle-tree -p mithril-metric -p mithril-persistence -p mithril-resource-pool \ -p mithril-ticker -p mithril-signed-entity-lock -p mithril-signed-entity-preloader \ -p mithril-aggregator -p mithril-signer -p mithril-client -p mithril-client-cli \ -p mithril-api-spec -p mithril-test-http-server \ diff --git a/CHANGELOG.md b/CHANGELOG.md index a169509601f..30a5c4e9570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ As a minor extension, we have adopted a slightly different versioning convention | mithril-signer | `-` | | mithril-stm | `-` | +- Reworked the Mithril aggregator file archiver to output byte stable archives systems. + - **REMOVED** support for `Gzip` compression/decompression in the Mithril aggregator and client: - The aggregator no longer produces or supports `Gzip` compression for snapshot-related artifacts: immutable files and ancillaries. - The Mithril client no longer supports `Gzip` decompression when downloading snapshot artifacts. diff --git a/Cargo.lock b/Cargo.lock index 37aaa19a061..d3408e24b04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4241,13 +4241,13 @@ dependencies = [ "mithril-dmq", "mithril-doc", "mithril-era", + "mithril-file-archiver", "mithril-metric", "mithril-persistence", "mithril-protocol-config", "mithril-resource-pool", "mithril-signed-entity-lock", "mithril-signed-entity-preloader", - "mithril-test-http-server", "mithril-ticker", "mockall", "paste", @@ -4265,15 +4265,12 @@ dependencies = [ "slog-scope", "slog-term", "sqlite", - "tar", - "tempfile", "thiserror 2.0.18", "tikv-jemallocator", "tokio", "tokio-util", "uuid", "warp", - "zstd", ] [[package]] @@ -4641,6 +4638,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "mithril-file-archiver" +version = "0.1.0" +dependencies = [ + "anyhow", + "hex", + "mithril-common", + "serde", + "serde_json", + "sha2 0.10.9", + "slog", + "slog-async", + "slog-term", + "tar", + "zstd", +] + [[package]] name = "mithril-merkle-tree" version = "0.1.4" diff --git a/Cargo.toml b/Cargo.toml index b9a66922369..3f3c1580227 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "internal/mithril-doc", "internal/mithril-doc-derive", "internal/mithril-era", + "internal/mithril-file-archiver", "internal/mithril-merkle-tree", "internal/mithril-metric", "internal/mithril-persistence", diff --git a/README.md b/README.md index 50b2f3cf138..04ffacb0f96 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ This repository consists of the following parts: - [**Mithril era**](./internal/mithril-era): mechanisms to read and check **Mithril Era markers**, used by **Mithril network** nodes. + - [**Mithril file archiver**](./internal/mithril-file-archiver): An API that generates byte stables `tar.zst` archives, used by the **Mithril aggregator**. + - [**Mithril metric**](./internal/mithril-metric): materials to expose **metrics** in **Mithril network** nodes. - [**Mithril persistence**](./internal/mithril-persistence): the **persistence** library that is used by **Mithril network** nodes. diff --git a/internal/mithril-file-archiver/Cargo.toml b/internal/mithril-file-archiver/Cargo.toml new file mode 100644 index 00000000000..f2a895402a5 --- /dev/null +++ b/internal/mithril-file-archiver/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "mithril-file-archiver" +version = "0.1.0" +authors.workspace = true +documentation.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +anyhow = { workspace = true } +mithril-common = { path = "../../mithril-common" } +serde = { workspace = true } +serde_json = { workspace = true } +slog = { workspace = true } +tar = "=0.4.46" # Pinned to ensure archive bytes stability across builds +zstd = { version = "=0.13.3", features = ["zstdmt"] } # Pinned to ensure archive bytes stability across builds + +[dev-dependencies] +hex = { workspace = true } +sha2 = "0.10.9" +slog-async = { workspace = true } +slog-term = { workspace = true } diff --git a/internal/mithril-file-archiver/Makefile b/internal/mithril-file-archiver/Makefile new file mode 100644 index 00000000000..d66d6d9fefc --- /dev/null +++ b/internal/mithril-file-archiver/Makefile @@ -0,0 +1,19 @@ +.PHONY: all build test check doc + +CARGO = cargo + +all: test build + +build: + ${CARGO} build --release + +test: + ${CARGO} test + +check: + ${CARGO} check --release --all-features --all-targets + ${CARGO} clippy --release --all-features --all-targets + ${CARGO} fmt --check + +doc: + ${CARGO} doc --no-deps --open diff --git a/internal/mithril-file-archiver/README.md b/internal/mithril-file-archiver/README.md new file mode 100644 index 00000000000..22702997335 --- /dev/null +++ b/internal/mithril-file-archiver/README.md @@ -0,0 +1,39 @@ +# Mithril-file-archiver + +**This is a work in progress** 🛠 + +An API to generate tar archives from files, directories, or serializable data (leveraging serde). + +Produced archives are byte stable across systems as long as the following invariants do not change: + +- The version of the zstandard compression library +- The parameters of the zstandard compression + +## Byte stability guarantees + +Given identical archive entry paths and contents, `FileArchiver` produces byte-identical `.tar.zst` archives across +runs and supported host systems. + +Archive bytes are unaffected by: + +- The source base directory +- File creation and modification times +- File permissions +- The order in which entries are supplied +- Equivalent path spellings, such as `foo/` and `foo`, or `./foo.txt` and `foo.txt` +- The order in which non-overlapping appenders are chained + +Entries are normalized and sorted by their archive paths. When chained appenders provide the same archive path, the +rightmost appender takes precedence. + +This guarantee requires the following archive-format invariants to remain unchanged: + +- Archive entry paths and contents +- The versions and behavior of the TAR and Zstandard libraries +- The Zstandard compression parameters, including the compression level and number of workers +- TAR header generation and metadata normalization +- JSON serialization output when using `AppenderData::from_json` + +Changing one of these invariants can change the resulting bytes and must be treated as an intentional archive-format +change. +Such a change requires bumping the archive-format version and updating the golden hashes that pin the expected output. diff --git a/mithril-aggregator/src/tools/file_archiver/api.rs b/internal/mithril-file-archiver/src/api.rs similarity index 85% rename from mithril-aggregator/src/tools/file_archiver/api.rs rename to internal/mithril-file-archiver/src/api.rs index bce2e595888..030abe97ba2 100644 --- a/mithril-aggregator/src/tools/file_archiver/api.rs +++ b/internal/mithril-file-archiver/src/api.rs @@ -6,19 +6,17 @@ use std::{ io::{Read, Seek, SeekFrom}, path::{Path, PathBuf}, }; -use tar::{Archive, Entry, EntryType}; +use tar::{Archive, Entry, EntryType, HeaderMode}; use zstd::{Decoder, Encoder}; use mithril_common::StdResult; use mithril_common::entities::CompressionAlgorithm; use mithril_common::logging::LoggerExtensions; -use crate::ZstandardCompressionParameters; +use crate::appender::TarAppender; +use crate::entities::{ArchiveParameters, FileArchive, ZstandardCompressionParameters}; use crate::tools::file_size; -use super::appender::TarAppender; -use super::{ArchiveParameters, FileArchive}; - /// Tool to archive files and directories. pub struct FileArchiver { zstandard_compression_parameter: ZstandardCompressionParameters, @@ -41,14 +39,18 @@ impl FileArchiver { } } - #[cfg(test)] - pub fn new_for_test(verification_temp_dir: PathBuf) -> Self { - use crate::test::TestLogger; - Self { - zstandard_compression_parameter: ZstandardCompressionParameters::default(), + /// Constructs a new `FileArchiver` that uses the default compression parameters. + pub fn new_with_default_parameters(verification_temp_dir: PathBuf, logger: Logger) -> Self { + Self::new( + ZstandardCompressionParameters::default(), verification_temp_dir, - logger: TestLogger::stdout(), - } + logger, + ) + } + + #[cfg(test)] + pub(crate) fn new_for_test(verification_temp_dir: PathBuf) -> Self { + Self::new_with_default_parameters(verification_temp_dir, crate::test::TestLogger::stdout()) } /// Archive the content of a directory. @@ -152,6 +154,7 @@ impl FileArchiver { enc.multithread(self.zstandard_compression_parameter.number_of_workers) .with_context(|| "ZstandardEncoder can not set the number of workers")?; let mut tar = tar::Builder::new(enc); + Self::configure_tar_builder(&mut tar); appender .append(&mut tar) @@ -272,16 +275,25 @@ impl FileArchiver { Ok(()) } + + fn configure_tar_builder(builder: &mut tar::Builder) { + builder.mode(HeaderMode::Deterministic); + builder.follow_symlinks(false); + // disable sparse files, as their support is not uniform across platforms and the size + // difference won't matter with zstandard compression + builder.sparse(false); + } } #[cfg(test)] mod tests { use std::fs::File; + use mithril_common::temp_dir_create; use mithril_common::test::assert_equivalent; - use crate::tools::file_archiver::appender::{AppenderDirAll, AppenderFile}; - use crate::tools::file_archiver::test_tools::*; + use crate::appender::{AppenderEntries, AppenderFile}; + use crate::test::{FileArchiveTestExtension, create_dir, create_file, double::FailAppender}; use super::*; @@ -294,18 +306,17 @@ mod tests { #[test] fn should_create_a_valid_archive_with_zstandard_compression() { - let test_dir = - get_test_directory("should_create_a_valid_archive_with_zstandard_compression"); + let test_dir = temp_dir_create!(); let target_archive = test_dir.join("archive.tar.zst"); - let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory")); - create_file(&archived_directory, "file_to_archive.txt"); + let source_dir = test_dir.join(create_dir(&test_dir, "source")); + let archived_file = source_dir.join(create_file(&source_dir, "file_to_archive.txt")); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); let archive = file_archiver .create_archive( &target_archive, - AppenderDirAll::new(archived_directory), + AppenderFile::append_at_archive_root(archived_file).unwrap(), CompressionAlgorithm::Zstandard, ) .expect("create_archive should not fail"); @@ -316,10 +327,7 @@ mod tests { #[test] fn should_delete_tmp_file_in_target_directory_if_archiving_fail() { - let test_dir = - get_test_directory("should_delete_tmp_file_in_target_directory_if_archiving_fail"); - // Note: the archived directory does not exist in order to make the archive process fail - let archived_directory = test_dir.join("db"); + let test_dir = temp_dir_create!(); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -332,7 +340,7 @@ mod tests { compression_algorithm: CompressionAlgorithm::Zstandard, }; let _ = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive(archive_params, FailAppender) .expect_err("FileArchiver::archive should fail if the target path doesn't exist."); let remaining_files: Vec = list_remaining_files(&test_dir); @@ -341,11 +349,7 @@ mod tests { #[test] fn should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail() { - let test_dir = get_test_directory( - "should_not_delete_an_already_existing_archive_with_same_name_if_archiving_fail", - ); - // Note: the archived directory does not exist in order to make the archive process fail - let archived_directory = test_dir.join("db"); + let test_dir = temp_dir_create!(); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -361,7 +365,7 @@ mod tests { compression_algorithm: CompressionAlgorithm::Zstandard, }; let _ = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive(archive_params, FailAppender) .expect_err("FileArchiver::archive should fail if the db is empty."); let remaining_files: Vec = list_remaining_files(&test_dir); @@ -373,11 +377,9 @@ mod tests { #[test] fn overwrite_already_existing_archive_when_archiving_succeed() { - let test_dir = - get_test_directory("overwrite_already_existing_archive_when_archiving_succeed"); - let archived_directory = test_dir.join(create_dir(&test_dir, "archived_directory")); - - create_file(&archived_directory, "file_to_archive.txt"); + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + let file_to_archive = create_file(&source, "file_to_archive.txt"); let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); @@ -389,15 +391,22 @@ mod tests { let first_archive = file_archiver .archive( archive_params.clone(), - AppenderDirAll::new(archived_directory.clone()), + AppenderEntries::new(vec![file_to_archive.clone()], source.clone()).unwrap(), ) .unwrap(); let first_archive_size = first_archive.get_archive_size(); - create_file(&archived_directory, "another_file_to_archive.txt"); + let another_file_to_archive = create_file(&source, "another_file_to_archive.txt"); let second_archive = file_archiver - .archive(archive_params, AppenderDirAll::new(archived_directory)) + .archive( + archive_params, + AppenderEntries::new( + vec![file_to_archive, another_file_to_archive], + source.clone(), + ) + .unwrap(), + ) .unwrap(); let second_archive_size = second_archive.get_archive_size(); @@ -409,7 +418,7 @@ mod tests { #[test] fn compute_size_of_uncompressed_data_and_archive() { - let test_dir = get_test_directory("compute_size_of_uncompressed_data_and_archive"); + let test_dir = temp_dir_create!(); let file_path = test_dir.join("file.txt"); let file = File::create(&file_path).unwrap(); diff --git a/internal/mithril-file-archiver/src/appender.rs b/internal/mithril-file-archiver/src/appender.rs new file mode 100644 index 00000000000..cc75fb5955c --- /dev/null +++ b/internal/mithril-file-archiver/src/appender.rs @@ -0,0 +1,926 @@ +//! Define how to append data to a [FileArchiver][crate::FileArchiver] + +use std::cmp::Ordering; +use std::collections::BTreeSet; +use std::fs::File; +use std::io::Write; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, anyhow}; +use serde::Serialize; + +use mithril_common::StdResult; + +use crate::tools::file_size; + +const READ_WRITE_PERMISSION: u32 = 0o666; +/// Timestamp arbitrarily chosen to `2026-01-01 00:00:00 UTC` +/// IMPORTANT: Do NOT change it, else the `AppenderData` archives bytes would change. +const FIXED_MTIME_ATTRIBUTE_FOR_DATA: u64 = 1767225600; + +/// Define multiple ways to append content to a tar archive. +pub trait TarAppender: Send { + /// Appends the contents of the current object to the given tar archive builder. + fn append(&self, tar: &mut tar::Builder) -> StdResult<()>; + + /// Computes the total uncompressed size of the data that will be added to the archive. + fn compute_uncompressed_data_size(&self) -> StdResult; +} + +/// Represents an object that can provide a list of entries to append to a tar archive. +pub trait ArchiveEntryProvider: Send { + /// Get the list of archive entries held by this provider. + fn collect_entries(&self) -> StdResult>; + + /// Chains this provider with another, combining their contents into a single archive. + /// + /// - when paths are overlapping, the rightmost provider takes precedence. + /// - if there are no overlapping paths, chaining is commutative `(A | B) = (B | A)`. + /// - chaining is always associative: `((A | B) | C) = (A | (B | C))`, even when paths are overlapping. + fn chain(self, provider_right: P2) -> ChainAppender + where + Self: Sized, + { + ChainAppender::new(self, provider_right) + } +} + +impl TarAppender for T { + fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { + for entry in self + .collect_entries() + .with_context(|| "Failed to collect entries from appender")? + { + entry.append_to_archive(tar)?; + } + Ok(()) + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + let mut size: u64 = 0; + for entry in self + .collect_entries() + .with_context(|| "Failed to collect entries from appender")? + { + size = size + .checked_add(entry.compute_uncompressed_data_size()?) + .with_context(|| "Failed to compute uncompressed data size")?; + } + Ok(size) + } +} + +/// Represents an entry to be added to a tar archive. +/// +/// Archive entries are identified and ordered solely by their normalized archive path. +/// Content and source fields do not participate in equality. +#[derive(Debug, Clone)] +pub enum ArchiveEntry { + /// A file entry. + File { + /// Path of the file in the archive. + location_in_archive: PathBuf, + /// Path to the source file on disk. + target_file: PathBuf, + }, + /// A directory entry. + Directory { + /// Path of the directory in the archive. + location_in_archive: PathBuf, + /// Path to the source directory on disk. + target_dir: PathBuf, + }, + /// Raw data entry. + Data { + /// Path of the data in the archive. + location_in_archive: PathBuf, + /// The data bytes. + data: Arc>, + }, +} + +impl ArchiveEntry { + /// Creates a directory entry. + pub fn from_dir(location_in_archive: PathBuf, target_dir: PathBuf) -> Self { + ArchiveEntry::Directory { + location_in_archive: Self::normalize_entry(location_in_archive), + target_dir, + } + } + + /// Creates a file entry. + pub fn from_file(location_in_archive: PathBuf, target_file: PathBuf) -> Self { + ArchiveEntry::File { + location_in_archive: Self::normalize_entry(location_in_archive), + target_file, + } + } + + /// Creates a data entry. + pub fn from_data(location_in_archive: PathBuf, data: Vec) -> Self { + ArchiveEntry::Data { + location_in_archive: Self::normalize_entry(location_in_archive), + data: Arc::new(data), + } + } + + /// Returns the location of this entry in the archive. + pub fn location_in_archive(&self) -> &Path { + match self { + ArchiveEntry::File { + location_in_archive, + .. + } => location_in_archive, + ArchiveEntry::Directory { + location_in_archive, + .. + } => location_in_archive, + ArchiveEntry::Data { + location_in_archive, + .. + } => location_in_archive, + } + } + + /// Appends this entry to the given tar archive builder. + pub fn append_to_archive(&self, tar: &mut tar::Builder) -> StdResult<()> { + match self { + ArchiveEntry::File { + location_in_archive, + target_file, + } => { + if !target_file.is_file() { + anyhow::bail!( + "File '{}' does not exist, can not add it to the archive at '{}'", + target_file.display(), + location_in_archive.display() + ); + } + + let mut file = File::open(target_file)?; + tar.append_file(location_in_archive, &mut file).with_context(|| { + format!( + "Can not add file: '{}' to the archive", + target_file.display() + ) + })?; + } + ArchiveEntry::Directory { + location_in_archive, + target_dir, + } => { + if !target_dir.is_dir() { + anyhow::bail!( + "Directory '{}' does not exist, can not add it to the archive at '{}'", + target_dir.display(), + location_in_archive.display() + ); + } + + tar.append_dir(location_in_archive, target_dir).with_context(|| { + format!( + "Can not add directory: '{}' to the archive", + location_in_archive.display() + ) + })?; + } + ArchiveEntry::Data { + location_in_archive, + data, + } => { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(READ_WRITE_PERMISSION); + header.set_mtime(FIXED_MTIME_ATTRIBUTE_FOR_DATA); + header.set_cksum(); + + tar.append_data(&mut header, location_in_archive, data.as_slice()) + .with_context(|| { + format!( + "Can not add file: '{}' to the archive", + location_in_archive.display() + ) + })?; + } + } + + Ok(()) + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + match self { + ArchiveEntry::File { target_file, .. } => file_size::compute_size_of_path(target_file), + ArchiveEntry::Directory { .. } => Ok(0), + ArchiveEntry::Data { data, .. } => Ok(data.len() as u64), + } + } + + fn normalize_entry(entry: PathBuf) -> PathBuf { + entry + .components() + .filter(|c| !matches!(c, Component::CurDir)) + .collect() + } +} + +impl Ord for ArchiveEntry { + fn cmp(&self, other: &Self) -> Ordering { + self.location_in_archive().cmp(other.location_in_archive()) + } +} + +impl PartialOrd for ArchiveEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for ArchiveEntry { + fn eq(&self, other: &Self) -> bool { + self.location_in_archive() == other.location_in_archive() + } +} + +impl Eq for ArchiveEntry {} + +/// An appender that adds one file. +pub struct AppenderFile { + entry: ArchiveEntry, +} + +impl AppenderFile { + /// Append the file at the root of the archive, keeping the same file name. + pub fn append_at_archive_root(target_file: PathBuf) -> StdResult { + if !target_file.is_file() { + return Err(anyhow!( + "The target file is not a file, path: {}", + target_file.display() + )); + } + + let location_in_archive = target_file + .file_name() + .with_context(|| { + format!( + "Can not get the file name from the target file path: '{}'", + target_file.display() + ) + })? + .to_owned(); + + Ok(Self { + entry: ArchiveEntry::from_file(PathBuf::from(location_in_archive), target_file), + }) + } +} + +impl ArchiveEntryProvider for AppenderFile { + fn collect_entries(&self) -> StdResult> { + Ok(BTreeSet::from([self.entry.clone()])) + } +} + +/// An appender that adds a list of entries, files, or directories. +/// +/// Directory contents are not added if not specified. +pub struct AppenderEntries { + entries: BTreeSet, +} + +impl AppenderEntries { + /// Create a new instance of `AppenderEntries`. + /// + /// Entries are normalized and sorted to ensure deterministic archive output. + /// + /// Returns an error if the `entries` are empty. + pub fn new(entries: Vec, base_directory: PathBuf) -> StdResult { + if entries.is_empty() { + return Err(anyhow!("The entries can not be empty")); + } + + let mut archive_entries: BTreeSet = BTreeSet::new(); + + for entry in entries { + let entry_path = base_directory.join(&entry); + if entry_path.is_dir() { + archive_entries.insert(ArchiveEntry::from_dir(entry, entry_path)); + } else if entry_path.is_file() { + archive_entries.insert(ArchiveEntry::from_file(entry, entry_path)); + } else { + anyhow::bail!("The entry: '{}' is not valid", entry_path.display()); + } + } + + Ok(Self { + entries: archive_entries, + }) + } +} + +impl ArchiveEntryProvider for AppenderEntries { + fn collect_entries(&self) -> StdResult> { + Ok(self.entries.clone()) + } +} + +/// An appender that adds either [serde::Serialize] serializable data or raw bytes. +pub struct AppenderData { + entry: ArchiveEntry, +} + +impl AppenderData { + /// Create a new instance of `AppenderData` from an object that will be serialized to JSON. + pub fn from_json( + location_in_archive: PathBuf, + object: &T, + ) -> StdResult { + let json_bytes = serde_json::to_vec(object).with_context(|| { + format!( + "Can not serialize JSON to file in archive: {:?}", + location_in_archive.display() + ) + })?; + + Ok(Self::from_raw_bytes(location_in_archive, json_bytes)) + } + + /// Create a new instance of `AppenderData` from a byte array. + pub fn from_raw_bytes(location_in_archive: PathBuf, bytes: Vec) -> Self { + Self { + entry: ArchiveEntry::from_data(location_in_archive, bytes), + } + } +} + +impl ArchiveEntryProvider for AppenderData { + fn collect_entries(&self) -> StdResult> { + Ok(BTreeSet::from([self.entry.clone()])) + } +} + +/// Combines multiple archive entry providers into one deterministic entry collection. +/// +/// - when paths are overlapping, the rightmost provider takes precedence. +/// - if there are no overlapping paths, chaining is commutative `(A | B) = (B | A)`. +/// - chaining is always associative: `((A | B) | C) = (A | (B | C))`, even when paths are overlapping. +pub struct ChainAppender { + provider_left: L, + provider_right: R, +} + +impl ChainAppender { + /// [ChainAppender] factory + pub fn new(provider_left: L, provider_right: R) -> Self { + Self { + provider_left, + provider_right, + } + } + + fn merge_entries_with_right_precedence(&self) -> StdResult> { + let mut entries = self.provider_right.collect_entries()?; + entries.extend(self.provider_left.collect_entries()?); + Ok(entries) + } +} + +impl ArchiveEntryProvider + for ChainAppender +{ + fn collect_entries(&self) -> StdResult> { + self.merge_entries_with_right_precedence() + } +} + +#[cfg(test)] +mod tests { + use mithril_common::entities::CompressionAlgorithm; + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use crate::api::FileArchiver; + use crate::entities::ArchiveParameters; + use crate::test::{FileArchiveTestExtension, create_dir, create_file}; + + use super::*; + + mod archive_entry { + use super::*; + + #[test] + fn removes_trailing_separator_from_directory_component() { + assert_eq!( + PathBuf::from("foo"), + ArchiveEntry::normalize_entry(PathBuf::from("foo/")), + ); + } + + #[cfg(windows)] + #[test] + fn removes_windows_trailing_separator_from_directory_component() { + assert_eq!( + PathBuf::from("foo"), + ArchiveEntry::normalize_entry(PathBuf::from("foo\\")), + ); + } + + #[test] + fn removes_leading_current_directory_component() { + assert_eq!( + PathBuf::from("foo/bar.txt"), + ArchiveEntry::normalize_entry(PathBuf::from("./foo/bar.txt")), + ); + } + + #[cfg(windows)] + #[test] + fn removes_windows_leading_current_directory_component() { + assert_eq!( + PathBuf::from("foo").join("bar.txt"), + ArchiveEntry::normalize_entry(PathBuf::from(r".\foo\bar.txt")), + ); + } + + #[cfg(windows)] + #[test] + fn forward_and_backward_separators_have_the_same_normalized_path() { + assert_eq!( + ArchiveEntry::normalize_entry(PathBuf::from("foo/bar.txt")), + ArchiveEntry::normalize_entry(PathBuf::from(r"foo\bar.txt")), + ); + } + + #[test] + fn entries_are_sorted_by_path() { + let mut entries = [ + ArchiveEntry::from_file(PathBuf::from("foo/bar.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("file_2.txt"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from("bar/"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from("foo/"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("foo/pika/"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("foo/pika/chuu.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from("file_1.txt"), PathBuf::new()), + ]; + entries.sort(); + + assert_eq!( + vec![ + PathBuf::from("bar"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from("foo"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("foo/pika"), + PathBuf::from("foo/pika/chuu.txt"), + ], + entries + .into_iter() + .map(|entry| entry.location_in_archive().to_path_buf()) + .collect::>() + ); + } + + #[cfg(windows)] + #[test] + fn entries_with_windows_path_are_sorted_by_path() { + let mut entries = [ + ArchiveEntry::from_file(PathBuf::from(r"foo\pika\chuu.txt"), PathBuf::new()), + ArchiveEntry::from_file(PathBuf::from(r"foo\bar.txt"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from(r"bar\\"), PathBuf::new()), + ArchiveEntry::from_dir(PathBuf::from(r"foo\\"), PathBuf::new()), + ]; + entries.sort(); + + assert_eq!( + vec![ + PathBuf::from("bar"), + PathBuf::from("foo"), + PathBuf::from("foo").join("bar.txt"), + PathBuf::from("foo").join("pika").join("chuu.txt"), + ], + entries + .into_iter() + .map(|entry| entry.location_in_archive().to_path_buf()) + .collect::>() + ); + } + + #[test] + fn appending_fails_if_source_file_does_not_exist() { + let entry = + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("not_exist.txt")); + let mut tar = tar::Builder::new(Vec::new()); + + let res = entry.append_to_archive(&mut tar); + assert!(res.is_err()); + } + + #[test] + fn appending_fails_if_source_dir_does_not_exist() { + let entry = ArchiveEntry::from_dir(PathBuf::from("foo/"), PathBuf::from("not_exist/")); + let mut tar = tar::Builder::new(Vec::new()); + + let res = entry.append_to_archive(&mut tar); + assert!(res.is_err()); + } + + #[test] + fn entries_with_the_same_archive_path_but_different_content_are_considered_equal() { + assert_eq!( + ArchiveEntry::from_data(PathBuf::from("foo.txt"), vec![0, 1, 2]), + ArchiveEntry::from_data(PathBuf::from("foo.txt"), vec![3, 4, 5]), + ); + + assert_eq!( + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("file.txt")), + ArchiveEntry::from_file(PathBuf::from("foo.txt"), PathBuf::from("other.txt")) + ); + } + } + + mod appender_entries { + use super::*; + + #[test] + fn create_archive_only_for_specified_directories_and_files() { + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + + let directory_to_archive_path = create_dir(&source, "directory_to_archive"); + let file_in_dir_to_archive_path = + create_file(&source, "directory_to_archive/file_in_dir_to_archive.txt"); + let file_to_archive_path = create_file(&source, "file_to_archive.txt"); + let empty_directory_to_archive_path = create_dir(&source, "empty_directory_to_archive"); + + create_dir(&source, "directory_not_to_archive"); + create_file(&source, "file_not_to_archive.txt"); + create_file( + &source, + "directory_to_archive/file_in_dir_not_to_archive.txt", + ); + + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + AppenderEntries::new( + vec![ + directory_to_archive_path, + file_in_dir_to_archive_path, + file_to_archive_path, + empty_directory_to_archive_path, + ], + source, + ) + .unwrap(), + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + + assert_dir_eq!( + &unpack_path, + "* directory_to_archive/ + ** file_in_dir_to_archive.txt + * empty_directory_to_archive/ + * file_to_archive.txt" + ); + } + + #[test] + fn creation_fails_when_entry_does_not_exist() { + let test_dir = temp_dir_create!(); + let res = AppenderEntries::new(vec![PathBuf::from("not_exist")], test_dir); + + assert!( + res.is_err(), + "AppenderEntries should return error when file or directory not exist" + ); + } + + #[test] + fn return_error_when_appending_empty_entries() { + let appender_creation_result = AppenderEntries::new(vec![], PathBuf::new()); + assert!(appender_creation_result.is_err(),); + } + + #[test] + fn can_append_duplicate_files_and_directories() { + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + + let directory_to_archive_path = create_dir(&source, "directory_to_archive"); + let file_to_archive_path = + create_file(&source, "directory_to_archive/file_to_archive.txt"); + + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + AppenderEntries::new( + vec![ + directory_to_archive_path.clone(), + directory_to_archive_path.clone(), + file_to_archive_path.clone(), + file_to_archive_path.clone(), + ], + source, + ) + .unwrap(), + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + + assert_dir_eq!( + &unpack_path, + "* directory_to_archive/ + ** file_to_archive.txt" + ); + } + + #[test] + fn compute_uncompressed_size_of_its_paths() { + fn create_file_with_len(path: &Path, len: u64) { + let file = File::create(path) + .unwrap_or_else(|_| panic!("failed to create '{}'", path.display())); + file.set_len(len).unwrap(); + } + + let test_dir = temp_dir_create!(); + let source = test_dir.join(create_dir(&test_dir, "source")); + let subdir = source.join(create_dir(&source, "subdir")); + create_file_with_len(&source.join("file_1"), 100); + create_file_with_len(&source.join("file_2"), 200); + create_file_with_len(&subdir.join("file_3"), 300); + create_file_with_len(&subdir.join("file_not_to_include"), 400); + + let appender_entries = AppenderEntries::new( + vec![ + PathBuf::from("file_1"), + PathBuf::from("file_2"), + PathBuf::from("subdir/"), + PathBuf::from("subdir/file_3"), + ], + source, + ) + .unwrap(); + + let entries_size = appender_entries.compute_uncompressed_data_size().unwrap(); + assert_eq!(600, entries_size); + } + } + + mod appender_file { + use super::*; + + #[test] + fn appending_file_to_tar() { + let test_dir = temp_dir_create!(); + let file_to_archive = create_file(&test_dir, "test_file.txt"); + + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + AppenderFile::append_at_archive_root(test_dir.join(&file_to_archive)).unwrap(), + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + + assert!(unpack_path.join(file_to_archive).exists()); + } + + #[test] + fn return_error_if_file_does_not_exist() { + let target_file_path = PathBuf::from("non_existent_file.txt"); + assert!(AppenderFile::append_at_archive_root(target_file_path).is_err()); + } + + #[test] + fn return_error_if_input_is_not_a_file() { + let test_dir = temp_dir_create!(); + assert!(AppenderFile::append_at_archive_root(test_dir).is_err()); + } + + #[test] + fn compute_uncompressed_size() { + let test_dir = temp_dir_create!(); + + let file_path = test_dir.join("file.txt"); + let file = File::create(&file_path).unwrap(); + file.set_len(777).unwrap(); + + let appender_file = AppenderFile::append_at_archive_root(file_path).unwrap(); + + let entries_size = appender_file.compute_uncompressed_data_size().unwrap(); + assert_eq!(777, entries_size); + } + } + + mod appender_data { + use serde::Deserialize; + use zstd::Decoder; + + use super::*; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct TestStruct { + field1: String, + field2: i32, + } + + #[test] + fn append_serializable_json() { + let test_dir = temp_dir_create!(); + let object = TestStruct { + field1: "test".to_string(), + field2: 42, + }; + let location_in_archive = PathBuf::from("folder").join("test.json"); + + let data_appender = + AppenderData::from_json(location_in_archive.clone(), &object).unwrap(); + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + data_appender, + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + let unpacked_file_path = unpack_path.join(&location_in_archive); + + assert!(unpacked_file_path.exists()); + + let deserialized_object: TestStruct = + serde_json::from_reader(File::open(unpacked_file_path).unwrap()).unwrap(); + assert_eq!(object, deserialized_object); + } + + #[test] + fn appended_entry_have_read_write_permissions_and_fixed_time_metadata() { + let test_dir = temp_dir_create!(); + let object = TestStruct { + field1: "test".to_string(), + field2: 42, + }; + let location_in_archive = PathBuf::from("folder").join("test.json"); + + let data_appender = + AppenderData::from_json(location_in_archive.clone(), &object).unwrap(); + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + data_appender, + ) + .unwrap(); + + let archive_file = File::open(archive.get_file_path()).unwrap(); + let mut archive = tar::Archive::new(Decoder::new(archive_file).unwrap()); + let mut archive_entries = archive.entries().unwrap(); + let appended_entry = archive_entries.next().unwrap().unwrap(); + + assert_eq!( + READ_WRITE_PERMISSION, + appended_entry.header().mode().unwrap() + ); + let mtime = appended_entry.header().mtime().unwrap(); + assert_eq!(FIXED_MTIME_ATTRIBUTE_FOR_DATA, mtime); + } + + #[test] + fn compute_uncompressed_size() { + let object = TestStruct { + field1: "test".to_string(), + field2: 42, + }; + + let data_appender = + AppenderData::from_json(PathBuf::from("whatever.json"), &object).unwrap(); + + let expected_size = serde_json::to_vec(&object).unwrap().len() as u64; + let entry_size = data_appender.compute_uncompressed_data_size().unwrap(); + assert_eq!(expected_size, entry_size); + } + } + + mod chain_appender { + use super::*; + + #[test] + fn chain_non_overlapping_appenders() { + let test_dir = temp_dir_create!(); + let file_to_archive = create_file(&test_dir, "test_file.txt"); + let json_location_in_archive = PathBuf::from("folder").join("test.json"); + + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + ChainAppender::new( + AppenderFile::append_at_archive_root(test_dir.join(&file_to_archive)) + .unwrap(), + AppenderData::from_json(json_location_in_archive.clone(), &"test").unwrap(), + ), + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + + assert!(unpack_path.join(file_to_archive).exists()); + assert!(unpack_path.join(json_location_in_archive).exists()); + } + + #[test] + fn chain_overlapping_appenders_data_from_right_appender_overwrite_left_appender_data() { + let test_dir = temp_dir_create!(); + let json_location_in_archive = PathBuf::from("test.json"); + + let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); + let archive = file_archiver + .archive( + ArchiveParameters { + archive_name_without_extension: "archive".to_string(), + target_directory: test_dir.clone(), + compression_algorithm: CompressionAlgorithm::Zstandard, + }, + ChainAppender::new( + AppenderData::from_json( + json_location_in_archive.clone(), + &"will be overwritten", + ) + .unwrap(), + AppenderData::from_json(json_location_in_archive.clone(), &"test").unwrap(), + ), + ) + .unwrap(); + + let unpack_path = archive.unpack_zstandard(&test_dir); + let unpacked_json_path = unpack_path.join(&json_location_in_archive); + + let deserialized_object: String = + serde_json::from_reader(File::open(&unpacked_json_path).unwrap()).unwrap(); + assert_eq!("test", deserialized_object); + } + + #[test] + fn compute_non_overlapping_uncompressed_size() { + let left_appender = + AppenderData::from_json(PathBuf::from("whatever1.json"), &"foo").unwrap(); + let right_appender = + AppenderData::from_json(PathBuf::from("whatever2.json"), &"bar").unwrap(); + + let expected_size = left_appender.compute_uncompressed_data_size().unwrap() + + right_appender.compute_uncompressed_data_size().unwrap(); + + let chain_appender = left_appender.chain(right_appender); + let size = chain_appender.compute_uncompressed_data_size().unwrap(); + assert_eq!(expected_size, size); + } + + #[test] + fn compute_overlapping_uncompressed_size() { + let overlapping_path = PathBuf::from("whatever.json"); + let left_appender = + AppenderData::from_json(overlapping_path.clone(), &"overwritten data").unwrap(); + let right_appender = + AppenderData::from_json(overlapping_path.clone(), &"final data").unwrap(); + + let expected_size = right_appender.compute_uncompressed_data_size().unwrap(); + + let chain_appender = left_appender.chain(right_appender); + let size = chain_appender.compute_uncompressed_data_size().unwrap(); + assert_eq!(expected_size, size); + } + } +} diff --git a/mithril-aggregator/src/tools/file_archiver/entities.rs b/internal/mithril-file-archiver/src/entities.rs similarity index 72% rename from mithril-aggregator/src/tools/file_archiver/entities.rs rename to internal/mithril-file-archiver/src/entities.rs index a4e9200688e..cefc7ba966f 100644 --- a/mithril-aggregator/src/tools/file_archiver/entities.rs +++ b/internal/mithril-file-archiver/src/entities.rs @@ -1,12 +1,36 @@ use std::path::{Path, PathBuf}; +use serde::Deserialize; + use mithril_common::entities::CompressionAlgorithm; +/// [Zstandard][CompressionAlgorithm::Zstandard] specific parameters +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +pub struct ZstandardCompressionParameters { + /// Level of compression, default to 9. + pub level: i32, + + /// Number of workers when compressing, 0 will disable multithreading, default to 4. + pub number_of_workers: u32, +} + +impl Default for ZstandardCompressionParameters { + fn default() -> Self { + Self { + level: 9, + number_of_workers: 4, + } + } +} + /// Parameters for creating an archive. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ArchiveParameters { + /// Archive name without file extension pub archive_name_without_extension: String, + /// Directory where the archive will be created pub target_directory: PathBuf, + /// Compression algorithm to use for the archive pub compression_algorithm: CompressionAlgorithm, } @@ -69,41 +93,6 @@ impl FileArchive { pub fn get_compression_algorithm(&self) -> CompressionAlgorithm { self.compression_algorithm } - - /// Unpack the archive to a directory. - /// - /// An 'unpack' directory will be created in the given parent directory. - #[cfg(test)] - pub fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf { - use super::test_tools::create_dir; - use std::fs::File; - use tar::Archive; - use zstd::stream::read::Decoder; - if self.compression_algorithm != CompressionAlgorithm::Zstandard { - panic!("Only Zstandard compression is supported"); - } - - let parent_dir = parent_dir.as_ref(); - let file_tar_zst = File::open(self.get_file_path()).unwrap(); - let file_tar_zst_decoder = Decoder::new(file_tar_zst).unwrap(); - let mut archive = Archive::new(file_tar_zst_decoder); - let unpack_path = parent_dir.join(create_dir(parent_dir, "unpack")); - archive.unpack(&unpack_path).unwrap(); - - unpack_path - } -} - -#[cfg(test)] -impl mithril_common::test::double::Dummy for FileArchive { - fn dummy() -> Self { - Self { - filepath: PathBuf::from("archive.tar.zst"), - archive_filesize: 10, - uncompressed_size: 789, - compression_algorithm: CompressionAlgorithm::Zstandard, - } - } } #[cfg(test)] diff --git a/internal/mithril-file-archiver/src/lib.rs b/internal/mithril-file-archiver/src/lib.rs new file mode 100644 index 00000000000..bac69444fbe --- /dev/null +++ b/internal/mithril-file-archiver/src/lib.rs @@ -0,0 +1,19 @@ +#![warn(missing_docs)] + +//! # Mithril-file-archiver +//! +//! An API to generate tar archives from files, directories, or serializable data (leveraging serde). +//! +//! Produced archives are byte stable across systems as long as the following invariants do not change: +//! * The version of the zstandard compression library +//! * The parameters of the zstandard compression +//! + +mod api; +pub mod appender; +mod entities; +pub mod test; +pub mod tools; + +pub use api::*; +pub use entities::*; diff --git a/internal/mithril-file-archiver/src/test/double/dummies.rs b/internal/mithril-file-archiver/src/test/double/dummies.rs new file mode 100644 index 00000000000..a59ae70320a --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/dummies.rs @@ -0,0 +1,17 @@ +use std::path::PathBuf; + +use mithril_common::entities::CompressionAlgorithm; +use mithril_common::test::double::Dummy; + +use crate::FileArchive; + +impl Dummy for FileArchive { + fn dummy() -> Self { + Self { + filepath: PathBuf::from("archive.tar.zst"), + archive_filesize: 10, + uncompressed_size: 789, + compression_algorithm: CompressionAlgorithm::Zstandard, + } + } +} diff --git a/internal/mithril-file-archiver/src/test/double/fail_appender.rs b/internal/mithril-file-archiver/src/test/double/fail_appender.rs new file mode 100644 index 00000000000..420e9ad78c8 --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/fail_appender.rs @@ -0,0 +1,21 @@ +use std::io::Write; +use tar::Builder; + +use mithril_common::StdResult; + +use crate::appender::TarAppender; + +/// A test double appender that always fails. +/// +/// Used in tests to verify error handling behavior when appending operations fail. +pub struct FailAppender; + +impl TarAppender for FailAppender { + fn append(&self, _tar: &mut Builder) -> StdResult<()> { + anyhow::bail!("FailAppender always fails (append)") + } + + fn compute_uncompressed_data_size(&self) -> StdResult { + anyhow::bail!("FailAppender always fails (compute_uncompressed_data_size)") + } +} diff --git a/internal/mithril-file-archiver/src/test/double/mod.rs b/internal/mithril-file-archiver/src/test/double/mod.rs new file mode 100644 index 00000000000..e569a50bff9 --- /dev/null +++ b/internal/mithril-file-archiver/src/test/double/mod.rs @@ -0,0 +1,8 @@ +//! Test doubles +//! +//! Enable unit testing with controlled inputs and predictable behavior. + +mod dummies; +mod fail_appender; + +pub use fail_appender::FailAppender; diff --git a/internal/mithril-file-archiver/src/test/extensions.rs b/internal/mithril-file-archiver/src/test/extensions.rs new file mode 100644 index 00000000000..830f551c82d --- /dev/null +++ b/internal/mithril-file-archiver/src/test/extensions.rs @@ -0,0 +1,29 @@ +use std::path::{Path, PathBuf}; + +use mithril_common::entities::CompressionAlgorithm; + +use crate::FileArchive; +use crate::test::unpack_archive; + +/// Extension trait adding test utilities to [FileArchive] +pub trait FileArchiveTestExtension { + /// `TEST ONLY` - Unpack the archive to a directory. + /// + /// An 'unpack' directory will be created in the given parent directory. + fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf; +} + +impl FileArchiveTestExtension for FileArchive { + fn unpack_zstandard>(&self, parent_dir: P) -> PathBuf { + if self.compression_algorithm != CompressionAlgorithm::Zstandard { + panic!("Only Zstandard compression is supported"); + } + + let unpack_path = parent_dir.as_ref().join("unpack"); + std::fs::create_dir(&unpack_path).unwrap(); + + unpack_archive(self.get_file_path(), &unpack_path).unwrap(); + + unpack_path + } +} diff --git a/internal/mithril-file-archiver/src/test/mod.rs b/internal/mithril-file-archiver/src/test/mod.rs new file mode 100644 index 00000000000..c782b4b0e7a --- /dev/null +++ b/internal/mithril-file-archiver/src/test/mod.rs @@ -0,0 +1,56 @@ +//! Test utilities. +//! +//! ⚠ Do not use in production code ⚠ +//! +//! This module provides in particular test doubles for the traits defined in this crate. + +pub mod double; +mod extensions; + +pub use extensions::*; + +#[cfg(test)] +pub(crate) use internal_tests_only::*; + +/// Unpack a zstandard-compressed tar archive to a specified directory. +/// +/// Note: `unpack_dir` must exist. +pub fn unpack_archive( + archive_path: &std::path::Path, + unpack_dir: &std::path::Path, +) -> mithril_common::StdResult<()> { + let mut archive = { + let file_tar_zst = std::fs::File::open(archive_path)?; + let file_tar_zst_decoder = zstd::Decoder::new(file_tar_zst)?; + tar::Archive::new(file_tar_zst_decoder) + }; + + archive.unpack(unpack_dir)?; + Ok(()) +} + +#[cfg(test)] +mod internal_tests_only { + use std::fs::File; + use std::path::{Path, PathBuf}; + + mithril_common::define_test_logger!(); + + /// Create a file in the root directory. + /// + /// Returns the relative path to the created file based on the root directory. + pub fn create_file(root: &Path, filename: &str) -> PathBuf { + let file_path = PathBuf::from(filename); + File::create(root.join(file_path.clone())).unwrap(); + file_path + } + + /// Create a directory in the root directory. + /// + /// Returns the relative path to the created directory based on the root directory. + pub fn create_dir(root: &Path, dirname: &str) -> PathBuf { + let dir_path = PathBuf::from(dirname); + std::fs::create_dir(root.join(dir_path.clone())).unwrap(); + dir_path + } +} diff --git a/mithril-aggregator/src/tools/file_size.rs b/internal/mithril-file-archiver/src/tools/file_size.rs similarity index 97% rename from mithril-aggregator/src/tools/file_size.rs rename to internal/mithril-file-archiver/src/tools/file_size.rs index d53be02d65b..f7ccbd9d446 100644 --- a/mithril-aggregator/src/tools/file_size.rs +++ b/internal/mithril-file-archiver/src/tools/file_size.rs @@ -1,3 +1,5 @@ +//! Tooling to compute File and Directory Sizes + use anyhow::Context; use std::{ collections::HashSet, @@ -7,7 +9,7 @@ use std::{ use mithril_common::StdResult; /// Compute the size of the given paths that could be files or folders. -pub(crate) fn compute_size(paths: Vec) -> StdResult { +pub fn compute_size(paths: Vec) -> StdResult { fn remove_duplicated_paths(paths: Vec) -> Vec { let mut result_folders = vec![]; let mut result_files = HashSet::new(); @@ -42,7 +44,7 @@ pub(crate) fn compute_size(paths: Vec) -> StdResult { /// Compute the size of one given path that could be a file or a folder. /// /// Returns 0 if the path is not a file or a folder. -pub(crate) fn compute_size_of_path(path: &Path) -> StdResult { +pub fn compute_size_of_path(path: &Path) -> StdResult { if path.is_file() { let metadata = std::fs::metadata(path) .with_context(|| format!("Failed to read metadata for file: {path:?}"))?; diff --git a/internal/mithril-file-archiver/src/tools/mod.rs b/internal/mithril-file-archiver/src/tools/mod.rs new file mode 100644 index 00000000000..ad45d2e7d2e --- /dev/null +++ b/internal/mithril-file-archiver/src/tools/mod.rs @@ -0,0 +1,3 @@ +//! Utilities used in file archiving + +pub mod file_size; diff --git a/internal/mithril-file-archiver/tests/extensions/helpers.rs b/internal/mithril-file-archiver/tests/extensions/helpers.rs new file mode 100644 index 00000000000..ee4b3116de6 --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/helpers.rs @@ -0,0 +1,170 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use mithril_common::entities::CompressionAlgorithm; +use mithril_common::temp_dir_create; +use mithril_file_archiver::{ArchiveParameters, FileArchiver}; + +/// Assert that two files have exactly the same bytes. +/// +/// SHA-256 hashes and file sizes are reported when they differ. +#[track_caller] +pub fn assert_files_are_byte_identical, A: AsRef>(expected: E, actual: A) { + let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); + let expected_bytes = read_file_bytes(expected_path); + let actual_bytes = read_file_bytes(actual_path); + + if expected_bytes != actual_bytes { + let expected_hash = hex::encode(Sha256::digest(&expected_bytes)); + let actual_hash = hex::encode(Sha256::digest(&actual_bytes)); + + panic!( + "Files are not byte-identical:\n\ + expected: '{}' ({} bytes, SHA-256: {})\n\ + actual: '{}' ({} bytes, SHA-256: {})", + expected_path.display(), + expected_bytes.len(), + expected_hash, + actual_path.display(), + actual_bytes.len(), + actual_hash, + ); + } +} + +/// Assert that two files are different. +/// +/// SHA-256 hashes and file sizes are reported when they are equal. +#[track_caller] +pub fn assert_files_are_byte_different, A: AsRef>(expected: E, actual: A) { + let (expected_path, actual_path) = (expected.as_ref(), actual.as_ref()); + let expected_bytes = read_file_bytes(expected_path); + let actual_bytes = read_file_bytes(actual_path); + + if expected_bytes == actual_bytes { + let hash = hex::encode(Sha256::digest(&expected_bytes)); + + panic!( + "Files are byte-identical:\n\ + expected: '{}'\n\ + actual: '{}'\n\ + bytes: {} bytes, SHA-256: {}", + expected_path.display(), + actual_path.display(), + expected_bytes.len(), + hash, + ); + } +} + +fn read_file_bytes(path: &Path) -> Vec { + std::fs::read(path) + .unwrap_or_else(|error| panic!("Could not read file '{}': {error}", path.display())) +} + +/// **IMPORTANT** Default zstandard compression parameters are used. +pub fn file_archiver(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + slog::Logger::root(slog::Discard, slog::o!()), + ) +} + +pub fn create_dir>(base_dir: &Path, dir_name: P) -> PathBuf { + let dir_path = base_dir.join(dir_name); + std::fs::create_dir(&dir_path).unwrap(); + dir_path +} + +pub fn create_file(root_dir: &Path, file_name: &str, size: Option) -> PathBuf { + let path = root_dir.join(file_name); + let mut file = File::create(&path).unwrap(); + + write!(file, "This is a test file named '{file_name}'").unwrap(); + writeln!(file).unwrap(); + + if let Some(file_size) = size { + file.set_len(file_size).unwrap(); + } + + path +} + +pub fn alter_file(path: &Path, alter_fn: F) { + let file = File::options().write(true).open(path).unwrap(); + alter_fn(&file); +} + +pub fn archive_parameters(filename: &str, target_dir: &Path) -> ArchiveParameters { + ArchiveParameters { + archive_name_without_extension: filename.to_string(), + target_directory: target_dir.to_path_buf(), + compression_algorithm: CompressionAlgorithm::Zstandard, + } +} + +pub fn compute_file_sha256(path: &Path) -> String { + let mut hasher = Sha256::new(); + + if path.is_file() { + hash_file(path, &mut hasher); + } else { + panic!("Path is not a file: {:?}", path.display()); + } + + hex::encode(hasher.finalize()) +} + +fn hash_file(path: &Path, hasher: &mut Sha256) { + let mut file = File::open(path).unwrap(); + let mut buffer = [0; 64 * 1024]; + + loop { + let bytes_read = file.read(&mut buffer).unwrap(); + if bytes_read == 0 { + break; + } + + hasher.update(&buffer[..bytes_read]); + } +} + +#[test] +fn assert_files_are_byte_identical_succeed_with_identical_files() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let identical_file_path = test_dir.join("altered_file"); + std::fs::copy(&reference_file_path, &identical_file_path).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &identical_file_path); +} + +#[test] +#[should_panic(expected = "Files are not byte-identical")] +fn assert_files_are_byte_identical_fails_if_one_byte_changes() { + let test_dir = temp_dir_create!(); + let content = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let altered_content = { + let mut altered_content = content.clone(); + altered_content[0] = !altered_content[0]; + altered_content + }; + + let reference_file_path = test_dir.join("reference_file"); + let mut reference_file = File::create_new(&reference_file_path).unwrap(); + reference_file.write_all(&content).unwrap(); + + let altered_file_path = test_dir.join("altered_file"); + let mut altered_file = File::create_new(&altered_file_path).unwrap(); + altered_file.write_all(&altered_content).unwrap(); + + assert_files_are_byte_identical(&reference_file_path, &altered_file_path); +} diff --git a/internal/mithril-file-archiver/tests/extensions/mod.rs b/internal/mithril-file-archiver/tests/extensions/mod.rs new file mode 100644 index 00000000000..128154a4ca4 --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/mod.rs @@ -0,0 +1,6 @@ +// Avoid clippy warnings generated by tests that doesn't use every tests_extensions (since each test +// is a different compilation target). +#![allow(dead_code)] + +pub mod helpers; +pub mod test_data; diff --git a/internal/mithril-file-archiver/tests/extensions/test_data.rs b/internal/mithril-file-archiver/tests/extensions/test_data.rs new file mode 100644 index 00000000000..ffe1f86ebbe --- /dev/null +++ b/internal/mithril-file-archiver/tests/extensions/test_data.rs @@ -0,0 +1,79 @@ +//! Fixed test data set for file archiver tests. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::Serialize; + +use crate::extensions::helpers; + +pub const TEST_BYTES: [u8; 10] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + +/// See: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks +pub(crate) const ZSTD_MAX_BLOCK_SIZE: u64 = 131_072; + +#[derive(Debug, PartialEq, Serialize)] +pub struct TestStruct { + field_str: String, + field_int: i32, + // Important note: using BTreeMap to ensure deterministic serialization, types like HashMap should + // not be used as the key order is not guaranteed + field_map: BTreeMap, +} + +impl Default for TestStruct { + fn default() -> Self { + Self { + field_str: "test".to_string(), + field_int: 42, + field_map: BTreeMap::from([ + ("key_3".to_string(), "value_3".to_string()), + ("key_1".to_string(), "value_1".to_string()), + ("key_2".to_string(), "value_2".to_string()), + ]), + } + } +} + +/// Create a file named `test.txt` in the given directory +pub fn create_test_txt(root_dir: &Path) -> PathBuf { + helpers::create_file(root_dir, "test.txt", Some(1048)) +} + +/// [AppenderEntries] ready list of entries created by [`create_test_dir`] +pub fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + ] +} + +/// Create a directory named `test_dir` in the given directory +/// +/// It will be filled with the following structure: +/// ```no_run +/// test_dir/ +/// ├── bar/ +/// ├── foo/ +/// │ └── bar.txt +/// ├── file_1.txt +/// ├── file_2.txt +/// ├── empty_file.txt +/// └── file_with_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt +/// ``` +pub fn create_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file(&test_dir, "file_2.txt", Some(ZSTD_MAX_BLOCK_SIZE * 2 + 1)); + + test_dir +} diff --git a/internal/mithril-file-archiver/tests/golden_master.rs b/internal/mithril-file-archiver/tests/golden_master.rs new file mode 100644 index 00000000000..56af53c94c6 --- /dev/null +++ b/internal/mithril-file-archiver/tests/golden_master.rs @@ -0,0 +1,220 @@ +//! The golden hashes are pinned to: the tar/zstd crate versions, GOLDEN_COMPRESSION_PARAMETERS +//! (multi-threaded zstd), and the exact bytes produced by `test_data::create_test_*`. +//! Any change to one of those invalidates them. + +mod extensions; + +use std::fs::File; +use std::path::{Path, PathBuf}; + +use mithril_common::temp_dir_create; + +use mithril_file_archiver::appender::*; +use mithril_file_archiver::{FileArchive, ZstandardCompressionParameters}; + +use extensions::*; + +// ** These hashes define TAR_ZSTD_V1. Update them only for an intentional archive-format change. ** +pub const TAR_ZSTD_V1_TEST_FILE_SHA256: &str = + "792b60f937bd348e5cfe8e4dc9fe7257b146888b8b30c52a547bd3ae4b7b1e4f"; +pub const TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256: &str = + "cf6c2755fe389e40891c5cb6e4b08e0f01e09846e58f229d4dfe138adc7d1157"; +pub const TAR_ZSTD_V1_TEST_DATA_SHA256: &str = + "fd5a178f1c717de39aef526d378e731871b74ee1ba11bdc159a4a872867d3748"; +pub const TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256: &str = + "6843b658c2c176afa156e82f7ee3d828f63cd7e9de16dc3166a5da96f0d3e15f"; +pub const TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256: &str = + "d2f8df8672bb6eca365d485445011bdd0adde34c978f4deded2cd548e97c24ab"; + +/// Create a directory named `test_dir` in the given directory +/// +/// It will be filled with the following structure: +/// ```no_run +/// test_dir/ +/// ├── bar/ +/// ├── foo/ +/// │ └── bar.txt +/// ├── empty_file.txt +/// ├── file_1.txt +/// ├── file_2.txt +/// └── file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt +/// ``` +pub fn create_golden_test_dir(root_dir: &Path) -> PathBuf { + let test_dir = helpers::create_dir(root_dir, "test_dir"); + + helpers::create_dir(&test_dir, "bar"); + + let foo_dir = helpers::create_dir(&test_dir, "foo"); + helpers::create_file(&foo_dir, "bar.txt", None); + + File::create(test_dir.join("empty.txt")).unwrap(); + + helpers::create_file(&test_dir, "file_1.txt", Some(511)); + helpers::create_file( + &test_dir, + "file_2.txt", + Some(test_data::ZSTD_MAX_BLOCK_SIZE * 2 + 1), + ); + helpers::create_file( + &test_dir, + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + None, + ); + + test_dir +} + +/// [AppenderEntries] ready list of entries created by [`create_golden_test_dir`] +pub fn golden_test_dir_entries() -> Vec { + vec![ + PathBuf::from("bar/"), + PathBuf::from("foo/"), + PathBuf::from("foo/bar.txt"), + PathBuf::from("empty.txt"), + PathBuf::from("file_1.txt"), + PathBuf::from("file_2.txt"), + PathBuf::from( + "file_with_a_very_very_long_name_a_name_longer_than_100_caracters_so_tar_switch_to_the_GNU_longname_extension.txt", + ), + ] +} + +#[track_caller] +fn assert_archive_not_empty(archive: &FileArchive) { + assert!( + archive.get_uncompressed_size() > 0, + "Archive '{}' has no content, fix the archive creation and try again", + archive.get_file_path().display() + ); +} + +/// Assert an archive matches its golden hash. +#[track_caller] +fn assert_archive_matches_golden_sha256(archive: &FileArchive, expected_sha256: &str) { + let actual_sha256 = helpers::compute_file_sha256(archive.get_file_path()); + + assert_eq!( + expected_sha256, + actual_sha256, + "Archive bytes changed ('{}', {} bytes).\n\ + Either the archive format is no longer reproducible, or the change is intentional \ + and the TAR_ZSTD_V* constants must be recomputed and their version bumped.", + archive.get_file_path().display(), + archive.get_archive_size(), + ); +} + +/// Compression parameters the golden hashes were computed with. +/// +/// They must mirror the production defaults, see [defaults_are_the_golden_parameters]. +const GOLDEN_COMPRESSION_PARAMETERS: ZstandardCompressionParameters = + ZstandardCompressionParameters { + level: 9, + number_of_workers: 4, + }; + +#[test] +fn defaults_are_the_golden_parameters() { + assert_eq!( + GOLDEN_COMPRESSION_PARAMETERS, + ZstandardCompressionParameters::default(), + "zstandard defaults changed: archives are no longer byte-compatible with the \ + previously published ones, the golden hashes must be recomputed and TAR_ZSTD_V1 \ + bumped to V2" + ); +} + +#[test] +fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_DATA_SHA256); +} + +#[test] +fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_RAW_BYTES_SHA256); +} + +#[test] +fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256(&archive, TAR_ZSTD_V1_TEST_FILE_SHA256); +} + +#[test] +fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + // Should construct the same archive as `AppenderDirAll` as we include all the source directory entries + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content).unwrap(), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_DIRECTORY_APPENDER_ENTRIES_SHA256, + ); +} + +#[test] +fn chain_appender() { + let test_dir = temp_dir_create!(); + let content = create_golden_test_dir(&helpers::create_dir(&test_dir, "source")); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("test", &test_dir), + AppenderEntries::new(golden_test_dir_entries(), content) + .unwrap() + .chain( + AppenderData::from_json( + PathBuf::from("test.json"), + &test_data::TestStruct::default(), + ) + .unwrap(), + ), + ) + .unwrap(); + + assert_archive_not_empty(&archive); + assert_archive_matches_golden_sha256( + &archive, + TAR_ZSTD_V1_TEST_CHAIN_DIRECTORY_AND_DATA_SHA256, + ); +} diff --git a/internal/mithril-file-archiver/tests/reproducibility.rs b/internal/mithril-file-archiver/tests/reproducibility.rs new file mode 100644 index 00000000000..bdc66d67ad8 --- /dev/null +++ b/internal/mithril-file-archiver/tests/reproducibility.rs @@ -0,0 +1,599 @@ +//! Reproducibility contract for archives produced by [`FileArchiver`]. +//! +//! Given identical archive entry paths and contents, `FileArchiver` must produce a +//! byte-identical `.tar.zst` archive regardless of the host system, source base +//! directory, creation time, modification times, permissions, or input entry order. +//! +//! This contract assumes identical archive-format dependencies and zstandard +//! compression parameters. Changing either is an archive-format change and requires +//! intentionally versioning the format and updating its golden hashes. + +mod extensions; + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use mithril_common::temp_dir_create; + +use mithril_file_archiver::appender::*; + +use extensions::*; + +mod repeated_archiving_produces_byte_identical_archives { + use std::time::Instant; + + use super::*; + + fn run_scenario(test_dir: PathBuf, ref_appender: T, repeated_appender: T) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || {}); + } + + fn run_scenario_in_different_unix_seconds( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + ) { + run_scenario_with_hook(test_dir, ref_appender, repeated_appender, || { + let seconds_since_unix_epoch = || { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + }; + + let reference_second = seconds_since_unix_epoch(); + let deadline = Instant::now() + Duration::from_millis(1500); + + while seconds_since_unix_epoch() == reference_second { + assert!( + Instant::now() < deadline, + "Unix time did not advance to the next second" + ); + std::thread::sleep(Duration::from_millis(10)); + } + }); + } + + fn run_scenario_with_hook( + test_dir: PathBuf, + ref_appender: T, + repeated_appender: T, + before_repeated_archive: F, + ) { + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + ref_appender, + ) + .unwrap(); + + before_repeated_archive(); + + let repeated_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("repeated", &test_dir), + repeated_appender, + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + repeated_archive.get_file_path(), + ); + } + + #[test] + fn appender_data_from_json() { + let test_dir = temp_dir_create!(); + let content = test_data::TestStruct::default(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + AppenderData::from_json(PathBuf::from("test_data.json"), &content).unwrap(), + ); + } + + #[test] + fn appender_data_from_raw_bytes() { + let test_dir = temp_dir_create!(); + let content = test_data::TEST_BYTES.to_vec(); + + // AppenderData create metadata itself when the archive is created, making it sensitive to the current unix time. + run_scenario_in_different_unix_seconds( + test_dir, + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content.clone()), + AppenderData::from_raw_bytes(PathBuf::from("bytes.txt"), content), + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_txt(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderFile::append_at_archive_root(content.clone()).unwrap(), + AppenderFile::append_at_archive_root(content).unwrap(), + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let content = test_data::create_test_dir(&helpers::create_dir(&test_dir, "source")); + + run_scenario( + test_dir, + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ); + } +} + +mod source_metadata_does_not_affect_archive_bytes { + use super::*; + + /// [AppenderEntries] ready list of the reference/altered dirs created by [setup_test_dirs]. + fn test_dir_entries() -> Vec { + vec![ + PathBuf::from("empty/"), + PathBuf::from("subdir/"), + PathBuf::from("subdir/file_2.txt"), + PathBuf::from("subdir/file_3.txt"), + PathBuf::from("file_1.txt"), + ] + } + + /// Create two directories with the same structure and applies the given function to each + /// reference/altered files pair. + /// + /// To be used with Appender that can work on a group of files and directories. + /// + /// Each directory contains the following files: + /// ```no_run + /// (reference|altered)/ + /// ├── empty/ + /// ├── subdir/ + /// │ ├── file_2.txt + /// │ └── file_3.txt + /// └── file_1.txt + /// ``` + fn setup_test_dirs( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + let subdirs = vec![Path::new("empty"), Path::new("subdir")]; + + for dir_path in &subdirs { + helpers::create_dir(&reference_dir, dir_path); + helpers::create_dir(&altered_dir, dir_path); + } + + for file_path in ["file_1.txt", "subdir/file_2.txt", "subdir/file_3.txt"] { + let reference_file = helpers::create_file(&reference_dir, file_path, None); + let altered_file = helpers::create_file(&altered_dir, file_path, None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + // Alter dirs after the files to avoid permission issues + for dir_path in &subdirs { + setup_reference_and_altered_path_fn( + &reference_dir.join(dir_path), + &altered_dir.join(dir_path), + ); + } + setup_reference_and_altered_path_fn(&reference_dir, &altered_dir); + } + + /// Create two directories with a single "test.txt" file and applies the given function + /// to the reference/altered file pair. + /// + /// To be used with Appender that can work on a single file. + fn setup_test_file( + test_dir: &Path, + setup_reference_and_altered_path_fn: F, + ) { + let source = helpers::create_dir(test_dir, "source"); + let reference_dir = helpers::create_dir(&source, "reference"); + let altered_dir = helpers::create_dir(&source, "altered"); + + let reference_file = helpers::create_file(&reference_dir, "file.txt", None); + let altered_file = helpers::create_file(&altered_dir, "file.txt", None); + + // Ensure the two files have the same content + helpers::assert_files_are_byte_identical(&reference_file, &altered_file); + + setup_reference_and_altered_path_fn(&reference_file, &altered_file); + } + + fn run_scenario A>(test_dir: PathBuf, build_tar_appender: B) { + let source = test_dir.join("source"); + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + build_tar_appender(source.join("reference")), + ) + .unwrap(); + + let archive_with_different_metadata = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("altered_metadata", &test_dir), + build_tar_appender(source.join("altered")), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_different_metadata.get_file_path(), + ); + } + + mod modification_time { + use super::*; + + fn setup_modification_time(reference_path: &Path, path_with_different_metadata: &Path) { + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + // can't modify dir times until + // is stabilized (currently planned for rust 1.99) + return; + } + + let base_time = SystemTime::UNIX_EPOCH; + helpers::alter_file(reference_path, |file| file.set_modified(base_time).unwrap()); + helpers::alter_file(path_with_different_metadata, |file| { + // note: in TAR, entries mtimes have a granularity of a second + file.set_modified(base_time + Duration::from_millis(5300)).unwrap() + }); + + assert_ne!( + reference_path.metadata().unwrap().modified().unwrap(), + path_with_different_metadata.metadata().unwrap().modified().unwrap() + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_modification_time); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } + + #[cfg(unix)] + mod permissions { + use std::fs::Permissions; + + use super::*; + + fn setup_permissions(reference_path: &Path, path_with_different_metadata: &Path) { + use std::os::unix::fs::PermissionsExt; + + let (reference_permission, altered_permission) = + // IMPORTANT: for directory the owner permission must be `7`, else this prevents + // `temp_dir_create` cleanup and make appenders fails on subdir files + if reference_path.is_dir() || path_with_different_metadata.is_dir() { + (Permissions::from_mode(0o766), Permissions::from_mode(0o767)) + } else { + (Permissions::from_mode(0o644), Permissions::from_mode(0o646)) + }; + + std::fs::set_permissions(reference_path, reference_permission).unwrap(); + std::fs::set_permissions(path_with_different_metadata, altered_permission).unwrap(); + + assert_ne!( + reference_path.metadata().unwrap().permissions(), + path_with_different_metadata.metadata().unwrap().permissions() + ); + } + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + setup_test_file(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderFile::append_at_archive_root(source.join("file.txt")).unwrap() + }) + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + setup_test_dirs(&test_dir, setup_permissions); + run_scenario(test_dir, |source| { + AppenderEntries::new(test_dir_entries(), source).unwrap() + }); + } + } +} + +mod source_base_directory_does_not_affect_archive { + use super::*; + + #[test] + fn appender_file() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_txt(&subdir_1); + let same_content_in_other_dir = test_data::create_test_txt(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderFile::append_at_archive_root(content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderFile::append_at_archive_root(same_content_in_other_dir).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), + ); + } + + #[test] + fn appender_entries() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let subdir_1 = helpers::create_dir(&source, "first"); + let subdir_2 = helpers::create_dir(&source, "second"); + + let content = test_data::create_test_dir(&subdir_1); + let same_content_in_other_dir = test_data::create_test_dir(&subdir_2); + + let archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content).unwrap(), + ) + .unwrap(); + let archive_with_same_content_but_from_another_dir = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("from_another_dir", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), same_content_in_other_dir) + .unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + archive.get_file_path(), + archive_with_same_content_but_from_another_dir.get_file_path(), + ); + } +} + +mod appender_entry_specifics { + use super::*; + + fn to_entries(paths: [&str; N]) -> Vec { + paths.into_iter().map(PathBuf::from).collect() + } + + #[test] + fn equivalent_entry_paths_produce_identical_archives() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_entries = ["bar/", "foo/", "foo/bar.txt", "file_1.txt", "file_2.txt"]; + let equivalent_entries = ["bar", "foo", "foo/bar.txt", "./file_1.txt", "file_2.txt"]; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(to_entries(reference_entries), content.clone()).unwrap(), + ) + .unwrap(); + + let archive_with_equivalent_entries_spelling = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("equivalent_spelling", &test_dir), + AppenderEntries::new(to_entries(equivalent_entries), content).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_equivalent_entries_spelling.get_file_path(), + ); + } + + #[test] + fn supplied_entry_order_does_not_affect_appender_entries_archive() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let content = test_data::create_test_dir(&source); + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderEntries::new(test_data::test_dir_entries(), content.clone()).unwrap(), + ) + .unwrap(); + + for (label, entries) in [ + ( + "child_before_parent_dir", + ["bar/", "foo/bar.txt", "foo/", "file_1.txt", "file_2.txt"], + ), + ( + "directories_before_files", + ["foo/", "bar/", "file_2.txt", "file_1.txt", "foo/bar.txt"], + ), + ( + "files_before_directories", + ["file_2.txt", "file_1.txt", "foo/bar.txt", "foo/", "bar/"], + ), + ( + "reverse_dirs_order", + ["foo/", "bar/", "foo/bar.txt", "file_1.txt", "file_2.txt"], + ), + ( + "reverse_files_order", + ["bar/", "foo/", "foo/bar.txt", "file_2.txt", "file_1.txt"], + ), + ] { + let archive_with_same_content_but_different_entries_order = + helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters(label, &test_dir), + AppenderEntries::new(to_entries(entries), content.clone()).unwrap(), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_with_same_content_but_different_entries_order.get_file_path(), + ); + } + } +} + +mod chain_specifics { + use super::*; + + #[test] + fn chaining_non_overlapping_appenders_is_commutative() { + let test_dir = temp_dir_create!(); + let source = helpers::create_dir(&test_dir, "source"); + let file_content = test_data::create_test_txt(&source); + let data_content = test_data::TEST_BYTES; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderFile::append_at_archive_root(file_content.clone()) + .unwrap() + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + data_content.to_vec(), + )), + ) + .unwrap(); + + let archive_chained_in_reverse = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("archive_chained_in_reverse", &test_dir), + AppenderData::from_raw_bytes(PathBuf::from("data.bytes"), data_content.to_vec()) + .chain(AppenderFile::append_at_archive_root(file_content.clone()).unwrap()), + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + reference_archive.get_file_path(), + archive_chained_in_reverse.get_file_path(), + ); + } + + #[test] + // Commutativity of chaining appenders is broken if they have overlapping paths as only the path + // from the rightmost appender is used. + fn chaining_overlapping_appenders_is_not_commutative() { + let test_dir = temp_dir_create!(); + let path_in_archive = PathBuf::from("data.bytes"); + let first_data_content = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + let second_data_content = [11, 12, 13, 14, 15, 16, 17, 18, 18, 19]; + + let reference_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("reference", &test_dir), + AppenderData::from_raw_bytes(path_in_archive.clone(), first_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + second_data_content.to_vec(), + )), + ) + .unwrap(); + + let archive_chained_in_reverse = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("archive_chained_in_reverse", &test_dir), + AppenderData::from_raw_bytes(path_in_archive.clone(), second_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("data.bytes"), + first_data_content.to_vec(), + )), + ) + .unwrap(); + + helpers::assert_files_are_byte_different( + reference_archive.get_file_path(), + archive_chained_in_reverse.get_file_path(), + ); + } + + #[test] + fn chaining_non_overlapping_appenders_is_associative() { + let test_dir = temp_dir_create!(); + let a_data_content = [1, 2, 3, 4, 5]; + let b_data_content = [6, 7, 8, 9, 10]; + let c_data_content = [11, 12, 13, 14, 15]; + + let left_grouped = + AppenderData::from_raw_bytes(PathBuf::from("c.txt"), c_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("a.txt"), + a_data_content.to_vec(), + )) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("b.txt"), + b_data_content.to_vec(), + )); + + let right_grouped = + AppenderData::from_raw_bytes(PathBuf::from("c.txt"), c_data_content.to_vec()).chain( + AppenderData::from_raw_bytes(PathBuf::from("a.txt"), a_data_content.to_vec()) + .chain(AppenderData::from_raw_bytes( + PathBuf::from("b.txt"), + b_data_content.to_vec(), + )), + ); + + let left_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("left-grouped", &test_dir), + left_grouped, + ) + .unwrap(); + + let right_archive = helpers::file_archiver(&test_dir) + .archive( + helpers::archive_parameters("right-grouped", &test_dir), + right_grouped, + ) + .unwrap(); + + helpers::assert_files_are_byte_identical( + left_archive.get_file_path(), + right_archive.get_file_path(), + ); + } +} diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 41c42557ba2..34afc261c6b 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -33,6 +33,7 @@ mithril-common = { path = "../mithril-common" } mithril-dmq = { path = "../internal/mithril-dmq" } mithril-doc = { path = "../internal/mithril-doc" } mithril-era = { path = "../internal/mithril-era" } +mithril-file-archiver = { path = "../internal/mithril-file-archiver" } mithril-metric = { path = "../internal/mithril-metric" } mithril-persistence = { path = "../internal/mithril-persistence" } mithril-protocol-config = { path = "../internal/mithril-protocol-config" } @@ -53,13 +54,11 @@ slog = { workspace = true, features = ["max_level_trace", "release_max_level_deb slog-async = { workspace = true } slog-bunyan = { workspace = true } sqlite = { version = "0.37.0", features = ["bundled"] } -tar = "0.4.46" thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } tokio-util = { version = "0.7.18", features = ["codec"] } uuid = { version = "1.23.4", features = ["v4", "fast-rng", "macro-diagnostics"] } warp = { workspace = true } -zstd = { version = "0.13.3", features = ["zstdmt"] } [target.'cfg(all(not(target_os = "windows"), not(all(target_os = "linux", target_arch = "aarch64"))))'.dependencies] tikv-jemallocator = { version = "0.7.0" } @@ -76,11 +75,9 @@ http = "1.4.2" httpmock = "0.8.3" mithril-api-spec = { path = "../internal/tests/mithril-api-spec" } mithril-common = { path = "../mithril-common", features = ["allow_skip_signer_certification"] } -mithril-test-http-server = { path = "../internal/tests/mithril-test-http-server" } mockall = { workspace = true } slog-scope = "4.4.1" slog-term = { workspace = true } -tempfile = "3.27.0" warp = { workspace = true, features = ["test"] } [[bench]] diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index 33fbe71d838..86e120481f4 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -104,6 +104,7 @@ mod tests { use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest; use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutable_trio_names}; + use mithril_file_archiver::FileArchiver; use mithril_common::{ CardanoNetwork, @@ -126,7 +127,7 @@ mod tests { services::CompressedArchiveSnapshotter, services::ancillary_signer::MockAncillarySigner, test::TestLogger, - tools::{file_archiver::FileArchiver, url_sanitizer::SanitizedUrlWithTrailingSlash}, + tools::url_sanitizer::SanitizedUrlWithTrailingSlash, }; use super::*; @@ -201,7 +202,10 @@ mod tests { cardano_db.get_dir().to_path_buf(), test_dir.join("ongoing_snapshots"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(FileArchiver::new_with_default_parameters( + test_dir.join("verification"), + TestLogger::stdout(), + )), Arc::new(MockAncillarySigner::that_succeeds_with_signature( ancillary_manifest_signature, )), @@ -271,8 +275,9 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("http://aggregator_uri").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test( + file_archiver: Arc::new(FileArchiver::new_with_default_parameters( test_dir.join("verification"), + TestLogger::stdout(), )), target_location: test_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs index 03e9f6494c1..4c07c17ef97 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs @@ -9,12 +9,12 @@ use mithril_common::{ entities::{AncillaryLocation, CardanoDbBeacon, CompressionAlgorithm}, logging::LoggerExtensions, }; +use mithril_file_archiver::FileArchive; use crate::{ DumbUploader, FileUploader, file_uploaders::{CloudUploader, LocalUploader}, services::Snapshotter, - tools::file_archiver::FileArchive, }; /// The [AncillaryFileUploader] trait allows identifying uploaders that return locations for ancillary archive files. diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs index 36dd8b368d2..885d84f7b3b 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs @@ -6,21 +6,20 @@ use std::{ use anyhow::Context; use async_trait::async_trait; +use slog::{Logger, error}; + use mithril_common::{ CardanoNetwork, StdResult, entities::{CardanoDbBeacon, CompressionAlgorithm, DigestLocation}, logging::LoggerExtensions, messages::CardanoDatabaseDigestListItemMessage, }; -use slog::{Logger, error}; +use mithril_file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::AppenderFile}; use crate::{ DumbUploader, FileUploader, ImmutableFileDigestMapper, file_uploaders::{CloudUploader, LocalUploader}, - tools::{ - file_archiver::{ArchiveParameters, FileArchive, FileArchiver, appender::AppenderFile}, - url_sanitizer::SanitizedUrlWithTrailingSlash, - }, + tools::url_sanitizer::SanitizedUrlWithTrailingSlash, }; /// The [DigestFileUploader] trait allows identifying uploaders that return locations for digest files. @@ -288,12 +287,7 @@ impl DigestArtifactBuilder { #[cfg(test)] mod tests { use anyhow::anyhow; - use std::{ - collections::BTreeMap, - fs::{File, read_to_string}, - }; - use tar::Archive; - use zstd::Decoder; + use std::{collections::BTreeMap, fs::read_to_string}; use mithril_common::{ current_function, @@ -301,11 +295,11 @@ mod tests { messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage}, test::{TempDir, assert_equivalent, double::Dummy}, }; + use mithril_file_archiver::{FileArchiver, test::unpack_archive}; use crate::{ file_uploaders::FileUploadRetryPolicy, immutable_file_digest_mapper::MockImmutableFileDigestMapper, test::TestLogger, - tools::file_archiver::FileArchiver, }; use super::*; @@ -360,15 +354,11 @@ mod tests { immutable_file_digest_mapper } - fn unpack_archive(archive_path: &Path, unpack_dir: &Path) -> StdResult<()> { - let mut archive = { - let file_tar_zst = File::open(archive_path)?; - let file_tar_zst_decoder = Decoder::new(file_tar_zst)?; - Archive::new(file_tar_zst_decoder) - }; - - archive.unpack(unpack_dir)?; - Ok(()) + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) } #[tokio::test] @@ -384,7 +374,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -417,7 +407,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -451,7 +441,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -476,7 +466,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -509,7 +499,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -549,7 +539,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -597,7 +587,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(temp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&temp_dir)), target_location: temp_dir.clone(), compression_algorithm: CompressionAlgorithm::Zstandard, }, @@ -639,7 +629,7 @@ mod tests { SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(build_local_uploader(&uploader_path))], DigestSnapshotter { - file_archiver: Arc::new(FileArchiver::new_for_test(tmp_dir.join("verification"))), + file_archiver: Arc::new(file_archiver_for_test(&tmp_dir)), target_location: digests_archive_dir.clone(), compression_algorithm, }, diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs index 7d83f5fe17a..70203f73e74 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/immutable.rs @@ -282,12 +282,12 @@ mod tests { entities::TemplateUri, test::{TempDir, assert_equivalent, equivalent_to}, }; + use mithril_file_archiver::FileArchiver; use semver::Version; use crate::services::ancillary_signer::MockAncillarySigner; use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter}; use crate::test::TestLogger; - use crate::tools::file_archiver::FileArchiver; use super::*; @@ -329,6 +329,13 @@ mod tests { uploader } + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + fn create_fake_file(path: &Path, content: &str) { let mut file = File::create(path).unwrap(); write!(file, "{content}").unwrap(); @@ -360,7 +367,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -454,7 +461,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -501,7 +508,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -536,7 +543,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -569,7 +576,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -626,7 +633,7 @@ mod tests { db_directory.clone(), db_directory.parent().unwrap().join("snapshot_dest"), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))), + Arc::new(file_archiver_for_test(&work_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs index 4ea033f231a..3eb89039aca 100644 --- a/mithril-aggregator/src/configuration.rs +++ b/mithril-aggregator/src/configuration.rs @@ -22,6 +22,7 @@ use mithril_common::{AggregateSignatureType, CardanoNetwork, StdResult}; use mithril_dmq::DmqNetwork; use mithril_doc::{Documenter, DocumenterDefault, StructDoc}; use mithril_era::adapters::EraReaderAdapterType; +use mithril_file_archiver::ZstandardCompressionParameters; use crate::entities::AggregatorEpochSettings; use crate::http_server::SERVER_BASE_PATH; @@ -706,25 +707,6 @@ pub enum SnapshotUploaderType { Local, } -/// [Zstandard][CompressionAlgorithm::Zstandard] specific parameters -#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] -pub struct ZstandardCompressionParameters { - /// Level of compression, default to 9. - pub level: i32, - - /// Number of workers when compressing, 0 will disable multithreading, default to 4. - pub number_of_workers: u32, -} - -impl Default for ZstandardCompressionParameters { - fn default() -> Self { - Self { - level: 9, - number_of_workers: 4, - } - } -} - /// Configuration to connect to the Blockfrost API. /// /// Currently only used to fetch the ticker and name for registered pools. diff --git a/mithril-aggregator/src/dependency_injection/builder/mod.rs b/mithril-aggregator/src/dependency_injection/builder/mod.rs index 9ddd29e84d8..e7b4c58ba7e 100644 --- a/mithril-aggregator/src/dependency_injection/builder/mod.rs +++ b/mithril-aggregator/src/dependency_injection/builder/mod.rs @@ -33,6 +33,7 @@ use mithril_common::{ signable_builder::{SignableBuilderService, SignableSeedBuilder}, }; use mithril_era::{EraChecker, EraReader, EraReaderAdapter}; +use mithril_file_archiver::FileArchiver; use mithril_persistence::sqlite::{SqliteConnection, SqliteConnectionPool}; use mithril_protocol_config::interface::{ MithrilNetworkConfigurationProvider, ProtocolConfigurationMarkersReader, @@ -61,7 +62,6 @@ use crate::{ SignedEntityService, SignerSynchronizer, Snapshotter, StakeDistributionService, UpkeepService, }, - tools::file_archiver::FileArchiver, }; use super::{ diff --git a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs index 4b70fd37c07..c579f2d2320 100644 --- a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs +++ b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use std::sync::Arc; use mithril_common::crypto_helper::ManifestSigner; +use mithril_file_archiver::FileArchiver; use crate::artifact_builder::{ AncillaryArtifactBuilder, AncillaryFileUploader, CardanoBlocksTransactionsArtifactBuilder, @@ -28,7 +29,6 @@ use crate::services::{ SignedEntityServiceArtifactsDependencies, Snapshotter, }; use crate::tools::DEFAULT_GCP_CREDENTIALS_JSON_ENV_VAR; -use crate::tools::file_archiver::FileArchiver; use crate::{DumbUploader, ExecutionEnvironment, FileUploader, SnapshotUploaderType}; impl DependenciesBuilder { diff --git a/mithril-aggregator/src/lib.rs b/mithril-aggregator/src/lib.rs index 939265118c2..0425fc455f0 100644 --- a/mithril-aggregator/src/lib.rs +++ b/mithril-aggregator/src/lib.rs @@ -34,7 +34,7 @@ mod tools; pub use crate::artifact_builder::ArtifactBuilder; pub use crate::configuration::{ ConfigurationSource, DefaultConfiguration, ExecutionEnvironment, ServeCommandConfiguration, - SnapshotUploaderType, ZstandardCompressionParameters, + SnapshotUploaderType, }; pub use crate::multi_signer::{MultiSigner, MultiSignerImpl}; pub use commands::{CommandType, MainOpts}; diff --git a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs index 2473397ab86..8327477c949 100644 --- a/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs +++ b/mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs @@ -13,11 +13,13 @@ use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutabl use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; use mithril_common::logging::LoggerExtensions; +use mithril_file_archiver::{ + ArchiveParameters, FileArchive, FileArchiver, + appender::{AppenderData, AppenderEntries, ArchiveEntryProvider, TarAppender}, + tools::file_size, +}; use crate::dependency_injection::DependenciesBuilderError; -use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender}; -use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver}; -use crate::tools::file_size; use super::{Snapshotter, ancillary_signer::AncillarySigner}; @@ -318,6 +320,7 @@ mod tests { use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; use mithril_common::test::assert_equivalent; use mithril_common::{assert_dir_eq, current_function, temp_dir_create}; + use mithril_file_archiver::test::FileArchiveTestExtension; use crate::services::ancillary_signer::MockAncillarySigner; use crate::test::TestLogger; @@ -333,6 +336,13 @@ mod tests { .collect() } + fn file_archiver_for_test(work_dir: &Path) -> FileArchiver { + FileArchiver::new_with_default_parameters( + work_dir.join("verification"), + TestLogger::stdout(), + ) + } + fn snapshotter_for_test( test_directory: &Path, db_directory: &Path, @@ -343,9 +353,7 @@ mod tests { db_directory.to_path_buf(), test_directory.join("ongoing_snapshot"), compression_algorithm, - Arc::new(FileArchiver::new_for_test( - test_directory.join("verification"), - )), + Arc::new(file_archiver_for_test(test_directory)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -378,7 +386,7 @@ mod tests { db_directory, ongoing_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -402,7 +410,7 @@ mod tests { db_directory, ongoing_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) @@ -425,7 +433,7 @@ mod tests { cardano_db.get_dir().clone(), pending_snapshot_directory.clone(), CompressionAlgorithm::Zstandard, - Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))), + Arc::new(file_archiver_for_test(&test_dir)), Arc::new(MockAncillarySigner::new()), TestLogger::stdout(), ) diff --git a/mithril-aggregator/src/services/snapshotter/interface.rs b/mithril-aggregator/src/services/snapshotter/interface.rs index 08584c15c3a..03f4bda22fc 100644 --- a/mithril-aggregator/src/services/snapshotter/interface.rs +++ b/mithril-aggregator/src/services/snapshotter/interface.rs @@ -2,8 +2,7 @@ use async_trait::async_trait; use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; - -use crate::tools::file_archiver::FileArchive; +use mithril_file_archiver::FileArchive; #[cfg_attr(test, mockall::automock)] #[async_trait] diff --git a/mithril-aggregator/src/services/snapshotter/test_doubles.rs b/mithril-aggregator/src/services/snapshotter/test_doubles.rs index ab2765cb663..dded8e14b17 100644 --- a/mithril-aggregator/src/services/snapshotter/test_doubles.rs +++ b/mithril-aggregator/src/services/snapshotter/test_doubles.rs @@ -6,9 +6,9 @@ use std::sync::RwLock; use mithril_common::StdResult; use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber}; +use mithril_file_archiver::FileArchive; use crate::services::Snapshotter; -use crate::tools::file_archiver::FileArchive; /// Snapshotter that does nothing. It is mainly used for test purposes. pub struct DumbSnapshotter { diff --git a/mithril-aggregator/src/tools/file_archiver/appender.rs b/mithril-aggregator/src/tools/file_archiver/appender.rs deleted file mode 100644 index 1e7b4de95ea..00000000000 --- a/mithril-aggregator/src/tools/file_archiver/appender.rs +++ /dev/null @@ -1,690 +0,0 @@ -use anyhow::{Context, anyhow}; -use serde::Serialize; -use std::fs::File; -use std::io::Write; -use std::path::PathBuf; - -use mithril_common::StdResult; - -use crate::tools::file_size; - -const READ_WRITE_PERMISSION: u32 = 0o666; - -/// Define multiple ways to append content to a tar archive. -pub trait TarAppender: Send { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()>; - - fn compute_uncompressed_data_size(&self) -> StdResult; - - fn chain(self, appender_right: A2) -> ChainAppender - where - Self: Sized, - { - ChainAppender::new(self, appender_right) - } -} - -#[cfg(test)] -pub struct AppenderDirAll { - target_directory: PathBuf, -} -#[cfg(test)] -impl AppenderDirAll { - // Note: Not used anymore outside of tests but useful tool to keep around if we ever need to archive a directory - pub fn new(target_directory: PathBuf) -> Self { - Self { target_directory } - } -} - -#[cfg(test)] -impl TarAppender for AppenderDirAll { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - tar.append_dir_all(".", &self.target_directory).with_context(|| { - format!( - "Create archive error: Can not add directory: '{}' to the archive", - self.target_directory.display() - ) - })?; - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - file_size::compute_size_of_path(&self.target_directory) - } -} - -pub struct AppenderFile { - /// Location of the file in the archive. - location_in_archive: PathBuf, - /// Path to the file to add to the archive. - target_file: PathBuf, -} - -impl AppenderFile { - /// Append the file at the root of the archive, keeping the same file name. - pub fn append_at_archive_root(target_file: PathBuf) -> StdResult { - if !target_file.is_file() { - return Err(anyhow!( - "The target file is not a file, path: {}", - target_file.display() - )); - } - - let location_in_archive = target_file - .file_name() - .with_context(|| { - format!( - "Can not get the file name from the target file path: '{}'", - target_file.display() - ) - })? - .to_owned(); - - Ok(Self { - location_in_archive: PathBuf::from(location_in_archive), - target_file, - }) - } -} - -impl TarAppender for AppenderFile { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - let mut file = File::open(&self.target_file) - .with_context(|| format!("Can not open file: '{}'", self.target_file.display()))?; - tar.append_file(&self.location_in_archive, &mut file) - .with_context(|| { - format!( - "Can not add file: '{}' to the archive", - self.target_file.display() - ) - })?; - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - file_size::compute_size_of_path(&self.target_file) - } -} - -pub struct AppenderEntries { - entries: Vec, - base_directory: PathBuf, -} - -impl AppenderEntries { - /// Create a new instance of `AppenderEntries`. - /// - /// Returns an error if the `entries` are empty. - pub fn new(entries: Vec, base_directory: PathBuf) -> StdResult { - if entries.is_empty() { - return Err(anyhow!("The entries can not be empty")); - } - - Ok(Self { - entries, - base_directory, - }) - } -} - -impl TarAppender for AppenderEntries { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - for entry in &self.entries { - let entry_path = self.base_directory.join(entry); - if entry_path.is_dir() { - tar.append_dir_all(entry, entry_path.clone()).with_context(|| { - format!( - "Can not add directory: '{}' to the archive", - entry_path.display() - ) - })?; - } else if entry_path.is_file() { - let mut file = File::open(entry_path.clone())?; - tar.append_file(entry, &mut file).with_context(|| { - format!( - "Can not add file: '{}' to the archive", - entry_path.display() - ) - })?; - } else { - return Err(anyhow!( - "The entry: '{}' is not valid", - entry_path.display() - )); - } - } - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - let full_entries_path = self - .entries - .iter() - .map(|entry| self.base_directory.join(entry)) - .collect(); - file_size::compute_size(full_entries_path) - } -} - -/// Append data to the archive. -pub struct AppenderData { - /// Location of the file in the archive where the data will be appended. - location_in_archive: PathBuf, - /// Byte array of the data to append. - bytes: Vec, -} - -impl AppenderData { - /// Create a new instance of `AppenderData` from an object that will be serialized to JSON. - pub fn from_json( - location_in_archive: PathBuf, - object: &T, - ) -> StdResult { - let json_bytes = serde_json::to_vec(object).with_context(|| { - format!( - "Can not serialize JSON to file in archive: {:?}", - location_in_archive.display() - ) - })?; - - Ok(Self::from_raw_bytes(location_in_archive, json_bytes)) - } - - /// Create a new instance of `AppenderData` from a byte array. - pub fn from_raw_bytes(location_in_archive: PathBuf, bytes: Vec) -> Self { - Self { - location_in_archive, - bytes, - } - } -} - -impl TarAppender for AppenderData { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - let mut header = tar::Header::new_gnu(); - header.set_size(self.bytes.len() as u64); - header.set_mode(READ_WRITE_PERMISSION); - header.set_mtime(chrono::Utc::now().timestamp() as u64); - header.set_cksum(); - - tar.append_data( - &mut header, - &self.location_in_archive, - self.bytes.as_slice(), - ) - .with_context(|| { - format!( - "Can not add file: '{}' to the archive", - self.location_in_archive.display() - ) - })?; - - Ok(()) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - Ok(self.bytes.len() as u64) - } -} - -/// Chain multiple `TarAppender` instances together. -pub struct ChainAppender { - appender_left: L, - appender_right: R, -} - -impl ChainAppender { - pub fn new(appender_left: L, appender_right: R) -> Self { - Self { - appender_left, - appender_right, - } - } -} - -impl TarAppender for ChainAppender { - fn append(&self, tar: &mut tar::Builder) -> StdResult<()> { - self.appender_left.append(tar)?; - self.appender_right.append(tar) - } - - fn compute_uncompressed_data_size(&self) -> StdResult { - // Size is aggregated even if the data is overwritten by the right appender because we - // can't know if there is an overlap or not - Ok(self.appender_left.compute_uncompressed_data_size()? - + self.appender_right.compute_uncompressed_data_size()?) - } -} - -#[cfg(test)] -mod tests { - use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder; - use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR}; - use mithril_common::entities::CompressionAlgorithm; - use mithril_common::{assert_dir_eq, temp_dir_create}; - - use crate::tools::file_archiver::test_tools::*; - use crate::tools::file_archiver::{ArchiveParameters, FileArchiver}; - - use super::*; - - mod appender_entries { - use super::*; - - #[test] - fn create_archive_only_for_specified_directories_and_files() { - let test_dir = temp_dir_create!(); - let source = test_dir.join(create_dir(&test_dir, "source")); - - let directory_to_archive_path = create_dir(&source, "directory_to_archive"); - let file_to_archive_path = create_file(&source, "file_to_archive.txt"); - create_dir(&source, "directory_not_to_archive"); - create_file(&source, "file_not_to_archive.txt"); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - AppenderEntries::new( - vec![directory_to_archive_path.clone(), file_to_archive_path.clone()], - source, - ) - .unwrap(), - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - - assert_dir_eq!( - &unpack_path, - "* directory_to_archive/ - * file_to_archive.txt" - ); - } - - #[test] - fn return_error_when_appending_file_or_directory_that_does_not_exist() { - let test_dir = temp_dir_create!(); - let target_archive = test_dir.join("whatever.tar.zst"); - let source = test_dir.join(create_dir(&test_dir, "source")); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - - file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - AppenderEntries::new(vec![PathBuf::from("not_exist")], source).unwrap(), - ) - .expect_err("AppenderEntries should return error when file or directory not exist"); - assert!(!target_archive.exists()); - } - - #[test] - fn return_error_when_appending_empty_entries() { - let appender_creation_result = AppenderEntries::new(vec![], PathBuf::new()); - assert!(appender_creation_result.is_err(),); - } - - #[test] - fn can_append_duplicate_files_and_directories() { - let test_dir = temp_dir_create!(); - let source = test_dir.join(create_dir(&test_dir, "source")); - - let directory_to_archive_path = create_dir(&source, "directory_to_archive"); - let file_to_archive_path = - create_file(&source, "directory_to_archive/file_to_archive.txt"); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - AppenderEntries::new( - vec![ - directory_to_archive_path.clone(), - directory_to_archive_path.clone(), - file_to_archive_path.clone(), - file_to_archive_path.clone(), - ], - source, - ) - .unwrap(), - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - - assert_dir_eq!( - &unpack_path, - "* directory_to_archive/ - ** file_to_archive.txt" - ); - } - - #[test] - fn compute_uncompressed_size_of_its_paths() { - let test_dir = "compute_uncompressed_size_of_its_paths"; - - let immutable_trio_file_size = 777; - let ledger_file_size = 6666; - let volatile_file_size = 99; - - let cardano_db = DummyCardanoDbBuilder::new(test_dir) - .with_immutables(&[1, 2, 3]) - .set_immutable_trio_file_size(immutable_trio_file_size) - .with_legacy_ledger_snapshots(&[437, 537, 637, 737]) - .set_ledger_file_size(ledger_file_size) - .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"]) - .set_volatile_file_size(volatile_file_size) - .build(); - - let appender_entries = AppenderEntries::new( - vec![ - PathBuf::from(IMMUTABLE_DIR), - PathBuf::from(LEDGER_DIR).join("437"), - PathBuf::from(LEDGER_DIR).join("537"), - PathBuf::from(VOLATILE_DIR).join("blocks-0.dat"), - ], - cardano_db.get_dir().clone(), - ) - .unwrap(); - - let entries_size = appender_entries.compute_uncompressed_data_size().unwrap(); - let expected_total_size = - (immutable_trio_file_size * 3) + (2 * ledger_file_size) + volatile_file_size; - assert_eq!(expected_total_size, entries_size); - } - } - - mod appender_file { - use super::*; - - #[test] - fn appending_file_to_tar() { - let test_dir = temp_dir_create!(); - let file_to_archive = create_file(&test_dir, "test_file.txt"); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - AppenderFile::append_at_archive_root(test_dir.join(&file_to_archive)).unwrap(), - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - - assert!(unpack_path.join(file_to_archive).exists()); - } - - #[test] - fn return_error_if_file_does_not_exist() { - let target_file_path = PathBuf::from("non_existent_file.txt"); - assert!(AppenderFile::append_at_archive_root(target_file_path).is_err()); - } - - #[test] - fn return_error_if_input_is_not_a_file() { - let test_dir = temp_dir_create!(); - assert!(AppenderFile::append_at_archive_root(test_dir).is_err()); - } - - #[test] - fn compute_uncompressed_size() { - let test_dir = temp_dir_create!(); - - let file_path = test_dir.join("file.txt"); - let file = File::create(&file_path).unwrap(); - file.set_len(777).unwrap(); - - let appender_file = AppenderFile::append_at_archive_root(file_path).unwrap(); - - let entries_size = appender_file.compute_uncompressed_data_size().unwrap(); - assert_eq!(777, entries_size); - } - } - - mod appender_dir_all { - use super::*; - - #[test] - fn compute_uncompressed_size() { - let test_dir = "appender_dir_all_compute_size"; - - let immutable_trio_file_size = 777; - let ledger_file_size = 6666; - let volatile_file_size = 99; - - let cardano_db = DummyCardanoDbBuilder::new(test_dir) - .with_immutables(&[1, 2]) - .set_immutable_trio_file_size(immutable_trio_file_size) - .with_legacy_ledger_snapshots(&[437, 537, 637]) - .set_ledger_file_size(ledger_file_size) - .with_volatile_files(&["blocks-0.dat"]) - .set_volatile_file_size(volatile_file_size) - .build(); - - let appender_dir_all = AppenderDirAll::new(cardano_db.get_dir().clone()); - - let entries_size = appender_dir_all.compute_uncompressed_data_size().unwrap(); - let expected_total_size = - (immutable_trio_file_size * 2) + (3 * ledger_file_size) + volatile_file_size; - assert_eq!(expected_total_size, entries_size); - } - } - - mod appender_data { - use serde::Deserialize; - use zstd::Decoder; - - use super::*; - - #[derive(Debug, PartialEq, Serialize, Deserialize)] - struct TestStruct { - field1: String, - field2: i32, - } - - #[test] - fn append_serializable_json() { - let test_dir = temp_dir_create!(); - let object = TestStruct { - field1: "test".to_string(), - field2: 42, - }; - let location_in_archive = PathBuf::from("folder").join("test.json"); - - let data_appender = - AppenderData::from_json(location_in_archive.clone(), &object).unwrap(); - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - data_appender, - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - let unpacked_file_path = unpack_path.join(&location_in_archive); - - assert!(unpacked_file_path.exists()); - - let deserialized_object: TestStruct = - serde_json::from_reader(File::open(unpacked_file_path).unwrap()).unwrap(); - assert_eq!(object, deserialized_object); - } - - #[test] - fn appended_entry_have_read_write_permissions_and_time_metadata() { - let test_dir = temp_dir_create!(); - let object = TestStruct { - field1: "test".to_string(), - field2: 42, - }; - let location_in_archive = PathBuf::from("folder").join("test.json"); - let start_time_stamp = chrono::Utc::now().timestamp() as u64; - - let data_appender = - AppenderData::from_json(location_in_archive.clone(), &object).unwrap(); - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - data_appender, - ) - .unwrap(); - - let archive_file = File::open(archive.get_file_path()).unwrap(); - let mut archive = tar::Archive::new(Decoder::new(archive_file).unwrap()); - let mut archive_entries = archive.entries().unwrap(); - let appended_entry = archive_entries.next().unwrap().unwrap(); - - assert_eq!( - READ_WRITE_PERMISSION, - appended_entry.header().mode().unwrap() - ); - let mtime = appended_entry.header().mtime().unwrap(); - assert!( - mtime >= start_time_stamp, - "entry mtime should be greater than or equal to the timestamp before the archive \ - creation:\n {mtime} < {start_time_stamp}" - ); - } - - #[test] - fn compute_uncompressed_size() { - let object = TestStruct { - field1: "test".to_string(), - field2: 42, - }; - - let data_appender = - AppenderData::from_json(PathBuf::from("whatever.json"), &object).unwrap(); - - let expected_size = serde_json::to_vec(&object).unwrap().len() as u64; - let entry_size = data_appender.compute_uncompressed_data_size().unwrap(); - assert_eq!(expected_size, entry_size); - } - } - - mod chain_appender { - use super::*; - - #[test] - fn chain_non_overlapping_appenders() { - let test_dir = temp_dir_create!(); - let file_to_archive = create_file(&test_dir, "test_file.txt"); - let json_location_in_archive = PathBuf::from("folder").join("test.json"); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - ChainAppender::new( - AppenderFile::append_at_archive_root(test_dir.join(&file_to_archive)) - .unwrap(), - AppenderData::from_json(json_location_in_archive.clone(), &"test").unwrap(), - ), - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - - assert!(unpack_path.join(file_to_archive).exists()); - assert!(unpack_path.join(json_location_in_archive).exists()); - } - - #[test] - fn chain_overlapping_appenders_data_from_right_appender_overwrite_left_appender_data() { - let test_dir = temp_dir_create!(); - let json_location_in_archive = PathBuf::from("test.json"); - - let file_archiver = FileArchiver::new_for_test(test_dir.join("verification")); - let archive = file_archiver - .archive( - ArchiveParameters { - archive_name_without_extension: "archive".to_string(), - target_directory: test_dir.clone(), - compression_algorithm: CompressionAlgorithm::Zstandard, - }, - ChainAppender::new( - AppenderData::from_json( - json_location_in_archive.clone(), - &"will be overwritten", - ) - .unwrap(), - AppenderData::from_json(json_location_in_archive.clone(), &"test").unwrap(), - ), - ) - .unwrap(); - - let unpack_path = archive.unpack_zstandard(&test_dir); - let unpacked_json_path = unpack_path.join(&json_location_in_archive); - - let deserialized_object: String = - serde_json::from_reader(File::open(&unpacked_json_path).unwrap()).unwrap(); - assert_eq!("test", deserialized_object); - } - - #[test] - fn compute_non_overlapping_uncompressed_size() { - let left_appender = - AppenderData::from_json(PathBuf::from("whatever1.json"), &"foo").unwrap(); - let right_appender = - AppenderData::from_json(PathBuf::from("whatever2.json"), &"bar").unwrap(); - - let expected_size = left_appender.compute_uncompressed_data_size().unwrap() - + right_appender.compute_uncompressed_data_size().unwrap(); - - let chain_appender = left_appender.chain(right_appender); - let size = chain_appender.compute_uncompressed_data_size().unwrap(); - assert_eq!(expected_size, size); - } - - #[test] - fn compute_uncompressed_size_cant_discriminate_overlaps_and_return_aggregated_appenders_sizes() - { - let overlapping_path = PathBuf::from("whatever.json"); - let left_appender = - AppenderData::from_json(overlapping_path.clone(), &"overwritten data").unwrap(); - let right_appender = - AppenderData::from_json(overlapping_path.clone(), &"final data").unwrap(); - - let expected_size = left_appender.compute_uncompressed_data_size().unwrap() - + right_appender.compute_uncompressed_data_size().unwrap(); - - let chain_appender = left_appender.chain(right_appender); - let size = chain_appender.compute_uncompressed_data_size().unwrap(); - assert_eq!(expected_size, size); - } - } -} diff --git a/mithril-aggregator/src/tools/file_archiver/mod.rs b/mithril-aggregator/src/tools/file_archiver/mod.rs deleted file mode 100644 index 024f937f9a0..00000000000 --- a/mithril-aggregator/src/tools/file_archiver/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -mod api; -pub mod appender; -mod entities; - -pub use api::*; -pub use entities::*; - -#[cfg(test)] -pub(crate) mod test_tools { - use std::fs::File; - use std::path::{Path, PathBuf}; - - use mithril_common::test::TempDir; - - pub fn get_test_directory(dir_name: &str) -> PathBuf { - TempDir::create("file_archiver", dir_name) - } - - /// Create a file in the root directory. - /// - /// Returns the relative path to the created file based on the root directory. - pub fn create_file(root: &Path, filename: &str) -> PathBuf { - let file_path = PathBuf::from(filename); - File::create(root.join(file_path.clone())).unwrap(); - file_path - } - - /// Create a directory in the root directory. - /// - /// Returns the relative path to the created directory based on the root directory. - pub fn create_dir(root: &Path, dirname: &str) -> PathBuf { - let dir_path = PathBuf::from(dirname); - std::fs::create_dir(root.join(dir_path.clone())).unwrap(); - dir_path - } -} diff --git a/mithril-aggregator/src/tools/mod.rs b/mithril-aggregator/src/tools/mod.rs index 44aeb35c791..93b3777bdde 100644 --- a/mithril-aggregator/src/tools/mod.rs +++ b/mithril-aggregator/src/tools/mod.rs @@ -1,7 +1,5 @@ mod certificates_hash_migrator; mod era; -pub mod file_archiver; -pub mod file_size; mod genesis; mod protocol_configuration; pub mod signer_importer;