diff --git a/Cargo.lock b/Cargo.lock index 0430d447d7e..76907c70c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -839,6 +848,25 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cast" version = "0.3.0" @@ -1155,6 +1183,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.6.0" @@ -1451,6 +1485,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "der" version = "0.7.10" @@ -1486,6 +1526,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "digest" version = "0.9.0" @@ -1749,6 +1800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -3109,6 +3161,26 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "liblzma" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -3577,6 +3649,15 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -3862,7 +3943,7 @@ dependencies = [ [[package]] name = "mithril-client-cli" -version = "0.12.6" +version = "0.12.7" dependencies = [ "anyhow", "async-trait", @@ -3870,22 +3951,28 @@ dependencies = [ "clap", "cli-table", "config", + "flate2", "fs2", "futures", + "httpmock", "human_bytes", "indicatif", "mithril-cli-helper", "mithril-client", "mithril-common", "mithril-doc", + "mockall", + "reqwest", "serde", "serde_json", "slog", "slog-async", "slog-bunyan", "slog-term", + "tar", "thiserror 2.0.12", "tokio", + "zip", ] [[package]] @@ -4131,7 +4218,7 @@ dependencies = [ [[package]] name = "mithril-stm" -version = "0.4.1" +version = "0.4.2" dependencies = [ "bincode", "blake2 0.10.6", @@ -4789,6 +4876,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac", +] + [[package]] name = "pem" version = "3.0.5" @@ -6131,6 +6228,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.7.0" @@ -8115,6 +8218,50 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "zip" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap 2.9.0", + "liblzma", + "memchr", + "pbkdf2", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 4a1749d509d..18303694869 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client-cli" -version = "0.12.6" +version = "0.12.7" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true } @@ -31,6 +31,7 @@ chrono = { workspace = true } clap = { workspace = true } cli-table = "0.5.0" config = { workspace = true } +flate2 = "1.1.1" fs2 = "0.4.3" futures = "0.3.31" human_bytes = { version = "0.4.3", features = ["fast"] } @@ -38,6 +39,13 @@ indicatif = { version = "0.17.11", features = ["tokio"] } mithril-cli-helper = { path = "../internal/mithril-cli-helper" } mithril-client = { path = "../mithril-client", features = ["fs", "unstable"] } mithril-doc = { path = "../internal/mithril-doc" } +reqwest = { workspace = true, features = [ + "default", + "gzip", + "zstd", + "deflate", + "brotli" +] } serde = { workspace = true } serde_json = { workspace = true } slog = { workspace = true, features = [ @@ -47,8 +55,12 @@ slog = { workspace = true, features = [ slog-async = { workspace = true } slog-bunyan = { workspace = true } slog-term = { workspace = true } +tar = "0.4.44" thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +zip = "4.0.0" [dev-dependencies] +httpmock = "0.7.0" mithril-common = { path = "../mithril-common", features = ["test_tools"] } +mockall = { workspace = true } diff --git a/mithril-client-cli/src/commands/cardano_db/download.rs b/mithril-client-cli/src/commands/cardano_db/download.rs index a593962646b..a6f75e974b3 100644 --- a/mithril-client-cli/src/commands/cardano_db/download.rs +++ b/mithril-client-cli/src/commands/cardano_db/download.rs @@ -185,6 +185,7 @@ impl PreparedCardanoDbDownload { &db_dir, &cardano_db_message, self.is_json_output_enabled(), + self.include_ancillary, )?; Ok(()) @@ -328,6 +329,7 @@ impl PreparedCardanoDbDownload { db_dir: &Path, cardano_db: &Snapshot, json_output: bool, + include_ancillary: bool, ) -> MithrilResult<()> { let canonicalized_filepath = &db_dir.canonicalize().with_context(|| { format!( @@ -336,12 +338,41 @@ impl PreparedCardanoDbDownload { ) })?; + let docker_cmd = format!( + "docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source=\"{}\",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{}", + canonicalized_filepath.display(), + cardano_db.network, + cardano_db.cardano_node_version + ); + + let snapshot_converter_cmd = |flavor| { + format!( + "mithril-client --unstable tools utxo-hd snapshot-converter --db-directory {} --cardano-node-version {} --utxo-hd-flavor {} --cardano-network {} --commit", + db_dir.display(), + cardano_db.cardano_node_version, + flavor, + cardano_db.network + ) + }; + if json_output { - println!( - r#"{{"timestamp": "{}", "db_directory": "{}"}}"#, - Utc::now().to_rfc3339(), - canonicalized_filepath.display() - ); + let json = if include_ancillary { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + "snapshot_converter_cmd_to_lmdb": snapshot_converter_cmd("LMDB"), + "snapshot_converter_cmd_to_legacy": snapshot_converter_cmd("Legacy") + }) + } else { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + }) + }; + + println!("{}", json); } else { let cardano_node_version = &cardano_db.cardano_node_version; println!( @@ -351,14 +382,29 @@ impl PreparedCardanoDbDownload { If you are using Cardano Docker image, you can restore a Cardano Node with: - docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source="{}",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{cardano_node_version} + {} "###, cardano_db.digest, db_dir.display(), - canonicalized_filepath.display(), - cardano_db.network, + docker_cmd, ); + + if include_ancillary { + println!( + r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command: + + {} + + Or to 'Legacy' flavor by running the command: + + {} + + "###, + snapshot_converter_cmd("LMDB"), + snapshot_converter_cmd("Legacy"), + ); + } } Ok(()) diff --git a/mithril-client-cli/src/commands/cardano_db_v2/download.rs b/mithril-client-cli/src/commands/cardano_db_v2/download.rs index b57d3a52552..0b000bd967a 100644 --- a/mithril-client-cli/src/commands/cardano_db_v2/download.rs +++ b/mithril-client-cli/src/commands/cardano_db_v2/download.rs @@ -245,6 +245,9 @@ impl PreparedCardanoDbV2Download { &restoration_options.db_dir, &cardano_db_message, self.is_json_output_enabled(), + restoration_options + .download_unpack_options + .include_ancillary, )?; Ok(()) @@ -488,6 +491,7 @@ impl PreparedCardanoDbV2Download { db_dir: &Path, cardano_db_snapshot: &CardanoDatabaseSnapshot, json_output: bool, + include_ancillary: bool, ) -> MithrilResult<()> { let canonicalized_filepath = &db_dir.canonicalize().with_context(|| { format!( @@ -496,12 +500,41 @@ impl PreparedCardanoDbV2Download { ) })?; + let docker_cmd = format!( + "docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source=\"{}\",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{}", + canonicalized_filepath.display(), + cardano_db_snapshot.network, + cardano_db_snapshot.cardano_node_version + ); + + let snapshot_converter_cmd = |flavor| { + format!( + "mithril-client --unstable tools utxo-hd snapshot-converter --db-directory {} --cardano-node-version {} --utxo-hd-flavor {} --cardano-network {} --commit", + db_dir.display(), + cardano_db_snapshot.cardano_node_version, + flavor, + cardano_db_snapshot.network + ) + }; + if json_output { - println!( - r#"{{"timestamp": "{}", "db_directory": "{}"}}"#, - Utc::now().to_rfc3339(), - canonicalized_filepath.display() - ); + let json = if include_ancillary { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd, + "snapshot_converter_cmd_to_lmdb": snapshot_converter_cmd("LMDB"), + "snapshot_converter_cmd_to_legacy": snapshot_converter_cmd("Legacy") + }) + } else { + serde_json::json!({ + "timestamp": Utc::now().to_rfc3339(), + "db_directory": canonicalized_filepath, + "run_docker_cmd": docker_cmd + }) + }; + + println!("{}", json); } else { let cardano_node_version = &cardano_db_snapshot.cardano_node_version; println!( @@ -511,14 +544,29 @@ impl PreparedCardanoDbV2Download { If you are using Cardano Docker image, you can restore a Cardano Node with: - docker run -v cardano-node-ipc:/ipc -v cardano-node-data:/data --mount type=bind,source="{}",target=/data/db/ -e NETWORK={} ghcr.io/intersectmbo/cardano-node:{cardano_node_version} + {} "###, cardano_db_snapshot.hash, db_dir.display(), - canonicalized_filepath.display(), - cardano_db_snapshot.network, + docker_cmd ); + + if include_ancillary { + println!( + r###"Upgrade and replace the restored ledger state snapshot to 'LMDB' flavor by running the command: + + {} + + Or to 'Legacy' flavor by running the command: + + {} + + "###, + snapshot_converter_cmd("LMDB"), + snapshot_converter_cmd("Legacy"), + ); + } } Ok(()) diff --git a/mithril-client-cli/src/commands/mod.rs b/mithril-client-cli/src/commands/mod.rs index 1f7d7229f19..60d782c366f 100644 --- a/mithril-client-cli/src/commands/mod.rs +++ b/mithril-client-cli/src/commands/mod.rs @@ -9,6 +9,7 @@ pub mod cardano_stake_distribution; pub mod cardano_transaction; mod deprecation; pub mod mithril_stake_distribution; +pub mod tools; pub use deprecation::{DeprecatedCommand, Deprecation}; diff --git a/mithril-client-cli/src/commands/tools/mod.rs b/mithril-client-cli/src/commands/tools/mod.rs new file mode 100644 index 00000000000..cc2651b52e4 --- /dev/null +++ b/mithril-client-cli/src/commands/tools/mod.rs @@ -0,0 +1,46 @@ +//! Tools commands +//! +//! Provides utility subcommands such as converting restored InMemory UTxO-HD ledger snapshot +//! to different flavors (Legacy, LMDB). + +mod snapshot_converter; + +pub use snapshot_converter::*; + +use clap::Subcommand; +use mithril_client::MithrilResult; + +/// Tools commands +#[derive(Subcommand, Debug, Clone)] +#[command(about = "[unstable] Tools commands")] +pub enum ToolsCommands { + /// UTxO-HD related commands + #[clap(subcommand, name = "utxo-hd")] + UTxOHD(UTxOHDCommands), +} + +impl ToolsCommands { + /// Execute Tools command + pub async fn execute(&self) -> MithrilResult<()> { + match self { + Self::UTxOHD(cmd) => cmd.execute().await, + } + } +} + +/// UTxO-HD related commands +#[derive(Subcommand, Debug, Clone)] +pub enum UTxOHDCommands { + /// Convert a restored `InMemory` ledger snapshot to another flavor. + #[clap(arg_required_else_help = false)] + SnapshotConverter(SnapshotConverterCommand), +} + +impl UTxOHDCommands { + /// Execute UTxO-HD command + pub async fn execute(&self) -> MithrilResult<()> { + match self { + Self::SnapshotConverter(cmd) => cmd.execute().await, + } + } +} diff --git a/mithril-client-cli/src/commands/tools/snapshot_converter.rs b/mithril-client-cli/src/commands/tools/snapshot_converter.rs new file mode 100644 index 00000000000..62784e9e6ac --- /dev/null +++ b/mithril-client-cli/src/commands/tools/snapshot_converter.rs @@ -0,0 +1,974 @@ +use std::{ + env, fmt, + fs::{create_dir, read_dir, remove_dir_all, rename}, + path::{Path, PathBuf}, + process::Command, +}; + +use anyhow::{anyhow, Context}; +use clap::{Parser, ValueEnum}; + +use mithril_client::MithrilResult; + +use crate::utils::{ + copy_dir, remove_dir_contents, ArchiveUnpacker, GitHubReleaseRetriever, HttpDownloader, + ReqwestGitHubApiClient, ReqwestHttpDownloader, +}; + +const GITHUB_ORGANIZATION: &str = "IntersectMBO"; +const GITHUB_REPOSITORY: &str = "cardano-node"; + +const LATEST_DISTRIBUTION_TAG: &str = "latest"; +const PRERELEASE_DISTRIBUTION_TAG: &str = "prerelease"; + +const WORK_DIR: &str = "tmp"; +const CARDANO_DISTRIBUTION_DIR: &str = "cardano-node-distribution"; +const SNAPSHOTS_DIR: &str = "snapshots"; + +const SNAPSHOT_CONVERTER_BIN_DIR: &str = "bin"; +const SNAPSHOT_CONVERTER_BIN_NAME_UNIX: &str = "snapshot-converter"; +const SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS: &str = "snapshot-converter.exe"; +const SNAPSHOT_CONVERTER_CONFIG_DIR: &str = "share"; +const SNAPSHOT_CONVERTER_CONFIG_FILE: &str = "config.json"; + +const LEDGER_DIR: &str = "ledger"; + +#[derive(Debug, Clone, ValueEnum)] +enum UTxOHDFlavor { + #[clap(name = "Legacy")] + Legacy, + #[clap(name = "LMDB")] + Lmdb, +} + +impl fmt::Display for UTxOHDFlavor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Legacy => write!(f, "Legacy"), + Self::Lmdb => write!(f, "LMDB"), + } + } +} + +#[derive(Debug, Clone, ValueEnum)] +enum CardanoNetwork { + Preview, + Preprod, + Mainnet, +} + +impl fmt::Display for CardanoNetwork { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Preview => write!(f, "preview"), + Self::Preprod => write!(f, "preprod"), + Self::Mainnet => write!(f, "mainnet"), + } + } +} + +/// Clap command to convert a restored `InMemory` Mithril snapshot to another flavor. +#[derive(Parser, Debug, Clone)] +pub struct SnapshotConverterCommand { + /// Path to the Cardano node database directory. + #[clap(long)] + db_directory: PathBuf, + + /// Cardano node version of the Mithril signed snapshot. + /// + /// `latest` and `prerelease` are also supported to download the latest or prerelease distribution. + #[clap(long)] + cardano_node_version: String, + + /// Cardano network. + #[clap(long)] + cardano_network: CardanoNetwork, + + /// UTxO-HD flavor to convert the ledger snapshot to. + #[clap(long)] + utxo_hd_flavor: UTxOHDFlavor, + + /// If set, the converted snapshot replaces the current ledger state in the `db_directory`. + #[clap(long)] + commit: bool, +} + +impl SnapshotConverterCommand { + /// Main command execution + pub async fn execute(&self) -> MithrilResult<()> { + let work_dir = self.db_directory.join(WORK_DIR); + create_dir(&work_dir).with_context(|| { + format!( + "Failed to create snapshot converter work directory: {}", + work_dir.display() + ) + })?; + let distribution_dir = work_dir.join(CARDANO_DISTRIBUTION_DIR); + + let result = { + create_dir(&distribution_dir).with_context(|| { + format!( + "Failed to create distribution directory: {}", + distribution_dir.display() + ) + })?; + let archive_path = Self::download_cardano_node_distribution( + ReqwestGitHubApiClient::new()?, + ReqwestHttpDownloader::new()?, + &self.cardano_node_version, + &distribution_dir, + ) + .await + .with_context(|| "Failed to download Cardano node distribution")?; + + println!( + "Unpacking distribution from archive: {}", + archive_path.display() + ); + ArchiveUnpacker::default() + .unpack(&archive_path, &distribution_dir) + .with_context(|| { + format!( + "Failed to unpack distribution to directory: {}", + distribution_dir.display() + ) + })?; + println!( + "Distribution unpacked successfully to: {}", + distribution_dir.display() + ); + + Self::convert_ledger_state_snapshot( + &work_dir, + &self.db_directory, + &distribution_dir, + &self.cardano_network, + &self.utxo_hd_flavor, + self.commit, + ) + .with_context(|| { + format!( + "Failed to convert ledger snapshot to flavor: {}", + self.utxo_hd_flavor + ) + })?; + + Ok(()) + }; + + if let Err(e) = Self::cleanup(&work_dir, &distribution_dir, self.commit, result.is_ok()) { + eprintln!( + "Failed to clean up temporary directory {} after execution: {}", + distribution_dir.display(), + e + ); + } + + result + } + + async fn download_cardano_node_distribution( + github_api_client: impl GitHubReleaseRetriever, + http_downloader: impl HttpDownloader, + tag: &str, + target_dir: &Path, + ) -> MithrilResult { + println!( + "Downloading Cardano node distribution for tag: '{}'...", + tag + ); + let release = match tag { + LATEST_DISTRIBUTION_TAG => github_api_client + .get_latest_release(GITHUB_ORGANIZATION, GITHUB_REPOSITORY) + .await + .with_context(|| "Failed to get latest release")?, + PRERELEASE_DISTRIBUTION_TAG => github_api_client + .get_prerelease(GITHUB_ORGANIZATION, GITHUB_REPOSITORY) + .await + .with_context(|| "Failed to get prerelease")?, + _ => github_api_client + .get_release_by_tag(GITHUB_ORGANIZATION, GITHUB_REPOSITORY, tag) + .await + .with_context(|| format!("Failed to get release by tag: {}", tag))?, + }; + let asset = release + .get_asset_for_os(env::consts::OS)? + .ok_or_else(|| anyhow!("No asset found for platform: {}", env::consts::OS)) + .with_context(|| { + format!( + "Failed to find asset for current platform: {}", + env::consts::OS + ) + })?; + let archive_path = http_downloader + .download_file(asset.browser_download_url.parse()?, target_dir, &asset.name) + .await?; + + println!( + "Distribution downloaded successfully. Archive location: {}", + archive_path.display() + ); + + Ok(archive_path) + } + + fn convert_ledger_state_snapshot( + work_dir: &Path, + db_dir: &Path, + distribution_dir: &Path, + cardano_network: &CardanoNetwork, + utxo_hd_flavor: &UTxOHDFlavor, + commit: bool, + ) -> MithrilResult<()> { + println!( + "Converting ledger state snapshot to '{}' flavor", + utxo_hd_flavor + ); + let snapshots_path = work_dir.join(SNAPSHOTS_DIR); + let copied_snapshot_path = + Self::copy_oldest_ledger_state_snapshot(db_dir, &snapshots_path)?; + let converted_snapshot_path = Self::compute_converted_snapshot_output_path( + &snapshots_path, + &copied_snapshot_path, + utxo_hd_flavor, + )?; + let converter_bin = + Self::get_snapshot_converter_binary_path(distribution_dir, env::consts::OS)?; + let config_path = + Self::get_snapshot_converter_config_path(distribution_dir, cardano_network); + Self::execute_snapshot_converter( + &converter_bin, + &copied_snapshot_path, + &converted_snapshot_path, + &config_path, + utxo_hd_flavor, + )?; + + if commit { + Self::commit_converted_snapshot(db_dir, &converted_snapshot_path).with_context( + || "Failed to overwrite the ledger state with the converted snapshot.", + )?; + } else { + println!("Snapshot location: {}", converted_snapshot_path.display()); + } + + Ok(()) + } + + fn execute_snapshot_converter( + bin_path: &Path, + input_path: &Path, + output_path: &Path, + config_path: &Path, + flavor: &UTxOHDFlavor, + ) -> MithrilResult<()> { + Command::new(bin_path) + .arg("Mem") + .arg(input_path) + .arg(flavor.to_string()) + .arg(output_path) + .arg("cardano") + .arg("--config") + .arg(config_path) + .status() + .with_context(|| { + format!( + "Failed to execute snapshot-converter binary at {}", + bin_path.display() + ) + })?; + + Ok(()) + } + + fn get_snapshot_converter_binary_path( + distribution_dir: &Path, + target_os: &str, + ) -> MithrilResult { + let base_path = distribution_dir.join(SNAPSHOT_CONVERTER_BIN_DIR); + let binary_name = match target_os { + "linux" | "macos" => SNAPSHOT_CONVERTER_BIN_NAME_UNIX, + "windows" => SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS, + _ => return Err(anyhow!("Unsupported platform: {}", target_os)), + }; + + Ok(base_path.join(binary_name)) + } + + fn get_snapshot_converter_config_path( + distribution_dir: &Path, + network: &CardanoNetwork, + ) -> PathBuf { + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + } + + /// Returns the list of valid ledger snapshot directories sorted in ascending order of slot number. + /// + /// Only directories with numeric names are considered valid snapshots. + fn get_sorted_snapshot_dirs(ledger_dir: &Path) -> MithrilResult> { + let entries = read_dir(ledger_dir).with_context(|| { + format!( + "Failed to read ledger state snapshots directory: {}", + ledger_dir.display() + ) + })?; + + let mut snapshots = entries + .filter_map(|entry| { + let path = entry.ok()?.path(); + if !path.is_dir() { + return None; + } + SnapshotConverterCommand::extract_slot_number(&path) + .ok() + .map(|slot| (slot, path)) + }) + .collect::>(); + + snapshots.sort_by_key(|(slot, _)| *slot); + + Ok(snapshots) + } + + /// Finds the oldest ledger snapshot (by slot number) in the `ledger/` directory of a Cardano node database. + fn find_oldest_ledger_state_snapshot(db_dir: &Path) -> MithrilResult { + let ledger_dir = db_dir.join(LEDGER_DIR); + let snapshots_by_slot = Self::get_sorted_snapshot_dirs(&ledger_dir)?; + snapshots_by_slot + .into_iter() + .map(|(_, path)| path) + .next() + .ok_or_else(|| { + anyhow!( + "No valid ledger state snapshot found in directory: {}", + ledger_dir.display() + ) + }) + } + + fn copy_oldest_ledger_state_snapshot( + db_dir: &Path, + target_dir: &Path, + ) -> MithrilResult { + let snapshot_path = Self::find_oldest_ledger_state_snapshot(db_dir)?; + let copied_snapshot_path = copy_dir(&snapshot_path, target_dir)?; + + Ok(copied_snapshot_path) + } + + fn compute_converted_snapshot_output_path( + snapshots_dir: &Path, + input_snapshot: &Path, + flavor: &UTxOHDFlavor, + ) -> MithrilResult { + let slot_number = Self::extract_slot_number(input_snapshot).with_context(|| { + format!( + "Failed to extract slot number from: {}", + input_snapshot.display() + ) + })?; + let converted_snapshot_path = snapshots_dir.join(format!( + "{}_{}", + slot_number, + flavor.to_string().to_lowercase() + )); + + Ok(converted_snapshot_path) + } + + fn extract_slot_number(path: &Path) -> MithrilResult { + let file_name = path + .file_name() + .ok_or_else(|| anyhow!("No filename in path: {}", path.display()))?; + let file_name_str = file_name + .to_str() + .ok_or_else(|| anyhow!("Invalid UTF-8 in path filename: {:?}", file_name))?; + + file_name_str + .parse::() + .with_context(|| format!("Invalid slot number in path filename: {}", file_name_str)) + } + + /// Commits the converted snapshot by replacing the current ledger state snapshots in the database directory. + fn commit_converted_snapshot( + db_dir: &Path, + converted_snapshot_path: &Path, + ) -> MithrilResult<()> { + let ledger_dir = db_dir.join(LEDGER_DIR); + println!( + "Upgrading and replacing ledger state in {} with converted snapshot: {}", + ledger_dir.display(), + converted_snapshot_path.display() + ); + let filename = converted_snapshot_path + .file_name() + .ok_or_else(|| anyhow!("Missing filename in converted snapshot path"))? + .to_string_lossy(); + let (slot_number, _) = filename + .split_once('_') + .ok_or_else(|| anyhow!("Invalid converted snapshot name format: {}", filename))?; + remove_dir_contents(&ledger_dir).with_context(|| { + format!( + "Failed to remove contents of ledger directory: {}", + ledger_dir.display() + ) + })?; + let destination = ledger_dir.join(slot_number); + rename(converted_snapshot_path, &destination).with_context(|| { + format!( + "Failed to move converted snapshot to ledger directory: {}", + destination.display() + ) + })?; + + Ok(()) + } + + fn cleanup( + work_dir: &Path, + distribution_dir: &Path, + commit: bool, + success: bool, + ) -> MithrilResult<()> { + match (success, commit) { + (true, true) => { + remove_dir_all(distribution_dir)?; + remove_dir_all(work_dir)?; + } + (true, false) => { + remove_dir_all(distribution_dir)?; + } + (false, _) => { + remove_dir_all(distribution_dir)?; + remove_dir_all(work_dir)?; + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use mithril_common::temp_dir_create; + + use super::*; + + mod download_cardano_node_distribution { + use mockall::predicate::eq; + use reqwest::Url; + + use crate::utils::{GitHubRelease, MockGitHubReleaseRetriever, MockHttpDownloader}; + + use super::*; + + #[tokio::test] + async fn downloads_latest_release_distribution() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_latest_release() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + LATEST_DISTRIBUTION_TAG, + &temp_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn downloads_prerelease_distribution() { + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_prerelease() + .with(eq(GITHUB_ORGANIZATION), eq(GITHUB_REPOSITORY)) + .returning(move |_, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + PRERELEASE_DISTRIBUTION_TAG, + &temp_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn downloads_tagged_release_distribution() { + let cardano_node_version = "10.3.1"; + let temp_dir = temp_dir_create!(); + let release = GitHubRelease::dummy_with_all_supported_assets(); + let asset = release.get_asset_for_os(env::consts::OS).unwrap().unwrap(); + + let cloned_release = release.clone(); + let mut github_api_client = MockGitHubReleaseRetriever::new(); + github_api_client + .expect_get_release_by_tag() + .with( + eq(GITHUB_ORGANIZATION), + eq(GITHUB_REPOSITORY), + eq(cardano_node_version), + ) + .returning(move |_, _, _| Ok(cloned_release.clone())); + + let mut http_downloader = MockHttpDownloader::new(); + http_downloader + .expect_download_file() + .with( + eq(Url::parse(&asset.browser_download_url).unwrap()), + eq(temp_dir.clone()), + eq(asset.name.clone()), + ) + .returning(|_, _, _| Ok(PathBuf::new())); + + SnapshotConverterCommand::download_cardano_node_distribution( + github_api_client, + http_downloader, + cardano_node_version, + &temp_dir, + ) + .await + .unwrap(); + } + } + + mod get_snapshot_converter_binary_path { + use super::*; + + #[test] + fn returns_correct_binary_path_for_linux() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "linux", + ) + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_UNIX) + ); + } + + #[test] + fn returns_correct_binary_path_for_macos() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "macos", + ) + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_UNIX) + ); + } + + #[test] + fn returns_correct_binary_path_for_windows() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + + let binary_path = SnapshotConverterCommand::get_snapshot_converter_binary_path( + &distribution_dir, + "windows", + ) + .unwrap(); + + assert_eq!( + binary_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_BIN_DIR) + .join(SNAPSHOT_CONVERTER_BIN_NAME_WINDOWS) + ); + } + } + + mod get_snapshot_converter_config_path { + use super::*; + + #[test] + fn returns_config_path_for_mainnet() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Mainnet; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + + #[test] + fn returns_config_path_for_preprod() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Preprod; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + + #[test] + fn returns_config_path_for_preview() { + let distribution_dir = PathBuf::from("/path/to/distribution"); + let network = CardanoNetwork::Preview; + + let config_path = SnapshotConverterCommand::get_snapshot_converter_config_path( + &distribution_dir, + &network, + ); + + assert_eq!( + config_path, + distribution_dir + .join(SNAPSHOT_CONVERTER_CONFIG_DIR) + .join(network.to_string()) + .join(SNAPSHOT_CONVERTER_CONFIG_FILE) + ); + } + } + + mod extract_slot_number { + use super::*; + + #[test] + fn parses_valid_numeric_path() { + let path = PathBuf::from("/whatever").join("123456"); + + let slot = SnapshotConverterCommand::extract_slot_number(&path).unwrap(); + + assert_eq!(slot, 123456); + } + + #[test] + fn fails_with_non_numeric_filename() { + let path = PathBuf::from("/whatever").join("notanumber"); + + SnapshotConverterCommand::extract_slot_number(&path) + .expect_err("Should fail with non-numeric filename"); + } + + #[test] + fn fails_if_no_filename() { + let path = PathBuf::from("/"); + + SnapshotConverterCommand::extract_slot_number(&path) + .expect_err("Should fail if path has no filename"); + } + } + + mod compute_converted_snapshot_output_path { + use super::*; + + #[test] + fn compute_output_path_from_numeric_file_name() { + let snapshots_dir = PathBuf::from("/snapshots"); + let input_snapshot = PathBuf::from("/whatever").join("123456"); + + { + let snapshot_path = + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Lmdb, + ) + .unwrap(); + + assert_eq!(snapshot_path, snapshots_dir.join("123456_lmdb")); + } + + { + let snapshot_path = + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Legacy, + ) + .unwrap(); + + assert_eq!(snapshot_path, snapshots_dir.join("123456_legacy")); + } + } + + #[test] + fn fails_with_invalid_slot_number() { + let snapshots_dir = PathBuf::from("/snapshots"); + let input_snapshot = PathBuf::from("/whatever/notanumber"); + + SnapshotConverterCommand::compute_converted_snapshot_output_path( + &snapshots_dir, + &input_snapshot, + &UTxOHDFlavor::Lmdb, + ) + .expect_err("Should fail with invalid slot number"); + } + } + + mod find_oldest_ledger_state_snapshot { + use std::fs::File; + + use mithril_common::temp_dir_create; + + use super::*; + + #[test] + fn finds_ledger_state_snapshot_with_lowest_slot_number() { + let db_dir = temp_dir_create!(); + let ledger_dir = db_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("1000")).unwrap(); + create_dir(ledger_dir.join("1500")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&db_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("500")); + } + + #[test] + fn returns_snapshot_when_only_one_valid_directory() { + let db_dir = temp_dir_create!(); + let ledger_dir = db_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("500")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&db_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("500")); + } + + #[test] + fn ignores_non_numeric_and_non_directory_entries() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("1000")).unwrap(); + File::create(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("notanumber")).unwrap(); + + let found = + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&temp_dir).unwrap(); + + assert_eq!(found, ledger_dir.join("1000")); + } + + #[test] + fn returns_error_if_no_valid_snapshot_found() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + File::create(ledger_dir.join("invalid")).unwrap(); + + SnapshotConverterCommand::find_oldest_ledger_state_snapshot(&temp_dir) + .expect_err("Should return error if no valid ledger snapshot directory found"); + } + + #[test] + fn get_sorted_snapshot_dirs_returns_sorted_valid_directories() { + let temp_dir = temp_dir_create!(); + let ledger_dir = temp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + + create_dir(ledger_dir.join("1500")).unwrap(); + create_dir(ledger_dir.join("1000")).unwrap(); + create_dir(ledger_dir.join("2000")).unwrap(); + File::create(ledger_dir.join("500")).unwrap(); + create_dir(ledger_dir.join("notanumber")).unwrap(); + + let snapshots = + SnapshotConverterCommand::get_sorted_snapshot_dirs(&ledger_dir).unwrap(); + + assert_eq!( + snapshots, + vec![ + (1000, ledger_dir.join("1000")), + (1500, ledger_dir.join("1500")), + (2000, ledger_dir.join("2000")), + ] + ); + } + } + + mod commit_converted_snapshot { + use std::fs::File; + + use super::*; + + #[test] + fn moves_converted_snapshot_to_ledger_directory() { + let tmp_dir = temp_dir_create!(); + let ledger_dir = tmp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + let previous_snapshot = ledger_dir.join("123"); + File::create(&previous_snapshot).unwrap(); + + let converted_snapshot = tmp_dir.join("456_lmdb"); + File::create(&converted_snapshot).unwrap(); + + assert!(previous_snapshot.exists()); + SnapshotConverterCommand::commit_converted_snapshot(&tmp_dir, &converted_snapshot) + .unwrap(); + + assert!(!previous_snapshot.exists()); + assert!(ledger_dir.join("456").exists()); + } + + #[test] + fn fails_if_converted_snapshot_has_invalid_filename() { + let tmp_dir = temp_dir_create!(); + let ledger_dir = tmp_dir.join(LEDGER_DIR); + create_dir(&ledger_dir).unwrap(); + let previous_snapshot = ledger_dir.join("123"); + File::create(&previous_snapshot).unwrap(); + + let converted_snapshot = tmp_dir.join("456"); + File::create(&converted_snapshot).unwrap(); + + SnapshotConverterCommand::commit_converted_snapshot(&tmp_dir, &converted_snapshot) + .expect_err("Should fail if converted snapshot has invalid filename"); + + assert!(previous_snapshot.exists()); + } + } + + mod cleanup { + use super::*; + + #[test] + fn removes_both_dirs_on_success_when_commit_is_true() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, true, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + + #[test] + fn removes_only_distribution_on_success_when_commit_is_false() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(work_dir.exists()); + } + + #[test] + fn removes_both_dirs_on_success_when_commit_is_true_and_distribution_is_nested() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = work_dir.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, true, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + + #[test] + fn removes_only_distribution_on_success_when_commit_is_false_and_distribution_is_nested() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = work_dir.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, true).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(work_dir.exists()); + } + + #[test] + fn removes_both_dirs_on_failure() { + let tmp = temp_dir_create!(); + let work_dir = tmp.join("workdir_dir"); + let distribution_dir = tmp.join("distribution_dir"); + create_dir(&work_dir).unwrap(); + create_dir(&distribution_dir).unwrap(); + + SnapshotConverterCommand::cleanup(&work_dir, &distribution_dir, false, false).unwrap(); + + assert!(!distribution_dir.exists()); + assert!(!work_dir.exists()); + } + } +} diff --git a/mithril-client-cli/src/main.rs b/mithril-client-cli/src/main.rs index df94a659955..57628c64faf 100644 --- a/mithril-client-cli/src/main.rs +++ b/mithril-client-cli/src/main.rs @@ -17,7 +17,8 @@ use mithril_client_cli::commands::{ cardano_db::CardanoDbCommands, cardano_db_v2::CardanoDbV2Commands, cardano_stake_distribution::CardanoStakeDistributionCommands, cardano_transaction::CardanoTransactionCommands, - mithril_stake_distribution::MithrilStakeDistributionCommands, DeprecatedCommand, Deprecation, + mithril_stake_distribution::MithrilStakeDistributionCommands, tools::ToolsCommands, + DeprecatedCommand, Deprecation, }; use mithril_client_cli::{ClapError, CommandContext}; @@ -209,6 +210,9 @@ enum ArtifactCommands { #[clap(alias("doc"), hide(true))] GenerateDoc(GenerateDocCommands), + + #[clap(subcommand)] + Tools(ToolsCommands), } impl ArtifactCommands { @@ -231,6 +235,16 @@ impl ArtifactCommands { Self::GenerateDoc(cmd) => cmd .execute(&mut Args::command()) .map_err(|message| anyhow!(message)), + Self::Tools(cmd) => { + if !context.is_unstable_enabled() { + Err(anyhow!(Self::unstable_flag_missing_message( + "tools", + "utxo-hd snapshot-converter" + ))) + } else { + cmd.execute().await + } + } } } @@ -274,4 +288,32 @@ mod tests { .to_string() .contains("subcommand is only accepted using the --unstable flag.")); } + + #[tokio::test] + async fn fail_if_tools_command_is_used_without_unstable_flag() { + let args = Args::try_parse_from([ + "mithril-client", + "tools", + "utxo-hd", + "snapshot-converter", + "--db-directory", + "whatever", + "--cardano-network", + "preview", + "--cardano-node-version", + "1.2.3", + "--utxo-hd-flavor", + "Legacy", + ]) + .unwrap(); + + let error = args + .execute(Logger::root(slog::Discard, slog::o!())) + .await + .expect_err("Should fail if unstable flag missing"); + + assert!(error + .to_string() + .contains("subcommand is only accepted using the --unstable flag.")); + } } diff --git a/mithril-client-cli/src/utils/archive_unpacker/interface.rs b/mithril-client-cli/src/utils/archive_unpacker/interface.rs new file mode 100644 index 00000000000..8afa1f737f0 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/interface.rs @@ -0,0 +1,13 @@ +use std::path::Path; + +use mithril_client::MithrilResult; + +/// Trait for supported archive formats (e.g. `.zip`, `.tar.gz`). +#[cfg_attr(test, mockall::automock)] +pub trait ArchiveFormat { + /// Checks whether this format can handle the given archive file. + fn supports(&self, archive_path: &Path) -> bool; + + /// Unpacks the archive into the target directory. + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()>; +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/mod.rs b/mithril-client-cli/src/utils/archive_unpacker/mod.rs new file mode 100644 index 00000000000..b6d895540b8 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/mod.rs @@ -0,0 +1,7 @@ +mod interface; +mod tar_gz_unpacker; +mod unpacker; +mod zip_unpacker; + +pub use interface::*; +pub use unpacker::*; diff --git a/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs new file mode 100644 index 00000000000..2e1494030a9 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/tar_gz_unpacker.rs @@ -0,0 +1,102 @@ +use std::{fs::File, path::Path}; + +use anyhow::Context; +use flate2::read::GzDecoder; +use tar::Archive; + +use mithril_client::MithrilResult; + +use super::ArchiveFormat; + +#[derive(Debug, Eq, PartialEq)] +pub struct TarGzUnpacker; + +impl ArchiveFormat for TarGzUnpacker { + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let archive = File::open(archive_path) + .with_context(|| format!("Could not open archive file '{}'", archive_path.display()))?; + let gzip_decoder = GzDecoder::new(archive); + let mut file_archive = Archive::new(gzip_decoder); + file_archive.unpack(unpack_dir).with_context(|| { + format!( + "Could not unpack '{}' with 'Gzip' to directory '{}'", + archive_path.display(), + unpack_dir.display() + ) + })?; + + Ok(()) + } + + fn supports(&self, path: &Path) -> bool { + path.extension().and_then(|e| e.to_str()) == Some("gz") + } +} + +#[cfg(test)] +mod tests { + use std::fs::{self, File}; + + use flate2::{write::GzEncoder, Compression}; + use tar::{Builder, Header}; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn unpack_tar_archive_extracts_all_files() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.tar.gz"); + + { + let tar_gz_file = File::create(&archive_path).unwrap(); + let encoder = GzEncoder::new(tar_gz_file, Compression::default()); + let mut tar_builder = Builder::new(encoder); + + let content = b"root content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "root.txt", &content[..]) + .unwrap(); + + let content = b"nested content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "nested/dir/nested-file.txt", &content[..]) + .unwrap(); + + tar_builder.finish().unwrap(); + } + + TarGzUnpacker.unpack(&archive_path, &temp_dir).unwrap(); + + assert_dir_eq! { + &temp_dir, + "* nested/ + ** dir/ + *** nested-file.txt + * archive.tar.gz + * root.txt" + }; + + let root_file_content = fs::read_to_string(temp_dir.join("root.txt")).unwrap(); + assert_eq!(root_file_content, "root content"); + + let nested_file_content = + fs::read_to_string(temp_dir.join("nested/dir/nested-file.txt")).unwrap(); + assert_eq!(nested_file_content, "nested content"); + } + + #[test] + fn supported_file_extension() { + assert!(TarGzUnpacker.supports(Path::new("archive.tar.gz"))); + assert!(TarGzUnpacker.supports(Path::new("archive.gz"))); + assert!(!TarGzUnpacker.supports(Path::new("archive.tar"))); + assert!(!TarGzUnpacker.supports(Path::new("archive.whatever"))); + } +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs new file mode 100644 index 00000000000..79e7715f71a --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/unpacker.rs @@ -0,0 +1,111 @@ +use std::path::Path; + +use anyhow::anyhow; + +use mithril_client::MithrilResult; + +use super::{tar_gz_unpacker::TarGzUnpacker, zip_unpacker::ZipUnpacker, ArchiveFormat}; + +pub struct ArchiveUnpacker { + supported_formats: Vec>, +} + +impl Default for ArchiveUnpacker { + fn default() -> Self { + Self { + supported_formats: vec![Box::new(TarGzUnpacker), Box::new(ZipUnpacker)], + } + } +} + +impl ArchiveUnpacker { + fn select_unpacker(&self, archive_path: &Path) -> MithrilResult<&dyn ArchiveFormat> { + self.supported_formats + .iter() + .find(|f| f.supports(archive_path)) + .map(|f| f.as_ref()) + .ok_or_else(|| anyhow!("Unsupported archive format: {}", archive_path.display())) + } + + pub fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let unpacker = self.select_unpacker(archive_path)?; + unpacker.unpack(archive_path, unpack_dir) + } +} + +#[cfg(test)] +mod tests { + use std::{fs::File, io::Write, path::Path}; + + use flate2::{write::GzEncoder, Compression}; + use tar::{Builder, Header}; + use zip::{write::FileOptions, ZipWriter}; + + use mithril_common::temp_dir_create; + + use super::*; + + #[test] + fn archive_unpacker_unpacks_tar_gz_archive() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.tar.gz"); + + { + let tar_gz_file = File::create(&archive_path).unwrap(); + let encoder = GzEncoder::new(tar_gz_file, Compression::default()); + let mut tar_builder = Builder::new(encoder); + + let content = b"whatever content"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_cksum(); + tar_builder + .append_data(&mut header, "file.txt", &content[..]) + .unwrap(); + tar_builder.finish().unwrap(); + } + + ArchiveUnpacker::default() + .unpack(&archive_path, &temp_dir) + .unwrap(); + + assert!(temp_dir.join("file.txt").exists()); + } + + #[test] + fn archive_unpacker_unpacks_zip_archive() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.zip"); + + { + let zip_file = File::create(&archive_path).unwrap(); + let mut zip_writer = ZipWriter::new(zip_file); + + zip_writer + .start_file("file.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"whatever content").unwrap(); + + zip_writer.finish().unwrap(); + } + + ArchiveUnpacker::default() + .unpack(&archive_path, &temp_dir) + .unwrap(); + + assert!(temp_dir.join("file.txt").exists()); + } + + #[test] + fn fails_with_unknown_extension() { + let path = Path::new("whatever.unknown"); + + let archive_unpacker = ArchiveUnpacker::default(); + let result = archive_unpacker.select_unpacker(path); + + assert!( + result.is_err(), + "Should fail with unsupported archive extension." + ); + } +} diff --git a/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs b/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs new file mode 100644 index 00000000000..8ddb54e81f8 --- /dev/null +++ b/mithril-client-cli/src/utils/archive_unpacker/zip_unpacker.rs @@ -0,0 +1,114 @@ +use std::{ + fs::{self, File}, + io, + path::Path, +}; + +use anyhow::Context; +use zip::ZipArchive; + +use mithril_client::MithrilResult; + +use super::ArchiveFormat; + +pub struct ZipUnpacker; + +impl ArchiveFormat for ZipUnpacker { + fn unpack(&self, archive_path: &Path, unpack_dir: &Path) -> MithrilResult<()> { + let file = File::open(archive_path) + .with_context(|| format!("Could not open archive file '{}'", archive_path.display()))?; + let mut archive = ZipArchive::new(file) + .with_context(|| format!("Could not read ZIP archive '{}'", archive_path.display()))?; + + for i in 0..archive.len() { + let mut file = archive.by_index(i)?; + let rel_path = file + .enclosed_name() + .ok_or_else(|| anyhow::anyhow!("File path is unsafe or malformed"))?; + let outpath = unpack_dir.join(rel_path); + + if file.is_dir() { + fs::create_dir_all(&outpath).with_context(|| { + format!("Could not create directory '{}'", outpath.display()) + })?; + } else { + if let Some(parent) = outpath.parent() { + if !parent.exists() { + fs::create_dir_all(parent).with_context(|| { + format!("Could not create directory '{}'", parent.display()) + })?; + } + } + let mut outfile = File::create(&outpath) + .with_context(|| format!("Could not create file '{}'", outpath.display()))?; + io::copy(&mut file, &mut outfile) + .with_context(|| format!("Failed to write file '{}'", outpath.display()))?; + } + } + + Ok(()) + } + + fn supports(&self, path: &Path) -> bool { + path.extension().and_then(|e| e.to_str()) == Some("zip") + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use zip::{write::FileOptions, ZipWriter}; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn unpack_zip_archive_extracts_all_files() { + let temp_dir = temp_dir_create!(); + let archive_path = temp_dir.join("archive.zip"); + + { + let zip_file = fs::File::create(&archive_path).unwrap(); + let mut zip_writer = ZipWriter::new(zip_file); + + zip_writer + .start_file("root.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"root content").unwrap(); + + zip_writer + .start_file("nested/dir/nested-file.txt", FileOptions::<()>::default()) + .unwrap(); + zip_writer.write_all(b"nested content").unwrap(); + + zip_writer.finish().unwrap(); + } + + ZipUnpacker.unpack(&archive_path, &temp_dir).unwrap(); + + assert_dir_eq! { + &temp_dir, + "* nested/ + ** dir/ + *** nested-file.txt + * archive.zip + * root.txt" + }; + + let root_file_content = fs::read_to_string(temp_dir.join("root.txt")).unwrap(); + assert_eq!(root_file_content, "root content"); + + let nested_file_content = + fs::read_to_string(temp_dir.join("nested/dir/nested-file.txt")).unwrap(); + assert_eq!(nested_file_content, "nested content"); + } + + #[test] + fn supported_file_extension() { + assert!(ZipUnpacker.supports(Path::new("archive.zip"))); + assert!(ZipUnpacker.supports(Path::new("archive.whatever.zip"))); + assert!(!ZipUnpacker.supports(Path::new("archive.whatever"))); + } +} diff --git a/mithril-client-cli/src/utils/fs.rs b/mithril-client-cli/src/utils/fs.rs new file mode 100644 index 00000000000..def181bbb02 --- /dev/null +++ b/mithril-client-cli/src/utils/fs.rs @@ -0,0 +1,145 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context}; + +use mithril_client::MithrilResult; + +/// Copies a directory and its contents to a new location. +pub fn copy_dir(source_dir: &Path, target_dir: &Path) -> MithrilResult { + let source_dir_name = source_dir + .file_name() + .ok_or_else(|| anyhow!("Invalid source directory: {}", source_dir.display()))?; + let destination_path = target_dir.join(source_dir_name); + copy_dir_contents(source_dir, &destination_path)?; + + Ok(destination_path) +} + +fn copy_dir_contents(source_dir: &Path, target_dir: &Path) -> MithrilResult<()> { + fs::create_dir_all(target_dir).with_context(|| { + format!( + "Failed to create target directory: {}", + target_dir.display() + ) + })?; + + for entry in fs::read_dir(source_dir)? { + let entry = entry?; + let file_type = entry.file_type()?; + let src_path = entry.path(); + let dst_path = target_dir.join(entry.file_name()); + + if file_type.is_dir() { + copy_dir_contents(&src_path, &dst_path)?; + } else { + fs::copy(&src_path, &dst_path).with_context(|| { + format!( + "Failed to copy file '{}' to '{}'", + src_path.display(), + dst_path.display() + ) + })?; + } + } + + Ok(()) +} + +/// Removes all contents inside the given directory. +pub fn remove_dir_contents(dir: &Path) -> MithrilResult<()> { + if !dir.exists() { + return Ok(()); + } + + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + fs::remove_dir_all(&path) + .with_context(|| format!("Failed to remove subdirectory: {}", path.display()))?; + } else { + fs::remove_file(&path) + .with_context(|| format!("Failed to remove file: {}", path.display()))?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs::File; + + use mithril_common::{assert_dir_eq, temp_dir_create}; + + use super::*; + + #[test] + fn fails_if_source_does_not_exist() { + let temp_dir = temp_dir_create!(); + let dir_not_exist = PathBuf::from("dir_not_exist"); + + copy_dir(&dir_not_exist, &temp_dir) + .expect_err("Expected error when source directory does not exist"); + } + + #[test] + fn returns_copied_directory_path() { + let temp_dir = temp_dir_create!(); + let src = temp_dir.join("dir_to_copy"); + fs::create_dir(&src).unwrap(); + let dst = temp_dir.join("dst"); + + let copied_dir_path = copy_dir(&src, &dst).unwrap(); + + assert_eq!(copied_dir_path, dst.join("dir_to_copy")); + } + + #[test] + fn copies_nested_directories_and_files() { + let temp_dir = temp_dir_create!(); + let src = temp_dir.join("dir_to_copy"); + fs::create_dir(&src).unwrap(); + File::create(src.join("root.txt")).unwrap(); + + let sub_dir1 = src.join("subdir1"); + fs::create_dir(&sub_dir1).unwrap(); + File::create(sub_dir1.join("subdir1.txt")).unwrap(); + + let sub_dir2 = src.join("subdir2"); + fs::create_dir(&sub_dir2).unwrap(); + File::create(sub_dir2.join("subdir2.txt")).unwrap(); + + let dst = temp_dir.join("dst"); + + copy_dir(&src, &dst).unwrap(); + + assert_dir_eq!( + &dst, + "* dir_to_copy/ + ** subdir1/ + *** subdir1.txt + ** subdir2/ + *** subdir2.txt + ** root.txt" + ); + } + + #[test] + fn cleans_directory_without_deleting_it() { + let dir = temp_dir_create!().join("dir_to_clean"); + fs::create_dir(&dir).unwrap(); + + File::create(dir.join("file1.txt")).unwrap(); + let sub_dir = dir.join("subdir"); + fs::create_dir(&sub_dir).unwrap(); + File::create(sub_dir.join("file2.txt")).unwrap(); + + remove_dir_contents(&dir).unwrap(); + + assert!(dir.exists()); + assert!(fs::read_dir(&dir).unwrap().next().is_none()); + } +} diff --git a/mithril-client-cli/src/utils/github_release_retriever/interface.rs b/mithril-client-cli/src/utils/github_release_retriever/interface.rs new file mode 100644 index 00000000000..7c2eec5a09a --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/interface.rs @@ -0,0 +1,27 @@ +use async_trait::async_trait; + +use mithril_client::MithrilResult; + +use super::model::GitHubRelease; + +/// Trait for interacting with the GitHub API to retrieve Cardano node release. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait GitHubReleaseRetriever { + /// Retrieves a release by its tag. + async fn get_release_by_tag( + &self, + owner: &str, + repo: &str, + tag: &str, + ) -> MithrilResult; + + /// Retrieves the latest release. + async fn get_latest_release(&self, owner: &str, repo: &str) -> MithrilResult; + + /// Retrieves the prerelease. + async fn get_prerelease(&self, owner: &str, repo: &str) -> MithrilResult; + + /// Retrieves all available releases. + async fn get_all_releases(&self, owner: &str, repo: &str) -> MithrilResult>; +} diff --git a/mithril-client-cli/src/utils/github_release_retriever/mod.rs b/mithril-client-cli/src/utils/github_release_retriever/mod.rs new file mode 100644 index 00000000000..d7676cf377a --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/mod.rs @@ -0,0 +1,7 @@ +mod interface; +mod model; +mod reqwest; + +pub use interface::*; +pub use model::*; +pub use reqwest::*; diff --git a/mithril-client-cli/src/utils/github_release_retriever/model.rs b/mithril-client-cli/src/utils/github_release_retriever/model.rs new file mode 100644 index 00000000000..937a97bdfc0 --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/model.rs @@ -0,0 +1,123 @@ +use anyhow::anyhow; +use serde::Deserialize; + +use mithril_client::MithrilResult; + +pub const ASSET_PLATFORM_LINUX: &str = "linux"; +pub const ASSET_PLATFORM_MACOS: &str = "macos"; +pub const ASSET_PLATFORM_WINDOWS: &str = "win64"; + +#[derive(Debug, Clone, Deserialize, Eq, PartialEq)] +pub struct GitHubAsset { + pub name: String, + pub browser_download_url: String, +} + +#[derive(Debug, Default, Clone, Deserialize)] +pub struct GitHubRelease { + pub assets: Vec, + pub prerelease: bool, +} + +impl GitHubRelease { + pub fn get_asset_for_os(&self, target_os: &str) -> MithrilResult> { + let os_in_asset_name = match target_os { + "linux" => ASSET_PLATFORM_LINUX, + "macos" => ASSET_PLATFORM_MACOS, + "windows" => ASSET_PLATFORM_WINDOWS, + _ => return Err(anyhow!("Unsupported platform: {}", target_os)), + }; + + let asset = self + .assets + .iter() + .find(|asset| asset.name.contains(os_in_asset_name)); + + Ok(asset) + } + + #[cfg(test)] + pub fn dummy_with_all_supported_assets() -> Self { + GitHubRelease { + assets: vec![ + GitHubAsset { + name: format!("asset-name-{}.tar.gz", ASSET_PLATFORM_LINUX), + browser_download_url: "https://release-assets.com/linux".to_string(), + }, + GitHubAsset { + name: format!("asset-name-{}.tar.gz", ASSET_PLATFORM_MACOS), + browser_download_url: "https://release-assets.com/macos".to_string(), + }, + GitHubAsset { + name: format!("asset-name-{}.zip", ASSET_PLATFORM_WINDOWS), + browser_download_url: "https://release-assets.com/windows".to_string(), + }, + ], + ..GitHubRelease::default() + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn dummy_asset(os: &str) -> GitHubAsset { + GitHubAsset { + name: format!("asset-name-{}.whatever", os), + browser_download_url: format!("https://release-assets.com/{}", os), + } + } + + #[test] + fn returns_expected_asset_for_each_supported_platform() { + let release = GitHubRelease { + assets: vec![ + dummy_asset(ASSET_PLATFORM_LINUX), + dummy_asset(ASSET_PLATFORM_MACOS), + dummy_asset(ASSET_PLATFORM_WINDOWS), + ], + ..GitHubRelease::default() + }; + + { + let asset = release.get_asset_for_os("linux").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_LINUX))); + } + + { + let asset = release.get_asset_for_os("macos").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_MACOS))); + } + + { + let asset = release.get_asset_for_os("windows").unwrap(); + assert_eq!(asset, Some(&dummy_asset(ASSET_PLATFORM_WINDOWS))); + } + } + + #[test] + fn returns_none_when_asset_is_missing() { + let release = GitHubRelease { + assets: vec![dummy_asset(ASSET_PLATFORM_LINUX)], + ..GitHubRelease::default() + }; + + let asset = release.get_asset_for_os("macos").unwrap(); + + assert!(asset.is_none()); + } + + #[test] + fn fails_for_unsupported_platform() { + let release = GitHubRelease { + assets: vec![dummy_asset(ASSET_PLATFORM_LINUX)], + ..GitHubRelease::default() + }; + + release + .get_asset_for_os("unsupported") + .expect_err("Should have failed for unsupported platform"); + } +} diff --git a/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs new file mode 100644 index 00000000000..d75472fc01c --- /dev/null +++ b/mithril-client-cli/src/utils/github_release_retriever/reqwest.rs @@ -0,0 +1,185 @@ +use anyhow::{anyhow, Context}; +use async_trait::async_trait; +use reqwest::{Client, IntoUrl}; +use serde::de::DeserializeOwned; + +use mithril_client::MithrilResult; + +use super::{GitHubRelease, GitHubReleaseRetriever}; + +pub struct ReqwestGitHubApiClient { + client: Client, +} + +impl ReqwestGitHubApiClient { + pub fn new() -> MithrilResult { + let client = Client::builder() + .user_agent("mithril-client") + .build() + .context("Failed to build Reqwest GitHub API client")?; + + Ok(Self { client }) + } + + async fn download(&self, source_url: U) -> MithrilResult { + let url = source_url + .into_url() + .with_context(|| "Given `source_url` is not a valid Url")?; + let response = self + .client + .get(url.clone()) + .send() + .await + .with_context(|| format!("Failed to send request to GitHub API: {}", url))?; + match response.status() { + reqwest::StatusCode::OK => {} + status => { + return Err(anyhow!( + "GitHub API request failed with status code: {}", + status + )); + } + } + let body = response.text().await?; + let parsed_body = serde_json::from_str::(&body) + .with_context(|| format!("Failed to parse response from GitHub API: {:?}", body))?; + + Ok(parsed_body) + } +} + +#[async_trait] +impl GitHubReleaseRetriever for ReqwestGitHubApiClient { + async fn get_release_by_tag( + &self, + organization: &str, + repository: &str, + tag: &str, + ) -> MithrilResult { + let url = + format!("https://api.github.com/repos/{organization}/{repository}/releases/tags/{tag}"); + let release = self.download(url).await?; + + Ok(release) + } + + async fn get_latest_release( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult { + let url = + format!("https://api.github.com/repos/{organization}/{repository}/releases/latest"); + let release = self.download(url).await?; + + Ok(release) + } + + async fn get_prerelease( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult { + let releases = self.get_all_releases(organization, repository).await?; + let prerelease = releases + .into_iter() + .find(|release| release.prerelease) + .ok_or_else(|| anyhow!("No prerelease found"))?; + + Ok(prerelease) + } + + async fn get_all_releases( + &self, + organization: &str, + repository: &str, + ) -> MithrilResult> { + let url = format!("https://api.github.com/repos/{organization}/{repository}/releases"); + let releases = self.download(url).await?; + + Ok(releases) + } +} + +#[cfg(test)] +mod tests { + use httpmock::{Method::GET, MockServer}; + use reqwest::StatusCode; + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize, PartialEq)] + struct FakeApiResponse { + key: String, + } + + #[tokio::test] + async fn download_succeeds_with_valid_json() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(200).body(r#"{ "key": "value" }"#); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: FakeApiResponse = client + .download(format!("{}/endpoint", server.base_url())) + .await + .unwrap(); + + assert_eq!( + result, + FakeApiResponse { + key: "value".into() + } + ); + } + + #[tokio::test] + async fn download_fails_on_invalid_json() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(200).body("this is not json"); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client + .download(format!("{}/endpoint", server.base_url())) + .await; + + assert!( + result.is_err(), + "Expected an error with invalid JSON response" + ); + } + + #[tokio::test] + async fn download_fails_on_invalid_url() { + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client.download("not a valid url").await; + + assert!(result.is_err(), "Expected an error for an invalid URL"); + } + + #[tokio::test] + async fn download_fails_when_server_returns_error_and_includes_status_in_error() { + let server = MockServer::start(); + let _mock = server.mock(|when, then| { + when.method(GET).path("/endpoint"); + then.status(StatusCode::INTERNAL_SERVER_ERROR.into()); + }); + let client = ReqwestGitHubApiClient::new().unwrap(); + + let result: MithrilResult = client + .download(format!("{}/endpoint", server.base_url())) + .await; + let error = result.expect_err("Expected an error due to 500 status"); + + assert!(error + .to_string() + .contains(&StatusCode::INTERNAL_SERVER_ERROR.to_string())); + } +} diff --git a/mithril-client-cli/src/utils/http_downloader/interface.rs b/mithril-client-cli/src/utils/http_downloader/interface.rs new file mode 100644 index 00000000000..6e6d2e065e1 --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/interface.rs @@ -0,0 +1,21 @@ +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use reqwest::Url; + +use mithril_client::MithrilResult; + +/// Trait for downloading a file over HTTP from a URL, +/// saving it to a target directory with the given filename. +/// +/// Returns the path to the downloaded file. +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait HttpDownloader { + async fn download_file( + &self, + url: Url, + download_dir: &Path, + filename: &str, + ) -> MithrilResult; +} diff --git a/mithril-client-cli/src/utils/http_downloader/mod.rs b/mithril-client-cli/src/utils/http_downloader/mod.rs new file mode 100644 index 00000000000..f3c8ff4a394 --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/mod.rs @@ -0,0 +1,5 @@ +mod interface; +mod reqwest; + +pub use interface::*; +pub use reqwest::*; diff --git a/mithril-client-cli/src/utils/http_downloader/reqwest.rs b/mithril-client-cli/src/utils/http_downloader/reqwest.rs new file mode 100644 index 00000000000..4781982dae0 --- /dev/null +++ b/mithril-client-cli/src/utils/http_downloader/reqwest.rs @@ -0,0 +1,53 @@ +use std::{ + fs::File, + io::Write, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use async_trait::async_trait; +use reqwest::{Client, Url}; + +use mithril_client::MithrilResult; + +use super::HttpDownloader; + +/// [ReqwestHttpDownloader] is an implementation of the [HttpDownloader]. +pub struct ReqwestHttpDownloader { + client: Client, +} + +impl ReqwestHttpDownloader { + /// Creates a new instance of [ReqwestHttpDownloader]. + pub fn new() -> MithrilResult { + let client = Client::builder() + .build() + .with_context(|| "Failed to build Reqwest HTTP client")?; + + Ok(Self { client }) + } +} + +#[async_trait] +impl HttpDownloader for ReqwestHttpDownloader { + async fn download_file( + &self, + url: Url, + download_dir: &Path, + filename: &str, + ) -> MithrilResult { + let response = self + .client + .get(url.clone()) + .send() + .await + .with_context(|| format!("Failed to download file from URL: {}", url))?; + + let bytes = response.bytes().await?; + let download_filepath = download_dir.join(filename); + let mut file = File::create(&download_filepath)?; + file.write_all(&bytes)?; + + Ok(download_filepath) + } +} diff --git a/mithril-client-cli/src/utils/mod.rs b/mithril-client-cli/src/utils/mod.rs index 6d70b6f01f0..631ae02d3ee 100644 --- a/mithril-client-cli/src/utils/mod.rs +++ b/mithril-client-cli/src/utils/mod.rs @@ -1,17 +1,25 @@ //! Utilities module //! This module contains tools needed for the commands layer. +mod archive_unpacker; mod cardano_db; mod cardano_db_download_checker; mod expander; mod feedback_receiver; +mod fs; +mod github_release_retriever; +mod http_downloader; mod multi_download_progress_reporter; mod progress_reporter; +pub use archive_unpacker::*; pub use cardano_db::*; pub use cardano_db_download_checker::*; pub use expander::*; pub use feedback_receiver::*; +pub use fs::*; +pub use github_release_retriever::*; +pub use http_downloader::*; pub use multi_download_progress_reporter::*; pub use progress_reporter::*;