Import electrsd

The `corepc` repo is for all things integration testing. Also it
provides `bitreq` (HTTP crate), and `corepc-types` (Bitcoin Core
JSONRPC types).

When integration testing some users (eg `bdk`) use `electrsd` along
side `corepc-node` (formerly `bitcoind`). `electrsd` depends on
`corepc-node` and every time we release `corepc-node` we have to do a
release of `electrsd`. Having them in separate repositories causes
friction and makes additional work. Instead we can put them together
to hopefully make everyone's life easier (well mine and Ricardo's
anyway).

Note that `bitcoind` was written by Ricardo. I re-named it, a move I'm
now regretting because of the original symmetry with `electrsd`.

This import has the blessing of Ricardo and needs an explicit ACK from
him before merging. When the CODEOWNERS file is added (#396) it should
include him as owner of this crate.

Import the `electrsd` crate from

  https://github.com/RCasatta/electrsd/

While HEAD is at commit hash: dc88d81520c9ec507fc830b88a3b92a034dbaf93

  `dc88d81: bump version 0.36.0 -> 0.36.1`

Author: tcharding

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = [ "bitreq", "client", "jsonrpc", "node", "types"]
+members = [ "bitreq", "client", "jsonrpc", "node", "types", "electrsd"]
 exclude = ["integration_test", "verify"]
 resolver = "2"
 

electrsd/Cargo.toml

@@ -0,0 +1,58 @@
+[package]
+name = "electrsd"
+version = "0.36.1"
+authors = ["Riccardo Casatta <[email protected]>"]
+description = "Utility to run a regtest electrs process, useful in integration testing environment"
+repository = "https://github.com/RCasatta/electrsd"
+documentation = "https://docs.rs/elecrtsd/"
+license = "MIT"
+edition = "2018"
+categories = ["cryptography::cryptocurrencies", "development-tools::testing"]
+
+[dependencies]
+corepc-node = { version = "0.10.0" }
+corepc-client = { version = "0.10.0" }
+electrum-client = { version = "0.24.0", default-features = false }
+log = { version = "0.4" }
+
+[target.'cfg(not(windows))'.dependencies]
+nix = { version = "0.25.0" }
+
+[dev-dependencies]
+env_logger = { version = "0.10" }
+
+[build-dependencies]
+bitcoin_hashes = { version = "0.14", optional = true }
+zip = { version = "0.6", default-features = false, optional = true, features = [
+  "bzip2",
+  "deflate",
+] }
+minreq = { version = "2.9.0", default-features = false, optional = true, features = [
+  "https",
+] }
+
+[features]
+legacy = []
+
+# download is not supposed to be used directly only through selecting one of the version feature
+download = ["bitcoin_hashes", "zip", "minreq"]
+
+esplora_a33e97e1 = ["download", "legacy"]
+electrs_0_8_10 = ["download"]
+electrs_0_9_1 = ["download"]
+electrs_0_9_11 = ["download"]
+electrs_0_10_6 = ["download"]
+
+corepc-node_29_0 = ["corepc-node/download", "corepc-node/29_0"]
+corepc-node_28_2 = ["corepc-node/download", "corepc-node/28_2"]
+corepc-node_27_2 = ["corepc-node/download", "corepc-node/27_2"]
+corepc-node_26_2 = ["corepc-node/download", "corepc-node/26_2"]
+corepc-node_25_2 = ["corepc-node/download", "corepc-node/25_2"]
+corepc-node_24_2 = ["corepc-node/download", "corepc-node/24_2"]
+corepc-node_23_1 = ["corepc-node/download", "corepc-node/23_2"]
+corepc-node_22_1 = ["corepc-node/download", "corepc-node/22_1"]
+corepc-node_0_21_2 = ["corepc-node/download", "corepc-node/0_21_2"]
+corepc-node_0_20_2 = ["corepc-node/download", "corepc-node/0_20_2"]
+corepc-node_0_19_1 = ["corepc-node/download", "corepc-node/0_19_1"]
+corepc-node_0_18_1 = ["corepc-node/download", "corepc-node/0_18_1"]
+corepc-node_0_17_2 = ["corepc-node/download", "corepc-node/0_17_2"]

electrsd/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Riccardo Casatta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

electrsd/README.md

@@ -0,0 +1,86 @@
+[![MIT license](https://img.shields.io/github/license/RCasatta/electrsd)](https://github.com/RCasatta/electrsd/blob/master/LICENSE)
+[![Crates](https://img.shields.io/crates/v/electrsd.svg)](https://crates.io/crates/electrsd)
+
+# Electrsd
+
+Utility to run a regtest [electrs](https://github.com/romanz/electrs/) process connected to a given [bitcoind](https://github.com/RCasatta/bitcoind) instance, 
+useful in integration testing environment.
+
+```rust
+let bitcoind = bitcoind::BitcoinD::new("/usr/local/bin/bitcoind").unwrap();
+let electrsd = electrsd::ElectrsD::new("/usr/local/bin/electrs", bitcoind).unwrap();
+let header = electrsd.client.block_headers_subscribe().unwrap();
+assert_eq!(header.height, 0);
+```
+
+## Automatic binaries download
+
+In your project Cargo.toml, activate the following features
+
+```yml
+electrsd = { version= "0.23", features = ["corepc-node_23_1", "electrs_0_9_1"] }
+```
+
+Then use it:
+
+```rust
+let bitcoind_exe = bitcoind::downloaded_exe_path().expect("bitcoind version feature must be enabled");
+let bitcoind = bitcoind::BitcoinD::new(bitcoind_exe).unwrap();
+let electrs_exe = electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
+let electrsd = electrsd::ElectrsD::new(electrs_exe, bitcoind).unwrap();
+```
+
+When the `ELECTRSD_DOWNLOAD_ENDPOINT`/`BITCOIND_DOWNLOAD_ENDPOINT` environment variables are set,
+`electrsd`/`bitcoind` will try to download the binaries from the given endpoints.
+
+When you don't use the auto-download feature you have the following options:
+
+- have `electrs` executable in the `PATH`
+- provide the `electrs` executable via the `ELECTRS_EXEC` env var
+
+```rust
+if let Ok(exe_path) = electrsd::exe_path() {
+  let electrsd = electrsd::electrsD::new(exe_path).unwrap();
+}
+```
+
+Startup options could be configured via the `Conf` struct using `electrsD::with_conf` or `electrsD::from_downloaded_with_conf`.
+
+## Nix
+
+For determinisim, in nix you cannot hit the internet within the `build.rs`. Moreover, some downstream crates cannot remove the auto-download feature from their dev-deps. In this case you can set the `ELECTRSD_SKIP_DOWNLOAD` env var and provide the electrs executable in the `PATH` (or skip the test execution).
+
+## Issues with traditional approach
+
+I used integration testing based on external bash script launching needed external processes, there are many issues with this approach like:
+
+* External script may interfere with local development environment https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/200fc8247c1896709a673b82a89ca0da5e7aa2ce/integration_test/run.sh#L9
+* Use of a single huge test to test everything https://github.com/rust-bitcoin/rust-bitcoincore-rpc/blob/200fc8247c1896709a673b82a89ca0da5e7aa2ce/integration_test/src/main.rs#L122-L203
+* If test are separated, a failing test may fail to leave a clean situation, causing other test to fail (because of the initial situation, not a real failure)
+* bash script are hard, especially support different OS and versions
+
+## Features
+
+  * electrsd use a temporary directory as db dir
+  * A free port is asked to the OS (a very low probability race condition is still possible) 
+  * The process is killed when the struct goes out of scope no matter how the test finishes
+  * Automatically download `electrs` executable with enabled features. Since there are no official binaries, they are built using the [manual workflow](.github/workflows/build_electrs.yml) under this project. Supported version are:
+    * [electrs 0.10.6](https://github.com/romanz/electrs/releases/tag/v0.10.6) (feature=electrs_0_10_6)
+    * [electrs 0.9.11](https://github.com/romanz/electrs/releases/tag/v0.9.11) (feature=electrs_0_9_11)
+    * [electrs 0.9.1](https://github.com/romanz/electrs/releases/tag/v0.9.1) (feature=electrs_0_9_1)
+    * [electrs 0.8.10](https://github.com/romanz/electrs/releases/tag/v0.8.10) (feature=electrs_0_8_10)
+    * [electrs esplora](https://github.com/Blockstream/electrs/tree/a33e97e1a1fc63fa9c20a116bb92579bbf43b254) (feature=esplora_a33e97e1)
+
+Thanks to these features every `#[test]` could easily run isolated with its own environment
+
+## Deprecations
+
+- Starting from version `0.26` the env var `ELECTRS_EXE` is deprecated in favor of `ELECTRS_EXEC`.
+
+
+## Used by
+
+  * [bdk](https://github.com/bitcoindevkit/bdk)
+  * [BEWallet](https://github.com/LeoComandini/BEWallet)
+  * [gdk rust](https://github.com/Blockstream/gdk/blob/master/subprojects/gdk_rust/)
+  * [lwk](https://github.com/Blockstream/lwk)

electrsd/build.rs

@@ -0,0 +1,83 @@
+#[cfg(not(feature = "download"))]
+fn main() {}
+
+#[cfg(feature = "download")]
+fn main() {
+    download::download()
+}
+
+#[cfg(feature = "download")]
+mod download {
+    use bitcoin_hashes::{sha256, Hash};
+    use std::fs::File;
+    use std::io::{BufRead, BufReader, Cursor};
+    use std::os::unix::fs::PermissionsExt;
+    use std::path::Path;
+    use std::str::FromStr;
+
+    include!("src/versions.rs");
+
+    const GITHUB_URL: &str =
+        "https://github.com/RCasatta/electrsd/releases/download/electrs_releases";
+
+    fn get_expected_sha256(filename: &str) -> Result<sha256::Hash, ()> {
+        let file = File::open("sha256").map_err(|_| ())?;
+        for line in BufReader::new(file).lines().flatten() {
+            let tokens: Vec<_> = line.split("  ").collect();
+            if tokens.len() == 2 && filename == tokens[1] {
+                return sha256::Hash::from_str(tokens[0]).map_err(|_| ());
+            }
+        }
+        Err(())
+    }
+
+    pub fn download() {
+        if std::env::var_os("ELECTRSD_SKIP_DOWNLOAD").is_some() {
+            return;
+        }
+
+        if !HAS_FEATURE {
+            return;
+        }
+        let download_filename_without_extension = electrs_name();
+        let download_filename = format!("{}.zip", download_filename_without_extension);
+        dbg!(&download_filename);
+        let expected_hash = get_expected_sha256(&download_filename).unwrap();
+        let out_dir = std::env::var_os("OUT_DIR").unwrap();
+        let electrs_exe_home = Path::new(&out_dir).join("electrs");
+        let destination_filename = electrs_exe_home
+            .join(&download_filename_without_extension)
+            .join("electrs");
+
+        dbg!(&destination_filename);
+
+        if !destination_filename.exists() {
+            println!(
+                "filename:{} version:{} hash:{}",
+                download_filename, VERSION, expected_hash
+            );
+
+            let download_endpoint =
+                std::env::var("ELECTRSD_DOWNLOAD_ENDPOINT").unwrap_or(GITHUB_URL.to_string());
+            let url = format!("{}/{}", download_endpoint, download_filename);
+
+            let downloaded_bytes = minreq::get(url).send().unwrap().into_bytes();
+
+            let downloaded_hash = sha256::Hash::hash(&downloaded_bytes);
+            assert_eq!(expected_hash, downloaded_hash);
+            let cursor = Cursor::new(downloaded_bytes);
+
+            let mut archive = zip::ZipArchive::new(cursor).unwrap();
+            let mut file = archive.by_index(0).unwrap();
+            std::fs::create_dir_all(destination_filename.parent().unwrap()).unwrap();
+            let mut outfile = std::fs::File::create(&destination_filename).unwrap();
+
+            std::io::copy(&mut file, &mut outfile).unwrap();
+            std::fs::set_permissions(
+                &destination_filename,
+                std::fs::Permissions::from_mode(0o755),
+            )
+            .unwrap();
+        }
+    }
+}

electrsd/sha256

@@ -0,0 +1,10 @@
+2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd  electrs_macos_esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254.zip
+865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1  electrs_linux_esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254.zip
+0459d493d399bdb9ef145c84125c3cd26c1993a48efe59fa9d3fa13a03b2f555  electrs_linux_v0.8.10.zip
+48c857ca953ea66ee31c4da5a801298c85815e792ab57291107e77a4871d5421  electrs_macos_v0.8.10.zip
+fee5cc9b6c8bbd3adc45c63c881844d948c1b4dd6817f99ee087a0ccc4ba3be0  electrs_linux_v0.9.1.zip
+10f468e9e617bfe8f9f4897fa4cbbb92fe809d977b747e4326f4c8e5dc1b3a51  electrs_macos_v0.9.1.zip
+2b2f8aef35cd8e16e109b948a903d010aa472f6cdf2147d47e01fd95cd1785da  electrs_linux_v0.9.11.zip
+b794287a5d98e590deadf07a3eb391cc1a53ef160c8cdcb8e6b14d856c7b181d  electrs_macos_v0.9.11.zip
+448693f42fa2e310bd86ba9a7304c9ab464854a3c7e4c3eaa8c774efeb0fbdd1  electrs_linux_v0.10.6.zip
+016ad9ef227c12ae6096ada1db09179d3330a4062ba2ab65390d6988f659be05  electrs_macos_v0.10.6.zip

electrsd/src/error.rs

@@ -0,0 +1,75 @@
+/// All the possible error in this crate
+#[derive(Debug)]
+pub enum Error {
+    /// Wrapper of io Error
+    Io(std::io::Error),
+
+    /// Wrapper of bitcoind Error
+    Bitcoind(corepc_node::Error),
+
+    /// Wrapper of electrum_client Error
+    ElectrumClient(electrum_client::Error),
+
+    /// Wrapper of nix Error
+    #[cfg(not(target_os = "windows"))]
+    Nix(nix::Error),
+
+    /// Wrapper of early exit status
+    EarlyExit(std::process::ExitStatus),
+
+    /// Returned when both tmpdir and staticdir is specified in `Conf` options
+    BothDirsSpecified,
+
+    /// Returned when calling methods requiring the bitcoind executable but none is found
+    /// (no feature, no `ELECTRS_EXEC`, no `electrs` in `PATH` )
+    NoElectrsExecutableFound,
+
+    /// Returned if both env vars `ELECTRS_EXEC` and `ELECTRS_EXE` are found
+    BothEnvVars,
+}
+
+impl std::error::Error for Error {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Error::Io(e) => Some(e),
+            Error::Bitcoind(e) => Some(e),
+            Error::ElectrumClient(e) => Some(e),
+            // Error::BitcoinCoreRpc(e) => Some(e),
+            #[cfg(not(target_os = "windows"))]
+            Error::Nix(e) => Some(e),
+
+            _ => None,
+        }
+    }
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Self {
+        Error::Io(e)
+    }
+}
+
+impl From<corepc_node::Error> for Error {
+    fn from(e: corepc_node::Error) -> Self {
+        Error::Bitcoind(e)
+    }
+}
+
+impl From<electrum_client::Error> for Error {
+    fn from(e: electrum_client::Error) -> Self {
+        Error::ElectrumClient(e)
+    }
+}
+
+#[cfg(not(target_os = "windows"))]
+impl From<nix::Error> for Error {
+    fn from(e: nix::Error) -> Self {
+        Error::Nix(e)
+    }
+}