Key image store service and server for router/store system (#2954)

* Ledger enclave support for router and store (#2896)

* Pulling changes in from milliec/ledger-router-dev

* Run cargo fmt

* Making requested changes and clarifications to Fog ledger router comments.

* Additional comment fixes

* Fix CI lint

* PR feedback nits

Co-authored-by: Andrew Wygle <[email protected]>

* Fog Ledger Router Admin service

* Sort itertools properly in Cargo.toml

* Key Image Router Server + Binary

* Update router config for parameterized retries

* Changes due to rebase

* normalize naming - ledger router, key image store

* Linting fixes

* Accept code review suggestions

* Updates for GRPCIO 0.12

* Remove some unwraps in ledger_router binary

* Pulling changes in from milliec/ledger-router-dev

* Making requested changes and clarifications to Fog ledger router comments.

* PR feedback nits

* Key Image Router Service

* Update fog/ledger/server/src/key_image_router_service.rs

Co-authored-by: Nick Santana <[email protected]>

* Address PR feedback around logging and comments.

* Parameterize allowed number of retries for query loop

* Fog Ledger Router Admin service

* Sort itertools properly in Cargo.toml

* Key Image Router Server + Binary

* Key image store changes pulled in from milliec/ledger-router-dev

* Cargo fmt

* Run clippy

* Sort itertools properly in Cargo.toml

* Rebase and update to match current fog ledger router branch

* Remove unused dependencies in fog-ledger-server

Co-authored-by: Nick Santana <[email protected]>

* Apply suggestions - Remove unused deps

Co-authored-by: Nick Santana <[email protected]>

* Apply suggestions from comments

Applying suggestions from @nick-mobilecoin's review

Co-authored-by: Nick Santana <[email protected]>

* Resolving some code quality issues in direct_key_image_store_check()

* Cargo fmt

* key image server and key image service moved to router server and router service

* Apply suggestions from code review

Co-authored-by: Sam Dealy <[email protected]>
Co-authored-by: Nick Santana <[email protected]>

---------

Co-authored-by: Andrew Wygle <[email protected]>
Co-authored-by: Nick Santana <[email protected]>
Co-authored-by: Sam Dealy <[email protected]>

fog/ledger/server/Cargo.toml

@@ -19,6 +19,10 @@ path = "src/bin/main.rs"
 name = "ledger_router"
 path = "src/bin/router.rs"
 
+[[bin]]
+name = "key_image_store"
+path = "src/bin/key_image_store.rs"
+
 [dependencies]
 mc-attest-api = { path = "../../../attest/api" }
 mc-attest-core = { path = "../../../attest/core" }
@@ -72,9 +76,11 @@ mc-util-build-sgx = { path = "../../../util/build/sgx" }
 # mobilecoin
 mc-account-keys = { path = "../../../account-keys" }
 mc-api = { path = "../../../api" }
+mc-attest-ake = { path = "../../../attest/ake" }
 mc-blockchain-test-utils = { path = "../../../blockchain/test-utils" }
 mc-common = { path = "../../../common", features = ["loggers"] }
 mc-crypto-keys = { path = "../../../crypto/keys" }
+mc-crypto-rand = { path = "../../../crypto/rand" }
 mc-util-test-helper = { path = "../../../util/test-helper" }
 mc-util-uri = { path = "../../../util/uri" }
 
@@ -86,3 +92,6 @@ mc-fog-test-infra = { path = "../../test_infra" }
 
 # third-party
 tempfile = "3.4"
+aes-gcm = "0.10.1"
+sha2 = "0.10"
+tempdir = "0.3"

fog/ledger/server/src/bin/key_image_store.rs

@@ -0,0 +1,74 @@
+// Copyright (c) 2018-2022 The MobileCoin Foundation
+
+use clap::Parser;
+use grpcio::{RpcStatus, RpcStatusCode};
+use mc_common::{logger::log, time::SystemTimeProvider};
+use mc_fog_ledger_enclave::{LedgerSgxEnclave, ENCLAVE_FILE};
+use mc_fog_ledger_server::{KeyImageStoreServer, LedgerStoreConfig};
+use mc_ledger_db::LedgerDB;
+use mc_util_grpc::AdminServer;
+use mc_watcher::watcher_db::WatcherDB;
+
+use std::{env, sync::Arc};
+
+fn main() {
+    let (logger, _global_logger_guard) =
+        mc_common::logger::create_app_logger(mc_common::logger::o!());
+    mc_common::setup_panic_handler();
+    let config = LedgerStoreConfig::parse();
+
+    let enclave_path = env::current_exe()
+        .expect("Could not get the path of our executable")
+        .with_file_name(ENCLAVE_FILE);
+    log::info!(
+        logger,
+        "enclave path {}, responder ID {}",
+        enclave_path
+            .to_str()
+            .expect("enclave path is not valid UTF-8"),
+        &config.client_responder_id
+    );
+    let enclave = LedgerSgxEnclave::new(
+        enclave_path,
+        &config.client_responder_id,
+        config.omap_capacity,
+        logger.clone(),
+    );
+
+    //Get our ledger connection started.
+    let db = LedgerDB::open(&config.ledger_db).expect("Could not read ledger DB");
+    let watcher =
+        WatcherDB::open_ro(&config.watcher_db, logger.clone()).expect("Could not open watcher DB");
+
+    let mut store_server = KeyImageStoreServer::new_from_config(
+        config.clone(),
+        enclave,
+        db,
+        watcher,
+        SystemTimeProvider::default(),
+        logger.clone(),
+    );
+    store_server.start();
+
+    //Initialize the admin api
+    let config2 = config.clone();
+    let get_config_json = Arc::new(move || {
+        serde_json::to_string(&config2)
+            .map_err(|err| RpcStatus::with_message(RpcStatusCode::INTERNAL, format!("{err:?}")))
+    });
+    let _admin_server = config.admin_listen_uri.as_ref().map(|admin_listen_uri| {
+        AdminServer::start(
+            None,
+            admin_listen_uri,
+            "Fog Ledger".to_owned(),
+            config.client_responder_id.to_string(),
+            Some(get_config_json),
+            logger,
+        )
+        .expect("Failed starting admin server")
+    });
+
+    loop {
+        std::thread::sleep(std::time::Duration::from_millis(1000));
+    }
+}

fog/ledger/server/src/config.rs

@@ -7,7 +7,7 @@
 use clap::Parser;
 use mc_attest_core::ProviderId;
 use mc_common::ResponderId;
-use mc_fog_uri::FogLedgerUri;
+use mc_fog_uri::{FogLedgerUri, KeyImageStoreUri};
 use mc_util_parse::parse_duration_in_seconds;
 use mc_util_uri::AdminUri;
 use serde::Serialize;
@@ -96,8 +96,8 @@ pub struct LedgerRouterConfig {
     #[clap(long, env = "MC_ADMIN_LISTEN_URI")]
     pub admin_listen_uri: AdminUri,
 
-    /// Number of query attempts with no forward progress before reporting an
-    /// error.
+    /// Number of query attempts with no forward progress
+    /// before reporting an error.
     #[clap(long, default_value = "3")]
     pub query_retries: usize,
 
@@ -115,3 +115,81 @@ pub struct LedgerRouterConfig {
     #[clap(long, default_value = "1048576", env = "MC_OMAP_CAPACITY")]
     pub omap_capacity: u64,
 }
+
+/// Configuration parameters for the Fog Ledger Store service.
+#[derive(Clone, Parser, Serialize)]
+#[clap(version)]
+pub struct LedgerStoreConfig {
+    /// The chain id of the network we are a part of
+    #[clap(long, env = "MC_CHAIN_ID")]
+    pub chain_id: String,
+
+    /// The ID with which to respond to client attestation requests.
+    ///
+    /// This ID needs to match the host:port clients use in their URI when
+    /// referencing this node.
+    #[clap(long, env = "MC_CLIENT_RESPONDER_ID")]
+    pub client_responder_id: ResponderId,
+
+    /// gRPC listening URI for client requests.
+    #[clap(long, env = "MC_CLIENT_LISTEN_URI")]
+    pub client_listen_uri: KeyImageStoreUri,
+
+    /// Path to ledger db (lmdb)
+    #[clap(long, value_parser(clap::value_parser!(PathBuf)), env = "MC_LEDGER_DB")]
+    pub ledger_db: PathBuf,
+
+    /// Path to watcher db (lmdb) - includes block timestamps
+    #[clap(long, value_parser(clap::value_parser!(PathBuf)), env = "MC_WATCHER_DB")]
+    pub watcher_db: PathBuf,
+
+    /// IAS Api Key.
+    #[clap(long, env = "MC_IAS_API_KEY")]
+    pub ias_api_key: String,
+
+    /// IAS Service Provider ID.
+    #[clap(long, env = "MC_IAS_SPID")]
+    pub ias_spid: ProviderId,
+
+    /// Optional admin listening URI.
+    #[clap(long, env = "MC_ADMIN_LISTEN_URI")]
+    pub admin_listen_uri: Option<AdminUri>,
+
+    /// Enables authenticating client requests using Authorization tokens using
+    /// the provided hex-encoded 32 bytes shared secret.
+    #[clap(long, value_parser = mc_util_parse::parse_hex::<[u8; 32]>, env = "MC_CLIENT_AUTH_TOKEN_SECRET")]
+    pub client_auth_token_secret: Option<[u8; 32]>,
+
+    /// Maximal client authentication token lifetime, in seconds (only relevant
+    /// when --client-auth-token-secret is used. Defaults to 86400 - 24
+    /// hours).
+    #[clap(long, default_value = "86400", value_parser = parse_duration_in_seconds, env = "MC_CLIENT_AUTH_TOKEN_MAX_LIFETIME")]
+    pub client_auth_token_max_lifetime: Duration,
+
+    /// The capacity to build the OMAP (ORAM hash table) with.
+    /// About 75% of this capacity can be used.
+    /// The hash table will overflow when there are more Keyimages than this,
+    /// and the server will have to be restarted with a larger number.
+    ///
+    /// Note: At time of writing, the hash table will be allocated to use all
+    /// available SGX EPC memory, and then beyond that it will be allocated on
+    /// the heap in the untrusted side. Once the needed capacity exceeds RAM,
+    /// you will either get killed by OOM killer, or it will start being swapped
+    /// to disk by linux kernel.
+    #[clap(long, default_value = "1048576", env = "MC_OMAP_CAPACITY")]
+    pub omap_capacity: u64,
+}
+
+/// Uri for any node in the key image store system.
+/// Old-style single-node servers and routers are both referred to with
+/// a KeyImageClientListenUri::ClientFacing(FogLedgerUri), whereas ledger
+/// store shard Uris will be KeyImageClientListenUri::Store(KeyImageStoreUri).
+#[derive(Clone, Serialize)]
+pub enum KeyImageClientListenUri {
+    /// URI used by the KeyImageStoreServer when fulfilling direct client
+    /// requests.
+    ClientFacing(FogLedgerUri),
+    /// URI used by the KeyImageStoreServer when fulfilling Fog Ledger Router
+    /// requests.
+    Store(KeyImageStoreUri),
+}