From b106fbad58ada9eec4aae22cba8fadd93736caef Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 24 Aug 2026 16:55:19 -0700 Subject: [PATCH 1/7] feat(object-store): add a provider-neutral storage seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Buzz keeps media blobs and the content-addressed Git object store on one bucket, but each held its own rust-s3 client and spoke in S3-shaped vocabulary: ETags, If-Match, If-None-Match: *. That made the storage provider a property of every call site rather than a property of the deployment, and left no place to add a second provider. Add `buzz-object-store`, which owns the seam: - `ObjectStore` — exactly the operations Buzz performs, no more: buffered and streaming upload, full/bounded/range/streaming read, head, create-only and revision-matched conditional writes, paginated prefix listing, single and bulk delete, connectivity and versioning admission checks. - `Revision` / `WriteCondition` / `ConditionalWrite` / `ImmutableWrite` — provider-safe types, so a compare-and-swap token is never a bare string with implied semantics. A revision minted by one provider cannot predicate a write against another; the accessor rejects it. - `ObjectStoreError` — an explicit taxonomy separating a *classified* provider answer (not found, precondition failed, throttled, transient, permanent) from an *unknown* transport outcome. That distinction is load bearing: the Git conformance probe drops unknown outcomes from its observer set rather than counting them as a lost race, so `TransportAmbiguous` stays reserved for pre-classification failures. - `ProviderKind`, so a deployment names its provider once. No behavior change: this commit adds the crate, its unit tests, and registers it in the workspace. The domain facades move onto it next. Signed-off-by: mozarthq --- Cargo.lock | 17 + Cargo.toml | 2 + crates/buzz-object-store/Cargo.toml | 21 + crates/buzz-object-store/src/error.rs | 197 +++++ crates/buzz-object-store/src/lib.rs | 245 ++++++ crates/buzz-object-store/src/providers/mod.rs | 7 + crates/buzz-object-store/src/providers/s3.rs | 701 ++++++++++++++++++ crates/buzz-object-store/src/revision.rs | 150 ++++ 8 files changed, 1340 insertions(+) create mode 100644 crates/buzz-object-store/Cargo.toml create mode 100644 crates/buzz-object-store/src/error.rs create mode 100644 crates/buzz-object-store/src/lib.rs create mode 100644 crates/buzz-object-store/src/providers/mod.rs create mode 100644 crates/buzz-object-store/src/providers/s3.rs create mode 100644 crates/buzz-object-store/src/revision.rs diff --git a/Cargo.lock b/Cargo.lock index 9544a63b899..ab337ce6b73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1157,6 +1157,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-object-store" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "bytes", + "futures-core", + "futures-util", + "rust-s3", + "serde", + "thiserror 2.0.18", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "buzz-pair-relay" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d6ee839f1b0..de373816410 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/buzz-deletion", "crates/buzz-workflow", "crates/buzz-media", + "crates/buzz-object-store", "crates/buzz-cli", "crates/buzz-pairing-cli", "crates/buzz-sdk", @@ -146,6 +147,7 @@ buzz-search = { path = "crates/buzz-search" } buzz-audit = { path = "crates/buzz-audit" } buzz-workflow = { path = "crates/buzz-workflow" } buzz-media = { path = "crates/buzz-media" } +buzz-object-store = { path = "crates/buzz-object-store" } buzz-sdk = { path = "crates/buzz-sdk" } buzz-ws-client = { path = "crates/buzz-ws-client" } buzz-relay-mesh = { path = "crates/buzz-relay-mesh" } diff --git a/crates/buzz-object-store/Cargo.toml b/crates/buzz-object-store/Cargo.toml new file mode 100644 index 00000000000..4a8df03b39f --- /dev/null +++ b/crates/buzz-object-store/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "buzz-object-store" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Provider-neutral object storage seam shared by Buzz media and Git-on-object-storage" + +[dependencies] +async-trait = "0.1" +axum = { workspace = true } +bytes = "1" +futures-core = "0.3" +futures-util = "0.3" +s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } +serde = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } diff --git a/crates/buzz-object-store/src/error.rs b/crates/buzz-object-store/src/error.rs new file mode 100644 index 00000000000..ddbf83acf54 --- /dev/null +++ b/crates/buzz-object-store/src/error.rs @@ -0,0 +1,197 @@ +//! Provider-neutral object-store error taxonomy. +//! +//! The distinction that matters most here is **classified vs. unknown**. A +//! provider that answered — with a status, a body, or a malformed response — +//! told us something about the object's state. A request that never got a +//! classified answer (socket refused, send flaked, mid-flight reset) told us +//! nothing: it is neither evidence that the write committed nor that it did +//! not. The Git conformance probe drops exactly the unknown outcomes from its +//! observer set, so [`ObjectStoreError::TransportAmbiguous`] must stay +//! reserved for pre-classification failures and nothing else. + +use std::time::Duration; + +use crate::revision::ProviderKind; + +/// Everything a provider operation can fail with. +/// +/// Losing a compare-and-swap race is deliberately *not* in here — that is +/// [`crate::ConditionalWrite::Conflict`], an ordinary outcome. +#[derive(Debug, thiserror::Error)] +pub enum ObjectStoreError { + /// The requested key does not exist. + #[error("object not found: {key}")] + NotFound { + /// Object key that was addressed. + key: String, + }, + + /// A precondition failed on an operation that is not a conditional write. + /// + /// Conditional writes surface a failed precondition as + /// [`crate::ConditionalWrite::Conflict`] instead; this variant catches a + /// precondition failure arriving where no CAS semantics were requested. + #[error("precondition failed on {key}")] + Conflict { + /// Object key that was addressed. + key: String, + }, + + /// The provider rejected the request for exceeding its request rate. + /// + /// Throttling is never evidence that a writer lost a CAS race. + #[error("object store throttled the {operation} request")] + Throttled { + /// Provider operation that was throttled. + operation: &'static str, + /// Provider-advertised backoff, when the response carried one. + retry_after: Option, + }, + + /// The provider answered with a transient failure; the operation may be + /// retried under the caller's policy (with its original precondition, if + /// it had one). + #[error("retryable object store failure during {operation}: {message}")] + TransportRetryable { + /// Provider operation that failed. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// The request never produced a classified provider response, so the + /// operation's outcome is unknown. + /// + /// A conditional write that fails this way must never be retried + /// unconditionally: reread the object and classify the committed revision + /// instead. + #[error("ambiguous object store outcome during {operation}: {message}")] + TransportAmbiguous { + /// Provider operation whose outcome is unknown. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// A permanent provider or authorization failure. + #[error("object store backend error during {operation}: {message}")] + Provider { + /// Provider operation that failed. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// Invalid storage configuration, detected at client construction. + #[error("object store config error: {0}")] + Config(String), + + /// The object is larger than the caller's bounded read budget. + #[error("object too large: {key} is {size} bytes (max {max})")] + ObjectTooLarge { + /// Object key that was read. + key: String, + /// Object size reported by the provider. + size: u64, + /// Maximum bytes the caller allows for this read. + max: u64, + }, + + /// A content-addressed read returned bytes that do not hash to the key. + #[error("digest mismatch on {key}: expected {expected}, got {actual}")] + DigestMismatch { + /// Object key that was read. + key: String, + /// Digest the caller expected (the content-addressed key). + expected: String, + /// Digest computed from the returned bytes. + actual: String, + }, + + /// A revision minted by one provider was presented to another. + #[error("revision provider mismatch: expected {expected}, got {actual}")] + RevisionMismatch { + /// Provider the operation is running against. + expected: ProviderKind, + /// Provider that minted the offered revision. + actual: ProviderKind, + }, +} + +impl ObjectStoreError { + /// Whether the operation's outcome is unknown rather than classified. + /// + /// Callers that reason about *observers* — the Git conformance probe, and + /// any future CAS retry policy — use this to drop unknown outcomes from + /// the evidence set rather than counting them as a lost race. + pub fn is_ambiguous(&self) -> bool { + matches!(self, Self::TransportAmbiguous { .. }) + } + + /// Whether the provider indicated the request may be retried as-is. + /// + /// A conditional write may only be retried with its original precondition. + pub fn is_retryable(&self) -> bool { + matches!( + self, + Self::Throttled { .. } | Self::TransportRetryable { .. } + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_pre_classification_failures_are_ambiguous() { + assert!(ObjectStoreError::TransportAmbiguous { + operation: "put", + message: "connection reset".into(), + } + .is_ambiguous()); + + for classified in [ + ObjectStoreError::NotFound { key: "k".into() }, + ObjectStoreError::Conflict { key: "k".into() }, + ObjectStoreError::Throttled { + operation: "put", + retry_after: None, + }, + ObjectStoreError::TransportRetryable { + operation: "put", + message: "503".into(), + }, + ObjectStoreError::Provider { + operation: "put", + message: "403".into(), + }, + ObjectStoreError::Config("bad".into()), + ] { + assert!( + !classified.is_ambiguous(), + "{classified} must stay a classified observation" + ); + } + } + + #[test] + fn throttling_and_transient_failures_are_retryable() { + assert!(ObjectStoreError::Throttled { + operation: "get", + retry_after: Some(Duration::from_secs(1)), + } + .is_retryable()); + assert!(ObjectStoreError::TransportRetryable { + operation: "get", + message: "503".into(), + } + .is_retryable()); + assert!(!ObjectStoreError::TransportAmbiguous { + operation: "get", + message: "reset".into(), + } + .is_retryable()); + assert!(!ObjectStoreError::NotFound { key: "k".into() }.is_retryable()); + } +} diff --git a/crates/buzz-object-store/src/lib.rs b/crates/buzz-object-store/src/lib.rs new file mode 100644 index 00000000000..590fd05f242 --- /dev/null +++ b/crates/buzz-object-store/src/lib.rs @@ -0,0 +1,245 @@ +//! Provider-neutral object storage for Buzz. +//! +//! Buzz keeps two very different workloads on one bucket: media blobs +//! (`buzz-media`) and the content-addressed Git object store with its +//! compare-and-swap ref pointer (`buzz-relay`'s `api::git::store`). Both used +//! to hold their own `rust-s3` client and speak in S3-shaped vocabulary — +//! ETags, `If-Match`, `If-None-Match: *`. That made the storage provider a +//! property of every call site rather than a property of the deployment. +//! +//! This crate is the seam. It owns: +//! +//! - the [`ObjectStore`] trait — exactly the operations Buzz performs, no more; +//! - provider-safe [`Revision`] / [`WriteCondition`] / [`ConditionalWrite`] +//! types, so a compare-and-swap token is never a bare string with implied +//! semantics; +//! - the [`ObjectStoreError`] taxonomy, which separates a *classified* +//! provider answer from an *unknown* transport outcome; +//! - the S3 provider ([`providers::s3`]), which is the only place an ETag +//! exists. +//! +//! Domain code above this seam — `MediaStorage`, `GitStore` — is a thin facade +//! that adds Buzz semantics (tenant-scoped sidecar keys, content addressing, +//! digest verification) and never names a provider. + +pub mod error; +pub mod providers; +pub mod revision; + +use std::path::Path; +use std::pin::Pin; + +use async_trait::async_trait; +use bytes::Bytes; + +pub use error::ObjectStoreError; +pub use providers::s3::{S3AddressingStyle, S3ObjectStore, S3StoreConfig}; +pub use revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; + +/// A stream of object byte chunks, usable with `axum::body::Body::from_stream()`. +pub type ByteStream = + Pin> + Send>>; + +/// Outcome of a create-only write of an immutable, content-addressed object. +/// +/// Distinct from [`ConditionalWrite`] because content addressing makes the +/// committed revision uninteresting: the key *is* the digest, so a key that +/// already exists already holds these exact bytes. Callers treat both variants +/// as success and only the conformance probe distinguishes them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImmutableWrite { + /// This call wrote the object. + Created, + /// The key already held an object — by content addressing, the same bytes. + AlreadyPresent, +} + +/// Object metadata from a HEAD, as much as every provider can supply. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectMeta { + /// Object size in bytes. + pub size: u64, + /// Current revision, when the provider reports one on HEAD. + pub revision: Option, +} + +/// One page of a prefix-scoped listing. +/// +/// Keys arrive in ascending UTF-8 binary order; callers rely on that for +/// streaming key-stream digests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ListPage { + /// `(key, size_in_bytes)` for each object in this page. + pub objects: Vec<(String, u64)>, + /// Token that fetches the next page, when one exists. + pub next_continuation_token: Option, + /// Whether the provider truncated the listing. + pub is_truncated: bool, +} + +/// Per-key outcomes of one bulk delete. +/// +/// Bulk deletion never fails on per-key outcomes: they are folded in here so +/// the caller owns retry and fail-closed policy. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BulkDeleteOutcome { + /// Keys the backend reported deleted (providers report already-missing + /// keys as deleted too — the API is idempotent by design). + pub deleted: u64, + /// Keys reported absent through a per-key "no such key" error; equivalent + /// to deleted for retry purposes. + pub already_missing: u64, + /// Keys whose deletion produced a version artifact (delete marker or + /// version id) — evidence of bucket versioning, which deletion must fail + /// closed on. + pub versioned_keys: Vec, + /// Remaining per-key failures as `(key, code, message)`. + pub failed: Vec<(String, String, String)>, +} + +/// The object-store operations Buzz actually performs. +/// +/// Implementations are shared across the process behind an `Arc`: the relay +/// constructs exactly one provider and hands it to both the media facade and +/// the Git facade. +#[async_trait] +pub trait ObjectStore: Send + Sync { + /// Which provider backs this client. + fn provider(&self) -> ProviderKind; + + /// Store an object from a byte slice. + /// + /// For large blobs prefer [`ObjectStore::put_file`], which never holds the + /// whole body in memory. + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError>; + + /// Stream a file from disk into the store without loading it into RAM. + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError>; + + /// Create-only write of an immutable, content-addressed object. + /// + /// A precondition failure is [`ImmutableWrite::AlreadyPresent`], not an + /// error: the key is the digest of the bytes, so a collision means the + /// stored bytes already equal these bytes. + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result; + + /// Write an object under a precondition. + /// + /// A failed precondition is [`ConditionalWrite::Conflict`], not an error. + /// On [`ConditionalWrite::Committed`] the returned [`Revision`] is read + /// from the write response and predicates the next conditional write; a + /// provider that commits without returning a revision is non-conforming + /// and must fail the operation rather than return an unusable token. + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result; + + /// Read an object's full body. + async fn get(&self, key: &str) -> Result; + + /// Read an inclusive byte range from an object. + /// + /// Only the requested slice is transferred; the full object is never + /// loaded into memory. + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result; + + /// Read an object as a chunk stream, without buffering the whole body. + async fn get_stream(&self, key: &str) -> Result; + + /// Read an object's body **and** its revision from a single response. + /// + /// Returns `Ok(None)` when the object does not exist. + /// + /// A HEAD followed by a GET can straddle a concurrent writer: the HEAD's + /// revision and the GET's body would describe different versions, and a + /// caller that predicated its next write on the HEAD revision would be + /// predicating on a version it never read. Both fields come from one + /// response so the snapshot stays consistent. + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError>; + + /// Read an object's metadata. Returns `Ok(None)` when it does not exist. + async fn head(&self, key: &str) -> Result, ObjectStoreError>; + + /// Fetch one page of a prefix-scoped listing. + /// + /// `max_keys` bounds a single provider response, not the caller's total + /// object budget; callers enforce the cumulative cap across pages. + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result; + + /// Delete a single object. Deleting an absent object is not an error. + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError>; + + /// Delete a bounded batch of objects, reporting per-key outcomes. + /// + /// Providers with a native batch API issue one request; providers without + /// one issue bounded-concurrency individual deletes. Either way the caller + /// sees the same per-key fold and decides retry policy. + async fn delete_objects(&self, keys: &[String]) -> Result; + + /// Probe connectivity and bucket access. + async fn ping(&self) -> Result<(), ObjectStoreError>; + + /// Whether the bucket retains non-current object versions. + /// + /// Deletion refuses versioned buckets: an ordinary delete on one inserts a + /// delete marker rather than proving logical absence, so a bulk delete + /// could report success while the bytes remain reachable. + async fn versioning_detected(&self) -> Result; + + /// Read an object after rejecting bodies larger than `max_bytes`. + /// + /// The HEAD bound is checked first so an oversized object is never + /// transferred, and the returned length is re-checked in case the provider + /// reported a bad size. + async fn get_limited(&self, key: &str, max_bytes: u64) -> Result { + let meta = self + .head(key) + .await? + .ok_or_else(|| ObjectStoreError::NotFound { key: key.into() })?; + if meta.size > max_bytes { + return Err(ObjectStoreError::ObjectTooLarge { + key: key.into(), + size: meta.size, + max: max_bytes, + }); + } + + let bytes = self.get(key).await?; + let size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if size > max_bytes { + return Err(ObjectStoreError::ObjectTooLarge { + key: key.into(), + size, + max: max_bytes, + }); + } + Ok(bytes) + } +} diff --git a/crates/buzz-object-store/src/providers/mod.rs b/crates/buzz-object-store/src/providers/mod.rs new file mode 100644 index 00000000000..cd2b2a741f1 --- /dev/null +++ b/crates/buzz-object-store/src/providers/mod.rs @@ -0,0 +1,7 @@ +//! Concrete [`crate::ObjectStore`] implementations. +//! +//! Exactly one provider is constructed per process and shared by every domain +//! facade. Provider-specific vocabulary — ETags, generations, addressing +//! styles, credential chains — stays inside these modules. + +pub mod s3; diff --git a/crates/buzz-object-store/src/providers/s3.rs b/crates/buzz-object-store/src/providers/s3.rs new file mode 100644 index 00000000000..124de555b4f --- /dev/null +++ b/crates/buzz-object-store/src/providers/s3.rs @@ -0,0 +1,701 @@ +//! S3 and S3-compatible provider, backed by `rust-s3`. +//! +//! This is the only module in Buzz that knows what an ETag is. Everything +//! above the seam holds a [`Revision`] and cannot tell an ETag from a Google +//! Cloud Storage generation. +//! +//! ## The 412 sharp edge +//! +//! `rust-s3`'s `fail-on-err` Cargo feature is unified ON across the build +//! graph, so non-2xx responses arrive here as `S3Error::HttpFailWithBody(code, +//! body)` *before* the caller sees `ResponseData`. A precondition failure +//! (412) is therefore an `Err` that must be reclassified as a *semantic* +//! result — [`ConditionalWrite::Conflict`] or [`ImmutableWrite::AlreadyPresent`] +//! — rather than propagated as a backend error. Empirically verified against +//! MinIO by the Git store's `probe::probe_412_surfacing`. +//! +//! ## Classified vs. unknown outcomes +//! +//! [`classify`] draws the line the Git conformance probe depends on: +//! `S3Error::{Reqwest, Http, Io}` are pre-classification failures — the racer +//! never got an answer from the backend — and map to +//! [`ObjectStoreError::TransportAmbiguous`]. Every other variant means the +//! backend *did* answer, in or out of contract, and stays a classified +//! observation. Do not widen the ambiguous set: it would let a genuine +//! conformance failure be silently dropped from the probe's observer set. + +use std::path::Path; +use std::str::FromStr; + +use async_trait::async_trait; +use bytes::Bytes; +use s3::creds::Credentials; +use s3::error::S3Error; +use s3::{Bucket, Region}; + +use crate::error::ObjectStoreError; +use crate::revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; +use crate::{BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore}; + +/// S3 URL addressing style shared by media and Git/CAS storage. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum S3AddressingStyle { + /// Put the bucket in the request path (`https://endpoint/bucket/key`). + /// + /// This preserves compatibility with the bundled MinIO deployments, whose + /// internal DNS only resolves the endpoint hostname. + #[default] + Path, + /// Put the bucket in the hostname (`https://bucket.endpoint/key`). + /// + /// This is the standard S3 form and is required by providers such as new + /// Railway Storage Buckets. + Virtual, +} + +impl FromStr for S3AddressingStyle { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "path" => Ok(Self::Path), + "virtual" => Ok(Self::Virtual), + _ => Err(format!( + "BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}" + )), + } + } +} + +/// Connection inputs for [`S3ObjectStore::new`]. +#[derive(Debug, Clone)] +pub struct S3StoreConfig { + /// S3-compatible endpoint URL (e.g. `http://localhost:9000`). + pub endpoint: String, + /// Static access key, or empty to use the AWS credential chain. + pub access_key: String, + /// Static secret key, or empty to use the AWS credential chain. + pub secret_key: String, + /// Bucket name. + pub bucket: String, + /// Region used for SigV4 request signing. + pub region: String, + /// URL addressing style. + pub addressing_style: S3AddressingStyle, +} + +/// S3-compatible object storage client. +pub struct S3ObjectStore { + bucket: Box, +} + +impl S3ObjectStore { + /// Build a client against an S3-compatible endpoint (e.g. MinIO). + /// + /// Credential selection: + /// - If both `access_key` and `secret_key` are non-empty, use them as + /// static credentials (MinIO/local/dev, or any static-key deployment). + /// - If both are empty, fall back to the AWS default credential chain via + /// [`Credentials::default`]: environment, shared profile, web-identity + /// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and + /// instance-metadata providers, in that order. This lets the relay use + /// the pod's IAM role without long-lived static keys. + /// - If exactly one is empty, fail: a half-configured static deployment + /// must surface rather than silently fall back to the chain. + pub fn new(config: &S3StoreConfig) -> Result { + let region = Region::Custom { + region: config.region.clone(), + endpoint: config.endpoint.clone(), + }; + let creds = match (config.access_key.is_empty(), config.secret_key.is_empty()) { + (false, false) => Credentials::new( + Some(&config.access_key), + Some(&config.secret_key), + None, + None, + None, + ), + (true, true) => Credentials::default(), + _ => { + return Err(ObjectStoreError::Config( + "s3 access_key and secret_key must be configured together, or both empty to use the AWS credential chain" + .to_string(), + )); + } + } + .map_err(|e| ObjectStoreError::Config(e.to_string()))?; + let bucket = Bucket::new(&config.bucket, region, creds) + .map_err(|e| ObjectStoreError::Config(e.to_string()))?; + let bucket = match config.addressing_style { + S3AddressingStyle::Path => bucket.with_path_style(), + S3AddressingStyle::Virtual => bucket, + }; + Ok(Self { bucket }) + } + + /// The bucket's public URL, for diagnostics and tests. + pub fn url(&self) -> String { + self.bucket.url() + } + + /// Whether the client signs and routes in path-addressing style. + pub fn is_path_style(&self) -> bool { + self.bucket.is_path_style() + } + + /// The signing region configured on the client. + pub fn region(&self) -> &Region { + &self.bucket.region + } + + /// Build the precondition headers for one conditional write. + fn condition_headers( + condition: &WriteCondition, + ) -> Result { + let mut headers = axum::http::HeaderMap::new(); + match condition { + WriteCondition::Absent => { + headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); + } + WriteCondition::Matches(revision) => { + let tag = revision.expect_s3_etag()?; + headers.insert( + axum::http::header::IF_MATCH, + tag.parse().map_err(|_| ObjectStoreError::Provider { + operation: "put_conditional", + message: format!("invalid etag {tag}"), + })?, + ); + } + } + Ok(headers) + } + + /// Read the ETag out of a response's headers, in either casing. + fn etag_of(headers: &std::collections::HashMap) -> Option { + headers.get("etag").or_else(|| headers.get("ETag")).cloned() + } +} + +/// Map a `rust-s3` failure into the provider-neutral taxonomy. +/// +/// See the module docs: only pre-classification failures may become +/// [`ObjectStoreError::TransportAmbiguous`]. +fn classify(operation: &'static str, key: &str, error: S3Error) -> ObjectStoreError { + let message = error.to_string(); + match error { + S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_) => { + ObjectStoreError::TransportAmbiguous { operation, message } + } + S3Error::HttpFailWithBody(404, _) => ObjectStoreError::NotFound { key: key.into() }, + S3Error::HttpFailWithBody(412, _) => ObjectStoreError::Conflict { key: key.into() }, + S3Error::HttpFailWithBody(429, _) => ObjectStoreError::Throttled { + operation, + retry_after: None, + }, + S3Error::HttpFailWithBody(500 | 502 | 503 | 504, _) => { + ObjectStoreError::TransportRetryable { operation, message } + } + _ => ObjectStoreError::Provider { operation, message }, + } +} + +/// Fold one `DeleteObjects` response into per-key outcomes. +/// +/// Historical MinIO releases report already-absent keys as +/// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to +/// `already_missing` to keep checkpointed retry idempotent. +fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + let mut outcome = BulkDeleteOutcome::default(); + for deleted in result.deleted { + if deleted.delete_marker == Some(true) + || deleted.delete_marker_version_id.is_some() + || deleted.version_id.is_some() + { + outcome.versioned_keys.push(deleted.key); + } else { + outcome.deleted += 1; + } + } + for error in result.errors { + if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { + outcome.already_missing += 1; + } else { + outcome.failed.push((error.key, error.code, error.message)); + } + } + outcome +} + +#[async_trait] +impl ObjectStore for S3ObjectStore { + fn provider(&self) -> ProviderKind { + ProviderKind::S3 + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.bucket + .put_object_with_content_type(key, bytes, content_type) + .await + .map_err(|e| classify("put", key, e))?; + Ok(()) + } + + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError> { + /// 8 MiB read buffer — the file is streamed, never held whole in RAM. + const BUF: usize = 8 * 1024 * 1024; + + let file = tokio::fs::File::open(path) + .await + .map_err(|e| ObjectStoreError::Provider { + operation: "put_file", + message: e.to_string(), + })?; + let mut reader = tokio::io::BufReader::with_capacity(BUF, file); + + self.bucket + .put_object_stream_with_content_type(&mut reader, key, content_type) + .await + .map_err(|e| classify("put_file", key, e))?; + Ok(()) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + let headers = Self::condition_headers(&WriteCondition::Absent)?; + match self + .bucket + .put_object_with_content_type_and_headers(key, bytes, content_type, Some(headers)) + .await + { + Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(ImmutableWrite::Created), + Err(S3Error::HttpFailWithBody(412, _)) => Ok(ImmutableWrite::AlreadyPresent), + Ok(resp) => Err(ObjectStoreError::Provider { + operation: "put_immutable", + message: format!("unexpected status {}", resp.status_code()), + }), + Err(e) => Err(classify("put_immutable", key, e)), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result { + let headers = Self::condition_headers(&condition)?; + match self + .bucket + .put_object_with_content_type_and_headers(key, bytes, content_type, Some(headers)) + .await + { + Ok(resp) if (200..300).contains(&resp.status_code()) => { + let etag = Self::etag_of(&resp.headers()).ok_or({ + // Fail closed: a conditional write we cannot chain (because + // the backend returned no ETag) is not a commit — it is a + // non-conforming backend. The conformance probe catches + // this at boot; in production we would rather refuse than + // hand the caller an empty token and force-fail the next + // compare-and-swap. + ObjectStoreError::Provider { + operation: "put_conditional", + message: "conditional write committed but the response carried no ETag \ + (backend does not satisfy revision-token consistency)" + .to_string(), + } + })?; + Ok(ConditionalWrite::Committed(Revision::S3Etag(etag))) + } + Err(S3Error::HttpFailWithBody(412, _)) => Ok(ConditionalWrite::Conflict), + Ok(resp) => Err(ObjectStoreError::Provider { + operation: "put_conditional", + message: format!("unexpected status {}", resp.status_code()), + }), + Err(e) => Err(classify("put_conditional", key, e)), + } + } + + async fn get(&self, key: &str) -> Result { + let resp = self + .bucket + .get_object(key) + .await + .map_err(|e| classify("get", key, e))?; + Ok(Bytes::from(resp.to_vec())) + } + + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result { + let resp = self + .bucket + .get_object_range(key, start, Some(end)) + .await + .map_err(|e| classify("get_range", key, e))?; + Ok(Bytes::from(resp.to_vec())) + } + + async fn get_stream(&self, key: &str) -> Result { + let response = self + .bucket + .get_object_stream(key) + .await + .map_err(|e| classify("get_stream", key, e))?; + + if response.status_code == 404 { + return Err(ObjectStoreError::NotFound { key: key.into() }); + } + + let operation = "get_stream"; + let stream = futures_util::StreamExt::map(response.bytes, move |chunk| { + chunk.map_err(|e| ObjectStoreError::Provider { + operation, + message: e.to_string(), + }) + }); + Ok(Box::pin(stream)) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + match self.bucket.get_object(key).await { + Ok(resp) => { + let etag = Self::etag_of(&resp.headers()).ok_or(ObjectStoreError::Provider { + operation: "get_with_revision", + message: "response carried no ETag".to_string(), + })?; + Ok(Some((Revision::S3Etag(etag), Bytes::from(resp.to_vec())))) + } + Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), + Err(e) => Err(classify("get_with_revision", key, e)), + } + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + match self.bucket.head_object(key).await { + Ok((result, _)) => Ok(Some(ObjectMeta { + size: result.content_length.unwrap_or(0) as u64, + revision: result.e_tag.map(Revision::S3Etag), + })), + Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), + Err(e) => Err(classify("head", key, e)), + } + } + + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result { + // Wraps rust-s3's manual `list_page`, NOT the auto-paginating `list`, + // which has no cap. + let (result, _status) = self + .bucket + .list_page( + prefix.to_string(), + None, + continuation_token, + None, + Some(max_keys), + ) + .await + .map_err(|e| classify("list_page", prefix, e))?; + Ok(ListPage { + objects: result + .contents + .into_iter() + .map(|obj| (obj.key, obj.size)) + .collect(), + next_continuation_token: result.next_continuation_token, + is_truncated: result.is_truncated, + }) + } + + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.bucket + .delete_object(key) + .await + .map_err(|e| classify("delete", key, e))?; + Ok(()) + } + + async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = keys + .iter() + .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| classify("delete_objects", "", e))?; + Ok(fold_bulk_delete_result(result)) + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + self.list_page("", None, 1).await.map(|_| ()) + } + + /// Detect whether the bucket has ever had versioning enabled. + /// + /// `rust-s3` exposes no `GetBucketVersioning`, so this writes and inspects + /// a short-lived probe object instead: versioning-enabled (and + /// versioning-suspended) buckets stamp new writes with a version id. + /// + /// This heuristic is S3-specific by construction and does not translate to + /// providers where every object carries a version token; those must + /// implement the check against real bucket metadata. + async fn versioning_detected(&self) -> Result { + let key = format!("probe/deletion-versioning-{}", uuid::Uuid::new_v4()); + self.put(&key, b"buzz deletion versioning probe", "text/plain") + .await?; + let inspected = self.bucket.head_object(&key).await; + let removed = self.bucket.delete_object(&key).await; + let (head, _) = inspected.map_err(|e| classify("versioning_detected", &key, e))?; + removed.map_err(|e| classify("versioning_detected", &key, e))?; + Ok(head.version_id.is_some()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(access: &str, secret: &str) -> S3StoreConfig { + S3StoreConfig { + endpoint: "http://localhost:9000".to_string(), + access_key: access.to_string(), + secret_key: secret.to_string(), + bucket: "buzz-media".to_string(), + region: "us-west-2".to_string(), + addressing_style: S3AddressingStyle::Path, + } + } + + /// Static keys present: builds a client without touching the AWS + /// credential chain (no env/metadata access), and the signing region + /// comes from config rather than a hardcoded "us-east-1". + #[test] + fn static_keys_build_client_with_configured_region() { + let store = + S3ObjectStore::new(&config("buzz_dev", "buzz_dev_secret")).expect("static creds"); + match store.region() { + Region::Custom { region, .. } => assert_eq!(region, "us-west-2"), + other => panic!("expected Custom region, got {other:?}"), + } + assert_eq!(store.provider(), ProviderKind::S3); + } + + #[test] + fn constructor_applies_both_addressing_styles() { + let path = + S3ObjectStore::new(&config("buzz_dev", "buzz_dev_secret")).expect("path-style client"); + assert!(path.is_path_style()); + assert_eq!(path.url(), "http://localhost:9000/buzz-media"); + + let mut virtual_config = config("buzz_dev", "buzz_dev_secret"); + virtual_config.addressing_style = S3AddressingStyle::Virtual; + let virtual_hosted = S3ObjectStore::new(&virtual_config).expect("virtual-hosted client"); + assert!(!virtual_hosted.is_path_style()); + assert_eq!(virtual_hosted.url(), "http://buzz-media.localhost:9000"); + } + + #[test] + fn partial_static_keys_are_rejected() { + for (access, secret) in [("buzz_dev", ""), ("", "buzz_dev_secret")] { + let err = match S3ObjectStore::new(&config(access, secret)) { + Ok(_) => panic!("partial static creds must not silently use credential chain"), + Err(err) => err, + }; + assert!( + matches!(err, ObjectStoreError::Config(ref msg) if msg.contains("must be configured together")), + "unexpected error: {err}" + ); + } + } + + #[test] + fn addressing_style_parses_supported_values() { + assert_eq!( + S3AddressingStyle::from_str("path"), + Ok(S3AddressingStyle::Path) + ); + assert_eq!( + S3AddressingStyle::from_str("virtual"), + Ok(S3AddressingStyle::Virtual) + ); + assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path); + } + + #[test] + fn addressing_style_rejects_unknown_or_ambiguous_values() { + for invalid in ["", "auto", "PATH", "virtual-hosted"] { + let error = + S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style"); + assert!( + error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"), + "unexpected error for {invalid:?}: {error}" + ); + } + } + + /// Pre-classification failures are the *only* ambiguous outcomes. The Git + /// conformance probe drops exactly this set from its observer count, so + /// widening it would let a real conformance failure vanish. + #[test] + fn transport_failures_classify_as_ambiguous() { + let io = classify("put", "k", S3Error::Io(std::io::Error::other("reset"))); + assert!(io.is_ambiguous(), "io error must be ambiguous: {io}"); + } + + #[test] + fn precondition_failure_classifies_as_conflict() { + let err = classify( + "put_conditional", + "pointers/x", + S3Error::HttpFailWithBody(412, "PreconditionFailed".into()), + ); + assert!(!err.is_ambiguous()); + assert!(matches!(err, ObjectStoreError::Conflict { ref key } if key == "pointers/x")); + } + + #[test] + fn missing_object_classifies_as_not_found() { + let err = classify( + "get", + "packs/x", + S3Error::HttpFailWithBody(404, "NoSuchKey".into()), + ); + assert!(matches!(err, ObjectStoreError::NotFound { ref key } if key == "packs/x")); + } + + #[test] + fn throttling_and_transient_statuses_stay_classified_but_retryable() { + let throttled = classify( + "put", + "k", + S3Error::HttpFailWithBody(429, "SlowDown".into()), + ); + assert!(matches!(throttled, ObjectStoreError::Throttled { .. })); + assert!(throttled.is_retryable() && !throttled.is_ambiguous()); + + let transient = classify( + "get", + "k", + S3Error::HttpFailWithBody(503, "SlowDown".into()), + ); + assert!(matches!( + transient, + ObjectStoreError::TransportRetryable { .. } + )); + assert!(transient.is_retryable() && !transient.is_ambiguous()); + } + + /// A 403 is a real backend answer: permanent, classified, never dropped. + #[test] + fn permission_denied_classifies_as_permanent_provider_failure() { + let err = classify( + "put", + "k", + S3Error::HttpFailWithBody(403, "AccessDenied".into()), + ); + assert!(matches!(err, ObjectStoreError::Provider { .. })); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + #[test] + fn conditional_write_rejects_a_foreign_provider_revision() { + let err = + S3ObjectStore::condition_headers(&WriteCondition::Matches(Revision::GcsGeneration(7))) + .expect_err("a GCS generation must never predicate an S3 If-Match"); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: ProviderKind::Gcs, + } + )); + } + + #[test] + fn conditional_write_accepts_an_s3_revision() { + let headers = S3ObjectStore::condition_headers(&WriteCondition::Matches(Revision::S3Etag( + "\"abc\"".into(), + ))) + .expect("s3 etag predicates If-Match"); + assert_eq!( + headers.get(axum::http::header::IF_MATCH).unwrap(), + "\"abc\"" + ); + } + + #[test] + fn create_only_write_uses_if_none_match_star() { + let headers = S3ObjectStore::condition_headers(&WriteCondition::Absent).expect("headers"); + assert_eq!(headers.get(axum::http::header::IF_NONE_MATCH).unwrap(), "*"); + } + + /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO + /// absent-key errors count as success, version artifacts are surfaced for + /// fail-closed handling, and anything else stays a per-key failure. + #[test] + fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let deleted_object = |key: &str, marker: bool| DeletedObject { + key: key.to_string(), + version_id: None, + delete_marker: marker.then_some(true), + delete_marker_version_id: marker.then(|| "v1".to_string()), + }; + let delete_error = |key: &str, code: &str, message: &str| DeleteError { + key: key.to_string(), + code: code.to_string(), + message: message.to_string(), + version_id: None, + }; + let result = DeleteObjectsResult { + deleted: vec![ + deleted_object("plain", false), + deleted_object("marked", true), + ], + errors: vec![ + delete_error("gone", "NoSuchKey", "absent"), + delete_error("gone-version", "NoSuchVersion", "absent"), + delete_error("denied", "AccessDenied", "nope"), + ], + }; + let outcome = fold_bulk_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 2); + assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); + assert_eq!( + outcome.failed, + vec![( + "denied".to_string(), + "AccessDenied".to_string(), + "nope".to_string() + )] + ); + } +} diff --git a/crates/buzz-object-store/src/revision.rs b/crates/buzz-object-store/src/revision.rs new file mode 100644 index 00000000000..ca0159c7c2d --- /dev/null +++ b/crates/buzz-object-store/src/revision.rs @@ -0,0 +1,150 @@ +//! Provider-safe revision tokens and conditional-write vocabulary. +//! +//! A revision is whatever the backing provider uses to name "the version of +//! this object I just observed": an ETag on S3, an object generation on Google +//! Cloud Storage. The two are not interchangeable, and the type keeps them +//! from being confused — a generation handed to an S3 `If-Match` would be a +//! silent correctness bug, so [`Revision::expect_s3_etag`] rejects it instead. + +use crate::error::ObjectStoreError; + +/// Which provider minted a [`Revision`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderKind { + /// S3 and S3-compatible backends (AWS S3, MinIO, Railway). + S3, + /// Google Cloud Storage, using native object generations. + Gcs, +} + +impl std::fmt::Display for ProviderKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::S3 => f.write_str("s3"), + Self::Gcs => f.write_str("gcs"), + } + } +} + +/// An opaque, provider-qualified object revision. +/// +/// Revisions are produced by reads and successful conditional writes, and are +/// consumed by the next conditional write. They are never parsed, compared +/// across providers, or synthesised by callers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Revision { + /// S3 entity tag, verbatim as the provider returned it (quotes included). + S3Etag(String), + /// Google Cloud Storage object generation. + GcsGeneration(i64), +} + +impl Revision { + /// Which provider this revision belongs to. + pub fn provider(&self) -> ProviderKind { + match self { + Self::S3Etag(_) => ProviderKind::S3, + Self::GcsGeneration(_) => ProviderKind::Gcs, + } + } + + /// Borrow the S3 entity tag, rejecting a revision from another provider. + /// + /// This is the guard that keeps a provider swap from degrading into a + /// blind overwrite: a caller holding a GCS generation cannot accidentally + /// predicate an S3 `If-Match` on it. + pub fn expect_s3_etag(&self) -> Result<&str, ObjectStoreError> { + match self { + Self::S3Etag(tag) => Ok(tag), + other => Err(ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: other.provider(), + }), + } + } + + /// Read the GCS object generation, rejecting a revision from another provider. + pub fn expect_gcs_generation(&self) -> Result { + match self { + Self::GcsGeneration(generation) => Ok(*generation), + other => Err(ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: other.provider(), + }), + } + } +} + +/// Precondition for a conditional write. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteCondition { + /// Create-only: commit iff the object does not yet exist. + Absent, + /// Compare-and-swap: commit iff the object is still at this revision. + Matches(Revision), +} + +/// Outcome of a conditional write. +/// +/// `Conflict` is *not* an error — it is the ordinary result of losing a +/// compare-and-swap race, and callers must classify it as such (retry, or +/// report a non-fast-forward) rather than as a backend failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConditionalWrite { + /// The write committed; the new revision predicates the next write. + Committed(Revision), + /// The precondition did not hold; nothing was written. + Conflict, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn revision_reports_its_provider() { + assert_eq!( + Revision::S3Etag("\"abc\"".into()).provider(), + ProviderKind::S3 + ); + assert_eq!(Revision::GcsGeneration(7).provider(), ProviderKind::Gcs); + } + + #[test] + fn s3_accessor_accepts_an_s3_revision() { + let revision = Revision::S3Etag("\"abc\"".into()); + assert_eq!(revision.expect_s3_etag().unwrap(), "\"abc\""); + } + + #[test] + fn s3_accessor_rejects_a_gcs_revision() { + let err = Revision::GcsGeneration(7).expect_s3_etag().unwrap_err(); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: ProviderKind::Gcs, + } + )); + } + + #[test] + fn gcs_accessor_rejects_an_s3_revision() { + let err = Revision::S3Etag("\"abc\"".into()) + .expect_gcs_generation() + .unwrap_err(); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + } + + #[test] + fn provider_kind_renders_a_stable_label() { + assert_eq!(ProviderKind::S3.to_string(), "s3"); + assert_eq!(ProviderKind::Gcs.to_string(), "gcs"); + } +} From d6b4679fc8550dad81db3ff5ab562910f1dc60cc Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 24 Aug 2026 16:57:44 -0700 Subject: [PATCH 2/7] refactor(media, git): move both facades onto the object-store seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MediaStorage` and `GitStore` were each a concrete `rust-s3` wrapper. They built their own `Bucket`, matched on `S3Error`, and carried S3 vocabulary through domain code. Move all of that into `providers::s3` — now the only module in the tree that knows what an ETag is — and leave behind the parts that are genuinely domain logic. Media keeps its content-addressed blob keys, the community-scoped metadata sidecar that gates tenant reads, and its error surface. `MediaStorage::new` still takes a `MediaConfig` and builds the S3 provider, so every existing caller is unchanged; `MediaStorage::with_store` wraps an already-constructed provider and `object_store()` hands it out, which lets the relay build one client per process instead of two against one bucket. `S3AddressingStyle` moves to the object-store crate — it configures the provider, not the media domain — and is re-exported from `buzz_media::config` so its parsing tests are untouched. Git keeps its content-addressed keys, digest verification, the idx sidecar layout, and the conformance probe. `ETag` becomes `Revision`, `Precond` becomes `WriteCondition`, and `CasOutcome` becomes `ConditionalWrite`. `StoreError::Backend` wraps `ObjectStoreError`, and the `From` impl lifts not-found / too-large / digest-mismatch into their own variants so every existing call site keeps matching on them directly. Probe semantics are unchanged. Its drop-and-floor rule — a racer that never got a classified provider response is dropped from the observer set rather than counted as a lost race — previously keyed off `S3Error::{Reqwest, Http, Io}` and now keys off `ObjectStoreError::is_ambiguous`, which the S3 provider maps from exactly those three variants. Everything else stays a classified observation that fails the probe closed. Behavior is preserved, including the paths that historically surfaced a backend 404 as a generic storage failure rather than a media `NotFound`. Neither `buzz-media` nor `buzz-relay` depends on `rust-s3` any more. Signed-off-by: mozarthq --- Cargo.lock | 6 +- crates/buzz-media/Cargo.toml | 3 +- crates/buzz-media/src/config.rs | 33 +- crates/buzz-media/src/error.rs | 13 +- crates/buzz-media/src/storage.rs | 920 +++--------------- crates/buzz-object-store/Cargo.toml | 1 + crates/buzz-object-store/src/lib.rs | 69 ++ crates/buzz-object-store/src/providers/s3.rs | 277 +++++- crates/buzz-relay/Cargo.toml | 2 +- crates/buzz-relay/src/api/git/cas_publish.rs | 64 +- crates/buzz-relay/src/api/git/hydrate.rs | 28 +- crates/buzz-relay/src/api/git/store.rs | 783 ++++++--------- .../buzz-relay/src/handlers/side_effects.rs | 16 +- crates/buzz-relay/src/state.rs | 13 +- 14 files changed, 823 insertions(+), 1405 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab337ce6b73..5f05cc9088b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1133,6 +1133,7 @@ dependencies = [ "axum", "blurhash", "buzz-core", + "buzz-object-store", "bytes", "chrono", "futures-core", @@ -1143,8 +1144,6 @@ dependencies = [ "infer", "mp4", "nostr 0.44.7", - "quick-xml 0.38.4", - "rust-s3", "serde", "serde_json", "sha2 0.11.0", @@ -1166,6 +1165,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "quick-xml 0.38.4", "rust-s3", "serde", "thiserror 2.0.18", @@ -1289,6 +1289,7 @@ dependencies = [ "buzz-db", "buzz-deletion", "buzz-media", + "buzz-object-store", "buzz-pubsub", "buzz-relay-mesh", "buzz-sdk", @@ -1321,7 +1322,6 @@ dependencies = [ "rand 0.10.1", "redis", "reqwest 0.13.4", - "rust-s3", "rustls", "serde", "serde_json", diff --git a/crates/buzz-media/Cargo.toml b/crates/buzz-media/Cargo.toml index 7808ecaff43..059653f7f61 100644 --- a/crates/buzz-media/Cargo.toml +++ b/crates/buzz-media/Cargo.toml @@ -9,6 +9,7 @@ description = "Media storage, validation, and thumbnail generation for Buzz" [dependencies] buzz-core = { workspace = true } +buzz-object-store = { workspace = true } nostr = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -21,7 +22,6 @@ chrono = { workspace = true } ulid = "1" uuid = { workspace = true } axum = { workspace = true } -s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } infer = "0.19" image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] } blurhash = "0.2" @@ -32,7 +32,6 @@ tempfile = "3" tokio-util = { version = "0.7", features = ["io"] } futures-util = "0.3" futures-core = "0.3" -quick-xml = { version = "0.38", features = ["serialize"] } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/buzz-media/src/config.rs b/crates/buzz-media/src/config.rs index 3c70e4afe19..8ca10c0161e 100644 --- a/crates/buzz-media/src/config.rs +++ b/crates/buzz-media/src/config.rs @@ -1,37 +1,6 @@ //! Media storage configuration. -use std::str::FromStr; - -/// S3 URL addressing style shared by media and Git/CAS storage. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum S3AddressingStyle { - /// Put the bucket in the request path (`https://endpoint/bucket/key`). - /// - /// This preserves compatibility with the bundled MinIO deployments, whose - /// internal DNS only resolves the endpoint hostname. - #[default] - Path, - /// Put the bucket in the hostname (`https://bucket.endpoint/key`). - /// - /// This is the standard S3 form and is required by providers such as new - /// Railway Storage Buckets. - Virtual, -} - -impl FromStr for S3AddressingStyle { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "path" => Ok(Self::Path), - "virtual" => Ok(Self::Virtual), - _ => Err(format!( - "BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}" - )), - } - } -} +pub use buzz_object_store::S3AddressingStyle; fn default_max_video_bytes() -> u64 { 524_288_000 // 500 MB diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index 5abbea6f580..0ea13a5104b 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -97,9 +97,16 @@ impl From for MediaError { } } -impl From for MediaError { - fn from(e: s3::error::S3Error) -> Self { - Self::StorageError(e.to_string()) +impl From for MediaError { + /// A missing object is a media-level `NotFound`; everything else — a + /// throttle, an ambiguous transport outcome, a permanent provider failure, + /// a misconfiguration — is opaque to media callers and collapses to + /// `StorageError`. + fn from(e: buzz_object_store::ObjectStoreError) -> Self { + match e { + buzz_object_store::ObjectStoreError::NotFound { .. } => Self::NotFound, + other => Self::StorageError(other.to_string()), + } } } diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index abe6bdd40ea..6ec4a1c1157 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -1,440 +1,177 @@ -//! S3/MinIO storage client. +//! Media storage facade. +//! +//! Thin domain layer over [`buzz_object_store::ObjectStore`]: it owns the +//! media key vocabulary (content-addressed blobs, community-scoped metadata +//! sidecars) and the media error surface, and knows nothing about which +//! provider is underneath. -use std::collections::HashMap; use std::path::Path; use std::pin::Pin; +use std::sync::Arc; use buzz_core::tenant::{CommunityId, TenantContext}; +use buzz_object_store::{ObjectStore, ObjectStoreError, S3ObjectStore, S3StoreConfig}; -use crate::config::{MediaConfig, S3AddressingStyle}; +use crate::config::MediaConfig; use crate::error::MediaError; use bytes::Bytes; -use quick_xml::events::{BytesStart, Event}; -use quick_xml::Reader; -use s3::creds::Credentials; -use s3::request::Request as _; -use s3::{Bucket, Region}; use serde::{Deserialize, Serialize}; -/// A stream of byte chunks from S3, usable with `axum::body::Body::from_stream()`. -pub type ByteStream = Pin> + Send>>; - -/// The kind of versioned S3 object-store entry. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ObjectVersionKind { - /// A concrete object version with bytes. - Object, - /// A delete-marker version hiding older bytes from live-object listing. - DeleteMarker, -} - -/// One S3 object version or delete marker under a tenant prefix. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObjectVersionEntry { - /// Object key. - pub key: String, - /// Concrete S3 version id. - pub version_id: String, - /// Whether this entry is a byte-bearing object or delete marker. - pub kind: ObjectVersionKind, - /// Byte size for object versions; zero for delete markers. - pub size: u64, -} - -/// Exact version identifier used for permanent deletion. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObjectVersionRef { - /// Object key. - pub key: String, - /// Concrete S3 version id. - pub version_id: String, -} - -/// One `ListObjectVersions` page. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ObjectVersionsPage { - /// Object versions and delete markers returned by this page. - pub entries: Vec, - /// Next key marker for truncated listings. - pub next_key_marker: Option, - /// Next version-id marker for truncated listings. - pub next_version_id_marker: Option, - /// Whether more pages remain. - pub is_truncated: bool, -} - -#[derive(Debug, Default)] -struct ListVersionFields { - key: Option, - version_id: Option, - size: Option, -} - -fn local_name(name: &[u8]) -> &[u8] { - name.rsplit(|byte| *byte == b':').next().unwrap_or(name) -} - -fn xml_error(error: impl std::fmt::Display) -> MediaError { - MediaError::StorageError(error.to_string()) -} - -fn read_element_text( - reader: &mut Reader<&[u8]>, - start: &BytesStart<'_>, -) -> Result { - reader - .read_text(start.to_end().name()) - .map(|text| text.into_owned()) - .map_err(xml_error) -} +pub use buzz_object_store::{ + BulkDeleteOutcome, ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, + ObjectVersionsPage, +}; -fn skip_element(reader: &mut Reader<&[u8]>, start: &BytesStart<'_>) -> Result<(), MediaError> { - reader - .read_to_end(start.to_end().name()) - .map_err(xml_error)?; - Ok(()) -} - -fn parse_list_version_entry( - reader: &mut Reader<&[u8]>, - start: &BytesStart<'_>, - kind: ObjectVersionKind, -) -> Result { - let mut fields = ListVersionFields::default(); - loop { - match reader.read_event().map_err(xml_error)? { - Event::Start(child) => match local_name(child.local_name().as_ref()) { - b"Key" => fields.key = Some(read_element_text(reader, &child)?), - b"VersionId" => fields.version_id = Some(read_element_text(reader, &child)?), - b"Size" => { - let size = read_element_text(reader, &child)?; - fields.size = Some(size.parse::().map_err(xml_error)?); - } - _ => skip_element(reader, &child)?, - }, - Event::Empty(child) => match local_name(child.local_name().as_ref()) { - b"Key" => fields.key = Some(String::new()), - b"VersionId" => fields.version_id = Some(String::new()), - b"Size" => fields.size = Some(0), - _ => {} - }, - Event::End(end) if end.name().as_ref() == start.to_end().name().as_ref() => { - let key = fields.key.ok_or_else(|| { - MediaError::StorageError("ListObjectVersions entry missing Key".to_string()) - })?; - let version_id = fields.version_id.ok_or_else(|| { - MediaError::StorageError( - "ListObjectVersions entry missing VersionId".to_string(), - ) - })?; - return Ok(ObjectVersionEntry { - key, - version_id, - kind, - size: if kind == ObjectVersionKind::Object { - fields.size.unwrap_or(0) - } else { - 0 - }, - }); - } - Event::Eof => { - return Err(MediaError::StorageError( - "unexpected EOF inside ListObjectVersions entry".to_string(), - )); - } - _ => {} - } - } -} - -fn parse_object_versions_page(xml: &[u8]) -> Result { - let mut reader = Reader::from_reader(xml); - reader.config_mut().trim_text(true); - let mut entries = Vec::new(); - let mut next_key_marker = None; - let mut next_version_id_marker = None; - let mut is_truncated = false; - - loop { - match reader.read_event().map_err(xml_error)? { - Event::Start(start) => match local_name(start.local_name().as_ref()) { - b"Version" => entries.push(parse_list_version_entry( - &mut reader, - &start, - ObjectVersionKind::Object, - )?), - b"DeleteMarker" => entries.push(parse_list_version_entry( - &mut reader, - &start, - ObjectVersionKind::DeleteMarker, - )?), - b"IsTruncated" => { - let value = read_element_text(&mut reader, &start)?; - is_truncated = value.eq_ignore_ascii_case("true"); - } - b"NextKeyMarker" => { - next_key_marker = Some(read_element_text(&mut reader, &start)?); - } - b"NextVersionIdMarker" => { - next_version_id_marker = Some(read_element_text(&mut reader, &start)?); - } - b"ListVersionsResult" => {} - _ => skip_element(&mut reader, &start)?, - }, - Event::Empty(start) => match local_name(start.local_name().as_ref()) { - b"NextKeyMarker" => next_key_marker = Some(String::new()), - b"NextVersionIdMarker" => next_version_id_marker = Some(String::new()), - _ => {} - }, - Event::Eof => break, - _ => {} - } - } - - Ok(ObjectVersionsPage { - entries, - next_key_marker, - next_version_id_marker, - is_truncated, - }) -} +/// A stream of byte chunks from object storage, usable with +/// `axum::body::Body::from_stream()`. +pub type ByteStream = Pin> + Send>>; -/// S3-compatible object storage client. +/// Media object storage client. pub struct MediaStorage { - bucket: Box, + store: Arc, } impl MediaStorage { - /// Create a new storage client from config. - /// - /// Credential selection: - /// - If both `s3_access_key` and `s3_secret_key` are non-empty, use them as - /// static credentials (MinIO/local/dev, or any static-key deployment). - /// - Otherwise, fall back to the AWS default credential chain via - /// [`Credentials::default`]: environment, shared profile, web-identity - /// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and - /// instance-metadata providers, in that order. This lets the relay use - /// the pod's IAM role without long-lived static keys. + /// Create a storage client from media config, over the S3 provider. pub fn new(config: &MediaConfig) -> Result { - let region = Region::Custom { - region: config.s3_region.clone(), + let store = S3ObjectStore::new(&S3StoreConfig { endpoint: config.s3_endpoint.clone(), - }; - let creds = match ( - config.s3_access_key.is_empty(), - config.s3_secret_key.is_empty(), - ) { - (false, false) => Credentials::new( - Some(&config.s3_access_key), - Some(&config.s3_secret_key), - None, - None, - None, - ), - (true, true) => { - // No static keys configured: resolve from the AWS credential chain - // (IRSA web-identity, env, profile, instance metadata). - Credentials::default() - } - _ => { - return Err(MediaError::StorageError( - "s3_access_key and s3_secret_key must be configured together, or both empty to use the AWS credential chain" - .to_string(), - )); - } - } - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let bucket = Bucket::new(&config.s3_bucket, region, creds) - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let bucket = match config.s3_addressing_style { - S3AddressingStyle::Path => bucket.with_path_style(), - S3AddressingStyle::Virtual => bucket, - }; - Ok(Self { bucket }) + access_key: config.s3_access_key.clone(), + secret_key: config.s3_secret_key.clone(), + bucket: config.s3_bucket.clone(), + region: config.s3_region.clone(), + addressing_style: config.s3_addressing_style, + })?; + Ok(Self::with_store(Arc::new(store))) + } + + /// Wrap an already-constructed object store. + /// + /// The relay builds one provider per process and shares it between media + /// and Git storage rather than opening a second client against the same + /// bucket. + pub fn with_store(store: Arc) -> Self { + Self { store } + } + + /// The shared object store behind this facade, for handing to another + /// domain facade (see [`MediaStorage::with_store`]). + pub fn object_store(&self) -> Arc { + Arc::clone(&self.store) } /// Store an object from a byte slice. /// /// Used for images, sidecars, and thumbnails. For large video files use - /// [`put_file`] to avoid loading the entire blob into RAM. + /// [`MediaStorage::put_file`] to avoid loading the entire blob into RAM. pub async fn put(&self, key: &str, bytes: &[u8], content_type: &str) -> Result<(), MediaError> { - self.bucket - .put_object_with_content_type(key, bytes, content_type) - .await?; - Ok(()) + self.store + .put(key, bytes, content_type) + .await + .map_err(storage_error) } - /// Stream a file from disk into S3 without loading it into RAM. + /// Stream a file from disk into object storage without loading it into RAM. /// - /// Uses rust-s3's `put_object_stream_with_content_type` which reads from - /// the file incrementally via an 8 MiB `BufReader`. The full file is never - /// held in memory simultaneously. Intended for video blobs (up to 500 MB). + /// The full file is never held in memory simultaneously. Intended for + /// video blobs (up to 500 MB). pub async fn put_file( &self, key: &str, path: &Path, content_type: &str, ) -> Result<(), MediaError> { - const BUF: usize = 8 * 1024 * 1024; // 8 MiB read buffer - - let file = tokio::fs::File::open(path) + self.store + .put_file(key, path, content_type) .await - .map_err(|e| MediaError::Io(e.to_string()))?; - let mut reader = tokio::io::BufReader::with_capacity(BUF, file); - - self.bucket - .put_object_stream_with_content_type(&mut reader, key, content_type) - .await?; - Ok(()) + .map_err(storage_error) } /// Retrieve an object's bytes. pub async fn get(&self, key: &str) -> Result, MediaError> { - match self.bucket.get_object(key).await { - Ok(response) => Ok(response.to_vec()), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.get(key).await?.to_vec()) } - /// Retrieve a byte range from an object via S3-native `Range` GET. + /// Retrieve a byte range from an object via a native ranged GET. /// /// `start` and `end` are inclusive byte offsets. Only the requested slice - /// is transferred from S3 — the full object is never loaded into RAM. - /// Intended for HTTP 206 range responses on large video blobs. + /// is transferred — the full object is never loaded into RAM. Intended for + /// HTTP 206 range responses on large video blobs. pub async fn get_range(&self, key: &str, start: u64, end: u64) -> Result, MediaError> { - match self.bucket.get_object_range(key, start, Some(end)).await { - Ok(response) => Ok(response.to_vec()), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.get_range(key, start, end).await?.to_vec()) } - /// Stream an object's bytes from S3 without loading into RAM. + /// Stream an object's bytes without loading them into RAM. /// /// Returns a pinned stream of `Result` chunks. /// The full object is never buffered — intended for streaming large /// blobs (video) directly into HTTP responses via `Body::from_stream()`. pub async fn get_stream(&self, key: &str) -> Result { - let response = self - .bucket - .get_object_stream(key) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - - if response.status_code == 404 { - return Err(MediaError::NotFound); - } - - let stream = futures_util::StreamExt::map(response.bytes, |chunk| { - chunk.map_err(|e| MediaError::StorageError(e.to_string())) - }); - Ok(Box::pin(stream)) + let stream = self.store.get_stream(key).await?; + Ok(Box::pin(futures_util::StreamExt::map(stream, |chunk| { + chunk.map_err(storage_error) + }))) } - /// Check if an object exists. Returns false on 404. + /// Check if an object exists. Returns false when absent. pub async fn head(&self, key: &str) -> Result { - match self.bucket.head_object(key).await { - Ok(_) => Ok(true), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(false), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.head(key).await.map_err(storage_error)?.is_some()) } /// Delete an object. Returns an error on failure — callers decide whether to propagate. pub async fn delete(&self, key: &str) -> Result<(), MediaError> { - self.bucket - .delete_object(key) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(()) + self.store.delete(key).await.map_err(storage_error) } - /// HEAD with metadata — returns Content-Length (size). + /// HEAD with metadata — returns the object size. pub async fn head_with_metadata(&self, key: &str) -> Result, MediaError> { - match self.bucket.head_object(key).await { - Ok((result, _)) => Ok(Some(BlobHeadMeta { - size: result.content_length.unwrap_or(0) as u64, - })), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(None), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self + .store + .head(key) + .await + .map_err(storage_error)? + .map(|meta| BlobHeadMeta { size: meta.size })) } - /// Bulk-delete up to one manifest chunk of keys via S3 `DeleteObjects`. + /// Detect whether the bucket retains non-current object versions. + /// + /// Deletion refuses versioned buckets because bulk deletes without a + /// version qualifier would only insert delete markers, not prove logical + /// absence. + pub async fn bucket_versioning_detected(&self) -> Result { + self.store + .versioning_detected() + .await + .map_err(storage_error) + } + + /// Bulk-delete up to one manifest chunk of keys. /// /// Never fails on per-key outcomes: they are folded into /// [`BulkDeleteOutcome`] so the caller owns retry/fail-closed policy. - /// Historical MinIO releases report already-absent keys as - /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to - /// `already_missing` to keep checkpointed retry idempotent. pub async fn delete_objects(&self, keys: &[String]) -> Result { - if keys.is_empty() { - return Ok(BulkDeleteOutcome::default()); - } - let identifiers = keys - .iter() - .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) - .collect::>(); - self.delete_object_identifiers(identifiers).await + self.store.delete_objects(keys).await.map_err(storage_error) } - /// Non-destructively verify that versioned bucket APIs are reachable. - /// - /// `ListObjectVersions` can be proven without mutation. S3 has no equivalent - /// dry-run for `DeleteObjectVersion`: `DeleteObjects` is always destructive, - /// even for exact versions, and deleting a fabricated version id does not - /// prove permission when policies can be prefix- or tag-constrained. - /// Operators must still provision `s3:DeleteObjectVersion`; the first exact - /// version deletion remains the destructive proof. + /// Non-destructively verify that exact-version listing is reachable. pub async fn preflight_version_listing(&self, prefix: &str) -> Result<(), MediaError> { self.list_prefix_versions_page(prefix, None, None, 1) .await .map(|_| ()) } - /// Bulk-delete exact object versions via S3 `DeleteObjects`. + /// Permanently delete exact provider versions. /// - /// Every identifier includes a version id, so this removes historical - /// versions and delete markers permanently instead of adding another - /// delete marker to a versioned bucket. + /// The provider translates the opaque `version_id` to an S3 version ID or + /// a GCS generation. Domain deletion code never performs that translation. pub async fn delete_object_versions( &self, versions: &[ObjectVersionRef], ) -> Result { - self.delete_object_versions_with_folding(versions, fold_version_delete_result) - .await - } - - async fn delete_object_versions_with_folding( - &self, - versions: &[ObjectVersionRef], - fold: fn(s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome, - ) -> Result { - if versions.is_empty() { - return Ok(BulkDeleteOutcome::default()); - } - let identifiers = object_version_identifiers(versions); - let result = self - .bucket - .delete_objects(identifiers) + self.store + .delete_versions(versions) .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(fold(result)) - } - - async fn delete_object_identifiers( - &self, - identifiers: Vec, - ) -> Result { - let result = self - .bucket - .delete_objects(identifiers) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(fold_bulk_delete_result(result)) + .map_err(storage_error) } /// Build the community-scoped sidecar key for a given sha256 (bare hash). @@ -459,8 +196,8 @@ impl MediaStorage { sha256: &str, ) -> Result { let key = Self::ctx_sidecar_key(ctx, sha256); - let resp = self.bucket.get_object(&key).await?; - let meta: BlobMeta = serde_json::from_slice(&resp.to_vec())?; + let bytes = self.store.get(&key).await.map_err(storage_error)?; + let meta: BlobMeta = serde_json::from_slice(&bytes)?; Ok(meta) } @@ -496,15 +233,14 @@ impl MediaStorage { /// Probe object-store connectivity and bucket access. pub async fn ping(&self) -> Result<(), MediaError> { - self.list_page(None, 1).await.map(|_| ()) + self.store.ping().await.map_err(storage_error) } - /// One page of a full-bucket listing, for the storage sweep. Wraps - /// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which - /// has no cap) and converts the result into the storage-agnostic - /// [`crate::bucket_index::Page`] shape the pure fold consumes. + /// One page of a full-bucket listing, for the storage sweep. Converts the + /// provider listing into the storage-agnostic [`crate::bucket_index::Page`] + /// shape the pure fold consumes. /// - /// `max_keys` bounds one HTTP response, not the sweep's total object + /// `max_keys` bounds one provider response, not the sweep's total object /// cap — the caller (`fold_bucket_listing`) enforces the cumulative cap /// across pages. pub async fn list_page( @@ -520,44 +256,32 @@ impl MediaStorage { /// /// Deletion enumerates the target community's exact key prefixes with /// this instead of listing the whole fleet bucket: cost stays - /// O(tenant objects) regardless of fleet size. `ListObjectsV2` returns - /// keys in ascending UTF-8 binary order, which callers rely on for - /// streaming key-stream digests. + /// O(tenant objects) regardless of fleet size. Listings return keys in + /// ascending UTF-8 binary order, which callers rely on for streaming + /// key-stream digests. pub async fn list_prefix_page( &self, prefix: &str, continuation_token: Option, max_keys: usize, ) -> Result { - let (result, _status) = self - .bucket - .list_page( - prefix.to_string(), - None, - continuation_token, - None, - Some(max_keys), - ) - .await?; + let page = self + .store + .list_page(prefix, continuation_token, max_keys) + .await + .map_err(storage_error)?; Ok(crate::bucket_index::Page { - objects: result - .contents - .into_iter() - .map(|obj| (obj.key, obj.size)) - .collect(), - next_continuation_token: result.next_continuation_token, - is_truncated: result.is_truncated, + objects: page.objects, + next_continuation_token: page.next_continuation_token, + is_truncated: page.is_truncated, }) } - /// One page of object versions and delete markers under a prefix. + /// One page of exact object versions under a prefix. /// - /// This uses S3 `ListObjectVersions` (`?versions`) instead of - /// `ListObjectsV2`: versioned buckets can be logically empty while still - /// retaining historical versions or delete markers, and permanent deletion - /// must enumerate both. Pagination must carry both `KeyMarker` and - /// `VersionIdMarker`; carrying only the key marker can skip siblings when a - /// key has multiple versions on a page boundary. + /// Both cursor fields are opaque and must be replayed together. S3 uses + /// the pair directly; GCS places its page token in `key_marker` and leaves + /// `version_id_marker` empty. pub async fn list_prefix_versions_page( &self, prefix: &str, @@ -565,113 +289,22 @@ impl MediaStorage { version_id_marker: Option, max_keys: usize, ) -> Result { - let mut query = HashMap::from([ - ("versions".to_string(), String::new()), - ("prefix".to_string(), prefix.to_string()), - ("max-keys".to_string(), max_keys.to_string()), - ]); - if let Some(marker) = key_marker { - query.insert("key-marker".to_string(), marker); - } - if let Some(marker) = version_id_marker { - query.insert("version-id-marker".to_string(), marker); - } - let bucket = self - .bucket - .with_extra_query(query) - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let request = s3::request::tokio_backend::ReqwestRequest::new( - &bucket, - "/", - s3::command::Command::GetObject, - ) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let response = request - .response_data(false) + self.store + .list_versions_page(prefix, key_marker, version_id_marker, max_keys) .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - if response.status_code() >= 300 { - return Err(MediaError::StorageError(format!( - "list object versions failed with status {}: {}", - response.status_code(), - response.as_str().unwrap_or("") - ))); - } - parse_object_versions_page(response.as_slice()) + .map_err(storage_error) } } -/// Per-key outcomes of one bulk `DeleteObjects` call. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct BulkDeleteOutcome { - /// Keys the backend reported deleted (S3 reports already-missing keys as - /// deleted too — the API is idempotent by design). - pub deleted: u64, - /// Keys reported absent via legacy MinIO `NoSuchKey`/`NoSuchVersion` - /// per-key errors; equivalent to deleted for retry purposes. - pub already_missing: u64, - /// Keys whose deletion produced a version artifact (delete marker or - /// version id) — evidence of bucket versioning, which deletion must - /// fail closed on. - pub versioned_keys: Vec, - /// Remaining per-key failures as `(key, code, message)`. - pub failed: Vec<(String, String, String)>, -} - -fn object_version_identifiers( - versions: &[ObjectVersionRef], -) -> Vec { - versions - .iter() - .map(|version| { - s3::serde_types::ObjectIdentifier::with_version( - version.key.clone(), - version.version_id.clone(), - ) - }) - .collect() -} - -fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { - fold_delete_result(result, DeleteMode::Unversioned) -} - -fn fold_version_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { - fold_delete_result(result, DeleteMode::ExplicitVersion) -} - -enum DeleteMode { - Unversioned, - ExplicitVersion, -} - -fn fold_delete_result( - result: s3::serde_types::DeleteObjectsResult, - mode: DeleteMode, -) -> BulkDeleteOutcome { - let mut outcome = BulkDeleteOutcome::default(); - for deleted in result.deleted { - let has_version_artifact = deleted.delete_marker == Some(true) - || deleted.delete_marker_version_id.is_some() - || deleted.version_id.is_some(); - match mode { - DeleteMode::Unversioned if has_version_artifact => { - outcome.versioned_keys.push(deleted.key); - } - DeleteMode::Unversioned | DeleteMode::ExplicitVersion => { - outcome.deleted += 1; - } - } - } - for error in result.errors { - if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { - outcome.already_missing += 1; - } else { - outcome.failed.push((error.key, error.code, error.message)); - } - } - outcome +/// Collapse any provider failure — including a missing object — into the +/// generic media storage error. +/// +/// Used on the paths that historically surfaced a backend 404 as a storage +/// failure rather than a media-level `NotFound`: sidecar reads are guarded by +/// an explicit HEAD, and a bare absence there means the bucket disagreed with +/// the guard. +fn storage_error(error: ObjectStoreError) -> MediaError { + MediaError::StorageError(error.to_string()) } #[cfg(test)] @@ -679,271 +312,6 @@ mod tests { use super::*; use std::collections::HashMap; - /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO - /// absent-key errors count as success, version artifacts are surfaced for - /// fail-closed handling, and anything else stays a per-key failure. - #[test] - fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { - use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; - let deleted_object = |key: &str, marker: bool| DeletedObject { - key: key.to_string(), - version_id: None, - delete_marker: marker.then_some(true), - delete_marker_version_id: marker.then(|| "v1".to_string()), - }; - let delete_error = |key: &str, code: &str, message: &str| DeleteError { - key: key.to_string(), - code: code.to_string(), - message: message.to_string(), - version_id: None, - }; - let result = DeleteObjectsResult { - deleted: vec![ - deleted_object("plain", false), - deleted_object("marked", true), - ], - errors: vec![ - delete_error("gone", "NoSuchKey", "absent"), - delete_error("gone-version", "NoSuchVersion", "absent"), - delete_error("denied", "AccessDenied", "nope"), - ], - }; - let outcome = fold_bulk_delete_result(result); - assert_eq!(outcome.deleted, 1); - assert_eq!(outcome.already_missing, 2); - assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); - assert_eq!( - outcome.failed, - vec![( - "denied".to_string(), - "AccessDenied".to_string(), - "nope".to_string() - )] - ); - } - - #[test] - fn version_delete_fold_counts_explicit_version_artifacts_as_deleted() { - use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; - let result = DeleteObjectsResult { - deleted: vec![DeletedObject { - key: "versioned".to_string(), - version_id: Some("v1".to_string()), - delete_marker: Some(true), - delete_marker_version_id: Some("v1".to_string()), - }], - errors: vec![ - DeleteError { - key: "retried-version".to_string(), - code: "NoSuchVersion".to_string(), - message: "already absent".to_string(), - version_id: Some("v-gone".to_string()), - }, - DeleteError { - key: "denied-version".to_string(), - code: "AccessDenied".to_string(), - message: "denied".to_string(), - version_id: Some("v-denied".to_string()), - }, - ], - }; - - let outcome = fold_version_delete_result(result); - assert_eq!(outcome.deleted, 1); - assert_eq!(outcome.already_missing, 1); - assert!(outcome.versioned_keys.is_empty()); - assert_eq!( - outcome.failed, - vec![( - "denied-version".to_string(), - "AccessDenied".to_string(), - "denied".to_string() - )] - ); - } - - #[test] - fn object_version_identifiers_include_explicit_version_ids() { - let identifiers = object_version_identifiers(&[ - ObjectVersionRef { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-object".to_string(), - }, - ObjectVersionRef { - key: "uploads/tenant/event/blob".to_string(), - version_id: "v-delete-marker".to_string(), - }, - ]); - - assert_eq!(identifiers.len(), 2); - assert_eq!(identifiers[0].key, "_meta/tenant/a.json"); - assert_eq!(identifiers[0].version_id.as_deref(), Some("v-object")); - assert_eq!(identifiers[1].key, "uploads/tenant/event/blob"); - assert_eq!( - identifiers[1].version_id.as_deref(), - Some("v-delete-marker") - ); - } - - #[tokio::test] - async fn delete_object_versions_empty_input_short_circuits_before_folding() { - let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("static client"); - let outcome = storage - .delete_object_versions_with_folding(&[], |_| BulkDeleteOutcome { - deleted: 0, - already_missing: 0, - versioned_keys: vec!["wrong-fold".to_string()], - failed: Vec::new(), - }) - .await - .expect("empty delete short-circuits before fold"); - assert_eq!(outcome, BulkDeleteOutcome::default()); - } - - #[test] - fn parse_object_versions_page_includes_objects_delete_markers_and_dual_markers() { - let page = parse_object_versions_page( - br#" - - buzz-media - _meta/tenant/ - _meta/tenant/a.json - v-old - 2 - true - _meta/tenant/a.json - v-new - - _meta/tenant/a.json - v-delete - true - - - _meta/tenant/a.json - v-new - false - 42 - -"#, - ) - .expect("parse versions page"); - - assert!(page.is_truncated); - assert_eq!(page.next_key_marker.as_deref(), Some("_meta/tenant/a.json")); - assert_eq!(page.next_version_id_marker.as_deref(), Some("v-new")); - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-delete".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-new".to_string(), - kind: ObjectVersionKind::Object, - size: 42, - }, - ] - ); - } - - #[test] - fn parse_object_versions_page_preserves_repeated_interleaved_aws_ordering() { - let page = parse_object_versions_page( - br#" - k-av33 - k-av2 - k-av11 - k-bm2 - k-bm110 -"#, - ) - .expect("parse interleaved versions page"); - - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v3".to_string(), - kind: ObjectVersionKind::Object, - size: 3, - }, - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v2".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v1".to_string(), - kind: ObjectVersionKind::Object, - size: 1, - }, - ObjectVersionEntry { - key: "k-b".to_string(), - version_id: "m2".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-b".to_string(), - version_id: "m1".to_string(), - kind: ObjectVersionKind::Object, - size: 10, - }, - ] - ); - } - - #[test] - fn parse_object_versions_page_handles_marker_only_key_before_versioned_key() { - let page = parse_object_versions_page( - br#" - k-marker-onlyd-only - k-versionedv220 - k-versionedd1 - k-versionedv110 -"#, - ) - .expect("parse marker-only and versioned keys"); - - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "k-marker-only".to_string(), - version_id: "d-only".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "v2".to_string(), - kind: ObjectVersionKind::Object, - size: 20, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "d1".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "v1".to_string(), - kind: ObjectVersionKind::Object, - size: 10, - }, - ] - ); - } - fn tenant(n: u128) -> TenantContext { TenantContext::resolved( CommunityId::from_uuid(uuid::Uuid::from_u128(n)), @@ -958,7 +326,7 @@ mod tests { s3_secret_key: secret.to_string(), s3_bucket: "buzz-media".to_string(), s3_region: "us-west-2".to_string(), - s3_addressing_style: S3AddressingStyle::Path, + s3_addressing_style: crate::config::S3AddressingStyle::Path, max_image_bytes: 50 * 1024 * 1024, max_gif_bytes: 10 * 1024 * 1024, max_video_bytes: 524_288_000, @@ -970,36 +338,6 @@ mod tests { } } - /// Static keys present: builds a client without touching the AWS - /// credential chain (no env/metadata access), and the signing region - /// comes from config rather than a hardcoded "us-east-1". - #[test] - fn static_keys_build_client_with_configured_region() { - let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("static creds should build a client"); - match storage.bucket.region { - Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"), - other => panic!("expected Custom region, got {other:?}"), - } - } - - #[test] - fn client_constructor_applies_both_addressing_styles() { - let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("path-style client"); - assert!(path.bucket.is_path_style()); - assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media"); - - let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret"); - virtual_config.s3_addressing_style = S3AddressingStyle::Virtual; - let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client"); - assert!(virtual_hosted.bucket.is_subdomain_style()); - assert_eq!( - virtual_hosted.bucket.url(), - "http://buzz-media.localhost:9000" - ); - } - #[test] fn partial_static_keys_are_rejected() { let err = match MediaStorage::new(&storage_config("buzz_dev", "")) { diff --git a/crates/buzz-object-store/Cargo.toml b/crates/buzz-object-store/Cargo.toml index 4a8df03b39f..62687518f32 100644 --- a/crates/buzz-object-store/Cargo.toml +++ b/crates/buzz-object-store/Cargo.toml @@ -13,6 +13,7 @@ axum = { workspace = true } bytes = "1" futures-core = "0.3" futures-util = "0.3" +quick-xml = { version = "0.38", features = ["serialize"] } s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } serde = { workspace = true } thiserror = { workspace = true } diff --git a/crates/buzz-object-store/src/lib.rs b/crates/buzz-object-store/src/lib.rs index 590fd05f242..4df3747a4b4 100644 --- a/crates/buzz-object-store/src/lib.rs +++ b/crates/buzz-object-store/src/lib.rs @@ -97,6 +97,58 @@ pub struct BulkDeleteOutcome { pub failed: Vec<(String, String, String)>, } +/// Provider-neutral kind of retained object version. +/// +/// S3 reports concrete versions and delete markers. GCS reports generations; +/// because Buzz admits only GCS buckets with versioning and soft delete off, +/// those entries are always concrete objects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObjectVersionKind { + /// A byte-bearing object version. + Object, + /// A marker that hides older bytes without removing them. + DeleteMarker, +} + +/// One provider version under an object prefix. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ObjectVersionEntry { + /// Object key. + pub key: String, + /// Opaque provider version token (S3 version ID or GCS generation). + pub version_id: String, + /// Whether this is a byte-bearing version or a delete marker. + pub kind: ObjectVersionKind, + /// Byte size for object versions; zero for delete markers. + pub size: u64, +} + +/// Exact provider version identifier used for permanent deletion. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ObjectVersionRef { + /// Object key. + pub key: String, + /// Opaque provider version token (S3 version ID or GCS generation). + pub version_id: String, +} + +/// One page of prefix-scoped provider versions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ObjectVersionsPage { + /// Object versions and delete markers returned by this page. + pub entries: Vec, + /// First half of the next-page cursor. + /// + /// S3 uses a key marker; GCS stores its opaque page token here. + pub next_key_marker: Option, + /// Second half of the next-page cursor, used by S3 when several versions + /// of one key straddle a page boundary. GCS leaves it empty. + pub next_version_id_marker: Option, + /// Whether more pages remain. + pub is_truncated: bool, +} + /// The object-store operations Buzz actually performs. /// /// Implementations are shared across the process behind an `Arc`: the relay @@ -203,6 +255,23 @@ pub trait ObjectStore: Send + Sync { /// sees the same per-key fold and decides retry policy. async fn delete_objects(&self, keys: &[String]) -> Result; + /// Fetch one page of exact object versions under a prefix. + /// + /// Cursor fields are opaque to callers and must be replayed together. + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result; + + /// Permanently delete exact provider versions. + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result; + /// Probe connectivity and bucket access. async fn ping(&self) -> Result<(), ObjectStoreError>; diff --git a/crates/buzz-object-store/src/providers/s3.rs b/crates/buzz-object-store/src/providers/s3.rs index 124de555b4f..9665b93e7eb 100644 --- a/crates/buzz-object-store/src/providers/s3.rs +++ b/crates/buzz-object-store/src/providers/s3.rs @@ -24,18 +24,25 @@ //! observation. Do not widen the ambiguous set: it would let a genuine //! conformance failure be silently dropped from the probe's observer set. +use std::collections::HashMap; use std::path::Path; use std::str::FromStr; use async_trait::async_trait; use bytes::Bytes; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::Reader; use s3::creds::Credentials; use s3::error::S3Error; +use s3::request::Request as _; use s3::{Bucket, Region}; use crate::error::ObjectStoreError; use crate::revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; -use crate::{BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore}; +use crate::{ + BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore, + ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, +}; /// S3 URL addressing style shared by media and Git/CAS storage. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] @@ -207,15 +214,32 @@ fn classify(operation: &'static str, key: &str, error: S3Error) -> ObjectStoreEr /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to /// `already_missing` to keep checkpointed retry idempotent. fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + fold_delete_result(result, DeleteMode::Unversioned) +} + +fn fold_version_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + fold_delete_result(result, DeleteMode::ExplicitVersion) +} + +enum DeleteMode { + Unversioned, + ExplicitVersion, +} + +fn fold_delete_result( + result: s3::serde_types::DeleteObjectsResult, + mode: DeleteMode, +) -> BulkDeleteOutcome { let mut outcome = BulkDeleteOutcome::default(); for deleted in result.deleted { - if deleted.delete_marker == Some(true) + let has_version_artifact = deleted.delete_marker == Some(true) || deleted.delete_marker_version_id.is_some() - || deleted.version_id.is_some() - { - outcome.versioned_keys.push(deleted.key); - } else { - outcome.deleted += 1; + || deleted.version_id.is_some(); + match mode { + DeleteMode::Unversioned if has_version_artifact => { + outcome.versioned_keys.push(deleted.key); + } + DeleteMode::Unversioned | DeleteMode::ExplicitVersion => outcome.deleted += 1, } } for error in result.errors { @@ -228,6 +252,150 @@ fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> Bulk outcome } +#[derive(Debug, Default)] +struct ListVersionFields { + key: Option, + version_id: Option, + size: Option, +} + +fn local_name(name: &[u8]) -> &[u8] { + name.rsplit(|byte| *byte == b':').next().unwrap_or(name) +} + +fn version_xml_error(error: impl std::fmt::Display) -> ObjectStoreError { + ObjectStoreError::Provider { + operation: "list_versions_page", + message: error.to_string(), + } +} + +fn read_element_text( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, +) -> Result { + reader + .read_text(start.to_end().name()) + .map(|text| text.into_owned()) + .map_err(version_xml_error) +} + +fn skip_element( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, +) -> Result<(), ObjectStoreError> { + reader + .read_to_end(start.to_end().name()) + .map_err(version_xml_error)?; + Ok(()) +} + +fn parse_list_version_entry( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, + kind: ObjectVersionKind, +) -> Result { + let mut fields = ListVersionFields::default(); + loop { + match reader.read_event().map_err(version_xml_error)? { + Event::Start(child) => match local_name(child.local_name().as_ref()) { + b"Key" => fields.key = Some(read_element_text(reader, &child)?), + b"VersionId" => fields.version_id = Some(read_element_text(reader, &child)?), + b"Size" => { + fields.size = Some( + read_element_text(reader, &child)? + .parse::() + .map_err(version_xml_error)?, + ); + } + _ => skip_element(reader, &child)?, + }, + Event::Empty(child) => match local_name(child.local_name().as_ref()) { + b"Key" => fields.key = Some(String::new()), + b"VersionId" => fields.version_id = Some(String::new()), + b"Size" => fields.size = Some(0), + _ => {} + }, + Event::End(end) if end.name().as_ref() == start.to_end().name().as_ref() => { + let key = fields.key.ok_or_else(|| { + version_xml_error("ListObjectVersions entry missing Key") + })?; + let version_id = fields.version_id.ok_or_else(|| { + version_xml_error("ListObjectVersions entry missing VersionId") + })?; + return Ok(ObjectVersionEntry { + key, + version_id, + kind, + size: if kind == ObjectVersionKind::Object { + fields.size.unwrap_or(0) + } else { + 0 + }, + }); + } + Event::Eof => { + return Err(version_xml_error( + "unexpected EOF inside ListObjectVersions entry", + )); + } + _ => {} + } + } +} + +fn parse_object_versions_page(xml: &[u8]) -> Result { + let mut reader = Reader::from_reader(xml); + reader.config_mut().trim_text(true); + let mut entries = Vec::new(); + let mut next_key_marker = None; + let mut next_version_id_marker = None; + let mut is_truncated = false; + + loop { + match reader.read_event().map_err(version_xml_error)? { + Event::Start(start) => match local_name(start.local_name().as_ref()) { + b"Version" => entries.push(parse_list_version_entry( + &mut reader, + &start, + ObjectVersionKind::Object, + )?), + b"DeleteMarker" => entries.push(parse_list_version_entry( + &mut reader, + &start, + ObjectVersionKind::DeleteMarker, + )?), + b"IsTruncated" => { + is_truncated = + read_element_text(&mut reader, &start)?.eq_ignore_ascii_case("true"); + } + b"NextKeyMarker" => { + next_key_marker = Some(read_element_text(&mut reader, &start)?); + } + b"NextVersionIdMarker" => { + next_version_id_marker = Some(read_element_text(&mut reader, &start)?); + } + b"ListVersionsResult" => {} + _ => skip_element(&mut reader, &start)?, + }, + Event::Empty(start) => match local_name(start.local_name().as_ref()) { + b"NextKeyMarker" => next_key_marker = Some(String::new()), + b"NextVersionIdMarker" => next_version_id_marker = Some(String::new()), + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + + Ok(ObjectVersionsPage { + entries, + next_key_marker, + next_version_id_marker, + is_truncated, + }) +} + #[async_trait] impl ObjectStore for S3ObjectStore { fn provider(&self) -> ProviderKind { @@ -453,6 +621,76 @@ impl ObjectStore for S3ObjectStore { Ok(fold_bulk_delete_result(result)) } + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result { + let mut query = HashMap::from([ + ("versions".to_string(), String::new()), + ("prefix".to_string(), prefix.to_string()), + ("max-keys".to_string(), max_keys.to_string()), + ]); + if let Some(marker) = key_marker { + query.insert("key-marker".to_string(), marker); + } + if let Some(marker) = version_id_marker { + query.insert("version-id-marker".to_string(), marker); + } + let bucket = self + .bucket + .with_extra_query(query) + .map_err(|e| classify("list_versions_page", prefix, e))?; + let request = s3::request::tokio_backend::ReqwestRequest::new( + &bucket, + "/", + s3::command::Command::GetObject, + ) + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + let response = request + .response_data(false) + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + if response.status_code() >= 300 { + return Err(ObjectStoreError::Provider { + operation: "list_versions_page", + message: format!( + "unexpected status {}: {}", + response.status_code(), + response.as_str().unwrap_or("") + ), + }); + } + parse_object_versions_page(response.as_slice()) + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + if versions.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = versions + .iter() + .map(|version| { + s3::serde_types::ObjectIdentifier::with_version( + version.key.clone(), + version.version_id.clone(), + ) + }) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| classify("delete_versions", "", e))?; + Ok(fold_version_delete_result(result)) + } + async fn ping(&self) -> Result<(), ObjectStoreError> { self.list_page("", None, 1).await.map(|_| ()) } @@ -535,31 +773,6 @@ mod tests { } } - #[test] - fn addressing_style_parses_supported_values() { - assert_eq!( - S3AddressingStyle::from_str("path"), - Ok(S3AddressingStyle::Path) - ); - assert_eq!( - S3AddressingStyle::from_str("virtual"), - Ok(S3AddressingStyle::Virtual) - ); - assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path); - } - - #[test] - fn addressing_style_rejects_unknown_or_ambiguous_values() { - for invalid in ["", "auto", "PATH", "virtual-hosted"] { - let error = - S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style"); - assert!( - error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"), - "unexpected error for {invalid:?}: {error}" - ); - } - } - /// Pre-classification failures are the *only* ambiguous outcomes. The Git /// conformance probe drops exactly this set from its observer count, so /// widening it would let a real conformance failure vanish. diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index deb2e7e16a5..f57175ddb67 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -64,7 +64,7 @@ base64 = "0.22" buzz-sdk = { workspace = true } buzz-workflow = { workspace = true, features = ["reqwest"] } buzz-media = { workspace = true } -s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } +buzz-object-store = { workspace = true } tempfile = "3" bytes = "1" infer = "0.19" diff --git a/crates/buzz-relay/src/api/git/cas_publish.rs b/crates/buzz-relay/src/api/git/cas_publish.rs index 50bb36d818c..aa456db90c9 100644 --- a/crates/buzz-relay/src/api/git/cas_publish.rs +++ b/crates/buzz-relay/src/api/git/cas_publish.rs @@ -75,7 +75,9 @@ use crate::api::git::manifest::{ pointer_key, Manifest, ManifestError, MANIFEST_VERSION, MAX_MANIFEST_PACKS, MAX_MANIFEST_REFS, PACK_COMPACTION_THRESHOLD, }; -use crate::api::git::store::{CasOutcome, ETag, GitStore, Precond, StoreError}; +use buzz_object_store::{ConditionalWrite, Revision, WriteCondition}; + +use crate::api::git::store::{GitStore, StoreError}; use buzz_core::TenantContext; const PACK_CAPTURE_TIMEOUT: Duration = Duration::from_secs(300); @@ -202,7 +204,7 @@ pub struct CasSuccess { /// this *before* running receive-pack against the hydrated workspace, and /// pass the same value into [`cas_publish`]. If the pointer advances /// between load and CAS (a concurrent push wins), the CAS fails with -/// `LostRace`/`Conflict` and the loser re-pushes — that is the only safe +/// `Conflict` and the loser re-pushes — that is the only safe /// retry path (the loser's receive-pack output is derived against the /// superseded parent, so reusing it would violate /// `Inv_RefDerivedFromParent`). @@ -213,10 +215,10 @@ pub struct CasSuccess { /// pointer happens to be live at CAS time. #[derive(Debug, Clone)] pub struct ParentState { - /// ETag predicating the next CAS write. `None` only when the pointer - /// does not yet exist (first push to an empty repo) — then the CAS - /// uses `If-None-Match: *`. - pub if_match: Option, + /// Revision predicating the next CAS write. `None` only when the + /// pointer does not yet exist (first push to an empty repo) — then the + /// CAS is create-only. + pub if_match: Option, /// The parent manifest's content-addressed *digest* (64-hex), not the /// full `manifests/` key. This lands in `Manifest.parent` and /// is what `Inv_RefDerivedFromParent` reasons over (parent = @@ -246,14 +248,14 @@ impl ParentState { /// Build a `ParentState` from already-loaded pointer state. /// /// The hydrate layer reads the pointer + verified manifest as part of - /// materializing the workspace, then hands the same `(etag, digest, + /// materializing the workspace, then hands the same `(revision, digest, /// manifest)` tuple back here. Centralizing the constructor in /// `cas_publish` means there's one place where `ParentState` /// invariants live; centralizing the I/O in `hydrate` means we read /// the pointer once per push, not twice. - pub fn from_loaded(etag: ETag, digest: String, parent: Manifest) -> Self { + pub fn from_loaded(revision: Revision, digest: String, parent: Manifest) -> Self { Self { - if_match: Some(etag), + if_match: Some(revision), parent_digest: Some(digest), parent, } @@ -385,10 +387,12 @@ fn digest_from_pack_key(key: &str) -> Result { .filter(|digest| digest.len() == 64 && digest.chars().all(|c| c.is_ascii_hexdigit())) .map(str::to_string) .ok_or_else(|| { - CasError::Backend(StoreError::Backend(s3::error::S3Error::HttpFailWithBody( - 500, - format!("put_pack returned non-standard key: {key}"), - ))) + CasError::Backend(StoreError::Backend( + buzz_object_store::ObjectStoreError::Provider { + operation: "put_pack", + message: format!("put_pack returned non-standard key: {key}"), + }, + )) }) } @@ -951,10 +955,12 @@ fn digest_from_manifest_key(key: &str) -> Result { key.strip_prefix("manifests/") .map(str::to_string) .ok_or_else(|| { - CasError::Backend(StoreError::Backend(s3::error::S3Error::HttpFailWithBody( - 500, - format!("put_manifest returned non-standard key: {key}"), - ))) + CasError::Backend(StoreError::Backend( + buzz_object_store::ObjectStoreError::Provider { + operation: "put_manifest", + message: format!("put_manifest returned non-standard key: {key}"), + }, + )) }) } @@ -1210,8 +1216,8 @@ async fn cas_publish_inner( // Step 7: CAS the pointer. let precond = match &parent_state.if_match { - Some(e) => Precond::IfMatch(e.clone()), - None => Precond::IfNoneMatchStar, + Some(revision) => WriteCondition::Matches(revision.clone()), + None => WriteCondition::Absent, }; let cas_outcome = match store .put_pointer(&pkey, manifest_digest.as_bytes(), precond) @@ -1232,7 +1238,7 @@ async fn cas_publish_inner( } }; match cas_outcome { - CasOutcome::Won(_new_etag) => { + ConditionalWrite::Committed(_new_revision) => { if let Some(observation) = &compaction_observation { record_compaction( "success", @@ -1247,7 +1253,7 @@ async fn cas_publish_inner( manifest_key, }) } - CasOutcome::LostRace => { + ConditionalWrite::Conflict => { if let Some(observation) = &compaction_observation { record_compaction( "cas_conflict", @@ -1266,11 +1272,11 @@ async fn cas_publish_inner( let expected = parent_state .if_match .as_ref() - .map(|e| e.0.as_str()) - .unwrap_or(""); + .map(|revision| format!("{revision:?}")) + .unwrap_or_else(|| "".to_string()); warn!( pointer = %pkey, - expected_etag = %expected, + expected_revision = %expected, attempted_manifest = %manifest_key, "CAS lost race; resolving winner for reconcile" ); @@ -1284,7 +1290,7 @@ async fn cas_publish_inner( } } -/// Re-read the pointer after a `LostRace` and fetch the winner's manifest. +/// Re-read the pointer after a CAS conflict and fetch the winner's manifest. /// /// Fail-closed at every step: if the pointer is now absent (a deletion /// raced in — currently impossible under the protocol's no-delete rule, @@ -1592,7 +1598,7 @@ mod tests { } fn live_store() -> GitStore { - GitStore::new( + GitStore::from_s3_config( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", @@ -1712,12 +1718,12 @@ mod tests { let repo = "history"; let pkey = pointer_key(ctx.community(), &owner, repo); match store - .put_pointer(&pkey, parent_digest.as_bytes(), Precond::IfNoneMatchStar) + .put_pointer(&pkey, parent_digest.as_bytes(), WriteCondition::Absent) .await .expect("put pointer") { - CasOutcome::Won(_) => {} - CasOutcome::LostRace => panic!("unique pointer must win"), + ConditionalWrite::Committed(_) => {} + ConditionalWrite::Conflict => panic!("unique pointer must win"), } let cache_parent = scratch.path().join("cache"); let cache = diff --git a/crates/buzz-relay/src/api/git/hydrate.rs b/crates/buzz-relay/src/api/git/hydrate.rs index 3ce809d18f7..4e1b9b0f77c 100644 --- a/crates/buzz-relay/src/api/git/hydrate.rs +++ b/crates/buzz-relay/src/api/git/hydrate.rs @@ -36,7 +36,9 @@ use tokio::process::Command; use super::cas_publish::ParentState; use super::manifest::{is_hex_oid, is_safe_refname, pointer_key, Manifest, ManifestError}; use super::pack_cache::GitPackCache; -use super::store::{ETag, GitStore, StoreError}; +use buzz_object_store::Revision; + +use super::store::{GitStore, StoreError}; use buzz_core::TenantContext; /// Manifests should stay small after ref/pack cardinality validation. Bound @@ -239,7 +241,7 @@ pub async fn hydrate_for_write( } } -/// Resolve the pointer to its `(ETag, digest, verified Manifest)` triple. +/// Resolve the pointer to its `(Revision, digest, verified Manifest)` triple. /// /// `Ok(None)` if the pointer is absent (caller decides 404 vs first-push /// per call site). `Err(_)` on any below-pointer failure. @@ -248,7 +250,7 @@ async fn load_pointer( ctx: &TenantContext, owner: &str, repo: &str, -) -> Result, HydrateError> { +) -> Result, HydrateError> { let pkey = pointer_key(ctx.community(), owner, repo); let (etag, pointer_bytes) = match store.get_pointer(&pkey).await? { Some(p) => p, @@ -543,7 +545,7 @@ mod tests { #[tokio::test] async fn materialized_repo_is_created_under_configured_scratch_dir() { let scratch = TempDir::new().unwrap(); - let store = GitStore::new( + let store = GitStore::from_s3_config( "http://localhost:9000", "x", "x", @@ -588,7 +590,7 @@ mod tests { } fn store() -> GitStore { - GitStore::new( + GitStore::from_s3_config( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", @@ -682,13 +684,15 @@ mod tests { .put_pointer( &pkey, manifest_digest.as_bytes(), - super::super::store::Precond::IfNoneMatchStar, + buzz_object_store::WriteCondition::Absent, ) .await .expect("put_pointer") { - super::super::store::CasOutcome::Won(_) => {} - super::super::store::CasOutcome::LostRace => panic!("first INM* must win"), + buzz_object_store::ConditionalWrite::Committed(_) => {} + buzz_object_store::ConditionalWrite::Conflict => { + panic!("first create-only write must commit") + } } // Hydrate. @@ -829,13 +833,15 @@ mod tests { .put_pointer( &pkey, manifest_digest.as_bytes(), - super::super::store::Precond::IfNoneMatchStar, + buzz_object_store::WriteCondition::Absent, ) .await .expect("put_pointer") { - super::super::store::CasOutcome::Won(_) => {} - super::super::store::CasOutcome::LostRace => panic!("first INM* must win"), + buzz_object_store::ConditionalWrite::Committed(_) => {} + buzz_object_store::ConditionalWrite::Conflict => { + panic!("first create-only write must commit") + } } let scratch = TempDir::new().unwrap(); diff --git a/crates/buzz-relay/src/api/git/store.rs b/crates/buzz-relay/src/api/git/store.rs index bdfca8dcf2d..10c364cef82 100644 --- a/crates/buzz-relay/src/api/git/store.rs +++ b/crates/buzz-relay/src/api/git/store.rs @@ -4,21 +4,27 @@ //! and the CAS pointer swap (axiom A3) described in //! `docs/git-on-object-storage.md`. //! -//! ## The 412 sharp edge +//! This is a domain facade over [`buzz_object_store::ObjectStore`]: it owns +//! the git key layout, digest verification, and the conformance probe, and +//! knows nothing about which provider is underneath. Compare-and-swap tokens +//! are [`Revision`]s, not ETags — the S3 provider is the only place an ETag +//! exists. //! -//! `rust-s3 = "0.37"` is shared across the workspace with `buzz-media`. The -//! `fail-on-err` Cargo feature is unified ON across the build graph, which -//! means non-2xx responses arrive here as `S3Error::HttpFailWithBody(code, -//! body)` *before* the caller sees `ResponseData`. The pointer-CAS path treats -//! the precondition-failure status (412) as a *semantic* result (`LostRace`), -//! not an error — see `classify_cas`. Empirically verified against MinIO in -//! `probe::probe_412_surfacing`. +//! ## Classified vs. unknown outcomes +//! +//! The pointer-CAS path treats a failed precondition as a *semantic* result +//! ([`ConditionalWrite::Conflict`]), not an error. A request that never +//! produced a classified provider response is different again: its outcome is +//! *unknown*, so the probe drops it from the observer set rather than counting +//! it as a lost race. That distinction lives in +//! [`buzz_object_store::ObjectStoreError::is_ambiguous`]; see the S3 provider +//! module for how each provider failure is classified. //! //! ## Content addressing (A1) //! -//! Pack and manifest keys are the SHA-256 of their bytes. Writes use -//! `If-None-Match: *` so the same key is never overwritten. Readers verify -//! object bytes against the expected digest on `get_verified`; any mismatch is +//! Pack and manifest keys are the SHA-256 of their bytes. Writes are +//! create-only so the same key is never overwritten. Readers verify object +//! bytes against the expected digest on `get_verified`; any mismatch is //! *detectable*, not silent — that is what A1's "create-only + content-address" //! discipline buys us, independent of bucket immutability features. @@ -26,43 +32,14 @@ use std::sync::Arc; +use buzz_object_store::{ + ConditionalWrite, ImmutableWrite, ObjectStore, ObjectStoreError, Revision, S3AddressingStyle, + S3ObjectStore, S3StoreConfig, WriteCondition, +}; use bytes::Bytes; -use s3::creds::Credentials; -use s3::error::S3Error; -use s3::{Bucket, Region}; use sha2::{Digest, Sha256}; -/// Opaque object-store ETag (used for `If-Match` on pointer CAS). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ETag(pub String); - -/// Precondition for `put_pointer`. -#[derive(Debug, Clone)] -pub enum Precond { - /// Create-only: succeed iff the pointer does not yet exist. - IfNoneMatchStar, - /// CAS: succeed iff the current ETag matches. - IfMatch(ETag), -} - -/// Result of a CAS pointer write. -/// -/// `LostRace` is *not* an error — it is the standard outcome of a losing CAS -/// and must be classified here so callers can decide retry vs. non-ff. On -/// `Won`, the returned `ETag` is the PUT response's ETag and can be fed -/// directly into the next `IfMatch` round (verified empirically against MinIO -/// in `probe::probe_full_roundtrip`). A backend that succeeds on the CAS PUT -/// but omits the response ETag is treated as non-conforming and fails the -/// operation with `StoreError::Backend` — see `classify_cas`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CasOutcome { - /// CAS succeeded; the new pointer ETag (suitable for the next `IfMatch`). - Won(ETag), - /// CAS lost the race (server returned 412). - LostRace, -} - -/// Errors that are *actually* errors — `LostRace` is not one. +/// Errors that are *actually* errors — a lost CAS race is not one. #[derive(Debug, thiserror::Error)] pub enum StoreError { /// The requested key does not exist. @@ -89,8 +66,8 @@ pub enum StoreError { actual: String, }, /// Any other backend / transport error. - #[error("s3 backend error: {0}")] - Backend(#[from] S3Error), + #[error("object store error: {0}")] + Backend(ObjectStoreError), /// Invalid storage configuration detected at client construction. #[error("git store config error: {0}")] Config(String), @@ -99,6 +76,35 @@ pub enum StoreError { Probe(ProbeFailure), } +impl From for StoreError { + /// Lift the provider errors that git treats as domain outcomes into their + /// own variants, so call sites keep matching on `NotFound` / + /// `ObjectTooLarge` / `DigestMismatch` directly. + fn from(error: ObjectStoreError) -> Self { + match error { + ObjectStoreError::NotFound { key } => Self::NotFound(key), + ObjectStoreError::ObjectTooLarge { key, size, max } => { + Self::ObjectTooLarge { key, size, max } + } + ObjectStoreError::DigestMismatch { + key, + expected, + actual, + } => Self::DigestMismatch { + key, + expected, + actual, + }, + other => Self::Backend(other), + } + } +} + +/// Build a backend error for a git-layer invariant the provider cannot express. +fn backend(operation: &'static str, message: String) -> StoreError { + StoreError::Backend(ObjectStoreError::Provider { operation, message }) +} + /// Configuration for `GitStore::run_conformance_probe`. /// /// Defaults: 32-way concurrency, 3 rounds. The probe is a deployment gate — @@ -131,7 +137,7 @@ pub struct ProbeReport { /// Total number of *transport-unknown* per-racer outcomes across all /// race rounds (sum of both `if_match_race` and `if_none_match_race` /// phases). A "transport-unknown" is a pre-classification failure — - /// `S3Error::{Reqwest, Http, Io}` — that means the racer never got a + /// [`ObjectStoreError::is_ambiguous`] — that means the racer never got a /// classified response from the backend, so its outcome is neither /// evidence for nor against A3 linearizability. Such racers are /// dropped from the observer set (see the race phases for the @@ -149,7 +155,7 @@ pub struct ProbeReport { #[derive(Debug, thiserror::Error)] #[error("conformance probe failed in phase '{phase}' (round {round}, key {key}): {reason}")] pub struct ProbeFailure { - /// One of `sequential`, `if_match_race`, `if_none_match_race`, `etag_consistency`. + /// One of `sequential`, `if_match_race`, `if_none_match_race`, `revision_consistency`. pub phase: &'static str, /// Round index (0-based) when this phase ran multiple rounds. pub round: usize, @@ -168,58 +174,44 @@ impl From for StoreError { /// Object-store client for git refs. #[derive(Clone)] pub struct GitStore { - bucket: Arc, + store: Arc, } impl GitStore { - /// Build a client against an S3-compatible endpoint (e.g. MinIO). + /// Build a git store over an existing object-store client. /// - /// `addressing_style` is shared with media storage so both paths sign and - /// route requests consistently. Path style supports the bundled MinIO DNS; - /// virtual-hosted style supports standard S3 and providers such as Railway. + /// The relay constructs exactly one provider per process and hands the + /// same client to media storage and to this facade. + pub fn new(store: Arc) -> Self { + Self { store } + } + + /// Build a git store over a freshly constructed S3 provider. /// - /// Credential selection mirrors [`buzz_media::MediaStorage::new`]: - /// - both `access_key` and `secret_key` non-empty → static credentials - /// (MinIO/local/dev, or any static-key deployment); - /// - both empty → the AWS default credential chain via - /// [`Credentials::default`] (env, profile, web-identity/IRSA, container, - /// instance metadata), so the relay can use its pod IAM role; - /// - exactly one empty → a config error, to surface a half-configured - /// static deployment instead of silently falling back to the chain. - pub fn new( + /// Convenience for tests and the backend conformance probe, which connect + /// to a bare S3-compatible endpoint without the rest of the relay. + /// Production shares one client via [`GitStore::new`]. + pub fn from_s3_config( endpoint: &str, access_key: &str, secret_key: &str, bucket_name: &str, region: &str, - addressing_style: buzz_media::config::S3AddressingStyle, + addressing_style: S3AddressingStyle, ) -> Result { - let region = Region::Custom { - region: region.into(), - endpoint: endpoint.into(), - }; - let creds = match (access_key.is_empty(), secret_key.is_empty()) { - (false, false) => Credentials::new(Some(access_key), Some(secret_key), None, None, None), - (true, true) => { - // No static keys configured: resolve from the AWS credential - // chain (IRSA web-identity, env, profile, instance metadata). - Credentials::default() - } - _ => { - return Err(StoreError::Config( - "s3 access_key and secret_key must be configured together, or both empty to use the AWS credential chain".to_string(), - )); - } - } - .map_err(|e| StoreError::Backend(S3Error::Credentials(e)))?; - let bucket = Bucket::new(bucket_name, region, creds).map_err(StoreError::Backend)?; - let bucket = match addressing_style { - buzz_media::config::S3AddressingStyle::Path => bucket.with_path_style(), - buzz_media::config::S3AddressingStyle::Virtual => bucket, - }; - Ok(Self { - bucket: Arc::from(bucket), + let store = S3ObjectStore::new(&S3StoreConfig { + endpoint: endpoint.to_string(), + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket_name.to_string(), + region: region.to_string(), + addressing_style, }) + .map_err(|e| match e { + ObjectStoreError::Config(message) => StoreError::Config(message), + other => StoreError::Backend(other), + })?; + Ok(Self::new(Arc::new(store))) } /// Compute the hex SHA-256 of `bytes`. The content-addressed key. @@ -237,10 +229,10 @@ impl GitStore { /// each manifest pack key. pub fn idx_key_for_pack_digest(pack_digest: &str) -> Result { if pack_digest.len() != 64 || !pack_digest.chars().all(|c| c.is_ascii_hexdigit()) { - return Err(StoreError::Backend(S3Error::HttpFailWithBody( - 400, + return Err(backend( + "idx_key_for_pack_digest", format!("invalid pack digest for idx sidecar: {pack_digest:?}"), - ))); + )); } Ok(format!("idx/{pack_digest}")) } @@ -249,8 +241,8 @@ impl GitStore { /// /// **The caller does not choose the key.** It is derived as /// `/` inside this method. This makes the - /// idempotency claim *constructive*: a 412 collision means the key already - /// holds bytes whose digest equals `sha256(these bytes)`, so by A1 + /// idempotency claim *constructive*: a precondition collision means the key + /// already holds bytes whose digest equals `sha256(these bytes)`, so by A1 /// (content-addressing) the stored bytes equal these bytes. Without this /// enforcement, a buggy caller passing the wrong key would silently break /// A1 detectability on read. @@ -263,24 +255,9 @@ impl GitStore { content_type: &str, ) -> Result { let key = Self::content_key(prefix, bytes); - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers(&key, bytes, content_type, Some(headers)) - .await - { - Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(key), - // 412 on a content-addressed key means the key already holds the - // same bytes (by construction — the key is the digest). A1 is - // preserved without a defensive GET. - Err(S3Error::HttpFailWithBody(412, _)) => Ok(key), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), - } + // Both outcomes are success: by construction the key holds these bytes. + self.store.put_immutable(&key, bytes, content_type).await?; + Ok(key) } /// Write a pack object. Returns the content-addressed key (`packs/`). @@ -293,31 +270,15 @@ impl GitStore { /// /// Unlike packs/manifests, the key is not the SHA-256 of `idx_bytes`; it is /// `idx/` so hydrates can derive it without changing manifest - /// bytes. A 412 is idempotent success for the cache layer: the first writer - /// already produced the sidecar for this pack, and hydrate validates before - /// trusting it. + /// bytes. A precondition collision is idempotent success for the cache + /// layer: the first writer already produced the sidecar for this pack, and + /// hydrate validates before trusting it. pub async fn put_idx(&self, pack_digest: &str, idx_bytes: &[u8]) -> Result { let key = Self::idx_key_for_pack_digest(pack_digest)?; - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers( - &key, - idx_bytes, - "application/x-git-index", - Some(headers), - ) - .await - { - Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(key), - Err(S3Error::HttpFailWithBody(412, _)) => Ok(key), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), - } + self.store + .put_immutable(&key, idx_bytes, "application/x-git-index") + .await?; + Ok(key) } /// Read an idx sidecar for `packs/`. @@ -350,11 +311,7 @@ impl GitStore { /// detectability. This raw `get` exists for the pointer (whose key is not a /// digest). pub async fn get(&self, key: &str) -> Result { - match self.bucket.get_object(key).await { - Ok(resp) => Ok(Bytes::from(resp.to_vec())), - Err(S3Error::HttpFailWithBody(404, _)) => Err(StoreError::NotFound(key.into())), - Err(e) => Err(StoreError::Backend(e)), - } + Ok(self.store.get(key).await?) } /// GET an object and verify its bytes hash to `expected_digest` (hex SHA-256). @@ -368,17 +325,7 @@ impl GitStore { expected_digest: &str, ) -> Result { let bytes = self.get(key).await?; - let mut hasher = Sha256::new(); - hasher.update(&bytes); - let actual = hex::encode(hasher.finalize()); - if actual != expected_digest { - return Err(StoreError::DigestMismatch { - key: key.into(), - expected: expected_digest.into(), - actual, - }); - } - Ok(bytes) + Self::verify_digest(key, expected_digest, bytes) } /// GET an immutable object after rejecting objects larger than `max_bytes`. @@ -386,7 +333,7 @@ impl GitStore { /// Pack and manifest objects are content addressed and create-only, so the /// HEAD result cannot race with a different body at the same key. The /// second length check protects against a backend that reports a bad - /// Content-Length header. + /// content length. pub async fn get_verified_limited( &self, key: &str, @@ -394,162 +341,61 @@ impl GitStore { max_bytes: u64, ) -> Result { let bytes = self.get_limited(key, max_bytes).await?; - let mut hasher = Sha256::new(); - hasher.update(&bytes); - let actual = hex::encode(hasher.finalize()); - if actual != expected_digest { - return Err(StoreError::DigestMismatch { - key: key.into(), - expected: expected_digest.into(), - actual, - }); - } - Ok(bytes) + Self::verify_digest(key, expected_digest, bytes) } /// GET an object after rejecting bodies larger than `max_bytes`. pub async fn get_limited(&self, key: &str, max_bytes: u64) -> Result { - let (head, status) = self.bucket.head_object(key).await.map_err(|e| match e { - S3Error::HttpFailWithBody(404, _) => StoreError::NotFound(key.into()), - other => StoreError::Backend(other), - })?; - if status == 404 { - return Err(StoreError::NotFound(key.into())); - } - if !(200..300).contains(&status) { - return Err(StoreError::Backend(S3Error::HttpFailWithBody( - status, - "unexpected status".into(), - ))); - } - if let Some(content_length) = head.content_length { - let size = u64::try_from(content_length).unwrap_or(u64::MAX); - if size > max_bytes { - return Err(StoreError::ObjectTooLarge { - key: key.into(), - size, - max: max_bytes, - }); - } - } - - let bytes = self.get(key).await?; - let size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - if size > max_bytes { - return Err(StoreError::ObjectTooLarge { - key: key.into(), - size, - max: max_bytes, - }); - } - Ok(bytes) + Ok(self.store.get_limited(key, max_bytes).await?) } - /// GET the pointer object, returning its ETag and bytes *from the same + /// GET the pointer object, returning its revision and bytes *from the same /// response* — atomic snapshot. /// /// Returns `Ok(None)` if the pointer does not exist (first-push case). /// /// **Why one GET, not HEAD-then-GET.** A separate HEAD followed by GET - /// can straddle a concurrent writer: the HEAD's ETag and the GET's body - /// would describe different pointer versions, and a caller that later - /// did `IfMatch(etag_from_head)` would be predicating on a version it - /// never actually read. Reading both fields from the GET response keeps - /// the snapshot consistent (A2: a single GET observes a single committed - /// object). Verified empirically in `probe::probe_get_exposes_etag`. - pub async fn get_pointer(&self, key: &str) -> Result, StoreError> { - match self.bucket.get_object(key).await { - Ok(resp) => { - let headers = resp.headers(); - let etag = headers - .get("etag") - .or_else(|| headers.get("ETag")) - .cloned() - .ok_or_else(|| { - StoreError::Backend(S3Error::HttpFailWithBody( - 500, - "GET pointer: response missing ETag".into(), - )) - })?; - Ok(Some((ETag(etag), Bytes::from(resp.to_vec())))) - } - Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), - Err(e) => Err(StoreError::Backend(e)), - } + /// can straddle a concurrent writer: the HEAD's revision and the GET's + /// body would describe different pointer versions, and a caller that + /// later predicated a CAS on the HEAD revision would be predicating on a + /// version it never actually read. Reading both fields from the GET + /// response keeps the snapshot consistent (A2: a single GET observes a + /// single committed object). + pub async fn get_pointer(&self, key: &str) -> Result, StoreError> { + Ok(self.store.get_with_revision(key).await?) } /// Write the pointer under a precondition (§Push step 7 — the CAS). /// - /// Returns `CasOutcome::LostRace` on 412 (the standard losing outcome). - /// On `CasOutcome::Won`, the returned `ETag` is read from the response - /// headers — callers use it as the `If-Match` value for the next CAS. + /// Returns [`ConditionalWrite::Conflict`] when the precondition did not + /// hold (the standard losing outcome). On + /// [`ConditionalWrite::Committed`], the returned [`Revision`] is read from + /// the write response — callers use it to predicate the next CAS. pub async fn put_pointer( &self, key: &str, body: &[u8], - precond: Precond, - ) -> Result { - let mut headers = axum::http::HeaderMap::new(); - match &precond { - Precond::IfNoneMatchStar => { - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - } - Precond::IfMatch(ETag(tag)) => { - headers.insert( - axum::http::header::IF_MATCH, - tag.parse().map_err(|_| { - StoreError::Backend(S3Error::HttpFailWithBody( - 400, - format!("invalid etag {tag}"), - )) - })?, - ); - } - } - let result = self - .bucket - .put_object_with_content_type_and_headers(key, body, "application/json", Some(headers)) - .await; - Self::classify_cas(result) + condition: WriteCondition, + ) -> Result { + Ok(self + .store + .put_conditional(key, body, "application/json", condition) + .await?) } - /// Map a rust-s3 PUT outcome to a `CasOutcome`. - /// - /// 412 → `LostRace`. 2xx → `Won(etag)` (etag read from response headers, - /// empty if missing — callers must tolerate empty etag and re-HEAD if they - /// need it strictly). Everything else bubbles as `StoreError::Backend`. - fn classify_cas( - result: Result, - ) -> Result { - match result { - Ok(resp) if (200..300).contains(&resp.status_code()) => { - let headers = resp.headers(); - let etag = headers - .get("etag") - .or_else(|| headers.get("ETag")) - .cloned() - .ok_or_else(|| { - // Fail closed: a CAS that we can't chain (because the - // backend didn't return an ETag) is not a `Won` — it's - // a non-conforming backend. The conformance probe will - // catch this; in production we'd rather refuse than - // hand the caller `ETag("")` and force-fail the next CAS. - StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "CAS succeeded but response missing ETag header \ - (backend does not satisfy ETag-token consistency)" - .into(), - )) - })?; - Ok(CasOutcome::Won(ETag(etag))) - } - Err(S3Error::HttpFailWithBody(412, _)) => Ok(CasOutcome::LostRace), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), + /// Hash `bytes` and reject anything that does not match `expected_digest`. + fn verify_digest(key: &str, expected_digest: &str, bytes: Bytes) -> Result { + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let actual = hex::encode(hasher.finalize()); + if actual != expected_digest { + return Err(StoreError::DigestMismatch { + key: key.into(), + expected: expected_digest.into(), + actual, + }); } + Ok(bytes) } /// Conformance probe — deployment gate per `docs/git-on-object-storage.md` @@ -563,16 +409,17 @@ impl GitStore { /// verify bytes. Tests A1 (content-addressed write) + A2 /// (read-after-write). /// 2. **`if_match_race`** — `race_width` parallel `put_pointer` calls - /// predicated on the same ETag. Exactly one must `Won`; the rest must - /// `LostRace`. Tests A3. + /// predicated on the same revision. Exactly one must commit; the rest + /// must conflict. Tests A3. /// 3. **`if_none_match_race`** — `race_width` parallel create-only writes /// against the same digest-shaped key (the same `put_immutable` path /// `put_pack`/`put_manifest` use). Tests A1 + A3 on the create-only - /// primitive. Counts raw HTTP outcomes (exactly one 2xx, rest 412) and - /// asserts final stored bytes equal the racers' bytes. - /// 4. **`etag_consistency`** — round-trip an ETag from `get_pointer` into - /// `put_pointer(IfMatch(...))` and assert `Won`. Tests that the token - /// is opaque and stable between read and CAS. + /// primitive. Counts raw outcomes (exactly one create, rest already + /// present) and asserts final stored bytes equal the racers' bytes. + /// 4. **`revision_consistency`** — round-trip a revision from + /// `get_pointer` into `put_pointer(Matches(...))` and assert it + /// commits. Tests that the token is opaque and stable between read and + /// CAS. pub async fn run_conformance_probe(&self, cfg: ProbeConfig) -> Result { use std::sync::Arc; if cfg.race_width < 2 || cfg.race_rounds == 0 { @@ -618,15 +465,15 @@ impl GitStore { } // -- Phase 2: if_match_race ----------------------------------------------- - // Seed the pointer with a known value, then race N IfMatch updates. + // Seed the pointer with a known value, then race N CAS updates. let seed = b"probe-pointer-seed".to_vec(); - let _ = self.bucket.delete_object(&pointer_key).await; // ignore 404 + let _ = self.store.delete(&pointer_key).await; // ignore absence let seed_outcome = self - .put_pointer(&pointer_key, &seed, Precond::IfNoneMatchStar) + .put_pointer(&pointer_key, &seed, WriteCondition::Absent) .await?; - let mut etag = match seed_outcome { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => { + let mut revision = match seed_outcome { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => { return Err(ProbeFailure { phase: "if_match_race", round: 0, @@ -642,39 +489,36 @@ impl GitStore { for i in 0..cfg.race_width { let me = Arc::clone(&arc_self); let pkey = pointer_key.clone(); - let et = etag.clone(); + let condition = WriteCondition::Matches(revision.clone()); let body = format!("round={round},racer={i},nonce={nonce}").into_bytes(); - tasks.push(async move { me.put_pointer(&pkey, &body, Precond::IfMatch(et)).await }); + tasks.push(async move { me.put_pointer(&pkey, &body, condition).await }); } let outcomes = futures_util::future::join_all(tasks).await; - // Drop-and-floor classification. A `Reqwest`/`Http`/`Io` error + // Drop-and-floor classification. An ambiguous provider outcome // means the racer never got a classified response from the // backend (couldn't open a socket, send flaked, etc.); its // outcome is *unknown*, not negative. A3 is a claim about // **observers**: dropping unknowns from the observer set // sharpens the assertion ("exactly one winner among observers") // and avoids smuggling a network-stack test into the - // conformance probe. Parse/decode errors (`Utf8`, - // `ReqwestHeaderToStr`, `SerdeXml`, ...) and `HttpFailWithBody` - // stay in the catch-all — those mean the backend *did* answer - // but not in the contract shape, which is a real conformance - // signal. + // conformance probe. Every other failure — a malformed + // response, an unexpected status — means the backend *did* + // answer but not in the contract shape, which is a real + // conformance signal and fails closed. let mut classified = 0usize; let mut winners = 0usize; - let mut new_etag: Option = None; + let mut new_revision: Option = None; for (i, outcome) in outcomes.into_iter().enumerate() { match outcome { - Ok(CasOutcome::Won(e)) => { + Ok(ConditionalWrite::Committed(committed)) => { classified += 1; winners += 1; - new_etag = Some(e); + new_revision = Some(committed); } - Ok(CasOutcome::LostRace) => { + Ok(ConditionalWrite::Conflict) => { classified += 1; } - Err(StoreError::Backend( - S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_), - )) => { + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { transport_drops += 1; tracing::warn!( phase = "if_match_race", @@ -720,17 +564,17 @@ impl GitStore { } .into()); } - etag = new_etag.expect("winner exists"); + revision = new_revision.expect("winner exists"); } // -- Phase 3: if_none_match_race ------------------------------------------ // N parallel create-only writes targeting the same digest-shaped key. - // Bypass `put_immutable`'s 412-swallow to count raw outcomes. + // Bypass `put_immutable`'s collision-swallow to count raw outcomes. for round in 0..cfg.race_rounds { let body = format!("probe-inm-race-{nonce}-{round}").into_bytes(); let key = Self::content_key("probe/inm-race", &body); // Clean slate. - let _ = self.bucket.delete_object(&key).await; + let _ = self.store.delete(&key).await; let arc_self: Arc<&Self> = Arc::new(self); let mut tasks = Vec::with_capacity(cfg.race_width); for _ in 0..cfg.race_width { @@ -741,35 +585,23 @@ impl GitStore { } let results = futures_util::future::join_all(tasks).await; // Drop-and-floor: same classification rule as Phase 2. Drop - // `Reqwest`/`Http`/`Io` (pre-classification — socket/send - // failure); count 2xx + 412 as the classified observers. Any - // other status or any non-transport `StoreError` is a real - // conformance signal and fails closed. + // ambiguous pre-classification failures; count created + + // already-present as the classified observers. Any other + // `StoreError` is a real conformance signal and fails closed. let mut classified = 0usize; - let mut twos = 0usize; - let mut twelves = 0usize; + let mut created = 0usize; + let mut collisions = 0usize; for (i, r) in results.into_iter().enumerate() { match r { - Ok(200..=299) => { + Ok(ImmutableWrite::Created) => { classified += 1; - twos += 1; + created += 1; } - Ok(412) => { + Ok(ImmutableWrite::AlreadyPresent) => { classified += 1; - twelves += 1; + collisions += 1; } - Ok(code) => { - return Err(ProbeFailure { - phase: "if_none_match_race", - round, - key, - reason: format!("racer {i}: unexpected status {code}"), - } - .into()) - } - Err(StoreError::Backend( - S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_), - )) => { + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { transport_drops += 1; tracing::warn!( phase = "if_none_match_race", @@ -802,17 +634,17 @@ impl GitStore { } .into()); } - // Create-only contract: exactly 1×2xx + (classified − 1)×412 - // *among observers*. The previous fixed `race_width − 1` would + // Create-only contract: exactly 1 create + (classified − 1) + // collisions *among observers*. A fixed `race_width − 1` would // false-positive on any transport drop; this expression honors // the drop-and-floor invariant. - if twos != 1 || twelves != classified - 1 { + if created != 1 || collisions != classified - 1 { return Err(ProbeFailure { phase: "if_none_match_race", round, key, reason: format!( - "expected 1×2xx + {}×412 among {classified} classified observers, got {twos}×2xx + {twelves}×412", + "expected 1 create + {} collisions among {classified} classified observers, got {created} creates + {collisions} collisions", classified - 1 ), } @@ -840,31 +672,31 @@ impl GitStore { } } - // -- Phase 4: etag_consistency -------------------------------------------- - // GET pointer, take its ETag, CAS-update with that ETag, expect Won. - // Proves the token round-trips opaquely between read and write. + // -- Phase 4: revision_consistency ---------------------------------------- + // GET pointer, take its revision, CAS-update with that revision, expect + // a commit. Proves the token round-trips opaquely between read and write. for round in 0..cfg.race_rounds { - let (et, _bytes) = + let (observed, _bytes) = self.get_pointer(&pointer_key) .await? .ok_or_else(|| ProbeFailure { - phase: "etag_consistency", + phase: "revision_consistency", round, key: pointer_key.clone(), reason: "pointer vanished mid-probe".into(), })?; - let body = format!("probe-etag-{round}-{nonce}").into_bytes(); + let body = format!("probe-revision-{round}-{nonce}").into_bytes(); match self - .put_pointer(&pointer_key, &body, Precond::IfMatch(et)) + .put_pointer(&pointer_key, &body, WriteCondition::Matches(observed)) .await? { - CasOutcome::Won(_) => {} - CasOutcome::LostRace => { + ConditionalWrite::Committed(_) => {} + ConditionalWrite::Conflict => { return Err(ProbeFailure { - phase: "etag_consistency", + phase: "revision_consistency", round, key: pointer_key, - reason: "GET-ETag → IfMatch chain lost race in a quiescent probe".into(), + reason: "GET-revision → CAS chain lost race in a quiescent probe".into(), } .into()) } @@ -873,7 +705,7 @@ impl GitStore { // Cleanup pointer (immutable probe writes accumulate by design; the // bucket's retention policy handles them, not the probe). - let _ = self.bucket.delete_object(&pointer_key).await; + let _ = self.store.delete(&pointer_key).await; Ok(ProbeReport { race_width: cfg.race_width, @@ -890,25 +722,17 @@ impl GitStore { } /// Raw create-only PUT exposed for the probe's race-counting phase, where - /// we need to *see* 412 outcomes rather than swallow them as idempotent. - /// Returns the HTTP status code on success-or-412; bubbles other errors. - async fn put_immutable_raw(&self, key: &str, bytes: &[u8]) -> Result { - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers( - key, - bytes, - "application/octet-stream", - Some(headers), - ) - .await - { - Ok(resp) => Ok(resp.status_code()), - Err(S3Error::HttpFailWithBody(412, _)) => Ok(412), - Err(e) => Err(StoreError::Backend(e)), - } + /// we need to *see* collision outcomes rather than swallow them as + /// idempotent. Bubbles anything that is not a create-or-collision. + async fn put_immutable_raw( + &self, + key: &str, + bytes: &[u8], + ) -> Result { + Ok(self + .store + .put_immutable(key, bytes, "application/octet-stream") + .await?) } } @@ -932,76 +756,73 @@ mod tests { } } + /// The provider errors git treats as domain outcomes must surface as their + /// own variants — call sites match on `NotFound` / `ObjectTooLarge` / + /// `DigestMismatch`, not on a wrapped backend error. #[test] - fn classify_cas_412_is_lost_race() { - let r = Err(S3Error::HttpFailWithBody(412, "PreconditionFailed".into())); - assert_eq!(GitStore::classify_cas(r).unwrap(), CasOutcome::LostRace); - } - - #[test] - fn classify_cas_other_4xx_bubbles() { - let r = Err(S3Error::HttpFailWithBody(403, "AccessDenied".into())); + fn provider_errors_lift_into_domain_variants() { + assert!(matches!( + StoreError::from(ObjectStoreError::NotFound { + key: "packs/x".into() + }), + StoreError::NotFound(ref key) if key == "packs/x" + )); assert!(matches!( - GitStore::classify_cas(r), - Err(StoreError::Backend(S3Error::HttpFailWithBody(403, _))) + StoreError::from(ObjectStoreError::ObjectTooLarge { + key: "packs/x".into(), + size: 9, + max: 4, + }), + StoreError::ObjectTooLarge { + size: 9, + max: 4, + .. + } + )); + assert!(matches!( + StoreError::from(ObjectStoreError::DigestMismatch { + key: "packs/x".into(), + expected: "a".into(), + actual: "b".into(), + }), + StoreError::DigestMismatch { .. } )); } + /// Everything else stays a backend error, and only pre-classification + /// failures are ambiguous — the probe's drop-and-floor rule reads exactly + /// this predicate to decide which racers leave the observer set. #[test] - fn static_keys_build_store_with_configured_region() { - let store = GitStore::new( - "http://localhost:9000", - "buzz_dev", - "buzz_dev_secret", - "buzz-git", - "us-west-2", - buzz_media::config::S3AddressingStyle::Path, - ) - .expect("static creds should build a git store"); - match store.bucket.region { - Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"), - ref other => panic!("expected Custom region, got {other:?}"), + fn other_provider_errors_stay_backend_and_keep_their_classification() { + let permanent = StoreError::from(ObjectStoreError::Provider { + operation: "put_conditional", + message: "AccessDenied".into(), + }); + match permanent { + StoreError::Backend(ref e) => assert!(!e.is_ambiguous()), + other => panic!("expected Backend, got {other:?}"), } - } - #[test] - fn constructor_applies_both_addressing_styles() { - for (style, expected_url, path_style) in [ - ( - buzz_media::config::S3AddressingStyle::Path, - "https://storage.example/buzz-git", - true, - ), - ( - buzz_media::config::S3AddressingStyle::Virtual, - "https://buzz-git.storage.example", - false, - ), - ] { - let store = GitStore::new( - "https://storage.example", - "buzz_dev", - "buzz_dev_secret", - "buzz-git", - "us-east-1", - style, - ) - .expect("construct git store"); - assert_eq!(store.bucket.url(), expected_url); - assert_eq!(store.bucket.is_path_style(), path_style); + let unknown = StoreError::from(ObjectStoreError::TransportAmbiguous { + operation: "put_conditional", + message: "connection reset".into(), + }); + match unknown { + StoreError::Backend(ref e) => assert!(e.is_ambiguous()), + other => panic!("expected Backend, got {other:?}"), } } #[test] fn partial_static_keys_are_rejected() { for (access, secret) in [("buzz_dev", ""), ("", "buzz_dev_secret")] { - let err = match GitStore::new( + let err = match GitStore::from_s3_config( "http://localhost:9000", access, secret, "buzz-git", "us-east-1", - buzz_media::config::S3AddressingStyle::Path, + S3AddressingStyle::Path, ) { Ok(_) => { panic!("partial static creds must not silently use the credential chain") @@ -1018,7 +839,8 @@ mod tests { #[cfg(test)] mod probe { - //! Empirical probe of rust-s3 + `fail-on-err` + MinIO surfacing of 412. + //! Empirical probe of the S3 provider's precondition surfacing against + //! live MinIO. //! //! Run manually: //! BUZZ_GIT_S3_PROBE=1 cargo test -p buzz-relay --lib \ @@ -1047,7 +869,7 @@ mod probe { .unwrap_or_else(|_| "path".into()) .parse() .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"); - GitStore::new( + GitStore::from_s3_config( &endpoint, &access_key, &secret_key, @@ -1072,24 +894,17 @@ mod probe { } let st = store(); let key = format!("probe/cas-{}.txt", uuid::Uuid::new_v4()); - let mut hdrs = axum::http::HeaderMap::new(); - hdrs.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - let r1 = st - .bucket - .put_object_with_content_type_and_headers( - &key, - b"first", - "text/plain", - Some(hdrs.clone()), - ) - .await; - assert!((200..300).contains(&r1.expect("first ok").status_code())); - let r2 = st - .bucket - .put_object_with_content_type_and_headers(&key, b"second", "text/plain", Some(hdrs)) - .await; - assert!(matches!(r2, Err(S3Error::HttpFailWithBody(412, _)))); - let _ = st.bucket.delete_object(&key).await; + let first = st + .put_immutable_raw(&key, b"first") + .await + .expect("first create-only write"); + assert_eq!(first, ImmutableWrite::Created); + let second = st + .put_immutable_raw(&key, b"second") + .await + .expect("second create-only write must classify, not error"); + assert_eq!(second, ImmutableWrite::AlreadyPresent); + let _ = st.store.delete(&key).await; } #[tokio::test] @@ -1118,57 +933,64 @@ mod probe { let err = st.get_verified(&key, &bogus).await.unwrap_err(); assert!(matches!(err, StoreError::DigestMismatch { .. })); - // 4. pointer lifecycle: get_pointer (None) → put_pointer(IfNoneMatchStar) - // → get_pointer (Some) → put_pointer(IfMatch correct) → put_pointer(IfMatch wrong, LostRace). + // 4. pointer lifecycle: get_pointer (None) → put_pointer(Absent) + // → get_pointer (Some) → put_pointer(Matches correct) + // → put_pointer(Matches stale, Conflict). let pkey = format!("pointers/{}.json", uuid::Uuid::new_v4()); assert!(st.get_pointer(&pkey).await.expect("get none").is_none()); let p1 = br#"{"manifest":"d1"}"#; let r = st - .put_pointer(&pkey, p1, Precond::IfNoneMatchStar) + .put_pointer(&pkey, p1, WriteCondition::Absent) .await .expect("first cas"); - let e1 = match r { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => panic!("first INM* should win"), + let r1 = match r { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("first create-only write should commit"), }; - eprintln!("Won.etag from PUT response: {:?}", e1.0); + eprintln!("committed revision from PUT response: {r1:?}"); - // Second INM* must lose. + // Second create-only write must conflict. let r = st - .put_pointer(&pkey, b"{}", Precond::IfNoneMatchStar) + .put_pointer(&pkey, b"{}", WriteCondition::Absent) .await .expect("second cas"); - assert_eq!(r, CasOutcome::LostRace, "second INM* must lose"); + assert_eq!( + r, + ConditionalWrite::Conflict, + "second create-only write must conflict" + ); - // Chain CAS directly on the PUT-returned ETag (no HEAD round-trip). + // Chain CAS directly on the PUT-returned revision (no HEAD round-trip). // MinIO returns the ETag in the PUT response; this proves callers can - // chain `Won → IfMatch → Won` without re-reading the pointer. - assert!(!e1.0.is_empty(), "MinIO should populate PUT response ETag"); + // chain commit → CAS → commit without re-reading the pointer. let p2 = br#"{"manifest":"d2"}"#; let r = st - .put_pointer(&pkey, p2, Precond::IfMatch(e1.clone())) + .put_pointer(&pkey, p2, WriteCondition::Matches(r1.clone())) .await .expect("cas2"); - let e2 = match r { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => panic!("IfMatch with fresh etag should win"), + let r2 = match r { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("CAS with fresh revision should commit"), }; - // Stale IfMatch (reuse the *first* etag, which has been superseded) → LostRace. + // Stale CAS (reuse the *first* revision, which has been superseded). let r = st - .put_pointer(&pkey, b"{}", Precond::IfMatch(e1)) + .put_pointer(&pkey, b"{}", WriteCondition::Matches(r1)) .await .expect("cas3"); - assert_eq!(r, CasOutcome::LostRace, "stale IfMatch must lose"); + assert_eq!(r, ConditionalWrite::Conflict, "stale CAS must conflict"); - // get_pointer's etag matches the most recent PUT-returned etag. - let (etag_now, _body) = st.get_pointer(&pkey).await.expect("get").expect("exists"); - assert_eq!(etag_now, e2, "get_pointer etag matches PUT-response etag"); + // get_pointer's revision matches the most recent PUT-returned revision. + let (revision_now, _body) = st.get_pointer(&pkey).await.expect("get").expect("exists"); + assert_eq!( + revision_now, r2, + "get_pointer revision matches PUT-response revision" + ); // Cleanup. - let _ = st.bucket.delete_object(&pkey).await; - let _ = st.bucket.delete_object(&key).await; + let _ = st.store.delete(&pkey).await; + let _ = st.store.delete(&key).await; } /// End-to-end conformance probe against MinIO. This is the same code path @@ -1191,24 +1013,17 @@ mod probe { assert_eq!(report.race_rounds, 2); } - /// Quick probe: confirm rust-s3's `get_object` exposes ETag on the response. + /// Quick probe: confirm a plain read exposes the object's revision. #[tokio::test] - async fn probe_get_exposes_etag() { + async fn probe_get_exposes_revision() { if !probe_enabled() { return; } let st = store(); - let key = format!("probe/etag-{}.txt", uuid::Uuid::new_v4()); - st.bucket - .put_object_with_content_type(&key, b"hi", "text/plain") - .await - .expect("put"); - let resp = st.bucket.get_object(&key).await.expect("get"); - let headers = resp.headers(); - eprintln!("GET headers: {headers:?}"); - let etag = headers.get("etag").or_else(|| headers.get("ETag")).cloned(); - assert!(etag.is_some(), "GET response must carry ETag header"); - eprintln!("ETag from GET: {etag:?}"); - let _ = st.bucket.delete_object(&key).await; + let key = format!("probe/revision-{}.txt", uuid::Uuid::new_v4()); + st.store.put(&key, b"hi", "text/plain").await.expect("put"); + let observed = st.get_pointer(&key).await.expect("get").expect("exists"); + eprintln!("revision from GET: {:?}", observed.0); + let _ = st.store.delete(&key).await; } } diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index d3416d673c5..1fc5034d485 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -2863,7 +2863,7 @@ const DEFAULT_HEAD: &str = "refs/heads/main"; /// Seed the manifest-pointer for a newly-announced repo with an empty manifest. /// -/// Idempotent: a `CasOutcome::LostRace` is treated as success **only if** the +/// Idempotent: a `ConditionalWrite::Conflict` is treated as success **only if** the /// existing pointer names the same empty manifest digest. Any other pre-existing /// pointer body (e.g. a non-empty manifest from a previous announce/push pair /// for the same `(owner, repo)`) surfaces as an error rather than silently @@ -2875,7 +2875,7 @@ async fn seed_manifest_pointer( repo_id: &str, ) -> anyhow::Result<()> { use crate::api::git::manifest::{pointer_key, Manifest, MANIFEST_VERSION}; - use crate::api::git::store::{CasOutcome, Precond}; + use buzz_object_store::{ConditionalWrite, WriteCondition}; use std::collections::BTreeMap; // The empty manifest. All empty manifests across all repos share canonical @@ -2906,22 +2906,22 @@ async fn seed_manifest_pointer( let pkey = pointer_key(tenant.community(), owner_hex, repo_id); let outcome = state .git_store - .put_pointer(&pkey, digest.as_bytes(), Precond::IfNoneMatchStar) + .put_pointer(&pkey, digest.as_bytes(), WriteCondition::Absent) .await .map_err(|e| anyhow::anyhow!("put_pointer: {e}"))?; match outcome { - CasOutcome::Won(_) => Ok(()), - CasOutcome::LostRace => { + ConditionalWrite::Committed(_) => Ok(()), + ConditionalWrite::Conflict => { // Pointer already exists. Idempotency check: only treat as success // if it names the same empty manifest digest. Any other value is // either a stale pointer from a prior repo lifecycle for the same // (owner, repo) or a real misconfiguration — surface, don't swallow. - let (_etag, body) = state + let (_revision, body) = state .git_store .get_pointer(&pkey) .await - .map_err(|e| anyhow::anyhow!("re-read pointer after LostRace: {e}"))? - .ok_or_else(|| anyhow::anyhow!("pointer vanished after LostRace race"))?; + .map_err(|e| anyhow::anyhow!("re-read pointer after CAS conflict: {e}"))? + .ok_or_else(|| anyhow::anyhow!("pointer vanished after CAS conflict"))?; let existing = std::str::from_utf8(&body) .map_err(|e| anyhow::anyhow!("pointer body not utf-8: {e}"))? .trim(); diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index efdb2846148..0d8d22c346b 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -836,15 +836,10 @@ impl AppState { let git_max_concurrent_ops = config.git_max_concurrent_ops; let media_max_concurrent_uploads = config.media_max_concurrent_uploads; - let git_store = crate::api::git::store::GitStore::new( - &config.media.s3_endpoint, - &config.media.s3_access_key, - &config.media.s3_secret_key, - &config.media.s3_bucket, - &config.media.s3_region, - config.media.s3_addressing_style, - ) - .expect("media storage was already constructed with this S3 config"); + // One provider per process: git and media share the client that was + // already constructed for media storage rather than opening a second + // one against the same bucket. + let git_store = crate::api::git::store::GitStore::new(media_storage.object_store()); let git_pack_cache = Arc::new( crate::api::git::pack_cache::GitPackCache::new( &config.git_pack_cache_path, From 5f8e937bb040aaff39b8c2f9dcfbd5c7bfc7630b Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 24 Aug 2026 17:50:02 -0700 Subject: [PATCH 3/7] feat(object-store, relay): add a native Google Cloud Storage provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud Storage's S3-interoperability endpoint accepts `If-Match` on PUT and then ignores it. Under the Git conformance probe that is not a degraded result but a semantic compare-and-swap violation: two racers predicating a write on the same revision both reported a commit, so the last writer silently destroyed the other's pointer update. Nothing above the provider can recover a lost update it was told did not happen, so Cloud Storage needs a provider that uses the native API, where the precondition is enforced. Add `providers::gcs`, implementing the seam against the official `google-cloud-storage` client with Application Default Credentials. No key file, HMAC pair, or S3 interoperability path is involved. Compare-and-swap uses native object generations rather than ETags: `ifGenerationMatch=0` is the create-only precondition and `ifGenerationMatch=` the revision-matched replace. A stale precondition returns HTTP 412 and is reported as an ordinary `ConditionalWrite::Conflict`, and a pointer read takes its body and its generation from the same response. Three details are worth calling out: - **Bucket contract at construction.** The S3 provider detects versioning by writing a probe object and looking for a version id. That heuristic is meaningless on Cloud Storage, where every object carries a generation whether or not old ones are retained, so this provider reads bucket metadata instead and fails closed unless object versioning is off and soft-delete retention is zero. Either setting would leave a restorable copy behind after a delete, so a deletion could otherwise report success while the bytes stayed reachable. Checking at construction also catches configuration drift on every boot. - **Retries are owned here.** The client's own retry loop is disabled and this module runs one bounded policy: capped exponential backoff with equal jitter, honouring `Retry-After`, drawing on a throttle budget separate from the transient-error budget so pacing cannot consume the allowance for real failures. An exhausted budget surfaces as `Throttled`, reaching the caller as backpressure rather than as a lost race. A conditional write is only ever retried carrying its exact original precondition. `put_file` is the one exception, delegating to the client's resumable upload retry so a large media blob resumes instead of restarting. - **Ambiguity is resolved by rereading, not guessing.** A 412 arriving after an attempt that never got a classified answer could be another writer's commit or this writer's own. The object is reread and its body decides. `ProviderSelection`/`ObjectStoreConfig` and an async `connect()` let a deployment select its provider once, in configuration. `BUZZ_OBJECT_STORE_PROVIDER` chooses it in the relay, defaulting to `s3` so an existing deployment is unaffected. Selecting `gcs` requires `BUZZ_OBJECT_STORE_BUCKET` and deliberately does not fall back to `BUZZ_S3_BUCKET`: a Cloud Storage deployment carries no `BUZZ_S3_*` values at all, and a default bucket name would address storage nobody configured. Startup connects through the seam, so a provider's admission checks run before the relay serves traffic — a bucket whose configuration would break deletion fails the boot rather than the first delete. Signed-off-by: mozarthq --- Cargo.lock | 450 +++++- crates/buzz-object-store/Cargo.toml | 6 + crates/buzz-object-store/src/lib.rs | 124 +- crates/buzz-object-store/src/providers/gcs.rs | 1302 +++++++++++++++++ crates/buzz-object-store/src/providers/mod.rs | 1 + crates/buzz-relay/src/config.rs | 104 ++ crates/buzz-relay/src/main.rs | 14 +- 7 files changed, 1975 insertions(+), 26 deletions(-) create mode 100644 crates/buzz-object-store/src/providers/gcs.rs diff --git a/Cargo.lock b/Cargo.lock index 5f05cc9088b..7c515f2c6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -581,6 +581,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -808,6 +814,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" version = "1.12.1" @@ -1113,7 +1128,7 @@ dependencies = [ "reqwest 0.13.4", "rmcp", "rustls", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "similar", @@ -1165,7 +1180,11 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "google-cloud-gax", + "google-cloud-storage", + "google-cloud-wkt", "quick-xml 0.38.4", + "rand 0.10.1", "rust-s3", "serde", "thiserror 2.0.18", @@ -1509,6 +1528,9 @@ name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] [[package]] name = "bzip2" @@ -1968,6 +1990,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2396,6 +2427,37 @@ dependencies = [ "tokio", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "deltae" version = "0.3.2" @@ -3279,6 +3341,226 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "google-cloud-auth" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff461519b1a948200f163574be072753bcfb462a323f0eb426629d89872dd685" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "google-cloud-gax", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "reqwest 0.13.4", + "rustc_version", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "time", + "tokio", + "url", +] + +[[package]] +name = "google-cloud-gax" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5615cff28ee59cfe52fbb4c11b8b1e77f650296e2ea4f4c2b7757ac6b19e752" +dependencies = [ + "bytes", + "futures", + "google-cloud-rpc", + "google-cloud-wkt", + "http", + "pin-project", + "rand 0.10.1", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-stream", +] + +[[package]] +name = "google-cloud-gax-internal" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2766757d877a7a8ac23da9884cb0e3f10ed9b75a0ce59801ce6b19bf9d5819e" +dependencies = [ + "bytes", + "futures", + "google-cloud-auth", + "google-cloud-gax", + "google-cloud-rpc", + "google-cloud-wkt", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "lazy_static", + "opentelemetry 0.32.0", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk 0.32.1", + "percent-encoding", + "pin-project", + "prost 0.14.3", + "prost-types 0.14.3", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tower", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "google-cloud-iam-v1" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f962b40234b1531e6ef73f7558871c96e117231e962086c98804323fb8d2c82" +dependencies = [ + "async-trait", + "bytes", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-type", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", + "tracing", +] + +[[package]] +name = "google-cloud-longrunning" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0363c5389ffda2b55cd8a86eef4b19a3481a48467c91dc5d77f626a9572766" +dependencies = [ + "async-trait", + "bytes", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-rpc", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", + "tracing", +] + +[[package]] +name = "google-cloud-lro" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47af3deef75c14a2983c430898d960c765bddbcc9f9188ca0563108e9227cfe7" +dependencies = [ + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-longrunning", + "google-cloud-rpc", + "google-cloud-wkt", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "google-cloud-rpc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2162c08a89118130979ba261080e960e44cdcb2d6e2ab8ca9b1da245285d353" +dependencies = [ + "bytes", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", +] + +[[package]] +name = "google-cloud-storage" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9227f65175fa91a6e41f246797917697efdadfe09dd8ea84ad8b737a71efbd28" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "futures", + "google-cloud-auth", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-iam-v1", + "google-cloud-longrunning", + "google-cloud-lro", + "google-cloud-rpc", + "google-cloud-type", + "google-cloud-wkt", + "hex", + "http", + "http-body", + "md5", + "percent-encoding", + "prost 0.14.3", + "prost-types 0.14.3", + "serde", + "serde_json", + "serde_with", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "google-cloud-type" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63acc3a92a85f96bab021c3a3e29b53bbacc97651e1b524d4c2991960a63eb82" +dependencies = [ + "bytes", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", +] + +[[package]] +name = "google-cloud-wkt" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7" +dependencies = [ + "base64 0.22.1", + "bytes", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "time", + "url", +] + [[package]] name = "h2" version = "0.4.16" @@ -3291,7 +3573,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3313,6 +3595,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3570,9 +3858,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -3626,9 +3914,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -3937,6 +4225,17 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09e54e57b4c48b40f7aec75635392b12b3421fa26fe8b4332e63138ed278459c" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -3945,6 +4244,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -4210,6 +4511,59 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link 0.2.1", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -5010,7 +5364,7 @@ dependencies = [ "rmcp", "rpassword", "rustls", - "schemars", + "schemars 1.2.1", "semver", "serde", "serde_json", @@ -5096,7 +5450,7 @@ dependencies = [ "prost-build 0.14.3", "protoc-bin-vendored", "rmcp", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "tokio", @@ -5295,7 +5649,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap", + "indexmap 2.14.0", "ipnet", "metrics", "metrics-util", @@ -5316,7 +5670,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "metrics", "ordered-float 5.3.0", "quanta", @@ -6441,6 +6795,12 @@ dependencies = [ "tonic-prost", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" + [[package]] name = "opentelemetry_sdk" version = "0.31.0" @@ -6730,7 +7090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap", + "indexmap 2.14.0", ] [[package]] @@ -6741,7 +7101,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", ] [[package]] @@ -6866,7 +7226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64 0.22.1", - "indexmap", + "indexmap 2.14.0", "quick-xml 0.39.4", "serde", "time", @@ -7063,7 +7423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e842efad9119158434d193c6682e2ebee4b44d6ad801d7b349623b3f57cdf55" dependencies = [ "futures", - "indexmap", + "indexmap 2.14.0", "nix 0.31.3", "tokio", "tracing", @@ -7975,7 +8335,7 @@ dependencies = [ "rand 0.10.1", "reqwest 0.13.4", "rmcp-macros", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "sse-stream", @@ -8250,6 +8610,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars" version = "1.2.1" @@ -8566,13 +8938,46 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "jiff", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -9035,7 +9440,7 @@ dependencies = [ "futures-util", "hashbrown 0.16.1", "hashlink", - "indexmap", + "indexmap 2.14.0", "log", "memchr", "percent-encoding", @@ -9985,7 +10390,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -10000,7 +10405,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 1.1.1+spec-1.1.0", @@ -10033,7 +10438,7 @@ version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ - "indexmap", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", @@ -10074,6 +10479,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "rustls-native-certs", "socket2", "sync_wrapper", "tokio", @@ -10115,7 +10521,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -11817,7 +12223,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.14.0", "memchr", "thiserror 2.0.18", "zopfli", diff --git a/crates/buzz-object-store/Cargo.toml b/crates/buzz-object-store/Cargo.toml index 62687518f32..3ef914a7356 100644 --- a/crates/buzz-object-store/Cargo.toml +++ b/crates/buzz-object-store/Cargo.toml @@ -13,10 +13,16 @@ axum = { workspace = true } bytes = "1" futures-core = "0.3" futures-util = "0.3" +google-cloud-gax = "1.13" +google-cloud-storage = "=1.17.0" quick-xml = { version = "0.38", features = ["serialize"] } +rand = { workspace = true } s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } serde = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } + +[dev-dependencies] +google-cloud-wkt = "1.7" diff --git a/crates/buzz-object-store/src/lib.rs b/crates/buzz-object-store/src/lib.rs index 4df3747a4b4..d4f5cef25f6 100644 --- a/crates/buzz-object-store/src/lib.rs +++ b/crates/buzz-object-store/src/lib.rs @@ -16,7 +16,8 @@ //! - the [`ObjectStoreError`] taxonomy, which separates a *classified* //! provider answer from an *unknown* transport outcome; //! - the S3 provider ([`providers::s3`]), which is the only place an ETag -//! exists. +//! exists, and the Google Cloud Storage provider ([`providers::gcs`]), which +//! is the only place an object generation exists. //! //! Domain code above this seam — `MediaStorage`, `GitStore` — is a thin facade //! that adds Buzz semantics (tenant-scoped sidecar keys, content addressing, @@ -28,14 +29,108 @@ pub mod revision; use std::path::Path; use std::pin::Pin; +use std::str::FromStr; +use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; pub use error::ObjectStoreError; +pub use providers::gcs::{GcsObjectStore, GcsRetryConfig, GcsStoreConfig}; pub use providers::s3::{S3AddressingStyle, S3ObjectStore, S3StoreConfig}; pub use revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; +/// Which provider a deployment selects, before its settings are resolved. +/// +/// Split from [`ObjectStoreConfig`] because the two callers that build a store +/// — the relay and the deletion tool — have different defaults for the S3 +/// settings but must agree on how the provider itself is chosen. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProviderSelection { + /// S3 or an S3-compatible backend, configured by the `BUZZ_S3_*` settings. + S3, + /// Google Cloud Storage, authenticated with Application Default + /// Credentials and addressed by bucket name alone. + Gcs { + /// Bucket name from `BUZZ_OBJECT_STORE_BUCKET`. + bucket: String, + }, +} + +impl ProviderSelection { + /// Read the provider selection from the environment. + /// + /// `BUZZ_OBJECT_STORE_PROVIDER` defaults to `s3`, so a deployment that has + /// never heard of this variable keeps its existing behavior. Selecting + /// `gcs` requires `BUZZ_OBJECT_STORE_BUCKET`: a Cloud Storage deployment + /// sets no `BUZZ_S3_*` values at all, so there is no bucket to fall back + /// to and guessing one would address the wrong data. + pub fn from_env() -> Result { + let provider = match std::env::var("BUZZ_OBJECT_STORE_PROVIDER") { + Ok(value) => value, + Err(std::env::VarError::NotPresent) => return Ok(Self::S3), + Err(std::env::VarError::NotUnicode(_)) => { + return Err( + "BUZZ_OBJECT_STORE_PROVIDER must be valid Unicode and one of 's3' or 'gcs'" + .to_string(), + ); + } + }; + match provider.parse::()? { + ProviderKind::S3 => Ok(Self::S3), + ProviderKind::Gcs => { + let bucket = std::env::var("BUZZ_OBJECT_STORE_BUCKET") + .ok() + .filter(|bucket| !bucket.trim().is_empty()) + .ok_or_else(|| { + "BUZZ_OBJECT_STORE_BUCKET must be set when \ + BUZZ_OBJECT_STORE_PROVIDER=gcs" + .to_string() + })?; + Ok(Self::Gcs { + bucket: bucket.trim().to_string(), + }) + } + } + } +} + +impl FromStr for ProviderKind { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "s3" => Ok(Self::S3), + "gcs" => Ok(Self::Gcs), + _ => Err(format!( + "BUZZ_OBJECT_STORE_PROVIDER must be 's3' or 'gcs', got {value:?}" + )), + } + } +} + +/// A fully resolved provider configuration, ready to connect. +#[derive(Debug, Clone)] +pub enum ObjectStoreConfig { + /// S3 or an S3-compatible backend. + S3(S3StoreConfig), + /// Google Cloud Storage. + Gcs(GcsStoreConfig), +} + +/// Build the single object-store client this process shares between media and +/// Git storage. +/// +/// Connecting is async because a provider may have admission checks to run: the +/// Cloud Storage provider reads bucket metadata here and refuses to return a +/// client for a bucket whose configuration would break deletion. +pub async fn connect(config: &ObjectStoreConfig) -> Result, ObjectStoreError> { + match config { + ObjectStoreConfig::S3(s3) => Ok(Arc::new(S3ObjectStore::new(s3)?)), + ObjectStoreConfig::Gcs(gcs) => Ok(Arc::new(GcsObjectStore::connect(gcs).await?)), + } +} + /// A stream of object byte chunks, usable with `axum::body::Body::from_stream()`. pub type ByteStream = Pin> + Send>>; @@ -312,3 +407,30 @@ pub trait ObjectStore: Send + Sync { Ok(bytes) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_names_parse() { + assert_eq!("s3".parse::(), Ok(ProviderKind::S3)); + assert_eq!("gcs".parse::(), Ok(ProviderKind::Gcs)); + } + + /// Unknown or near-miss spellings fail rather than silently selecting the + /// default provider: a typo in `BUZZ_OBJECT_STORE_PROVIDER` must not + /// quietly point a Cloud Storage deployment at S3. + #[test] + fn unknown_provider_names_are_rejected() { + for invalid in ["", "S3", "GCS", "google", "gs", "minio"] { + let error = invalid + .parse::() + .expect_err("must reject unknown provider"); + assert!( + error.contains("must be 's3' or 'gcs'"), + "unexpected error for {invalid:?}: {error}" + ); + } + } +} diff --git a/crates/buzz-object-store/src/providers/gcs.rs b/crates/buzz-object-store/src/providers/gcs.rs new file mode 100644 index 00000000000..5dd05b7e16c --- /dev/null +++ b/crates/buzz-object-store/src/providers/gcs.rs @@ -0,0 +1,1302 @@ +//! Google Cloud Storage provider, backed by Google's official client. +//! +//! This is the only module in Buzz that knows what an object *generation* is. +//! Everything above the seam holds a [`Revision`] and cannot tell a generation +//! from an S3 ETag. +//! +//! ## Why generations, not ETags +//! +//! Cloud Storage stamps every object write with a monotonically increasing +//! `generation`, and accepts it back as the `ifGenerationMatch` precondition. +//! That is a first-class compare-and-swap token: `ifGenerationMatch=0` means +//! "commit only if the object does not exist", and `ifGenerationMatch=` +//! means "commit only if the object is still at ``". A stale precondition +//! is refused with HTTP 412, which this module reports as +//! [`ConditionalWrite::Conflict`] — an ordinary lost race, never a backend +//! error. +//! +//! ## Bucket contract, checked at construction +//! +//! The S3 provider detects bucket versioning empirically, by writing a probe +//! object and looking for a version id on the response. That heuristic is +//! meaningless here: *every* Cloud Storage object carries a generation whether +//! or not the bucket retains old ones. [`GcsObjectStore::connect`] reads the +//! bucket's metadata instead and refuses to build a client unless object +//! versioning is off **and** soft-delete retention is zero. Both settings would +//! otherwise leave a restorable copy behind after a delete, so a deletion +//! request could report success while the bytes remain reachable. Checking at +//! construction also catches out-of-band configuration drift on every boot. +//! +//! ## Retries +//! +//! The client's own retry loop is disabled ([`no_client_retries`]) and this +//! module owns one bounded policy instead, so that three rules hold visibly: +//! +//! - a conditional write is only ever retried carrying its exact original +//! precondition — it is never downgraded to an unconditional write; +//! - HTTP 429 is throttling, never evidence of a lost race, so it paces the +//! caller and, if the budget runs out, surfaces as +//! [`ObjectStoreError::Throttled`] for the caller to absorb as backpressure; +//! - when an attempt fails without a classified answer, a subsequent 412 is +//! *not* assumed to be someone else's commit. The object is reread and the +//! committed body decides — see [`GcsObjectStore::put_conditional`]. +//! +//! The single exception is [`ObjectStore::put_file`], which streams a +//! multi-hundred-megabyte media blob: it delegates to the client's resumable +//! upload retry so a transient failure resumes mid-object instead of +//! restarting the transfer. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use futures_util::StreamExt; +use google_cloud_gax::error::Error as GcsError; +use google_cloud_gax::retry_policy::RetryPolicyExt; +use google_cloud_storage::client::{Storage, StorageControl}; +use google_cloud_storage::model_ext::ReadRange; +use google_cloud_storage::retry_policy::RetryableErrors; + +use crate::error::ObjectStoreError; +use crate::revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; +use crate::{ + BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore, + ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, +}; + +/// How many deletes a bulk delete keeps in flight. +/// +/// Cloud Storage has no batch-delete RPC, so a bulk delete is N individual +/// deletes. The width is bounded to keep one caller from consuming the whole +/// per-bucket request budget and throttling every other operation. +const BULK_DELETE_CONCURRENCY: usize = 12; + +/// Upper bound on a provider-advertised `Retry-After`. +/// +/// Honouring the header is required, but an unbounded sleep would let one +/// response stall a request for minutes. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(30); + +/// Longest provider detail kept on an error. +/// +/// Cloud Storage error bodies are JSON diagnostics, but they can echo request +/// detail; the message is bounded so an error can never grow into a log of the +/// request. +const MAX_ERROR_DETAIL: usize = 512; + +/// Bounded retry policy for one [`GcsObjectStore`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GcsRetryConfig { + /// Total attempts, including the first. `1` disables retries. + pub max_attempts: u32, + /// Backoff ceiling used for the first retry, doubled thereafter. + pub initial_backoff: Duration, + /// Ceiling on any single computed backoff. + pub max_backoff: Duration, +} + +impl Default for GcsRetryConfig { + fn default() -> Self { + Self { + max_attempts: 6, + initial_backoff: Duration::from_millis(200), + max_backoff: Duration::from_secs(8), + } + } +} + +/// Connection inputs for [`GcsObjectStore::connect`]. +#[derive(Debug, Clone)] +pub struct GcsStoreConfig { + /// Bucket name, without any `gs://` or resource-path decoration. + pub bucket: String, + /// Bounded retry policy for this client. + pub retry: GcsRetryConfig, +} + +impl GcsStoreConfig { + /// Configure a bucket with the default retry policy. + pub fn new(bucket: impl Into) -> Self { + Self { + bucket: bucket.into(), + retry: GcsRetryConfig::default(), + } + } +} + +/// The two bucket settings that decide whether a delete proves absence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BucketContract { + versioning_enabled: bool, + soft_delete_retention: Duration, +} + +impl BucketContract { + fn of(bucket: &google_cloud_storage::model::Bucket) -> Self { + let versioning_enabled = bucket.versioning.as_ref().is_some_and(|v| v.enabled); + let soft_delete_retention = bucket + .soft_delete_policy + .as_ref() + .and_then(|policy| policy.retention_duration.as_ref()) + .map(|d| { + let seconds = u64::try_from(d.seconds()).unwrap_or(0); + let nanos = u32::try_from(d.nanos()).unwrap_or(0); + Duration::new(seconds, nanos) + }) + .unwrap_or(Duration::ZERO); + Self { + versioning_enabled, + soft_delete_retention, + } + } + + /// Whether a delete on this bucket leaves a restorable copy behind. + /// + /// Object versioning and soft delete are different mechanisms with the + /// same consequence for Buzz: the deleted bytes stay reachable, so a + /// deletion cannot claim absence. Both therefore answer the seam's + /// "does this bucket retain non-current versions" question with yes. + fn retains_noncurrent_versions(&self) -> bool { + self.versioning_enabled || !self.soft_delete_retention.is_zero() + } + + /// Fail closed unless the bucket satisfies the deletion contract. + fn admit(&self, bucket: &str) -> Result<(), ObjectStoreError> { + let mut violations = Vec::new(); + if self.versioning_enabled { + violations.push("object versioning is enabled".to_string()); + } + if !self.soft_delete_retention.is_zero() { + violations.push(format!( + "soft-delete retention is {}s (must be 0)", + self.soft_delete_retention.as_secs() + )); + } + if violations.is_empty() { + return Ok(()); + } + Err(ObjectStoreError::Config(format!( + "bucket {bucket} does not satisfy the deletion contract ({}): a delete would leave a \ + restorable copy behind, so deletion could report success while the object stays \ + reachable", + violations.join("; ") + ))) + } +} + +/// Google Cloud Storage client, authenticated with Application Default +/// Credentials. +/// +/// Credentials are never configured in code: the client resolves them through +/// ADC (attached service account on GCE/GKE/Cloud Run, workload identity, or a +/// developer's `gcloud` credentials). There is no key material, HMAC pair, or +/// key file on this path. +pub struct GcsObjectStore { + bucket: String, + /// `projects/_/buckets/`, the resource name both clients address. + resource: String, + data: Storage, + control: StorageControl, + retry: GcsRetryConfig, +} + +/// Retry policy handed to the client so it performs exactly one attempt. +/// +/// This module owns retries (see the module docs); a second, invisible loop +/// underneath would multiply attempts and hide throttling from the caller. +fn no_client_retries() -> impl google_cloud_gax::retry_policy::RetryPolicy { + RetryableErrors.with_attempt_limit(1) +} + +impl GcsObjectStore { + /// Build a client against a bucket and verify its deletion contract. + /// + /// Fails closed when object versioning is enabled or soft-delete retention + /// is non-zero. + pub async fn connect(config: &GcsStoreConfig) -> Result { + if config.bucket.is_empty() { + return Err(ObjectStoreError::Config( + "gcs bucket must be configured".to_string(), + )); + } + if config.retry.max_attempts == 0 { + return Err(ObjectStoreError::Config( + "gcs retry max_attempts must be at least 1".to_string(), + )); + } + + let data = Storage::builder() + .with_retry_policy(no_client_retries()) + .build() + .await + .map_err(|e| ObjectStoreError::Config(format!("gcs storage client: {e}")))?; + let control = StorageControl::builder() + .with_retry_policy(no_client_retries()) + .build() + .await + .map_err(|e| ObjectStoreError::Config(format!("gcs storage control client: {e}")))?; + + let store = Self { + resource: bucket_resource(&config.bucket), + bucket: config.bucket.clone(), + data, + control, + retry: config.retry, + }; + store.read_bucket_contract().await?.admit(&store.bucket)?; + Ok(store) + } + + /// The bucket this client addresses. + pub fn bucket(&self) -> &str { + &self.bucket + } + + /// Read the bucket's versioning and soft-delete settings. + async fn read_bucket_contract(&self) -> Result { + let bucket = self + .with_retries("get_bucket", &self.bucket, || async { + self.control + .get_bucket() + .set_name(self.resource.clone()) + .send() + .await + .map_err(|e| classify("get_bucket", &self.bucket, e)) + }) + .await?; + Ok(BucketContract::of(&bucket)) + } + + /// Run `attempt` under this client's bounded retry policy. + /// + /// Only throttling, transient backend answers, and unclassified transport + /// failures are retried; everything else is the caller's answer on the + /// first attempt. Conditional writes do not use this helper — they need + /// per-attempt bookkeeping and have their own loop. + async fn with_retries( + &self, + operation: &'static str, + key: &str, + mut attempt: F, + ) -> Result + where + F: FnMut() -> Fut, + Fut: Future>, + { + for attempt_index in 0..self.retry.max_attempts { + match attempt().await { + Ok(value) => return Ok(value), + Err(error) => match self.retry_delay(&error, attempt_index) { + Some(delay) if attempt_index + 1 < self.retry.max_attempts => { + tracing::debug!( + provider = "gcs", + operation, + attempt = attempt_index + 1, + delay_ms = delay.as_millis() as u64, + "retrying object store operation" + ); + tokio::time::sleep(delay).await; + } + _ => return Err(error), + }, + } + } + // Unreachable while `max_attempts >= 1`, which the constructor enforces. + Err(ObjectStoreError::TransportRetryable { + operation, + message: format!("retry budget exhausted for {key:?}"), + }) + } + + /// How long to wait before retrying, or `None` when the error is final. + fn retry_delay(&self, error: &ObjectStoreError, attempt_index: u32) -> Option { + match error { + ObjectStoreError::Throttled { retry_after, .. } => Some( + retry_after + .map(|hint| hint.min(MAX_RETRY_AFTER)) + .unwrap_or_else(|| self.backoff(attempt_index)), + ), + ObjectStoreError::TransportRetryable { .. } + | ObjectStoreError::TransportAmbiguous { .. } => Some(self.backoff(attempt_index)), + _ => None, + } + } + + /// Capped exponential backoff with full jitter. + /// + /// Full jitter (a uniform draw from `[1ms, cap]`) rather than the raw + /// exponent: a hot pointer is written by several racers at once, and + /// unjittered backoff would keep them synchronised into the same retry + /// instants. + fn backoff(&self, attempt_index: u32) -> Duration { + let cap = self + .retry + .initial_backoff + .saturating_mul(1u32 << attempt_index.min(16)) + .min(self.retry.max_backoff); + let cap_ms = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(1); + Duration::from_millis(1 + rand::random::() % cap_ms) + } + + /// One conditional-write attempt, carrying `precondition` verbatim. + async fn write_once( + &self, + operation: &'static str, + key: &str, + bytes: &[u8], + content_type: &str, + precondition: Option, + ) -> Result { + let mut write = self + .data + .write_object( + self.resource.clone(), + key.to_string(), + Bytes::copy_from_slice(bytes), + ) + .set_content_type(content_type.to_string()); + if let Some(generation) = precondition { + write = write.set_if_generation_match(generation); + } + write + .send_unbuffered() + .await + .map(|object| Revision::GcsGeneration(object.generation)) + .map_err(|e| classify(operation, key, e)) + } + + /// After an unclassified failure, let the stored object decide whether the + /// write committed. + /// + /// A 412 arriving after an attempt whose outcome was never classified is + /// genuinely ambiguous: either another writer won the race, or *our own* + /// earlier attempt committed and the retry then found its own generation in + /// place. Guessing either way is a correctness bug, so the object is reread + /// and its body answers. Bodies are compared rather than generations + /// because the generation the winning attempt would have returned was never + /// received. + async fn classify_ambiguous_commit( + &self, + key: &str, + written: &[u8], + ) -> Result { + match self.get_with_revision(key).await? { + Some((revision, body)) if body.as_ref() == written => { + Ok(ConditionalWrite::Committed(revision)) + } + _ => Ok(ConditionalWrite::Conflict), + } + } +} + +/// The resource name both Cloud Storage clients address a bucket by. +fn bucket_resource(bucket: &str) -> String { + format!("projects/_/buckets/{bucket}") +} + +/// The `ifGenerationMatch` value implementing a [`WriteCondition`]. +/// +/// `0` is Cloud Storage's create-only precondition: no live generation can be +/// zero, so it holds exactly when the object is absent. +fn generation_precondition(condition: &WriteCondition) -> Result { + match condition { + WriteCondition::Absent => Ok(0), + WriteCondition::Matches(revision) => revision.expect_gcs_generation(), + } +} + +/// Parse a `Retry-After` delta-seconds value, if the response carried one. +/// +/// Only the delta-seconds form is read. Cloud Storage does not send the +/// HTTP-date form, and misreading a date as a duration would produce an absurd +/// sleep; an unparsed header simply falls back to computed backoff. +fn retry_after_of(error: &GcsError) -> Option { + error + .http_headers()? + .get("retry-after")? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +/// Bound and tidy a provider error for inclusion in an [`ObjectStoreError`]. +fn detail(error: &GcsError) -> String { + let mut message = error.to_string(); + if message.len() > MAX_ERROR_DETAIL { + message.truncate(MAX_ERROR_DETAIL); + message.push('…'); + } + message +} + +/// Map a client failure into the provider-neutral taxonomy. +/// +/// The ordering is the whole point: a response that carried a status told us +/// something about the object, and stays a *classified* observation no matter +/// how the transport behaved afterwards. Only a failure with no status at all +/// becomes [`ObjectStoreError::TransportAmbiguous`], which is the set the Git +/// conformance probe drops from its observers. Widening it would let a real +/// conformance failure disappear. +fn classify(operation: &'static str, key: &str, error: GcsError) -> ObjectStoreError { + let message = detail(&error); + + if let Some(status) = error.http_status_code() { + return match status { + 404 => ObjectStoreError::NotFound { key: key.into() }, + 412 => ObjectStoreError::Conflict { key: key.into() }, + 429 => ObjectStoreError::Throttled { + operation, + retry_after: retry_after_of(&error), + }, + 408 | 500..=599 => ObjectStoreError::TransportRetryable { operation, message }, + _ => ObjectStoreError::Provider { operation, message }, + }; + } + + // gRPC-shaped answers carry an RPC status instead of an HTTP one. They are + // still answers, so they classify the same way. + if let Some(code) = error.status().map(|status| status.code) { + use google_cloud_gax::error::rpc::Code; + return match code { + Code::NotFound => ObjectStoreError::NotFound { key: key.into() }, + Code::Aborted | Code::FailedPrecondition | Code::AlreadyExists => { + ObjectStoreError::Conflict { key: key.into() } + } + Code::ResourceExhausted => ObjectStoreError::Throttled { + operation, + retry_after: retry_after_of(&error), + }, + Code::Unavailable | Code::Internal | Code::DeadlineExceeded => { + ObjectStoreError::TransportRetryable { operation, message } + } + _ => ObjectStoreError::Provider { operation, message }, + }; + } + + if error.is_connect() || error.is_io() || error.is_timeout() || error.is_transport() { + return ObjectStoreError::TransportAmbiguous { operation, message }; + } + + // The client gave up before any attempt was classified, so the outcome of + // the last one is unknown. + if error.is_exhausted() { + return ObjectStoreError::TransportAmbiguous { operation, message }; + } + + ObjectStoreError::Provider { operation, message } +} + +/// Whether a failed attempt might still have reached the backend. +/// +/// A refused connection never delivered the request, so a later 412 cannot be +/// this writer's own commit. Any other unclassified failure could have been a +/// response that was lost on the way back. +fn may_have_committed(error: &ObjectStoreError) -> bool { + match error { + ObjectStoreError::TransportAmbiguous { message, .. } => !is_connect_refusal(message), + ObjectStoreError::TransportRetryable { .. } | ObjectStoreError::Throttled { .. } => false, + _ => false, + } +} + +/// Whether an ambiguous failure's detail describes a connection that was never +/// established. +fn is_connect_refusal(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + message.contains("connection refused") + || message.contains("dns error") + || message.contains("failed to lookup address") +} + +#[async_trait] +impl ObjectStore for GcsObjectStore { + fn provider(&self) -> ProviderKind { + ProviderKind::Gcs + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.with_retries("put", key, || { + self.write_once("put", key, bytes, content_type, None) + }) + .await + .map(|_| ()) + } + + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError> { + let file = tokio::fs::File::open(path) + .await + .map_err(|e| ObjectStoreError::Provider { + operation: "put_file", + message: e.to_string(), + })?; + + // The one operation that keeps the client's own retry loop: a media + // blob can be hundreds of megabytes, and the client's resumable upload + // resumes mid-object where this module's loop would restart the whole + // transfer. The upload is unconditional, so a retry cannot disturb any + // precondition. + self.data + .write_object(self.resource.clone(), key.to_string(), file) + .set_content_type(content_type.to_string()) + .with_retry_policy( + RetryableErrors + .with_attempt_limit(self.retry.max_attempts) + .with_time_limit(Duration::from_secs(300)), + ) + .send_unbuffered() + .await + .map_err(|e| classify("put_file", key, e))?; + Ok(()) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + match self + .put_conditional(key, bytes, content_type, WriteCondition::Absent) + .await? + { + ConditionalWrite::Committed(_) => Ok(ImmutableWrite::Created), + ConditionalWrite::Conflict => Ok(ImmutableWrite::AlreadyPresent), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result { + let precondition = generation_precondition(&condition)?; + // Set once an attempt fails without a classified answer. From then on a + // 412 is no longer self-evidently someone else's commit. + let mut outcome_unknown = false; + + for attempt_index in 0..self.retry.max_attempts { + let error = match self + .write_once( + "put_conditional", + key, + bytes, + content_type, + Some(precondition), + ) + .await + { + Ok(revision) => return Ok(ConditionalWrite::Committed(revision)), + Err(ObjectStoreError::Conflict { .. }) if !outcome_unknown => { + return Ok(ConditionalWrite::Conflict); + } + Err(ObjectStoreError::Conflict { .. }) => { + return self.classify_ambiguous_commit(key, bytes).await; + } + Err(error) => error, + }; + + outcome_unknown |= may_have_committed(&error); + + // Retrying always replays `precondition` verbatim: the loop never + // relaxes a compare-and-swap into a blind overwrite. + match self.retry_delay(&error, attempt_index) { + Some(delay) if attempt_index + 1 < self.retry.max_attempts => { + tracing::debug!( + provider = "gcs", + operation = "put_conditional", + attempt = attempt_index + 1, + delay_ms = delay.as_millis() as u64, + outcome_unknown, + "retrying conditional write with its original precondition" + ); + tokio::time::sleep(delay).await; + } + _ => return Err(error), + } + } + + Err(ObjectStoreError::TransportRetryable { + operation: "put_conditional", + message: "retry budget exhausted".to_string(), + }) + } + + async fn get(&self, key: &str) -> Result { + self.with_retries("get", key, || async { + let mut response = self + .data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + .map_err(|e| classify("get", key, e))?; + let mut body = Vec::with_capacity(response.object().size.max(0) as usize); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get", key, e))?); + } + Ok(Bytes::from(body)) + }) + .await + } + + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result { + if end < start { + return Err(ObjectStoreError::Provider { + operation: "get_range", + message: format!("inverted byte range {start}..={end}"), + }); + } + // The seam's range is inclusive on both ends; Cloud Storage takes an + // offset and a length. + let length = end - start + 1; + + self.with_retries("get_range", key, || async { + let mut response = self + .data + .read_object(self.resource.clone(), key.to_string()) + .set_read_range(ReadRange::segment(start, length)) + .send() + .await + .map_err(|e| classify("get_range", key, e))?; + let mut body = Vec::new(); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get_range", key, e))?); + } + Ok(Bytes::from(body)) + }) + .await + } + + async fn get_stream(&self, key: &str) -> Result { + // Only opening the stream is retried. Once bytes are flowing the + // caller owns the body, and restarting mid-response would silently + // splice two reads together. + let response = self + .with_retries("get_stream", key, || async { + self.data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + .map_err(|e| classify("get_stream", key, e)) + }) + .await?; + + // `ReadObjectResponse` only exposes a `Stream` adapter behind the + // client's `unstable-stream` feature, so the stream is unfolded from + // its stable chunk-at-a-time API instead of opting into an unstable + // surface. + let key = key.to_string(); + Ok(Box::pin(futures_util::stream::unfold( + response, + move |mut response| { + let key = key.clone(); + async move { + let chunk = response.next().await?; + Some((chunk.map_err(|e| classify("get_stream", &key, e)), response)) + } + }, + ))) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + self.with_retries("get_with_revision", key, || async { + let mut response = match self + .data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + { + Ok(response) => response, + Err(error) => { + return match classify("get_with_revision", key, error) { + ObjectStoreError::NotFound { .. } => Ok(None), + other => Err(other), + }; + } + }; + + // Body and generation both come off this one response, so the + // revision a caller predicates its next write on always describes + // the bytes it just read. + let revision = Revision::GcsGeneration(response.object().generation); + let mut body = Vec::with_capacity(response.object().size.max(0) as usize); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get_with_revision", key, e))?); + } + Ok(Some((revision, Bytes::from(body)))) + }) + .await + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + self.with_retries("head", key, || async { + match self + .control + .get_object() + .set_bucket(self.resource.clone()) + .set_object(key.to_string()) + .send() + .await + { + Ok(object) => Ok(Some(ObjectMeta { + size: u64::try_from(object.size).unwrap_or(0), + revision: Some(Revision::GcsGeneration(object.generation)), + })), + Err(error) => match classify("head", key, error) { + ObjectStoreError::NotFound { .. } => Ok(None), + other => Err(other), + }, + } + }) + .await + } + + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result { + let page_size = i32::try_from(max_keys).unwrap_or(i32::MAX); + + self.with_retries("list_page", prefix, || async { + let mut request = self + .control + .list_objects() + .set_parent(self.resource.clone()) + .set_prefix(prefix.to_string()) + .set_page_size(page_size); + if let Some(token) = continuation_token.clone() { + request = request.set_page_token(token); + } + let response = request + .send() + .await + .map_err(|e| classify("list_page", prefix, e))?; + + // Cloud Storage signals "no more pages" with an empty token rather + // than an absent one. + let next = Some(response.next_page_token).filter(|token| !token.is_empty()); + Ok(ListPage { + objects: response + .objects + .into_iter() + .map(|object| (object.name, u64::try_from(object.size).unwrap_or(0))) + .collect(), + is_truncated: next.is_some(), + next_continuation_token: next, + }) + }) + .await + } + + /// Delete one object. + /// + /// The delete is not generation-qualified. It does not need to be: the + /// constructor refuses buckets with object versioning or soft delete, so a + /// key has exactly one live generation and an unqualified delete removes + /// precisely the object the caller addressed. Deleting an absent object is + /// not an error. + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.with_retries("delete", key, || async { + match self + .control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.to_string()) + .send() + .await + { + Ok(()) => Ok(()), + Err(error) => match classify("delete", key, error) { + ObjectStoreError::NotFound { .. } => Ok(()), + other => Err(other), + }, + } + }) + .await + } + + async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + + // Cloud Storage has no batch-delete RPC, so this is bounded-concurrency + // individual deletes folded into the same per-key outcome the S3 + // provider reports from its batch response. + let outcomes = futures_util::stream::iter(keys.iter().cloned()) + .map(|key| async move { + let result = self + .with_retries("delete_objects", &key, || async { + self.control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.clone()) + .send() + .await + .map_err(|e| classify("delete_objects", &key, e)) + }) + .await; + (key, result) + }) + .buffer_unordered(BULK_DELETE_CONCURRENCY) + .collect::>() + .await; + + let mut outcome = BulkDeleteOutcome::default(); + for (key, result) in outcomes { + match result { + Ok(()) => outcome.deleted += 1, + Err(ObjectStoreError::NotFound { .. }) => outcome.already_missing += 1, + Err(error) => { + outcome + .failed + .push((key, error_code(&error).to_string(), error.to_string())) + } + } + } + // `versioned_keys` stays empty by construction: a delete on a bucket + // that passed the admission check cannot produce a version artifact. + Ok(outcome) + } + + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result { + if version_id_marker.is_some() { + return Err(ObjectStoreError::Provider { + operation: "list_versions_page", + message: "GCS pagination accepts one opaque page token; a second cursor component is invalid" + .to_string(), + }); + } + let page_size = i32::try_from(max_keys).unwrap_or(i32::MAX); + + self.with_retries("list_versions_page", prefix, || async { + let mut request = self + .control + .list_objects() + .set_parent(self.resource.clone()) + .set_prefix(prefix.to_string()) + .set_versions(true) + .set_page_size(page_size); + if let Some(token) = key_marker.clone() { + request = request.set_page_token(token); + } + let response = request + .send() + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + let next = Some(response.next_page_token).filter(|token| !token.is_empty()); + Ok(ObjectVersionsPage { + entries: response + .objects + .into_iter() + .map(|object| ObjectVersionEntry { + key: object.name, + version_id: object.generation.to_string(), + kind: ObjectVersionKind::Object, + size: u64::try_from(object.size).unwrap_or(0), + }) + .collect(), + is_truncated: next.is_some(), + next_key_marker: next, + next_version_id_marker: None, + }) + }) + .await + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + if versions.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + + let mut parsed = Vec::with_capacity(versions.len()); + for version in versions { + let generation = version.version_id.parse::().map_err(|_| { + ObjectStoreError::Provider { + operation: "delete_versions", + message: format!( + "invalid GCS generation for object {:?}", + version.key + ), + } + })?; + if generation <= 0 { + return Err(ObjectStoreError::Provider { + operation: "delete_versions", + message: format!( + "non-positive GCS generation for object {:?}", + version.key + ), + }); + } + parsed.push((version.key.clone(), generation)); + } + + let outcomes = futures_util::stream::iter(parsed) + .map(|(key, generation)| async move { + let result = self + .with_retries("delete_versions", &key, || async { + self.control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.clone()) + .set_generation(generation) + .send() + .await + .map_err(|e| classify("delete_versions", &key, e)) + }) + .await; + (key, result) + }) + .buffer_unordered(BULK_DELETE_CONCURRENCY) + .collect::>() + .await; + + let mut outcome = BulkDeleteOutcome::default(); + for (key, result) in outcomes { + match result { + Ok(()) => outcome.deleted += 1, + Err(ObjectStoreError::NotFound { .. }) => outcome.already_missing += 1, + Err(error) => outcome.failed.push(( + key, + error_code(&error).to_string(), + error.to_string(), + )), + } + } + Ok(outcome) + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + self.list_page("", None, 1).await.map(|_| ()) + } + + /// Whether a delete on this bucket would leave a restorable copy behind. + /// + /// Read from bucket metadata rather than probed. Every Cloud Storage object + /// carries a generation whether or not old ones are retained, so the S3 + /// provider's "did the response have a version id" heuristic would answer + /// yes on a correctly configured bucket. Soft-delete retention counts too: + /// it retains a restorable copy for exactly the same reason versioning + /// does. + async fn versioning_detected(&self) -> Result { + Ok(self + .read_bucket_contract() + .await? + .retains_noncurrent_versions()) + } +} + +/// Stable, bounded label for a per-key bulk-delete failure. +fn error_code(error: &ObjectStoreError) -> &'static str { + match error { + ObjectStoreError::NotFound { .. } => "NotFound", + ObjectStoreError::Conflict { .. } => "PreconditionFailed", + ObjectStoreError::Throttled { .. } => "Throttled", + ObjectStoreError::TransportRetryable { .. } => "TransportRetryable", + ObjectStoreError::TransportAmbiguous { .. } => "TransportAmbiguous", + ObjectStoreError::Config(_) => "Config", + ObjectStoreError::ObjectTooLarge { .. } => "ObjectTooLarge", + ObjectStoreError::DigestMismatch { .. } => "DigestMismatch", + ObjectStoreError::RevisionMismatch { .. } => "RevisionMismatch", + ObjectStoreError::Provider { .. } => "Provider", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use google_cloud_storage::http::HeaderMap; + use google_cloud_storage::model::bucket::{SoftDeletePolicy, Versioning}; + use google_cloud_storage::model::Bucket; + + fn http_error(status: u16) -> GcsError { + GcsError::http(status, HeaderMap::new(), bytes::Bytes::new()) + } + + fn throttled_error(retry_after: &str) -> GcsError { + let mut headers = HeaderMap::new(); + headers.insert("retry-after", retry_after.parse().unwrap()); + GcsError::http(429, headers, bytes::Bytes::new()) + } + + #[test] + fn bucket_resource_uses_the_wildcard_project_form() { + assert_eq!( + bucket_resource("buzz-objects"), + "projects/_/buckets/buzz-objects" + ); + } + + /// Create-only is `ifGenerationMatch=0`; a compare-and-swap carries the + /// observed generation verbatim. + #[test] + fn write_conditions_map_to_generation_preconditions() { + assert_eq!(generation_precondition(&WriteCondition::Absent).unwrap(), 0); + assert_eq!( + generation_precondition(&WriteCondition::Matches(Revision::GcsGeneration( + 1_700_000_000_000_042 + ))) + .unwrap(), + 1_700_000_000_000_042 + ); + } + + /// An ETag can never predicate a generation precondition. Accepting one + /// would have to mean dropping the precondition, which is a blind + /// overwrite. + #[test] + fn a_foreign_revision_is_rejected_rather_than_downgraded() { + let err = + generation_precondition(&WriteCondition::Matches(Revision::S3Etag("\"abc\"".into()))) + .expect_err("an S3 ETag must never predicate a GCS generation match"); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + } + + #[test] + fn missing_object_classifies_as_not_found() { + let err = classify("get", "packs/x", http_error(404)); + assert!(matches!(err, ObjectStoreError::NotFound { ref key } if key == "packs/x")); + assert!(!err.is_ambiguous()); + } + + /// A stale generation is a normal compare-and-swap conflict, never a + /// backend failure and never a throttle. + #[test] + fn stale_generation_classifies_as_conflict() { + let err = classify("put_conditional", "pointers/x", http_error(412)); + assert!(matches!(err, ObjectStoreError::Conflict { ref key } if key == "pointers/x")); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + #[test] + fn throttling_classifies_as_retryable_backpressure() { + let err = classify("put_conditional", "pointers/x", http_error(429)); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: None, + .. + } + )); + assert!(err.is_retryable() && !err.is_ambiguous()); + } + + #[test] + fn retry_after_seconds_are_honoured_and_capped() { + let err = classify("put_conditional", "pointers/x", throttled_error("3")); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: Some(d), + .. + } if d == Duration::from_secs(3) + )); + + let store_retry = GcsRetryConfig::default(); + let hint = Duration::from_secs(3600).min(MAX_RETRY_AFTER); + assert_eq!(hint, MAX_RETRY_AFTER); + assert!(store_retry.max_backoff < MAX_RETRY_AFTER); + } + + /// An HTTP-date `Retry-After` is not misread as a duration; the caller + /// falls back to computed backoff instead of sleeping for aeons. + #[test] + fn unparseable_retry_after_falls_back_to_backoff() { + let err = classify("get", "k", throttled_error("Wed, 21 Oct 2026 07:28:00 GMT")); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: None, + .. + } + )); + } + + #[test] + fn transient_statuses_stay_classified_but_retryable() { + for status in [408, 500, 502, 503, 504] { + let err = classify("get", "k", http_error(status)); + assert!( + matches!(err, ObjectStoreError::TransportRetryable { .. }), + "status {status} should be retryable, got {err}" + ); + assert!(err.is_retryable() && !err.is_ambiguous()); + } + } + + /// A 403 is a real answer: permanent, classified, never dropped from the + /// conformance probe's observer set. + #[test] + fn permission_denied_classifies_as_permanent_provider_failure() { + let err = classify("put", "k", http_error(403)); + assert!(matches!(err, ObjectStoreError::Provider { .. })); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + /// Pre-classification failures are the only ambiguous outcomes. + #[test] + fn transport_failures_without_a_status_classify_as_ambiguous() { + for error in [ + GcsError::io("connection reset by peer"), + GcsError::connect("connection refused"), + GcsError::timeout("deadline exceeded"), + ] { + let err = classify("put_conditional", "k", error); + assert!(err.is_ambiguous(), "expected ambiguous, got {err}"); + } + } + + /// A conditional write whose attempt was refused at connect time never + /// reached the backend, so a later 412 is somebody else's commit. + #[test] + fn a_refused_connection_cannot_have_committed() { + let refused = classify( + "put_conditional", + "k", + GcsError::connect("connection refused"), + ); + assert!(!may_have_committed(&refused)); + + let reset = classify( + "put_conditional", + "k", + GcsError::io("connection reset by peer"), + ); + assert!(may_have_committed(&reset)); + } + + /// Classified answers say what happened, so they never arm the + /// reread-and-decide path. + #[test] + fn classified_failures_do_not_arm_ambiguity() { + for error in [ + classify("put_conditional", "k", http_error(429)), + classify("put_conditional", "k", http_error(503)), + classify("put_conditional", "k", http_error(403)), + ] { + assert!(!may_have_committed(&error), "{error}"); + } + } + + fn bucket_with(versioning: Option, soft_delete_seconds: Option) -> Bucket { + let mut bucket = Bucket::new(); + if let Some(enabled) = versioning { + bucket.versioning = Some(Versioning::new().set_enabled(enabled)); + } + if let Some(seconds) = soft_delete_seconds { + let retention = google_cloud_wkt::Duration::try_from(Duration::from_secs(seconds)) + .expect("representable retention"); + bucket.soft_delete_policy = + Some(SoftDeletePolicy::new().set_retention_duration(retention)); + } + bucket + } + + #[test] + fn a_conforming_bucket_is_admitted() { + for bucket in [ + bucket_with(None, None), + bucket_with(Some(false), Some(0)), + bucket_with(Some(false), None), + ] { + let contract = BucketContract::of(&bucket); + assert!(!contract.retains_noncurrent_versions()); + contract.admit("buzz-objects").expect("bucket conforms"); + } + } + + /// Versioning and soft delete are different mechanisms with the same + /// consequence: a delete stops proving absence. Both fail construction. + #[test] + fn versioning_or_soft_delete_fails_admission_closed() { + let versioned = BucketContract::of(&bucket_with(Some(true), Some(0))); + assert!(versioned.retains_noncurrent_versions()); + let err = versioned.admit("buzz-objects").unwrap_err(); + assert!( + matches!(err, ObjectStoreError::Config(ref m) if m.contains("object versioning is enabled")), + "unexpected error: {err}" + ); + + let soft_deleted = BucketContract::of(&bucket_with(Some(false), Some(604_800))); + assert!(soft_deleted.retains_noncurrent_versions()); + let err = soft_deleted.admit("buzz-objects").unwrap_err(); + assert!( + matches!(err, ObjectStoreError::Config(ref m) if m.contains("soft-delete retention is 604800s")), + "unexpected error: {err}" + ); + } + + #[test] + fn both_violations_are_reported_together() { + let contract = BucketContract::of(&bucket_with(Some(true), Some(604_800))); + let err = contract.admit("buzz-objects").unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("object versioning is enabled"), + "{message}" + ); + assert!(message.contains("soft-delete retention"), "{message}"); + } + + #[tokio::test] + async fn empty_bucket_and_zero_attempts_are_rejected_at_construction() { + // Both checks run before any credential resolution or network call, so + // they are provable without an environment. + let mut config = GcsStoreConfig::new(""); + assert!(matches!( + GcsObjectStore::connect(&config).await, + Err(ObjectStoreError::Config(ref m)) if m.contains("bucket must be configured") + )); + + config = GcsStoreConfig::new("buzz-objects"); + config.retry.max_attempts = 0; + assert!(matches!( + GcsObjectStore::connect(&config).await, + Err(ObjectStoreError::Config(ref m)) if m.contains("max_attempts") + )); + } + + #[test] + fn error_detail_is_bounded() { + let payload = bytes::Bytes::from("x".repeat(4096)); + let error = GcsError::http(500, HeaderMap::new(), payload); + assert!(detail(&error).len() <= MAX_ERROR_DETAIL + 4); + } +} diff --git a/crates/buzz-object-store/src/providers/mod.rs b/crates/buzz-object-store/src/providers/mod.rs index cd2b2a741f1..af4ca84c936 100644 --- a/crates/buzz-object-store/src/providers/mod.rs +++ b/crates/buzz-object-store/src/providers/mod.rs @@ -4,4 +4,5 @@ //! facade. Provider-specific vocabulary — ETags, generations, addressing //! styles, credential chains — stays inside these modules. +pub mod gcs; pub mod s3; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index e035752ec3a..5e86ac21ba7 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -292,6 +292,13 @@ pub struct Config { /// Media storage configuration (S3/MinIO). pub media: buzz_media::MediaConfig, + /// Object-store provider backing both media blobs and Git storage. + /// + /// The relay builds exactly one client from this and shares it between the + /// two facades, so the provider is a property of the deployment rather + /// than of each call site. Selected by `BUZZ_OBJECT_STORE_PROVIDER`, which + /// defaults to the S3 settings above. + pub object_store: buzz_object_store::ObjectStoreConfig, /// Maximum concurrent media uploads handled by one relay process. pub media_max_concurrent_uploads: usize, /// Maximum concurrent media uploads accepted from one pubkey. @@ -880,6 +887,29 @@ impl Config { .map(|s| s.trim().to_lowercase()) .filter(|s| !s.is_empty()), }; + // The provider is chosen once here; a Cloud Storage deployment carries + // no `BUZZ_S3_*` values at all, so its bucket comes from + // `BUZZ_OBJECT_STORE_BUCKET` rather than from the media settings. + let object_store = match buzz_object_store::ProviderSelection::from_env() + .map_err(ConfigError::InvalidValue)? + { + buzz_object_store::ProviderSelection::S3 => { + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + endpoint: media.s3_endpoint.clone(), + access_key: media.s3_access_key.clone(), + secret_key: media.s3_secret_key.clone(), + bucket: media.s3_bucket.clone(), + region: media.s3_region.clone(), + addressing_style: media.s3_addressing_style, + }) + } + buzz_object_store::ProviderSelection::Gcs { bucket } => { + buzz_object_store::ObjectStoreConfig::Gcs(buzz_object_store::GcsStoreConfig::new( + bucket, + )) + } + }; + let media_max_concurrent_uploads: usize = std::env::var("BUZZ_MEDIA_MAX_CONCURRENT_UPLOADS") .ok() @@ -1235,6 +1265,7 @@ impl Config { allow_nip_oa_auth, klipy, media, + object_store, media_max_concurrent_uploads, media_max_concurrent_uploads_per_pubkey, media_uploads_per_minute, @@ -1772,6 +1803,79 @@ mod tests { )); } + /// Restore an environment variable to whatever it was before a test. + fn restore_env(name: &str, previous: Option) { + match previous { + Some(value) => std::env::set_var(name, value), + None => std::env::remove_var(name), + } + } + + /// A deployment that has never heard of `BUZZ_OBJECT_STORE_PROVIDER` keeps + /// its S3 settings, including the endpoint and credentials that reach the + /// bundled MinIO. + #[test] + fn object_store_defaults_to_the_s3_settings() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous = std::env::var_os("BUZZ_OBJECT_STORE_PROVIDER"); + std::env::remove_var("BUZZ_OBJECT_STORE_PROVIDER"); + + let config = Config::from_env().expect("default config loads"); + + restore_env("BUZZ_OBJECT_STORE_PROVIDER", previous); + + match config.object_store { + buzz_object_store::ObjectStoreConfig::S3(s3) => { + assert_eq!(s3.endpoint, config.media.s3_endpoint); + assert_eq!(s3.bucket, config.media.s3_bucket); + assert_eq!(s3.region, config.media.s3_region); + assert_eq!(s3.addressing_style, config.media.s3_addressing_style); + } + other => panic!("expected the S3 provider by default, got {other:?}"), + } + } + + /// Selecting Cloud Storage takes its bucket from + /// `BUZZ_OBJECT_STORE_BUCKET`, never from `BUZZ_S3_BUCKET`: a Cloud + /// Storage deployment sets no `BUZZ_S3_*` values, so falling back to that + /// default would address a bucket nobody configured. + #[test] + fn gcs_provider_requires_its_own_bucket() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous_provider = std::env::var_os("BUZZ_OBJECT_STORE_PROVIDER"); + let previous_bucket = std::env::var_os("BUZZ_OBJECT_STORE_BUCKET"); + + std::env::set_var("BUZZ_OBJECT_STORE_PROVIDER", "gcs"); + std::env::remove_var("BUZZ_OBJECT_STORE_BUCKET"); + let without_bucket = Config::from_env(); + + std::env::set_var("BUZZ_OBJECT_STORE_BUCKET", "buzz-objects"); + let with_bucket = Config::from_env(); + + std::env::set_var("BUZZ_OBJECT_STORE_PROVIDER", "minio"); + let unknown_provider = Config::from_env(); + + restore_env("BUZZ_OBJECT_STORE_PROVIDER", previous_provider); + restore_env("BUZZ_OBJECT_STORE_BUCKET", previous_bucket); + + assert!(matches!( + without_bucket, + Err(ConfigError::InvalidValue(ref message)) + if message.contains("BUZZ_OBJECT_STORE_BUCKET must be set") + )); + match with_bucket.expect("gcs config loads").object_store { + buzz_object_store::ObjectStoreConfig::Gcs(gcs) => { + assert_eq!(gcs.bucket, "buzz-objects"); + } + other => panic!("expected the GCS provider, got {other:?}"), + } + assert!(matches!( + unknown_provider, + Err(ConfigError::InvalidValue(ref message)) + if message.contains("must be 's3' or 'gcs'") + )); + } + #[cfg(unix)] #[test] fn s3_addressing_style_env_rejects_non_unicode_values() { diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index d81602e2019..f6ec581d330 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -449,9 +449,17 @@ async fn main() -> anyhow::Result<()> { .media .validate() .map_err(|e| anyhow::anyhow!("invalid media config: {e}"))?; - let media_storage = buzz_media::MediaStorage::new(&config.media) - .map_err(|e| anyhow::anyhow!("failed to initialize media storage: {e}"))?; - info!("Media storage connected"); + // One provider client per process, shared by the media facade and (via + // `AppState`) the Git facade. Connecting can perform provider admission + // checks — Cloud Storage refuses a bucket whose configuration would break + // deletion — so a misconfigured backend fails startup here rather than + // during the first delete. + let object_store = buzz_object_store::connect(&config.object_store) + .await + .map_err(|e| anyhow::anyhow!("failed to initialize object storage: {e}"))?; + let provider = object_store.provider(); + let media_storage = buzz_media::MediaStorage::with_store(object_store); + info!(%provider, "Object storage connected"); let (app_state, audit_shutdown) = AppState::new( config.clone(), From 61064ff52387ba371f8c7c76dceb28e212c080bc Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 24 Aug 2026 18:45:14 -0700 Subject: [PATCH 4/7] feat(git, relay): add a Cloud Storage conformance profile and wire it up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conformance probe's S3 profile races 32 writers against one object name, three rounds over, as fast as it can. Cloud Storage publishes a roughly one-write-per-second ceiling per object name, so that shape would have spent the whole probe being throttled and then read the resulting throttles as conformance evidence — a gate that passes because the backend refused to answer is not a gate. Add a second profile the provider's own characteristics justify: a narrow race of 3 writers, 2 rounds, with same-key rounds spaced past the published ceiling (>1 s). It proves the same properties the S3 profile does, on a backend that must be paced to prove anything: - body and generation read from one response and checked against what was just committed; - a compare-and-swap on the observed generation, which must report a different one; - a replay of the superseded generation, which must conflict; - a narrow race on one generation, repeated; - the winning generation predicating the next successful write. Three rules keep pacing from becoming leniency. Two committed racers is always fatal. A commit reported with no generation is fatal wherever it appears — the caller would have nothing to predicate its next write on, and dropping the precondition is a blind overwrite. And a round that proves nothing, because every racer was throttled or too few were classified to witness a race, is re-run within a bounded budget rather than scored; exhausting the budget fails the probe. `Throttled` is never counted as a lost race. Conflicts with no acknowledged winner stay fatal, since that is a lost update announcing itself. Startup previously built the probe config from hardcoded S3 defaults, so a Cloud Storage deployment would have run the wrong gate. The defaults now come from the provider the process actually connected to, and the two environment overrides continue to apply on top for either profile. The gate stays enabled and fatal; only the shape of the evidence changed. The report and the admission log line carry the profile, the throttle and re-run counts, the shortest same-key interval actually observed, and any probe object cleanup could not remove — so a backend that is degrading is visible while it is still passing. Probe objects are deleted on the success and failure paths alike: a failed probe is the one that gets re-run. Tests drive a scripted store through the seam, because a live bucket cannot be asked to commit two writers on one generation — covering two winners, an all-throttled round re-run and then bounded, a mixed conflict/throttle round, a commit with no generation on both the create and the race path, a store that honours a stale generation, a round where everyone loses, and the measured spacing. Signed-off-by: mozarthq --- crates/buzz-relay/src/api/git/store.rs | 1394 +++++++++++++++++++++++- crates/buzz-relay/src/main.rs | 35 +- 2 files changed, 1395 insertions(+), 34 deletions(-) diff --git a/crates/buzz-relay/src/api/git/store.rs b/crates/buzz-relay/src/api/git/store.rs index 10c364cef82..b4fac01b9aa 100644 --- a/crates/buzz-relay/src/api/git/store.rs +++ b/crates/buzz-relay/src/api/git/store.rs @@ -31,10 +31,11 @@ #![allow(dead_code)] // wired in by the push path in a follow-up commit use std::sync::Arc; +use std::time::{Duration, Instant}; use buzz_object_store::{ - ConditionalWrite, ImmutableWrite, ObjectStore, ObjectStoreError, Revision, S3AddressingStyle, - S3ObjectStore, S3StoreConfig, WriteCondition, + ConditionalWrite, ImmutableWrite, ObjectStore, ObjectStoreError, ProviderKind, Revision, + S3AddressingStyle, S3ObjectStore, S3StoreConfig, WriteCondition, }; use bytes::Bytes; use sha2::{Digest, Sha256}; @@ -107,22 +108,65 @@ fn backend(operation: &'static str, message: String) -> StoreError { /// Configuration for `GitStore::run_conformance_probe`. /// -/// Defaults: 32-way concurrency, 3 rounds. The probe is a deployment gate — -/// run at startup, fail-closed. See `docs/git-on-object-storage.md` §Conformance. -#[derive(Debug, Clone)] +/// The probe is a deployment gate — run at startup, fail-closed. See +/// `docs/git-on-object-storage.md` §Conformance. +/// +/// Defaults are per-provider ([`ProbeConfig::for_provider`]) because the two +/// profiles are proving the same axiom against backends with very different +/// admission costs: an S3-compatible store answers a wide race cheaply, while +/// Cloud Storage publishes a one-write-per-second ceiling per object name, so a +/// wide, tightly-spaced race there measures the rate limiter rather than the +/// conditional-write semantics. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ProbeConfig { /// How many tasks race per round. Must be ≥ 2. pub race_width: usize, /// How many rounds to run each race phase. pub race_rounds: usize, + /// How many times a round that proved nothing may be re-run before the + /// probe gives up. + /// + /// A round proves nothing when no racer's outcome distinguishes a + /// conforming backend from a broken one — every racer throttled, or too few + /// racers were classified to have witnessed a race at all. Retrying is not + /// leniency: an unproven round is neither pass nor fail, and exhausting the + /// budget without ever proving a round fails the probe. + pub unproven_round_retries: usize, + /// Minimum wall-clock spacing between mutations of the same key. + /// + /// Zero for backends with no documented per-object write ceiling. Cloud + /// Storage documents one write per second per object name, so its profile + /// spaces same-key rounds beyond that interval — the probe proves the + /// store's conditional-write semantics, and deliberately violating the + /// published rate limit would only prove that the rate limiter works. + pub same_key_spacing: Duration, +} + +impl ProbeConfig { + /// Defaults for `provider`. + pub fn for_provider(provider: ProviderKind) -> Self { + match provider { + ProviderKind::S3 => Self { + race_width: 32, + race_rounds: 3, + unproven_round_retries: 3, + same_key_spacing: Duration::ZERO, + }, + ProviderKind::Gcs => Self { + race_width: 3, + race_rounds: 2, + unproven_round_retries: 3, + // Just past Cloud Storage's documented one-write-per-second + // per-object ceiling. + same_key_spacing: Duration::from_millis(1_100), + }, + } + } } impl Default for ProbeConfig { fn default() -> Self { - Self { - race_width: 32, - race_rounds: 3, - } + Self::for_provider(ProviderKind::S3) } } @@ -130,10 +174,40 @@ impl Default for ProbeConfig { /// detail lives in `ProbeFailure` (the error variant). #[derive(Debug, Clone)] pub struct ProbeReport { + /// Which provider profile ran. + pub profile: ProviderKind, /// Concurrency used. pub race_width: usize, /// Rounds executed per race phase. pub race_rounds: usize, + /// Racers the backend answered with throttling across all race rounds. + /// + /// A throttled racer is *never* a lost race — the write was refused before + /// the precondition was evaluated, so it is evidence about request rate and + /// about nothing else. Counting them separately is what keeps backpressure + /// from masquerading as conformance. Non-zero here on a passing probe means + /// "admitted, and the backend was pacing us", which is the expected shape on + /// a provider with a per-object write ceiling. + pub throttled_racers: usize, + /// Race rounds that proved nothing and were re-run. + /// + /// See [`ProbeConfig::unproven_round_retries`]. Non-zero on a passing probe + /// means every round eventually proved itself, but the backend needed more + /// attempts than a quiet one would. + pub throttled_rounds_retried: usize, + /// Shortest observed interval between two same-key mutation rounds, when + /// the profile spaces them. + /// + /// Reported so a passing probe can be checked against the spacing it + /// claimed to honour rather than trusted to have slept. + pub min_same_key_gap: Option, + /// Probe objects the cleanup pass could not remove. + /// + /// Cleanup failure does not fail the probe — a store that satisfies every + /// conformance axiom is admitted even if a tidy-up delete flaked — but it + /// is surfaced because silent probe-key accumulation in a shared bucket is + /// exactly the kind of leak that is invisible until it is large. + pub cleanup_failures: usize, /// Total number of *transport-unknown* per-racer outcomes across all /// race rounds (sum of both `if_match_race` and `if_none_match_race` /// phases). A "transport-unknown" is a pre-classification failure — @@ -155,7 +229,12 @@ pub struct ProbeReport { #[derive(Debug, thiserror::Error)] #[error("conformance probe failed in phase '{phase}' (round {round}, key {key}): {reason}")] pub struct ProbeFailure { - /// One of `sequential`, `if_match_race`, `if_none_match_race`, `revision_consistency`. + /// The profile phase that failed. + /// + /// S3 profile: `sequential`, `if_match_race`, `if_none_match_race`, + /// `revision_consistency`. Cloud Storage profile: `immutable`, + /// `pointer_create`, `pointer_read`, `cas_replace`, `stale_cas`, + /// `cas_race`, `generation_roundtrip`. pub phase: &'static str, /// Round index (0-based) when this phase ran multiple rounds. pub round: usize, @@ -171,6 +250,103 @@ impl From for StoreError { } } +/// Marks a body written by the Cloud Storage profile's racing writers. +/// +/// Public to the crate so a test double can recognise a racer's write without +/// hard-coding the probe's body format. +pub(crate) const GCS_RACE_BODY_PREFIX: &str = "probe-gcs-race:"; + +/// Build a Cloud Storage profile failure. +fn gcs_failure(phase: &'static str, round: usize, key: &str, reason: String) -> ProbeFailure { + ProbeFailure { + phase, + round, + key: key.to_string(), + reason, + } +} + +/// Read the object generation out of a revision the store reported committing. +/// +/// A commit with no generation is fatal, not cosmetic: the generation is the +/// only token that can predicate the next write, so a caller handed nothing +/// would have to either stop writing or drop its precondition — and dropping it +/// turns the pointer swap into a blind overwrite. Cloud Storage has no live +/// generation `0`, so zero *is* the absent case. +fn gcs_generation( + phase: &'static str, + round: usize, + key: &str, + revision: &Revision, +) -> Result { + match revision.expect_gcs_generation() { + Ok(generation) if generation > 0 => Ok(generation), + Ok(_) => Err(gcs_failure( + phase, + round, + key, + "the store reported a successful write with no object generation".to_string(), + )), + Err(error) => Err(gcs_failure( + phase, + round, + key, + format!("expected an object generation: {error}"), + )), + } +} + +/// What one Cloud Storage race round established. +enum GcsRaceOutcome { + /// Exactly one racer committed, witnessed by at least one other classified + /// racer. + Committed { + /// Generation the winner committed. + generation: i64, + /// Bytes the winner wrote. + body: Vec, + }, + /// The round is neither pass nor fail — nothing in it distinguishes a + /// conforming store from a broken one — so it must be re-run. + Unproven(String), +} + +/// Counters and pacing state carried across the Cloud Storage phases. +#[derive(Default)] +struct GcsProbeState { + transport_drops: usize, + throttled_racers: usize, + throttled_rounds_retried: usize, + min_same_key_gap: Option, + last_same_key_write: Option, +} + +impl GcsProbeState { + /// Wait until `spacing` has elapsed since the previous same-key mutation, + /// and record the interval actually observed. + /// + /// The recorded gap is measured, not assumed: the report carries it so a + /// passing probe can be checked against the spacing it claimed to honour. + async fn pace(&mut self, spacing: Duration) { + let now = match self.last_same_key_write { + None => Instant::now(), + Some(previous) => { + let elapsed = previous.elapsed(); + if elapsed < spacing { + tokio::time::sleep(spacing - elapsed).await; + } + let gap = previous.elapsed(); + self.min_same_key_gap = Some( + self.min_same_key_gap + .map_or(gap, |shortest| shortest.min(gap)), + ); + Instant::now() + } + }; + self.last_same_key_write = Some(now); + } +} + /// Object-store client for git refs. #[derive(Clone)] pub struct GitStore { @@ -403,6 +579,32 @@ impl GitStore { /// `StoreError::Probe(ProbeFailure)` and the caller (relay startup) MUST /// refuse to come up. /// + /// The profile is chosen by the configured provider, not by the caller: the + /// axioms are the same for every backend, but the evidence that admits one + /// is provider-shaped. An S3-compatible store is admitted by a wide race on + /// ETag preconditions; Cloud Storage is admitted by a paced race on object + /// generations, where a refused (throttled) write is not a lost race. + pub async fn run_conformance_probe(&self, cfg: ProbeConfig) -> Result { + if cfg.race_width < 2 || cfg.race_rounds == 0 { + return Err(ProbeFailure { + phase: "config", + round: 0, + key: String::new(), + reason: format!( + "race_width must be ≥ 2 and race_rounds ≥ 1, got {}/{}", + cfg.race_width, cfg.race_rounds + ), + } + .into()); + } + match self.store.provider() { + ProviderKind::S3 => self.run_s3_conformance_probe(cfg).await, + ProviderKind::Gcs => self.run_gcs_conformance_probe(cfg).await, + } + } + + /// The S3 profile: revision-token compare-and-swap under a wide race. + /// /// Four phases: /// /// 1. **`sequential`** — write a content-addressed object, read it back, @@ -420,20 +622,8 @@ impl GitStore { /// `get_pointer` into `put_pointer(Matches(...))` and assert it /// commits. Tests that the token is opaque and stable between read and /// CAS. - pub async fn run_conformance_probe(&self, cfg: ProbeConfig) -> Result { + async fn run_s3_conformance_probe(&self, cfg: ProbeConfig) -> Result { use std::sync::Arc; - if cfg.race_width < 2 || cfg.race_rounds == 0 { - return Err(ProbeFailure { - phase: "config", - round: 0, - key: String::new(), - reason: format!( - "race_width must be ≥ 2 and race_rounds ≥ 1, got {}/{}", - cfg.race_width, cfg.race_rounds - ), - } - .into()); - } let nonce = uuid::Uuid::new_v4(); let pointer_key = format!("probe/pointer-{nonce}"); // Accumulator for *transport-unknown* per-racer outcomes across both @@ -708,12 +898,532 @@ impl GitStore { let _ = self.store.delete(&pointer_key).await; Ok(ProbeReport { + profile: ProviderKind::S3, race_width: cfg.race_width, race_rounds: cfg.race_rounds, transport_drops, + throttled_racers: 0, + throttled_rounds_retried: 0, + min_same_key_gap: None, + cleanup_failures: 0, }) } + /// The Cloud Storage profile: object-generation compare-and-swap, paced to + /// the provider's published per-object write ceiling. + /// + /// Same axioms, different evidence. Cloud Storage answers a stale + /// precondition with 412 (an ordinary conflict) and an over-rate write with + /// 429 (a refusal to evaluate the precondition at all), and it publishes a + /// maximum of one write per second to a single object name. A profile that + /// ignored either fact would be measuring the rate limiter: a burst of 429s + /// would either be miscounted as lost races — turning "the backend paced us" + /// into "the backend admitted two winners' worth of losers" — or would make + /// a round pass with no race in it. So this profile races narrowly, spaces + /// same-key rounds past the ceiling, classifies throttles separately from + /// conflicts, and re-runs a round that proved nothing rather than scoring + /// it. + /// + /// Phases, in order: + /// + /// 1. **`immutable`** — create-only write of a content-addressed object, + /// read back, digest verified. A1 plus read-after-write. + /// 2. **`pointer_create`** — create the pointer under the create-only + /// precondition, which Cloud Storage spells as generation `0`. + /// 3. **`pointer_read`** — read body and generation from one response and + /// check both against what was just committed. A2. + /// 4. **`cas_replace`** — replace under the observed generation; the commit + /// must report a *different* generation. + /// 5. **`stale_cas`** — replay the superseded generation; must conflict. A + /// store that commits here is performing blind overwrites. + /// 6. **`cas_race`** — `race_width` writers on one generation, `race_rounds` + /// times: exactly one commit, every other classified racer a conflict or + /// a throttle, and the stored object equal to the winner's. + /// 7. **`generation_roundtrip`** — the winning generation predicates the + /// next successful compare-and-swap, closing the loop the push path + /// depends on. + /// + /// Probe objects are removed afterwards on both the success and the failure + /// path; a cleanup failure is reported, not fatal. + async fn run_gcs_conformance_probe(&self, cfg: ProbeConfig) -> Result { + let nonce = uuid::Uuid::new_v4(); + let mut written = Vec::new(); + let mut state = GcsProbeState::default(); + + let outcome = self + .gcs_probe_phases(&cfg, nonce, &mut written, &mut state) + .await; + let cleanup_failures = self.remove_probe_objects(&written).await; + outcome?; + + Ok(ProbeReport { + profile: ProviderKind::Gcs, + race_width: cfg.race_width, + race_rounds: cfg.race_rounds, + transport_drops: state.transport_drops, + throttled_racers: state.throttled_racers, + throttled_rounds_retried: state.throttled_rounds_retried, + min_same_key_gap: state.min_same_key_gap, + cleanup_failures, + }) + } + + /// The Cloud Storage phases, factored out so cleanup runs on every path. + async fn gcs_probe_phases( + &self, + cfg: &ProbeConfig, + nonce: uuid::Uuid, + written: &mut Vec, + state: &mut GcsProbeState, + ) -> Result<(), StoreError> { + // -- Phase 1: immutable ------------------------------------------------ + // The nonce makes this key new, so the create-only write must report a + // create rather than a collision. + let body = format!("probe-gcs-immutable-{nonce}").into_bytes(); + let key = Self::content_key("probe/gcs-immutable", &body); + written.push(key.clone()); + let outcome = self + .put_immutable_raw(&key, &body) + .await + .map_err(|e| gcs_failure("immutable", 0, &key, format!("create-only write: {e}")))?; + if outcome != ImmutableWrite::Created { + return Err(gcs_failure( + "immutable", + 0, + &key, + "a freshly nonced key reported a collision, so the create-only \ + precondition is not being evaluated" + .to_string(), + ) + .into()); + } + let read = self + .get_verified(&key, &Self::digest_hex(&body)) + .await + .map_err(|e| gcs_failure("immutable", 0, &key, format!("verified read: {e}")))?; + if read[..] != body[..] { + return Err(gcs_failure( + "immutable", + 0, + &key, + "read-after-write returned different bytes".to_string(), + ) + .into()); + } + + // -- Phase 2: pointer_create ------------------------------------------- + let pointer_key = format!("probe/gcs-pointer-{nonce}"); + written.push(pointer_key.clone()); + let seed = format!("probe-gcs-pointer-seed-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + let created = self + .put_pointer(&pointer_key, &seed, WriteCondition::Absent) + .await + .map_err(|e| { + gcs_failure( + "pointer_create", + 0, + &pointer_key, + format!("create-only pointer write: {e}"), + ) + })?; + let mut generation = match created { + ConditionalWrite::Committed(revision) => { + gcs_generation("pointer_create", 0, &pointer_key, &revision)? + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "pointer_create", + 0, + &pointer_key, + "a freshly nonced pointer key was already taken".to_string(), + ) + .into()) + } + }; + + // -- Phase 3: pointer_read --------------------------------------------- + // Body and generation must describe the same committed object, or the + // generation a caller predicates its next write on names a version it + // never read. + let (revision, stored) = self + .get_pointer(&pointer_key) + .await + .map_err(|e| gcs_failure("pointer_read", 0, &pointer_key, format!("read: {e}")))? + .ok_or_else(|| { + gcs_failure( + "pointer_read", + 0, + &pointer_key, + "the pointer just committed does not exist".to_string(), + ) + })?; + let observed = gcs_generation("pointer_read", 0, &pointer_key, &revision)?; + if observed != generation { + return Err(gcs_failure( + "pointer_read", + 0, + &pointer_key, + format!( + "read reported generation {observed} for the object committed as {generation}" + ), + ) + .into()); + } + if stored[..] != seed[..] { + return Err(gcs_failure( + "pointer_read", + 0, + &pointer_key, + "read returned bytes other than the committed body".to_string(), + ) + .into()); + } + + // -- Phase 4: cas_replace ---------------------------------------------- + let replacement = format!("probe-gcs-replace-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + let superseded = generation; + generation = match self + .put_pointer( + &pointer_key, + &replacement, + WriteCondition::Matches(Revision::GcsGeneration(generation)), + ) + .await + .map_err(|e| gcs_failure("cas_replace", 0, &pointer_key, format!("write: {e}")))? + { + ConditionalWrite::Committed(revision) => { + let committed = gcs_generation("cas_replace", 0, &pointer_key, &revision)?; + if committed == superseded { + return Err(gcs_failure( + "cas_replace", + 0, + &pointer_key, + format!( + "the replacement reported the same generation {superseded} it \ + replaced, so the token cannot distinguish versions" + ), + ) + .into()); + } + committed + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "cas_replace", + 0, + &pointer_key, + "a compare-and-swap on the just-read generation conflicted with no \ + competing writer" + .to_string(), + ) + .into()) + } + }; + + // -- Phase 5: stale_cas ------------------------------------------------ + let stale_body = format!("probe-gcs-stale-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + match self + .put_pointer( + &pointer_key, + &stale_body, + WriteCondition::Matches(Revision::GcsGeneration(superseded)), + ) + .await + .map_err(|e| gcs_failure("stale_cas", 0, &pointer_key, format!("write: {e}")))? + { + ConditionalWrite::Conflict => {} + ConditionalWrite::Committed(_) => { + return Err(gcs_failure( + "stale_cas", + 0, + &pointer_key, + format!( + "a write predicated on superseded generation {superseded} committed: \ + the precondition is not being enforced, so every pointer update is a \ + blind overwrite" + ), + ) + .into()) + } + } + + // -- Phase 6: cas_race ------------------------------------------------- + for round in 0..cfg.race_rounds { + let mut attempt = 0usize; + let winner = loop { + state.pace(cfg.same_key_spacing).await; + match self + .gcs_race_round(round, attempt, &pointer_key, nonce, generation, cfg, state) + .await? + { + GcsRaceOutcome::Committed { generation, body } => { + break (generation, body); + } + GcsRaceOutcome::Unproven(reason) => { + if attempt >= cfg.unproven_round_retries { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + format!( + "no round proved a race in {} attempts: {reason}", + attempt + 1 + ), + ) + .into()); + } + attempt += 1; + state.throttled_rounds_retried += 1; + tracing::warn!( + phase = "cas_race", + round, + attempt, + reason = %reason, + "conformance race round proved nothing; re-running" + ); + } + } + }; + let (committed, body) = winner; + + // The stored object must be the winner's, at the winner's + // generation: a loser's payload surviving the race is the failure + // mode the whole pointer protocol exists to exclude. + let (revision, stored) = self + .get_pointer(&pointer_key) + .await + .map_err(|e| { + gcs_failure( + "cas_race", + round, + &pointer_key, + format!("post-race read: {e}"), + ) + })? + .ok_or_else(|| { + gcs_failure( + "cas_race", + round, + &pointer_key, + "the pointer vanished during the race".to_string(), + ) + })?; + let settled = gcs_generation("cas_race", round, &pointer_key, &revision)?; + if settled != committed { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + format!( + "the winner committed generation {committed} but the object settled at \ + {settled}" + ), + ) + .into()); + } + if stored[..] != body[..] { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + "the object holds bytes no racer reported committing".to_string(), + ) + .into()); + } + generation = committed; + } + + // -- Phase 7: generation_roundtrip ------------------------------------- + // The generation a racer won with must predicate the next write; that + // chain — commit, then compare-and-swap on the returned token — is + // exactly what the push path does between two pushes. + let final_body = format!("probe-gcs-roundtrip-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + match self + .put_pointer( + &pointer_key, + &final_body, + WriteCondition::Matches(Revision::GcsGeneration(generation)), + ) + .await + .map_err(|e| { + gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + format!("write: {e}"), + ) + })? { + ConditionalWrite::Committed(revision) => { + let committed = gcs_generation("generation_roundtrip", 0, &pointer_key, &revision)?; + if committed == generation { + return Err(gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + format!("the write reported the same generation {generation} it replaced"), + ) + .into()); + } + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + "the generation a racer committed did not predicate the next write, so a \ + winner cannot chain its own pushes" + .to_string(), + ) + .into()) + } + } + + Ok(()) + } + + /// One race round: `cfg.race_width` writers on the same generation. + /// + /// Returns the winner when the round proved something, and + /// [`GcsRaceOutcome::Unproven`] when it did not. Only a semantic violation — + /// two winners, a commit with no generation, an unclassifiable backend + /// answer — fails here. + #[allow(clippy::too_many_arguments)] + async fn gcs_race_round( + &self, + round: usize, + attempt: usize, + pointer_key: &str, + nonce: uuid::Uuid, + generation: i64, + cfg: &ProbeConfig, + state: &mut GcsProbeState, + ) -> Result { + let mut tasks = Vec::with_capacity(cfg.race_width); + for racer in 0..cfg.race_width { + let body = + format!("{GCS_RACE_BODY_PREFIX}{round}:{attempt}:{racer}:{nonce}").into_bytes(); + let condition = WriteCondition::Matches(Revision::GcsGeneration(generation)); + tasks.push(async move { + let outcome = self.put_pointer(pointer_key, &body, condition).await; + (racer, body, outcome) + }); + } + + let mut winners: Vec<(i64, Vec)> = Vec::new(); + let mut conflicts = 0usize; + let mut throttled = 0usize; + let mut drops = 0usize; + for (racer, body, outcome) in futures_util::future::join_all(tasks).await { + match outcome { + Ok(ConditionalWrite::Committed(revision)) => { + let committed = gcs_generation("cas_race", round, pointer_key, &revision)?; + winners.push((committed, body)); + } + Ok(ConditionalWrite::Conflict) => conflicts += 1, + // Throttling is a refusal to evaluate the precondition, so it + // says nothing about who won. It is counted, never scored. + Err(StoreError::Backend(ObjectStoreError::Throttled { .. })) => { + throttled += 1; + state.throttled_racers += 1; + } + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { + drops += 1; + state.transport_drops += 1; + tracing::warn!( + phase = "cas_race", + round, + racer, + "transport drop (pre-classification: socket/send failure)" + ); + } + Err(e) => { + return Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!("racer {racer}: {e}"), + ) + .into()) + } + } + } + + if winners.len() > 1 { + return Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!( + "{} racers committed on one generation: the store is not linearizing \ + conditional writes", + winners.len() + ), + ) + .into()); + } + + // Classified observers. A throttled racer is one: it proves the round + // ran, even though it proves nothing about the precondition. + let classified = winners.len() + conflicts + throttled; + if let Some((committed, body)) = winners.pop() { + if classified < 2 { + return Ok(GcsRaceOutcome::Unproven(format!( + "only {classified} of {} racers were classified, so no race was witnessed \ + ({drops} transport drops)", + cfg.race_width + ))); + } + return Ok(GcsRaceOutcome::Committed { + generation: committed, + body, + }); + } + + if conflicts == 0 { + return Ok(GcsRaceOutcome::Unproven(format!( + "no racer committed and none saw the generation move ({throttled} throttled, \ + {drops} transport drops)" + ))); + } + if drops > 0 { + return Ok(GcsRaceOutcome::Unproven(format!( + "{conflicts} racers saw the generation move but the committing racer's outcome \ + was never classified ({drops} transport drops)" + ))); + } + Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!( + "{conflicts} racers were told the generation had moved, but no racer committed \ + and every outcome was classified: the object changed without an acknowledged \ + writer" + ), + ) + .into()) + } + + /// Delete the probe's objects, returning how many could not be removed. + /// + /// Runs on the failure path too: a failed probe is the one that gets + /// re-run, so that is exactly when leaking keys into the deployment's own + /// bucket must not happen. + async fn remove_probe_objects(&self, keys: &[String]) -> usize { + let mut failures = 0usize; + for key in keys { + if let Err(error) = self.store.delete(key).await { + failures += 1; + tracing::warn!(%error, "conformance probe could not remove its object"); + } + } + failures + } + /// Helper: hex SHA-256 of bytes. fn digest_hex(bytes: &[u8]) -> String { let mut h = Sha256::new(); @@ -837,6 +1547,643 @@ mod tests { } } +#[cfg(test)] +mod profiles { + //! Profile behaviour against a scripted store. + //! + //! The probe consumes the object-store seam, so a test double can answer a + //! race any way a real backend could — including ways no conforming backend + //! ever would. That is the point: a conformance gate is only worth its boot + //! time if it *fails* on the answers it claims to reject, and a live bucket + //! cannot be asked to commit two writers on one generation. + + use std::collections::{HashMap, VecDeque}; + use std::sync::atomic::{AtomicI64, Ordering}; + use std::sync::Mutex; + + use async_trait::async_trait; + use buzz_object_store::{BulkDeleteOutcome, ByteStream, ListPage, ObjectMeta}; + + use super::*; + + /// How a scripted racer answers. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum RacerOutcome { + /// Commit regardless of the precondition — the only way to stage two + /// winners on one generation. + Win, + /// Commit, but report no object generation. + WinWithoutGeneration, + /// The ordinary lost-race answer. + Conflict, + /// Refuse the write for request rate. + Throttle, + /// Never produce a classified answer. + Drop, + } + + /// An in-memory object store with real compare-and-swap semantics, plus a + /// script that can override the answers to the probe's racing writers. + struct ScriptedStore { + provider: ProviderKind, + objects: Mutex>, + next_generation: AtomicI64, + race_script: Mutex>, + /// Blind-overwrite bug: commit even when the precondition is stale. + accept_stale_precondition: bool, + /// Report a created object as committed with no generation. + zero_generation_on_create: bool, + } + + impl ScriptedStore { + fn new(provider: ProviderKind) -> Self { + Self { + provider, + objects: Mutex::new(HashMap::new()), + // No live object has generation 0; that value means "absent". + next_generation: AtomicI64::new(1), + race_script: Mutex::new(VecDeque::new()), + accept_stale_precondition: false, + zero_generation_on_create: false, + } + } + + /// Script consecutive race rounds; rounds past the script race for real. + fn scripting(self, rounds: impl IntoIterator>) -> Self { + self.race_script + .lock() + .unwrap() + .extend(rounds.into_iter().flatten()); + self + } + + fn accepting_stale_preconditions(mut self) -> Self { + self.accept_stale_precondition = true; + self + } + + fn without_create_generation(mut self) -> Self { + self.zero_generation_on_create = true; + self + } + + fn keys(&self) -> Vec { + let mut keys: Vec<_> = self.objects.lock().unwrap().keys().cloned().collect(); + keys.sort(); + keys + } + + /// Mint the revision token this provider would report. + fn revision(&self, generation: i64) -> Revision { + match self.provider { + ProviderKind::S3 => Revision::S3Etag(format!("\"{generation}\"")), + ProviderKind::Gcs => Revision::GcsGeneration(generation), + } + } + + /// Read a caller's revision back, rejecting one from another provider + /// exactly as a real provider does. + fn generation_of(&self, revision: &Revision) -> Result { + match self.provider { + ProviderKind::S3 => revision + .expect_s3_etag() + .map(|tag| tag.trim_matches('"').parse().unwrap_or(-1)), + ProviderKind::Gcs => revision.expect_gcs_generation(), + } + } + + fn commit(&self, key: &str, bytes: &[u8]) -> i64 { + let generation = self.next_generation.fetch_add(1, Ordering::SeqCst); + self.objects + .lock() + .unwrap() + .insert(key.to_string(), (generation, Bytes::copy_from_slice(bytes))); + generation + } + + fn current_generation(&self, key: &str) -> i64 { + self.objects + .lock() + .unwrap() + .get(key) + .map(|(generation, _)| *generation) + .unwrap_or(0) + } + } + + #[async_trait] + impl ObjectStore for ScriptedStore { + fn provider(&self) -> ProviderKind { + self.provider + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + _content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.commit(key, bytes); + Ok(()) + } + + async fn put_file( + &self, + _key: &str, + _path: &std::path::Path, + _content_type: &str, + ) -> Result<(), ObjectStoreError> { + Err(ObjectStoreError::Provider { + operation: "put_file", + message: "unused by the conformance probe".into(), + }) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + match self + .put_conditional(key, bytes, content_type, WriteCondition::Absent) + .await? + { + ConditionalWrite::Committed(_) => Ok(ImmutableWrite::Created), + ConditionalWrite::Conflict => Ok(ImmutableWrite::AlreadyPresent), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + _content_type: &str, + condition: WriteCondition, + ) -> Result { + let expected = match &condition { + WriteCondition::Absent => 0, + WriteCondition::Matches(revision) => self.generation_of(revision)?, + }; + + // Only the profile's racing writers are scripted; every other write + // gets real compare-and-swap semantics, so the phases around the + // race behave like a conforming store unless a test says otherwise. + if bytes.starts_with(GCS_RACE_BODY_PREFIX.as_bytes()) { + let scripted = self.race_script.lock().unwrap().pop_front(); + match scripted { + Some(RacerOutcome::Win) => { + return Ok(ConditionalWrite::Committed( + self.revision(self.commit(key, bytes)), + )) + } + Some(RacerOutcome::WinWithoutGeneration) => { + self.commit(key, bytes); + return Ok(ConditionalWrite::Committed(Revision::GcsGeneration(0))); + } + Some(RacerOutcome::Conflict) => return Ok(ConditionalWrite::Conflict), + Some(RacerOutcome::Throttle) => { + return Err(ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }) + } + Some(RacerOutcome::Drop) => { + return Err(ObjectStoreError::TransportAmbiguous { + operation: "put_conditional", + message: "connection reset by peer".into(), + }) + } + None => {} + } + } + + let current = self.current_generation(key); + let honour_stale = + self.accept_stale_precondition && matches!(condition, WriteCondition::Matches(_)); + if current != expected && !honour_stale { + return Ok(ConditionalWrite::Conflict); + } + let generation = self.commit(key, bytes); + if self.zero_generation_on_create && current == 0 { + return Ok(ConditionalWrite::Committed(Revision::GcsGeneration(0))); + } + Ok(ConditionalWrite::Committed(self.revision(generation))) + } + + async fn get(&self, key: &str) -> Result { + self.objects + .lock() + .unwrap() + .get(key) + .map(|(_, bytes)| bytes.clone()) + .ok_or_else(|| ObjectStoreError::NotFound { key: key.into() }) + } + + async fn get_range( + &self, + key: &str, + start: u64, + end: u64, + ) -> Result { + let bytes = self.get(key).await?; + Ok(bytes.slice(start as usize..=(end as usize))) + } + + async fn get_stream(&self, _key: &str) -> Result { + Err(ObjectStoreError::Provider { + operation: "get_stream", + message: "unused by the conformance probe".into(), + }) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + let found = self + .objects + .lock() + .unwrap() + .get(key) + .map(|(generation, bytes)| (*generation, bytes.clone())); + Ok(found.map(|(generation, bytes)| (self.revision(generation), bytes))) + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + let found = self + .objects + .lock() + .unwrap() + .get(key) + .map(|(generation, bytes)| (*generation, bytes.len() as u64)); + Ok(found.map(|(generation, size)| ObjectMeta { + size, + revision: Some(self.revision(generation)), + })) + } + + async fn list_page( + &self, + _prefix: &str, + _continuation_token: Option, + _max_keys: usize, + ) -> Result { + Ok(ListPage::default()) + } + + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.objects.lock().unwrap().remove(key); + Ok(()) + } + + async fn delete_objects( + &self, + keys: &[String], + ) -> Result { + let mut objects = self.objects.lock().unwrap(); + let mut outcome = BulkDeleteOutcome::default(); + for key in keys { + objects.remove(key); + outcome.deleted += 1; + } + Ok(outcome) + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + Ok(()) + } + + async fn versioning_detected(&self) -> Result { + Ok(false) + } + } + + /// A Cloud Storage profile config with the sleeps taken out, so the tests + /// that are not about pacing run at memory speed. + fn unpaced(race_rounds: usize) -> ProbeConfig { + ProbeConfig { + race_width: 3, + race_rounds, + unproven_round_retries: 3, + same_key_spacing: Duration::ZERO, + } + } + + fn probe_failure(error: StoreError) -> ProbeFailure { + match error { + StoreError::Probe(failure) => failure, + other => panic!("expected a probe failure, got {other:?}"), + } + } + + /// Each profile's defaults are the ones the provider can actually answer. + /// The S3 numbers are unchanged — this profile is already admitted in + /// production and a conformance gate that quietly narrows is worse than one + /// that never widened. + #[test] + fn profile_defaults_follow_the_provider() { + let s3 = ProbeConfig::for_provider(ProviderKind::S3); + assert_eq!(s3, ProbeConfig::default()); + assert_eq!(s3.race_width, 32); + assert_eq!(s3.race_rounds, 3); + assert_eq!(s3.same_key_spacing, Duration::ZERO); + + let gcs = ProbeConfig::for_provider(ProviderKind::Gcs); + assert!(gcs.race_width >= 2 && gcs.race_width < s3.race_width); + assert!(gcs.race_rounds >= 1); + assert!( + gcs.same_key_spacing > Duration::from_secs(1), + "same-key rounds must be spaced past Cloud Storage's one-write-per-second ceiling" + ); + assert!(gcs.unproven_round_retries >= 1); + } + + /// A conforming store passes, and the probe leaves nothing behind. + #[tokio::test] + async fn a_conforming_store_is_admitted_and_cleans_up() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs)); + let store = GitStore::new(backend.clone()); + + let report = store + .run_conformance_probe(unpaced(2)) + .await + .expect("a conforming store is admitted"); + + assert_eq!(report.profile, ProviderKind::Gcs); + assert_eq!(report.race_width, 3); + assert_eq!(report.race_rounds, 2); + assert_eq!(report.throttled_racers, 0); + assert_eq!(report.throttled_rounds_retried, 0); + assert_eq!(report.transport_drops, 0); + assert_eq!(report.cleanup_failures, 0); + assert!( + backend.keys().is_empty(), + "probe objects left behind: {:?}", + backend.keys() + ); + } + + /// The failure the pointer protocol exists to exclude. Two winners is never + /// a pacing artefact, a retryable round, or a degraded observation — it is + /// the store admitting a lost update. + #[tokio::test] + async fn two_committed_racers_fail_the_probe() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::Win, + RacerOutcome::Win, + RacerOutcome::Conflict, + ]])); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("two winners on one generation must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure.reason.contains("2 racers committed"), + "unexpected reason: {}", + failure.reason + ); + } + + /// A round in which every racer was refused for rate proves nothing: no + /// precondition was ever evaluated. Scoring it as "no winner" would fail a + /// conforming store for being paced, so the round is re-run instead. + #[tokio::test] + async fn a_fully_throttled_round_is_re_run_rather_than_scored() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Throttle; 3]]), + ); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a throttled round is re-run, and the re-run proves the race"); + + assert_eq!(report.throttled_rounds_retried, 1); + assert_eq!(report.throttled_racers, 3); + } + + /// Re-running is bounded. A store that only ever throttles is never + /// admitted — the probe fails rather than waiting forever or passing on no + /// evidence. + #[tokio::test] + async fn re_runs_are_bounded_and_an_unproven_race_fails_closed() { + let mut cfg = unpaced(1); + cfg.unproven_round_retries = 2; + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting( + vec![vec![RacerOutcome::Throttle; 3]; cfg.unproven_round_retries + 1], + )); + + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(cfg) + .await + .expect_err("a store that only throttles is never admitted"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure + .reason + .contains("no round proved a race in 3 attempts"), + "unexpected reason: {}", + failure.reason + ); + } + + /// One winner among a mix of conflicts and throttles is a pass: the + /// throttled racer is counted, not treated as a loser, and one classified + /// witness is enough to have seen the race. + #[tokio::test] + async fn a_throttled_racer_is_never_a_lost_race() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::Win, + RacerOutcome::Conflict, + RacerOutcome::Throttle, + ]])); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a round with one winner, one conflict and one throttle is proven"); + + assert_eq!(report.throttled_racers, 1); + assert_eq!(report.throttled_rounds_retried, 0); + } + + /// A commit the store cannot name is unusable: the caller has nothing to + /// predicate its next write on. Fatal wherever it appears. + #[tokio::test] + async fn a_commit_without_a_generation_fails_the_probe() { + let on_create = Arc::new(ScriptedStore::new(ProviderKind::Gcs).without_create_generation()); + let failure = probe_failure( + GitStore::new(on_create) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("a create with no generation must fail closed"), + ); + assert_eq!(failure.phase, "pointer_create"); + assert!( + failure.reason.contains("no object generation"), + "unexpected reason: {}", + failure.reason + ); + + let on_race = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::WinWithoutGeneration, + RacerOutcome::Conflict, + RacerOutcome::Conflict, + ]])); + let failure = probe_failure( + GitStore::new(on_race) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("a race winner with no generation must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure.reason.contains("no object generation"), + "unexpected reason: {}", + failure.reason + ); + } + + /// A store that commits on a superseded generation is doing blind + /// overwrites, which silently loses pushes. The stale phase is what catches + /// it, and it must catch it before any race runs. + #[tokio::test] + async fn a_store_that_honours_a_stale_generation_fails_the_probe() { + let backend = + Arc::new(ScriptedStore::new(ProviderKind::Gcs).accepting_stale_preconditions()); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("an unenforced precondition must fail closed"), + ); + assert_eq!(failure.phase, "stale_cas"); + assert!( + failure.reason.contains("blind overwrite"), + "unexpected reason: {}", + failure.reason + ); + } + + /// Every racer was told the generation had moved, and every outcome was + /// classified — so the object changed with no writer acknowledged. That is + /// a lost update announcing itself, not an unproven round: the probe must + /// not retry its way past it. + #[tokio::test] + async fn a_round_where_every_racer_loses_fails_the_probe() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Conflict; 3]]), + ); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("conflicts with no acknowledged winner must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure + .reason + .contains("the object changed without an acknowledged writer"), + "unexpected reason: {}", + failure.reason + ); + } + + /// Every racer's outcome was unknown, so the round is unproven rather than + /// a failure — the probe admits stores, not networks. + #[tokio::test] + async fn transport_drops_leave_the_observer_set_rather_than_failing() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Drop; 3]]), + ); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a dropped round is re-run"); + + assert_eq!(report.transport_drops, 3); + assert_eq!(report.throttled_rounds_retried, 1); + } + + /// Pacing is measured, not asserted: the probe sleeps between same-key + /// mutations and reports the shortest interval it actually observed. + #[tokio::test] + async fn same_key_mutations_are_spaced_by_the_configured_interval() { + let spacing = Duration::from_millis(40); + let mut cfg = unpaced(2); + cfg.race_width = 2; + cfg.same_key_spacing = spacing; + + // create, replace, stale, two race rounds, round-trip: six same-key + // mutations, so five paced gaps. + let paced_gaps = 5; + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs)); + let started = Instant::now(); + let report = GitStore::new(backend) + .run_conformance_probe(cfg) + .await + .expect("a conforming store is admitted"); + let elapsed = started.elapsed(); + + let observed = report + .min_same_key_gap + .expect("a paced profile reports the interval it observed"); + assert!( + observed >= spacing, + "shortest observed gap {observed:?} is under the configured {spacing:?}" + ); + assert!( + elapsed >= spacing * paced_gaps, + "the whole probe took {elapsed:?}, less than {paced_gaps} gaps of {spacing:?}" + ); + } + + /// The provider selects the profile. An S3 store still runs the ETag + /// profile, unchanged, and reports none of the Cloud Storage counters. + #[tokio::test] + async fn an_s3_store_runs_the_s3_profile() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::S3)); + let report = GitStore::new(backend) + .run_conformance_probe(ProbeConfig { + race_width: 3, + race_rounds: 1, + ..ProbeConfig::default() + }) + .await + .expect("a conforming S3 store is admitted"); + + assert_eq!(report.profile, ProviderKind::S3); + assert_eq!(report.transport_drops, 0); + assert_eq!(report.throttled_racers, 0); + assert_eq!(report.min_same_key_gap, None); + } + + /// The width floor is a property of the gate, not of a profile: one writer + /// cannot witness a race whichever provider is underneath. + #[tokio::test] + async fn a_race_narrower_than_two_writers_is_rejected() { + for provider in [ProviderKind::S3, ProviderKind::Gcs] { + let backend = Arc::new(ScriptedStore::new(provider)); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(ProbeConfig { + race_width: 1, + race_rounds: 1, + ..ProbeConfig::for_provider(provider) + }) + .await + .expect_err("a single writer cannot witness a race"), + ); + assert_eq!(failure.phase, "config"); + } + } +} + #[cfg(test)] mod probe { //! Empirical probe of the S3 provider's precondition surfacing against @@ -1005,6 +2352,7 @@ mod probe { .run_conformance_probe(ProbeConfig { race_width: 8, race_rounds: 2, + ..ProbeConfig::for_provider(ProviderKind::S3) }) .await .expect("conformance probe"); diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index f6ec581d330..25e225932af 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -504,29 +504,36 @@ async fn main() -> anyhow::Result<()> { info!(runtime_id = %runtime_id, "Inter-relay mesh started"); } - // Git-on-object-storage: admit the configured S3/MinIO backend against the + // Git-on-object-storage: admit the configured object store against the // linearizable conditional-write axiom (A3) before serving git traffic. // Failure is fatal: a backend that cannot satisfy pointer CAS invalidates // the manifest-pointer protocol. This is a deployment gate, not a proof. + // + // The profile follows the provider — a wide ETag race for S3, a paced + // generation race for Cloud Storage — so the defaults below are the + // profile's, and the environment only overrides them. if std::env::var("BUZZ_GIT_CONFORMANCE_PROBE") .map(|v| v != "false") .unwrap_or(true) { - let race_width = std::env::var("BUZZ_GIT_PROBE_WRITERS") + let mut cfg = buzz_relay::api::git::store::ProbeConfig::for_provider(provider); + if let Some(race_width) = std::env::var("BUZZ_GIT_PROBE_WRITERS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(32); - let race_rounds = std::env::var("BUZZ_GIT_PROBE_ROUNDS") + { + cfg.race_width = race_width; + } + if let Some(race_rounds) = std::env::var("BUZZ_GIT_PROBE_ROUNDS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(3); - let cfg = buzz_relay::api::git::store::ProbeConfig { - race_width, - race_rounds, - }; + { + cfg.race_rounds = race_rounds; + } tracing::info!( - race_width, - race_rounds, + profile = %provider, + race_width = cfg.race_width, + race_rounds = cfg.race_rounds, + same_key_spacing_ms = cfg.same_key_spacing.as_millis() as u64, "running git object-store conformance probe (A3 gate)" ); let report = state @@ -535,9 +542,15 @@ async fn main() -> anyhow::Result<()> { .await .map_err(|e| anyhow::anyhow!("git conformance probe failed: {e}"))?; tracing::info!( + profile = %report.profile, race_width = report.race_width, race_rounds = report.race_rounds, transport_drops = report.transport_drops, + throttled_racers = report.throttled_racers, + throttled_rounds_retried = report.throttled_rounds_retried, + min_same_key_gap_ms = + report.min_same_key_gap.map(|gap| gap.as_millis() as u64), + cleanup_failures = report.cleanup_failures, "git object-store backend admitted: A3 conformance probe passed" ); } From 8afe1c27a464950825b107ddd85d22695856b6c1 Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 24 Aug 2026 17:29:09 -0700 Subject: [PATCH 5/7] test(object-store, git): add live and scale suites, and document the profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests prove how the code judges answers; only a real bucket proves which answers Cloud Storage actually gives. Add the suites that close that gap, all `#[ignore]`d and additionally gated on `BUZZ_GCS_LIVE=1` so a bare `--ignored` run without credentials skips rather than fails. Credentials come from Application Default Credentials, exactly as in production, and every test works under its own uuid-scoped namespace and asserts its own cleanup emptied it. - `tests/gcs_live.rs` — provider conformance against a live bucket: bucket admission accepted and refused, permission denied refusing to hand out a client, foreign revisions never reaching the backend, create-only idempotence, compare-and-swap admitting exactly one writer, reads agreeing across full/range/stream, large objects streaming through a resumable upload, prefix pagination, idempotent deletion with per-key outcomes, and rapid same-key transitions being paced rather than failed. - `tests/gcs_scale.rs` — store-level scale evidence, additionally gated on `BUZZ_GCS_SCALE=1`: a repeated hot-pointer race that must never admit two winners, concurrent creates across distinct names that must all commit, and a sustained mixed read/write/list load reporting its latency percentiles and outcome mix. Each reports its numbers so a regression in throttling or contention behaviour is legible rather than merely non-fatal. - `api::git::store::gcs_live` — the deployment gate itself, run against the provider it was written for: the profile admitting a real bucket, the pointer cycle publishing exactly one state, content-addressed objects round tripping with corruption detected, and a chunked seed committing every sequential transition. Also documents the two conformance profiles and the provider-neutral implementation vocabulary in `docs/git-on-object-storage.md`, so the reason the Cloud Storage profile is shaped differently is recorded next to the design it belongs to rather than only in the probe source. Signed-off-by: mozarthq --- Cargo.lock | 3 + crates/buzz-object-store/Cargo.toml | 10 +- crates/buzz-object-store/src/providers/gcs.rs | 300 +++++-- crates/buzz-object-store/tests/gcs_live.rs | 824 ++++++++++++++++++ crates/buzz-object-store/tests/gcs_scale.rs | 586 +++++++++++++ crates/buzz-relay/src/api/git/store.rs | 471 +++++++++- docs/git-on-object-storage.md | 59 +- 7 files changed, 2188 insertions(+), 65 deletions(-) create mode 100644 crates/buzz-object-store/tests/gcs_live.rs create mode 100644 crates/buzz-object-store/tests/gcs_scale.rs diff --git a/Cargo.lock b/Cargo.lock index 7c515f2c6ef..1efd93590bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1183,10 +1183,13 @@ dependencies = [ "google-cloud-gax", "google-cloud-storage", "google-cloud-wkt", + "hex", "quick-xml 0.38.4", "rand 0.10.1", "rust-s3", + "rustls", "serde", + "sha2 0.11.0", "thiserror 2.0.18", "tokio", "tracing", diff --git a/crates/buzz-object-store/Cargo.toml b/crates/buzz-object-store/Cargo.toml index 3ef914a7356..6f88d62a8d8 100644 --- a/crates/buzz-object-store/Cargo.toml +++ b/crates/buzz-object-store/Cargo.toml @@ -14,7 +14,10 @@ bytes = "1" futures-core = "0.3" futures-util = "0.3" google-cloud-gax = "1.13" -google-cloud-storage = "=1.17.0" +# `default-rustls-provider` would install aws-lc-rs as the process-wide rustls +# provider. Buzz installs ring at startup instead (see `buzz-relay`'s `main`), +# and two providers racing for the default is a panic at first use. +google-cloud-storage = { version = "=1.17.0", default-features = false } quick-xml = { version = "0.38", features = ["serialize"] } rand = { workspace = true } s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } @@ -26,3 +29,8 @@ uuid = { workspace = true } [dev-dependencies] google-cloud-wkt = "1.7" +hex = { workspace = true } +sha2 = { workspace = true } +# The live GCS tests are their own process, so they install the same rustls +# provider `buzz-relay`'s `main` does before making a TLS request. +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } diff --git a/crates/buzz-object-store/src/providers/gcs.rs b/crates/buzz-object-store/src/providers/gcs.rs index 5dd05b7e16c..4451d87d378 100644 --- a/crates/buzz-object-store/src/providers/gcs.rs +++ b/crates/buzz-object-store/src/providers/gcs.rs @@ -79,6 +79,14 @@ const BULK_DELETE_CONCURRENCY: usize = 12; /// response stall a request for minutes. const MAX_RETRY_AFTER: Duration = Duration::from_secs(30); +/// Floor on the pause after a throttled request that carried no `Retry-After`. +/// +/// Cloud Storage documents a maximum of one write per second to a single object +/// name. A writer already being throttled on one key therefore cannot succeed +/// by retrying sooner, and would only spend its bounded attempt budget — which +/// is what turns absorbed backpressure into a failed push. +const MIN_THROTTLE_BACKOFF: Duration = Duration::from_secs(1); + /// Longest provider detail kept on an error. /// /// Cloud Storage error bodies are JSON diagnostics, but they can echo request @@ -89,8 +97,16 @@ const MAX_ERROR_DETAIL: usize = 512; /// Bounded retry policy for one [`GcsObjectStore`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct GcsRetryConfig { - /// Total attempts, including the first. `1` disables retries. + /// Attempts allowed while the backend is failing, including the first. + /// `1` disables retries. pub max_attempts: u32, + /// Attempts allowed while the backend is throttling, including the first. + /// + /// Larger than `max_attempts` because a 429 is not a failure: it is the + /// service asking to be approached more slowly, at a published rate. Giving + /// up on it turns backpressure the caller should have absorbed into a + /// failed write. + pub max_throttled_attempts: u32, /// Backoff ceiling used for the first retry, doubled thereafter. pub initial_backoff: Duration, /// Ceiling on any single computed backoff. @@ -101,6 +117,7 @@ impl Default for GcsRetryConfig { fn default() -> Self { Self { max_attempts: 6, + max_throttled_attempts: 12, initial_backoff: Duration::from_millis(200), max_backoff: Duration::from_secs(8), } @@ -221,9 +238,9 @@ impl GcsObjectStore { "gcs bucket must be configured".to_string(), )); } - if config.retry.max_attempts == 0 { + if config.retry.max_attempts == 0 || config.retry.max_throttled_attempts == 0 { return Err(ObjectStoreError::Config( - "gcs retry max_attempts must be at least 1".to_string(), + "gcs retry max_attempts and max_throttled_attempts must be at least 1".to_string(), )); } @@ -278,66 +295,32 @@ impl GcsObjectStore { async fn with_retries( &self, operation: &'static str, - key: &str, + _key: &str, mut attempt: F, ) -> Result where F: FnMut() -> Fut, Fut: Future>, { - for attempt_index in 0..self.retry.max_attempts { + let mut budget = RetryBudget::new(&self.retry); + loop { match attempt().await { Ok(value) => return Ok(value), - Err(error) => match self.retry_delay(&error, attempt_index) { - Some(delay) if attempt_index + 1 < self.retry.max_attempts => { + Err(error) => match budget.next_delay(&error) { + Some(delay) => { tracing::debug!( provider = "gcs", operation, - attempt = attempt_index + 1, + attempt = budget.attempts(), delay_ms = delay.as_millis() as u64, "retrying object store operation" ); tokio::time::sleep(delay).await; } - _ => return Err(error), + None => return Err(error), }, } } - // Unreachable while `max_attempts >= 1`, which the constructor enforces. - Err(ObjectStoreError::TransportRetryable { - operation, - message: format!("retry budget exhausted for {key:?}"), - }) - } - - /// How long to wait before retrying, or `None` when the error is final. - fn retry_delay(&self, error: &ObjectStoreError, attempt_index: u32) -> Option { - match error { - ObjectStoreError::Throttled { retry_after, .. } => Some( - retry_after - .map(|hint| hint.min(MAX_RETRY_AFTER)) - .unwrap_or_else(|| self.backoff(attempt_index)), - ), - ObjectStoreError::TransportRetryable { .. } - | ObjectStoreError::TransportAmbiguous { .. } => Some(self.backoff(attempt_index)), - _ => None, - } - } - - /// Capped exponential backoff with full jitter. - /// - /// Full jitter (a uniform draw from `[1ms, cap]`) rather than the raw - /// exponent: a hot pointer is written by several racers at once, and - /// unjittered backoff would keep them synchronised into the same retry - /// instants. - fn backoff(&self, attempt_index: u32) -> Duration { - let cap = self - .retry - .initial_backoff - .saturating_mul(1u32 << attempt_index.min(16)) - .min(self.retry.max_backoff); - let cap_ms = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(1); - Duration::from_millis(1 + rand::random::() % cap_ms) } /// One conditional-write attempt, carrying `precondition` verbatim. @@ -391,6 +374,88 @@ impl GcsObjectStore { } } +/// One operation's progress through the bounded retry policy. +/// +/// Throttling and transient failure are counted separately on purpose. +/// A 5xx or a dropped connection is the backend failing, and a caller waiting +/// on a long series of them is waiting for nothing. A 429 is the backend +/// working correctly and asking to be approached more slowly, with a published +/// recovery interval — a sole writer pushing a repository in chunks has to wait +/// its turn, and giving up on it would turn absorbed backpressure into a failed +/// push. So throttling gets its own, larger allowance. +struct RetryBudget<'a> { + retry: &'a GcsRetryConfig, + failures: u32, + throttles: u32, +} + +impl<'a> RetryBudget<'a> { + fn new(retry: &'a GcsRetryConfig) -> Self { + Self { + retry, + failures: 0, + throttles: 0, + } + } + + /// Attempts made so far. + fn attempts(&self) -> u32 { + self.failures + } + + /// Record `error` and return how long to wait, or `None` to give up and + /// return it to the caller. + fn next_delay(&mut self, error: &ObjectStoreError) -> Option { + let attempt_index = self.failures; + self.failures += 1; + + match error { + ObjectStoreError::Throttled { retry_after, .. } => { + self.throttles += 1; + if self.throttles >= self.retry.max_throttled_attempts { + return None; + } + Some(match retry_after { + Some(hint) => (*hint).min(MAX_RETRY_AFTER), + // Cloud Storage publishes a one-write-per-second ceiling + // per object name. Once a writer is being throttled on a + // single key, retrying sooner than that cannot succeed — it + // only spends the budget — so the throttle path takes a + // floor the transient path does not. + None => backoff(self.retry, attempt_index).max(MIN_THROTTLE_BACKOFF), + }) + } + ObjectStoreError::TransportRetryable { .. } + | ObjectStoreError::TransportAmbiguous { .. } => { + if self.failures - self.throttles >= self.retry.max_attempts { + return None; + } + Some(backoff(self.retry, attempt_index)) + } + _ => None, + } + } +} + +/// Capped exponential backoff with equal jitter. +/// +/// The wait is half the exponent plus a uniform draw over the other half, +/// rather than a uniform draw over the whole of it. Both forms de-synchronise +/// racers on a hot pointer, which is the point of jitter, but full jitter also +/// makes very short waits likely — and against a hard per-object rate limit a +/// short wait cannot succeed, so it burns an attempt from a bounded budget for +/// nothing. Keeping the lower half fixed means every retry is meaningfully +/// later than the one before it. +fn backoff(retry: &GcsRetryConfig, attempt_index: u32) -> Duration { + let cap = retry + .initial_backoff + .saturating_mul(1u32 << attempt_index.min(16)) + .min(retry.max_backoff); + let cap_ms = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(2); + let fixed = cap_ms / 2; + Duration::from_millis(fixed + rand::random::() % (cap_ms - fixed)) +} + /// The resource name both Cloud Storage clients address a bucket by. fn bucket_resource(bucket: &str) -> String { format!("projects/_/buckets/{bucket}") @@ -590,8 +655,9 @@ impl ObjectStore for GcsObjectStore { // Set once an attempt fails without a classified answer. From then on a // 412 is no longer self-evidently someone else's commit. let mut outcome_unknown = false; + let mut budget = RetryBudget::new(&self.retry); - for attempt_index in 0..self.retry.max_attempts { + loop { let error = match self .write_once( "put_conditional", @@ -616,26 +682,21 @@ impl ObjectStore for GcsObjectStore { // Retrying always replays `precondition` verbatim: the loop never // relaxes a compare-and-swap into a blind overwrite. - match self.retry_delay(&error, attempt_index) { - Some(delay) if attempt_index + 1 < self.retry.max_attempts => { + match budget.next_delay(&error) { + Some(delay) => { tracing::debug!( provider = "gcs", operation = "put_conditional", - attempt = attempt_index + 1, + attempt = budget.attempts(), delay_ms = delay.as_millis() as u64, outcome_unknown, "retrying conditional write with its original precondition" ); tokio::time::sleep(delay).await; } - _ => return Err(error), + None => return Err(error), } } - - Err(ObjectStoreError::TransportRetryable { - operation: "put_conditional", - message: "retry budget exhausted".to_string(), - }) } async fn get(&self, key: &str) -> Result { @@ -1120,6 +1181,8 @@ mod tests { #[test] fn retry_after_seconds_are_honoured_and_capped() { + let retry = GcsRetryConfig::default(); + let err = classify("put_conditional", "pointers/x", throttled_error("3")); assert!(matches!( err, @@ -1128,11 +1191,132 @@ mod tests { .. } if d == Duration::from_secs(3) )); + assert_eq!( + RetryBudget::new(&retry).next_delay(&err), + Some(Duration::from_secs(3)), + "an advertised backoff is used verbatim" + ); + + let absurd = classify("put_conditional", "pointers/x", throttled_error("86400")); + assert_eq!( + RetryBudget::new(&retry).next_delay(&absurd), + Some(MAX_RETRY_AFTER), + "one response must not stall a request indefinitely" + ); + } + + /// Cloud Storage caps writes to one per second per object name, so a + /// throttled writer that retries sooner cannot succeed — it only spends an + /// attempt from a bounded budget. Live testing found exactly this: with a + /// sub-second first retry the budget ran out and absorbed backpressure + /// surfaced as a failed write. + #[test] + fn throttling_without_a_hint_waits_at_least_the_per_object_write_interval() { + let retry = GcsRetryConfig::default(); + let throttled = ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }; + let mut budget = RetryBudget::new(&retry); + for attempt in 1..retry.max_throttled_attempts { + let delay = budget + .next_delay(&throttled) + .expect("throttling stays retryable within its own budget"); + assert!( + delay >= MIN_THROTTLE_BACKOFF, + "attempt {attempt} would retry after {delay:?}, sooner than the per-object \ + write interval" + ); + } + assert!( + budget.next_delay(&throttled).is_none(), + "the throttle budget is bounded" + ); + } - let store_retry = GcsRetryConfig::default(); - let hint = Duration::from_secs(3600).min(MAX_RETRY_AFTER); - assert_eq!(hint, MAX_RETRY_AFTER); - assert!(store_retry.max_backoff < MAX_RETRY_AFTER); + /// Throttling and transient failure draw on separate allowances. A long + /// series of 429s must not consume the budget reserved for a genuinely + /// failing backend, and vice versa — otherwise a paced writer and a broken + /// one are indistinguishable to the policy. + #[test] + fn throttling_and_transient_failure_have_independent_budgets() { + let retry = GcsRetryConfig::default(); + assert!( + retry.max_throttled_attempts > retry.max_attempts, + "backpressure deserves more patience than failure" + ); + let throttled = ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }; + let transient = ObjectStoreError::TransportRetryable { + operation: "put_conditional", + message: "503".into(), + }; + + // Spend the throttle allowance down to its last attempt, then show the + // transient allowance is still whole. + let mut budget = RetryBudget::new(&retry); + for _ in 1..retry.max_throttled_attempts { + assert!(budget.next_delay(&throttled).is_some()); + } + for _ in 1..retry.max_attempts { + assert!( + budget.next_delay(&transient).is_some(), + "throttling must not have eaten the transient budget" + ); + } + assert!( + budget.next_delay(&transient).is_none(), + "the transient budget is bounded" + ); + } + + /// Equal jitter: every wait is at least half its exponential cap, and never + /// more than the cap. Full jitter would satisfy the upper bound alone while + /// making near-zero waits common. + #[test] + fn backoff_grows_and_never_collapses_to_zero() { + let retry = GcsRetryConfig::default(); + let mut previous_floor = Duration::ZERO; + for attempt in 0..6 { + let cap = retry + .initial_backoff + .saturating_mul(1 << attempt) + .min(retry.max_backoff); + for _ in 0..64 { + let delay = backoff(&retry, attempt); + assert!(delay <= cap, "attempt {attempt}: {delay:?} exceeds {cap:?}"); + assert!( + delay >= cap / 2, + "attempt {attempt}: {delay:?} is below half of {cap:?}" + ); + } + assert!(cap >= previous_floor, "backoff must not shrink"); + previous_floor = cap; + } + } + + /// Only backpressure and unknown outcomes are retried. A stale generation, + /// a missing object, or a permission failure is the caller's answer on the + /// first attempt. + #[test] + fn classified_final_answers_are_not_retried() { + let retry = GcsRetryConfig::default(); + for error in [ + ObjectStoreError::NotFound { key: "k".into() }, + ObjectStoreError::Conflict { key: "k".into() }, + ObjectStoreError::Provider { + operation: "put", + message: "403".into(), + }, + ObjectStoreError::Config("bad".into()), + ] { + assert!( + RetryBudget::new(&retry).next_delay(&error).is_none(), + "{error} must not be retried" + ); + } } /// An HTTP-date `Retry-After` is not misread as a duration; the caller diff --git a/crates/buzz-object-store/tests/gcs_live.rs b/crates/buzz-object-store/tests/gcs_live.rs new file mode 100644 index 00000000000..0e0fab10172 --- /dev/null +++ b/crates/buzz-object-store/tests/gcs_live.rs @@ -0,0 +1,824 @@ +//! Live round-trip tests for the Google Cloud Storage provider. +//! +//! These talk to a real bucket, so they are `#[ignore]`d and additionally +//! gated on `BUZZ_GCS_LIVE=1` — a bare `--ignored` run in an environment +//! without credentials skips instead of failing. +//! +//! ```bash +//! BUZZ_GCS_LIVE=1 \ +//! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ +//! cargo test -p buzz-object-store --test gcs_live -- --ignored +//! ``` +//! +//! Credentials come from Application Default Credentials, exactly as they do +//! in production. The bucket must satisfy the provider's admission check — +//! object versioning disabled and soft-delete retention zero — which the first +//! test asserts explicitly. +//! +//! Every test works under a unique `a2//` prefix and deletes what it +//! wrote, so a bucket accumulates nothing across runs and concurrent runs +//! cannot collide. +//! +//! Two arms need a bucket that is *not* the disposable one, and stay skipped +//! unless the operator names it. Both are read-only — they never write, and +//! the provider refuses to hand out a client for either: +//! +//! ```bash +//! BUZZ_GCS_DENIED_BUCKET=a-bucket-this-identity-cannot-read \ +//! BUZZ_GCS_MISCONFIGURED_BUCKET=a-bucket-with-versioning-or-soft-delete-on \ +//! … +//! ``` + +use std::panic::AssertUnwindSafe; +use std::time::{Duration, Instant}; + +use futures_util::FutureExt; + +use buzz_object_store::{ + ConditionalWrite, GcsObjectStore, GcsStoreConfig, ImmutableWrite, ObjectStore, + ObjectStoreError, ProviderKind, Revision, WriteCondition, +}; + +const CONTENT_TYPE: &str = "application/octet-stream"; + +/// Install the process-wide rustls provider these tests need. +/// +/// The relay does this in `main` before any TLS request. A test binary must do +/// the same: both ring and aws-lc-rs are in the build graph, so rustls refuses +/// to pick one on its own. +fn install_crypto_provider() { + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + +/// Whether this environment opted in to talking to a real bucket. +fn live_enabled(test: &str) -> bool { + install_crypto_provider(); + if std::env::var("BUZZ_GCS_LIVE").as_deref() == Ok("1") { + return true; + } + eprintln!("skipping {test}: set BUZZ_GCS_LIVE=1 to run against a live bucket"); + false +} + +/// A bucket named by `variable`, or `None` when the operator did not supply one. +fn named_bucket(variable: &str, test: &str) -> Option { + match std::env::var(variable) { + Ok(bucket) if !bucket.trim().is_empty() => Some(bucket.trim().to_string()), + _ => { + eprintln!("skipping {test}: set {variable} to run this arm"); + None + } + } +} + +/// Connect to the test bucket, or `None` when this environment has not opted +/// in to live tests. +async fn live_store(test: &str) -> Option<(GcsObjectStore, String)> { + if !live_enabled(test) { + return None; + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_LIVE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + let store = GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"); + let prefix = format!("a2/{}/{test}", uuid::Uuid::new_v4()); + Some((store, prefix)) +} + +/// Delete everything this test wrote. +/// +/// Failing to clean up is a test failure: these buckets are shared with other +/// runs and a leak would be invisible until it was large. +async fn cleanup(store: &GcsObjectStore, prefix: &str) { + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(prefix, token, 1000) + .await + .expect("list for cleanup"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + break; + } + } + if keys.is_empty() { + return; + } + let outcome = store + .delete_objects(&keys) + .await + .expect("bulk delete for cleanup"); + assert!( + outcome.failed.is_empty(), + "cleanup left objects behind: {:?}", + outcome.failed + ); +} + +/// Run one live test body, then clean up its prefix whether it passed or not. +/// +/// A panicking body would otherwise leak its objects into a bucket shared with +/// every other run — and a failing test is precisely the one that gets re-run, +/// so that is when cleanup matters most. +async fn with_prefix(test: &str, body: F) +where + F: AsyncFnOnce(&GcsObjectStore, &str), +{ + let Some((store, prefix)) = live_store(test).await else { + return; + }; + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + cleanup(&store, &prefix).await; + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } +} + +/// The provider reports itself as Cloud Storage, and the bucket satisfies the +/// deletion contract the constructor enforces. +/// +/// `versioning_detected()` is read from bucket metadata rather than probed, so +/// a correctly configured bucket answers `false` even though every Cloud +/// Storage object carries a generation. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn admits_a_bucket_that_can_prove_deletion() { + with_prefix("admission", async |store, _prefix| { + assert_eq!(store.provider(), ProviderKind::Gcs); + assert!( + !store.versioning_detected().await.expect("bucket metadata"), + "the test bucket must have object versioning and soft delete disabled" + ); + store.ping().await.expect("bucket is reachable"); + }) + .await; +} + +/// Create-only writes: the first commits, the second finds the key taken. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn create_only_write_is_idempotent() { + with_prefix("immutable", async |store, prefix| { + let key = format!("{prefix}/packs/object"); + + assert_eq!( + store + .put_immutable(&key, b"immutable bytes", CONTENT_TYPE) + .await + .expect("first create-only write"), + ImmutableWrite::Created + ); + assert_eq!( + store + .put_immutable(&key, b"immutable bytes", CONTENT_TYPE) + .await + .expect("second create-only write"), + ImmutableWrite::AlreadyPresent, + "a create-only precondition failure is success, not an error" + ); + assert_eq!( + store.get(&key).await.expect("read back").as_ref(), + b"immutable bytes" + ); + }) + .await; +} + +/// The compare-and-swap contract, end to end. +/// +/// Two writers hold the same generation. The first commits and the second — +/// now holding a stale generation — must lose, with exactly one winner and no +/// silent overwrite. The winning generation then predicates the next write, so +/// a caller can chain transitions without ever rereading. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn compare_and_swap_admits_exactly_one_writer() { + with_prefix("cas", async |store, prefix| { + let key = format!("{prefix}/pointers/manifest"); + + // Create with the absent precondition. + let created = store + .put_conditional(&key, b"state-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer"); + let ConditionalWrite::Committed(generation_0) = created else { + panic!("creating an absent pointer must commit"); + }; + assert!(matches!(generation_0, Revision::GcsGeneration(g) if g > 0)); + + // A second create-only write cannot commit over it. + assert_eq!( + store + .put_conditional(&key, b"state-x", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("second create"), + ConditionalWrite::Conflict + ); + + // Body and revision come from one read, and the revision predicates the + // next write. + let (read_generation, body) = store + .get_with_revision(&key) + .await + .expect("pointer read") + .expect("pointer exists"); + assert_eq!(body.as_ref(), b"state-0"); + assert_eq!(read_generation, generation_0); + + // Two sequential writers, both holding `generation_0`. Exactly one wins. + let winner = store + .put_conditional( + &key, + b"state-1", + CONTENT_TYPE, + WriteCondition::Matches(read_generation.clone()), + ) + .await + .expect("first writer"); + let loser = store + .put_conditional( + &key, + b"state-2", + CONTENT_TYPE, + WriteCondition::Matches(read_generation), + ) + .await + .expect("second writer"); + + let ConditionalWrite::Committed(generation_1) = winner else { + panic!("the writer holding the current generation must commit"); + }; + assert_ne!( + generation_1, generation_0, + "a commit mints a new generation" + ); + assert_eq!( + loser, + ConditionalWrite::Conflict, + "a stale generation must lose the race rather than overwrite" + ); + assert_eq!( + store.get(&key).await.expect("read winner").as_ref(), + b"state-1", + "the loser must not have written anything" + ); + + // The winner's generation chains straight into the next transition. + assert!(matches!( + store + .put_conditional( + &key, + b"state-3", + CONTENT_TYPE, + WriteCondition::Matches(generation_1), + ) + .await + .expect("chained write"), + ConditionalWrite::Committed(_) + )); + }) + .await; +} + +/// An ETag can never predicate a generation match; the write is refused rather +/// than silently downgraded to an unconditional overwrite. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn a_foreign_revision_never_reaches_the_backend() { + with_prefix("foreign-revision", async |store, prefix| { + let key = format!("{prefix}/pointers/manifest"); + + store + .put_conditional(&key, b"state-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer"); + + let error = store + .put_conditional( + &key, + b"clobbered", + CONTENT_TYPE, + WriteCondition::Matches(Revision::S3Etag("\"deadbeef\"".into())), + ) + .await + .expect_err("an S3 ETag must not predicate a GCS write"); + assert!(matches!( + error, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + assert_eq!( + store.get(&key).await.expect("read back").as_ref(), + b"state-0", + "the refused write must not have touched the object" + ); + }) + .await; +} + +/// Full, ranged, and streamed reads agree, and metadata reports the size and +/// generation. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn reads_agree_across_full_range_and_stream() { + use futures_util::StreamExt; + + with_prefix("reads", async |store, prefix| { + let key = format!("{prefix}/blobs/payload"); + let body: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); + + store.put(&key, &body, CONTENT_TYPE).await.expect("put"); + + assert_eq!( + store.get(&key).await.expect("full read").as_ref(), + &body[..] + ); + assert_eq!( + store.get_range(&key, 100, 199).await.expect("range read"), + &body[100..=199], + "the seam's range is inclusive on both ends" + ); + assert_eq!( + store.get_range(&key, 4095, 4095).await.expect("last byte"), + &body[4095..], + ); + + let mut streamed = Vec::new(); + let mut chunks = store.get_stream(&key).await.expect("open stream"); + while let Some(chunk) = chunks.next().await { + streamed.extend_from_slice(&chunk.expect("stream chunk")); + } + assert_eq!(streamed, body); + + let meta = store + .head(&key) + .await + .expect("head") + .expect("object exists"); + assert_eq!(meta.size, body.len() as u64); + assert!(matches!(meta.revision, Some(Revision::GcsGeneration(g)) if g > 0)); + + assert!(store + .head(&format!("{prefix}/blobs/absent")) + .await + .expect("head of an absent object is not an error") + .is_none()); + assert!(store + .get_with_revision(&format!("{prefix}/blobs/absent")) + .await + .expect("pointer read of an absent object is not an error") + .is_none()); + assert!(matches!( + store.get(&format!("{prefix}/blobs/absent")).await, + Err(ObjectStoreError::NotFound { .. }) + )); + }) + .await; +} + +/// Prefix listing pages, in ascending key order, without dropping or repeating +/// a key across the page boundary. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn listing_paginates_over_a_prefix() { + with_prefix("listing", async |store, prefix| { + let expected: Vec = (0..5).map(|i| format!("{prefix}/page/{i:03}")).collect(); + for key in &expected { + store.put(key, b"x", CONTENT_TYPE).await.expect("put"); + } + + let mut seen = Vec::new(); + let mut token = None; + let mut pages = 0; + loop { + let page = store.list_page(prefix, token, 2).await.expect("list page"); + pages += 1; + assert!(page.objects.len() <= 2, "max_keys must bound one response"); + seen.extend(page.objects.into_iter().map(|(key, size)| { + assert_eq!(size, 1, "listing reports object size"); + key + })); + token = page.next_continuation_token; + match token { + Some(_) => assert!(page.is_truncated, "a continuation token means truncated"), + None => { + assert!(!page.is_truncated, "the last page is not truncated"); + break; + } + } + } + + assert!(pages > 1, "five keys at two per page must span pages"); + assert_eq!( + seen, expected, + "keys arrive in ascending order, exactly once" + ); + }) + .await; +} + +/// Deletion means deletion, deleting an absent key is not an error, and a bulk +/// delete folds present and absent keys into distinct counters. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn deletion_is_idempotent_and_reports_per_key_outcomes() { + with_prefix("deletion", async |store, prefix| { + let key = format!("{prefix}/single"); + store.put(&key, b"x", CONTENT_TYPE).await.expect("put"); + store.delete(&key).await.expect("delete"); + // Absence has to hold on every read path, immediately. A bucket with + // soft delete on would answer the HEAD with a live object or resurrect + // one on the next read; the admission check exists to keep that bucket + // from ever getting this far, and this is the observable consequence. + assert!( + store.head(&key).await.expect("head").is_none(), + "with versioning and soft delete off, a delete proves absence" + ); + assert!( + matches!( + store.get(&key).await, + Err(ObjectStoreError::NotFound { .. }) + ), + "a deleted object must not be readable" + ); + assert!( + store + .get_with_revision(&key) + .await + .expect("pointer read of a deleted object is not an error") + .is_none(), + "a deleted object must not still carry a revision" + ); + assert!( + store + .list_page(&key, None, 10) + .await + .expect("list") + .objects + .is_empty(), + "a deleted object must not still be listed" + ); + store + .delete(&key) + .await + .expect("deleting an absent object is not an error"); + + let present: Vec = (0..3).map(|i| format!("{prefix}/bulk/{i}")).collect(); + for key in &present { + store.put(key, b"x", CONTENT_TYPE).await.expect("put"); + } + let absent: Vec = (0..2).map(|i| format!("{prefix}/bulk/gone-{i}")).collect(); + + let mut keys = present.clone(); + keys.extend(absent); + let outcome = store.delete_objects(&keys).await.expect("bulk delete"); + + assert_eq!(outcome.deleted, 3); + assert_eq!(outcome.already_missing, 2); + assert!(outcome.failed.is_empty(), "{:?}", outcome.failed); + assert!( + outcome.versioned_keys.is_empty(), + "an admitted bucket cannot produce version artifacts" + ); + + assert!(store + .list_page(prefix, None, 10) + .await + .expect("list") + .objects + .is_empty()); + }) + .await; +} + +/// A sole writer driving the pointer faster than Cloud Storage's documented +/// one-write-per-second-per-object limit. +/// +/// This is the shape of the mirror's chunked repository seed: many sequential +/// transitions against a single object name, each predicated on the generation +/// the previous one returned. Throttling must pace the writer — never fail the +/// push, and never masquerade as a lost race. +/// +/// The two phases are ordered deliberately. The first uses a client configured +/// for a single attempt, so every 429 reaches the caller and can be counted; +/// it also spends whatever burst allowance the service was willing to grant. +/// The second then drives the same key just as hard through the provider's own +/// bounded policy, with that allowance already gone — which is what makes it a +/// real test of absorption rather than of a quiet service. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn rapid_sequential_transitions_are_paced_not_failed() { + // Close to a real chunked seed of a large mirrored repository (~4.5k refs + // at 200 refs per chunk), and comfortably past the burst allowance the + // service grants a fresh object name — which is what makes the throttling + // in this test real rather than incidental. + const TRANSITIONS: usize = 20; + + with_prefix("pacing", async |store, prefix| { + let key = format!("{prefix}/pointers/hot"); + + let mut revision = match store + .put_conditional(&key, b"transition-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer") + { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("creating an absent pointer must commit"), + }; + + // Phase 1: no in-provider retries, so throttling is visible and counted. + let mut config = GcsStoreConfig::new(store.bucket()); + config.retry.max_attempts = 1; + config.retry.max_throttled_attempts = 1; + let unpaced = GcsObjectStore::connect(&config) + .await + .expect("connect a single-attempt client"); + + let mut observed_429s = 0usize; + for i in 0..TRANSITIONS { + let body = format!("unpaced-{i}"); + loop { + match unpaced + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision.clone()), + ) + .await + { + Ok(ConditionalWrite::Committed(next)) => { + revision = next; + break; + } + Ok(ConditionalWrite::Conflict) => { + panic!("transition {i} lost a race it was the only entrant in") + } + // Throttling is backpressure. The retry carries the same + // precondition; dropping it here would be the exact bug this + // test exists to prevent. + Err(ObjectStoreError::Throttled { retry_after, .. }) => { + observed_429s += 1; + tokio::time::sleep(retry_after.unwrap_or(Duration::from_millis(400))).await; + } + Err(other) => panic!("transition {i} failed: {other}"), + } + } + } + + // Phase 2: the provider's own bounded policy, against a spent allowance. + let started = Instant::now(); + let mut generations = vec![revision.clone()]; + for i in 0..TRANSITIONS { + let body = format!("paced-{i}"); + revision = match store + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision), + ) + .await + .unwrap_or_else(|e| panic!("transition {i} must not fail: {e}")) + { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => { + panic!("transition {i} lost a race it was the only entrant in") + } + }; + generations.push(revision.clone()); + } + let elapsed = started.elapsed(); + + let mut distinct = generations.clone(); + distinct.dedup(); + assert_eq!( + distinct.len(), + generations.len(), + "every transition must mint a distinct generation" + ); + assert_eq!( + store.get(&key).await.expect("final read").as_ref(), + format!("paced-{}", TRANSITIONS - 1).as_bytes(), + "the last transition must be the published state" + ); + + eprintln!( + "pacing: {TRANSITIONS} single-attempt transitions observed {observed_429s} throttled \ + attempts; {TRANSITIONS} further transitions then completed through the provider's own \ + policy in {elapsed:?}" + ); + }) + .await; +} + +/// Streaming a large object to the backend and back without ever holding it in +/// memory. +/// +/// `put_file` is the media path for blobs that do not fit in a buffer: it hands +/// the client an open file and lets the resumable upload protocol do the +/// chunking. The read side is the mirror image — the byte stream is folded into +/// a digest as it arrives, so nothing larger than one chunk is resident at +/// either end. The assertions that matter are that the round trip is +/// byte-exact, that the provider reports the size it stored, and that a range +/// read still addresses the far end of a multi-chunk object correctly. +/// +/// The size is configurable because the deployment's ceiling (500 MiB) is +/// larger than a routine test run wants to move; the default is big enough to +/// require a resumable upload rather than a single-request one. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn streams_a_large_object_through_a_resumable_upload() { + use std::io::Write; + + use sha2::{Digest, Sha256}; + + let mebibytes: usize = std::env::var("BUZZ_GCS_LARGE_OBJECT_MIB") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(96); + + with_prefix("large-object", async |store, prefix| { + let key = format!("{prefix}/media/large.bin"); + let total = mebibytes * 1024 * 1024; + + // A pseudo-random, non-repeating body: a run of identical blocks would + // let a chunking bug (a dropped or duplicated chunk) round-trip + // undetected. + let mut block = vec![0u8; 1024 * 1024]; + let mut expected = Sha256::new(); + let source = std::env::temp_dir().join(format!("buzz-a4-large-{}", uuid::Uuid::new_v4())); + { + let mut file = std::fs::File::create(&source).expect("create the upload source"); + let mut state = 0x2545_F491_4F6C_DD1Du64; + for _ in 0..mebibytes { + for byte in block.iter_mut() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = (state >> 24) as u8; + } + expected.update(&block); + file.write_all(&block).expect("write the upload source"); + } + file.flush().expect("flush the upload source"); + } + let expected = hex::encode(expected.finalize()); + + let upload = Instant::now(); + let outcome = store.put_file(&key, &source, CONTENT_TYPE).await; + let upload = upload.elapsed(); + // The temp file is this test's, not the harness's: remove it before + // asserting so a failure cannot leave a large file behind. + let _ = std::fs::remove_file(&source); + outcome.expect("streaming upload"); + + let meta = store + .head(&key) + .await + .expect("head") + .expect("the uploaded object exists"); + assert_eq!(meta.size, total as u64, "the provider stored every byte"); + + let download = Instant::now(); + let mut streamed = Sha256::new(); + let mut length = 0u64; + { + use futures_util::StreamExt; + let mut chunks = store.get_stream(&key).await.expect("open the read stream"); + while let Some(chunk) = chunks.next().await { + let chunk = chunk.expect("stream chunk"); + length += chunk.len() as u64; + streamed.update(&chunk); + } + } + let download = download.elapsed(); + assert_eq!(length, total as u64, "the stream delivered every byte"); + assert_eq!( + hex::encode(streamed.finalize()), + expected, + "the streamed body must be byte-exact" + ); + + // A range read against the far end of a multi-chunk object: the offset + // is past every upload chunk boundary, so an implementation that + // resolved ranges against a single chunk would miss here. + let tail = store + .get_range(&key, total as u64 - 8, total as u64 - 1) + .await + .expect("range read at the tail"); + assert_eq!(tail.len(), 8, "the seam's range is inclusive on both ends"); + + eprintln!( + "large object: {mebibytes} MiB uploaded in {upload:?}, streamed back in {download:?}" + ); + }) + .await; +} + +/// Permission denied is a classified, permanent answer — never an ambiguous +/// outcome, and never a reason to come up anyway. +/// +/// The bucket-metadata read is the first request the provider makes, so a +/// client for a bucket this identity cannot read never gets constructed. That +/// is the fail-closed behaviour that matters: an unauthorised deployment stops +/// at connect rather than discovering its authorisation one operation at a +/// time. +/// +/// The arm is read-only. It reads bucket metadata and nothing else, and the +/// only outcome it accepts is a refusal. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn permission_denied_refuses_to_hand_out_a_client() { + let test = "permission-denied"; + if !live_enabled(test) { + return; + } + let Some(bucket) = named_bucket("BUZZ_GCS_DENIED_BUCKET", test) else { + return; + }; + + let error = GcsObjectStore::connect(&GcsStoreConfig::new(&bucket)) + .await + .err() + .unwrap_or_else(|| { + panic!("BUZZ_GCS_DENIED_BUCKET={bucket} is readable by this identity — pick a bucket it cannot read") + }); + + match &error { + ObjectStoreError::Provider { operation, .. } => { + assert_eq!(*operation, "get_bucket", "the refusal names the operation"); + } + other => panic!( + "permission denied must be a classified provider answer, not {other:?}: an \ + authorisation failure that read as ambiguous or retryable would be retried \ + forever, and one that read as a conflict would be scored as a lost race" + ), + } + assert!( + !error.is_ambiguous(), + "a 403 is an answer: the request was evaluated and refused" + ); + assert!( + !error.is_retryable(), + "retrying a permission failure cannot change its outcome" + ); + eprintln!("permission denied: {error}"); +} + +/// A bucket that cannot prove deletion is refused at connect, with the +/// violation named. +/// +/// This is the A2 admission check against a real misconfigured bucket rather +/// than a constructed one: object versioning or a nonzero soft-delete retention +/// both mean a delete leaves a restorable copy, so `delete` could report success +/// while the bytes stay reachable. Buzz's deletion contract cannot hold on such +/// a bucket, so the provider refuses to return a client for it at all. +/// +/// Read-only: the run reads bucket metadata and stops. Nothing is written to +/// the misconfigured bucket, which is the point — the client never exists. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn a_bucket_that_cannot_prove_deletion_is_refused() { + let test = "misconfigured-bucket"; + if !live_enabled(test) { + return; + } + let Some(bucket) = named_bucket("BUZZ_GCS_MISCONFIGURED_BUCKET", test) else { + return; + }; + + let error = GcsObjectStore::connect(&GcsStoreConfig::new(&bucket)) + .await + .err() + .unwrap_or_else(|| { + panic!( + "BUZZ_GCS_MISCONFIGURED_BUCKET={bucket} satisfies the deletion contract — point \ + this at a bucket with object versioning or a nonzero soft-delete retention" + ) + }); + + let ObjectStoreError::Config(message) = &error else { + panic!("a misconfigured bucket must be refused as a configuration error, not {error:?}"); + }; + assert!( + message.contains("does not satisfy the deletion contract"), + "the refusal must name the contract it failed: {message}" + ); + assert!( + message.contains("object versioning is enabled") || message.contains("soft-delete"), + "the refusal must name the violating setting: {message}" + ); + eprintln!("misconfigured bucket refused: {error}"); +} diff --git a/crates/buzz-object-store/tests/gcs_scale.rs b/crates/buzz-object-store/tests/gcs_scale.rs new file mode 100644 index 00000000000..2e0be997c59 --- /dev/null +++ b/crates/buzz-object-store/tests/gcs_scale.rs @@ -0,0 +1,586 @@ +//! Store-level scale evidence for the Google Cloud Storage provider. +//! +//! These arms answer the questions the functional suite cannot: whether the +//! provider's behaviour holds when many writers run at once, whether the +//! compare-and-swap contract still admits exactly one writer per round when +//! that round is repeated dozens of times, and what latency and outcome mix a +//! sustained load actually produces. +//! +//! They cost minutes and thousands of requests, so they are gated twice — +//! `BUZZ_GCS_LIVE=1` like the functional suite, plus `BUZZ_GCS_SCALE=1`: +//! +//! ```bash +//! BUZZ_GCS_LIVE=1 BUZZ_GCS_SCALE=1 \ +//! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ +//! cargo test -p buzz-object-store --test gcs_scale -- --ignored --nocapture +//! ``` +//! +//! Every knob has an environment override so a rehearsal can widen a run +//! without editing the source; the defaults are what a routine run should cost. +//! +//! ## What these arms measure, and what they do not +//! +//! They are *store-level* evidence: pointer objects and blobs through the +//! object-store seam. They are not a repository-shaped corpus — no packs of +//! realistic size, no ref counts, no cold materialisation — and they should not +//! be read as one. +//! +//! Throttling deserves one note. The provider absorbs 429 inside its own +//! bounded policy, so a caller running the production configuration sees almost +//! none; a caller-visible count near zero here is evidence that pacing works, +//! not that the service never throttled. The functional suite's pacing arm +//! drives the same key through a single-attempt client precisely so the 429s +//! become visible and countable. + +use std::collections::HashMap; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures_util::FutureExt; + +use buzz_object_store::{ + ConditionalWrite, GcsObjectStore, GcsStoreConfig, ObjectStore, ObjectStoreError, Revision, + WriteCondition, +}; + +const CONTENT_TYPE: &str = "application/octet-stream"; + +/// How many pointer creates the distinct-name arm issues at once. +const DEFAULT_DISTINCT_WRITERS: usize = 500; +/// How many hot-pointer contention rounds to run. +const DEFAULT_CONTENTION_ROUNDS: usize = 25; +/// How many writers race for the same pointer in one round. +const DEFAULT_CONTENTION_WIDTH: usize = 3; +/// How long the sustained mixed-load arm runs. +const DEFAULT_MIXED_LOAD_SECONDS: u64 = 120; +/// How many independent workers the mixed-load arm runs. +const DEFAULT_MIXED_LOAD_WORKERS: usize = 8; +/// How long each mixed-load worker waits between operations. +const DEFAULT_MIXED_LOAD_INTERVAL_MS: u64 = 250; + +/// Just past Cloud Storage's documented one-write-per-second per-object +/// ceiling, so a contention round measures the conditional-write semantics +/// rather than the rate limiter. +const SAME_KEY_SPACING: Duration = Duration::from_millis(1_100); + +/// Read a `usize` knob from the environment. +fn knob(variable: &str, default: usize) -> usize { + std::env::var(variable) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +/// Connect to the test bucket, or `None` when this environment has not opted +/// in to the scale arms. +async fn scale_store(test: &str) -> Option<(Arc, String)> { + // The relay installs this in `main` before any TLS request. These tests are + // their own process and must do the same: both ring and aws-lc-rs are in + // the build graph, so rustls refuses to pick one on its own. + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); + + for (variable, value) in [("BUZZ_GCS_LIVE", "1"), ("BUZZ_GCS_SCALE", "1")] { + if std::env::var(variable).as_deref() != Ok(value) { + eprintln!("skipping {test}: set {variable}={value} to run the scale arms"); + return None; + } + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_SCALE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + let store = Arc::new( + GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"), + ); + Some((store, format!("a4-scale/{}/{test}", uuid::Uuid::new_v4()))) +} + +/// Run one scale arm, then remove everything it wrote whether it passed or not. +/// +/// These arms write hundreds to thousands of objects, so a leak on the failure +/// path is not a rounding error — and a failing arm is the one that gets re-run. +async fn with_prefix(test: &str, body: F) +where + F: AsyncFnOnce(&Arc, &str), +{ + let Some((store, prefix)) = scale_store(test).await else { + return; + }; + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(&prefix, token, 1000) + .await + .expect("list for cleanup"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + break; + } + } + let removed = keys.len(); + let cleanup = if keys.is_empty() { + None + } else { + Some(store.delete_objects(&keys).await) + }; + + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } + if let Some(cleanup) = cleanup { + let cleanup = cleanup.expect("bulk delete for cleanup"); + assert!( + cleanup.failed.is_empty(), + "cleanup left objects behind: {:?}", + cleanup.failed + ); + } + let leaked = store + .list_page(&prefix, None, 10) + .await + .expect("verify the namespace is empty"); + assert!( + leaked.objects.is_empty(), + "the namespace must be empty after cleanup: {:?}", + leaked.objects + ); + eprintln!("{test}: removed {removed} objects, namespace verified empty"); +} + +/// Latency samples for one class of operation. +#[derive(Default)] +struct Latencies(Vec); + +impl Latencies { + fn record(&mut self, elapsed: Duration) { + self.0.push(elapsed); + } + + /// The sample at `fraction` through the sorted distribution. + /// + /// Nearest-rank, so a small sample reports a real observation rather than + /// an interpolation between two of them. + fn percentile(&mut self, fraction: f64) -> Duration { + if self.0.is_empty() { + return Duration::ZERO; + } + self.0.sort_unstable(); + let rank = ((self.0.len() as f64) * fraction).ceil() as usize; + self.0[rank.clamp(1, self.0.len()) - 1] + } + + fn summary(&mut self, label: &str) -> String { + let count = self.0.len(); + let p50 = self.percentile(0.50); + let p95 = self.percentile(0.95); + let p99 = self.percentile(0.99); + format!("{label}: n={count} p50={p50:?} p95={p95:?} p99={p99:?}") + } +} + +/// How one attempt was answered, folded across a whole run. +#[derive(Default, Debug, PartialEq, Eq)] +struct Outcomes { + committed: u64, + conflicts: u64, + reads: u64, + lists: u64, + throttled: u64, + retryable: u64, + ambiguous: u64, + terminal: u64, +} + +impl Outcomes { + /// Fold a provider error into the counter its classification names. + fn record_error(&mut self, error: &ObjectStoreError) { + match error { + ObjectStoreError::Throttled { .. } => self.throttled += 1, + ObjectStoreError::TransportRetryable { .. } => self.retryable += 1, + ObjectStoreError::TransportAmbiguous { .. } => self.ambiguous += 1, + _ => self.terminal += 1, + } + } +} + +/// Concurrent creates across distinct object names: the axis Cloud Storage +/// scales horizontally on. +/// +/// The per-object write ceiling applies to one object name. Independent +/// repositories publish to independent pointers, so this is the shape that has +/// to scale — and every writer must commit, because none of them is contending +/// with another. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn concurrent_creates_across_distinct_names_all_commit() { + let writers = knob("BUZZ_GCS_SCALE_DISTINCT_WRITERS", DEFAULT_DISTINCT_WRITERS); + + with_prefix("distinct-names", async |store, prefix| { + let started = Instant::now(); + let mut tasks = Vec::with_capacity(writers); + for i in 0..writers { + let store = Arc::clone(store); + let key = format!("{prefix}/pointers/{i:05}"); + tasks.push(tokio::spawn(async move { + let attempt = Instant::now(); + let outcome = store + .put_conditional( + &key, + format!("writer-{i}").as_bytes(), + CONTENT_TYPE, + WriteCondition::Absent, + ) + .await; + (outcome, attempt.elapsed()) + })); + } + + let mut outcomes = Outcomes::default(); + let mut latencies = Latencies::default(); + let mut revisions = Vec::with_capacity(writers); + for task in tasks { + let (outcome, elapsed) = task.await.expect("writer task"); + latencies.record(elapsed); + match outcome { + Ok(ConditionalWrite::Committed(revision)) => { + outcomes.committed += 1; + revisions.push(revision); + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => outcomes.record_error(&error), + } + } + let elapsed = started.elapsed(); + + assert_eq!( + outcomes.committed, writers as u64, + "every writer addressed its own object name, so every one must commit: {outcomes:?}" + ); + assert_eq!( + revisions.len(), + writers, + "a commit without a revision leaves the caller nothing to predicate its next write on" + ); + + eprintln!( + "distinct names: {writers} concurrent creates in {elapsed:?} ({:.0}/sec), {}", + writers as f64 / elapsed.as_secs_f64(), + latencies.summary("create") + ); + eprintln!("distinct names: outcomes {outcomes:?}"); + }) + .await; +} + +/// The hot-pointer race, repeated: exactly one winner per round, every round, +/// and the published state is always the winner's. +/// +/// One round proves the contract holds; repeating it is what would surface a +/// rare double commit or a lost update. The second assertion is the one that +/// catches a silent lost update — a round can report one winner and still have +/// published a loser's body if the backend committed out of order. +/// +/// Rounds are spaced past the published per-object ceiling on purpose: this arm +/// is measuring conditional-write semantics under contention, and a deliberately +/// over-rate round would only measure the rate limiter. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn a_repeated_hot_pointer_race_never_admits_two_winners() { + let rounds = knob( + "BUZZ_GCS_SCALE_CONTENTION_ROUNDS", + DEFAULT_CONTENTION_ROUNDS, + ); + let width = knob("BUZZ_GCS_SCALE_CONTENTION_WIDTH", DEFAULT_CONTENTION_WIDTH); + assert!(width >= 2, "a race needs at least two entrants"); + + with_prefix("hot-pointer", async |store, prefix| { + let key = format!("{prefix}/pointers/hot"); + + let ConditionalWrite::Committed(mut revision) = store + .put_conditional(&key, b"round-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let mut outcomes = Outcomes::default(); + let mut latencies = Latencies::default(); + let mut last_round_end = Instant::now(); + let started = Instant::now(); + + for round in 1..=rounds { + // Space same-key rounds past the documented ceiling. + let since = last_round_end.elapsed(); + if since < SAME_KEY_SPACING { + tokio::time::sleep(SAME_KEY_SPACING - since).await; + } + + let mut racers = Vec::with_capacity(width); + for racer in 0..width { + let store = Arc::clone(store); + let key = key.clone(); + let revision = revision.clone(); + let body = format!("round-{round}-racer-{racer}"); + racers.push(tokio::spawn(async move { + let attempt = Instant::now(); + let outcome = store + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision), + ) + .await; + (body, outcome, attempt.elapsed()) + })); + } + + let mut winners: Vec<(String, Revision)> = Vec::new(); + for racer in racers { + let (body, outcome, elapsed) = racer.await.expect("racer task"); + latencies.record(elapsed); + match outcome { + Ok(ConditionalWrite::Committed(next)) => { + outcomes.committed += 1; + winners.push((body, next)); + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => { + // Throttling is a refusal to evaluate the precondition, + // so it is never evidence of a lost race. + outcomes.record_error(&error); + } + } + } + last_round_end = Instant::now(); + + assert_eq!( + winners.len(), + 1, + "round {round} admitted {} winners on one revision — two committed writers on the \ + same precondition is a lost update announcing itself", + winners.len() + ); + let (winning_body, winning_revision) = winners.pop().expect("exactly one winner"); + assert_ne!( + winning_revision, revision, + "round {round} committed without minting a new revision" + ); + + // The published state must be the winner's, not a loser's: one + // acknowledged winner over somebody else's bytes is exactly the + // silent lost update this arm exists to rule out. + let (published_revision, published) = store + .get_with_revision(&key) + .await + .expect("read the published state") + .expect("the pointer exists"); + assert_eq!( + published.as_ref(), + winning_body.as_bytes(), + "round {round} published a body no acknowledged winner wrote" + ); + assert_eq!( + published_revision, winning_revision, + "round {round} published a revision the winner did not commit" + ); + revision = winning_revision; + } + let elapsed = started.elapsed(); + + assert_eq!( + outcomes.committed, rounds as u64, + "exactly one winner per round: {outcomes:?}" + ); + + eprintln!( + "hot pointer: {rounds} rounds of width {width} in {elapsed:?}, {}", + latencies.summary("cas") + ); + eprintln!("hot pointer: outcomes {outcomes:?}"); + }) + .await; +} + +/// A sustained mixed read/write/list load at a bounded rate. +/// +/// Each worker owns its own pointer, which is the deployment's real shape: +/// independent repositories publish to independent object names, so a worker's +/// compare-and-swap chain is sequential on its own object and stays inside the +/// per-object ceiling while the fleet as a whole is concurrent. +/// +/// This arm is a measurement, not a threshold. It asserts only what must never +/// happen — a writer losing a race it was the only entrant in, or a terminal +/// error — and reports the rest for the record. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn sustained_mixed_load_reports_its_latency_and_outcome_mix() { + let seconds = knob( + "BUZZ_GCS_SCALE_MIXED_SECONDS", + DEFAULT_MIXED_LOAD_SECONDS as usize, + ) as u64; + let workers = knob("BUZZ_GCS_SCALE_MIXED_WORKERS", DEFAULT_MIXED_LOAD_WORKERS); + let interval = Duration::from_millis(knob( + "BUZZ_GCS_SCALE_MIXED_INTERVAL_MS", + DEFAULT_MIXED_LOAD_INTERVAL_MS as usize, + ) as u64); + + with_prefix("mixed-load", async |store, prefix| { + let deadline = Instant::now() + Duration::from_secs(seconds); + + let mut tasks = Vec::with_capacity(workers); + for worker in 0..workers { + let store = Arc::clone(store); + let pointer = format!("{prefix}/pointers/worker-{worker:03}"); + let blob_prefix = format!("{prefix}/blobs/worker-{worker:03}"); + tasks.push(tokio::spawn(async move { + let mut outcomes = Outcomes::default(); + let mut latencies: HashMap<&'static str, Latencies> = HashMap::new(); + + let ConditionalWrite::Committed(mut revision) = store + .put_conditional(&pointer, b"seed", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("seed the worker's pointer") + else { + panic!("creating an absent pointer must commit"); + }; + outcomes.committed += 1; + + let mut step = 0u64; + while Instant::now() < deadline { + step += 1; + // 4 reads : 2 writes : 1 list. Reads dominate because + // serving a repository resolves its pointer per request. + let attempt = Instant::now(); + match step % 7 { + 0 => { + let outcome = store.list_page(&blob_prefix, None, 100).await; + latencies + .entry("list") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(_) => outcomes.lists += 1, + Err(error) => outcomes.record_error(&error), + } + } + 1 | 3 => { + // An immutable blob write, then a pointer swap + // publishing it — the two writes a push performs. + let blob = format!("{blob_prefix}/{step:06}"); + let outcome = store + .put_immutable( + &blob, + format!("blob-{step}").as_bytes(), + CONTENT_TYPE, + ) + .await; + latencies + .entry("write") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(_) => outcomes.committed += 1, + Err(error) => outcomes.record_error(&error), + } + + let swap = Instant::now(); + let outcome = store + .put_conditional( + &pointer, + format!("step-{step}").as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision.clone()), + ) + .await; + latencies.entry("cas").or_default().record(swap.elapsed()); + match outcome { + Ok(ConditionalWrite::Committed(next)) => { + outcomes.committed += 1; + revision = next; + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => outcomes.record_error(&error), + } + } + _ => { + let outcome = store.get_with_revision(&pointer).await; + latencies + .entry("read") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(Some(_)) => outcomes.reads += 1, + Ok(None) => panic!("the worker's own pointer disappeared"), + Err(error) => outcomes.record_error(&error), + } + } + } + + let spent = attempt.elapsed(); + if spent < interval { + tokio::time::sleep(interval - spent).await; + } + } + + (outcomes, latencies) + })); + } + + let started = Instant::now(); + let mut totals = Outcomes::default(); + let mut latencies: HashMap<&'static str, Latencies> = HashMap::new(); + for task in tasks { + let (outcomes, worker_latencies) = task.await.expect("mixed-load worker"); + totals.committed += outcomes.committed; + totals.conflicts += outcomes.conflicts; + totals.reads += outcomes.reads; + totals.lists += outcomes.lists; + totals.throttled += outcomes.throttled; + totals.retryable += outcomes.retryable; + totals.ambiguous += outcomes.ambiguous; + totals.terminal += outcomes.terminal; + for (class, samples) in worker_latencies { + latencies.entry(class).or_default().0.extend(samples.0); + } + } + let elapsed = started.elapsed(); + + assert_eq!( + totals.conflicts, 0, + "every worker owns its own pointer, so no worker can lose a race: {totals:?}" + ); + assert_eq!( + totals.terminal, 0, + "a terminal error under nominal load is a failure, not a measurement: {totals:?}" + ); + + let operations = totals.committed + totals.reads + totals.lists; + eprintln!( + "mixed load: {workers} workers for {elapsed:?}, {operations} operations \ + ({:.1}/sec), outcomes {totals:?}", + operations as f64 / elapsed.as_secs_f64() + ); + let mut classes: Vec<&'static str> = latencies.keys().copied().collect(); + classes.sort_unstable(); + for class in classes { + eprintln!( + "mixed load: {}", + latencies.get_mut(class).expect("class").summary(class) + ); + } + }) + .await; +} diff --git a/crates/buzz-relay/src/api/git/store.rs b/crates/buzz-relay/src/api/git/store.rs index b4fac01b9aa..6a4082e7965 100644 --- a/crates/buzz-relay/src/api/git/store.rs +++ b/crates/buzz-relay/src/api/git/store.rs @@ -140,6 +140,15 @@ pub struct ProbeConfig { /// store's conditional-write semantics, and deliberately violating the /// published rate limit would only prove that the rate limiter works. pub same_key_spacing: Duration, + /// Key namespace for the objects the probe writes for itself. + /// + /// The default reproduces the keys the probe has always used. Overriding it + /// gives a run its own disposable namespace, which is what lets a test + /// prove the cleanup pass emptied it. (The S3 profile's content-addressed + /// phase writes a pack object under the store's own `packs/` namespace by + /// construction — it exercises the production write path — and is not + /// affected by this setting.) + pub key_prefix: String, } impl ProbeConfig { @@ -151,6 +160,7 @@ impl ProbeConfig { race_rounds: 3, unproven_round_retries: 3, same_key_spacing: Duration::ZERO, + key_prefix: DEFAULT_PROBE_KEY_PREFIX.to_string(), }, ProviderKind::Gcs => Self { race_width: 3, @@ -159,11 +169,21 @@ impl ProbeConfig { // Just past Cloud Storage's documented one-write-per-second // per-object ceiling. same_key_spacing: Duration::from_millis(1_100), + key_prefix: DEFAULT_PROBE_KEY_PREFIX.to_string(), }, } } + + /// Run under `prefix` instead of the default namespace. + pub fn under_key_prefix(mut self, prefix: impl Into) -> Self { + self.key_prefix = prefix.into(); + self + } } +/// Namespace the probe writes its own objects under. +const DEFAULT_PROBE_KEY_PREFIX: &str = "probe"; + impl Default for ProbeConfig { fn default() -> Self { Self::for_provider(ProviderKind::S3) @@ -625,7 +645,7 @@ impl GitStore { async fn run_s3_conformance_probe(&self, cfg: ProbeConfig) -> Result { use std::sync::Arc; let nonce = uuid::Uuid::new_v4(); - let pointer_key = format!("probe/pointer-{nonce}"); + let pointer_key = format!("{}/pointer-{nonce}", cfg.key_prefix); // Accumulator for *transport-unknown* per-racer outcomes across both // race phases. See `ProbeReport::transport_drops` for the rationale. let mut transport_drops = 0usize; @@ -762,7 +782,7 @@ impl GitStore { // Bypass `put_immutable`'s collision-swallow to count raw outcomes. for round in 0..cfg.race_rounds { let body = format!("probe-inm-race-{nonce}-{round}").into_bytes(); - let key = Self::content_key("probe/inm-race", &body); + let key = Self::content_key(&format!("{}/inm-race", cfg.key_prefix), &body); // Clean slate. let _ = self.store.delete(&key).await; let arc_self: Arc<&Self> = Arc::new(self); @@ -980,7 +1000,7 @@ impl GitStore { // The nonce makes this key new, so the create-only write must report a // create rather than a collision. let body = format!("probe-gcs-immutable-{nonce}").into_bytes(); - let key = Self::content_key("probe/gcs-immutable", &body); + let key = Self::content_key(&format!("{}/gcs-immutable", cfg.key_prefix), &body); written.push(key.clone()); let outcome = self .put_immutable_raw(&key, &body) @@ -1012,7 +1032,7 @@ impl GitStore { } // -- Phase 2: pointer_create ------------------------------------------- - let pointer_key = format!("probe/gcs-pointer-{nonce}"); + let pointer_key = format!("{}/gcs-pointer-{nonce}", cfg.key_prefix); written.push(pointer_key.clone()); let seed = format!("probe-gcs-pointer-seed-{nonce}").into_bytes(); state.pace(cfg.same_key_spacing).await; @@ -1865,8 +1885,8 @@ mod profiles { ProbeConfig { race_width: 3, race_rounds, - unproven_round_retries: 3, same_key_spacing: Duration::ZERO, + ..ProbeConfig::for_provider(ProviderKind::Gcs) } } @@ -2375,3 +2395,444 @@ mod probe { let _ = st.store.delete(&key).await; } } + +#[cfg(test)] +mod gcs_live { + //! The Cloud Storage profile against a real bucket. + //! + //! This is the deployment gate itself, run against the provider it was + //! written for — the scripted-store tests prove how the profile *judges* + //! answers, and only a live bucket proves which answers Cloud Storage + //! actually gives. + //! + //! Talks to a real bucket, so it is `#[ignore]`d and additionally gated on + //! `BUZZ_GCS_LIVE=1`; a bare `--ignored` run without credentials skips + //! rather than fails. + //! + //! ```bash + //! BUZZ_GCS_LIVE=1 \ + //! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ + //! cargo test -p buzz-relay --lib api::git::store::gcs_live -- --ignored --nocapture + //! ``` + //! + //! Credentials come from Application Default Credentials, exactly as in + //! production. Every test works under its own `a3//` namespace: the + //! probe run asserts the probe's own cleanup pass emptied it, and the + //! git-facade runs delete what they wrote whether they passed or not. + + use std::panic::AssertUnwindSafe; + + use buzz_object_store::{GcsObjectStore, GcsStoreConfig}; + use futures_util::FutureExt; + + use super::*; + + /// Every key left under `prefix`. + async fn remaining(store: &Arc, prefix: &str) -> Vec { + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(prefix, token, 1000) + .await + .expect("list the probe namespace"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + return keys; + } + } + } + + /// Connect to the test bucket, or `None` when this environment has not + /// opted in to live tests. + async fn live_backend() -> Option> { + // The relay installs this in `main` before any TLS request; a test + // binary must do the same, because both ring and aws-lc-rs are in the + // build graph and rustls will not choose between them. + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); + + if std::env::var("BUZZ_GCS_LIVE").as_deref() != Ok("1") { + eprintln!("skipping: set BUZZ_GCS_LIVE=1 to run against a live bucket"); + return None; + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_LIVE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + // Connecting runs the provider's admission check: a bucket with + // versioning or soft delete never reaches these tests at all. + Some(Arc::new( + GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"), + )) + } + + /// Run one git-facade test under its own namespace, then remove what it + /// wrote whether it passed or not. + /// + /// A panicking body would otherwise leak into a bucket shared with every + /// other run — and a failing test is precisely the one that gets re-run. + async fn with_git_store(test: &str, body: F) + where + F: AsyncFnOnce(&GitStore, &str), + { + let Some(backend) = live_backend().await else { + return; + }; + let prefix = format!("a4/{}/{test}", uuid::Uuid::new_v4()); + let store = GitStore::new(backend.clone()); + + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + + let keys = remaining(&backend, &prefix).await; + let cleanup = if keys.is_empty() { + Ok(Default::default()) + } else { + backend.delete_objects(&keys).await + }; + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } + let cleanup = cleanup.expect("bulk delete for cleanup"); + assert!( + cleanup.failed.is_empty(), + "cleanup left objects behind: {:?}", + cleanup.failed + ); + assert!( + remaining(&backend, &prefix).await.is_empty(), + "the namespace must be empty after cleanup" + ); + } + + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn the_cloud_storage_profile_admits_a_real_bucket() { + let Some(backend) = live_backend().await else { + return; + }; + let prefix = format!("a3/{}", uuid::Uuid::new_v4()); + let cfg = ProbeConfig::for_provider(ProviderKind::Gcs).under_key_prefix(&prefix); + let spacing = cfg.same_key_spacing; + + let report = GitStore::new(backend.clone()) + .run_conformance_probe(cfg) + .await + .expect("Cloud Storage satisfies the conditional-write axioms"); + eprintln!("✓ probe report: {report:?}"); + + assert_eq!(report.profile, ProviderKind::Gcs); + assert_eq!(report.race_width, 3); + assert_eq!(report.race_rounds, 2); + let gap = report + .min_same_key_gap + .expect("the paced profile reports the interval it observed"); + assert!( + gap >= spacing, + "same-key rounds were {gap:?} apart, inside the configured {spacing:?}" + ); + + // The bucket is shared with other runs, so an unremoved probe object + // would be invisible until it was large. + let leaked = remaining(&backend, &prefix).await; + assert_eq!(report.cleanup_failures, 0, "cleanup reported failures"); + assert!(leaked.is_empty(), "probe objects left behind: {leaked:?}"); + } + + /// The publication cycle a push performs, against a real bucket. + /// + /// Create the pointer under the create-only precondition, read body and + /// revision from one response, swap on the revision that read returned, and + /// then replay the superseded revision — which must lose rather than + /// overwrite. The last assertion is the one that matters most: an absent + /// pointer reads as `None`, distinct from every failure, because a pointer + /// read that could not be answered must never be mistaken for an empty + /// repository. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn the_pointer_cycle_publishes_exactly_one_state() { + with_git_store("pointer-cycle", async |store, prefix| { + let key = format!("{prefix}/pointers/repo"); + + assert!( + store + .get_pointer(&key) + .await + .expect("reading an absent pointer is not an error") + .is_none(), + "an unpublished repository is `None`, never an error and never empty bytes" + ); + + let ConditionalWrite::Committed(created) = store + .put_pointer(&key, b"{\"refs\":0}", WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let (observed, body) = store + .get_pointer(&key) + .await + .expect("pointer read") + .expect("the pointer exists"); + assert_eq!(body.as_ref(), b"{\"refs\":0}"); + assert_eq!( + observed, created, + "the revision a read reports must be the one the write committed" + ); + + let ConditionalWrite::Committed(swapped) = store + .put_pointer( + &key, + b"{\"refs\":1}", + WriteCondition::Matches(observed.clone()), + ) + .await + .expect("swap on the observed revision") + else { + panic!("a writer holding the current revision must commit"); + }; + assert_ne!(swapped, created, "a commit mints a new revision"); + + assert_eq!( + store + .put_pointer(&key, b"{\"refs\":99}", WriteCondition::Matches(observed)) + .await + .expect("replay the superseded revision"), + ConditionalWrite::Conflict, + "a superseded revision must lose rather than overwrite" + ); + assert_eq!( + store.get(&key).await.expect("read the published state"), + &b"{\"refs\":1}"[..], + "the losing writer must not have published anything" + ); + }) + .await; + } + + /// Content addressing end to end: what a push writes, what a hydrate reads, + /// and what happens when the bytes are not what the key says they are. + /// + /// The idx sidecar is the one object here whose key is *not* its own digest + /// — it is derived from the pack digest so a hydrate can find it without + /// changing manifest bytes — so it gets its own miss/hit assertions. The + /// corruption arm is the point of the whole discipline: bytes that do not + /// hash to their key are a detected error on read, not silent corruption + /// handed to git. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn content_addressed_objects_round_trip_and_corruption_is_detected() { + with_git_store("content-addressing", async |store, prefix| { + // Production derives these keys from the bytes, so a test cannot + // place them under its own namespace without going around the + // facade. Writing them where production would and cleaning up by + // exact key keeps the write path honest. + let pack_bytes = b"PACK\x00\x00\x00\x02 pretend pack payload".to_vec(); + let pack_key = store.put_pack(&pack_bytes).await.expect("write the pack"); + let pack_digest = pack_key + .strip_prefix("packs/") + .expect("a pack key is namespaced") + .to_string(); + assert_eq!( + pack_key, + GitStore::content_key("packs", &pack_bytes), + "the facade, not the caller, derives the key from the bytes" + ); + assert_eq!( + store + .get_verified(&pack_key, &pack_digest) + .await + .expect("verified read"), + pack_bytes, + "a hydrate reads the pack back byte-exact" + ); + + assert!( + store + .get_idx(&pack_digest, 1 << 20) + .await + .expect("an idx miss is not an error") + .is_none(), + "a missing idx is a cache miss the hydrate regenerates" + ); + let idx_bytes = b"\xfftOc\x00\x00\x00\x02 pretend idx".to_vec(); + let idx_key = store + .put_idx(&pack_digest, &idx_bytes) + .await + .expect("write the idx sidecar"); + assert_eq!(idx_key, format!("idx/{pack_digest}")); + assert_eq!( + store + .get_idx(&pack_digest, 1 << 20) + .await + .expect("idx read") + .expect("the idx exists"), + idx_bytes + ); + + let manifest_bytes = format!("{{\"packs\":[\"{pack_key}\"]}}").into_bytes(); + let manifest_key = store + .put_manifest(&manifest_bytes) + .await + .expect("write the manifest"); + let manifest_digest = manifest_key + .strip_prefix("manifests/") + .expect("a manifest key is namespaced") + .to_string(); + assert_eq!( + store + .get_verified_limited(&manifest_key, &manifest_digest, 1 << 20) + .await + .expect("bounded verified read"), + manifest_bytes + ); + assert!( + matches!( + store + .get_verified_limited(&manifest_key, &manifest_digest, 4) + .await, + Err(StoreError::ObjectTooLarge { .. }) + ), + "a bounded read rejects an oversized object before transferring it" + ); + + assert!( + matches!( + store.get(&format!("{prefix}/packs/absent")).await, + Err(StoreError::NotFound(_)) + ), + "a missing object is a not-found, never empty bytes" + ); + + // Corruption: bytes that do not hash to the key they sit under. + // Production cannot produce this (writes are create-only and the + // facade derives the key), so the test writes through the backend + // directly — which is exactly the shape of a backend that returned + // the wrong object. + let claimed = b"the bytes this key names".to_vec(); + let corrupt_key = GitStore::content_key(&format!("{prefix}/packs"), &claimed); + let claimed_digest = corrupt_key + .rsplit('/') + .next() + .expect("a content key ends in its digest") + .to_string(); + store + .store + .put(&corrupt_key, b"different bytes", "application/x-git-pack") + .await + .expect("plant mismatched bytes"); + + let error = store + .get_verified(&corrupt_key, &claimed_digest) + .await + .expect_err("a verified read must reject bytes that do not hash to their key"); + let StoreError::DigestMismatch { + key, + expected, + actual, + } = &error + else { + panic!("corruption must surface as a digest mismatch, not {error:?}"); + }; + assert_eq!(key, &corrupt_key); + assert_eq!(expected, &claimed_digest); + assert_ne!(actual, expected, "the report names the digest it computed"); + + // Only the corruption arm's key is under the test namespace; the + // rest live where production writes them, so remove them by hand. + let planted = [pack_key, idx_key, manifest_key]; + let outcome = store + .store + .delete_objects(&planted) + .await + .expect("remove the objects written outside the test namespace"); + assert!(outcome.failed.is_empty(), "{:?}", outcome.failed); + }) + .await; + } + + /// A chunked repository seed: one writer, one pointer, many sequential + /// transitions, faster than the published per-object write ceiling. + /// + /// This is the mirror's initial seed of a large repository — roughly 4.5k + /// refs at 200 refs per chunk, each chunk a pointer transition predicated + /// on the revision the previous one committed. Cloud Storage documents one + /// write per second to a single object name, so the sequence deliberately + /// exceeds it. Every transition must still commit: throttling is pacing, + /// absorbed inside the provider's bounded policy, and must never reach the + /// caller as a failed push or be mistaken for a lost race in a sequence + /// that has exactly one entrant. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn a_chunked_seed_commits_every_sequential_transition() { + /// One more than the ~23 chunks a 4.5k-ref seed needs. + const CHUNKS: usize = 24; + + with_git_store("chunked-seed", async |store, prefix| { + let key = format!("{prefix}/pointers/seed"); + + let ConditionalWrite::Committed(mut revision) = store + .put_pointer(&key, b"{\"chunk\":0,\"refs\":0}", WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let mut revisions = vec![revision.clone()]; + let started = Instant::now(); + for chunk in 1..=CHUNKS { + let body = format!("{{\"chunk\":{chunk},\"refs\":{}}}", chunk * 200); + let outcome = store + .put_pointer( + &key, + body.as_bytes(), + WriteCondition::Matches(revision.clone()), + ) + .await; + revision = match outcome { + Ok(ConditionalWrite::Committed(next)) => next, + Ok(ConditionalWrite::Conflict) => panic!( + "chunk {chunk} lost a race it was the only entrant in — throttling was \ + misclassified as a lost CAS" + ), + Err(StoreError::Backend(ObjectStoreError::Throttled { .. })) => panic!( + "chunk {chunk} surfaced throttling as a push failure; pacing belongs \ + inside the provider's bounded policy, never at the caller" + ), + Err(other) => panic!("chunk {chunk} failed: {other}"), + }; + revisions.push(revision.clone()); + } + let elapsed = started.elapsed(); + + let mut distinct = revisions.clone(); + distinct.dedup(); + assert_eq!( + distinct.len(), + revisions.len(), + "every transition must publish a fresh revision" + ); + assert_eq!( + store.get(&key).await.expect("read the published state"), + &format!("{{\"chunk\":{CHUNKS},\"refs\":{}}}", CHUNKS * 200).into_bytes()[..], + "the last chunk must be the published state" + ); + + eprintln!( + "chunked seed: {CHUNKS} sequential pointer transitions committed in {elapsed:?} \ + ({:.2} transitions/sec), zero failures", + CHUNKS as f64 / elapsed.as_secs_f64() + ); + }) + .await; + } +} diff --git a/docs/git-on-object-storage.md b/docs/git-on-object-storage.md index 1d87720edc7..40476137d8e 100644 --- a/docs/git-on-object-storage.md +++ b/docs/git-on-object-storage.md @@ -368,6 +368,51 @@ conditional writes cannot admit a backend against it. a quote mismatch between the read path and the write path silently tests the wrong thing. The probe must use the exact token format the pointer write uses. +### Provider profiles + +The axioms are the same for every backend; the *evidence* that admits one is +provider-shaped, so the probe runs the profile the configured provider needs. + +The four phases above are the **S3 profile**: a wide race (default 32 writers × +3 rounds) on revision-token preconditions, with unclassified transport failures +dropped from the observer set and a floor of two classified observers per round. + +A backend that publishes a per-object write ceiling needs a different shape. +Google Cloud Storage documents a maximum of one write per second to a single +object name and answers an over-rate write with `429` — a refusal to evaluate +the precondition at all, which is categorically not a lost race. Running the S3 +profile against it would measure the rate limiter rather than the store: a burst +of `429`s would either be scored as losers, making a correctly paced backend look +like one admitting lost updates, or would leave a round with no race in it. The +**Cloud Storage profile** therefore races narrowly (default 3 × 2), spaces +same-key rounds past the published ceiling, and adds the phases that only exist +where the revision is an object generation: + +1. create-only content-addressed write, read back, digest verified; +2. pointer creation under the create-only precondition (generation `0`); +3. body and generation read from one response, checked against the commit; +4. compare-and-swap on the observed generation, which must report a new one; +5. replay of the superseded generation, which must conflict; +6. a race on one generation, repeated, with the stored object required to equal + the winner's; +7. the winning generation predicating the next successful write. + +Three rules stop pacing from becoming leniency: + +- **Two committed racers is always fatal.** No throttle, drop, or retry budget + can explain it. +- **A commit reported without a generation is fatal**, wherever it appears. The + caller would have nothing to predicate its next write on, and dropping the + precondition turns the pointer swap into a blind overwrite. +- **A round that proves nothing is re-run, not scored.** Every racer throttled, + or too few classified to have witnessed a race, is neither pass nor fail; + the round is re-run within a bounded budget, and exhausting it fails the probe. + Conflicts with no acknowledged winner stay fatal — that is a lost update + announcing itself. + +Both profiles remain fail-closed deployment gates, and both remove their own +objects afterwards on the success and failure paths alike. + **Proof surface (explicit non-goals of the probe and the design).** The protocol depends only on conditional writes of *small single objects* (the manifest pointer). It does **not** depend on, and the probe does **not** test: @@ -413,6 +458,18 @@ knowledge, new. ## Implementation Correspondence +> **A note on vocabulary.** This specification states A1 and A3 in S3 terms — +> ETags, `If-Match`, `If-None-Match: *` — because that is the backend the +> axioms were first admitted against. The implementation states them in +> provider-neutral terms: `crates/buzz-object-store` defines an opaque +> `Revision` (the CAS token), `WriteCondition::{Absent, Matches}` (the +> precondition), and `ConditionalWrite::{Committed, Conflict}` (the outcome). +> An ETag is one inhabitant of `Revision` and exists only inside the S3 +> provider. The mapping is exact — `Absent` is `If-None-Match: *`, `Matches(e)` +> is `If-Match: e`, `Conflict` is the 412 — so every theorem below transfers +> unchanged; a backend is admitted by the §Conformance probe, not by speaking +> S3's header vocabulary. + The fence (Theorem 1) maps to a single structural obligation on the implementation, stated here as a requirement the code must meet for the proof to transfer: @@ -434,7 +491,7 @@ transfer: - **Parent observed once.** `hydrate_for_write` reads the pointer, fetches and verifies the parent manifest, materializes the workspace from it, and returns a `(HydratedRepo, ParentState)` pair where `ParentState` carries the exact - `(ETag, digest, Manifest)` triple the workspace was hydrated against. That + `(Revision, digest, Manifest)` triple the workspace was hydrated against. That same `ParentState` rides on the `PushContext` through receive-pack, and `cas_publish` predicates the CAS on `parent_state.if_match` — it never re-reads the pointer. The "build on `d_old`, publish against `d_new`" hazard is closed From 3f545eb0ef4ceec7016e23a42c1fd736b46badc2 Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 31 Aug 2026 05:29:16 -0700 Subject: [PATCH 6/7] refactor(object-store): isolate provider composition Keep media, Git, and deletion domain behavior behind ObjectStore while relay and standalone process roots select GCS or S3. Add provider-neutral exact-version deletion, fail-closed provider tokens, and an explicit GCS-to-S3 cutover proof. Signed-off-by: mozarthq --- Cargo.lock | 2 + crates/buzz-deletion/Cargo.toml | 1 + crates/buzz-deletion/src/lib.rs | 118 +++++++++--------- crates/buzz-media/src/config.rs | 70 +---------- crates/buzz-media/src/lib.rs | 2 +- crates/buzz-media/src/storage.rs | 59 +-------- crates/buzz-media/src/upload.rs | 6 - crates/buzz-media/src/validation.rs | 6 - crates/buzz-media/tests/static_creds_minio.rs | 32 ++--- crates/buzz-media/tests/versioned_minio.rs | 39 +++--- crates/buzz-object-store/src/providers/gcs.rs | 105 ++++++++++------ crates/buzz-object-store/src/providers/s3.rs | 71 ++++++++++- crates/buzz-object-store/tests/gcs_live.rs | 39 +++++- crates/buzz-relay/src/api/admin/mod.rs | 18 +-- crates/buzz-relay/src/api/bridge.rs | 2 +- crates/buzz-relay/src/api/gifs.rs | 3 +- crates/buzz-relay/src/api/git/cas_publish.rs | 4 +- crates/buzz-relay/src/api/git/hydrate.rs | 16 +-- crates/buzz-relay/src/api/git/policy.rs | 2 +- crates/buzz-relay/src/api/git/store.rs | 88 +++++-------- crates/buzz-relay/src/api/git/transport.rs | 2 +- crates/buzz-relay/src/api/invites.rs | 2 +- crates/buzz-relay/src/api/media.rs | 2 +- crates/buzz-relay/src/api/operator.rs | 4 +- crates/buzz-relay/src/config.rs | 94 ++++++++------ crates/buzz-relay/src/handlers/event.rs | 5 +- .../src/handlers/identity_archive.rs | 2 +- crates/buzz-relay/src/handlers/relay_admin.rs | 2 +- crates/buzz-relay/src/lib.rs | 42 +++++++ crates/buzz-relay/src/router.rs | 2 +- crates/buzz-relay/src/state.rs | 2 +- crates/buzz-relay/src/workflow_sink.rs | 2 +- crates/buzz-test-client/Cargo.toml | 1 + crates/buzz-test-client/tests/e2e_git.rs | 2 +- docs/git-on-object-storage.md | 35 +++++- 35 files changed, 467 insertions(+), 415 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1efd93590bc..d8f9362f65b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1098,6 +1098,7 @@ dependencies = [ "buzz-core", "buzz-db", "buzz-media", + "buzz-object-store", "chrono", "clap", "deadpool-redis", @@ -1422,6 +1423,7 @@ dependencies = [ "base64 0.22.1", "buzz-core", "buzz-media", + "buzz-object-store", "buzz-sdk", "buzz-ws-client", "chrono", diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index 8c308b0a8c0..2a3c177f994 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -13,6 +13,7 @@ thiserror = { workspace = true } buzz-core = { workspace = true } buzz-db = { workspace = true } buzz-media = { workspace = true } +buzz-object-store = { workspace = true } chrono = { workspace = true } clap = { version = "4", features = ["derive"] } deadpool-redis = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index d963a7f261f..8f0ee9d833c 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -561,27 +561,8 @@ async fn connect_services() -> Result { } async fn connect_services_with_store(store: DeletionStore) -> Result { - let (s3_access_key, s3_secret_key) = s3_key_pair_from_env(); - let media_config = buzz_media::MediaConfig { - s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, - s3_access_key, - s3_secret_key, - s3_bucket: required_env("BUZZ_S3_BUCKET")?, - s3_region: s3_region_from_env(), - s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") - .unwrap_or_else(|_| "path".to_string()) - .parse() - .map_err(anyhow::Error::msg)?, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }; - let media = Arc::new(MediaStorage::new(&media_config)?); + let object_store = buzz_object_store::connect(&deletion_object_store_config()?).await?; + let media = Arc::new(MediaStorage::with_store(object_store)); let redis_url = required_env("REDIS_URL")?; let mut redis_config = deadpool_redis::Config::from_url(&redis_url); redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( @@ -598,6 +579,37 @@ async fn connect_services_with_store(store: DeletionStore) -> Result { }) } +/// Resolve the same deployment-level provider used by the relay. +/// +/// The deletion executable is a separate composition root, so it must select +/// the provider independently while preserving exactly the relay's environment +/// contract. Domain deletion code continues to consume `MediaStorage` only. +fn deletion_object_store_config() -> Result { + match buzz_object_store::ProviderSelection::from_env().map_err(anyhow::Error::msg)? { + buzz_object_store::ProviderSelection::Gcs { bucket } => { + Ok(buzz_object_store::ObjectStoreConfig::Gcs( + buzz_object_store::GcsStoreConfig::new(bucket), + )) + } + buzz_object_store::ProviderSelection::S3 => { + let (access_key, secret_key) = s3_key_pair_from_env(); + Ok(buzz_object_store::ObjectStoreConfig::S3( + buzz_object_store::S3StoreConfig { + endpoint: required_env("BUZZ_S3_ENDPOINT")?, + access_key, + secret_key, + bucket: required_env("BUZZ_S3_BUCKET")?, + region: s3_region_from_env(), + addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + .unwrap_or_else(|_| "path".to_string()) + .parse() + .map_err(anyhow::Error::msg)?, + }, + )) + } + } +} + fn s3_region_from_env() -> String { resolve_s3_region( std::env::var("BUZZ_S3_REGION").ok(), @@ -1605,6 +1617,12 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + fn s3_media_storage(config: buzz_object_store::S3StoreConfig) -> Arc { + let store = + buzz_object_store::S3ObjectStore::new(&config).expect("construct S3 test object store"); + Arc::new(MediaStorage::with_store(Arc::new(store))) + } + #[test] fn submit_host_prefers_explicit_host() { assert_eq!( @@ -1693,25 +1711,14 @@ mod tests { .expect("runnable deletion request"); let services = Services { store, - media: Arc::new( - MediaStorage::new(&buzz_media::MediaConfig { - s3_endpoint: "http://127.0.0.1:1".to_string(), - s3_access_key: "unused".to_string(), - s3_secret_key: "unused".to_string(), - s3_bucket: "unused".to_string(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: buzz_media::S3AddressingStyle::Path, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }) - .expect("construct unused media service"), - ), + media: s3_media_storage(buzz_object_store::S3StoreConfig { + endpoint: "http://127.0.0.1:1".to_string(), + access_key: "unused".to_string(), + secret_key: "unused".to_string(), + bucket: "unused".to_string(), + region: "us-east-1".to_string(), + addressing_style: buzz_object_store::S3AddressingStyle::Path, + }), redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .expect("construct unused Redis pool"), @@ -1794,27 +1801,16 @@ mod tests { let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) .expect("BUZZ_TEST_S3_BUCKET or BUZZ_S3_BUCKET is required"); - Arc::new( - MediaStorage::new(&buzz_media::MediaConfig { - s3_endpoint: endpoint, - s3_access_key: access_key, - s3_secret_key: secret_key, - s3_bucket: bucket, - s3_region: std::env::var("BUZZ_TEST_S3_REGION") - .or_else(|_| std::env::var("BUZZ_S3_REGION")) - .unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style: buzz_media::S3AddressingStyle::Path, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }) - .expect("construct deletion test media service"), - ) + s3_media_storage(buzz_object_store::S3StoreConfig { + endpoint, + access_key, + secret_key, + bucket, + region: std::env::var("BUZZ_TEST_S3_REGION") + .or_else(|_| std::env::var("BUZZ_S3_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style: buzz_object_store::S3AddressingStyle::Path, + }) } #[tokio::test] diff --git a/crates/buzz-media/src/config.rs b/crates/buzz-media/src/config.rs index 8ca10c0161e..f1b9c9a5675 100644 --- a/crates/buzz-media/src/config.rs +++ b/crates/buzz-media/src/config.rs @@ -1,6 +1,4 @@ -//! Media storage configuration. - -pub use buzz_object_store::S3AddressingStyle; +//! Provider-neutral media behavior configuration. fn default_max_video_bytes() -> u64 { 524_288_000 // 500 MB @@ -10,32 +8,12 @@ fn default_max_file_bytes() -> u64 { 104_857_600 // 100 MB } -fn default_s3_region() -> String { - "us-east-1".to_string() -} - -/// Configuration for media storage (S3/MinIO). +/// Configuration for media validation, upload policy, and public URLs. +/// +/// Provider connection settings belong to the process composition root and +/// are deliberately absent here. #[derive(Debug, Clone, serde::Deserialize)] pub struct MediaConfig { - /// S3-compatible endpoint URL (e.g. "http://localhost:9000"). - pub s3_endpoint: String, - /// S3 access key. - pub s3_access_key: String, - /// S3 secret key. - pub s3_secret_key: String, - /// S3 bucket name. - pub s3_bucket: String, - /// AWS region for SigV4 request signing (e.g. "us-west-2"). - /// - /// Must match the region of `s3_endpoint` for real AWS S3, otherwise - /// requests are signed with the wrong credential scope and AWS rejects - /// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where - /// the value is not meaningfully checked. - #[serde(default = "default_s3_region")] - pub s3_region: String, - /// S3 URL addressing style. Defaults to path style for MinIO compatibility. - #[serde(default)] - pub s3_addressing_style: S3AddressingStyle, /// Maximum upload size for images (bytes). Default: 50 MB. pub max_image_bytes: u64, /// Maximum upload size for animated GIFs (bytes). Default: 10 MB. @@ -128,17 +106,10 @@ impl MediaConfig { #[cfg(test)] mod tests { - use super::{MediaConfig, S3AddressingStyle}; - use std::str::FromStr; + use super::MediaConfig; fn valid_config() -> MediaConfig { MediaConfig { - s3_endpoint: "http://localhost:9000".to_string(), - s3_access_key: "k".to_string(), - s3_secret_key: "s".to_string(), - s3_bucket: "buzz-media".to_string(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: S3AddressingStyle::Path, max_image_bytes: 1, max_gif_bytes: 1, max_video_bytes: 1, @@ -150,35 +121,6 @@ mod tests { } } - #[test] - fn addressing_style_parses_supported_values() { - assert_eq!( - S3AddressingStyle::from_str("path"), - Ok(S3AddressingStyle::Path) - ); - assert_eq!( - S3AddressingStyle::from_str("virtual"), - Ok(S3AddressingStyle::Virtual) - ); - } - - #[test] - fn addressing_style_defaults_to_path() { - assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path); - } - - #[test] - fn addressing_style_rejects_unknown_or_ambiguous_values() { - for invalid in ["", "auto", "PATH", "virtual-hosted"] { - let error = - S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style"); - assert!( - error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"), - "unexpected error for {invalid:?}: {error}" - ); - } - } - #[test] fn upload_record_knobs_default_off_and_validate() { assert!(valid_config().validate().is_ok()); diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 3198e1f8301..5bf598683d9 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -18,7 +18,7 @@ pub use bucket_index::{ BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, Page, SweepError, TaxonomySweepOutcome, }; -pub use config::{MediaConfig, S3AddressingStyle}; +pub use config::MediaConfig; pub use error::MediaError; pub use storage::{ BlobHeadMeta, BlobMeta, BulkDeleteOutcome, ByteStream, MediaStorage, ObjectVersionEntry, diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index 6ec4a1c1157..574434d0306 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -10,16 +10,14 @@ use std::pin::Pin; use std::sync::Arc; use buzz_core::tenant::{CommunityId, TenantContext}; -use buzz_object_store::{ObjectStore, ObjectStoreError, S3ObjectStore, S3StoreConfig}; +use buzz_object_store::{ObjectStore, ObjectStoreError}; -use crate::config::MediaConfig; use crate::error::MediaError; use bytes::Bytes; use serde::{Deserialize, Serialize}; pub use buzz_object_store::{ - BulkDeleteOutcome, ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, - ObjectVersionsPage, + BulkDeleteOutcome, ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, }; /// A stream of byte chunks from object storage, usable with @@ -32,19 +30,6 @@ pub struct MediaStorage { } impl MediaStorage { - /// Create a storage client from media config, over the S3 provider. - pub fn new(config: &MediaConfig) -> Result { - let store = S3ObjectStore::new(&S3StoreConfig { - endpoint: config.s3_endpoint.clone(), - access_key: config.s3_access_key.clone(), - secret_key: config.s3_secret_key.clone(), - bucket: config.s3_bucket.clone(), - region: config.s3_region.clone(), - addressing_style: config.s3_addressing_style, - })?; - Ok(Self::with_store(Arc::new(store))) - } - /// Wrap an already-constructed object store. /// /// The relay builds one provider per process and shares it between media @@ -319,46 +304,6 @@ mod tests { ) } - fn storage_config(access: &str, secret: &str) -> crate::config::MediaConfig { - crate::config::MediaConfig { - s3_endpoint: "http://localhost:9000".to_string(), - s3_access_key: access.to_string(), - s3_secret_key: secret.to_string(), - s3_bucket: "buzz-media".to_string(), - s3_region: "us-west-2".to_string(), - s3_addressing_style: crate::config::S3AddressingStyle::Path, - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - } - } - - #[test] - fn partial_static_keys_are_rejected() { - let err = match MediaStorage::new(&storage_config("buzz_dev", "")) { - Ok(_) => panic!("partial static creds must not silently use credential chain"), - Err(err) => err, - }; - assert!( - err.to_string().contains("must be configured together"), - "unexpected error: {err}" - ); - - let err = match MediaStorage::new(&storage_config("", "buzz_dev_secret")) { - Ok(_) => panic!("partial static creds must not silently use credential chain"), - Err(err) => err, - }; - assert!( - err.to_string().contains("must be configured together"), - "unexpected error: {err}" - ); - } - #[test] fn tenant_key_writers_are_covered_by_deletion_taxonomy() { let ctx = tenant(1); diff --git a/crates/buzz-media/src/upload.rs b/crates/buzz-media/src/upload.rs index 524b033280d..b9b11904943 100644 --- a/crates/buzz-media/src/upload.rs +++ b/crates/buzz-media/src/upload.rs @@ -565,12 +565,6 @@ mod tests { fn test_config() -> MediaConfig { MediaConfig { - s3_endpoint: String::new(), - s3_access_key: String::new(), - s3_secret_key: String::new(), - s3_bucket: String::new(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: crate::config::S3AddressingStyle::Path, max_image_bytes: 50 * 1024 * 1024, max_gif_bytes: 10 * 1024 * 1024, max_video_bytes: 524_288_000, diff --git a/crates/buzz-media/src/validation.rs b/crates/buzz-media/src/validation.rs index 706c354d043..62ecc9553d3 100644 --- a/crates/buzz-media/src/validation.rs +++ b/crates/buzz-media/src/validation.rs @@ -960,12 +960,6 @@ mod tests { fn test_config() -> MediaConfig { MediaConfig { - s3_endpoint: String::new(), - s3_access_key: String::new(), - s3_secret_key: String::new(), - s3_bucket: String::new(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: crate::config::S3AddressingStyle::Path, max_image_bytes: 50 * 1024 * 1024, max_gif_bytes: 10 * 1024 * 1024, max_video_bytes: 524_288_000, diff --git a/crates/buzz-media/tests/static_creds_minio.rs b/crates/buzz-media/tests/static_creds_minio.rs index 4c8c10702cb..eb8c9d5ec8a 100644 --- a/crates/buzz-media/tests/static_creds_minio.rs +++ b/crates/buzz-media/tests/static_creds_minio.rs @@ -18,39 +18,31 @@ //! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET` / `BUZZ_S3_REGION` / //! `BUZZ_S3_ADDRESSING_STYLE`. The default remains `path` for MinIO. -use buzz_media::config::MediaConfig; use buzz_media::storage::MediaStorage; +use buzz_object_store::{S3ObjectStore, S3StoreConfig}; -fn minio_config() -> MediaConfig { - MediaConfig { - s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") +fn minio_config() -> S3StoreConfig { + S3StoreConfig { + endpoint: std::env::var("BUZZ_S3_ENDPOINT") .unwrap_or_else(|_| "http://localhost:9000".to_string()), - s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") - .unwrap_or_else(|_| "buzz_dev".to_string()), - s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") + access_key: std::env::var("BUZZ_S3_ACCESS_KEY").unwrap_or_else(|_| "buzz_dev".to_string()), + secret_key: std::env::var("BUZZ_S3_SECRET_KEY") .unwrap_or_else(|_| "buzz_dev_secret".to_string()), - s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), - s3_region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), + region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") .unwrap_or_else(|_| "path".to_string()) .parse() .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"), - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, } } #[tokio::test] #[ignore = "requires a live MinIO (docker compose up -d minio minio-init)"] async fn static_creds_round_trip_against_minio() { - let storage = - MediaStorage::new(&minio_config()).expect("static creds should build a storage client"); + let store = S3ObjectStore::new(&minio_config()) + .expect("static creds should build an object-store client"); + let storage = MediaStorage::with_store(std::sync::Arc::new(store)); let key = format!("_test/static-creds-{}.bin", std::process::id()); let body = b"hardcoded-creds-still-work"; diff --git a/crates/buzz-media/tests/versioned_minio.rs b/crates/buzz-media/tests/versioned_minio.rs index 1e0db4b481f..7d2ac0005f0 100644 --- a/crates/buzz-media/tests/versioned_minio.rs +++ b/crates/buzz-media/tests/versioned_minio.rs @@ -18,34 +18,31 @@ use std::process::Command; -use buzz_media::config::MediaConfig; use buzz_media::storage::{MediaStorage, ObjectVersionKind, ObjectVersionRef}; +use buzz_object_store::{S3ObjectStore, S3StoreConfig}; fn env_or(name: &str, default: &str) -> String { std::env::var(name).unwrap_or_else(|_| default.to_string()) } -fn minio_config(bucket: String) -> MediaConfig { - MediaConfig { - s3_endpoint: env_or("BUZZ_S3_ENDPOINT", "http://localhost:9000"), - s3_access_key: env_or("BUZZ_S3_ACCESS_KEY", "buzz_dev"), - s3_secret_key: env_or("BUZZ_S3_SECRET_KEY", "buzz_dev_secret"), - s3_bucket: bucket, - s3_region: env_or("BUZZ_S3_REGION", "us-east-1"), - s3_addressing_style: env_or("BUZZ_S3_ADDRESSING_STYLE", "path") +fn minio_config(bucket: String) -> S3StoreConfig { + S3StoreConfig { + endpoint: env_or("BUZZ_S3_ENDPOINT", "http://localhost:9000"), + access_key: env_or("BUZZ_S3_ACCESS_KEY", "buzz_dev"), + secret_key: env_or("BUZZ_S3_SECRET_KEY", "buzz_dev_secret"), + bucket, + region: env_or("BUZZ_S3_REGION", "us-east-1"), + addressing_style: env_or("BUZZ_S3_ADDRESSING_STYLE", "path") .parse() .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"), - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, } } +fn media_storage(config: &S3StoreConfig) -> MediaStorage { + let store = S3ObjectStore::new(config).expect("static MinIO object-store client"); + MediaStorage::with_store(std::sync::Arc::new(store)) +} + fn run_mc(args: &[String]) -> Result<(), String> { let container = env_or("BUZZ_MINIO_CONTAINER", "buzz-minio"); let output = Command::new("docker") @@ -133,7 +130,7 @@ async fn never_versioned_bucket_lists_null_versions_and_exact_delete_empties_lis let bucket = format!("buzz-media-never-versioned-{}", std::process::id()); let bucket_path = format!("local/{bucket}"); let config = minio_config(bucket.clone()); - mc_alias(&config.s3_access_key, &config.s3_secret_key).expect("configure mc alias"); + mc_alias(&config.access_key, &config.secret_key).expect("configure mc alias"); run_mc(&[ "mb".to_string(), "--ignore-existing".to_string(), @@ -141,7 +138,7 @@ async fn never_versioned_bucket_lists_null_versions_and_exact_delete_empties_lis ]) .expect("create isolated never-versioned test bucket"); - let storage = MediaStorage::new(&config).expect("static MinIO storage client"); + let storage = media_storage(&config); let prefix = format!("_test/never-versioned-{}/", uuid::Uuid::new_v4()); let key = format!("{prefix}plain.bin"); storage @@ -187,7 +184,7 @@ async fn versioned_bucket_exact_version_delete_reaches_final_list_versions_empti let bucket = format!("buzz-media-versioned-{}", std::process::id()); let bucket_path = format!("local/{bucket}"); let config = minio_config(bucket.clone()); - mc_alias(&config.s3_access_key, &config.s3_secret_key).expect("configure mc alias"); + mc_alias(&config.access_key, &config.secret_key).expect("configure mc alias"); run_mc(&[ "mb".to_string(), "--ignore-existing".to_string(), @@ -201,7 +198,7 @@ async fn versioned_bucket_exact_version_delete_reaches_final_list_versions_empti ]) .expect("enable bucket versioning"); - let storage = MediaStorage::new(&config).expect("static MinIO storage client"); + let storage = media_storage(&config); let prefix = format!("_test/versioned-{}/", uuid::Uuid::new_v4()); let historical_key = format!("{prefix}historical.bin"); let marker_only_key = format!("{prefix}marker-only.bin"); diff --git a/crates/buzz-object-store/src/providers/gcs.rs b/crates/buzz-object-store/src/providers/gcs.rs index 4451d87d378..4e4dc2c07ca 100644 --- a/crates/buzz-object-store/src/providers/gcs.rs +++ b/crates/buzz-object-store/src/providers/gcs.rs @@ -461,6 +461,36 @@ fn bucket_resource(bucket: &str) -> String { format!("projects/_/buckets/{bucket}") } +/// Translate one Cloud Storage object descriptor into the provider-neutral +/// version vocabulary used by deletion. +fn version_entry_of(object: google_cloud_storage::model::Object) -> ObjectVersionEntry { + ObjectVersionEntry { + key: object.name, + version_id: object.generation.to_string(), + kind: ObjectVersionKind::Object, + size: u64::try_from(object.size).unwrap_or(0), + } +} + +/// Parse an opaque seam token back into a generation without ever accepting a +/// token from a different provider or a sentinel generation. +fn generation_of(version: &ObjectVersionRef) -> Result { + let generation = version + .version_id + .parse::() + .map_err(|_| ObjectStoreError::Provider { + operation: "delete_versions", + message: format!("invalid GCS generation for object {:?}", version.key), + })?; + if generation <= 0 { + return Err(ObjectStoreError::Provider { + operation: "delete_versions", + message: format!("non-positive GCS generation for object {:?}", version.key), + }); + } + Ok(generation) +} + /// The `ifGenerationMatch` value implementing a [`WriteCondition`]. /// /// `0` is Cloud Storage's create-only precondition: no live generation can be @@ -974,16 +1004,7 @@ impl ObjectStore for GcsObjectStore { .map_err(|e| classify("list_versions_page", prefix, e))?; let next = Some(response.next_page_token).filter(|token| !token.is_empty()); Ok(ObjectVersionsPage { - entries: response - .objects - .into_iter() - .map(|object| ObjectVersionEntry { - key: object.name, - version_id: object.generation.to_string(), - kind: ObjectVersionKind::Object, - size: u64::try_from(object.size).unwrap_or(0), - }) - .collect(), + entries: response.objects.into_iter().map(version_entry_of).collect(), is_truncated: next.is_some(), next_key_marker: next, next_version_id_marker: None, @@ -1002,25 +1023,7 @@ impl ObjectStore for GcsObjectStore { let mut parsed = Vec::with_capacity(versions.len()); for version in versions { - let generation = version.version_id.parse::().map_err(|_| { - ObjectStoreError::Provider { - operation: "delete_versions", - message: format!( - "invalid GCS generation for object {:?}", - version.key - ), - } - })?; - if generation <= 0 { - return Err(ObjectStoreError::Provider { - operation: "delete_versions", - message: format!( - "non-positive GCS generation for object {:?}", - version.key - ), - }); - } - parsed.push((version.key.clone(), generation)); + parsed.push((version.key.clone(), generation_of(version)?)); } let outcomes = futures_util::stream::iter(parsed) @@ -1048,11 +1051,11 @@ impl ObjectStore for GcsObjectStore { match result { Ok(()) => outcome.deleted += 1, Err(ObjectStoreError::NotFound { .. }) => outcome.already_missing += 1, - Err(error) => outcome.failed.push(( - key, - error_code(&error).to_string(), - error.to_string(), - )), + Err(error) => { + outcome + .failed + .push((key, error_code(&error).to_string(), error.to_string())) + } } } Ok(outcome) @@ -1099,7 +1102,7 @@ mod tests { use super::*; use google_cloud_storage::http::HeaderMap; use google_cloud_storage::model::bucket::{SoftDeletePolicy, Versioning}; - use google_cloud_storage::model::Bucket; + use google_cloud_storage::model::{Bucket, Object}; fn http_error(status: u16) -> GcsError { GcsError::http(status, HeaderMap::new(), bytes::Bytes::new()) @@ -1119,6 +1122,38 @@ mod tests { ); } + #[test] + fn object_generations_map_to_provider_neutral_versions() { + let entry = version_entry_of( + Object::new() + .set_name("_meta/tenant/a.json") + .set_generation(1_700_000_000_000_042_i64) + .set_size(42_i64), + ); + assert_eq!(entry.key, "_meta/tenant/a.json"); + assert_eq!(entry.version_id, "1700000000000042"); + assert_eq!(entry.kind, ObjectVersionKind::Object); + assert_eq!(entry.size, 42); + } + + #[test] + fn exact_delete_rejects_foreign_or_sentinel_generation_tokens() { + for version_id in ["v-s3-token", "0", "-1"] { + let error = generation_of(&ObjectVersionRef { + key: "object".to_string(), + version_id: version_id.to_string(), + }) + .expect_err("invalid generation must fail closed"); + assert!(matches!( + error, + ObjectStoreError::Provider { + operation: "delete_versions", + .. + } + )); + } + } + /// Create-only is `ifGenerationMatch=0`; a compare-and-swap carries the /// observed generation verbatim. #[test] diff --git a/crates/buzz-object-store/src/providers/s3.rs b/crates/buzz-object-store/src/providers/s3.rs index 9665b93e7eb..779f9591b11 100644 --- a/crates/buzz-object-store/src/providers/s3.rs +++ b/crates/buzz-object-store/src/providers/s3.rs @@ -317,9 +317,9 @@ fn parse_list_version_entry( _ => {} }, Event::End(end) if end.name().as_ref() == start.to_end().name().as_ref() => { - let key = fields.key.ok_or_else(|| { - version_xml_error("ListObjectVersions entry missing Key") - })?; + let key = fields + .key + .ok_or_else(|| version_xml_error("ListObjectVersions entry missing Key"))?; let version_id = fields.version_id.ok_or_else(|| { version_xml_error("ListObjectVersions entry missing VersionId") })?; @@ -911,4 +911,69 @@ mod tests { )] ); } + + #[test] + fn explicit_version_delete_counts_version_artifacts_as_deleted() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let result = DeleteObjectsResult { + deleted: vec![DeletedObject { + key: "versioned".to_string(), + version_id: Some("v1".to_string()), + delete_marker: Some(true), + delete_marker_version_id: Some("v1".to_string()), + }], + errors: vec![DeleteError { + key: "already-gone".to_string(), + code: "NoSuchVersion".to_string(), + message: "absent".to_string(), + version_id: Some("v0".to_string()), + }], + }; + + let outcome = fold_version_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 1); + assert!(outcome.versioned_keys.is_empty()); + assert!(outcome.failed.is_empty()); + } + + #[test] + fn version_listing_preserves_objects_markers_and_dual_cursor() { + let page = parse_object_versions_page( + br#" + + true + _meta/tenant/a.json + v-new + + _meta/tenant/a.jsonv-delete + + + _meta/tenant/a.jsonv-new42 + +"#, + ) + .expect("parse S3 version page"); + + assert!(page.is_truncated); + assert_eq!(page.next_key_marker.as_deref(), Some("_meta/tenant/a.json")); + assert_eq!(page.next_version_id_marker.as_deref(), Some("v-new")); + assert_eq!( + page.entries, + vec![ + ObjectVersionEntry { + key: "_meta/tenant/a.json".to_string(), + version_id: "v-delete".to_string(), + kind: ObjectVersionKind::DeleteMarker, + size: 0, + }, + ObjectVersionEntry { + key: "_meta/tenant/a.json".to_string(), + version_id: "v-new".to_string(), + kind: ObjectVersionKind::Object, + size: 42, + }, + ] + ); + } } diff --git a/crates/buzz-object-store/tests/gcs_live.rs b/crates/buzz-object-store/tests/gcs_live.rs index 0e0fab10172..27cd0a9bd32 100644 --- a/crates/buzz-object-store/tests/gcs_live.rs +++ b/crates/buzz-object-store/tests/gcs_live.rs @@ -36,7 +36,7 @@ use futures_util::FutureExt; use buzz_object_store::{ ConditionalWrite, GcsObjectStore, GcsStoreConfig, ImmutableWrite, ObjectStore, - ObjectStoreError, ProviderKind, Revision, WriteCondition, + ObjectStoreError, ObjectVersionKind, ObjectVersionRef, ProviderKind, Revision, WriteCondition, }; const CONTENT_TYPE: &str = "application/octet-stream"; @@ -163,6 +163,43 @@ async fn admits_a_bucket_that_can_prove_deletion() { .await; } +/// The provider-neutral exact-version contract maps GCS generations without +/// leaking generation vocabulary into media or deletion code. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn lists_and_deletes_an_exact_generation() { + with_prefix("exact-version", async |store, prefix| { + let key = format!("{prefix}/_meta/a.json"); + store + .put(&key, b"version body", CONTENT_TYPE) + .await + .expect("write object"); + + let page = store + .list_versions_page(prefix, None, None, 100) + .await + .expect("list exact versions"); + let entry = page + .entries + .into_iter() + .find(|entry| entry.key == key) + .expect("written generation is listed"); + assert_eq!(entry.kind, ObjectVersionKind::Object); + + let outcome = store + .delete_versions(&[ObjectVersionRef { + key: entry.key, + version_id: entry.version_id, + }]) + .await + .expect("delete exact generation"); + assert_eq!(outcome.deleted, 1); + assert!(outcome.failed.is_empty()); + assert!(store.head(&key).await.expect("head after delete").is_none()); + }) + .await; +} + /// Create-only writes: the first commits, the second finds the key taken. #[tokio::test] #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] diff --git a/crates/buzz-relay/src/api/admin/mod.rs b/crates/buzz-relay/src/api/admin/mod.rs index 2f0d128fc87..786bbba29a0 100644 --- a/crates/buzz-relay/src/api/admin/mod.rs +++ b/crates/buzz-relay/src/api/admin/mod.rs @@ -1309,7 +1309,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -1814,7 +1814,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -2307,7 +2307,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2388,7 +2388,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -2515,7 +2515,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2884,7 +2884,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2957,7 +2957,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -3027,7 +3027,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -4780,7 +4780,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = crate::state::AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 31fdb019bfc..ae7cc5f44da 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -3858,7 +3858,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = crate::state::AppState::new( config, diff --git a/crates/buzz-relay/src/api/gifs.rs b/crates/buzz-relay/src/api/gifs.rs index c29df6746bb..384e151e385 100644 --- a/crates/buzz-relay/src/api/gifs.rs +++ b/crates/buzz-relay/src/api/gifs.rs @@ -381,8 +381,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = - buzz_media::MediaStorage::new(&config.media).expect("test media storage config"); + let media_storage = crate::test_media_storage(&config).expect("test media storage config"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/git/cas_publish.rs b/crates/buzz-relay/src/api/git/cas_publish.rs index aa456db90c9..7c746342ab6 100644 --- a/crates/buzz-relay/src/api/git/cas_publish.rs +++ b/crates/buzz-relay/src/api/git/cas_publish.rs @@ -1598,13 +1598,13 @@ mod tests { } fn live_store() -> GitStore { - GitStore::from_s3_config( + crate::test_git_store( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", "buzz-media", "us-east-1", - buzz_media::config::S3AddressingStyle::Path, + "path", ) .expect("connect local MinIO") } diff --git a/crates/buzz-relay/src/api/git/hydrate.rs b/crates/buzz-relay/src/api/git/hydrate.rs index 4e1b9b0f77c..74a977ea60a 100644 --- a/crates/buzz-relay/src/api/git/hydrate.rs +++ b/crates/buzz-relay/src/api/git/hydrate.rs @@ -545,15 +545,9 @@ mod tests { #[tokio::test] async fn materialized_repo_is_created_under_configured_scratch_dir() { let scratch = TempDir::new().unwrap(); - let store = GitStore::from_s3_config( - "http://localhost:9000", - "x", - "x", - "x", - "us-east-1", - buzz_media::config::S3AddressingStyle::Path, - ) - .expect("construct store"); + let store = + crate::test_git_store("http://localhost:9000", "x", "x", "x", "us-east-1", "path") + .expect("construct store"); let manifest = Manifest { version: 1, head: "refs/heads/main".into(), @@ -590,13 +584,13 @@ mod tests { } fn store() -> GitStore { - GitStore::from_s3_config( + crate::test_git_store( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", "buzz-git", "us-east-1", - buzz_media::config::S3AddressingStyle::Path, + "path", ) .expect("connect local MinIO") } diff --git a/crates/buzz-relay/src/api/git/policy.rs b/crates/buzz-relay/src/api/git/policy.rs index 32d63f46008..f09219faf0d 100644 --- a/crates/buzz-relay/src/api/git/policy.rs +++ b/crates/buzz-relay/src/api/git/policy.rs @@ -850,7 +850,7 @@ printf '%s' "$HMAC_INPUT" | openssl dgst -sha256 -hmac "{secret}" -hex 2>/dev/nu db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/git/store.rs b/crates/buzz-relay/src/api/git/store.rs index 6a4082e7965..b69003655ee 100644 --- a/crates/buzz-relay/src/api/git/store.rs +++ b/crates/buzz-relay/src/api/git/store.rs @@ -35,7 +35,7 @@ use std::time::{Duration, Instant}; use buzz_object_store::{ ConditionalWrite, ImmutableWrite, ObjectStore, ObjectStoreError, ProviderKind, Revision, - S3AddressingStyle, S3ObjectStore, S3StoreConfig, WriteCondition, + WriteCondition, }; use bytes::Bytes; use sha2::{Digest, Sha256}; @@ -382,34 +382,6 @@ impl GitStore { Self { store } } - /// Build a git store over a freshly constructed S3 provider. - /// - /// Convenience for tests and the backend conformance probe, which connect - /// to a bare S3-compatible endpoint without the rest of the relay. - /// Production shares one client via [`GitStore::new`]. - pub fn from_s3_config( - endpoint: &str, - access_key: &str, - secret_key: &str, - bucket_name: &str, - region: &str, - addressing_style: S3AddressingStyle, - ) -> Result { - let store = S3ObjectStore::new(&S3StoreConfig { - endpoint: endpoint.to_string(), - access_key: access_key.to_string(), - secret_key: secret_key.to_string(), - bucket: bucket_name.to_string(), - region: region.to_string(), - addressing_style, - }) - .map_err(|e| match e { - ObjectStoreError::Config(message) => StoreError::Config(message), - other => StoreError::Backend(other), - })?; - Ok(Self::new(Arc::new(store))) - } - /// Compute the hex SHA-256 of `bytes`. The content-addressed key. pub fn content_key(prefix: &str, bytes: &[u8]) -> String { let mut h = Sha256::new(); @@ -1542,29 +1514,6 @@ mod tests { other => panic!("expected Backend, got {other:?}"), } } - - #[test] - fn partial_static_keys_are_rejected() { - for (access, secret) in [("buzz_dev", ""), ("", "buzz_dev_secret")] { - let err = match GitStore::from_s3_config( - "http://localhost:9000", - access, - secret, - "buzz-git", - "us-east-1", - S3AddressingStyle::Path, - ) { - Ok(_) => { - panic!("partial static creds must not silently use the credential chain") - } - Err(err) => err, - }; - assert!( - matches!(err, StoreError::Config(_)), - "expected Config error, got {err:?}" - ); - } - } } #[cfg(test)] @@ -1582,7 +1531,9 @@ mod profiles { use std::sync::Mutex; use async_trait::async_trait; - use buzz_object_store::{BulkDeleteOutcome, ByteStream, ListPage, ObjectMeta}; + use buzz_object_store::{ + BulkDeleteOutcome, ByteStream, ListPage, ObjectMeta, ObjectVersionRef, ObjectVersionsPage, + }; use super::*; @@ -1870,6 +1821,27 @@ mod profiles { Ok(outcome) } + async fn list_versions_page( + &self, + _prefix: &str, + _key_marker: Option, + _version_id_marker: Option, + _max_keys: usize, + ) -> Result { + Ok(ObjectVersionsPage::default()) + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + let keys = versions + .iter() + .map(|version| version.key.clone()) + .collect::>(); + self.delete_objects(&keys).await + } + async fn ping(&self) -> Result<(), ObjectStoreError> { Ok(()) } @@ -2232,17 +2204,15 @@ mod probe { std::env::var("BUZZ_S3_SECRET_KEY").unwrap_or_else(|_| "buzz_dev_secret".into()); let bucket = std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-git".into()); let region = std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".into()); - let addressing_style = std::env::var("BUZZ_S3_ADDRESSING_STYLE") - .unwrap_or_else(|_| "path".into()) - .parse() - .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"); - GitStore::from_s3_config( + let addressing_style = + std::env::var("BUZZ_S3_ADDRESSING_STYLE").unwrap_or_else(|_| "path".into()); + crate::test_git_store( &endpoint, &access_key, &secret_key, &bucket, ®ion, - addressing_style, + &addressing_style, ) .expect("connect S3-compatible storage") } diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 94fd7f8758e..41baa3d1c15 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2314,7 +2314,7 @@ mod track_c_tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index e3d05165e0d..35253918702 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -694,7 +694,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 780532ec5d0..540ac313770 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -1158,7 +1158,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index b59fd840c6d..7dea2958320 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -603,7 +603,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = AppState::new( config, db, @@ -1288,7 +1288,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 5e86ac21ba7..edd7e26477e 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -832,28 +832,7 @@ impl Config { .and_then(|v| v.parse().ok()) .unwrap_or(9102); - let s3_addressing_style = match std::env::var("BUZZ_S3_ADDRESSING_STYLE") { - Ok(value) => value.parse().map_err(ConfigError::InvalidValue)?, - Err(std::env::VarError::NotPresent) => buzz_media::config::S3AddressingStyle::default(), - Err(std::env::VarError::NotUnicode(_)) => { - return Err(ConfigError::InvalidValue( - "BUZZ_S3_ADDRESSING_STYLE must be valid Unicode and one of 'path' or 'virtual'" - .to_string(), - )); - } - }; let media = buzz_media::MediaConfig { - s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") - .unwrap_or_else(|_| "http://localhost:9000".to_string()), - s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") - .unwrap_or_else(|_| "buzz_dev".to_string()), - s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") - .unwrap_or_else(|_| "buzz_dev_secret".to_string()), - s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), - s3_region: std::env::var("BUZZ_S3_REGION") - .or_else(|_| std::env::var("AWS_REGION")) - .unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style, max_image_bytes: std::env::var("BUZZ_MAX_IMAGE_BYTES") .ok() .and_then(|v| v.parse().ok()) @@ -887,20 +866,38 @@ impl Config { .map(|s| s.trim().to_lowercase()) .filter(|s| !s.is_empty()), }; - // The provider is chosen once here; a Cloud Storage deployment carries - // no `BUZZ_S3_*` values at all, so its bucket comes from - // `BUZZ_OBJECT_STORE_BUCKET` rather than from the media settings. + // Choose the provider before reading provider-native settings. A GCS + // deployment is unaffected by stale or malformed `BUZZ_S3_*` values, + // and vice versa. let object_store = match buzz_object_store::ProviderSelection::from_env() .map_err(ConfigError::InvalidValue)? { buzz_object_store::ProviderSelection::S3 => { + let addressing_style = match std::env::var("BUZZ_S3_ADDRESSING_STYLE") { + Ok(value) => value.parse().map_err(ConfigError::InvalidValue)?, + Err(std::env::VarError::NotPresent) => { + buzz_object_store::S3AddressingStyle::default() + } + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ConfigError::InvalidValue( + "BUZZ_S3_ADDRESSING_STYLE must be valid Unicode and one of 'path' or 'virtual'" + .to_string(), + )); + } + }; buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { - endpoint: media.s3_endpoint.clone(), - access_key: media.s3_access_key.clone(), - secret_key: media.s3_secret_key.clone(), - bucket: media.s3_bucket.clone(), - region: media.s3_region.clone(), - addressing_style: media.s3_addressing_style, + endpoint: std::env::var("BUZZ_S3_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:9000".to_string()), + access_key: std::env::var("BUZZ_S3_ACCESS_KEY") + .unwrap_or_else(|_| "buzz_dev".to_string()), + secret_key: std::env::var("BUZZ_S3_SECRET_KEY") + .unwrap_or_else(|_| "buzz_dev_secret".to_string()), + bucket: std::env::var("BUZZ_S3_BUCKET") + .unwrap_or_else(|_| "buzz-media".to_string()), + region: std::env::var("BUZZ_S3_REGION") + .or_else(|_| std::env::var("AWS_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style, }) } buzz_object_store::ProviderSelection::Gcs { bucket } => { @@ -1402,9 +1399,14 @@ mod tests { !config.serve_git_web_gui, "serve_git_web_gui should default to false" ); - assert_eq!( - config.media.s3_addressing_style, - buzz_media::config::S3AddressingStyle::Path, + assert!( + matches!( + config.object_store, + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + addressing_style: buzz_object_store::S3AddressingStyle::Path, + .. + }) + ), "S3 addressing must default to path style for bundled MinIO compatibility" ); assert!( @@ -1783,8 +1785,7 @@ mod tests { std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "virtual"); let configured = Config::from_env() .expect("virtual style config") - .media - .s3_addressing_style; + .object_store; std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "auto"); let invalid = Config::from_env(); @@ -1795,7 +1796,13 @@ mod tests { std::env::remove_var("BUZZ_S3_ADDRESSING_STYLE"); } - assert_eq!(configured, buzz_media::config::S3AddressingStyle::Virtual); + assert!(matches!( + configured, + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + addressing_style: buzz_object_store::S3AddressingStyle::Virtual, + .. + }) + )); assert!(matches!( invalid, Err(ConfigError::InvalidValue(ref message)) @@ -1826,10 +1833,13 @@ mod tests { match config.object_store { buzz_object_store::ObjectStoreConfig::S3(s3) => { - assert_eq!(s3.endpoint, config.media.s3_endpoint); - assert_eq!(s3.bucket, config.media.s3_bucket); - assert_eq!(s3.region, config.media.s3_region); - assert_eq!(s3.addressing_style, config.media.s3_addressing_style); + assert_eq!(s3.endpoint, "http://localhost:9000"); + assert_eq!(s3.bucket, "buzz-media"); + assert_eq!(s3.region, "us-east-1"); + assert_eq!( + s3.addressing_style, + buzz_object_store::S3AddressingStyle::Path + ); } other => panic!("expected the S3 provider by default, got {other:?}"), } @@ -1844,8 +1854,11 @@ mod tests { let _guard = ENV_MUTEX.lock().unwrap(); let previous_provider = std::env::var_os("BUZZ_OBJECT_STORE_PROVIDER"); let previous_bucket = std::env::var_os("BUZZ_OBJECT_STORE_BUCKET"); + let previous_s3_style = std::env::var_os("BUZZ_S3_ADDRESSING_STYLE"); std::env::set_var("BUZZ_OBJECT_STORE_PROVIDER", "gcs"); + // Provider-native settings from an inactive provider are not parsed. + std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "not-an-s3-style"); std::env::remove_var("BUZZ_OBJECT_STORE_BUCKET"); let without_bucket = Config::from_env(); @@ -1857,6 +1870,7 @@ mod tests { restore_env("BUZZ_OBJECT_STORE_PROVIDER", previous_provider); restore_env("BUZZ_OBJECT_STORE_BUCKET", previous_bucket); + restore_env("BUZZ_S3_ADDRESSING_STYLE", previous_s3_style); assert!(matches!( without_bucket, diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index ccba40f3282..afa4e97fde6 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -2031,8 +2031,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = - buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, @@ -2084,7 +2083,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (state, audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/handlers/identity_archive.rs b/crates/buzz-relay/src/handlers/identity_archive.rs index 40c54647837..b106c2fc88b 100644 --- a/crates/buzz-relay/src/handlers/identity_archive.rs +++ b/crates/buzz-relay/src/handlers/identity_archive.rs @@ -459,7 +459,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (state, _audit_shutdown) = crate::state::AppState::new( config, db, diff --git a/crates/buzz-relay/src/handlers/relay_admin.rs b/crates/buzz-relay/src/handlers/relay_admin.rs index 3782f2c516d..50dac6f2051 100644 --- a/crates/buzz-relay/src/handlers/relay_admin.rs +++ b/crates/buzz-relay/src/handlers/relay_admin.rs @@ -739,7 +739,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index 800433a8498..a8afbb60a33 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -55,3 +55,45 @@ pub mod workflow_sink; pub use config::Config; pub use error::{RelayError, Result}; pub use state::AppState; + +/// Build the default S3-backed media facade used by unit-test fixtures. +/// +/// Provider construction belongs here, outside media and Git domain modules. +#[cfg(test)] +pub(crate) fn test_media_storage( + config: &Config, +) -> std::result::Result { + let buzz_object_store::ObjectStoreConfig::S3(s3) = &config.object_store else { + return Err(buzz_object_store::ObjectStoreError::Config( + "synchronous test fixture requires the S3 test provider".to_string(), + )); + }; + let store = buzz_object_store::S3ObjectStore::new(s3)?; + Ok(buzz_media::MediaStorage::with_store(std::sync::Arc::new( + store, + ))) +} + +/// Build an S3-backed Git facade for MinIO/unit-test fixtures. +#[cfg(test)] +pub(crate) fn test_git_store( + endpoint: &str, + access_key: &str, + secret_key: &str, + bucket: &str, + region: &str, + addressing_style: &str, +) -> std::result::Result { + let addressing_style = addressing_style + .parse() + .map_err(buzz_object_store::ObjectStoreError::Config)?; + let store = buzz_object_store::S3ObjectStore::new(&buzz_object_store::S3StoreConfig { + endpoint: endpoint.to_string(), + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + region: region.to_string(), + addressing_style, + })?; + Ok(api::git::store::GitStore::new(std::sync::Arc::new(store))) +} diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index dd0fde6fdcd..6d2a9fa6ac5 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -578,7 +578,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 0d8d22c346b..451fc3b9b00 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -1441,7 +1441,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 8ce23a2e8ea..fdb11fd6a97 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -660,7 +660,7 @@ mod integration_tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-test-client/Cargo.toml b/crates/buzz-test-client/Cargo.toml index e495c163004..ddb8efeaef3 100644 --- a/crates/buzz-test-client/Cargo.toml +++ b/crates/buzz-test-client/Cargo.toml @@ -37,6 +37,7 @@ sqlx = { workspace = true } chrono = { workspace = true } s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } buzz-media = { workspace = true } +buzz-object-store = { workspace = true } buzz-sdk = { workspace = true } [[bin]] diff --git a/crates/buzz-test-client/tests/e2e_git.rs b/crates/buzz-test-client/tests/e2e_git.rs index 3c82e317649..6d0fdac17b2 100644 --- a/crates/buzz-test-client/tests/e2e_git.rs +++ b/crates/buzz-test-client/tests/e2e_git.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; -use buzz_media::S3AddressingStyle; +use buzz_object_store::S3AddressingStyle; use nostr::{EventBuilder, Keys, Kind, Tag}; use s3::creds::Credentials; use s3::{Bucket, Region}; diff --git a/docs/git-on-object-storage.md b/docs/git-on-object-storage.md index 40476137d8e..1bce2445184 100644 --- a/docs/git-on-object-storage.md +++ b/docs/git-on-object-storage.md @@ -5,7 +5,7 @@ ## Abstract This document specifies a protocol for hosting git repositories on an object -store (S3 and S3-compatible backends such as MinIO) with **no persistent +store (currently Google Cloud Storage, with an AWS S3 adapter) with **no persistent filesystem**, and gives a formal proof of its safety properties. Repository content is stored as create-only, content-addressed pack objects (immutable by protocol discipline — see A1, not by assuming an immutable store); the current @@ -65,6 +65,39 @@ local. Cache misses, restarts, and evictions only affect performance; object storage remains the source of truth, and per-request refs/HEAD are still materialized from the current manifest. +### Provider portability and cutover + +Object storage is a deployment choice, not a media or Git domain choice. +`crates/buzz-object-store::ObjectStore` is the only interface those domains +consume. The relay constructs one provider at composition time and shares it +between `MediaStorage` and `GitStore`; `BUZZ_OBJECT_STORE_PROVIDER` selects +`gcs` or `s3`. Provider-native vocabulary is confined to its adapter: + +- GCS generations implement revisions, conditional writes, version listing, + and exact-version deletion; Application Default Credentials provide identity. +- S3 ETags/version IDs implement the same contract; the AWS credential chain + provides identity when static interoperability credentials are absent. + +The current Mozart deployment selects GCS. Moving Buzz to AWS does not require +changes in media, Git, deletion, or key layout. It is an explicit data cutover: + +1. provision the target bucket with the deletion contract satisfied (no object + versioning or recoverable soft-delete retention), and grant only the Buzz + runtime identity; +2. run the target adapter's conformance profile before serving traffic; +3. drain writers, copy every key and byte exactly, and prove source/target key + stream digests and byte totals agree for both tenant and Git prefixes; +4. switch `BUZZ_OBJECT_STORE_PROVIDER` and its provider configuration in the + deployment layer, then prove media reads, Git hydrate/clone, CAS publication, + version enumeration, and exact deletion on the target; +5. retain the source read-only for the rollback window. A rollback after target + writes requires a reverse delta copy before switching; never point two live + writers at different buckets. + +The bucket name is configuration, never embedded in object keys or domain +records. This keeps a future GCS-to-S3 move a copy-and-composition change rather +than a schema or application rewrite. + The accepted v1 tradeoff: under concurrent same-repo pushes, every contender hydrates and runs receive-pack, and the CAS losers' subprocess work is discarded. This is wasted CPU/IO under contention, not a correctness bug — From c91702dfbc4d1acdb49fef08f1d75b7de25793b1 Mon Sep 17 00:00:00 2001 From: mozarthq Date: Mon, 31 Aug 2026 08:19:03 -0700 Subject: [PATCH 7/7] fix(db): pin function search paths for restore Signed-off-by: mozarthq --- crates/buzz-db/src/runtime/migration.rs | 122 +++++++++++++++++++++- migrations/0041_function_search_paths.sql | 36 +++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 migrations/0041_function_search_paths.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 00cd81c6940..c5684e3931f 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -699,7 +699,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 40); + assert_eq!(migrations.len(), 41); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -2400,6 +2400,75 @@ mod tests { assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn function_search_path_migration_repairs_restore_context() { + let pool = connect_test_pool().await; + reset_public_schema(&pool).await; + run_migrations_through(&pool, 40) + .await + .expect("apply migrations through 40"); + + let community = uuid::Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community) + .bind(format!("restore-proof-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert restore proof community"); + + let mut before = pool.acquire().await.expect("pre-migration connection"); + sqlx::query("BEGIN") + .execute(&mut *before) + .await + .expect("begin pre-migration transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *before) + .await + .expect("clear pre-migration search path"); + let before_error = sqlx::query("SELECT public.assert_community_write_allowed($1)") + .bind(community) + .execute(&mut *before) + .await + .expect_err("migration 40 must reproduce restore name-resolution failure"); + assert_eq!( + before_error + .as_database_error() + .and_then(sqlx::error::DatabaseError::code) + .as_deref(), + Some("42883") + ); + sqlx::query("ROLLBACK") + .execute(&mut *before) + .await + .expect("rollback pre-migration transaction"); + drop(before); + + run_migrations(&pool) + .await + .expect("apply function search-path migration"); + + let mut after = pool.acquire().await.expect("post-migration connection"); + sqlx::query("BEGIN") + .execute(&mut *after) + .await + .expect("begin restore-like transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *after) + .await + .expect("clear restore-like search path"); + sqlx::query("INSERT INTO public.users (community_id, pubkey) VALUES ($1, $2)") + .bind(community) + .bind(vec![0x41_u8; 32]) + .execute(&mut *after) + .await + .expect("restore-like insert must traverse the repaired write fence"); + sqlx::query("ROLLBACK") + .execute(&mut *after) + .await + .expect("finish restore-like transaction"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn run_migrations_applies_consolidated_initial_schema_on_fresh_database() { @@ -2475,6 +2544,57 @@ mod tests { .await .expect("insert late-table test community"); } + + let unpinned_application_functions: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT \ + FROM pg_proc AS procedure \ + JOIN pg_namespace AS namespace \ + ON namespace.oid = procedure.pronamespace \ + WHERE namespace.nspname = 'public' \ + AND procedure.prokind = 'f' \ + AND procedure.proowner = (SELECT oid FROM pg_roles WHERE rolname = current_user) \ + AND NOT EXISTS (\ + SELECT 1 FROM pg_depend AS dependency \ + WHERE dependency.classid = 'pg_proc'::REGCLASS \ + AND dependency.objid = procedure.oid \ + AND dependency.deptype = 'e'\ + ) \ + AND NOT coalesce(procedure.proconfig, ARRAY[]::TEXT[]) \ + @> ARRAY['search_path=public, pg_catalog']", + ) + .fetch_one(&pool) + .await + .expect("inspect application function search paths"); + assert_eq!( + unpinned_application_functions, 0, + "every application-owned function must resolve independently of the restore session" + ); + + let mut restore_connection = pool.acquire().await.expect("restore-like connection"); + sqlx::query("BEGIN") + .execute(&mut *restore_connection) + .await + .expect("begin restore-like transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *restore_connection) + .await + .expect("clear restore-like search path"); + sqlx::query("SELECT public.assert_community_write_allowed($1)") + .bind(active_a) + .execute(&mut *restore_connection) + .await + .expect("write fence must resolve with an empty invoker search path"); + sqlx::query("INSERT INTO public.users (community_id, pubkey) VALUES ($1, $2)") + .bind(active_a) + .bind(vec![0x41_u8; 32]) + .execute(&mut *restore_connection) + .await + .expect("restore-like insert must traverse the write-fence trigger"); + sqlx::query("ROLLBACK") + .execute(&mut *restore_connection) + .await + .expect("finish restore-like transaction"); + sqlx::query( "CREATE TABLE late_created_scoped (\ community_id UUID NOT NULL, id BIGINT PRIMARY KEY, value TEXT NOT NULL\ diff --git a/migrations/0041_function_search_paths.sql b/migrations/0041_function_search_paths.sql new file mode 100644 index 00000000000..2df99143b68 --- /dev/null +++ b/migrations/0041_function_search_paths.sql @@ -0,0 +1,36 @@ +-- PostgreSQL restore sessions deliberately clear search_path. Buzz trigger +-- functions call other Buzz functions and tables by their unqualified names, +-- so data-only restores must not inherit name resolution from the invoker. +-- Pin every application-owned public function while leaving extension-owned +-- and provider-owned functions untouched. + +SET LOCAL search_path TO public, pg_catalog; + +DO $$ +DECLARE + function_identity REGPROCEDURE; +BEGIN + FOR function_identity IN + SELECT procedure.oid::REGPROCEDURE + FROM pg_proc AS procedure + JOIN pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname = 'public' + AND procedure.prokind = 'f' + AND procedure.proowner = (SELECT oid FROM pg_roles WHERE rolname = current_user) + AND NOT EXISTS ( + SELECT 1 + FROM pg_depend AS dependency + WHERE dependency.classid = 'pg_proc'::REGCLASS + AND dependency.objid = procedure.oid + AND dependency.deptype = 'e' + ) + ORDER BY procedure.oid + LOOP + EXECUTE format( + 'ALTER FUNCTION %s SET search_path TO public, pg_catalog', + function_identity + ); + END LOOP; +END +$$;