diff --git a/Cargo.lock b/Cargo.lock index 9544a63b899..d8f9362f65b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -581,6 +581,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -808,6 +814,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" version = "1.12.1" @@ -1083,6 +1098,7 @@ dependencies = [ "buzz-core", "buzz-db", "buzz-media", + "buzz-object-store", "chrono", "clap", "deadpool-redis", @@ -1113,7 +1129,7 @@ dependencies = [ "reqwest 0.13.4", "rmcp", "rustls", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "similar", @@ -1133,6 +1149,7 @@ dependencies = [ "axum", "blurhash", "buzz-core", + "buzz-object-store", "bytes", "chrono", "futures-core", @@ -1143,8 +1160,6 @@ dependencies = [ "infer", "mp4", "nostr 0.44.7", - "quick-xml 0.38.4", - "rust-s3", "serde", "serde_json", "sha2 0.11.0", @@ -1157,6 +1172,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-object-store" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum", + "bytes", + "futures-core", + "futures-util", + "google-cloud-gax", + "google-cloud-storage", + "google-cloud-wkt", + "hex", + "quick-xml 0.38.4", + "rand 0.10.1", + "rust-s3", + "rustls", + "serde", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "buzz-pair-relay" version = "0.1.0" @@ -1272,6 +1312,7 @@ dependencies = [ "buzz-db", "buzz-deletion", "buzz-media", + "buzz-object-store", "buzz-pubsub", "buzz-relay-mesh", "buzz-sdk", @@ -1304,7 +1345,6 @@ dependencies = [ "rand 0.10.1", "redis", "reqwest 0.13.4", - "rust-s3", "rustls", "serde", "serde_json", @@ -1383,6 +1423,7 @@ dependencies = [ "base64 0.22.1", "buzz-core", "buzz-media", + "buzz-object-store", "buzz-sdk", "buzz-ws-client", "chrono", @@ -1492,6 +1533,9 @@ name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] [[package]] name = "bzip2" @@ -1951,6 +1995,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2379,6 +2432,37 @@ dependencies = [ "tokio", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "deltae" version = "0.3.2" @@ -3262,6 +3346,226 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "google-cloud-auth" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff461519b1a948200f163574be072753bcfb462a323f0eb426629d89872dd685" +dependencies = [ + "async-trait", + "base64 0.23.1", + "bytes", + "google-cloud-gax", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "reqwest 0.13.4", + "rustc_version", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "time", + "tokio", + "url", +] + +[[package]] +name = "google-cloud-gax" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5615cff28ee59cfe52fbb4c11b8b1e77f650296e2ea4f4c2b7757ac6b19e752" +dependencies = [ + "bytes", + "futures", + "google-cloud-rpc", + "google-cloud-wkt", + "http", + "pin-project", + "rand 0.10.1", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-stream", +] + +[[package]] +name = "google-cloud-gax-internal" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2766757d877a7a8ac23da9884cb0e3f10ed9b75a0ce59801ce6b19bf9d5819e" +dependencies = [ + "bytes", + "futures", + "google-cloud-auth", + "google-cloud-gax", + "google-cloud-rpc", + "google-cloud-wkt", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "lazy_static", + "opentelemetry 0.32.0", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk 0.32.1", + "percent-encoding", + "pin-project", + "prost 0.14.3", + "prost-types 0.14.3", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tower", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "google-cloud-iam-v1" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f962b40234b1531e6ef73f7558871c96e117231e962086c98804323fb8d2c82" +dependencies = [ + "async-trait", + "bytes", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-type", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", + "tracing", +] + +[[package]] +name = "google-cloud-longrunning" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0363c5389ffda2b55cd8a86eef4b19a3481a48467c91dc5d77f626a9572766" +dependencies = [ + "async-trait", + "bytes", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-rpc", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", + "tracing", +] + +[[package]] +name = "google-cloud-lro" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47af3deef75c14a2983c430898d960c765bddbcc9f9188ca0563108e9227cfe7" +dependencies = [ + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-longrunning", + "google-cloud-rpc", + "google-cloud-wkt", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "google-cloud-rpc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2162c08a89118130979ba261080e960e44cdcb2d6e2ab8ca9b1da245285d353" +dependencies = [ + "bytes", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", +] + +[[package]] +name = "google-cloud-storage" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9227f65175fa91a6e41f246797917697efdadfe09dd8ea84ad8b737a71efbd28" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "crc32c", + "futures", + "google-cloud-auth", + "google-cloud-gax", + "google-cloud-gax-internal", + "google-cloud-iam-v1", + "google-cloud-longrunning", + "google-cloud-lro", + "google-cloud-rpc", + "google-cloud-type", + "google-cloud-wkt", + "hex", + "http", + "http-body", + "md5", + "percent-encoding", + "prost 0.14.3", + "prost-types 0.14.3", + "serde", + "serde_json", + "serde_with", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "google-cloud-type" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63acc3a92a85f96bab021c3a3e29b53bbacc97651e1b524d4c2991960a63eb82" +dependencies = [ + "bytes", + "google-cloud-wkt", + "serde", + "serde_json", + "serde_with", +] + +[[package]] +name = "google-cloud-wkt" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fccf98cfd5481a5f5a285181ab0c62123d7d47cd2bb7299448440649349e4e7" +dependencies = [ + "base64 0.22.1", + "bytes", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "time", + "url", +] + [[package]] name = "h2" version = "0.4.16" @@ -3274,7 +3578,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3296,6 +3600,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -3553,9 +3863,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -3609,9 +3919,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -3920,6 +4230,17 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09e54e57b4c48b40f7aec75635392b12b3421fa26fe8b4332e63138ed278459c" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -3928,6 +4249,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -4193,6 +4516,59 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link 0.2.1", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -4993,7 +5369,7 @@ dependencies = [ "rmcp", "rpassword", "rustls", - "schemars", + "schemars 1.2.1", "semver", "serde", "serde_json", @@ -5079,7 +5455,7 @@ dependencies = [ "prost-build 0.14.3", "protoc-bin-vendored", "rmcp", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "tokio", @@ -5278,7 +5654,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap", + "indexmap 2.14.0", "ipnet", "metrics", "metrics-util", @@ -5299,7 +5675,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.14.0", "metrics", "ordered-float 5.3.0", "quanta", @@ -6424,6 +6800,12 @@ dependencies = [ "tonic-prost", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" + [[package]] name = "opentelemetry_sdk" version = "0.31.0" @@ -6713,7 +7095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap", + "indexmap 2.14.0", ] [[package]] @@ -6724,7 +7106,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset 0.5.7", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", ] [[package]] @@ -6849,7 +7231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64 0.22.1", - "indexmap", + "indexmap 2.14.0", "quick-xml 0.39.4", "serde", "time", @@ -7046,7 +7428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e842efad9119158434d193c6682e2ebee4b44d6ad801d7b349623b3f57cdf55" dependencies = [ "futures", - "indexmap", + "indexmap 2.14.0", "nix 0.31.3", "tokio", "tracing", @@ -7958,7 +8340,7 @@ dependencies = [ "rand 0.10.1", "reqwest 0.13.4", "rmcp-macros", - "schemars", + "schemars 1.2.1", "serde", "serde_json", "sse-stream", @@ -8233,6 +8615,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars" version = "1.2.1" @@ -8549,13 +8943,46 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "jiff", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -9018,7 +9445,7 @@ dependencies = [ "futures-util", "hashbrown 0.16.1", "hashlink", - "indexmap", + "indexmap 2.14.0", "log", "memchr", "percent-encoding", @@ -9968,7 +10395,7 @@ version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 0.7.5+spec-1.1.0", @@ -9983,7 +10410,7 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde_core", "serde_spanned", "toml_datetime 1.1.1+spec-1.1.0", @@ -10016,7 +10443,7 @@ version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ - "indexmap", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", @@ -10057,6 +10484,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "rustls-native-certs", "socket2", "sync_wrapper", "tokio", @@ -10098,7 +10526,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -11800,7 +12228,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.14.0", "memchr", "thiserror 2.0.18", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index d6ee839f1b0..de373816410 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/buzz-deletion", "crates/buzz-workflow", "crates/buzz-media", + "crates/buzz-object-store", "crates/buzz-cli", "crates/buzz-pairing-cli", "crates/buzz-sdk", @@ -146,6 +147,7 @@ buzz-search = { path = "crates/buzz-search" } buzz-audit = { path = "crates/buzz-audit" } buzz-workflow = { path = "crates/buzz-workflow" } buzz-media = { path = "crates/buzz-media" } +buzz-object-store = { path = "crates/buzz-object-store" } buzz-sdk = { path = "crates/buzz-sdk" } buzz-ws-client = { path = "crates/buzz-ws-client" } buzz-relay-mesh = { path = "crates/buzz-relay-mesh" } diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 00cd81c6940..c5684e3931f 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -699,7 +699,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 40); + assert_eq!(migrations.len(), 41); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -2400,6 +2400,75 @@ mod tests { assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn function_search_path_migration_repairs_restore_context() { + let pool = connect_test_pool().await; + reset_public_schema(&pool).await; + run_migrations_through(&pool, 40) + .await + .expect("apply migrations through 40"); + + let community = uuid::Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community) + .bind(format!("restore-proof-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert restore proof community"); + + let mut before = pool.acquire().await.expect("pre-migration connection"); + sqlx::query("BEGIN") + .execute(&mut *before) + .await + .expect("begin pre-migration transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *before) + .await + .expect("clear pre-migration search path"); + let before_error = sqlx::query("SELECT public.assert_community_write_allowed($1)") + .bind(community) + .execute(&mut *before) + .await + .expect_err("migration 40 must reproduce restore name-resolution failure"); + assert_eq!( + before_error + .as_database_error() + .and_then(sqlx::error::DatabaseError::code) + .as_deref(), + Some("42883") + ); + sqlx::query("ROLLBACK") + .execute(&mut *before) + .await + .expect("rollback pre-migration transaction"); + drop(before); + + run_migrations(&pool) + .await + .expect("apply function search-path migration"); + + let mut after = pool.acquire().await.expect("post-migration connection"); + sqlx::query("BEGIN") + .execute(&mut *after) + .await + .expect("begin restore-like transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *after) + .await + .expect("clear restore-like search path"); + sqlx::query("INSERT INTO public.users (community_id, pubkey) VALUES ($1, $2)") + .bind(community) + .bind(vec![0x41_u8; 32]) + .execute(&mut *after) + .await + .expect("restore-like insert must traverse the repaired write fence"); + sqlx::query("ROLLBACK") + .execute(&mut *after) + .await + .expect("finish restore-like transaction"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn run_migrations_applies_consolidated_initial_schema_on_fresh_database() { @@ -2475,6 +2544,57 @@ mod tests { .await .expect("insert late-table test community"); } + + let unpinned_application_functions: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT \ + FROM pg_proc AS procedure \ + JOIN pg_namespace AS namespace \ + ON namespace.oid = procedure.pronamespace \ + WHERE namespace.nspname = 'public' \ + AND procedure.prokind = 'f' \ + AND procedure.proowner = (SELECT oid FROM pg_roles WHERE rolname = current_user) \ + AND NOT EXISTS (\ + SELECT 1 FROM pg_depend AS dependency \ + WHERE dependency.classid = 'pg_proc'::REGCLASS \ + AND dependency.objid = procedure.oid \ + AND dependency.deptype = 'e'\ + ) \ + AND NOT coalesce(procedure.proconfig, ARRAY[]::TEXT[]) \ + @> ARRAY['search_path=public, pg_catalog']", + ) + .fetch_one(&pool) + .await + .expect("inspect application function search paths"); + assert_eq!( + unpinned_application_functions, 0, + "every application-owned function must resolve independently of the restore session" + ); + + let mut restore_connection = pool.acquire().await.expect("restore-like connection"); + sqlx::query("BEGIN") + .execute(&mut *restore_connection) + .await + .expect("begin restore-like transaction"); + sqlx::query("SET LOCAL search_path TO ''") + .execute(&mut *restore_connection) + .await + .expect("clear restore-like search path"); + sqlx::query("SELECT public.assert_community_write_allowed($1)") + .bind(active_a) + .execute(&mut *restore_connection) + .await + .expect("write fence must resolve with an empty invoker search path"); + sqlx::query("INSERT INTO public.users (community_id, pubkey) VALUES ($1, $2)") + .bind(active_a) + .bind(vec![0x41_u8; 32]) + .execute(&mut *restore_connection) + .await + .expect("restore-like insert must traverse the write-fence trigger"); + sqlx::query("ROLLBACK") + .execute(&mut *restore_connection) + .await + .expect("finish restore-like transaction"); + sqlx::query( "CREATE TABLE late_created_scoped (\ community_id UUID NOT NULL, id BIGINT PRIMARY KEY, value TEXT NOT NULL\ diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index 8c308b0a8c0..2a3c177f994 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -13,6 +13,7 @@ thiserror = { workspace = true } buzz-core = { workspace = true } buzz-db = { workspace = true } buzz-media = { workspace = true } +buzz-object-store = { workspace = true } chrono = { workspace = true } clap = { version = "4", features = ["derive"] } deadpool-redis = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index d963a7f261f..8f0ee9d833c 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -561,27 +561,8 @@ async fn connect_services() -> Result { } async fn connect_services_with_store(store: DeletionStore) -> Result { - let (s3_access_key, s3_secret_key) = s3_key_pair_from_env(); - let media_config = buzz_media::MediaConfig { - s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, - s3_access_key, - s3_secret_key, - s3_bucket: required_env("BUZZ_S3_BUCKET")?, - s3_region: s3_region_from_env(), - s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") - .unwrap_or_else(|_| "path".to_string()) - .parse() - .map_err(anyhow::Error::msg)?, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }; - let media = Arc::new(MediaStorage::new(&media_config)?); + let object_store = buzz_object_store::connect(&deletion_object_store_config()?).await?; + let media = Arc::new(MediaStorage::with_store(object_store)); let redis_url = required_env("REDIS_URL")?; let mut redis_config = deadpool_redis::Config::from_url(&redis_url); redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( @@ -598,6 +579,37 @@ async fn connect_services_with_store(store: DeletionStore) -> Result { }) } +/// Resolve the same deployment-level provider used by the relay. +/// +/// The deletion executable is a separate composition root, so it must select +/// the provider independently while preserving exactly the relay's environment +/// contract. Domain deletion code continues to consume `MediaStorage` only. +fn deletion_object_store_config() -> Result { + match buzz_object_store::ProviderSelection::from_env().map_err(anyhow::Error::msg)? { + buzz_object_store::ProviderSelection::Gcs { bucket } => { + Ok(buzz_object_store::ObjectStoreConfig::Gcs( + buzz_object_store::GcsStoreConfig::new(bucket), + )) + } + buzz_object_store::ProviderSelection::S3 => { + let (access_key, secret_key) = s3_key_pair_from_env(); + Ok(buzz_object_store::ObjectStoreConfig::S3( + buzz_object_store::S3StoreConfig { + endpoint: required_env("BUZZ_S3_ENDPOINT")?, + access_key, + secret_key, + bucket: required_env("BUZZ_S3_BUCKET")?, + region: s3_region_from_env(), + addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + .unwrap_or_else(|_| "path".to_string()) + .parse() + .map_err(anyhow::Error::msg)?, + }, + )) + } + } +} + fn s3_region_from_env() -> String { resolve_s3_region( std::env::var("BUZZ_S3_REGION").ok(), @@ -1605,6 +1617,12 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + fn s3_media_storage(config: buzz_object_store::S3StoreConfig) -> Arc { + let store = + buzz_object_store::S3ObjectStore::new(&config).expect("construct S3 test object store"); + Arc::new(MediaStorage::with_store(Arc::new(store))) + } + #[test] fn submit_host_prefers_explicit_host() { assert_eq!( @@ -1693,25 +1711,14 @@ mod tests { .expect("runnable deletion request"); let services = Services { store, - media: Arc::new( - MediaStorage::new(&buzz_media::MediaConfig { - s3_endpoint: "http://127.0.0.1:1".to_string(), - s3_access_key: "unused".to_string(), - s3_secret_key: "unused".to_string(), - s3_bucket: "unused".to_string(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: buzz_media::S3AddressingStyle::Path, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }) - .expect("construct unused media service"), - ), + media: s3_media_storage(buzz_object_store::S3StoreConfig { + endpoint: "http://127.0.0.1:1".to_string(), + access_key: "unused".to_string(), + secret_key: "unused".to_string(), + bucket: "unused".to_string(), + region: "us-east-1".to_string(), + addressing_style: buzz_object_store::S3AddressingStyle::Path, + }), redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .expect("construct unused Redis pool"), @@ -1794,27 +1801,16 @@ mod tests { let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) .expect("BUZZ_TEST_S3_BUCKET or BUZZ_S3_BUCKET is required"); - Arc::new( - MediaStorage::new(&buzz_media::MediaConfig { - s3_endpoint: endpoint, - s3_access_key: access_key, - s3_secret_key: secret_key, - s3_bucket: bucket, - s3_region: std::env::var("BUZZ_TEST_S3_REGION") - .or_else(|_| std::env::var("BUZZ_S3_REGION")) - .unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style: buzz_media::S3AddressingStyle::Path, - max_image_bytes: 1, - max_gif_bytes: 1, - max_video_bytes: 1, - max_file_bytes: 1, - public_base_url: "http://localhost/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - }) - .expect("construct deletion test media service"), - ) + s3_media_storage(buzz_object_store::S3StoreConfig { + endpoint, + access_key, + secret_key, + bucket, + region: std::env::var("BUZZ_TEST_S3_REGION") + .or_else(|_| std::env::var("BUZZ_S3_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style: buzz_object_store::S3AddressingStyle::Path, + }) } #[tokio::test] diff --git a/crates/buzz-media/Cargo.toml b/crates/buzz-media/Cargo.toml index 7808ecaff43..059653f7f61 100644 --- a/crates/buzz-media/Cargo.toml +++ b/crates/buzz-media/Cargo.toml @@ -9,6 +9,7 @@ description = "Media storage, validation, and thumbnail generation for Buzz" [dependencies] buzz-core = { workspace = true } +buzz-object-store = { workspace = true } nostr = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -21,7 +22,6 @@ chrono = { workspace = true } ulid = "1" uuid = { workspace = true } axum = { workspace = true } -s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } infer = "0.19" image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] } blurhash = "0.2" @@ -32,7 +32,6 @@ tempfile = "3" tokio-util = { version = "0.7", features = ["io"] } futures-util = "0.3" futures-core = "0.3" -quick-xml = { version = "0.38", features = ["serialize"] } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/buzz-media/src/config.rs b/crates/buzz-media/src/config.rs index 3c70e4afe19..f1b9c9a5675 100644 --- a/crates/buzz-media/src/config.rs +++ b/crates/buzz-media/src/config.rs @@ -1,37 +1,4 @@ -//! Media storage configuration. - -use std::str::FromStr; - -/// S3 URL addressing style shared by media and Git/CAS storage. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum S3AddressingStyle { - /// Put the bucket in the request path (`https://endpoint/bucket/key`). - /// - /// This preserves compatibility with the bundled MinIO deployments, whose - /// internal DNS only resolves the endpoint hostname. - #[default] - Path, - /// Put the bucket in the hostname (`https://bucket.endpoint/key`). - /// - /// This is the standard S3 form and is required by providers such as new - /// Railway Storage Buckets. - Virtual, -} - -impl FromStr for S3AddressingStyle { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "path" => Ok(Self::Path), - "virtual" => Ok(Self::Virtual), - _ => Err(format!( - "BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}" - )), - } - } -} +//! Provider-neutral media behavior configuration. fn default_max_video_bytes() -> u64 { 524_288_000 // 500 MB @@ -41,32 +8,12 @@ fn default_max_file_bytes() -> u64 { 104_857_600 // 100 MB } -fn default_s3_region() -> String { - "us-east-1".to_string() -} - -/// Configuration for media storage (S3/MinIO). +/// Configuration for media validation, upload policy, and public URLs. +/// +/// Provider connection settings belong to the process composition root and +/// are deliberately absent here. #[derive(Debug, Clone, serde::Deserialize)] pub struct MediaConfig { - /// S3-compatible endpoint URL (e.g. "http://localhost:9000"). - pub s3_endpoint: String, - /// S3 access key. - pub s3_access_key: String, - /// S3 secret key. - pub s3_secret_key: String, - /// S3 bucket name. - pub s3_bucket: String, - /// AWS region for SigV4 request signing (e.g. "us-west-2"). - /// - /// Must match the region of `s3_endpoint` for real AWS S3, otherwise - /// requests are signed with the wrong credential scope and AWS rejects - /// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where - /// the value is not meaningfully checked. - #[serde(default = "default_s3_region")] - pub s3_region: String, - /// S3 URL addressing style. Defaults to path style for MinIO compatibility. - #[serde(default)] - pub s3_addressing_style: S3AddressingStyle, /// Maximum upload size for images (bytes). Default: 50 MB. pub max_image_bytes: u64, /// Maximum upload size for animated GIFs (bytes). Default: 10 MB. @@ -159,17 +106,10 @@ impl MediaConfig { #[cfg(test)] mod tests { - use super::{MediaConfig, S3AddressingStyle}; - use std::str::FromStr; + use super::MediaConfig; fn valid_config() -> MediaConfig { MediaConfig { - s3_endpoint: "http://localhost:9000".to_string(), - s3_access_key: "k".to_string(), - s3_secret_key: "s".to_string(), - s3_bucket: "buzz-media".to_string(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: S3AddressingStyle::Path, max_image_bytes: 1, max_gif_bytes: 1, max_video_bytes: 1, @@ -181,35 +121,6 @@ mod tests { } } - #[test] - fn addressing_style_parses_supported_values() { - assert_eq!( - S3AddressingStyle::from_str("path"), - Ok(S3AddressingStyle::Path) - ); - assert_eq!( - S3AddressingStyle::from_str("virtual"), - Ok(S3AddressingStyle::Virtual) - ); - } - - #[test] - fn addressing_style_defaults_to_path() { - assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path); - } - - #[test] - fn addressing_style_rejects_unknown_or_ambiguous_values() { - for invalid in ["", "auto", "PATH", "virtual-hosted"] { - let error = - S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style"); - assert!( - error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"), - "unexpected error for {invalid:?}: {error}" - ); - } - } - #[test] fn upload_record_knobs_default_off_and_validate() { assert!(valid_config().validate().is_ok()); diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index 5abbea6f580..0ea13a5104b 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -97,9 +97,16 @@ impl From for MediaError { } } -impl From for MediaError { - fn from(e: s3::error::S3Error) -> Self { - Self::StorageError(e.to_string()) +impl From for MediaError { + /// A missing object is a media-level `NotFound`; everything else — a + /// throttle, an ambiguous transport outcome, a permanent provider failure, + /// a misconfiguration — is opaque to media callers and collapses to + /// `StorageError`. + fn from(e: buzz_object_store::ObjectStoreError) -> Self { + match e { + buzz_object_store::ObjectStoreError::NotFound { .. } => Self::NotFound, + other => Self::StorageError(other.to_string()), + } } } diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 3198e1f8301..5bf598683d9 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -18,7 +18,7 @@ pub use bucket_index::{ BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, Page, SweepError, TaxonomySweepOutcome, }; -pub use config::{MediaConfig, S3AddressingStyle}; +pub use config::MediaConfig; pub use error::MediaError; pub use storage::{ BlobHeadMeta, BlobMeta, BulkDeleteOutcome, ByteStream, MediaStorage, ObjectVersionEntry, diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index abe6bdd40ea..574434d0306 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -1,440 +1,162 @@ -//! S3/MinIO storage client. +//! Media storage facade. +//! +//! Thin domain layer over [`buzz_object_store::ObjectStore`]: it owns the +//! media key vocabulary (content-addressed blobs, community-scoped metadata +//! sidecars) and the media error surface, and knows nothing about which +//! provider is underneath. -use std::collections::HashMap; use std::path::Path; use std::pin::Pin; +use std::sync::Arc; use buzz_core::tenant::{CommunityId, TenantContext}; +use buzz_object_store::{ObjectStore, ObjectStoreError}; -use crate::config::{MediaConfig, S3AddressingStyle}; use crate::error::MediaError; use bytes::Bytes; -use quick_xml::events::{BytesStart, Event}; -use quick_xml::Reader; -use s3::creds::Credentials; -use s3::request::Request as _; -use s3::{Bucket, Region}; use serde::{Deserialize, Serialize}; -/// A stream of byte chunks from S3, usable with `axum::body::Body::from_stream()`. -pub type ByteStream = Pin> + Send>>; - -/// The kind of versioned S3 object-store entry. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ObjectVersionKind { - /// A concrete object version with bytes. - Object, - /// A delete-marker version hiding older bytes from live-object listing. - DeleteMarker, -} - -/// One S3 object version or delete marker under a tenant prefix. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObjectVersionEntry { - /// Object key. - pub key: String, - /// Concrete S3 version id. - pub version_id: String, - /// Whether this entry is a byte-bearing object or delete marker. - pub kind: ObjectVersionKind, - /// Byte size for object versions; zero for delete markers. - pub size: u64, -} - -/// Exact version identifier used for permanent deletion. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObjectVersionRef { - /// Object key. - pub key: String, - /// Concrete S3 version id. - pub version_id: String, -} - -/// One `ListObjectVersions` page. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ObjectVersionsPage { - /// Object versions and delete markers returned by this page. - pub entries: Vec, - /// Next key marker for truncated listings. - pub next_key_marker: Option, - /// Next version-id marker for truncated listings. - pub next_version_id_marker: Option, - /// Whether more pages remain. - pub is_truncated: bool, -} - -#[derive(Debug, Default)] -struct ListVersionFields { - key: Option, - version_id: Option, - size: Option, -} - -fn local_name(name: &[u8]) -> &[u8] { - name.rsplit(|byte| *byte == b':').next().unwrap_or(name) -} - -fn xml_error(error: impl std::fmt::Display) -> MediaError { - MediaError::StorageError(error.to_string()) -} - -fn read_element_text( - reader: &mut Reader<&[u8]>, - start: &BytesStart<'_>, -) -> Result { - reader - .read_text(start.to_end().name()) - .map(|text| text.into_owned()) - .map_err(xml_error) -} +pub use buzz_object_store::{ + BulkDeleteOutcome, ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, +}; -fn skip_element(reader: &mut Reader<&[u8]>, start: &BytesStart<'_>) -> Result<(), MediaError> { - reader - .read_to_end(start.to_end().name()) - .map_err(xml_error)?; - Ok(()) -} - -fn parse_list_version_entry( - reader: &mut Reader<&[u8]>, - start: &BytesStart<'_>, - kind: ObjectVersionKind, -) -> Result { - let mut fields = ListVersionFields::default(); - loop { - match reader.read_event().map_err(xml_error)? { - Event::Start(child) => match local_name(child.local_name().as_ref()) { - b"Key" => fields.key = Some(read_element_text(reader, &child)?), - b"VersionId" => fields.version_id = Some(read_element_text(reader, &child)?), - b"Size" => { - let size = read_element_text(reader, &child)?; - fields.size = Some(size.parse::().map_err(xml_error)?); - } - _ => skip_element(reader, &child)?, - }, - Event::Empty(child) => match local_name(child.local_name().as_ref()) { - b"Key" => fields.key = Some(String::new()), - b"VersionId" => fields.version_id = Some(String::new()), - b"Size" => fields.size = Some(0), - _ => {} - }, - Event::End(end) if end.name().as_ref() == start.to_end().name().as_ref() => { - let key = fields.key.ok_or_else(|| { - MediaError::StorageError("ListObjectVersions entry missing Key".to_string()) - })?; - let version_id = fields.version_id.ok_or_else(|| { - MediaError::StorageError( - "ListObjectVersions entry missing VersionId".to_string(), - ) - })?; - return Ok(ObjectVersionEntry { - key, - version_id, - kind, - size: if kind == ObjectVersionKind::Object { - fields.size.unwrap_or(0) - } else { - 0 - }, - }); - } - Event::Eof => { - return Err(MediaError::StorageError( - "unexpected EOF inside ListObjectVersions entry".to_string(), - )); - } - _ => {} - } - } -} - -fn parse_object_versions_page(xml: &[u8]) -> Result { - let mut reader = Reader::from_reader(xml); - reader.config_mut().trim_text(true); - let mut entries = Vec::new(); - let mut next_key_marker = None; - let mut next_version_id_marker = None; - let mut is_truncated = false; - - loop { - match reader.read_event().map_err(xml_error)? { - Event::Start(start) => match local_name(start.local_name().as_ref()) { - b"Version" => entries.push(parse_list_version_entry( - &mut reader, - &start, - ObjectVersionKind::Object, - )?), - b"DeleteMarker" => entries.push(parse_list_version_entry( - &mut reader, - &start, - ObjectVersionKind::DeleteMarker, - )?), - b"IsTruncated" => { - let value = read_element_text(&mut reader, &start)?; - is_truncated = value.eq_ignore_ascii_case("true"); - } - b"NextKeyMarker" => { - next_key_marker = Some(read_element_text(&mut reader, &start)?); - } - b"NextVersionIdMarker" => { - next_version_id_marker = Some(read_element_text(&mut reader, &start)?); - } - b"ListVersionsResult" => {} - _ => skip_element(&mut reader, &start)?, - }, - Event::Empty(start) => match local_name(start.local_name().as_ref()) { - b"NextKeyMarker" => next_key_marker = Some(String::new()), - b"NextVersionIdMarker" => next_version_id_marker = Some(String::new()), - _ => {} - }, - Event::Eof => break, - _ => {} - } - } - - Ok(ObjectVersionsPage { - entries, - next_key_marker, - next_version_id_marker, - is_truncated, - }) -} +/// A stream of byte chunks from object storage, usable with +/// `axum::body::Body::from_stream()`. +pub type ByteStream = Pin> + Send>>; -/// S3-compatible object storage client. +/// Media object storage client. pub struct MediaStorage { - bucket: Box, + store: Arc, } impl MediaStorage { - /// Create a new storage client from config. + /// Wrap an already-constructed object store. /// - /// Credential selection: - /// - If both `s3_access_key` and `s3_secret_key` are non-empty, use them as - /// static credentials (MinIO/local/dev, or any static-key deployment). - /// - Otherwise, fall back to the AWS default credential chain via - /// [`Credentials::default`]: environment, shared profile, web-identity - /// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and - /// instance-metadata providers, in that order. This lets the relay use - /// the pod's IAM role without long-lived static keys. - pub fn new(config: &MediaConfig) -> Result { - let region = Region::Custom { - region: config.s3_region.clone(), - endpoint: config.s3_endpoint.clone(), - }; - let creds = match ( - config.s3_access_key.is_empty(), - config.s3_secret_key.is_empty(), - ) { - (false, false) => Credentials::new( - Some(&config.s3_access_key), - Some(&config.s3_secret_key), - None, - None, - None, - ), - (true, true) => { - // No static keys configured: resolve from the AWS credential chain - // (IRSA web-identity, env, profile, instance metadata). - Credentials::default() - } - _ => { - return Err(MediaError::StorageError( - "s3_access_key and s3_secret_key must be configured together, or both empty to use the AWS credential chain" - .to_string(), - )); - } - } - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let bucket = Bucket::new(&config.s3_bucket, region, creds) - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let bucket = match config.s3_addressing_style { - S3AddressingStyle::Path => bucket.with_path_style(), - S3AddressingStyle::Virtual => bucket, - }; - Ok(Self { bucket }) + /// The relay builds one provider per process and shares it between media + /// and Git storage rather than opening a second client against the same + /// bucket. + pub fn with_store(store: Arc) -> Self { + Self { store } + } + + /// The shared object store behind this facade, for handing to another + /// domain facade (see [`MediaStorage::with_store`]). + pub fn object_store(&self) -> Arc { + Arc::clone(&self.store) } /// Store an object from a byte slice. /// /// Used for images, sidecars, and thumbnails. For large video files use - /// [`put_file`] to avoid loading the entire blob into RAM. + /// [`MediaStorage::put_file`] to avoid loading the entire blob into RAM. pub async fn put(&self, key: &str, bytes: &[u8], content_type: &str) -> Result<(), MediaError> { - self.bucket - .put_object_with_content_type(key, bytes, content_type) - .await?; - Ok(()) + self.store + .put(key, bytes, content_type) + .await + .map_err(storage_error) } - /// Stream a file from disk into S3 without loading it into RAM. + /// Stream a file from disk into object storage without loading it into RAM. /// - /// Uses rust-s3's `put_object_stream_with_content_type` which reads from - /// the file incrementally via an 8 MiB `BufReader`. The full file is never - /// held in memory simultaneously. Intended for video blobs (up to 500 MB). + /// The full file is never held in memory simultaneously. Intended for + /// video blobs (up to 500 MB). pub async fn put_file( &self, key: &str, path: &Path, content_type: &str, ) -> Result<(), MediaError> { - const BUF: usize = 8 * 1024 * 1024; // 8 MiB read buffer - - let file = tokio::fs::File::open(path) + self.store + .put_file(key, path, content_type) .await - .map_err(|e| MediaError::Io(e.to_string()))?; - let mut reader = tokio::io::BufReader::with_capacity(BUF, file); - - self.bucket - .put_object_stream_with_content_type(&mut reader, key, content_type) - .await?; - Ok(()) + .map_err(storage_error) } /// Retrieve an object's bytes. pub async fn get(&self, key: &str) -> Result, MediaError> { - match self.bucket.get_object(key).await { - Ok(response) => Ok(response.to_vec()), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.get(key).await?.to_vec()) } - /// Retrieve a byte range from an object via S3-native `Range` GET. + /// Retrieve a byte range from an object via a native ranged GET. /// /// `start` and `end` are inclusive byte offsets. Only the requested slice - /// is transferred from S3 — the full object is never loaded into RAM. - /// Intended for HTTP 206 range responses on large video blobs. + /// is transferred — the full object is never loaded into RAM. Intended for + /// HTTP 206 range responses on large video blobs. pub async fn get_range(&self, key: &str, start: u64, end: u64) -> Result, MediaError> { - match self.bucket.get_object_range(key, start, Some(end)).await { - Ok(response) => Ok(response.to_vec()), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.get_range(key, start, end).await?.to_vec()) } - /// Stream an object's bytes from S3 without loading into RAM. + /// Stream an object's bytes without loading them into RAM. /// /// Returns a pinned stream of `Result` chunks. /// The full object is never buffered — intended for streaming large /// blobs (video) directly into HTTP responses via `Body::from_stream()`. pub async fn get_stream(&self, key: &str) -> Result { - let response = self - .bucket - .get_object_stream(key) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - - if response.status_code == 404 { - return Err(MediaError::NotFound); - } - - let stream = futures_util::StreamExt::map(response.bytes, |chunk| { - chunk.map_err(|e| MediaError::StorageError(e.to_string())) - }); - Ok(Box::pin(stream)) + let stream = self.store.get_stream(key).await?; + Ok(Box::pin(futures_util::StreamExt::map(stream, |chunk| { + chunk.map_err(storage_error) + }))) } - /// Check if an object exists. Returns false on 404. + /// Check if an object exists. Returns false when absent. pub async fn head(&self, key: &str) -> Result { - match self.bucket.head_object(key).await { - Ok(_) => Ok(true), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(false), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self.store.head(key).await.map_err(storage_error)?.is_some()) } /// Delete an object. Returns an error on failure — callers decide whether to propagate. pub async fn delete(&self, key: &str) -> Result<(), MediaError> { - self.bucket - .delete_object(key) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(()) + self.store.delete(key).await.map_err(storage_error) } - /// HEAD with metadata — returns Content-Length (size). + /// HEAD with metadata — returns the object size. pub async fn head_with_metadata(&self, key: &str) -> Result, MediaError> { - match self.bucket.head_object(key).await { - Ok((result, _)) => Ok(Some(BlobHeadMeta { - size: result.content_length.unwrap_or(0) as u64, - })), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(None), - Err(e) => Err(MediaError::StorageError(e.to_string())), - } + Ok(self + .store + .head(key) + .await + .map_err(storage_error)? + .map(|meta| BlobHeadMeta { size: meta.size })) + } + + /// Detect whether the bucket retains non-current object versions. + /// + /// Deletion refuses versioned buckets because bulk deletes without a + /// version qualifier would only insert delete markers, not prove logical + /// absence. + pub async fn bucket_versioning_detected(&self) -> Result { + self.store + .versioning_detected() + .await + .map_err(storage_error) } - /// Bulk-delete up to one manifest chunk of keys via S3 `DeleteObjects`. + /// Bulk-delete up to one manifest chunk of keys. /// /// Never fails on per-key outcomes: they are folded into /// [`BulkDeleteOutcome`] so the caller owns retry/fail-closed policy. - /// Historical MinIO releases report already-absent keys as - /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to - /// `already_missing` to keep checkpointed retry idempotent. pub async fn delete_objects(&self, keys: &[String]) -> Result { - if keys.is_empty() { - return Ok(BulkDeleteOutcome::default()); - } - let identifiers = keys - .iter() - .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) - .collect::>(); - self.delete_object_identifiers(identifiers).await + self.store.delete_objects(keys).await.map_err(storage_error) } - /// Non-destructively verify that versioned bucket APIs are reachable. - /// - /// `ListObjectVersions` can be proven without mutation. S3 has no equivalent - /// dry-run for `DeleteObjectVersion`: `DeleteObjects` is always destructive, - /// even for exact versions, and deleting a fabricated version id does not - /// prove permission when policies can be prefix- or tag-constrained. - /// Operators must still provision `s3:DeleteObjectVersion`; the first exact - /// version deletion remains the destructive proof. + /// Non-destructively verify that exact-version listing is reachable. pub async fn preflight_version_listing(&self, prefix: &str) -> Result<(), MediaError> { self.list_prefix_versions_page(prefix, None, None, 1) .await .map(|_| ()) } - /// Bulk-delete exact object versions via S3 `DeleteObjects`. + /// Permanently delete exact provider versions. /// - /// Every identifier includes a version id, so this removes historical - /// versions and delete markers permanently instead of adding another - /// delete marker to a versioned bucket. + /// The provider translates the opaque `version_id` to an S3 version ID or + /// a GCS generation. Domain deletion code never performs that translation. pub async fn delete_object_versions( &self, versions: &[ObjectVersionRef], ) -> Result { - self.delete_object_versions_with_folding(versions, fold_version_delete_result) - .await - } - - async fn delete_object_versions_with_folding( - &self, - versions: &[ObjectVersionRef], - fold: fn(s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome, - ) -> Result { - if versions.is_empty() { - return Ok(BulkDeleteOutcome::default()); - } - let identifiers = object_version_identifiers(versions); - let result = self - .bucket - .delete_objects(identifiers) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(fold(result)) - } - - async fn delete_object_identifiers( - &self, - identifiers: Vec, - ) -> Result { - let result = self - .bucket - .delete_objects(identifiers) + self.store + .delete_versions(versions) .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - Ok(fold_bulk_delete_result(result)) + .map_err(storage_error) } /// Build the community-scoped sidecar key for a given sha256 (bare hash). @@ -459,8 +181,8 @@ impl MediaStorage { sha256: &str, ) -> Result { let key = Self::ctx_sidecar_key(ctx, sha256); - let resp = self.bucket.get_object(&key).await?; - let meta: BlobMeta = serde_json::from_slice(&resp.to_vec())?; + let bytes = self.store.get(&key).await.map_err(storage_error)?; + let meta: BlobMeta = serde_json::from_slice(&bytes)?; Ok(meta) } @@ -496,15 +218,14 @@ impl MediaStorage { /// Probe object-store connectivity and bucket access. pub async fn ping(&self) -> Result<(), MediaError> { - self.list_page(None, 1).await.map(|_| ()) + self.store.ping().await.map_err(storage_error) } - /// One page of a full-bucket listing, for the storage sweep. Wraps - /// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which - /// has no cap) and converts the result into the storage-agnostic - /// [`crate::bucket_index::Page`] shape the pure fold consumes. + /// One page of a full-bucket listing, for the storage sweep. Converts the + /// provider listing into the storage-agnostic [`crate::bucket_index::Page`] + /// shape the pure fold consumes. /// - /// `max_keys` bounds one HTTP response, not the sweep's total object + /// `max_keys` bounds one provider response, not the sweep's total object /// cap — the caller (`fold_bucket_listing`) enforces the cumulative cap /// across pages. pub async fn list_page( @@ -520,44 +241,32 @@ impl MediaStorage { /// /// Deletion enumerates the target community's exact key prefixes with /// this instead of listing the whole fleet bucket: cost stays - /// O(tenant objects) regardless of fleet size. `ListObjectsV2` returns - /// keys in ascending UTF-8 binary order, which callers rely on for - /// streaming key-stream digests. + /// O(tenant objects) regardless of fleet size. Listings return keys in + /// ascending UTF-8 binary order, which callers rely on for streaming + /// key-stream digests. pub async fn list_prefix_page( &self, prefix: &str, continuation_token: Option, max_keys: usize, ) -> Result { - let (result, _status) = self - .bucket - .list_page( - prefix.to_string(), - None, - continuation_token, - None, - Some(max_keys), - ) - .await?; + let page = self + .store + .list_page(prefix, continuation_token, max_keys) + .await + .map_err(storage_error)?; Ok(crate::bucket_index::Page { - objects: result - .contents - .into_iter() - .map(|obj| (obj.key, obj.size)) - .collect(), - next_continuation_token: result.next_continuation_token, - is_truncated: result.is_truncated, + objects: page.objects, + next_continuation_token: page.next_continuation_token, + is_truncated: page.is_truncated, }) } - /// One page of object versions and delete markers under a prefix. + /// One page of exact object versions under a prefix. /// - /// This uses S3 `ListObjectVersions` (`?versions`) instead of - /// `ListObjectsV2`: versioned buckets can be logically empty while still - /// retaining historical versions or delete markers, and permanent deletion - /// must enumerate both. Pagination must carry both `KeyMarker` and - /// `VersionIdMarker`; carrying only the key marker can skip siblings when a - /// key has multiple versions on a page boundary. + /// Both cursor fields are opaque and must be replayed together. S3 uses + /// the pair directly; GCS places its page token in `key_marker` and leaves + /// `version_id_marker` empty. pub async fn list_prefix_versions_page( &self, prefix: &str, @@ -565,113 +274,22 @@ impl MediaStorage { version_id_marker: Option, max_keys: usize, ) -> Result { - let mut query = HashMap::from([ - ("versions".to_string(), String::new()), - ("prefix".to_string(), prefix.to_string()), - ("max-keys".to_string(), max_keys.to_string()), - ]); - if let Some(marker) = key_marker { - query.insert("key-marker".to_string(), marker); - } - if let Some(marker) = version_id_marker { - query.insert("version-id-marker".to_string(), marker); - } - let bucket = self - .bucket - .with_extra_query(query) - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let request = s3::request::tokio_backend::ReqwestRequest::new( - &bucket, - "/", - s3::command::Command::GetObject, - ) - .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - let response = request - .response_data(false) + self.store + .list_versions_page(prefix, key_marker, version_id_marker, max_keys) .await - .map_err(|e| MediaError::StorageError(e.to_string()))?; - if response.status_code() >= 300 { - return Err(MediaError::StorageError(format!( - "list object versions failed with status {}: {}", - response.status_code(), - response.as_str().unwrap_or("") - ))); - } - parse_object_versions_page(response.as_slice()) + .map_err(storage_error) } } -/// Per-key outcomes of one bulk `DeleteObjects` call. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct BulkDeleteOutcome { - /// Keys the backend reported deleted (S3 reports already-missing keys as - /// deleted too — the API is idempotent by design). - pub deleted: u64, - /// Keys reported absent via legacy MinIO `NoSuchKey`/`NoSuchVersion` - /// per-key errors; equivalent to deleted for retry purposes. - pub already_missing: u64, - /// Keys whose deletion produced a version artifact (delete marker or - /// version id) — evidence of bucket versioning, which deletion must - /// fail closed on. - pub versioned_keys: Vec, - /// Remaining per-key failures as `(key, code, message)`. - pub failed: Vec<(String, String, String)>, -} - -fn object_version_identifiers( - versions: &[ObjectVersionRef], -) -> Vec { - versions - .iter() - .map(|version| { - s3::serde_types::ObjectIdentifier::with_version( - version.key.clone(), - version.version_id.clone(), - ) - }) - .collect() -} - -fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { - fold_delete_result(result, DeleteMode::Unversioned) -} - -fn fold_version_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { - fold_delete_result(result, DeleteMode::ExplicitVersion) -} - -enum DeleteMode { - Unversioned, - ExplicitVersion, -} - -fn fold_delete_result( - result: s3::serde_types::DeleteObjectsResult, - mode: DeleteMode, -) -> BulkDeleteOutcome { - let mut outcome = BulkDeleteOutcome::default(); - for deleted in result.deleted { - let has_version_artifact = deleted.delete_marker == Some(true) - || deleted.delete_marker_version_id.is_some() - || deleted.version_id.is_some(); - match mode { - DeleteMode::Unversioned if has_version_artifact => { - outcome.versioned_keys.push(deleted.key); - } - DeleteMode::Unversioned | DeleteMode::ExplicitVersion => { - outcome.deleted += 1; - } - } - } - for error in result.errors { - if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { - outcome.already_missing += 1; - } else { - outcome.failed.push((error.key, error.code, error.message)); - } - } - outcome +/// Collapse any provider failure — including a missing object — into the +/// generic media storage error. +/// +/// Used on the paths that historically surfaced a backend 404 as a storage +/// failure rather than a media-level `NotFound`: sidecar reads are guarded by +/// an explicit HEAD, and a bare absence there means the bucket disagreed with +/// the guard. +fn storage_error(error: ObjectStoreError) -> MediaError { + MediaError::StorageError(error.to_string()) } #[cfg(test)] @@ -679,271 +297,6 @@ mod tests { use super::*; use std::collections::HashMap; - /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO - /// absent-key errors count as success, version artifacts are surfaced for - /// fail-closed handling, and anything else stays a per-key failure. - #[test] - fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { - use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; - let deleted_object = |key: &str, marker: bool| DeletedObject { - key: key.to_string(), - version_id: None, - delete_marker: marker.then_some(true), - delete_marker_version_id: marker.then(|| "v1".to_string()), - }; - let delete_error = |key: &str, code: &str, message: &str| DeleteError { - key: key.to_string(), - code: code.to_string(), - message: message.to_string(), - version_id: None, - }; - let result = DeleteObjectsResult { - deleted: vec![ - deleted_object("plain", false), - deleted_object("marked", true), - ], - errors: vec![ - delete_error("gone", "NoSuchKey", "absent"), - delete_error("gone-version", "NoSuchVersion", "absent"), - delete_error("denied", "AccessDenied", "nope"), - ], - }; - let outcome = fold_bulk_delete_result(result); - assert_eq!(outcome.deleted, 1); - assert_eq!(outcome.already_missing, 2); - assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); - assert_eq!( - outcome.failed, - vec![( - "denied".to_string(), - "AccessDenied".to_string(), - "nope".to_string() - )] - ); - } - - #[test] - fn version_delete_fold_counts_explicit_version_artifacts_as_deleted() { - use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; - let result = DeleteObjectsResult { - deleted: vec![DeletedObject { - key: "versioned".to_string(), - version_id: Some("v1".to_string()), - delete_marker: Some(true), - delete_marker_version_id: Some("v1".to_string()), - }], - errors: vec![ - DeleteError { - key: "retried-version".to_string(), - code: "NoSuchVersion".to_string(), - message: "already absent".to_string(), - version_id: Some("v-gone".to_string()), - }, - DeleteError { - key: "denied-version".to_string(), - code: "AccessDenied".to_string(), - message: "denied".to_string(), - version_id: Some("v-denied".to_string()), - }, - ], - }; - - let outcome = fold_version_delete_result(result); - assert_eq!(outcome.deleted, 1); - assert_eq!(outcome.already_missing, 1); - assert!(outcome.versioned_keys.is_empty()); - assert_eq!( - outcome.failed, - vec![( - "denied-version".to_string(), - "AccessDenied".to_string(), - "denied".to_string() - )] - ); - } - - #[test] - fn object_version_identifiers_include_explicit_version_ids() { - let identifiers = object_version_identifiers(&[ - ObjectVersionRef { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-object".to_string(), - }, - ObjectVersionRef { - key: "uploads/tenant/event/blob".to_string(), - version_id: "v-delete-marker".to_string(), - }, - ]); - - assert_eq!(identifiers.len(), 2); - assert_eq!(identifiers[0].key, "_meta/tenant/a.json"); - assert_eq!(identifiers[0].version_id.as_deref(), Some("v-object")); - assert_eq!(identifiers[1].key, "uploads/tenant/event/blob"); - assert_eq!( - identifiers[1].version_id.as_deref(), - Some("v-delete-marker") - ); - } - - #[tokio::test] - async fn delete_object_versions_empty_input_short_circuits_before_folding() { - let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("static client"); - let outcome = storage - .delete_object_versions_with_folding(&[], |_| BulkDeleteOutcome { - deleted: 0, - already_missing: 0, - versioned_keys: vec!["wrong-fold".to_string()], - failed: Vec::new(), - }) - .await - .expect("empty delete short-circuits before fold"); - assert_eq!(outcome, BulkDeleteOutcome::default()); - } - - #[test] - fn parse_object_versions_page_includes_objects_delete_markers_and_dual_markers() { - let page = parse_object_versions_page( - br#" - - buzz-media - _meta/tenant/ - _meta/tenant/a.json - v-old - 2 - true - _meta/tenant/a.json - v-new - - _meta/tenant/a.json - v-delete - true - - - _meta/tenant/a.json - v-new - false - 42 - -"#, - ) - .expect("parse versions page"); - - assert!(page.is_truncated); - assert_eq!(page.next_key_marker.as_deref(), Some("_meta/tenant/a.json")); - assert_eq!(page.next_version_id_marker.as_deref(), Some("v-new")); - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-delete".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "_meta/tenant/a.json".to_string(), - version_id: "v-new".to_string(), - kind: ObjectVersionKind::Object, - size: 42, - }, - ] - ); - } - - #[test] - fn parse_object_versions_page_preserves_repeated_interleaved_aws_ordering() { - let page = parse_object_versions_page( - br#" - k-av33 - k-av2 - k-av11 - k-bm2 - k-bm110 -"#, - ) - .expect("parse interleaved versions page"); - - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v3".to_string(), - kind: ObjectVersionKind::Object, - size: 3, - }, - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v2".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-a".to_string(), - version_id: "v1".to_string(), - kind: ObjectVersionKind::Object, - size: 1, - }, - ObjectVersionEntry { - key: "k-b".to_string(), - version_id: "m2".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-b".to_string(), - version_id: "m1".to_string(), - kind: ObjectVersionKind::Object, - size: 10, - }, - ] - ); - } - - #[test] - fn parse_object_versions_page_handles_marker_only_key_before_versioned_key() { - let page = parse_object_versions_page( - br#" - k-marker-onlyd-only - k-versionedv220 - k-versionedd1 - k-versionedv110 -"#, - ) - .expect("parse marker-only and versioned keys"); - - assert_eq!( - page.entries, - vec![ - ObjectVersionEntry { - key: "k-marker-only".to_string(), - version_id: "d-only".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "v2".to_string(), - kind: ObjectVersionKind::Object, - size: 20, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "d1".to_string(), - kind: ObjectVersionKind::DeleteMarker, - size: 0, - }, - ObjectVersionEntry { - key: "k-versioned".to_string(), - version_id: "v1".to_string(), - kind: ObjectVersionKind::Object, - size: 10, - }, - ] - ); - } - fn tenant(n: u128) -> TenantContext { TenantContext::resolved( CommunityId::from_uuid(uuid::Uuid::from_u128(n)), @@ -951,76 +304,6 @@ mod tests { ) } - fn storage_config(access: &str, secret: &str) -> crate::config::MediaConfig { - crate::config::MediaConfig { - s3_endpoint: "http://localhost:9000".to_string(), - s3_access_key: access.to_string(), - s3_secret_key: secret.to_string(), - s3_bucket: "buzz-media".to_string(), - s3_region: "us-west-2".to_string(), - s3_addressing_style: S3AddressingStyle::Path, - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, - } - } - - /// Static keys present: builds a client without touching the AWS - /// credential chain (no env/metadata access), and the signing region - /// comes from config rather than a hardcoded "us-east-1". - #[test] - fn static_keys_build_client_with_configured_region() { - let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("static creds should build a client"); - match storage.bucket.region { - Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"), - other => panic!("expected Custom region, got {other:?}"), - } - } - - #[test] - fn client_constructor_applies_both_addressing_styles() { - let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret")) - .expect("path-style client"); - assert!(path.bucket.is_path_style()); - assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media"); - - let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret"); - virtual_config.s3_addressing_style = S3AddressingStyle::Virtual; - let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client"); - assert!(virtual_hosted.bucket.is_subdomain_style()); - assert_eq!( - virtual_hosted.bucket.url(), - "http://buzz-media.localhost:9000" - ); - } - - #[test] - fn partial_static_keys_are_rejected() { - let err = match MediaStorage::new(&storage_config("buzz_dev", "")) { - Ok(_) => panic!("partial static creds must not silently use credential chain"), - Err(err) => err, - }; - assert!( - err.to_string().contains("must be configured together"), - "unexpected error: {err}" - ); - - let err = match MediaStorage::new(&storage_config("", "buzz_dev_secret")) { - Ok(_) => panic!("partial static creds must not silently use credential chain"), - Err(err) => err, - }; - assert!( - err.to_string().contains("must be configured together"), - "unexpected error: {err}" - ); - } - #[test] fn tenant_key_writers_are_covered_by_deletion_taxonomy() { let ctx = tenant(1); diff --git a/crates/buzz-media/src/upload.rs b/crates/buzz-media/src/upload.rs index 524b033280d..b9b11904943 100644 --- a/crates/buzz-media/src/upload.rs +++ b/crates/buzz-media/src/upload.rs @@ -565,12 +565,6 @@ mod tests { fn test_config() -> MediaConfig { MediaConfig { - s3_endpoint: String::new(), - s3_access_key: String::new(), - s3_secret_key: String::new(), - s3_bucket: String::new(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: crate::config::S3AddressingStyle::Path, max_image_bytes: 50 * 1024 * 1024, max_gif_bytes: 10 * 1024 * 1024, max_video_bytes: 524_288_000, diff --git a/crates/buzz-media/src/validation.rs b/crates/buzz-media/src/validation.rs index 706c354d043..62ecc9553d3 100644 --- a/crates/buzz-media/src/validation.rs +++ b/crates/buzz-media/src/validation.rs @@ -960,12 +960,6 @@ mod tests { fn test_config() -> MediaConfig { MediaConfig { - s3_endpoint: String::new(), - s3_access_key: String::new(), - s3_secret_key: String::new(), - s3_bucket: String::new(), - s3_region: "us-east-1".to_string(), - s3_addressing_style: crate::config::S3AddressingStyle::Path, max_image_bytes: 50 * 1024 * 1024, max_gif_bytes: 10 * 1024 * 1024, max_video_bytes: 524_288_000, diff --git a/crates/buzz-media/tests/static_creds_minio.rs b/crates/buzz-media/tests/static_creds_minio.rs index 4c8c10702cb..eb8c9d5ec8a 100644 --- a/crates/buzz-media/tests/static_creds_minio.rs +++ b/crates/buzz-media/tests/static_creds_minio.rs @@ -18,39 +18,31 @@ //! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET` / `BUZZ_S3_REGION` / //! `BUZZ_S3_ADDRESSING_STYLE`. The default remains `path` for MinIO. -use buzz_media::config::MediaConfig; use buzz_media::storage::MediaStorage; +use buzz_object_store::{S3ObjectStore, S3StoreConfig}; -fn minio_config() -> MediaConfig { - MediaConfig { - s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") +fn minio_config() -> S3StoreConfig { + S3StoreConfig { + endpoint: std::env::var("BUZZ_S3_ENDPOINT") .unwrap_or_else(|_| "http://localhost:9000".to_string()), - s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") - .unwrap_or_else(|_| "buzz_dev".to_string()), - s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") + access_key: std::env::var("BUZZ_S3_ACCESS_KEY").unwrap_or_else(|_| "buzz_dev".to_string()), + secret_key: std::env::var("BUZZ_S3_SECRET_KEY") .unwrap_or_else(|_| "buzz_dev_secret".to_string()), - s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), - s3_region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), + region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") .unwrap_or_else(|_| "path".to_string()) .parse() .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"), - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, } } #[tokio::test] #[ignore = "requires a live MinIO (docker compose up -d minio minio-init)"] async fn static_creds_round_trip_against_minio() { - let storage = - MediaStorage::new(&minio_config()).expect("static creds should build a storage client"); + let store = S3ObjectStore::new(&minio_config()) + .expect("static creds should build an object-store client"); + let storage = MediaStorage::with_store(std::sync::Arc::new(store)); let key = format!("_test/static-creds-{}.bin", std::process::id()); let body = b"hardcoded-creds-still-work"; diff --git a/crates/buzz-media/tests/versioned_minio.rs b/crates/buzz-media/tests/versioned_minio.rs index 1e0db4b481f..7d2ac0005f0 100644 --- a/crates/buzz-media/tests/versioned_minio.rs +++ b/crates/buzz-media/tests/versioned_minio.rs @@ -18,34 +18,31 @@ use std::process::Command; -use buzz_media::config::MediaConfig; use buzz_media::storage::{MediaStorage, ObjectVersionKind, ObjectVersionRef}; +use buzz_object_store::{S3ObjectStore, S3StoreConfig}; fn env_or(name: &str, default: &str) -> String { std::env::var(name).unwrap_or_else(|_| default.to_string()) } -fn minio_config(bucket: String) -> MediaConfig { - MediaConfig { - s3_endpoint: env_or("BUZZ_S3_ENDPOINT", "http://localhost:9000"), - s3_access_key: env_or("BUZZ_S3_ACCESS_KEY", "buzz_dev"), - s3_secret_key: env_or("BUZZ_S3_SECRET_KEY", "buzz_dev_secret"), - s3_bucket: bucket, - s3_region: env_or("BUZZ_S3_REGION", "us-east-1"), - s3_addressing_style: env_or("BUZZ_S3_ADDRESSING_STYLE", "path") +fn minio_config(bucket: String) -> S3StoreConfig { + S3StoreConfig { + endpoint: env_or("BUZZ_S3_ENDPOINT", "http://localhost:9000"), + access_key: env_or("BUZZ_S3_ACCESS_KEY", "buzz_dev"), + secret_key: env_or("BUZZ_S3_SECRET_KEY", "buzz_dev_secret"), + bucket, + region: env_or("BUZZ_S3_REGION", "us-east-1"), + addressing_style: env_or("BUZZ_S3_ADDRESSING_STYLE", "path") .parse() .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"), - max_image_bytes: 50 * 1024 * 1024, - max_gif_bytes: 10 * 1024 * 1024, - max_video_bytes: 524_288_000, - max_file_bytes: 104_857_600, - public_base_url: "http://localhost:3000/media".to_string(), - upload_records_enabled: false, - upload_ip_header: None, - upload_port_header: None, } } +fn media_storage(config: &S3StoreConfig) -> MediaStorage { + let store = S3ObjectStore::new(config).expect("static MinIO object-store client"); + MediaStorage::with_store(std::sync::Arc::new(store)) +} + fn run_mc(args: &[String]) -> Result<(), String> { let container = env_or("BUZZ_MINIO_CONTAINER", "buzz-minio"); let output = Command::new("docker") @@ -133,7 +130,7 @@ async fn never_versioned_bucket_lists_null_versions_and_exact_delete_empties_lis let bucket = format!("buzz-media-never-versioned-{}", std::process::id()); let bucket_path = format!("local/{bucket}"); let config = minio_config(bucket.clone()); - mc_alias(&config.s3_access_key, &config.s3_secret_key).expect("configure mc alias"); + mc_alias(&config.access_key, &config.secret_key).expect("configure mc alias"); run_mc(&[ "mb".to_string(), "--ignore-existing".to_string(), @@ -141,7 +138,7 @@ async fn never_versioned_bucket_lists_null_versions_and_exact_delete_empties_lis ]) .expect("create isolated never-versioned test bucket"); - let storage = MediaStorage::new(&config).expect("static MinIO storage client"); + let storage = media_storage(&config); let prefix = format!("_test/never-versioned-{}/", uuid::Uuid::new_v4()); let key = format!("{prefix}plain.bin"); storage @@ -187,7 +184,7 @@ async fn versioned_bucket_exact_version_delete_reaches_final_list_versions_empti let bucket = format!("buzz-media-versioned-{}", std::process::id()); let bucket_path = format!("local/{bucket}"); let config = minio_config(bucket.clone()); - mc_alias(&config.s3_access_key, &config.s3_secret_key).expect("configure mc alias"); + mc_alias(&config.access_key, &config.secret_key).expect("configure mc alias"); run_mc(&[ "mb".to_string(), "--ignore-existing".to_string(), @@ -201,7 +198,7 @@ async fn versioned_bucket_exact_version_delete_reaches_final_list_versions_empti ]) .expect("enable bucket versioning"); - let storage = MediaStorage::new(&config).expect("static MinIO storage client"); + let storage = media_storage(&config); let prefix = format!("_test/versioned-{}/", uuid::Uuid::new_v4()); let historical_key = format!("{prefix}historical.bin"); let marker_only_key = format!("{prefix}marker-only.bin"); diff --git a/crates/buzz-object-store/Cargo.toml b/crates/buzz-object-store/Cargo.toml new file mode 100644 index 00000000000..6f88d62a8d8 --- /dev/null +++ b/crates/buzz-object-store/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "buzz-object-store" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Provider-neutral object storage seam shared by Buzz media and Git-on-object-storage" + +[dependencies] +async-trait = "0.1" +axum = { workspace = true } +bytes = "1" +futures-core = "0.3" +futures-util = "0.3" +google-cloud-gax = "1.13" +# `default-rustls-provider` would install aws-lc-rs as the process-wide rustls +# provider. Buzz installs ring at startup instead (see `buzz-relay`'s `main`), +# and two providers racing for the default is a panic at first use. +google-cloud-storage = { version = "=1.17.0", default-features = false } +quick-xml = { version = "0.38", features = ["serialize"] } +rand = { workspace = true } +s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } +serde = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +google-cloud-wkt = "1.7" +hex = { workspace = true } +sha2 = { workspace = true } +# The live GCS tests are their own process, so they install the same rustls +# provider `buzz-relay`'s `main` does before making a TLS request. +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } diff --git a/crates/buzz-object-store/src/error.rs b/crates/buzz-object-store/src/error.rs new file mode 100644 index 00000000000..ddbf83acf54 --- /dev/null +++ b/crates/buzz-object-store/src/error.rs @@ -0,0 +1,197 @@ +//! Provider-neutral object-store error taxonomy. +//! +//! The distinction that matters most here is **classified vs. unknown**. A +//! provider that answered — with a status, a body, or a malformed response — +//! told us something about the object's state. A request that never got a +//! classified answer (socket refused, send flaked, mid-flight reset) told us +//! nothing: it is neither evidence that the write committed nor that it did +//! not. The Git conformance probe drops exactly the unknown outcomes from its +//! observer set, so [`ObjectStoreError::TransportAmbiguous`] must stay +//! reserved for pre-classification failures and nothing else. + +use std::time::Duration; + +use crate::revision::ProviderKind; + +/// Everything a provider operation can fail with. +/// +/// Losing a compare-and-swap race is deliberately *not* in here — that is +/// [`crate::ConditionalWrite::Conflict`], an ordinary outcome. +#[derive(Debug, thiserror::Error)] +pub enum ObjectStoreError { + /// The requested key does not exist. + #[error("object not found: {key}")] + NotFound { + /// Object key that was addressed. + key: String, + }, + + /// A precondition failed on an operation that is not a conditional write. + /// + /// Conditional writes surface a failed precondition as + /// [`crate::ConditionalWrite::Conflict`] instead; this variant catches a + /// precondition failure arriving where no CAS semantics were requested. + #[error("precondition failed on {key}")] + Conflict { + /// Object key that was addressed. + key: String, + }, + + /// The provider rejected the request for exceeding its request rate. + /// + /// Throttling is never evidence that a writer lost a CAS race. + #[error("object store throttled the {operation} request")] + Throttled { + /// Provider operation that was throttled. + operation: &'static str, + /// Provider-advertised backoff, when the response carried one. + retry_after: Option, + }, + + /// The provider answered with a transient failure; the operation may be + /// retried under the caller's policy (with its original precondition, if + /// it had one). + #[error("retryable object store failure during {operation}: {message}")] + TransportRetryable { + /// Provider operation that failed. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// The request never produced a classified provider response, so the + /// operation's outcome is unknown. + /// + /// A conditional write that fails this way must never be retried + /// unconditionally: reread the object and classify the committed revision + /// instead. + #[error("ambiguous object store outcome during {operation}: {message}")] + TransportAmbiguous { + /// Provider operation whose outcome is unknown. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// A permanent provider or authorization failure. + #[error("object store backend error during {operation}: {message}")] + Provider { + /// Provider operation that failed. + operation: &'static str, + /// Redacted provider detail — never object bytes or credentials. + message: String, + }, + + /// Invalid storage configuration, detected at client construction. + #[error("object store config error: {0}")] + Config(String), + + /// The object is larger than the caller's bounded read budget. + #[error("object too large: {key} is {size} bytes (max {max})")] + ObjectTooLarge { + /// Object key that was read. + key: String, + /// Object size reported by the provider. + size: u64, + /// Maximum bytes the caller allows for this read. + max: u64, + }, + + /// A content-addressed read returned bytes that do not hash to the key. + #[error("digest mismatch on {key}: expected {expected}, got {actual}")] + DigestMismatch { + /// Object key that was read. + key: String, + /// Digest the caller expected (the content-addressed key). + expected: String, + /// Digest computed from the returned bytes. + actual: String, + }, + + /// A revision minted by one provider was presented to another. + #[error("revision provider mismatch: expected {expected}, got {actual}")] + RevisionMismatch { + /// Provider the operation is running against. + expected: ProviderKind, + /// Provider that minted the offered revision. + actual: ProviderKind, + }, +} + +impl ObjectStoreError { + /// Whether the operation's outcome is unknown rather than classified. + /// + /// Callers that reason about *observers* — the Git conformance probe, and + /// any future CAS retry policy — use this to drop unknown outcomes from + /// the evidence set rather than counting them as a lost race. + pub fn is_ambiguous(&self) -> bool { + matches!(self, Self::TransportAmbiguous { .. }) + } + + /// Whether the provider indicated the request may be retried as-is. + /// + /// A conditional write may only be retried with its original precondition. + pub fn is_retryable(&self) -> bool { + matches!( + self, + Self::Throttled { .. } | Self::TransportRetryable { .. } + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_pre_classification_failures_are_ambiguous() { + assert!(ObjectStoreError::TransportAmbiguous { + operation: "put", + message: "connection reset".into(), + } + .is_ambiguous()); + + for classified in [ + ObjectStoreError::NotFound { key: "k".into() }, + ObjectStoreError::Conflict { key: "k".into() }, + ObjectStoreError::Throttled { + operation: "put", + retry_after: None, + }, + ObjectStoreError::TransportRetryable { + operation: "put", + message: "503".into(), + }, + ObjectStoreError::Provider { + operation: "put", + message: "403".into(), + }, + ObjectStoreError::Config("bad".into()), + ] { + assert!( + !classified.is_ambiguous(), + "{classified} must stay a classified observation" + ); + } + } + + #[test] + fn throttling_and_transient_failures_are_retryable() { + assert!(ObjectStoreError::Throttled { + operation: "get", + retry_after: Some(Duration::from_secs(1)), + } + .is_retryable()); + assert!(ObjectStoreError::TransportRetryable { + operation: "get", + message: "503".into(), + } + .is_retryable()); + assert!(!ObjectStoreError::TransportAmbiguous { + operation: "get", + message: "reset".into(), + } + .is_retryable()); + assert!(!ObjectStoreError::NotFound { key: "k".into() }.is_retryable()); + } +} diff --git a/crates/buzz-object-store/src/lib.rs b/crates/buzz-object-store/src/lib.rs new file mode 100644 index 00000000000..d4f5cef25f6 --- /dev/null +++ b/crates/buzz-object-store/src/lib.rs @@ -0,0 +1,436 @@ +//! Provider-neutral object storage for Buzz. +//! +//! Buzz keeps two very different workloads on one bucket: media blobs +//! (`buzz-media`) and the content-addressed Git object store with its +//! compare-and-swap ref pointer (`buzz-relay`'s `api::git::store`). Both used +//! to hold their own `rust-s3` client and speak in S3-shaped vocabulary — +//! ETags, `If-Match`, `If-None-Match: *`. That made the storage provider a +//! property of every call site rather than a property of the deployment. +//! +//! This crate is the seam. It owns: +//! +//! - the [`ObjectStore`] trait — exactly the operations Buzz performs, no more; +//! - provider-safe [`Revision`] / [`WriteCondition`] / [`ConditionalWrite`] +//! types, so a compare-and-swap token is never a bare string with implied +//! semantics; +//! - the [`ObjectStoreError`] taxonomy, which separates a *classified* +//! provider answer from an *unknown* transport outcome; +//! - the S3 provider ([`providers::s3`]), which is the only place an ETag +//! exists, and the Google Cloud Storage provider ([`providers::gcs`]), which +//! is the only place an object generation exists. +//! +//! Domain code above this seam — `MediaStorage`, `GitStore` — is a thin facade +//! that adds Buzz semantics (tenant-scoped sidecar keys, content addressing, +//! digest verification) and never names a provider. + +pub mod error; +pub mod providers; +pub mod revision; + +use std::path::Path; +use std::pin::Pin; +use std::str::FromStr; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; + +pub use error::ObjectStoreError; +pub use providers::gcs::{GcsObjectStore, GcsRetryConfig, GcsStoreConfig}; +pub use providers::s3::{S3AddressingStyle, S3ObjectStore, S3StoreConfig}; +pub use revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; + +/// Which provider a deployment selects, before its settings are resolved. +/// +/// Split from [`ObjectStoreConfig`] because the two callers that build a store +/// — the relay and the deletion tool — have different defaults for the S3 +/// settings but must agree on how the provider itself is chosen. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProviderSelection { + /// S3 or an S3-compatible backend, configured by the `BUZZ_S3_*` settings. + S3, + /// Google Cloud Storage, authenticated with Application Default + /// Credentials and addressed by bucket name alone. + Gcs { + /// Bucket name from `BUZZ_OBJECT_STORE_BUCKET`. + bucket: String, + }, +} + +impl ProviderSelection { + /// Read the provider selection from the environment. + /// + /// `BUZZ_OBJECT_STORE_PROVIDER` defaults to `s3`, so a deployment that has + /// never heard of this variable keeps its existing behavior. Selecting + /// `gcs` requires `BUZZ_OBJECT_STORE_BUCKET`: a Cloud Storage deployment + /// sets no `BUZZ_S3_*` values at all, so there is no bucket to fall back + /// to and guessing one would address the wrong data. + pub fn from_env() -> Result { + let provider = match std::env::var("BUZZ_OBJECT_STORE_PROVIDER") { + Ok(value) => value, + Err(std::env::VarError::NotPresent) => return Ok(Self::S3), + Err(std::env::VarError::NotUnicode(_)) => { + return Err( + "BUZZ_OBJECT_STORE_PROVIDER must be valid Unicode and one of 's3' or 'gcs'" + .to_string(), + ); + } + }; + match provider.parse::()? { + ProviderKind::S3 => Ok(Self::S3), + ProviderKind::Gcs => { + let bucket = std::env::var("BUZZ_OBJECT_STORE_BUCKET") + .ok() + .filter(|bucket| !bucket.trim().is_empty()) + .ok_or_else(|| { + "BUZZ_OBJECT_STORE_BUCKET must be set when \ + BUZZ_OBJECT_STORE_PROVIDER=gcs" + .to_string() + })?; + Ok(Self::Gcs { + bucket: bucket.trim().to_string(), + }) + } + } + } +} + +impl FromStr for ProviderKind { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "s3" => Ok(Self::S3), + "gcs" => Ok(Self::Gcs), + _ => Err(format!( + "BUZZ_OBJECT_STORE_PROVIDER must be 's3' or 'gcs', got {value:?}" + )), + } + } +} + +/// A fully resolved provider configuration, ready to connect. +#[derive(Debug, Clone)] +pub enum ObjectStoreConfig { + /// S3 or an S3-compatible backend. + S3(S3StoreConfig), + /// Google Cloud Storage. + Gcs(GcsStoreConfig), +} + +/// Build the single object-store client this process shares between media and +/// Git storage. +/// +/// Connecting is async because a provider may have admission checks to run: the +/// Cloud Storage provider reads bucket metadata here and refuses to return a +/// client for a bucket whose configuration would break deletion. +pub async fn connect(config: &ObjectStoreConfig) -> Result, ObjectStoreError> { + match config { + ObjectStoreConfig::S3(s3) => Ok(Arc::new(S3ObjectStore::new(s3)?)), + ObjectStoreConfig::Gcs(gcs) => Ok(Arc::new(GcsObjectStore::connect(gcs).await?)), + } +} + +/// A stream of object byte chunks, usable with `axum::body::Body::from_stream()`. +pub type ByteStream = + Pin> + Send>>; + +/// Outcome of a create-only write of an immutable, content-addressed object. +/// +/// Distinct from [`ConditionalWrite`] because content addressing makes the +/// committed revision uninteresting: the key *is* the digest, so a key that +/// already exists already holds these exact bytes. Callers treat both variants +/// as success and only the conformance probe distinguishes them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImmutableWrite { + /// This call wrote the object. + Created, + /// The key already held an object — by content addressing, the same bytes. + AlreadyPresent, +} + +/// Object metadata from a HEAD, as much as every provider can supply. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectMeta { + /// Object size in bytes. + pub size: u64, + /// Current revision, when the provider reports one on HEAD. + pub revision: Option, +} + +/// One page of a prefix-scoped listing. +/// +/// Keys arrive in ascending UTF-8 binary order; callers rely on that for +/// streaming key-stream digests. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ListPage { + /// `(key, size_in_bytes)` for each object in this page. + pub objects: Vec<(String, u64)>, + /// Token that fetches the next page, when one exists. + pub next_continuation_token: Option, + /// Whether the provider truncated the listing. + pub is_truncated: bool, +} + +/// Per-key outcomes of one bulk delete. +/// +/// Bulk deletion never fails on per-key outcomes: they are folded in here so +/// the caller owns retry and fail-closed policy. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BulkDeleteOutcome { + /// Keys the backend reported deleted (providers report already-missing + /// keys as deleted too — the API is idempotent by design). + pub deleted: u64, + /// Keys reported absent through a per-key "no such key" error; equivalent + /// to deleted for retry purposes. + pub already_missing: u64, + /// Keys whose deletion produced a version artifact (delete marker or + /// version id) — evidence of bucket versioning, which deletion must fail + /// closed on. + pub versioned_keys: Vec, + /// Remaining per-key failures as `(key, code, message)`. + pub failed: Vec<(String, String, String)>, +} + +/// Provider-neutral kind of retained object version. +/// +/// S3 reports concrete versions and delete markers. GCS reports generations; +/// because Buzz admits only GCS buckets with versioning and soft delete off, +/// those entries are always concrete objects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObjectVersionKind { + /// A byte-bearing object version. + Object, + /// A marker that hides older bytes without removing them. + DeleteMarker, +} + +/// One provider version under an object prefix. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ObjectVersionEntry { + /// Object key. + pub key: String, + /// Opaque provider version token (S3 version ID or GCS generation). + pub version_id: String, + /// Whether this is a byte-bearing version or a delete marker. + pub kind: ObjectVersionKind, + /// Byte size for object versions; zero for delete markers. + pub size: u64, +} + +/// Exact provider version identifier used for permanent deletion. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct ObjectVersionRef { + /// Object key. + pub key: String, + /// Opaque provider version token (S3 version ID or GCS generation). + pub version_id: String, +} + +/// One page of prefix-scoped provider versions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ObjectVersionsPage { + /// Object versions and delete markers returned by this page. + pub entries: Vec, + /// First half of the next-page cursor. + /// + /// S3 uses a key marker; GCS stores its opaque page token here. + pub next_key_marker: Option, + /// Second half of the next-page cursor, used by S3 when several versions + /// of one key straddle a page boundary. GCS leaves it empty. + pub next_version_id_marker: Option, + /// Whether more pages remain. + pub is_truncated: bool, +} + +/// The object-store operations Buzz actually performs. +/// +/// Implementations are shared across the process behind an `Arc`: the relay +/// constructs exactly one provider and hands it to both the media facade and +/// the Git facade. +#[async_trait] +pub trait ObjectStore: Send + Sync { + /// Which provider backs this client. + fn provider(&self) -> ProviderKind; + + /// Store an object from a byte slice. + /// + /// For large blobs prefer [`ObjectStore::put_file`], which never holds the + /// whole body in memory. + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError>; + + /// Stream a file from disk into the store without loading it into RAM. + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError>; + + /// Create-only write of an immutable, content-addressed object. + /// + /// A precondition failure is [`ImmutableWrite::AlreadyPresent`], not an + /// error: the key is the digest of the bytes, so a collision means the + /// stored bytes already equal these bytes. + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result; + + /// Write an object under a precondition. + /// + /// A failed precondition is [`ConditionalWrite::Conflict`], not an error. + /// On [`ConditionalWrite::Committed`] the returned [`Revision`] is read + /// from the write response and predicates the next conditional write; a + /// provider that commits without returning a revision is non-conforming + /// and must fail the operation rather than return an unusable token. + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result; + + /// Read an object's full body. + async fn get(&self, key: &str) -> Result; + + /// Read an inclusive byte range from an object. + /// + /// Only the requested slice is transferred; the full object is never + /// loaded into memory. + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result; + + /// Read an object as a chunk stream, without buffering the whole body. + async fn get_stream(&self, key: &str) -> Result; + + /// Read an object's body **and** its revision from a single response. + /// + /// Returns `Ok(None)` when the object does not exist. + /// + /// A HEAD followed by a GET can straddle a concurrent writer: the HEAD's + /// revision and the GET's body would describe different versions, and a + /// caller that predicated its next write on the HEAD revision would be + /// predicating on a version it never read. Both fields come from one + /// response so the snapshot stays consistent. + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError>; + + /// Read an object's metadata. Returns `Ok(None)` when it does not exist. + async fn head(&self, key: &str) -> Result, ObjectStoreError>; + + /// Fetch one page of a prefix-scoped listing. + /// + /// `max_keys` bounds a single provider response, not the caller's total + /// object budget; callers enforce the cumulative cap across pages. + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result; + + /// Delete a single object. Deleting an absent object is not an error. + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError>; + + /// Delete a bounded batch of objects, reporting per-key outcomes. + /// + /// Providers with a native batch API issue one request; providers without + /// one issue bounded-concurrency individual deletes. Either way the caller + /// sees the same per-key fold and decides retry policy. + async fn delete_objects(&self, keys: &[String]) -> Result; + + /// Fetch one page of exact object versions under a prefix. + /// + /// Cursor fields are opaque to callers and must be replayed together. + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result; + + /// Permanently delete exact provider versions. + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result; + + /// Probe connectivity and bucket access. + async fn ping(&self) -> Result<(), ObjectStoreError>; + + /// Whether the bucket retains non-current object versions. + /// + /// Deletion refuses versioned buckets: an ordinary delete on one inserts a + /// delete marker rather than proving logical absence, so a bulk delete + /// could report success while the bytes remain reachable. + async fn versioning_detected(&self) -> Result; + + /// Read an object after rejecting bodies larger than `max_bytes`. + /// + /// The HEAD bound is checked first so an oversized object is never + /// transferred, and the returned length is re-checked in case the provider + /// reported a bad size. + async fn get_limited(&self, key: &str, max_bytes: u64) -> Result { + let meta = self + .head(key) + .await? + .ok_or_else(|| ObjectStoreError::NotFound { key: key.into() })?; + if meta.size > max_bytes { + return Err(ObjectStoreError::ObjectTooLarge { + key: key.into(), + size: meta.size, + max: max_bytes, + }); + } + + let bytes = self.get(key).await?; + let size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if size > max_bytes { + return Err(ObjectStoreError::ObjectTooLarge { + key: key.into(), + size, + max: max_bytes, + }); + } + Ok(bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_names_parse() { + assert_eq!("s3".parse::(), Ok(ProviderKind::S3)); + assert_eq!("gcs".parse::(), Ok(ProviderKind::Gcs)); + } + + /// Unknown or near-miss spellings fail rather than silently selecting the + /// default provider: a typo in `BUZZ_OBJECT_STORE_PROVIDER` must not + /// quietly point a Cloud Storage deployment at S3. + #[test] + fn unknown_provider_names_are_rejected() { + for invalid in ["", "S3", "GCS", "google", "gs", "minio"] { + let error = invalid + .parse::() + .expect_err("must reject unknown provider"); + assert!( + error.contains("must be 's3' or 'gcs'"), + "unexpected error for {invalid:?}: {error}" + ); + } + } +} diff --git a/crates/buzz-object-store/src/providers/gcs.rs b/crates/buzz-object-store/src/providers/gcs.rs new file mode 100644 index 00000000000..4e4dc2c07ca --- /dev/null +++ b/crates/buzz-object-store/src/providers/gcs.rs @@ -0,0 +1,1521 @@ +//! Google Cloud Storage provider, backed by Google's official client. +//! +//! This is the only module in Buzz that knows what an object *generation* is. +//! Everything above the seam holds a [`Revision`] and cannot tell a generation +//! from an S3 ETag. +//! +//! ## Why generations, not ETags +//! +//! Cloud Storage stamps every object write with a monotonically increasing +//! `generation`, and accepts it back as the `ifGenerationMatch` precondition. +//! That is a first-class compare-and-swap token: `ifGenerationMatch=0` means +//! "commit only if the object does not exist", and `ifGenerationMatch=` +//! means "commit only if the object is still at ``". A stale precondition +//! is refused with HTTP 412, which this module reports as +//! [`ConditionalWrite::Conflict`] — an ordinary lost race, never a backend +//! error. +//! +//! ## Bucket contract, checked at construction +//! +//! The S3 provider detects bucket versioning empirically, by writing a probe +//! object and looking for a version id on the response. That heuristic is +//! meaningless here: *every* Cloud Storage object carries a generation whether +//! or not the bucket retains old ones. [`GcsObjectStore::connect`] reads the +//! bucket's metadata instead and refuses to build a client unless object +//! versioning is off **and** soft-delete retention is zero. Both settings would +//! otherwise leave a restorable copy behind after a delete, so a deletion +//! request could report success while the bytes remain reachable. Checking at +//! construction also catches out-of-band configuration drift on every boot. +//! +//! ## Retries +//! +//! The client's own retry loop is disabled ([`no_client_retries`]) and this +//! module owns one bounded policy instead, so that three rules hold visibly: +//! +//! - a conditional write is only ever retried carrying its exact original +//! precondition — it is never downgraded to an unconditional write; +//! - HTTP 429 is throttling, never evidence of a lost race, so it paces the +//! caller and, if the budget runs out, surfaces as +//! [`ObjectStoreError::Throttled`] for the caller to absorb as backpressure; +//! - when an attempt fails without a classified answer, a subsequent 412 is +//! *not* assumed to be someone else's commit. The object is reread and the +//! committed body decides — see [`GcsObjectStore::put_conditional`]. +//! +//! The single exception is [`ObjectStore::put_file`], which streams a +//! multi-hundred-megabyte media blob: it delegates to the client's resumable +//! upload retry so a transient failure resumes mid-object instead of +//! restarting the transfer. + +use std::future::Future; +use std::path::Path; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use futures_util::StreamExt; +use google_cloud_gax::error::Error as GcsError; +use google_cloud_gax::retry_policy::RetryPolicyExt; +use google_cloud_storage::client::{Storage, StorageControl}; +use google_cloud_storage::model_ext::ReadRange; +use google_cloud_storage::retry_policy::RetryableErrors; + +use crate::error::ObjectStoreError; +use crate::revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; +use crate::{ + BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore, + ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, +}; + +/// How many deletes a bulk delete keeps in flight. +/// +/// Cloud Storage has no batch-delete RPC, so a bulk delete is N individual +/// deletes. The width is bounded to keep one caller from consuming the whole +/// per-bucket request budget and throttling every other operation. +const BULK_DELETE_CONCURRENCY: usize = 12; + +/// Upper bound on a provider-advertised `Retry-After`. +/// +/// Honouring the header is required, but an unbounded sleep would let one +/// response stall a request for minutes. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(30); + +/// Floor on the pause after a throttled request that carried no `Retry-After`. +/// +/// Cloud Storage documents a maximum of one write per second to a single object +/// name. A writer already being throttled on one key therefore cannot succeed +/// by retrying sooner, and would only spend its bounded attempt budget — which +/// is what turns absorbed backpressure into a failed push. +const MIN_THROTTLE_BACKOFF: Duration = Duration::from_secs(1); + +/// Longest provider detail kept on an error. +/// +/// Cloud Storage error bodies are JSON diagnostics, but they can echo request +/// detail; the message is bounded so an error can never grow into a log of the +/// request. +const MAX_ERROR_DETAIL: usize = 512; + +/// Bounded retry policy for one [`GcsObjectStore`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GcsRetryConfig { + /// Attempts allowed while the backend is failing, including the first. + /// `1` disables retries. + pub max_attempts: u32, + /// Attempts allowed while the backend is throttling, including the first. + /// + /// Larger than `max_attempts` because a 429 is not a failure: it is the + /// service asking to be approached more slowly, at a published rate. Giving + /// up on it turns backpressure the caller should have absorbed into a + /// failed write. + pub max_throttled_attempts: u32, + /// Backoff ceiling used for the first retry, doubled thereafter. + pub initial_backoff: Duration, + /// Ceiling on any single computed backoff. + pub max_backoff: Duration, +} + +impl Default for GcsRetryConfig { + fn default() -> Self { + Self { + max_attempts: 6, + max_throttled_attempts: 12, + initial_backoff: Duration::from_millis(200), + max_backoff: Duration::from_secs(8), + } + } +} + +/// Connection inputs for [`GcsObjectStore::connect`]. +#[derive(Debug, Clone)] +pub struct GcsStoreConfig { + /// Bucket name, without any `gs://` or resource-path decoration. + pub bucket: String, + /// Bounded retry policy for this client. + pub retry: GcsRetryConfig, +} + +impl GcsStoreConfig { + /// Configure a bucket with the default retry policy. + pub fn new(bucket: impl Into) -> Self { + Self { + bucket: bucket.into(), + retry: GcsRetryConfig::default(), + } + } +} + +/// The two bucket settings that decide whether a delete proves absence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BucketContract { + versioning_enabled: bool, + soft_delete_retention: Duration, +} + +impl BucketContract { + fn of(bucket: &google_cloud_storage::model::Bucket) -> Self { + let versioning_enabled = bucket.versioning.as_ref().is_some_and(|v| v.enabled); + let soft_delete_retention = bucket + .soft_delete_policy + .as_ref() + .and_then(|policy| policy.retention_duration.as_ref()) + .map(|d| { + let seconds = u64::try_from(d.seconds()).unwrap_or(0); + let nanos = u32::try_from(d.nanos()).unwrap_or(0); + Duration::new(seconds, nanos) + }) + .unwrap_or(Duration::ZERO); + Self { + versioning_enabled, + soft_delete_retention, + } + } + + /// Whether a delete on this bucket leaves a restorable copy behind. + /// + /// Object versioning and soft delete are different mechanisms with the + /// same consequence for Buzz: the deleted bytes stay reachable, so a + /// deletion cannot claim absence. Both therefore answer the seam's + /// "does this bucket retain non-current versions" question with yes. + fn retains_noncurrent_versions(&self) -> bool { + self.versioning_enabled || !self.soft_delete_retention.is_zero() + } + + /// Fail closed unless the bucket satisfies the deletion contract. + fn admit(&self, bucket: &str) -> Result<(), ObjectStoreError> { + let mut violations = Vec::new(); + if self.versioning_enabled { + violations.push("object versioning is enabled".to_string()); + } + if !self.soft_delete_retention.is_zero() { + violations.push(format!( + "soft-delete retention is {}s (must be 0)", + self.soft_delete_retention.as_secs() + )); + } + if violations.is_empty() { + return Ok(()); + } + Err(ObjectStoreError::Config(format!( + "bucket {bucket} does not satisfy the deletion contract ({}): a delete would leave a \ + restorable copy behind, so deletion could report success while the object stays \ + reachable", + violations.join("; ") + ))) + } +} + +/// Google Cloud Storage client, authenticated with Application Default +/// Credentials. +/// +/// Credentials are never configured in code: the client resolves them through +/// ADC (attached service account on GCE/GKE/Cloud Run, workload identity, or a +/// developer's `gcloud` credentials). There is no key material, HMAC pair, or +/// key file on this path. +pub struct GcsObjectStore { + bucket: String, + /// `projects/_/buckets/`, the resource name both clients address. + resource: String, + data: Storage, + control: StorageControl, + retry: GcsRetryConfig, +} + +/// Retry policy handed to the client so it performs exactly one attempt. +/// +/// This module owns retries (see the module docs); a second, invisible loop +/// underneath would multiply attempts and hide throttling from the caller. +fn no_client_retries() -> impl google_cloud_gax::retry_policy::RetryPolicy { + RetryableErrors.with_attempt_limit(1) +} + +impl GcsObjectStore { + /// Build a client against a bucket and verify its deletion contract. + /// + /// Fails closed when object versioning is enabled or soft-delete retention + /// is non-zero. + pub async fn connect(config: &GcsStoreConfig) -> Result { + if config.bucket.is_empty() { + return Err(ObjectStoreError::Config( + "gcs bucket must be configured".to_string(), + )); + } + if config.retry.max_attempts == 0 || config.retry.max_throttled_attempts == 0 { + return Err(ObjectStoreError::Config( + "gcs retry max_attempts and max_throttled_attempts must be at least 1".to_string(), + )); + } + + let data = Storage::builder() + .with_retry_policy(no_client_retries()) + .build() + .await + .map_err(|e| ObjectStoreError::Config(format!("gcs storage client: {e}")))?; + let control = StorageControl::builder() + .with_retry_policy(no_client_retries()) + .build() + .await + .map_err(|e| ObjectStoreError::Config(format!("gcs storage control client: {e}")))?; + + let store = Self { + resource: bucket_resource(&config.bucket), + bucket: config.bucket.clone(), + data, + control, + retry: config.retry, + }; + store.read_bucket_contract().await?.admit(&store.bucket)?; + Ok(store) + } + + /// The bucket this client addresses. + pub fn bucket(&self) -> &str { + &self.bucket + } + + /// Read the bucket's versioning and soft-delete settings. + async fn read_bucket_contract(&self) -> Result { + let bucket = self + .with_retries("get_bucket", &self.bucket, || async { + self.control + .get_bucket() + .set_name(self.resource.clone()) + .send() + .await + .map_err(|e| classify("get_bucket", &self.bucket, e)) + }) + .await?; + Ok(BucketContract::of(&bucket)) + } + + /// Run `attempt` under this client's bounded retry policy. + /// + /// Only throttling, transient backend answers, and unclassified transport + /// failures are retried; everything else is the caller's answer on the + /// first attempt. Conditional writes do not use this helper — they need + /// per-attempt bookkeeping and have their own loop. + async fn with_retries( + &self, + operation: &'static str, + _key: &str, + mut attempt: F, + ) -> Result + where + F: FnMut() -> Fut, + Fut: Future>, + { + let mut budget = RetryBudget::new(&self.retry); + loop { + match attempt().await { + Ok(value) => return Ok(value), + Err(error) => match budget.next_delay(&error) { + Some(delay) => { + tracing::debug!( + provider = "gcs", + operation, + attempt = budget.attempts(), + delay_ms = delay.as_millis() as u64, + "retrying object store operation" + ); + tokio::time::sleep(delay).await; + } + None => return Err(error), + }, + } + } + } + + /// One conditional-write attempt, carrying `precondition` verbatim. + async fn write_once( + &self, + operation: &'static str, + key: &str, + bytes: &[u8], + content_type: &str, + precondition: Option, + ) -> Result { + let mut write = self + .data + .write_object( + self.resource.clone(), + key.to_string(), + Bytes::copy_from_slice(bytes), + ) + .set_content_type(content_type.to_string()); + if let Some(generation) = precondition { + write = write.set_if_generation_match(generation); + } + write + .send_unbuffered() + .await + .map(|object| Revision::GcsGeneration(object.generation)) + .map_err(|e| classify(operation, key, e)) + } + + /// After an unclassified failure, let the stored object decide whether the + /// write committed. + /// + /// A 412 arriving after an attempt whose outcome was never classified is + /// genuinely ambiguous: either another writer won the race, or *our own* + /// earlier attempt committed and the retry then found its own generation in + /// place. Guessing either way is a correctness bug, so the object is reread + /// and its body answers. Bodies are compared rather than generations + /// because the generation the winning attempt would have returned was never + /// received. + async fn classify_ambiguous_commit( + &self, + key: &str, + written: &[u8], + ) -> Result { + match self.get_with_revision(key).await? { + Some((revision, body)) if body.as_ref() == written => { + Ok(ConditionalWrite::Committed(revision)) + } + _ => Ok(ConditionalWrite::Conflict), + } + } +} + +/// One operation's progress through the bounded retry policy. +/// +/// Throttling and transient failure are counted separately on purpose. +/// A 5xx or a dropped connection is the backend failing, and a caller waiting +/// on a long series of them is waiting for nothing. A 429 is the backend +/// working correctly and asking to be approached more slowly, with a published +/// recovery interval — a sole writer pushing a repository in chunks has to wait +/// its turn, and giving up on it would turn absorbed backpressure into a failed +/// push. So throttling gets its own, larger allowance. +struct RetryBudget<'a> { + retry: &'a GcsRetryConfig, + failures: u32, + throttles: u32, +} + +impl<'a> RetryBudget<'a> { + fn new(retry: &'a GcsRetryConfig) -> Self { + Self { + retry, + failures: 0, + throttles: 0, + } + } + + /// Attempts made so far. + fn attempts(&self) -> u32 { + self.failures + } + + /// Record `error` and return how long to wait, or `None` to give up and + /// return it to the caller. + fn next_delay(&mut self, error: &ObjectStoreError) -> Option { + let attempt_index = self.failures; + self.failures += 1; + + match error { + ObjectStoreError::Throttled { retry_after, .. } => { + self.throttles += 1; + if self.throttles >= self.retry.max_throttled_attempts { + return None; + } + Some(match retry_after { + Some(hint) => (*hint).min(MAX_RETRY_AFTER), + // Cloud Storage publishes a one-write-per-second ceiling + // per object name. Once a writer is being throttled on a + // single key, retrying sooner than that cannot succeed — it + // only spends the budget — so the throttle path takes a + // floor the transient path does not. + None => backoff(self.retry, attempt_index).max(MIN_THROTTLE_BACKOFF), + }) + } + ObjectStoreError::TransportRetryable { .. } + | ObjectStoreError::TransportAmbiguous { .. } => { + if self.failures - self.throttles >= self.retry.max_attempts { + return None; + } + Some(backoff(self.retry, attempt_index)) + } + _ => None, + } + } +} + +/// Capped exponential backoff with equal jitter. +/// +/// The wait is half the exponent plus a uniform draw over the other half, +/// rather than a uniform draw over the whole of it. Both forms de-synchronise +/// racers on a hot pointer, which is the point of jitter, but full jitter also +/// makes very short waits likely — and against a hard per-object rate limit a +/// short wait cannot succeed, so it burns an attempt from a bounded budget for +/// nothing. Keeping the lower half fixed means every retry is meaningfully +/// later than the one before it. +fn backoff(retry: &GcsRetryConfig, attempt_index: u32) -> Duration { + let cap = retry + .initial_backoff + .saturating_mul(1u32 << attempt_index.min(16)) + .min(retry.max_backoff); + let cap_ms = u64::try_from(cap.as_millis()).unwrap_or(u64::MAX).max(2); + let fixed = cap_ms / 2; + Duration::from_millis(fixed + rand::random::() % (cap_ms - fixed)) +} + +/// The resource name both Cloud Storage clients address a bucket by. +fn bucket_resource(bucket: &str) -> String { + format!("projects/_/buckets/{bucket}") +} + +/// Translate one Cloud Storage object descriptor into the provider-neutral +/// version vocabulary used by deletion. +fn version_entry_of(object: google_cloud_storage::model::Object) -> ObjectVersionEntry { + ObjectVersionEntry { + key: object.name, + version_id: object.generation.to_string(), + kind: ObjectVersionKind::Object, + size: u64::try_from(object.size).unwrap_or(0), + } +} + +/// Parse an opaque seam token back into a generation without ever accepting a +/// token from a different provider or a sentinel generation. +fn generation_of(version: &ObjectVersionRef) -> Result { + let generation = version + .version_id + .parse::() + .map_err(|_| ObjectStoreError::Provider { + operation: "delete_versions", + message: format!("invalid GCS generation for object {:?}", version.key), + })?; + if generation <= 0 { + return Err(ObjectStoreError::Provider { + operation: "delete_versions", + message: format!("non-positive GCS generation for object {:?}", version.key), + }); + } + Ok(generation) +} + +/// The `ifGenerationMatch` value implementing a [`WriteCondition`]. +/// +/// `0` is Cloud Storage's create-only precondition: no live generation can be +/// zero, so it holds exactly when the object is absent. +fn generation_precondition(condition: &WriteCondition) -> Result { + match condition { + WriteCondition::Absent => Ok(0), + WriteCondition::Matches(revision) => revision.expect_gcs_generation(), + } +} + +/// Parse a `Retry-After` delta-seconds value, if the response carried one. +/// +/// Only the delta-seconds form is read. Cloud Storage does not send the +/// HTTP-date form, and misreading a date as a duration would produce an absurd +/// sleep; an unparsed header simply falls back to computed backoff. +fn retry_after_of(error: &GcsError) -> Option { + error + .http_headers()? + .get("retry-after")? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(Duration::from_secs) +} + +/// Bound and tidy a provider error for inclusion in an [`ObjectStoreError`]. +fn detail(error: &GcsError) -> String { + let mut message = error.to_string(); + if message.len() > MAX_ERROR_DETAIL { + message.truncate(MAX_ERROR_DETAIL); + message.push('…'); + } + message +} + +/// Map a client failure into the provider-neutral taxonomy. +/// +/// The ordering is the whole point: a response that carried a status told us +/// something about the object, and stays a *classified* observation no matter +/// how the transport behaved afterwards. Only a failure with no status at all +/// becomes [`ObjectStoreError::TransportAmbiguous`], which is the set the Git +/// conformance probe drops from its observers. Widening it would let a real +/// conformance failure disappear. +fn classify(operation: &'static str, key: &str, error: GcsError) -> ObjectStoreError { + let message = detail(&error); + + if let Some(status) = error.http_status_code() { + return match status { + 404 => ObjectStoreError::NotFound { key: key.into() }, + 412 => ObjectStoreError::Conflict { key: key.into() }, + 429 => ObjectStoreError::Throttled { + operation, + retry_after: retry_after_of(&error), + }, + 408 | 500..=599 => ObjectStoreError::TransportRetryable { operation, message }, + _ => ObjectStoreError::Provider { operation, message }, + }; + } + + // gRPC-shaped answers carry an RPC status instead of an HTTP one. They are + // still answers, so they classify the same way. + if let Some(code) = error.status().map(|status| status.code) { + use google_cloud_gax::error::rpc::Code; + return match code { + Code::NotFound => ObjectStoreError::NotFound { key: key.into() }, + Code::Aborted | Code::FailedPrecondition | Code::AlreadyExists => { + ObjectStoreError::Conflict { key: key.into() } + } + Code::ResourceExhausted => ObjectStoreError::Throttled { + operation, + retry_after: retry_after_of(&error), + }, + Code::Unavailable | Code::Internal | Code::DeadlineExceeded => { + ObjectStoreError::TransportRetryable { operation, message } + } + _ => ObjectStoreError::Provider { operation, message }, + }; + } + + if error.is_connect() || error.is_io() || error.is_timeout() || error.is_transport() { + return ObjectStoreError::TransportAmbiguous { operation, message }; + } + + // The client gave up before any attempt was classified, so the outcome of + // the last one is unknown. + if error.is_exhausted() { + return ObjectStoreError::TransportAmbiguous { operation, message }; + } + + ObjectStoreError::Provider { operation, message } +} + +/// Whether a failed attempt might still have reached the backend. +/// +/// A refused connection never delivered the request, so a later 412 cannot be +/// this writer's own commit. Any other unclassified failure could have been a +/// response that was lost on the way back. +fn may_have_committed(error: &ObjectStoreError) -> bool { + match error { + ObjectStoreError::TransportAmbiguous { message, .. } => !is_connect_refusal(message), + ObjectStoreError::TransportRetryable { .. } | ObjectStoreError::Throttled { .. } => false, + _ => false, + } +} + +/// Whether an ambiguous failure's detail describes a connection that was never +/// established. +fn is_connect_refusal(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + message.contains("connection refused") + || message.contains("dns error") + || message.contains("failed to lookup address") +} + +#[async_trait] +impl ObjectStore for GcsObjectStore { + fn provider(&self) -> ProviderKind { + ProviderKind::Gcs + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.with_retries("put", key, || { + self.write_once("put", key, bytes, content_type, None) + }) + .await + .map(|_| ()) + } + + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError> { + let file = tokio::fs::File::open(path) + .await + .map_err(|e| ObjectStoreError::Provider { + operation: "put_file", + message: e.to_string(), + })?; + + // The one operation that keeps the client's own retry loop: a media + // blob can be hundreds of megabytes, and the client's resumable upload + // resumes mid-object where this module's loop would restart the whole + // transfer. The upload is unconditional, so a retry cannot disturb any + // precondition. + self.data + .write_object(self.resource.clone(), key.to_string(), file) + .set_content_type(content_type.to_string()) + .with_retry_policy( + RetryableErrors + .with_attempt_limit(self.retry.max_attempts) + .with_time_limit(Duration::from_secs(300)), + ) + .send_unbuffered() + .await + .map_err(|e| classify("put_file", key, e))?; + Ok(()) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + match self + .put_conditional(key, bytes, content_type, WriteCondition::Absent) + .await? + { + ConditionalWrite::Committed(_) => Ok(ImmutableWrite::Created), + ConditionalWrite::Conflict => Ok(ImmutableWrite::AlreadyPresent), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result { + let precondition = generation_precondition(&condition)?; + // Set once an attempt fails without a classified answer. From then on a + // 412 is no longer self-evidently someone else's commit. + let mut outcome_unknown = false; + let mut budget = RetryBudget::new(&self.retry); + + loop { + let error = match self + .write_once( + "put_conditional", + key, + bytes, + content_type, + Some(precondition), + ) + .await + { + Ok(revision) => return Ok(ConditionalWrite::Committed(revision)), + Err(ObjectStoreError::Conflict { .. }) if !outcome_unknown => { + return Ok(ConditionalWrite::Conflict); + } + Err(ObjectStoreError::Conflict { .. }) => { + return self.classify_ambiguous_commit(key, bytes).await; + } + Err(error) => error, + }; + + outcome_unknown |= may_have_committed(&error); + + // Retrying always replays `precondition` verbatim: the loop never + // relaxes a compare-and-swap into a blind overwrite. + match budget.next_delay(&error) { + Some(delay) => { + tracing::debug!( + provider = "gcs", + operation = "put_conditional", + attempt = budget.attempts(), + delay_ms = delay.as_millis() as u64, + outcome_unknown, + "retrying conditional write with its original precondition" + ); + tokio::time::sleep(delay).await; + } + None => return Err(error), + } + } + } + + async fn get(&self, key: &str) -> Result { + self.with_retries("get", key, || async { + let mut response = self + .data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + .map_err(|e| classify("get", key, e))?; + let mut body = Vec::with_capacity(response.object().size.max(0) as usize); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get", key, e))?); + } + Ok(Bytes::from(body)) + }) + .await + } + + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result { + if end < start { + return Err(ObjectStoreError::Provider { + operation: "get_range", + message: format!("inverted byte range {start}..={end}"), + }); + } + // The seam's range is inclusive on both ends; Cloud Storage takes an + // offset and a length. + let length = end - start + 1; + + self.with_retries("get_range", key, || async { + let mut response = self + .data + .read_object(self.resource.clone(), key.to_string()) + .set_read_range(ReadRange::segment(start, length)) + .send() + .await + .map_err(|e| classify("get_range", key, e))?; + let mut body = Vec::new(); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get_range", key, e))?); + } + Ok(Bytes::from(body)) + }) + .await + } + + async fn get_stream(&self, key: &str) -> Result { + // Only opening the stream is retried. Once bytes are flowing the + // caller owns the body, and restarting mid-response would silently + // splice two reads together. + let response = self + .with_retries("get_stream", key, || async { + self.data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + .map_err(|e| classify("get_stream", key, e)) + }) + .await?; + + // `ReadObjectResponse` only exposes a `Stream` adapter behind the + // client's `unstable-stream` feature, so the stream is unfolded from + // its stable chunk-at-a-time API instead of opting into an unstable + // surface. + let key = key.to_string(); + Ok(Box::pin(futures_util::stream::unfold( + response, + move |mut response| { + let key = key.clone(); + async move { + let chunk = response.next().await?; + Some((chunk.map_err(|e| classify("get_stream", &key, e)), response)) + } + }, + ))) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + self.with_retries("get_with_revision", key, || async { + let mut response = match self + .data + .read_object(self.resource.clone(), key.to_string()) + .send() + .await + { + Ok(response) => response, + Err(error) => { + return match classify("get_with_revision", key, error) { + ObjectStoreError::NotFound { .. } => Ok(None), + other => Err(other), + }; + } + }; + + // Body and generation both come off this one response, so the + // revision a caller predicates its next write on always describes + // the bytes it just read. + let revision = Revision::GcsGeneration(response.object().generation); + let mut body = Vec::with_capacity(response.object().size.max(0) as usize); + while let Some(chunk) = response.next().await { + body.extend_from_slice(&chunk.map_err(|e| classify("get_with_revision", key, e))?); + } + Ok(Some((revision, Bytes::from(body)))) + }) + .await + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + self.with_retries("head", key, || async { + match self + .control + .get_object() + .set_bucket(self.resource.clone()) + .set_object(key.to_string()) + .send() + .await + { + Ok(object) => Ok(Some(ObjectMeta { + size: u64::try_from(object.size).unwrap_or(0), + revision: Some(Revision::GcsGeneration(object.generation)), + })), + Err(error) => match classify("head", key, error) { + ObjectStoreError::NotFound { .. } => Ok(None), + other => Err(other), + }, + } + }) + .await + } + + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result { + let page_size = i32::try_from(max_keys).unwrap_or(i32::MAX); + + self.with_retries("list_page", prefix, || async { + let mut request = self + .control + .list_objects() + .set_parent(self.resource.clone()) + .set_prefix(prefix.to_string()) + .set_page_size(page_size); + if let Some(token) = continuation_token.clone() { + request = request.set_page_token(token); + } + let response = request + .send() + .await + .map_err(|e| classify("list_page", prefix, e))?; + + // Cloud Storage signals "no more pages" with an empty token rather + // than an absent one. + let next = Some(response.next_page_token).filter(|token| !token.is_empty()); + Ok(ListPage { + objects: response + .objects + .into_iter() + .map(|object| (object.name, u64::try_from(object.size).unwrap_or(0))) + .collect(), + is_truncated: next.is_some(), + next_continuation_token: next, + }) + }) + .await + } + + /// Delete one object. + /// + /// The delete is not generation-qualified. It does not need to be: the + /// constructor refuses buckets with object versioning or soft delete, so a + /// key has exactly one live generation and an unqualified delete removes + /// precisely the object the caller addressed. Deleting an absent object is + /// not an error. + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.with_retries("delete", key, || async { + match self + .control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.to_string()) + .send() + .await + { + Ok(()) => Ok(()), + Err(error) => match classify("delete", key, error) { + ObjectStoreError::NotFound { .. } => Ok(()), + other => Err(other), + }, + } + }) + .await + } + + async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + + // Cloud Storage has no batch-delete RPC, so this is bounded-concurrency + // individual deletes folded into the same per-key outcome the S3 + // provider reports from its batch response. + let outcomes = futures_util::stream::iter(keys.iter().cloned()) + .map(|key| async move { + let result = self + .with_retries("delete_objects", &key, || async { + self.control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.clone()) + .send() + .await + .map_err(|e| classify("delete_objects", &key, e)) + }) + .await; + (key, result) + }) + .buffer_unordered(BULK_DELETE_CONCURRENCY) + .collect::>() + .await; + + let mut outcome = BulkDeleteOutcome::default(); + for (key, result) in outcomes { + match result { + Ok(()) => outcome.deleted += 1, + Err(ObjectStoreError::NotFound { .. }) => outcome.already_missing += 1, + Err(error) => { + outcome + .failed + .push((key, error_code(&error).to_string(), error.to_string())) + } + } + } + // `versioned_keys` stays empty by construction: a delete on a bucket + // that passed the admission check cannot produce a version artifact. + Ok(outcome) + } + + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result { + if version_id_marker.is_some() { + return Err(ObjectStoreError::Provider { + operation: "list_versions_page", + message: "GCS pagination accepts one opaque page token; a second cursor component is invalid" + .to_string(), + }); + } + let page_size = i32::try_from(max_keys).unwrap_or(i32::MAX); + + self.with_retries("list_versions_page", prefix, || async { + let mut request = self + .control + .list_objects() + .set_parent(self.resource.clone()) + .set_prefix(prefix.to_string()) + .set_versions(true) + .set_page_size(page_size); + if let Some(token) = key_marker.clone() { + request = request.set_page_token(token); + } + let response = request + .send() + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + let next = Some(response.next_page_token).filter(|token| !token.is_empty()); + Ok(ObjectVersionsPage { + entries: response.objects.into_iter().map(version_entry_of).collect(), + is_truncated: next.is_some(), + next_key_marker: next, + next_version_id_marker: None, + }) + }) + .await + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + if versions.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + + let mut parsed = Vec::with_capacity(versions.len()); + for version in versions { + parsed.push((version.key.clone(), generation_of(version)?)); + } + + let outcomes = futures_util::stream::iter(parsed) + .map(|(key, generation)| async move { + let result = self + .with_retries("delete_versions", &key, || async { + self.control + .delete_object() + .set_bucket(self.resource.clone()) + .set_object(key.clone()) + .set_generation(generation) + .send() + .await + .map_err(|e| classify("delete_versions", &key, e)) + }) + .await; + (key, result) + }) + .buffer_unordered(BULK_DELETE_CONCURRENCY) + .collect::>() + .await; + + let mut outcome = BulkDeleteOutcome::default(); + for (key, result) in outcomes { + match result { + Ok(()) => outcome.deleted += 1, + Err(ObjectStoreError::NotFound { .. }) => outcome.already_missing += 1, + Err(error) => { + outcome + .failed + .push((key, error_code(&error).to_string(), error.to_string())) + } + } + } + Ok(outcome) + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + self.list_page("", None, 1).await.map(|_| ()) + } + + /// Whether a delete on this bucket would leave a restorable copy behind. + /// + /// Read from bucket metadata rather than probed. Every Cloud Storage object + /// carries a generation whether or not old ones are retained, so the S3 + /// provider's "did the response have a version id" heuristic would answer + /// yes on a correctly configured bucket. Soft-delete retention counts too: + /// it retains a restorable copy for exactly the same reason versioning + /// does. + async fn versioning_detected(&self) -> Result { + Ok(self + .read_bucket_contract() + .await? + .retains_noncurrent_versions()) + } +} + +/// Stable, bounded label for a per-key bulk-delete failure. +fn error_code(error: &ObjectStoreError) -> &'static str { + match error { + ObjectStoreError::NotFound { .. } => "NotFound", + ObjectStoreError::Conflict { .. } => "PreconditionFailed", + ObjectStoreError::Throttled { .. } => "Throttled", + ObjectStoreError::TransportRetryable { .. } => "TransportRetryable", + ObjectStoreError::TransportAmbiguous { .. } => "TransportAmbiguous", + ObjectStoreError::Config(_) => "Config", + ObjectStoreError::ObjectTooLarge { .. } => "ObjectTooLarge", + ObjectStoreError::DigestMismatch { .. } => "DigestMismatch", + ObjectStoreError::RevisionMismatch { .. } => "RevisionMismatch", + ObjectStoreError::Provider { .. } => "Provider", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use google_cloud_storage::http::HeaderMap; + use google_cloud_storage::model::bucket::{SoftDeletePolicy, Versioning}; + use google_cloud_storage::model::{Bucket, Object}; + + fn http_error(status: u16) -> GcsError { + GcsError::http(status, HeaderMap::new(), bytes::Bytes::new()) + } + + fn throttled_error(retry_after: &str) -> GcsError { + let mut headers = HeaderMap::new(); + headers.insert("retry-after", retry_after.parse().unwrap()); + GcsError::http(429, headers, bytes::Bytes::new()) + } + + #[test] + fn bucket_resource_uses_the_wildcard_project_form() { + assert_eq!( + bucket_resource("buzz-objects"), + "projects/_/buckets/buzz-objects" + ); + } + + #[test] + fn object_generations_map_to_provider_neutral_versions() { + let entry = version_entry_of( + Object::new() + .set_name("_meta/tenant/a.json") + .set_generation(1_700_000_000_000_042_i64) + .set_size(42_i64), + ); + assert_eq!(entry.key, "_meta/tenant/a.json"); + assert_eq!(entry.version_id, "1700000000000042"); + assert_eq!(entry.kind, ObjectVersionKind::Object); + assert_eq!(entry.size, 42); + } + + #[test] + fn exact_delete_rejects_foreign_or_sentinel_generation_tokens() { + for version_id in ["v-s3-token", "0", "-1"] { + let error = generation_of(&ObjectVersionRef { + key: "object".to_string(), + version_id: version_id.to_string(), + }) + .expect_err("invalid generation must fail closed"); + assert!(matches!( + error, + ObjectStoreError::Provider { + operation: "delete_versions", + .. + } + )); + } + } + + /// Create-only is `ifGenerationMatch=0`; a compare-and-swap carries the + /// observed generation verbatim. + #[test] + fn write_conditions_map_to_generation_preconditions() { + assert_eq!(generation_precondition(&WriteCondition::Absent).unwrap(), 0); + assert_eq!( + generation_precondition(&WriteCondition::Matches(Revision::GcsGeneration( + 1_700_000_000_000_042 + ))) + .unwrap(), + 1_700_000_000_000_042 + ); + } + + /// An ETag can never predicate a generation precondition. Accepting one + /// would have to mean dropping the precondition, which is a blind + /// overwrite. + #[test] + fn a_foreign_revision_is_rejected_rather_than_downgraded() { + let err = + generation_precondition(&WriteCondition::Matches(Revision::S3Etag("\"abc\"".into()))) + .expect_err("an S3 ETag must never predicate a GCS generation match"); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + } + + #[test] + fn missing_object_classifies_as_not_found() { + let err = classify("get", "packs/x", http_error(404)); + assert!(matches!(err, ObjectStoreError::NotFound { ref key } if key == "packs/x")); + assert!(!err.is_ambiguous()); + } + + /// A stale generation is a normal compare-and-swap conflict, never a + /// backend failure and never a throttle. + #[test] + fn stale_generation_classifies_as_conflict() { + let err = classify("put_conditional", "pointers/x", http_error(412)); + assert!(matches!(err, ObjectStoreError::Conflict { ref key } if key == "pointers/x")); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + #[test] + fn throttling_classifies_as_retryable_backpressure() { + let err = classify("put_conditional", "pointers/x", http_error(429)); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: None, + .. + } + )); + assert!(err.is_retryable() && !err.is_ambiguous()); + } + + #[test] + fn retry_after_seconds_are_honoured_and_capped() { + let retry = GcsRetryConfig::default(); + + let err = classify("put_conditional", "pointers/x", throttled_error("3")); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: Some(d), + .. + } if d == Duration::from_secs(3) + )); + assert_eq!( + RetryBudget::new(&retry).next_delay(&err), + Some(Duration::from_secs(3)), + "an advertised backoff is used verbatim" + ); + + let absurd = classify("put_conditional", "pointers/x", throttled_error("86400")); + assert_eq!( + RetryBudget::new(&retry).next_delay(&absurd), + Some(MAX_RETRY_AFTER), + "one response must not stall a request indefinitely" + ); + } + + /// Cloud Storage caps writes to one per second per object name, so a + /// throttled writer that retries sooner cannot succeed — it only spends an + /// attempt from a bounded budget. Live testing found exactly this: with a + /// sub-second first retry the budget ran out and absorbed backpressure + /// surfaced as a failed write. + #[test] + fn throttling_without_a_hint_waits_at_least_the_per_object_write_interval() { + let retry = GcsRetryConfig::default(); + let throttled = ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }; + let mut budget = RetryBudget::new(&retry); + for attempt in 1..retry.max_throttled_attempts { + let delay = budget + .next_delay(&throttled) + .expect("throttling stays retryable within its own budget"); + assert!( + delay >= MIN_THROTTLE_BACKOFF, + "attempt {attempt} would retry after {delay:?}, sooner than the per-object \ + write interval" + ); + } + assert!( + budget.next_delay(&throttled).is_none(), + "the throttle budget is bounded" + ); + } + + /// Throttling and transient failure draw on separate allowances. A long + /// series of 429s must not consume the budget reserved for a genuinely + /// failing backend, and vice versa — otherwise a paced writer and a broken + /// one are indistinguishable to the policy. + #[test] + fn throttling_and_transient_failure_have_independent_budgets() { + let retry = GcsRetryConfig::default(); + assert!( + retry.max_throttled_attempts > retry.max_attempts, + "backpressure deserves more patience than failure" + ); + let throttled = ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }; + let transient = ObjectStoreError::TransportRetryable { + operation: "put_conditional", + message: "503".into(), + }; + + // Spend the throttle allowance down to its last attempt, then show the + // transient allowance is still whole. + let mut budget = RetryBudget::new(&retry); + for _ in 1..retry.max_throttled_attempts { + assert!(budget.next_delay(&throttled).is_some()); + } + for _ in 1..retry.max_attempts { + assert!( + budget.next_delay(&transient).is_some(), + "throttling must not have eaten the transient budget" + ); + } + assert!( + budget.next_delay(&transient).is_none(), + "the transient budget is bounded" + ); + } + + /// Equal jitter: every wait is at least half its exponential cap, and never + /// more than the cap. Full jitter would satisfy the upper bound alone while + /// making near-zero waits common. + #[test] + fn backoff_grows_and_never_collapses_to_zero() { + let retry = GcsRetryConfig::default(); + let mut previous_floor = Duration::ZERO; + for attempt in 0..6 { + let cap = retry + .initial_backoff + .saturating_mul(1 << attempt) + .min(retry.max_backoff); + for _ in 0..64 { + let delay = backoff(&retry, attempt); + assert!(delay <= cap, "attempt {attempt}: {delay:?} exceeds {cap:?}"); + assert!( + delay >= cap / 2, + "attempt {attempt}: {delay:?} is below half of {cap:?}" + ); + } + assert!(cap >= previous_floor, "backoff must not shrink"); + previous_floor = cap; + } + } + + /// Only backpressure and unknown outcomes are retried. A stale generation, + /// a missing object, or a permission failure is the caller's answer on the + /// first attempt. + #[test] + fn classified_final_answers_are_not_retried() { + let retry = GcsRetryConfig::default(); + for error in [ + ObjectStoreError::NotFound { key: "k".into() }, + ObjectStoreError::Conflict { key: "k".into() }, + ObjectStoreError::Provider { + operation: "put", + message: "403".into(), + }, + ObjectStoreError::Config("bad".into()), + ] { + assert!( + RetryBudget::new(&retry).next_delay(&error).is_none(), + "{error} must not be retried" + ); + } + } + + /// An HTTP-date `Retry-After` is not misread as a duration; the caller + /// falls back to computed backoff instead of sleeping for aeons. + #[test] + fn unparseable_retry_after_falls_back_to_backoff() { + let err = classify("get", "k", throttled_error("Wed, 21 Oct 2026 07:28:00 GMT")); + assert!(matches!( + err, + ObjectStoreError::Throttled { + retry_after: None, + .. + } + )); + } + + #[test] + fn transient_statuses_stay_classified_but_retryable() { + for status in [408, 500, 502, 503, 504] { + let err = classify("get", "k", http_error(status)); + assert!( + matches!(err, ObjectStoreError::TransportRetryable { .. }), + "status {status} should be retryable, got {err}" + ); + assert!(err.is_retryable() && !err.is_ambiguous()); + } + } + + /// A 403 is a real answer: permanent, classified, never dropped from the + /// conformance probe's observer set. + #[test] + fn permission_denied_classifies_as_permanent_provider_failure() { + let err = classify("put", "k", http_error(403)); + assert!(matches!(err, ObjectStoreError::Provider { .. })); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + /// Pre-classification failures are the only ambiguous outcomes. + #[test] + fn transport_failures_without_a_status_classify_as_ambiguous() { + for error in [ + GcsError::io("connection reset by peer"), + GcsError::connect("connection refused"), + GcsError::timeout("deadline exceeded"), + ] { + let err = classify("put_conditional", "k", error); + assert!(err.is_ambiguous(), "expected ambiguous, got {err}"); + } + } + + /// A conditional write whose attempt was refused at connect time never + /// reached the backend, so a later 412 is somebody else's commit. + #[test] + fn a_refused_connection_cannot_have_committed() { + let refused = classify( + "put_conditional", + "k", + GcsError::connect("connection refused"), + ); + assert!(!may_have_committed(&refused)); + + let reset = classify( + "put_conditional", + "k", + GcsError::io("connection reset by peer"), + ); + assert!(may_have_committed(&reset)); + } + + /// Classified answers say what happened, so they never arm the + /// reread-and-decide path. + #[test] + fn classified_failures_do_not_arm_ambiguity() { + for error in [ + classify("put_conditional", "k", http_error(429)), + classify("put_conditional", "k", http_error(503)), + classify("put_conditional", "k", http_error(403)), + ] { + assert!(!may_have_committed(&error), "{error}"); + } + } + + fn bucket_with(versioning: Option, soft_delete_seconds: Option) -> Bucket { + let mut bucket = Bucket::new(); + if let Some(enabled) = versioning { + bucket.versioning = Some(Versioning::new().set_enabled(enabled)); + } + if let Some(seconds) = soft_delete_seconds { + let retention = google_cloud_wkt::Duration::try_from(Duration::from_secs(seconds)) + .expect("representable retention"); + bucket.soft_delete_policy = + Some(SoftDeletePolicy::new().set_retention_duration(retention)); + } + bucket + } + + #[test] + fn a_conforming_bucket_is_admitted() { + for bucket in [ + bucket_with(None, None), + bucket_with(Some(false), Some(0)), + bucket_with(Some(false), None), + ] { + let contract = BucketContract::of(&bucket); + assert!(!contract.retains_noncurrent_versions()); + contract.admit("buzz-objects").expect("bucket conforms"); + } + } + + /// Versioning and soft delete are different mechanisms with the same + /// consequence: a delete stops proving absence. Both fail construction. + #[test] + fn versioning_or_soft_delete_fails_admission_closed() { + let versioned = BucketContract::of(&bucket_with(Some(true), Some(0))); + assert!(versioned.retains_noncurrent_versions()); + let err = versioned.admit("buzz-objects").unwrap_err(); + assert!( + matches!(err, ObjectStoreError::Config(ref m) if m.contains("object versioning is enabled")), + "unexpected error: {err}" + ); + + let soft_deleted = BucketContract::of(&bucket_with(Some(false), Some(604_800))); + assert!(soft_deleted.retains_noncurrent_versions()); + let err = soft_deleted.admit("buzz-objects").unwrap_err(); + assert!( + matches!(err, ObjectStoreError::Config(ref m) if m.contains("soft-delete retention is 604800s")), + "unexpected error: {err}" + ); + } + + #[test] + fn both_violations_are_reported_together() { + let contract = BucketContract::of(&bucket_with(Some(true), Some(604_800))); + let err = contract.admit("buzz-objects").unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("object versioning is enabled"), + "{message}" + ); + assert!(message.contains("soft-delete retention"), "{message}"); + } + + #[tokio::test] + async fn empty_bucket_and_zero_attempts_are_rejected_at_construction() { + // Both checks run before any credential resolution or network call, so + // they are provable without an environment. + let mut config = GcsStoreConfig::new(""); + assert!(matches!( + GcsObjectStore::connect(&config).await, + Err(ObjectStoreError::Config(ref m)) if m.contains("bucket must be configured") + )); + + config = GcsStoreConfig::new("buzz-objects"); + config.retry.max_attempts = 0; + assert!(matches!( + GcsObjectStore::connect(&config).await, + Err(ObjectStoreError::Config(ref m)) if m.contains("max_attempts") + )); + } + + #[test] + fn error_detail_is_bounded() { + let payload = bytes::Bytes::from("x".repeat(4096)); + let error = GcsError::http(500, HeaderMap::new(), payload); + assert!(detail(&error).len() <= MAX_ERROR_DETAIL + 4); + } +} diff --git a/crates/buzz-object-store/src/providers/mod.rs b/crates/buzz-object-store/src/providers/mod.rs new file mode 100644 index 00000000000..af4ca84c936 --- /dev/null +++ b/crates/buzz-object-store/src/providers/mod.rs @@ -0,0 +1,8 @@ +//! Concrete [`crate::ObjectStore`] implementations. +//! +//! Exactly one provider is constructed per process and shared by every domain +//! facade. Provider-specific vocabulary — ETags, generations, addressing +//! styles, credential chains — stays inside these modules. + +pub mod gcs; +pub mod s3; diff --git a/crates/buzz-object-store/src/providers/s3.rs b/crates/buzz-object-store/src/providers/s3.rs new file mode 100644 index 00000000000..779f9591b11 --- /dev/null +++ b/crates/buzz-object-store/src/providers/s3.rs @@ -0,0 +1,979 @@ +//! S3 and S3-compatible provider, backed by `rust-s3`. +//! +//! This is the only module in Buzz that knows what an ETag is. Everything +//! above the seam holds a [`Revision`] and cannot tell an ETag from a Google +//! Cloud Storage generation. +//! +//! ## The 412 sharp edge +//! +//! `rust-s3`'s `fail-on-err` Cargo feature is unified ON across the build +//! graph, so non-2xx responses arrive here as `S3Error::HttpFailWithBody(code, +//! body)` *before* the caller sees `ResponseData`. A precondition failure +//! (412) is therefore an `Err` that must be reclassified as a *semantic* +//! result — [`ConditionalWrite::Conflict`] or [`ImmutableWrite::AlreadyPresent`] +//! — rather than propagated as a backend error. Empirically verified against +//! MinIO by the Git store's `probe::probe_412_surfacing`. +//! +//! ## Classified vs. unknown outcomes +//! +//! [`classify`] draws the line the Git conformance probe depends on: +//! `S3Error::{Reqwest, Http, Io}` are pre-classification failures — the racer +//! never got an answer from the backend — and map to +//! [`ObjectStoreError::TransportAmbiguous`]. Every other variant means the +//! backend *did* answer, in or out of contract, and stays a classified +//! observation. Do not widen the ambiguous set: it would let a genuine +//! conformance failure be silently dropped from the probe's observer set. + +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; + +use async_trait::async_trait; +use bytes::Bytes; +use quick_xml::events::{BytesStart, Event}; +use quick_xml::Reader; +use s3::creds::Credentials; +use s3::error::S3Error; +use s3::request::Request as _; +use s3::{Bucket, Region}; + +use crate::error::ObjectStoreError; +use crate::revision::{ConditionalWrite, ProviderKind, Revision, WriteCondition}; +use crate::{ + BulkDeleteOutcome, ByteStream, ImmutableWrite, ListPage, ObjectMeta, ObjectStore, + ObjectVersionEntry, ObjectVersionKind, ObjectVersionRef, ObjectVersionsPage, +}; + +/// S3 URL addressing style shared by media and Git/CAS storage. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum S3AddressingStyle { + /// Put the bucket in the request path (`https://endpoint/bucket/key`). + /// + /// This preserves compatibility with the bundled MinIO deployments, whose + /// internal DNS only resolves the endpoint hostname. + #[default] + Path, + /// Put the bucket in the hostname (`https://bucket.endpoint/key`). + /// + /// This is the standard S3 form and is required by providers such as new + /// Railway Storage Buckets. + Virtual, +} + +impl FromStr for S3AddressingStyle { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "path" => Ok(Self::Path), + "virtual" => Ok(Self::Virtual), + _ => Err(format!( + "BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}" + )), + } + } +} + +/// Connection inputs for [`S3ObjectStore::new`]. +#[derive(Debug, Clone)] +pub struct S3StoreConfig { + /// S3-compatible endpoint URL (e.g. `http://localhost:9000`). + pub endpoint: String, + /// Static access key, or empty to use the AWS credential chain. + pub access_key: String, + /// Static secret key, or empty to use the AWS credential chain. + pub secret_key: String, + /// Bucket name. + pub bucket: String, + /// Region used for SigV4 request signing. + pub region: String, + /// URL addressing style. + pub addressing_style: S3AddressingStyle, +} + +/// S3-compatible object storage client. +pub struct S3ObjectStore { + bucket: Box, +} + +impl S3ObjectStore { + /// Build a client against an S3-compatible endpoint (e.g. MinIO). + /// + /// Credential selection: + /// - If both `access_key` and `secret_key` are non-empty, use them as + /// static credentials (MinIO/local/dev, or any static-key deployment). + /// - If both are empty, fall back to the AWS default credential chain via + /// [`Credentials::default`]: environment, shared profile, web-identity + /// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and + /// instance-metadata providers, in that order. This lets the relay use + /// the pod's IAM role without long-lived static keys. + /// - If exactly one is empty, fail: a half-configured static deployment + /// must surface rather than silently fall back to the chain. + pub fn new(config: &S3StoreConfig) -> Result { + let region = Region::Custom { + region: config.region.clone(), + endpoint: config.endpoint.clone(), + }; + let creds = match (config.access_key.is_empty(), config.secret_key.is_empty()) { + (false, false) => Credentials::new( + Some(&config.access_key), + Some(&config.secret_key), + None, + None, + None, + ), + (true, true) => Credentials::default(), + _ => { + return Err(ObjectStoreError::Config( + "s3 access_key and secret_key must be configured together, or both empty to use the AWS credential chain" + .to_string(), + )); + } + } + .map_err(|e| ObjectStoreError::Config(e.to_string()))?; + let bucket = Bucket::new(&config.bucket, region, creds) + .map_err(|e| ObjectStoreError::Config(e.to_string()))?; + let bucket = match config.addressing_style { + S3AddressingStyle::Path => bucket.with_path_style(), + S3AddressingStyle::Virtual => bucket, + }; + Ok(Self { bucket }) + } + + /// The bucket's public URL, for diagnostics and tests. + pub fn url(&self) -> String { + self.bucket.url() + } + + /// Whether the client signs and routes in path-addressing style. + pub fn is_path_style(&self) -> bool { + self.bucket.is_path_style() + } + + /// The signing region configured on the client. + pub fn region(&self) -> &Region { + &self.bucket.region + } + + /// Build the precondition headers for one conditional write. + fn condition_headers( + condition: &WriteCondition, + ) -> Result { + let mut headers = axum::http::HeaderMap::new(); + match condition { + WriteCondition::Absent => { + headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); + } + WriteCondition::Matches(revision) => { + let tag = revision.expect_s3_etag()?; + headers.insert( + axum::http::header::IF_MATCH, + tag.parse().map_err(|_| ObjectStoreError::Provider { + operation: "put_conditional", + message: format!("invalid etag {tag}"), + })?, + ); + } + } + Ok(headers) + } + + /// Read the ETag out of a response's headers, in either casing. + fn etag_of(headers: &std::collections::HashMap) -> Option { + headers.get("etag").or_else(|| headers.get("ETag")).cloned() + } +} + +/// Map a `rust-s3` failure into the provider-neutral taxonomy. +/// +/// See the module docs: only pre-classification failures may become +/// [`ObjectStoreError::TransportAmbiguous`]. +fn classify(operation: &'static str, key: &str, error: S3Error) -> ObjectStoreError { + let message = error.to_string(); + match error { + S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_) => { + ObjectStoreError::TransportAmbiguous { operation, message } + } + S3Error::HttpFailWithBody(404, _) => ObjectStoreError::NotFound { key: key.into() }, + S3Error::HttpFailWithBody(412, _) => ObjectStoreError::Conflict { key: key.into() }, + S3Error::HttpFailWithBody(429, _) => ObjectStoreError::Throttled { + operation, + retry_after: None, + }, + S3Error::HttpFailWithBody(500 | 502 | 503 | 504, _) => { + ObjectStoreError::TransportRetryable { operation, message } + } + _ => ObjectStoreError::Provider { operation, message }, + } +} + +/// Fold one `DeleteObjects` response into per-key outcomes. +/// +/// Historical MinIO releases report already-absent keys as +/// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to +/// `already_missing` to keep checkpointed retry idempotent. +fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + fold_delete_result(result, DeleteMode::Unversioned) +} + +fn fold_version_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + fold_delete_result(result, DeleteMode::ExplicitVersion) +} + +enum DeleteMode { + Unversioned, + ExplicitVersion, +} + +fn fold_delete_result( + result: s3::serde_types::DeleteObjectsResult, + mode: DeleteMode, +) -> BulkDeleteOutcome { + let mut outcome = BulkDeleteOutcome::default(); + for deleted in result.deleted { + let has_version_artifact = deleted.delete_marker == Some(true) + || deleted.delete_marker_version_id.is_some() + || deleted.version_id.is_some(); + match mode { + DeleteMode::Unversioned if has_version_artifact => { + outcome.versioned_keys.push(deleted.key); + } + DeleteMode::Unversioned | DeleteMode::ExplicitVersion => outcome.deleted += 1, + } + } + for error in result.errors { + if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { + outcome.already_missing += 1; + } else { + outcome.failed.push((error.key, error.code, error.message)); + } + } + outcome +} + +#[derive(Debug, Default)] +struct ListVersionFields { + key: Option, + version_id: Option, + size: Option, +} + +fn local_name(name: &[u8]) -> &[u8] { + name.rsplit(|byte| *byte == b':').next().unwrap_or(name) +} + +fn version_xml_error(error: impl std::fmt::Display) -> ObjectStoreError { + ObjectStoreError::Provider { + operation: "list_versions_page", + message: error.to_string(), + } +} + +fn read_element_text( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, +) -> Result { + reader + .read_text(start.to_end().name()) + .map(|text| text.into_owned()) + .map_err(version_xml_error) +} + +fn skip_element( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, +) -> Result<(), ObjectStoreError> { + reader + .read_to_end(start.to_end().name()) + .map_err(version_xml_error)?; + Ok(()) +} + +fn parse_list_version_entry( + reader: &mut Reader<&[u8]>, + start: &BytesStart<'_>, + kind: ObjectVersionKind, +) -> Result { + let mut fields = ListVersionFields::default(); + loop { + match reader.read_event().map_err(version_xml_error)? { + Event::Start(child) => match local_name(child.local_name().as_ref()) { + b"Key" => fields.key = Some(read_element_text(reader, &child)?), + b"VersionId" => fields.version_id = Some(read_element_text(reader, &child)?), + b"Size" => { + fields.size = Some( + read_element_text(reader, &child)? + .parse::() + .map_err(version_xml_error)?, + ); + } + _ => skip_element(reader, &child)?, + }, + Event::Empty(child) => match local_name(child.local_name().as_ref()) { + b"Key" => fields.key = Some(String::new()), + b"VersionId" => fields.version_id = Some(String::new()), + b"Size" => fields.size = Some(0), + _ => {} + }, + Event::End(end) if end.name().as_ref() == start.to_end().name().as_ref() => { + let key = fields + .key + .ok_or_else(|| version_xml_error("ListObjectVersions entry missing Key"))?; + let version_id = fields.version_id.ok_or_else(|| { + version_xml_error("ListObjectVersions entry missing VersionId") + })?; + return Ok(ObjectVersionEntry { + key, + version_id, + kind, + size: if kind == ObjectVersionKind::Object { + fields.size.unwrap_or(0) + } else { + 0 + }, + }); + } + Event::Eof => { + return Err(version_xml_error( + "unexpected EOF inside ListObjectVersions entry", + )); + } + _ => {} + } + } +} + +fn parse_object_versions_page(xml: &[u8]) -> Result { + let mut reader = Reader::from_reader(xml); + reader.config_mut().trim_text(true); + let mut entries = Vec::new(); + let mut next_key_marker = None; + let mut next_version_id_marker = None; + let mut is_truncated = false; + + loop { + match reader.read_event().map_err(version_xml_error)? { + Event::Start(start) => match local_name(start.local_name().as_ref()) { + b"Version" => entries.push(parse_list_version_entry( + &mut reader, + &start, + ObjectVersionKind::Object, + )?), + b"DeleteMarker" => entries.push(parse_list_version_entry( + &mut reader, + &start, + ObjectVersionKind::DeleteMarker, + )?), + b"IsTruncated" => { + is_truncated = + read_element_text(&mut reader, &start)?.eq_ignore_ascii_case("true"); + } + b"NextKeyMarker" => { + next_key_marker = Some(read_element_text(&mut reader, &start)?); + } + b"NextVersionIdMarker" => { + next_version_id_marker = Some(read_element_text(&mut reader, &start)?); + } + b"ListVersionsResult" => {} + _ => skip_element(&mut reader, &start)?, + }, + Event::Empty(start) => match local_name(start.local_name().as_ref()) { + b"NextKeyMarker" => next_key_marker = Some(String::new()), + b"NextVersionIdMarker" => next_version_id_marker = Some(String::new()), + _ => {} + }, + Event::Eof => break, + _ => {} + } + } + + Ok(ObjectVersionsPage { + entries, + next_key_marker, + next_version_id_marker, + is_truncated, + }) +} + +#[async_trait] +impl ObjectStore for S3ObjectStore { + fn provider(&self) -> ProviderKind { + ProviderKind::S3 + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.bucket + .put_object_with_content_type(key, bytes, content_type) + .await + .map_err(|e| classify("put", key, e))?; + Ok(()) + } + + async fn put_file( + &self, + key: &str, + path: &Path, + content_type: &str, + ) -> Result<(), ObjectStoreError> { + /// 8 MiB read buffer — the file is streamed, never held whole in RAM. + const BUF: usize = 8 * 1024 * 1024; + + let file = tokio::fs::File::open(path) + .await + .map_err(|e| ObjectStoreError::Provider { + operation: "put_file", + message: e.to_string(), + })?; + let mut reader = tokio::io::BufReader::with_capacity(BUF, file); + + self.bucket + .put_object_stream_with_content_type(&mut reader, key, content_type) + .await + .map_err(|e| classify("put_file", key, e))?; + Ok(()) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + let headers = Self::condition_headers(&WriteCondition::Absent)?; + match self + .bucket + .put_object_with_content_type_and_headers(key, bytes, content_type, Some(headers)) + .await + { + Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(ImmutableWrite::Created), + Err(S3Error::HttpFailWithBody(412, _)) => Ok(ImmutableWrite::AlreadyPresent), + Ok(resp) => Err(ObjectStoreError::Provider { + operation: "put_immutable", + message: format!("unexpected status {}", resp.status_code()), + }), + Err(e) => Err(classify("put_immutable", key, e)), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + condition: WriteCondition, + ) -> Result { + let headers = Self::condition_headers(&condition)?; + match self + .bucket + .put_object_with_content_type_and_headers(key, bytes, content_type, Some(headers)) + .await + { + Ok(resp) if (200..300).contains(&resp.status_code()) => { + let etag = Self::etag_of(&resp.headers()).ok_or({ + // Fail closed: a conditional write we cannot chain (because + // the backend returned no ETag) is not a commit — it is a + // non-conforming backend. The conformance probe catches + // this at boot; in production we would rather refuse than + // hand the caller an empty token and force-fail the next + // compare-and-swap. + ObjectStoreError::Provider { + operation: "put_conditional", + message: "conditional write committed but the response carried no ETag \ + (backend does not satisfy revision-token consistency)" + .to_string(), + } + })?; + Ok(ConditionalWrite::Committed(Revision::S3Etag(etag))) + } + Err(S3Error::HttpFailWithBody(412, _)) => Ok(ConditionalWrite::Conflict), + Ok(resp) => Err(ObjectStoreError::Provider { + operation: "put_conditional", + message: format!("unexpected status {}", resp.status_code()), + }), + Err(e) => Err(classify("put_conditional", key, e)), + } + } + + async fn get(&self, key: &str) -> Result { + let resp = self + .bucket + .get_object(key) + .await + .map_err(|e| classify("get", key, e))?; + Ok(Bytes::from(resp.to_vec())) + } + + async fn get_range(&self, key: &str, start: u64, end: u64) -> Result { + let resp = self + .bucket + .get_object_range(key, start, Some(end)) + .await + .map_err(|e| classify("get_range", key, e))?; + Ok(Bytes::from(resp.to_vec())) + } + + async fn get_stream(&self, key: &str) -> Result { + let response = self + .bucket + .get_object_stream(key) + .await + .map_err(|e| classify("get_stream", key, e))?; + + if response.status_code == 404 { + return Err(ObjectStoreError::NotFound { key: key.into() }); + } + + let operation = "get_stream"; + let stream = futures_util::StreamExt::map(response.bytes, move |chunk| { + chunk.map_err(|e| ObjectStoreError::Provider { + operation, + message: e.to_string(), + }) + }); + Ok(Box::pin(stream)) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + match self.bucket.get_object(key).await { + Ok(resp) => { + let etag = Self::etag_of(&resp.headers()).ok_or(ObjectStoreError::Provider { + operation: "get_with_revision", + message: "response carried no ETag".to_string(), + })?; + Ok(Some((Revision::S3Etag(etag), Bytes::from(resp.to_vec())))) + } + Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), + Err(e) => Err(classify("get_with_revision", key, e)), + } + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + match self.bucket.head_object(key).await { + Ok((result, _)) => Ok(Some(ObjectMeta { + size: result.content_length.unwrap_or(0) as u64, + revision: result.e_tag.map(Revision::S3Etag), + })), + Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), + Err(e) => Err(classify("head", key, e)), + } + } + + async fn list_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, + ) -> Result { + // Wraps rust-s3's manual `list_page`, NOT the auto-paginating `list`, + // which has no cap. + let (result, _status) = self + .bucket + .list_page( + prefix.to_string(), + None, + continuation_token, + None, + Some(max_keys), + ) + .await + .map_err(|e| classify("list_page", prefix, e))?; + Ok(ListPage { + objects: result + .contents + .into_iter() + .map(|obj| (obj.key, obj.size)) + .collect(), + next_continuation_token: result.next_continuation_token, + is_truncated: result.is_truncated, + }) + } + + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.bucket + .delete_object(key) + .await + .map_err(|e| classify("delete", key, e))?; + Ok(()) + } + + async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = keys + .iter() + .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| classify("delete_objects", "", e))?; + Ok(fold_bulk_delete_result(result)) + } + + async fn list_versions_page( + &self, + prefix: &str, + key_marker: Option, + version_id_marker: Option, + max_keys: usize, + ) -> Result { + let mut query = HashMap::from([ + ("versions".to_string(), String::new()), + ("prefix".to_string(), prefix.to_string()), + ("max-keys".to_string(), max_keys.to_string()), + ]); + if let Some(marker) = key_marker { + query.insert("key-marker".to_string(), marker); + } + if let Some(marker) = version_id_marker { + query.insert("version-id-marker".to_string(), marker); + } + let bucket = self + .bucket + .with_extra_query(query) + .map_err(|e| classify("list_versions_page", prefix, e))?; + let request = s3::request::tokio_backend::ReqwestRequest::new( + &bucket, + "/", + s3::command::Command::GetObject, + ) + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + let response = request + .response_data(false) + .await + .map_err(|e| classify("list_versions_page", prefix, e))?; + if response.status_code() >= 300 { + return Err(ObjectStoreError::Provider { + operation: "list_versions_page", + message: format!( + "unexpected status {}: {}", + response.status_code(), + response.as_str().unwrap_or("") + ), + }); + } + parse_object_versions_page(response.as_slice()) + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + if versions.is_empty() { + return Ok(BulkDeleteOutcome::default()); + } + let identifiers = versions + .iter() + .map(|version| { + s3::serde_types::ObjectIdentifier::with_version( + version.key.clone(), + version.version_id.clone(), + ) + }) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| classify("delete_versions", "", e))?; + Ok(fold_version_delete_result(result)) + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + self.list_page("", None, 1).await.map(|_| ()) + } + + /// Detect whether the bucket has ever had versioning enabled. + /// + /// `rust-s3` exposes no `GetBucketVersioning`, so this writes and inspects + /// a short-lived probe object instead: versioning-enabled (and + /// versioning-suspended) buckets stamp new writes with a version id. + /// + /// This heuristic is S3-specific by construction and does not translate to + /// providers where every object carries a version token; those must + /// implement the check against real bucket metadata. + async fn versioning_detected(&self) -> Result { + let key = format!("probe/deletion-versioning-{}", uuid::Uuid::new_v4()); + self.put(&key, b"buzz deletion versioning probe", "text/plain") + .await?; + let inspected = self.bucket.head_object(&key).await; + let removed = self.bucket.delete_object(&key).await; + let (head, _) = inspected.map_err(|e| classify("versioning_detected", &key, e))?; + removed.map_err(|e| classify("versioning_detected", &key, e))?; + Ok(head.version_id.is_some()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(access: &str, secret: &str) -> S3StoreConfig { + S3StoreConfig { + endpoint: "http://localhost:9000".to_string(), + access_key: access.to_string(), + secret_key: secret.to_string(), + bucket: "buzz-media".to_string(), + region: "us-west-2".to_string(), + addressing_style: S3AddressingStyle::Path, + } + } + + /// Static keys present: builds a client without touching the AWS + /// credential chain (no env/metadata access), and the signing region + /// comes from config rather than a hardcoded "us-east-1". + #[test] + fn static_keys_build_client_with_configured_region() { + let store = + S3ObjectStore::new(&config("buzz_dev", "buzz_dev_secret")).expect("static creds"); + match store.region() { + Region::Custom { region, .. } => assert_eq!(region, "us-west-2"), + other => panic!("expected Custom region, got {other:?}"), + } + assert_eq!(store.provider(), ProviderKind::S3); + } + + #[test] + fn constructor_applies_both_addressing_styles() { + let path = + S3ObjectStore::new(&config("buzz_dev", "buzz_dev_secret")).expect("path-style client"); + assert!(path.is_path_style()); + assert_eq!(path.url(), "http://localhost:9000/buzz-media"); + + let mut virtual_config = config("buzz_dev", "buzz_dev_secret"); + virtual_config.addressing_style = S3AddressingStyle::Virtual; + let virtual_hosted = S3ObjectStore::new(&virtual_config).expect("virtual-hosted client"); + assert!(!virtual_hosted.is_path_style()); + assert_eq!(virtual_hosted.url(), "http://buzz-media.localhost:9000"); + } + + #[test] + fn partial_static_keys_are_rejected() { + for (access, secret) in [("buzz_dev", ""), ("", "buzz_dev_secret")] { + let err = match S3ObjectStore::new(&config(access, secret)) { + Ok(_) => panic!("partial static creds must not silently use credential chain"), + Err(err) => err, + }; + assert!( + matches!(err, ObjectStoreError::Config(ref msg) if msg.contains("must be configured together")), + "unexpected error: {err}" + ); + } + } + + /// Pre-classification failures are the *only* ambiguous outcomes. The Git + /// conformance probe drops exactly this set from its observer count, so + /// widening it would let a real conformance failure vanish. + #[test] + fn transport_failures_classify_as_ambiguous() { + let io = classify("put", "k", S3Error::Io(std::io::Error::other("reset"))); + assert!(io.is_ambiguous(), "io error must be ambiguous: {io}"); + } + + #[test] + fn precondition_failure_classifies_as_conflict() { + let err = classify( + "put_conditional", + "pointers/x", + S3Error::HttpFailWithBody(412, "PreconditionFailed".into()), + ); + assert!(!err.is_ambiguous()); + assert!(matches!(err, ObjectStoreError::Conflict { ref key } if key == "pointers/x")); + } + + #[test] + fn missing_object_classifies_as_not_found() { + let err = classify( + "get", + "packs/x", + S3Error::HttpFailWithBody(404, "NoSuchKey".into()), + ); + assert!(matches!(err, ObjectStoreError::NotFound { ref key } if key == "packs/x")); + } + + #[test] + fn throttling_and_transient_statuses_stay_classified_but_retryable() { + let throttled = classify( + "put", + "k", + S3Error::HttpFailWithBody(429, "SlowDown".into()), + ); + assert!(matches!(throttled, ObjectStoreError::Throttled { .. })); + assert!(throttled.is_retryable() && !throttled.is_ambiguous()); + + let transient = classify( + "get", + "k", + S3Error::HttpFailWithBody(503, "SlowDown".into()), + ); + assert!(matches!( + transient, + ObjectStoreError::TransportRetryable { .. } + )); + assert!(transient.is_retryable() && !transient.is_ambiguous()); + } + + /// A 403 is a real backend answer: permanent, classified, never dropped. + #[test] + fn permission_denied_classifies_as_permanent_provider_failure() { + let err = classify( + "put", + "k", + S3Error::HttpFailWithBody(403, "AccessDenied".into()), + ); + assert!(matches!(err, ObjectStoreError::Provider { .. })); + assert!(!err.is_ambiguous() && !err.is_retryable()); + } + + #[test] + fn conditional_write_rejects_a_foreign_provider_revision() { + let err = + S3ObjectStore::condition_headers(&WriteCondition::Matches(Revision::GcsGeneration(7))) + .expect_err("a GCS generation must never predicate an S3 If-Match"); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: ProviderKind::Gcs, + } + )); + } + + #[test] + fn conditional_write_accepts_an_s3_revision() { + let headers = S3ObjectStore::condition_headers(&WriteCondition::Matches(Revision::S3Etag( + "\"abc\"".into(), + ))) + .expect("s3 etag predicates If-Match"); + assert_eq!( + headers.get(axum::http::header::IF_MATCH).unwrap(), + "\"abc\"" + ); + } + + #[test] + fn create_only_write_uses_if_none_match_star() { + let headers = S3ObjectStore::condition_headers(&WriteCondition::Absent).expect("headers"); + assert_eq!(headers.get(axum::http::header::IF_NONE_MATCH).unwrap(), "*"); + } + + /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO + /// absent-key errors count as success, version artifacts are surfaced for + /// fail-closed handling, and anything else stays a per-key failure. + #[test] + fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let deleted_object = |key: &str, marker: bool| DeletedObject { + key: key.to_string(), + version_id: None, + delete_marker: marker.then_some(true), + delete_marker_version_id: marker.then(|| "v1".to_string()), + }; + let delete_error = |key: &str, code: &str, message: &str| DeleteError { + key: key.to_string(), + code: code.to_string(), + message: message.to_string(), + version_id: None, + }; + let result = DeleteObjectsResult { + deleted: vec![ + deleted_object("plain", false), + deleted_object("marked", true), + ], + errors: vec![ + delete_error("gone", "NoSuchKey", "absent"), + delete_error("gone-version", "NoSuchVersion", "absent"), + delete_error("denied", "AccessDenied", "nope"), + ], + }; + let outcome = fold_bulk_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 2); + assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); + assert_eq!( + outcome.failed, + vec![( + "denied".to_string(), + "AccessDenied".to_string(), + "nope".to_string() + )] + ); + } + + #[test] + fn explicit_version_delete_counts_version_artifacts_as_deleted() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let result = DeleteObjectsResult { + deleted: vec![DeletedObject { + key: "versioned".to_string(), + version_id: Some("v1".to_string()), + delete_marker: Some(true), + delete_marker_version_id: Some("v1".to_string()), + }], + errors: vec![DeleteError { + key: "already-gone".to_string(), + code: "NoSuchVersion".to_string(), + message: "absent".to_string(), + version_id: Some("v0".to_string()), + }], + }; + + let outcome = fold_version_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 1); + assert!(outcome.versioned_keys.is_empty()); + assert!(outcome.failed.is_empty()); + } + + #[test] + fn version_listing_preserves_objects_markers_and_dual_cursor() { + let page = parse_object_versions_page( + br#" + + true + _meta/tenant/a.json + v-new + + _meta/tenant/a.jsonv-delete + + + _meta/tenant/a.jsonv-new42 + +"#, + ) + .expect("parse S3 version page"); + + assert!(page.is_truncated); + assert_eq!(page.next_key_marker.as_deref(), Some("_meta/tenant/a.json")); + assert_eq!(page.next_version_id_marker.as_deref(), Some("v-new")); + assert_eq!( + page.entries, + vec![ + ObjectVersionEntry { + key: "_meta/tenant/a.json".to_string(), + version_id: "v-delete".to_string(), + kind: ObjectVersionKind::DeleteMarker, + size: 0, + }, + ObjectVersionEntry { + key: "_meta/tenant/a.json".to_string(), + version_id: "v-new".to_string(), + kind: ObjectVersionKind::Object, + size: 42, + }, + ] + ); + } +} diff --git a/crates/buzz-object-store/src/revision.rs b/crates/buzz-object-store/src/revision.rs new file mode 100644 index 00000000000..ca0159c7c2d --- /dev/null +++ b/crates/buzz-object-store/src/revision.rs @@ -0,0 +1,150 @@ +//! Provider-safe revision tokens and conditional-write vocabulary. +//! +//! A revision is whatever the backing provider uses to name "the version of +//! this object I just observed": an ETag on S3, an object generation on Google +//! Cloud Storage. The two are not interchangeable, and the type keeps them +//! from being confused — a generation handed to an S3 `If-Match` would be a +//! silent correctness bug, so [`Revision::expect_s3_etag`] rejects it instead. + +use crate::error::ObjectStoreError; + +/// Which provider minted a [`Revision`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProviderKind { + /// S3 and S3-compatible backends (AWS S3, MinIO, Railway). + S3, + /// Google Cloud Storage, using native object generations. + Gcs, +} + +impl std::fmt::Display for ProviderKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::S3 => f.write_str("s3"), + Self::Gcs => f.write_str("gcs"), + } + } +} + +/// An opaque, provider-qualified object revision. +/// +/// Revisions are produced by reads and successful conditional writes, and are +/// consumed by the next conditional write. They are never parsed, compared +/// across providers, or synthesised by callers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Revision { + /// S3 entity tag, verbatim as the provider returned it (quotes included). + S3Etag(String), + /// Google Cloud Storage object generation. + GcsGeneration(i64), +} + +impl Revision { + /// Which provider this revision belongs to. + pub fn provider(&self) -> ProviderKind { + match self { + Self::S3Etag(_) => ProviderKind::S3, + Self::GcsGeneration(_) => ProviderKind::Gcs, + } + } + + /// Borrow the S3 entity tag, rejecting a revision from another provider. + /// + /// This is the guard that keeps a provider swap from degrading into a + /// blind overwrite: a caller holding a GCS generation cannot accidentally + /// predicate an S3 `If-Match` on it. + pub fn expect_s3_etag(&self) -> Result<&str, ObjectStoreError> { + match self { + Self::S3Etag(tag) => Ok(tag), + other => Err(ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: other.provider(), + }), + } + } + + /// Read the GCS object generation, rejecting a revision from another provider. + pub fn expect_gcs_generation(&self) -> Result { + match self { + Self::GcsGeneration(generation) => Ok(*generation), + other => Err(ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: other.provider(), + }), + } + } +} + +/// Precondition for a conditional write. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteCondition { + /// Create-only: commit iff the object does not yet exist. + Absent, + /// Compare-and-swap: commit iff the object is still at this revision. + Matches(Revision), +} + +/// Outcome of a conditional write. +/// +/// `Conflict` is *not* an error — it is the ordinary result of losing a +/// compare-and-swap race, and callers must classify it as such (retry, or +/// report a non-fast-forward) rather than as a backend failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConditionalWrite { + /// The write committed; the new revision predicates the next write. + Committed(Revision), + /// The precondition did not hold; nothing was written. + Conflict, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn revision_reports_its_provider() { + assert_eq!( + Revision::S3Etag("\"abc\"".into()).provider(), + ProviderKind::S3 + ); + assert_eq!(Revision::GcsGeneration(7).provider(), ProviderKind::Gcs); + } + + #[test] + fn s3_accessor_accepts_an_s3_revision() { + let revision = Revision::S3Etag("\"abc\"".into()); + assert_eq!(revision.expect_s3_etag().unwrap(), "\"abc\""); + } + + #[test] + fn s3_accessor_rejects_a_gcs_revision() { + let err = Revision::GcsGeneration(7).expect_s3_etag().unwrap_err(); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::S3, + actual: ProviderKind::Gcs, + } + )); + } + + #[test] + fn gcs_accessor_rejects_an_s3_revision() { + let err = Revision::S3Etag("\"abc\"".into()) + .expect_gcs_generation() + .unwrap_err(); + assert!(matches!( + err, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + } + + #[test] + fn provider_kind_renders_a_stable_label() { + assert_eq!(ProviderKind::S3.to_string(), "s3"); + assert_eq!(ProviderKind::Gcs.to_string(), "gcs"); + } +} diff --git a/crates/buzz-object-store/tests/gcs_live.rs b/crates/buzz-object-store/tests/gcs_live.rs new file mode 100644 index 00000000000..27cd0a9bd32 --- /dev/null +++ b/crates/buzz-object-store/tests/gcs_live.rs @@ -0,0 +1,861 @@ +//! Live round-trip tests for the Google Cloud Storage provider. +//! +//! These talk to a real bucket, so they are `#[ignore]`d and additionally +//! gated on `BUZZ_GCS_LIVE=1` — a bare `--ignored` run in an environment +//! without credentials skips instead of failing. +//! +//! ```bash +//! BUZZ_GCS_LIVE=1 \ +//! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ +//! cargo test -p buzz-object-store --test gcs_live -- --ignored +//! ``` +//! +//! Credentials come from Application Default Credentials, exactly as they do +//! in production. The bucket must satisfy the provider's admission check — +//! object versioning disabled and soft-delete retention zero — which the first +//! test asserts explicitly. +//! +//! Every test works under a unique `a2//` prefix and deletes what it +//! wrote, so a bucket accumulates nothing across runs and concurrent runs +//! cannot collide. +//! +//! Two arms need a bucket that is *not* the disposable one, and stay skipped +//! unless the operator names it. Both are read-only — they never write, and +//! the provider refuses to hand out a client for either: +//! +//! ```bash +//! BUZZ_GCS_DENIED_BUCKET=a-bucket-this-identity-cannot-read \ +//! BUZZ_GCS_MISCONFIGURED_BUCKET=a-bucket-with-versioning-or-soft-delete-on \ +//! … +//! ``` + +use std::panic::AssertUnwindSafe; +use std::time::{Duration, Instant}; + +use futures_util::FutureExt; + +use buzz_object_store::{ + ConditionalWrite, GcsObjectStore, GcsStoreConfig, ImmutableWrite, ObjectStore, + ObjectStoreError, ObjectVersionKind, ObjectVersionRef, ProviderKind, Revision, WriteCondition, +}; + +const CONTENT_TYPE: &str = "application/octet-stream"; + +/// Install the process-wide rustls provider these tests need. +/// +/// The relay does this in `main` before any TLS request. A test binary must do +/// the same: both ring and aws-lc-rs are in the build graph, so rustls refuses +/// to pick one on its own. +fn install_crypto_provider() { + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); +} + +/// Whether this environment opted in to talking to a real bucket. +fn live_enabled(test: &str) -> bool { + install_crypto_provider(); + if std::env::var("BUZZ_GCS_LIVE").as_deref() == Ok("1") { + return true; + } + eprintln!("skipping {test}: set BUZZ_GCS_LIVE=1 to run against a live bucket"); + false +} + +/// A bucket named by `variable`, or `None` when the operator did not supply one. +fn named_bucket(variable: &str, test: &str) -> Option { + match std::env::var(variable) { + Ok(bucket) if !bucket.trim().is_empty() => Some(bucket.trim().to_string()), + _ => { + eprintln!("skipping {test}: set {variable} to run this arm"); + None + } + } +} + +/// Connect to the test bucket, or `None` when this environment has not opted +/// in to live tests. +async fn live_store(test: &str) -> Option<(GcsObjectStore, String)> { + if !live_enabled(test) { + return None; + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_LIVE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + let store = GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"); + let prefix = format!("a2/{}/{test}", uuid::Uuid::new_v4()); + Some((store, prefix)) +} + +/// Delete everything this test wrote. +/// +/// Failing to clean up is a test failure: these buckets are shared with other +/// runs and a leak would be invisible until it was large. +async fn cleanup(store: &GcsObjectStore, prefix: &str) { + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(prefix, token, 1000) + .await + .expect("list for cleanup"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + break; + } + } + if keys.is_empty() { + return; + } + let outcome = store + .delete_objects(&keys) + .await + .expect("bulk delete for cleanup"); + assert!( + outcome.failed.is_empty(), + "cleanup left objects behind: {:?}", + outcome.failed + ); +} + +/// Run one live test body, then clean up its prefix whether it passed or not. +/// +/// A panicking body would otherwise leak its objects into a bucket shared with +/// every other run — and a failing test is precisely the one that gets re-run, +/// so that is when cleanup matters most. +async fn with_prefix(test: &str, body: F) +where + F: AsyncFnOnce(&GcsObjectStore, &str), +{ + let Some((store, prefix)) = live_store(test).await else { + return; + }; + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + cleanup(&store, &prefix).await; + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } +} + +/// The provider reports itself as Cloud Storage, and the bucket satisfies the +/// deletion contract the constructor enforces. +/// +/// `versioning_detected()` is read from bucket metadata rather than probed, so +/// a correctly configured bucket answers `false` even though every Cloud +/// Storage object carries a generation. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn admits_a_bucket_that_can_prove_deletion() { + with_prefix("admission", async |store, _prefix| { + assert_eq!(store.provider(), ProviderKind::Gcs); + assert!( + !store.versioning_detected().await.expect("bucket metadata"), + "the test bucket must have object versioning and soft delete disabled" + ); + store.ping().await.expect("bucket is reachable"); + }) + .await; +} + +/// The provider-neutral exact-version contract maps GCS generations without +/// leaking generation vocabulary into media or deletion code. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn lists_and_deletes_an_exact_generation() { + with_prefix("exact-version", async |store, prefix| { + let key = format!("{prefix}/_meta/a.json"); + store + .put(&key, b"version body", CONTENT_TYPE) + .await + .expect("write object"); + + let page = store + .list_versions_page(prefix, None, None, 100) + .await + .expect("list exact versions"); + let entry = page + .entries + .into_iter() + .find(|entry| entry.key == key) + .expect("written generation is listed"); + assert_eq!(entry.kind, ObjectVersionKind::Object); + + let outcome = store + .delete_versions(&[ObjectVersionRef { + key: entry.key, + version_id: entry.version_id, + }]) + .await + .expect("delete exact generation"); + assert_eq!(outcome.deleted, 1); + assert!(outcome.failed.is_empty()); + assert!(store.head(&key).await.expect("head after delete").is_none()); + }) + .await; +} + +/// Create-only writes: the first commits, the second finds the key taken. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn create_only_write_is_idempotent() { + with_prefix("immutable", async |store, prefix| { + let key = format!("{prefix}/packs/object"); + + assert_eq!( + store + .put_immutable(&key, b"immutable bytes", CONTENT_TYPE) + .await + .expect("first create-only write"), + ImmutableWrite::Created + ); + assert_eq!( + store + .put_immutable(&key, b"immutable bytes", CONTENT_TYPE) + .await + .expect("second create-only write"), + ImmutableWrite::AlreadyPresent, + "a create-only precondition failure is success, not an error" + ); + assert_eq!( + store.get(&key).await.expect("read back").as_ref(), + b"immutable bytes" + ); + }) + .await; +} + +/// The compare-and-swap contract, end to end. +/// +/// Two writers hold the same generation. The first commits and the second — +/// now holding a stale generation — must lose, with exactly one winner and no +/// silent overwrite. The winning generation then predicates the next write, so +/// a caller can chain transitions without ever rereading. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn compare_and_swap_admits_exactly_one_writer() { + with_prefix("cas", async |store, prefix| { + let key = format!("{prefix}/pointers/manifest"); + + // Create with the absent precondition. + let created = store + .put_conditional(&key, b"state-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer"); + let ConditionalWrite::Committed(generation_0) = created else { + panic!("creating an absent pointer must commit"); + }; + assert!(matches!(generation_0, Revision::GcsGeneration(g) if g > 0)); + + // A second create-only write cannot commit over it. + assert_eq!( + store + .put_conditional(&key, b"state-x", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("second create"), + ConditionalWrite::Conflict + ); + + // Body and revision come from one read, and the revision predicates the + // next write. + let (read_generation, body) = store + .get_with_revision(&key) + .await + .expect("pointer read") + .expect("pointer exists"); + assert_eq!(body.as_ref(), b"state-0"); + assert_eq!(read_generation, generation_0); + + // Two sequential writers, both holding `generation_0`. Exactly one wins. + let winner = store + .put_conditional( + &key, + b"state-1", + CONTENT_TYPE, + WriteCondition::Matches(read_generation.clone()), + ) + .await + .expect("first writer"); + let loser = store + .put_conditional( + &key, + b"state-2", + CONTENT_TYPE, + WriteCondition::Matches(read_generation), + ) + .await + .expect("second writer"); + + let ConditionalWrite::Committed(generation_1) = winner else { + panic!("the writer holding the current generation must commit"); + }; + assert_ne!( + generation_1, generation_0, + "a commit mints a new generation" + ); + assert_eq!( + loser, + ConditionalWrite::Conflict, + "a stale generation must lose the race rather than overwrite" + ); + assert_eq!( + store.get(&key).await.expect("read winner").as_ref(), + b"state-1", + "the loser must not have written anything" + ); + + // The winner's generation chains straight into the next transition. + assert!(matches!( + store + .put_conditional( + &key, + b"state-3", + CONTENT_TYPE, + WriteCondition::Matches(generation_1), + ) + .await + .expect("chained write"), + ConditionalWrite::Committed(_) + )); + }) + .await; +} + +/// An ETag can never predicate a generation match; the write is refused rather +/// than silently downgraded to an unconditional overwrite. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn a_foreign_revision_never_reaches_the_backend() { + with_prefix("foreign-revision", async |store, prefix| { + let key = format!("{prefix}/pointers/manifest"); + + store + .put_conditional(&key, b"state-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer"); + + let error = store + .put_conditional( + &key, + b"clobbered", + CONTENT_TYPE, + WriteCondition::Matches(Revision::S3Etag("\"deadbeef\"".into())), + ) + .await + .expect_err("an S3 ETag must not predicate a GCS write"); + assert!(matches!( + error, + ObjectStoreError::RevisionMismatch { + expected: ProviderKind::Gcs, + actual: ProviderKind::S3, + } + )); + assert_eq!( + store.get(&key).await.expect("read back").as_ref(), + b"state-0", + "the refused write must not have touched the object" + ); + }) + .await; +} + +/// Full, ranged, and streamed reads agree, and metadata reports the size and +/// generation. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn reads_agree_across_full_range_and_stream() { + use futures_util::StreamExt; + + with_prefix("reads", async |store, prefix| { + let key = format!("{prefix}/blobs/payload"); + let body: Vec = (0..4096u32).map(|i| (i % 251) as u8).collect(); + + store.put(&key, &body, CONTENT_TYPE).await.expect("put"); + + assert_eq!( + store.get(&key).await.expect("full read").as_ref(), + &body[..] + ); + assert_eq!( + store.get_range(&key, 100, 199).await.expect("range read"), + &body[100..=199], + "the seam's range is inclusive on both ends" + ); + assert_eq!( + store.get_range(&key, 4095, 4095).await.expect("last byte"), + &body[4095..], + ); + + let mut streamed = Vec::new(); + let mut chunks = store.get_stream(&key).await.expect("open stream"); + while let Some(chunk) = chunks.next().await { + streamed.extend_from_slice(&chunk.expect("stream chunk")); + } + assert_eq!(streamed, body); + + let meta = store + .head(&key) + .await + .expect("head") + .expect("object exists"); + assert_eq!(meta.size, body.len() as u64); + assert!(matches!(meta.revision, Some(Revision::GcsGeneration(g)) if g > 0)); + + assert!(store + .head(&format!("{prefix}/blobs/absent")) + .await + .expect("head of an absent object is not an error") + .is_none()); + assert!(store + .get_with_revision(&format!("{prefix}/blobs/absent")) + .await + .expect("pointer read of an absent object is not an error") + .is_none()); + assert!(matches!( + store.get(&format!("{prefix}/blobs/absent")).await, + Err(ObjectStoreError::NotFound { .. }) + )); + }) + .await; +} + +/// Prefix listing pages, in ascending key order, without dropping or repeating +/// a key across the page boundary. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn listing_paginates_over_a_prefix() { + with_prefix("listing", async |store, prefix| { + let expected: Vec = (0..5).map(|i| format!("{prefix}/page/{i:03}")).collect(); + for key in &expected { + store.put(key, b"x", CONTENT_TYPE).await.expect("put"); + } + + let mut seen = Vec::new(); + let mut token = None; + let mut pages = 0; + loop { + let page = store.list_page(prefix, token, 2).await.expect("list page"); + pages += 1; + assert!(page.objects.len() <= 2, "max_keys must bound one response"); + seen.extend(page.objects.into_iter().map(|(key, size)| { + assert_eq!(size, 1, "listing reports object size"); + key + })); + token = page.next_continuation_token; + match token { + Some(_) => assert!(page.is_truncated, "a continuation token means truncated"), + None => { + assert!(!page.is_truncated, "the last page is not truncated"); + break; + } + } + } + + assert!(pages > 1, "five keys at two per page must span pages"); + assert_eq!( + seen, expected, + "keys arrive in ascending order, exactly once" + ); + }) + .await; +} + +/// Deletion means deletion, deleting an absent key is not an error, and a bulk +/// delete folds present and absent keys into distinct counters. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn deletion_is_idempotent_and_reports_per_key_outcomes() { + with_prefix("deletion", async |store, prefix| { + let key = format!("{prefix}/single"); + store.put(&key, b"x", CONTENT_TYPE).await.expect("put"); + store.delete(&key).await.expect("delete"); + // Absence has to hold on every read path, immediately. A bucket with + // soft delete on would answer the HEAD with a live object or resurrect + // one on the next read; the admission check exists to keep that bucket + // from ever getting this far, and this is the observable consequence. + assert!( + store.head(&key).await.expect("head").is_none(), + "with versioning and soft delete off, a delete proves absence" + ); + assert!( + matches!( + store.get(&key).await, + Err(ObjectStoreError::NotFound { .. }) + ), + "a deleted object must not be readable" + ); + assert!( + store + .get_with_revision(&key) + .await + .expect("pointer read of a deleted object is not an error") + .is_none(), + "a deleted object must not still carry a revision" + ); + assert!( + store + .list_page(&key, None, 10) + .await + .expect("list") + .objects + .is_empty(), + "a deleted object must not still be listed" + ); + store + .delete(&key) + .await + .expect("deleting an absent object is not an error"); + + let present: Vec = (0..3).map(|i| format!("{prefix}/bulk/{i}")).collect(); + for key in &present { + store.put(key, b"x", CONTENT_TYPE).await.expect("put"); + } + let absent: Vec = (0..2).map(|i| format!("{prefix}/bulk/gone-{i}")).collect(); + + let mut keys = present.clone(); + keys.extend(absent); + let outcome = store.delete_objects(&keys).await.expect("bulk delete"); + + assert_eq!(outcome.deleted, 3); + assert_eq!(outcome.already_missing, 2); + assert!(outcome.failed.is_empty(), "{:?}", outcome.failed); + assert!( + outcome.versioned_keys.is_empty(), + "an admitted bucket cannot produce version artifacts" + ); + + assert!(store + .list_page(prefix, None, 10) + .await + .expect("list") + .objects + .is_empty()); + }) + .await; +} + +/// A sole writer driving the pointer faster than Cloud Storage's documented +/// one-write-per-second-per-object limit. +/// +/// This is the shape of the mirror's chunked repository seed: many sequential +/// transitions against a single object name, each predicated on the generation +/// the previous one returned. Throttling must pace the writer — never fail the +/// push, and never masquerade as a lost race. +/// +/// The two phases are ordered deliberately. The first uses a client configured +/// for a single attempt, so every 429 reaches the caller and can be counted; +/// it also spends whatever burst allowance the service was willing to grant. +/// The second then drives the same key just as hard through the provider's own +/// bounded policy, with that allowance already gone — which is what makes it a +/// real test of absorption rather than of a quiet service. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn rapid_sequential_transitions_are_paced_not_failed() { + // Close to a real chunked seed of a large mirrored repository (~4.5k refs + // at 200 refs per chunk), and comfortably past the burst allowance the + // service grants a fresh object name — which is what makes the throttling + // in this test real rather than incidental. + const TRANSITIONS: usize = 20; + + with_prefix("pacing", async |store, prefix| { + let key = format!("{prefix}/pointers/hot"); + + let mut revision = match store + .put_conditional(&key, b"transition-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create pointer") + { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("creating an absent pointer must commit"), + }; + + // Phase 1: no in-provider retries, so throttling is visible and counted. + let mut config = GcsStoreConfig::new(store.bucket()); + config.retry.max_attempts = 1; + config.retry.max_throttled_attempts = 1; + let unpaced = GcsObjectStore::connect(&config) + .await + .expect("connect a single-attempt client"); + + let mut observed_429s = 0usize; + for i in 0..TRANSITIONS { + let body = format!("unpaced-{i}"); + loop { + match unpaced + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision.clone()), + ) + .await + { + Ok(ConditionalWrite::Committed(next)) => { + revision = next; + break; + } + Ok(ConditionalWrite::Conflict) => { + panic!("transition {i} lost a race it was the only entrant in") + } + // Throttling is backpressure. The retry carries the same + // precondition; dropping it here would be the exact bug this + // test exists to prevent. + Err(ObjectStoreError::Throttled { retry_after, .. }) => { + observed_429s += 1; + tokio::time::sleep(retry_after.unwrap_or(Duration::from_millis(400))).await; + } + Err(other) => panic!("transition {i} failed: {other}"), + } + } + } + + // Phase 2: the provider's own bounded policy, against a spent allowance. + let started = Instant::now(); + let mut generations = vec![revision.clone()]; + for i in 0..TRANSITIONS { + let body = format!("paced-{i}"); + revision = match store + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision), + ) + .await + .unwrap_or_else(|e| panic!("transition {i} must not fail: {e}")) + { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => { + panic!("transition {i} lost a race it was the only entrant in") + } + }; + generations.push(revision.clone()); + } + let elapsed = started.elapsed(); + + let mut distinct = generations.clone(); + distinct.dedup(); + assert_eq!( + distinct.len(), + generations.len(), + "every transition must mint a distinct generation" + ); + assert_eq!( + store.get(&key).await.expect("final read").as_ref(), + format!("paced-{}", TRANSITIONS - 1).as_bytes(), + "the last transition must be the published state" + ); + + eprintln!( + "pacing: {TRANSITIONS} single-attempt transitions observed {observed_429s} throttled \ + attempts; {TRANSITIONS} further transitions then completed through the provider's own \ + policy in {elapsed:?}" + ); + }) + .await; +} + +/// Streaming a large object to the backend and back without ever holding it in +/// memory. +/// +/// `put_file` is the media path for blobs that do not fit in a buffer: it hands +/// the client an open file and lets the resumable upload protocol do the +/// chunking. The read side is the mirror image — the byte stream is folded into +/// a digest as it arrives, so nothing larger than one chunk is resident at +/// either end. The assertions that matter are that the round trip is +/// byte-exact, that the provider reports the size it stored, and that a range +/// read still addresses the far end of a multi-chunk object correctly. +/// +/// The size is configurable because the deployment's ceiling (500 MiB) is +/// larger than a routine test run wants to move; the default is big enough to +/// require a resumable upload rather than a single-request one. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn streams_a_large_object_through_a_resumable_upload() { + use std::io::Write; + + use sha2::{Digest, Sha256}; + + let mebibytes: usize = std::env::var("BUZZ_GCS_LARGE_OBJECT_MIB") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(96); + + with_prefix("large-object", async |store, prefix| { + let key = format!("{prefix}/media/large.bin"); + let total = mebibytes * 1024 * 1024; + + // A pseudo-random, non-repeating body: a run of identical blocks would + // let a chunking bug (a dropped or duplicated chunk) round-trip + // undetected. + let mut block = vec![0u8; 1024 * 1024]; + let mut expected = Sha256::new(); + let source = std::env::temp_dir().join(format!("buzz-a4-large-{}", uuid::Uuid::new_v4())); + { + let mut file = std::fs::File::create(&source).expect("create the upload source"); + let mut state = 0x2545_F491_4F6C_DD1Du64; + for _ in 0..mebibytes { + for byte in block.iter_mut() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + *byte = (state >> 24) as u8; + } + expected.update(&block); + file.write_all(&block).expect("write the upload source"); + } + file.flush().expect("flush the upload source"); + } + let expected = hex::encode(expected.finalize()); + + let upload = Instant::now(); + let outcome = store.put_file(&key, &source, CONTENT_TYPE).await; + let upload = upload.elapsed(); + // The temp file is this test's, not the harness's: remove it before + // asserting so a failure cannot leave a large file behind. + let _ = std::fs::remove_file(&source); + outcome.expect("streaming upload"); + + let meta = store + .head(&key) + .await + .expect("head") + .expect("the uploaded object exists"); + assert_eq!(meta.size, total as u64, "the provider stored every byte"); + + let download = Instant::now(); + let mut streamed = Sha256::new(); + let mut length = 0u64; + { + use futures_util::StreamExt; + let mut chunks = store.get_stream(&key).await.expect("open the read stream"); + while let Some(chunk) = chunks.next().await { + let chunk = chunk.expect("stream chunk"); + length += chunk.len() as u64; + streamed.update(&chunk); + } + } + let download = download.elapsed(); + assert_eq!(length, total as u64, "the stream delivered every byte"); + assert_eq!( + hex::encode(streamed.finalize()), + expected, + "the streamed body must be byte-exact" + ); + + // A range read against the far end of a multi-chunk object: the offset + // is past every upload chunk boundary, so an implementation that + // resolved ranges against a single chunk would miss here. + let tail = store + .get_range(&key, total as u64 - 8, total as u64 - 1) + .await + .expect("range read at the tail"); + assert_eq!(tail.len(), 8, "the seam's range is inclusive on both ends"); + + eprintln!( + "large object: {mebibytes} MiB uploaded in {upload:?}, streamed back in {download:?}" + ); + }) + .await; +} + +/// Permission denied is a classified, permanent answer — never an ambiguous +/// outcome, and never a reason to come up anyway. +/// +/// The bucket-metadata read is the first request the provider makes, so a +/// client for a bucket this identity cannot read never gets constructed. That +/// is the fail-closed behaviour that matters: an unauthorised deployment stops +/// at connect rather than discovering its authorisation one operation at a +/// time. +/// +/// The arm is read-only. It reads bucket metadata and nothing else, and the +/// only outcome it accepts is a refusal. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn permission_denied_refuses_to_hand_out_a_client() { + let test = "permission-denied"; + if !live_enabled(test) { + return; + } + let Some(bucket) = named_bucket("BUZZ_GCS_DENIED_BUCKET", test) else { + return; + }; + + let error = GcsObjectStore::connect(&GcsStoreConfig::new(&bucket)) + .await + .err() + .unwrap_or_else(|| { + panic!("BUZZ_GCS_DENIED_BUCKET={bucket} is readable by this identity — pick a bucket it cannot read") + }); + + match &error { + ObjectStoreError::Provider { operation, .. } => { + assert_eq!(*operation, "get_bucket", "the refusal names the operation"); + } + other => panic!( + "permission denied must be a classified provider answer, not {other:?}: an \ + authorisation failure that read as ambiguous or retryable would be retried \ + forever, and one that read as a conflict would be scored as a lost race" + ), + } + assert!( + !error.is_ambiguous(), + "a 403 is an answer: the request was evaluated and refused" + ); + assert!( + !error.is_retryable(), + "retrying a permission failure cannot change its outcome" + ); + eprintln!("permission denied: {error}"); +} + +/// A bucket that cannot prove deletion is refused at connect, with the +/// violation named. +/// +/// This is the A2 admission check against a real misconfigured bucket rather +/// than a constructed one: object versioning or a nonzero soft-delete retention +/// both mean a delete leaves a restorable copy, so `delete` could report success +/// while the bytes stay reachable. Buzz's deletion contract cannot hold on such +/// a bucket, so the provider refuses to return a client for it at all. +/// +/// Read-only: the run reads bucket metadata and stops. Nothing is written to +/// the misconfigured bucket, which is the point — the client never exists. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] +async fn a_bucket_that_cannot_prove_deletion_is_refused() { + let test = "misconfigured-bucket"; + if !live_enabled(test) { + return; + } + let Some(bucket) = named_bucket("BUZZ_GCS_MISCONFIGURED_BUCKET", test) else { + return; + }; + + let error = GcsObjectStore::connect(&GcsStoreConfig::new(&bucket)) + .await + .err() + .unwrap_or_else(|| { + panic!( + "BUZZ_GCS_MISCONFIGURED_BUCKET={bucket} satisfies the deletion contract — point \ + this at a bucket with object versioning or a nonzero soft-delete retention" + ) + }); + + let ObjectStoreError::Config(message) = &error else { + panic!("a misconfigured bucket must be refused as a configuration error, not {error:?}"); + }; + assert!( + message.contains("does not satisfy the deletion contract"), + "the refusal must name the contract it failed: {message}" + ); + assert!( + message.contains("object versioning is enabled") || message.contains("soft-delete"), + "the refusal must name the violating setting: {message}" + ); + eprintln!("misconfigured bucket refused: {error}"); +} diff --git a/crates/buzz-object-store/tests/gcs_scale.rs b/crates/buzz-object-store/tests/gcs_scale.rs new file mode 100644 index 00000000000..2e0be997c59 --- /dev/null +++ b/crates/buzz-object-store/tests/gcs_scale.rs @@ -0,0 +1,586 @@ +//! Store-level scale evidence for the Google Cloud Storage provider. +//! +//! These arms answer the questions the functional suite cannot: whether the +//! provider's behaviour holds when many writers run at once, whether the +//! compare-and-swap contract still admits exactly one writer per round when +//! that round is repeated dozens of times, and what latency and outcome mix a +//! sustained load actually produces. +//! +//! They cost minutes and thousands of requests, so they are gated twice — +//! `BUZZ_GCS_LIVE=1` like the functional suite, plus `BUZZ_GCS_SCALE=1`: +//! +//! ```bash +//! BUZZ_GCS_LIVE=1 BUZZ_GCS_SCALE=1 \ +//! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ +//! cargo test -p buzz-object-store --test gcs_scale -- --ignored --nocapture +//! ``` +//! +//! Every knob has an environment override so a rehearsal can widen a run +//! without editing the source; the defaults are what a routine run should cost. +//! +//! ## What these arms measure, and what they do not +//! +//! They are *store-level* evidence: pointer objects and blobs through the +//! object-store seam. They are not a repository-shaped corpus — no packs of +//! realistic size, no ref counts, no cold materialisation — and they should not +//! be read as one. +//! +//! Throttling deserves one note. The provider absorbs 429 inside its own +//! bounded policy, so a caller running the production configuration sees almost +//! none; a caller-visible count near zero here is evidence that pacing works, +//! not that the service never throttled. The functional suite's pacing arm +//! drives the same key through a single-attempt client precisely so the 429s +//! become visible and countable. + +use std::collections::HashMap; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures_util::FutureExt; + +use buzz_object_store::{ + ConditionalWrite, GcsObjectStore, GcsStoreConfig, ObjectStore, ObjectStoreError, Revision, + WriteCondition, +}; + +const CONTENT_TYPE: &str = "application/octet-stream"; + +/// How many pointer creates the distinct-name arm issues at once. +const DEFAULT_DISTINCT_WRITERS: usize = 500; +/// How many hot-pointer contention rounds to run. +const DEFAULT_CONTENTION_ROUNDS: usize = 25; +/// How many writers race for the same pointer in one round. +const DEFAULT_CONTENTION_WIDTH: usize = 3; +/// How long the sustained mixed-load arm runs. +const DEFAULT_MIXED_LOAD_SECONDS: u64 = 120; +/// How many independent workers the mixed-load arm runs. +const DEFAULT_MIXED_LOAD_WORKERS: usize = 8; +/// How long each mixed-load worker waits between operations. +const DEFAULT_MIXED_LOAD_INTERVAL_MS: u64 = 250; + +/// Just past Cloud Storage's documented one-write-per-second per-object +/// ceiling, so a contention round measures the conditional-write semantics +/// rather than the rate limiter. +const SAME_KEY_SPACING: Duration = Duration::from_millis(1_100); + +/// Read a `usize` knob from the environment. +fn knob(variable: &str, default: usize) -> usize { + std::env::var(variable) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +/// Connect to the test bucket, or `None` when this environment has not opted +/// in to the scale arms. +async fn scale_store(test: &str) -> Option<(Arc, String)> { + // The relay installs this in `main` before any TLS request. These tests are + // their own process and must do the same: both ring and aws-lc-rs are in + // the build graph, so rustls refuses to pick one on its own. + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); + + for (variable, value) in [("BUZZ_GCS_LIVE", "1"), ("BUZZ_GCS_SCALE", "1")] { + if std::env::var(variable).as_deref() != Ok(value) { + eprintln!("skipping {test}: set {variable}={value} to run the scale arms"); + return None; + } + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_SCALE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + let store = Arc::new( + GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"), + ); + Some((store, format!("a4-scale/{}/{test}", uuid::Uuid::new_v4()))) +} + +/// Run one scale arm, then remove everything it wrote whether it passed or not. +/// +/// These arms write hundreds to thousands of objects, so a leak on the failure +/// path is not a rounding error — and a failing arm is the one that gets re-run. +async fn with_prefix(test: &str, body: F) +where + F: AsyncFnOnce(&Arc, &str), +{ + let Some((store, prefix)) = scale_store(test).await else { + return; + }; + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(&prefix, token, 1000) + .await + .expect("list for cleanup"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + break; + } + } + let removed = keys.len(); + let cleanup = if keys.is_empty() { + None + } else { + Some(store.delete_objects(&keys).await) + }; + + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } + if let Some(cleanup) = cleanup { + let cleanup = cleanup.expect("bulk delete for cleanup"); + assert!( + cleanup.failed.is_empty(), + "cleanup left objects behind: {:?}", + cleanup.failed + ); + } + let leaked = store + .list_page(&prefix, None, 10) + .await + .expect("verify the namespace is empty"); + assert!( + leaked.objects.is_empty(), + "the namespace must be empty after cleanup: {:?}", + leaked.objects + ); + eprintln!("{test}: removed {removed} objects, namespace verified empty"); +} + +/// Latency samples for one class of operation. +#[derive(Default)] +struct Latencies(Vec); + +impl Latencies { + fn record(&mut self, elapsed: Duration) { + self.0.push(elapsed); + } + + /// The sample at `fraction` through the sorted distribution. + /// + /// Nearest-rank, so a small sample reports a real observation rather than + /// an interpolation between two of them. + fn percentile(&mut self, fraction: f64) -> Duration { + if self.0.is_empty() { + return Duration::ZERO; + } + self.0.sort_unstable(); + let rank = ((self.0.len() as f64) * fraction).ceil() as usize; + self.0[rank.clamp(1, self.0.len()) - 1] + } + + fn summary(&mut self, label: &str) -> String { + let count = self.0.len(); + let p50 = self.percentile(0.50); + let p95 = self.percentile(0.95); + let p99 = self.percentile(0.99); + format!("{label}: n={count} p50={p50:?} p95={p95:?} p99={p99:?}") + } +} + +/// How one attempt was answered, folded across a whole run. +#[derive(Default, Debug, PartialEq, Eq)] +struct Outcomes { + committed: u64, + conflicts: u64, + reads: u64, + lists: u64, + throttled: u64, + retryable: u64, + ambiguous: u64, + terminal: u64, +} + +impl Outcomes { + /// Fold a provider error into the counter its classification names. + fn record_error(&mut self, error: &ObjectStoreError) { + match error { + ObjectStoreError::Throttled { .. } => self.throttled += 1, + ObjectStoreError::TransportRetryable { .. } => self.retryable += 1, + ObjectStoreError::TransportAmbiguous { .. } => self.ambiguous += 1, + _ => self.terminal += 1, + } + } +} + +/// Concurrent creates across distinct object names: the axis Cloud Storage +/// scales horizontally on. +/// +/// The per-object write ceiling applies to one object name. Independent +/// repositories publish to independent pointers, so this is the shape that has +/// to scale — and every writer must commit, because none of them is contending +/// with another. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn concurrent_creates_across_distinct_names_all_commit() { + let writers = knob("BUZZ_GCS_SCALE_DISTINCT_WRITERS", DEFAULT_DISTINCT_WRITERS); + + with_prefix("distinct-names", async |store, prefix| { + let started = Instant::now(); + let mut tasks = Vec::with_capacity(writers); + for i in 0..writers { + let store = Arc::clone(store); + let key = format!("{prefix}/pointers/{i:05}"); + tasks.push(tokio::spawn(async move { + let attempt = Instant::now(); + let outcome = store + .put_conditional( + &key, + format!("writer-{i}").as_bytes(), + CONTENT_TYPE, + WriteCondition::Absent, + ) + .await; + (outcome, attempt.elapsed()) + })); + } + + let mut outcomes = Outcomes::default(); + let mut latencies = Latencies::default(); + let mut revisions = Vec::with_capacity(writers); + for task in tasks { + let (outcome, elapsed) = task.await.expect("writer task"); + latencies.record(elapsed); + match outcome { + Ok(ConditionalWrite::Committed(revision)) => { + outcomes.committed += 1; + revisions.push(revision); + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => outcomes.record_error(&error), + } + } + let elapsed = started.elapsed(); + + assert_eq!( + outcomes.committed, writers as u64, + "every writer addressed its own object name, so every one must commit: {outcomes:?}" + ); + assert_eq!( + revisions.len(), + writers, + "a commit without a revision leaves the caller nothing to predicate its next write on" + ); + + eprintln!( + "distinct names: {writers} concurrent creates in {elapsed:?} ({:.0}/sec), {}", + writers as f64 / elapsed.as_secs_f64(), + latencies.summary("create") + ); + eprintln!("distinct names: outcomes {outcomes:?}"); + }) + .await; +} + +/// The hot-pointer race, repeated: exactly one winner per round, every round, +/// and the published state is always the winner's. +/// +/// One round proves the contract holds; repeating it is what would surface a +/// rare double commit or a lost update. The second assertion is the one that +/// catches a silent lost update — a round can report one winner and still have +/// published a loser's body if the backend committed out of order. +/// +/// Rounds are spaced past the published per-object ceiling on purpose: this arm +/// is measuring conditional-write semantics under contention, and a deliberately +/// over-rate round would only measure the rate limiter. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn a_repeated_hot_pointer_race_never_admits_two_winners() { + let rounds = knob( + "BUZZ_GCS_SCALE_CONTENTION_ROUNDS", + DEFAULT_CONTENTION_ROUNDS, + ); + let width = knob("BUZZ_GCS_SCALE_CONTENTION_WIDTH", DEFAULT_CONTENTION_WIDTH); + assert!(width >= 2, "a race needs at least two entrants"); + + with_prefix("hot-pointer", async |store, prefix| { + let key = format!("{prefix}/pointers/hot"); + + let ConditionalWrite::Committed(mut revision) = store + .put_conditional(&key, b"round-0", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let mut outcomes = Outcomes::default(); + let mut latencies = Latencies::default(); + let mut last_round_end = Instant::now(); + let started = Instant::now(); + + for round in 1..=rounds { + // Space same-key rounds past the documented ceiling. + let since = last_round_end.elapsed(); + if since < SAME_KEY_SPACING { + tokio::time::sleep(SAME_KEY_SPACING - since).await; + } + + let mut racers = Vec::with_capacity(width); + for racer in 0..width { + let store = Arc::clone(store); + let key = key.clone(); + let revision = revision.clone(); + let body = format!("round-{round}-racer-{racer}"); + racers.push(tokio::spawn(async move { + let attempt = Instant::now(); + let outcome = store + .put_conditional( + &key, + body.as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision), + ) + .await; + (body, outcome, attempt.elapsed()) + })); + } + + let mut winners: Vec<(String, Revision)> = Vec::new(); + for racer in racers { + let (body, outcome, elapsed) = racer.await.expect("racer task"); + latencies.record(elapsed); + match outcome { + Ok(ConditionalWrite::Committed(next)) => { + outcomes.committed += 1; + winners.push((body, next)); + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => { + // Throttling is a refusal to evaluate the precondition, + // so it is never evidence of a lost race. + outcomes.record_error(&error); + } + } + } + last_round_end = Instant::now(); + + assert_eq!( + winners.len(), + 1, + "round {round} admitted {} winners on one revision — two committed writers on the \ + same precondition is a lost update announcing itself", + winners.len() + ); + let (winning_body, winning_revision) = winners.pop().expect("exactly one winner"); + assert_ne!( + winning_revision, revision, + "round {round} committed without minting a new revision" + ); + + // The published state must be the winner's, not a loser's: one + // acknowledged winner over somebody else's bytes is exactly the + // silent lost update this arm exists to rule out. + let (published_revision, published) = store + .get_with_revision(&key) + .await + .expect("read the published state") + .expect("the pointer exists"); + assert_eq!( + published.as_ref(), + winning_body.as_bytes(), + "round {round} published a body no acknowledged winner wrote" + ); + assert_eq!( + published_revision, winning_revision, + "round {round} published a revision the winner did not commit" + ); + revision = winning_revision; + } + let elapsed = started.elapsed(); + + assert_eq!( + outcomes.committed, rounds as u64, + "exactly one winner per round: {outcomes:?}" + ); + + eprintln!( + "hot pointer: {rounds} rounds of width {width} in {elapsed:?}, {}", + latencies.summary("cas") + ); + eprintln!("hot pointer: outcomes {outcomes:?}"); + }) + .await; +} + +/// A sustained mixed read/write/list load at a bounded rate. +/// +/// Each worker owns its own pointer, which is the deployment's real shape: +/// independent repositories publish to independent object names, so a worker's +/// compare-and-swap chain is sequential on its own object and stays inside the +/// per-object ceiling while the fleet as a whole is concurrent. +/// +/// This arm is a measurement, not a threshold. It asserts only what must never +/// happen — a writer losing a race it was the only entrant in, or a terminal +/// error — and reports the rest for the record. +#[tokio::test] +#[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1, BUZZ_GCS_SCALE=1)"] +async fn sustained_mixed_load_reports_its_latency_and_outcome_mix() { + let seconds = knob( + "BUZZ_GCS_SCALE_MIXED_SECONDS", + DEFAULT_MIXED_LOAD_SECONDS as usize, + ) as u64; + let workers = knob("BUZZ_GCS_SCALE_MIXED_WORKERS", DEFAULT_MIXED_LOAD_WORKERS); + let interval = Duration::from_millis(knob( + "BUZZ_GCS_SCALE_MIXED_INTERVAL_MS", + DEFAULT_MIXED_LOAD_INTERVAL_MS as usize, + ) as u64); + + with_prefix("mixed-load", async |store, prefix| { + let deadline = Instant::now() + Duration::from_secs(seconds); + + let mut tasks = Vec::with_capacity(workers); + for worker in 0..workers { + let store = Arc::clone(store); + let pointer = format!("{prefix}/pointers/worker-{worker:03}"); + let blob_prefix = format!("{prefix}/blobs/worker-{worker:03}"); + tasks.push(tokio::spawn(async move { + let mut outcomes = Outcomes::default(); + let mut latencies: HashMap<&'static str, Latencies> = HashMap::new(); + + let ConditionalWrite::Committed(mut revision) = store + .put_conditional(&pointer, b"seed", CONTENT_TYPE, WriteCondition::Absent) + .await + .expect("seed the worker's pointer") + else { + panic!("creating an absent pointer must commit"); + }; + outcomes.committed += 1; + + let mut step = 0u64; + while Instant::now() < deadline { + step += 1; + // 4 reads : 2 writes : 1 list. Reads dominate because + // serving a repository resolves its pointer per request. + let attempt = Instant::now(); + match step % 7 { + 0 => { + let outcome = store.list_page(&blob_prefix, None, 100).await; + latencies + .entry("list") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(_) => outcomes.lists += 1, + Err(error) => outcomes.record_error(&error), + } + } + 1 | 3 => { + // An immutable blob write, then a pointer swap + // publishing it — the two writes a push performs. + let blob = format!("{blob_prefix}/{step:06}"); + let outcome = store + .put_immutable( + &blob, + format!("blob-{step}").as_bytes(), + CONTENT_TYPE, + ) + .await; + latencies + .entry("write") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(_) => outcomes.committed += 1, + Err(error) => outcomes.record_error(&error), + } + + let swap = Instant::now(); + let outcome = store + .put_conditional( + &pointer, + format!("step-{step}").as_bytes(), + CONTENT_TYPE, + WriteCondition::Matches(revision.clone()), + ) + .await; + latencies.entry("cas").or_default().record(swap.elapsed()); + match outcome { + Ok(ConditionalWrite::Committed(next)) => { + outcomes.committed += 1; + revision = next; + } + Ok(ConditionalWrite::Conflict) => outcomes.conflicts += 1, + Err(error) => outcomes.record_error(&error), + } + } + _ => { + let outcome = store.get_with_revision(&pointer).await; + latencies + .entry("read") + .or_default() + .record(attempt.elapsed()); + match outcome { + Ok(Some(_)) => outcomes.reads += 1, + Ok(None) => panic!("the worker's own pointer disappeared"), + Err(error) => outcomes.record_error(&error), + } + } + } + + let spent = attempt.elapsed(); + if spent < interval { + tokio::time::sleep(interval - spent).await; + } + } + + (outcomes, latencies) + })); + } + + let started = Instant::now(); + let mut totals = Outcomes::default(); + let mut latencies: HashMap<&'static str, Latencies> = HashMap::new(); + for task in tasks { + let (outcomes, worker_latencies) = task.await.expect("mixed-load worker"); + totals.committed += outcomes.committed; + totals.conflicts += outcomes.conflicts; + totals.reads += outcomes.reads; + totals.lists += outcomes.lists; + totals.throttled += outcomes.throttled; + totals.retryable += outcomes.retryable; + totals.ambiguous += outcomes.ambiguous; + totals.terminal += outcomes.terminal; + for (class, samples) in worker_latencies { + latencies.entry(class).or_default().0.extend(samples.0); + } + } + let elapsed = started.elapsed(); + + assert_eq!( + totals.conflicts, 0, + "every worker owns its own pointer, so no worker can lose a race: {totals:?}" + ); + assert_eq!( + totals.terminal, 0, + "a terminal error under nominal load is a failure, not a measurement: {totals:?}" + ); + + let operations = totals.committed + totals.reads + totals.lists; + eprintln!( + "mixed load: {workers} workers for {elapsed:?}, {operations} operations \ + ({:.1}/sec), outcomes {totals:?}", + operations as f64 / elapsed.as_secs_f64() + ); + let mut classes: Vec<&'static str> = latencies.keys().copied().collect(); + classes.sort_unstable(); + for class in classes { + eprintln!( + "mixed load: {}", + latencies.get_mut(class).expect("class").summary(class) + ); + } + }) + .await; +} diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index deb2e7e16a5..f57175ddb67 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -64,7 +64,7 @@ base64 = "0.22" buzz-sdk = { workspace = true } buzz-workflow = { workspace = true, features = ["reqwest"] } buzz-media = { workspace = true } -s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } +buzz-object-store = { workspace = true } tempfile = "3" bytes = "1" infer = "0.19" diff --git a/crates/buzz-relay/src/api/admin/mod.rs b/crates/buzz-relay/src/api/admin/mod.rs index 2f0d128fc87..786bbba29a0 100644 --- a/crates/buzz-relay/src/api/admin/mod.rs +++ b/crates/buzz-relay/src/api/admin/mod.rs @@ -1309,7 +1309,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -1814,7 +1814,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -2307,7 +2307,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2388,7 +2388,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _audit_shutdown) = crate::state::AppState::new( config, db, @@ -2515,7 +2515,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2884,7 +2884,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -2957,7 +2957,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -3027,7 +3027,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (mut state, _) = crate::state::AppState::new( config, db, @@ -4780,7 +4780,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = crate::state::AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 31fdb019bfc..ae7cc5f44da 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -3858,7 +3858,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = crate::state::AppState::new( config, diff --git a/crates/buzz-relay/src/api/gifs.rs b/crates/buzz-relay/src/api/gifs.rs index c29df6746bb..384e151e385 100644 --- a/crates/buzz-relay/src/api/gifs.rs +++ b/crates/buzz-relay/src/api/gifs.rs @@ -381,8 +381,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = - buzz_media::MediaStorage::new(&config.media).expect("test media storage config"); + let media_storage = crate::test_media_storage(&config).expect("test media storage config"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/git/cas_publish.rs b/crates/buzz-relay/src/api/git/cas_publish.rs index 50bb36d818c..7c746342ab6 100644 --- a/crates/buzz-relay/src/api/git/cas_publish.rs +++ b/crates/buzz-relay/src/api/git/cas_publish.rs @@ -75,7 +75,9 @@ use crate::api::git::manifest::{ pointer_key, Manifest, ManifestError, MANIFEST_VERSION, MAX_MANIFEST_PACKS, MAX_MANIFEST_REFS, PACK_COMPACTION_THRESHOLD, }; -use crate::api::git::store::{CasOutcome, ETag, GitStore, Precond, StoreError}; +use buzz_object_store::{ConditionalWrite, Revision, WriteCondition}; + +use crate::api::git::store::{GitStore, StoreError}; use buzz_core::TenantContext; const PACK_CAPTURE_TIMEOUT: Duration = Duration::from_secs(300); @@ -202,7 +204,7 @@ pub struct CasSuccess { /// this *before* running receive-pack against the hydrated workspace, and /// pass the same value into [`cas_publish`]. If the pointer advances /// between load and CAS (a concurrent push wins), the CAS fails with -/// `LostRace`/`Conflict` and the loser re-pushes — that is the only safe +/// `Conflict` and the loser re-pushes — that is the only safe /// retry path (the loser's receive-pack output is derived against the /// superseded parent, so reusing it would violate /// `Inv_RefDerivedFromParent`). @@ -213,10 +215,10 @@ pub struct CasSuccess { /// pointer happens to be live at CAS time. #[derive(Debug, Clone)] pub struct ParentState { - /// ETag predicating the next CAS write. `None` only when the pointer - /// does not yet exist (first push to an empty repo) — then the CAS - /// uses `If-None-Match: *`. - pub if_match: Option, + /// Revision predicating the next CAS write. `None` only when the + /// pointer does not yet exist (first push to an empty repo) — then the + /// CAS is create-only. + pub if_match: Option, /// The parent manifest's content-addressed *digest* (64-hex), not the /// full `manifests/` key. This lands in `Manifest.parent` and /// is what `Inv_RefDerivedFromParent` reasons over (parent = @@ -246,14 +248,14 @@ impl ParentState { /// Build a `ParentState` from already-loaded pointer state. /// /// The hydrate layer reads the pointer + verified manifest as part of - /// materializing the workspace, then hands the same `(etag, digest, + /// materializing the workspace, then hands the same `(revision, digest, /// manifest)` tuple back here. Centralizing the constructor in /// `cas_publish` means there's one place where `ParentState` /// invariants live; centralizing the I/O in `hydrate` means we read /// the pointer once per push, not twice. - pub fn from_loaded(etag: ETag, digest: String, parent: Manifest) -> Self { + pub fn from_loaded(revision: Revision, digest: String, parent: Manifest) -> Self { Self { - if_match: Some(etag), + if_match: Some(revision), parent_digest: Some(digest), parent, } @@ -385,10 +387,12 @@ fn digest_from_pack_key(key: &str) -> Result { .filter(|digest| digest.len() == 64 && digest.chars().all(|c| c.is_ascii_hexdigit())) .map(str::to_string) .ok_or_else(|| { - CasError::Backend(StoreError::Backend(s3::error::S3Error::HttpFailWithBody( - 500, - format!("put_pack returned non-standard key: {key}"), - ))) + CasError::Backend(StoreError::Backend( + buzz_object_store::ObjectStoreError::Provider { + operation: "put_pack", + message: format!("put_pack returned non-standard key: {key}"), + }, + )) }) } @@ -951,10 +955,12 @@ fn digest_from_manifest_key(key: &str) -> Result { key.strip_prefix("manifests/") .map(str::to_string) .ok_or_else(|| { - CasError::Backend(StoreError::Backend(s3::error::S3Error::HttpFailWithBody( - 500, - format!("put_manifest returned non-standard key: {key}"), - ))) + CasError::Backend(StoreError::Backend( + buzz_object_store::ObjectStoreError::Provider { + operation: "put_manifest", + message: format!("put_manifest returned non-standard key: {key}"), + }, + )) }) } @@ -1210,8 +1216,8 @@ async fn cas_publish_inner( // Step 7: CAS the pointer. let precond = match &parent_state.if_match { - Some(e) => Precond::IfMatch(e.clone()), - None => Precond::IfNoneMatchStar, + Some(revision) => WriteCondition::Matches(revision.clone()), + None => WriteCondition::Absent, }; let cas_outcome = match store .put_pointer(&pkey, manifest_digest.as_bytes(), precond) @@ -1232,7 +1238,7 @@ async fn cas_publish_inner( } }; match cas_outcome { - CasOutcome::Won(_new_etag) => { + ConditionalWrite::Committed(_new_revision) => { if let Some(observation) = &compaction_observation { record_compaction( "success", @@ -1247,7 +1253,7 @@ async fn cas_publish_inner( manifest_key, }) } - CasOutcome::LostRace => { + ConditionalWrite::Conflict => { if let Some(observation) = &compaction_observation { record_compaction( "cas_conflict", @@ -1266,11 +1272,11 @@ async fn cas_publish_inner( let expected = parent_state .if_match .as_ref() - .map(|e| e.0.as_str()) - .unwrap_or(""); + .map(|revision| format!("{revision:?}")) + .unwrap_or_else(|| "".to_string()); warn!( pointer = %pkey, - expected_etag = %expected, + expected_revision = %expected, attempted_manifest = %manifest_key, "CAS lost race; resolving winner for reconcile" ); @@ -1284,7 +1290,7 @@ async fn cas_publish_inner( } } -/// Re-read the pointer after a `LostRace` and fetch the winner's manifest. +/// Re-read the pointer after a CAS conflict and fetch the winner's manifest. /// /// Fail-closed at every step: if the pointer is now absent (a deletion /// raced in — currently impossible under the protocol's no-delete rule, @@ -1592,13 +1598,13 @@ mod tests { } fn live_store() -> GitStore { - GitStore::new( + crate::test_git_store( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", "buzz-media", "us-east-1", - buzz_media::config::S3AddressingStyle::Path, + "path", ) .expect("connect local MinIO") } @@ -1712,12 +1718,12 @@ mod tests { let repo = "history"; let pkey = pointer_key(ctx.community(), &owner, repo); match store - .put_pointer(&pkey, parent_digest.as_bytes(), Precond::IfNoneMatchStar) + .put_pointer(&pkey, parent_digest.as_bytes(), WriteCondition::Absent) .await .expect("put pointer") { - CasOutcome::Won(_) => {} - CasOutcome::LostRace => panic!("unique pointer must win"), + ConditionalWrite::Committed(_) => {} + ConditionalWrite::Conflict => panic!("unique pointer must win"), } let cache_parent = scratch.path().join("cache"); let cache = diff --git a/crates/buzz-relay/src/api/git/hydrate.rs b/crates/buzz-relay/src/api/git/hydrate.rs index 3ce809d18f7..74a977ea60a 100644 --- a/crates/buzz-relay/src/api/git/hydrate.rs +++ b/crates/buzz-relay/src/api/git/hydrate.rs @@ -36,7 +36,9 @@ use tokio::process::Command; use super::cas_publish::ParentState; use super::manifest::{is_hex_oid, is_safe_refname, pointer_key, Manifest, ManifestError}; use super::pack_cache::GitPackCache; -use super::store::{ETag, GitStore, StoreError}; +use buzz_object_store::Revision; + +use super::store::{GitStore, StoreError}; use buzz_core::TenantContext; /// Manifests should stay small after ref/pack cardinality validation. Bound @@ -239,7 +241,7 @@ pub async fn hydrate_for_write( } } -/// Resolve the pointer to its `(ETag, digest, verified Manifest)` triple. +/// Resolve the pointer to its `(Revision, digest, verified Manifest)` triple. /// /// `Ok(None)` if the pointer is absent (caller decides 404 vs first-push /// per call site). `Err(_)` on any below-pointer failure. @@ -248,7 +250,7 @@ async fn load_pointer( ctx: &TenantContext, owner: &str, repo: &str, -) -> Result, HydrateError> { +) -> Result, HydrateError> { let pkey = pointer_key(ctx.community(), owner, repo); let (etag, pointer_bytes) = match store.get_pointer(&pkey).await? { Some(p) => p, @@ -543,15 +545,9 @@ mod tests { #[tokio::test] async fn materialized_repo_is_created_under_configured_scratch_dir() { let scratch = TempDir::new().unwrap(); - let store = GitStore::new( - "http://localhost:9000", - "x", - "x", - "x", - "us-east-1", - buzz_media::config::S3AddressingStyle::Path, - ) - .expect("construct store"); + let store = + crate::test_git_store("http://localhost:9000", "x", "x", "x", "us-east-1", "path") + .expect("construct store"); let manifest = Manifest { version: 1, head: "refs/heads/main".into(), @@ -588,13 +584,13 @@ mod tests { } fn store() -> GitStore { - GitStore::new( + crate::test_git_store( "http://localhost:9000", "buzz_dev", "buzz_dev_secret", "buzz-git", "us-east-1", - buzz_media::config::S3AddressingStyle::Path, + "path", ) .expect("connect local MinIO") } @@ -682,13 +678,15 @@ mod tests { .put_pointer( &pkey, manifest_digest.as_bytes(), - super::super::store::Precond::IfNoneMatchStar, + buzz_object_store::WriteCondition::Absent, ) .await .expect("put_pointer") { - super::super::store::CasOutcome::Won(_) => {} - super::super::store::CasOutcome::LostRace => panic!("first INM* must win"), + buzz_object_store::ConditionalWrite::Committed(_) => {} + buzz_object_store::ConditionalWrite::Conflict => { + panic!("first create-only write must commit") + } } // Hydrate. @@ -829,13 +827,15 @@ mod tests { .put_pointer( &pkey, manifest_digest.as_bytes(), - super::super::store::Precond::IfNoneMatchStar, + buzz_object_store::WriteCondition::Absent, ) .await .expect("put_pointer") { - super::super::store::CasOutcome::Won(_) => {} - super::super::store::CasOutcome::LostRace => panic!("first INM* must win"), + buzz_object_store::ConditionalWrite::Committed(_) => {} + buzz_object_store::ConditionalWrite::Conflict => { + panic!("first create-only write must commit") + } } let scratch = TempDir::new().unwrap(); diff --git a/crates/buzz-relay/src/api/git/policy.rs b/crates/buzz-relay/src/api/git/policy.rs index 32d63f46008..f09219faf0d 100644 --- a/crates/buzz-relay/src/api/git/policy.rs +++ b/crates/buzz-relay/src/api/git/policy.rs @@ -850,7 +850,7 @@ printf '%s' "$HMAC_INPUT" | openssl dgst -sha256 -hmac "{secret}" -hex 2>/dev/nu db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/git/store.rs b/crates/buzz-relay/src/api/git/store.rs index bdfca8dcf2d..b69003655ee 100644 --- a/crates/buzz-relay/src/api/git/store.rs +++ b/crates/buzz-relay/src/api/git/store.rs @@ -4,65 +4,43 @@ //! and the CAS pointer swap (axiom A3) described in //! `docs/git-on-object-storage.md`. //! -//! ## The 412 sharp edge +//! This is a domain facade over [`buzz_object_store::ObjectStore`]: it owns +//! the git key layout, digest verification, and the conformance probe, and +//! knows nothing about which provider is underneath. Compare-and-swap tokens +//! are [`Revision`]s, not ETags — the S3 provider is the only place an ETag +//! exists. //! -//! `rust-s3 = "0.37"` is shared across the workspace with `buzz-media`. The -//! `fail-on-err` Cargo feature is unified ON across the build graph, which -//! means non-2xx responses arrive here as `S3Error::HttpFailWithBody(code, -//! body)` *before* the caller sees `ResponseData`. The pointer-CAS path treats -//! the precondition-failure status (412) as a *semantic* result (`LostRace`), -//! not an error — see `classify_cas`. Empirically verified against MinIO in -//! `probe::probe_412_surfacing`. +//! ## Classified vs. unknown outcomes +//! +//! The pointer-CAS path treats a failed precondition as a *semantic* result +//! ([`ConditionalWrite::Conflict`]), not an error. A request that never +//! produced a classified provider response is different again: its outcome is +//! *unknown*, so the probe drops it from the observer set rather than counting +//! it as a lost race. That distinction lives in +//! [`buzz_object_store::ObjectStoreError::is_ambiguous`]; see the S3 provider +//! module for how each provider failure is classified. //! //! ## Content addressing (A1) //! -//! Pack and manifest keys are the SHA-256 of their bytes. Writes use -//! `If-None-Match: *` so the same key is never overwritten. Readers verify -//! object bytes against the expected digest on `get_verified`; any mismatch is +//! Pack and manifest keys are the SHA-256 of their bytes. Writes are +//! create-only so the same key is never overwritten. Readers verify object +//! bytes against the expected digest on `get_verified`; any mismatch is //! *detectable*, not silent — that is what A1's "create-only + content-address" //! discipline buys us, independent of bucket immutability features. #![allow(dead_code)] // wired in by the push path in a follow-up commit use std::sync::Arc; +use std::time::{Duration, Instant}; +use buzz_object_store::{ + ConditionalWrite, ImmutableWrite, ObjectStore, ObjectStoreError, ProviderKind, Revision, + WriteCondition, +}; use bytes::Bytes; -use s3::creds::Credentials; -use s3::error::S3Error; -use s3::{Bucket, Region}; use sha2::{Digest, Sha256}; -/// Opaque object-store ETag (used for `If-Match` on pointer CAS). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ETag(pub String); - -/// Precondition for `put_pointer`. -#[derive(Debug, Clone)] -pub enum Precond { - /// Create-only: succeed iff the pointer does not yet exist. - IfNoneMatchStar, - /// CAS: succeed iff the current ETag matches. - IfMatch(ETag), -} - -/// Result of a CAS pointer write. -/// -/// `LostRace` is *not* an error — it is the standard outcome of a losing CAS -/// and must be classified here so callers can decide retry vs. non-ff. On -/// `Won`, the returned `ETag` is the PUT response's ETag and can be fed -/// directly into the next `IfMatch` round (verified empirically against MinIO -/// in `probe::probe_full_roundtrip`). A backend that succeeds on the CAS PUT -/// but omits the response ETag is treated as non-conforming and fails the -/// operation with `StoreError::Backend` — see `classify_cas`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CasOutcome { - /// CAS succeeded; the new pointer ETag (suitable for the next `IfMatch`). - Won(ETag), - /// CAS lost the race (server returned 412). - LostRace, -} - -/// Errors that are *actually* errors — `LostRace` is not one. +/// Errors that are *actually* errors — a lost CAS race is not one. #[derive(Debug, thiserror::Error)] pub enum StoreError { /// The requested key does not exist. @@ -89,8 +67,8 @@ pub enum StoreError { actual: String, }, /// Any other backend / transport error. - #[error("s3 backend error: {0}")] - Backend(#[from] S3Error), + #[error("object store error: {0}")] + Backend(ObjectStoreError), /// Invalid storage configuration detected at client construction. #[error("git store config error: {0}")] Config(String), @@ -99,24 +77,116 @@ pub enum StoreError { Probe(ProbeFailure), } +impl From for StoreError { + /// Lift the provider errors that git treats as domain outcomes into their + /// own variants, so call sites keep matching on `NotFound` / + /// `ObjectTooLarge` / `DigestMismatch` directly. + fn from(error: ObjectStoreError) -> Self { + match error { + ObjectStoreError::NotFound { key } => Self::NotFound(key), + ObjectStoreError::ObjectTooLarge { key, size, max } => { + Self::ObjectTooLarge { key, size, max } + } + ObjectStoreError::DigestMismatch { + key, + expected, + actual, + } => Self::DigestMismatch { + key, + expected, + actual, + }, + other => Self::Backend(other), + } + } +} + +/// Build a backend error for a git-layer invariant the provider cannot express. +fn backend(operation: &'static str, message: String) -> StoreError { + StoreError::Backend(ObjectStoreError::Provider { operation, message }) +} + /// Configuration for `GitStore::run_conformance_probe`. /// -/// Defaults: 32-way concurrency, 3 rounds. The probe is a deployment gate — -/// run at startup, fail-closed. See `docs/git-on-object-storage.md` §Conformance. -#[derive(Debug, Clone)] +/// The probe is a deployment gate — run at startup, fail-closed. See +/// `docs/git-on-object-storage.md` §Conformance. +/// +/// Defaults are per-provider ([`ProbeConfig::for_provider`]) because the two +/// profiles are proving the same axiom against backends with very different +/// admission costs: an S3-compatible store answers a wide race cheaply, while +/// Cloud Storage publishes a one-write-per-second ceiling per object name, so a +/// wide, tightly-spaced race there measures the rate limiter rather than the +/// conditional-write semantics. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ProbeConfig { /// How many tasks race per round. Must be ≥ 2. pub race_width: usize, /// How many rounds to run each race phase. pub race_rounds: usize, + /// How many times a round that proved nothing may be re-run before the + /// probe gives up. + /// + /// A round proves nothing when no racer's outcome distinguishes a + /// conforming backend from a broken one — every racer throttled, or too few + /// racers were classified to have witnessed a race at all. Retrying is not + /// leniency: an unproven round is neither pass nor fail, and exhausting the + /// budget without ever proving a round fails the probe. + pub unproven_round_retries: usize, + /// Minimum wall-clock spacing between mutations of the same key. + /// + /// Zero for backends with no documented per-object write ceiling. Cloud + /// Storage documents one write per second per object name, so its profile + /// spaces same-key rounds beyond that interval — the probe proves the + /// store's conditional-write semantics, and deliberately violating the + /// published rate limit would only prove that the rate limiter works. + pub same_key_spacing: Duration, + /// Key namespace for the objects the probe writes for itself. + /// + /// The default reproduces the keys the probe has always used. Overriding it + /// gives a run its own disposable namespace, which is what lets a test + /// prove the cleanup pass emptied it. (The S3 profile's content-addressed + /// phase writes a pack object under the store's own `packs/` namespace by + /// construction — it exercises the production write path — and is not + /// affected by this setting.) + pub key_prefix: String, +} + +impl ProbeConfig { + /// Defaults for `provider`. + pub fn for_provider(provider: ProviderKind) -> Self { + match provider { + ProviderKind::S3 => Self { + race_width: 32, + race_rounds: 3, + unproven_round_retries: 3, + same_key_spacing: Duration::ZERO, + key_prefix: DEFAULT_PROBE_KEY_PREFIX.to_string(), + }, + ProviderKind::Gcs => Self { + race_width: 3, + race_rounds: 2, + unproven_round_retries: 3, + // Just past Cloud Storage's documented one-write-per-second + // per-object ceiling. + same_key_spacing: Duration::from_millis(1_100), + key_prefix: DEFAULT_PROBE_KEY_PREFIX.to_string(), + }, + } + } + + /// Run under `prefix` instead of the default namespace. + pub fn under_key_prefix(mut self, prefix: impl Into) -> Self { + self.key_prefix = prefix.into(); + self + } } +/// Namespace the probe writes its own objects under. +const DEFAULT_PROBE_KEY_PREFIX: &str = "probe"; + impl Default for ProbeConfig { fn default() -> Self { - Self { - race_width: 32, - race_rounds: 3, - } + Self::for_provider(ProviderKind::S3) } } @@ -124,14 +194,44 @@ impl Default for ProbeConfig { /// detail lives in `ProbeFailure` (the error variant). #[derive(Debug, Clone)] pub struct ProbeReport { + /// Which provider profile ran. + pub profile: ProviderKind, /// Concurrency used. pub race_width: usize, /// Rounds executed per race phase. pub race_rounds: usize, + /// Racers the backend answered with throttling across all race rounds. + /// + /// A throttled racer is *never* a lost race — the write was refused before + /// the precondition was evaluated, so it is evidence about request rate and + /// about nothing else. Counting them separately is what keeps backpressure + /// from masquerading as conformance. Non-zero here on a passing probe means + /// "admitted, and the backend was pacing us", which is the expected shape on + /// a provider with a per-object write ceiling. + pub throttled_racers: usize, + /// Race rounds that proved nothing and were re-run. + /// + /// See [`ProbeConfig::unproven_round_retries`]. Non-zero on a passing probe + /// means every round eventually proved itself, but the backend needed more + /// attempts than a quiet one would. + pub throttled_rounds_retried: usize, + /// Shortest observed interval between two same-key mutation rounds, when + /// the profile spaces them. + /// + /// Reported so a passing probe can be checked against the spacing it + /// claimed to honour rather than trusted to have slept. + pub min_same_key_gap: Option, + /// Probe objects the cleanup pass could not remove. + /// + /// Cleanup failure does not fail the probe — a store that satisfies every + /// conformance axiom is admitted even if a tidy-up delete flaked — but it + /// is surfaced because silent probe-key accumulation in a shared bucket is + /// exactly the kind of leak that is invisible until it is large. + pub cleanup_failures: usize, /// Total number of *transport-unknown* per-racer outcomes across all /// race rounds (sum of both `if_match_race` and `if_none_match_race` /// phases). A "transport-unknown" is a pre-classification failure — - /// `S3Error::{Reqwest, Http, Io}` — that means the racer never got a + /// [`ObjectStoreError::is_ambiguous`] — that means the racer never got a /// classified response from the backend, so its outcome is neither /// evidence for nor against A3 linearizability. Such racers are /// dropped from the observer set (see the race phases for the @@ -149,7 +249,12 @@ pub struct ProbeReport { #[derive(Debug, thiserror::Error)] #[error("conformance probe failed in phase '{phase}' (round {round}, key {key}): {reason}")] pub struct ProbeFailure { - /// One of `sequential`, `if_match_race`, `if_none_match_race`, `etag_consistency`. + /// The profile phase that failed. + /// + /// S3 profile: `sequential`, `if_match_race`, `if_none_match_race`, + /// `revision_consistency`. Cloud Storage profile: `immutable`, + /// `pointer_create`, `pointer_read`, `cas_replace`, `stale_cas`, + /// `cas_race`, `generation_roundtrip`. pub phase: &'static str, /// Round index (0-based) when this phase ran multiple rounds. pub round: usize, @@ -165,61 +270,116 @@ impl From for StoreError { } } +/// Marks a body written by the Cloud Storage profile's racing writers. +/// +/// Public to the crate so a test double can recognise a racer's write without +/// hard-coding the probe's body format. +pub(crate) const GCS_RACE_BODY_PREFIX: &str = "probe-gcs-race:"; + +/// Build a Cloud Storage profile failure. +fn gcs_failure(phase: &'static str, round: usize, key: &str, reason: String) -> ProbeFailure { + ProbeFailure { + phase, + round, + key: key.to_string(), + reason, + } +} + +/// Read the object generation out of a revision the store reported committing. +/// +/// A commit with no generation is fatal, not cosmetic: the generation is the +/// only token that can predicate the next write, so a caller handed nothing +/// would have to either stop writing or drop its precondition — and dropping it +/// turns the pointer swap into a blind overwrite. Cloud Storage has no live +/// generation `0`, so zero *is* the absent case. +fn gcs_generation( + phase: &'static str, + round: usize, + key: &str, + revision: &Revision, +) -> Result { + match revision.expect_gcs_generation() { + Ok(generation) if generation > 0 => Ok(generation), + Ok(_) => Err(gcs_failure( + phase, + round, + key, + "the store reported a successful write with no object generation".to_string(), + )), + Err(error) => Err(gcs_failure( + phase, + round, + key, + format!("expected an object generation: {error}"), + )), + } +} + +/// What one Cloud Storage race round established. +enum GcsRaceOutcome { + /// Exactly one racer committed, witnessed by at least one other classified + /// racer. + Committed { + /// Generation the winner committed. + generation: i64, + /// Bytes the winner wrote. + body: Vec, + }, + /// The round is neither pass nor fail — nothing in it distinguishes a + /// conforming store from a broken one — so it must be re-run. + Unproven(String), +} + +/// Counters and pacing state carried across the Cloud Storage phases. +#[derive(Default)] +struct GcsProbeState { + transport_drops: usize, + throttled_racers: usize, + throttled_rounds_retried: usize, + min_same_key_gap: Option, + last_same_key_write: Option, +} + +impl GcsProbeState { + /// Wait until `spacing` has elapsed since the previous same-key mutation, + /// and record the interval actually observed. + /// + /// The recorded gap is measured, not assumed: the report carries it so a + /// passing probe can be checked against the spacing it claimed to honour. + async fn pace(&mut self, spacing: Duration) { + let now = match self.last_same_key_write { + None => Instant::now(), + Some(previous) => { + let elapsed = previous.elapsed(); + if elapsed < spacing { + tokio::time::sleep(spacing - elapsed).await; + } + let gap = previous.elapsed(); + self.min_same_key_gap = Some( + self.min_same_key_gap + .map_or(gap, |shortest| shortest.min(gap)), + ); + Instant::now() + } + }; + self.last_same_key_write = Some(now); + } +} + /// Object-store client for git refs. #[derive(Clone)] pub struct GitStore { - bucket: Arc, + store: Arc, } impl GitStore { - /// Build a client against an S3-compatible endpoint (e.g. MinIO). + /// Build a git store over an existing object-store client. /// - /// `addressing_style` is shared with media storage so both paths sign and - /// route requests consistently. Path style supports the bundled MinIO DNS; - /// virtual-hosted style supports standard S3 and providers such as Railway. - /// - /// Credential selection mirrors [`buzz_media::MediaStorage::new`]: - /// - both `access_key` and `secret_key` non-empty → static credentials - /// (MinIO/local/dev, or any static-key deployment); - /// - both empty → the AWS default credential chain via - /// [`Credentials::default`] (env, profile, web-identity/IRSA, container, - /// instance metadata), so the relay can use its pod IAM role; - /// - exactly one empty → a config error, to surface a half-configured - /// static deployment instead of silently falling back to the chain. - pub fn new( - endpoint: &str, - access_key: &str, - secret_key: &str, - bucket_name: &str, - region: &str, - addressing_style: buzz_media::config::S3AddressingStyle, - ) -> Result { - let region = Region::Custom { - region: region.into(), - endpoint: endpoint.into(), - }; - let creds = match (access_key.is_empty(), secret_key.is_empty()) { - (false, false) => Credentials::new(Some(access_key), Some(secret_key), None, None, None), - (true, true) => { - // No static keys configured: resolve from the AWS credential - // chain (IRSA web-identity, env, profile, instance metadata). - Credentials::default() - } - _ => { - return Err(StoreError::Config( - "s3 access_key and secret_key must be configured together, or both empty to use the AWS credential chain".to_string(), - )); - } - } - .map_err(|e| StoreError::Backend(S3Error::Credentials(e)))?; - let bucket = Bucket::new(bucket_name, region, creds).map_err(StoreError::Backend)?; - let bucket = match addressing_style { - buzz_media::config::S3AddressingStyle::Path => bucket.with_path_style(), - buzz_media::config::S3AddressingStyle::Virtual => bucket, - }; - Ok(Self { - bucket: Arc::from(bucket), - }) + /// The relay constructs exactly one provider per process and hands the + /// same client to media storage and to this facade. + pub fn new(store: Arc) -> Self { + Self { store } } /// Compute the hex SHA-256 of `bytes`. The content-addressed key. @@ -237,10 +397,10 @@ impl GitStore { /// each manifest pack key. pub fn idx_key_for_pack_digest(pack_digest: &str) -> Result { if pack_digest.len() != 64 || !pack_digest.chars().all(|c| c.is_ascii_hexdigit()) { - return Err(StoreError::Backend(S3Error::HttpFailWithBody( - 400, + return Err(backend( + "idx_key_for_pack_digest", format!("invalid pack digest for idx sidecar: {pack_digest:?}"), - ))); + )); } Ok(format!("idx/{pack_digest}")) } @@ -249,8 +409,8 @@ impl GitStore { /// /// **The caller does not choose the key.** It is derived as /// `/` inside this method. This makes the - /// idempotency claim *constructive*: a 412 collision means the key already - /// holds bytes whose digest equals `sha256(these bytes)`, so by A1 + /// idempotency claim *constructive*: a precondition collision means the key + /// already holds bytes whose digest equals `sha256(these bytes)`, so by A1 /// (content-addressing) the stored bytes equal these bytes. Without this /// enforcement, a buggy caller passing the wrong key would silently break /// A1 detectability on read. @@ -263,24 +423,9 @@ impl GitStore { content_type: &str, ) -> Result { let key = Self::content_key(prefix, bytes); - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers(&key, bytes, content_type, Some(headers)) - .await - { - Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(key), - // 412 on a content-addressed key means the key already holds the - // same bytes (by construction — the key is the digest). A1 is - // preserved without a defensive GET. - Err(S3Error::HttpFailWithBody(412, _)) => Ok(key), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), - } + // Both outcomes are success: by construction the key holds these bytes. + self.store.put_immutable(&key, bytes, content_type).await?; + Ok(key) } /// Write a pack object. Returns the content-addressed key (`packs/`). @@ -293,31 +438,15 @@ impl GitStore { /// /// Unlike packs/manifests, the key is not the SHA-256 of `idx_bytes`; it is /// `idx/` so hydrates can derive it without changing manifest - /// bytes. A 412 is idempotent success for the cache layer: the first writer - /// already produced the sidecar for this pack, and hydrate validates before - /// trusting it. + /// bytes. A precondition collision is idempotent success for the cache + /// layer: the first writer already produced the sidecar for this pack, and + /// hydrate validates before trusting it. pub async fn put_idx(&self, pack_digest: &str, idx_bytes: &[u8]) -> Result { let key = Self::idx_key_for_pack_digest(pack_digest)?; - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers( - &key, - idx_bytes, - "application/x-git-index", - Some(headers), - ) - .await - { - Ok(resp) if (200..300).contains(&resp.status_code()) => Ok(key), - Err(S3Error::HttpFailWithBody(412, _)) => Ok(key), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), - } + self.store + .put_immutable(&key, idx_bytes, "application/x-git-index") + .await?; + Ok(key) } /// Read an idx sidecar for `packs/`. @@ -350,11 +479,7 @@ impl GitStore { /// detectability. This raw `get` exists for the pointer (whose key is not a /// digest). pub async fn get(&self, key: &str) -> Result { - match self.bucket.get_object(key).await { - Ok(resp) => Ok(Bytes::from(resp.to_vec())), - Err(S3Error::HttpFailWithBody(404, _)) => Err(StoreError::NotFound(key.into())), - Err(e) => Err(StoreError::Backend(e)), - } + Ok(self.store.get(key).await?) } /// GET an object and verify its bytes hash to `expected_digest` (hex SHA-256). @@ -368,17 +493,7 @@ impl GitStore { expected_digest: &str, ) -> Result { let bytes = self.get(key).await?; - let mut hasher = Sha256::new(); - hasher.update(&bytes); - let actual = hex::encode(hasher.finalize()); - if actual != expected_digest { - return Err(StoreError::DigestMismatch { - key: key.into(), - expected: expected_digest.into(), - actual, - }); - } - Ok(bytes) + Self::verify_digest(key, expected_digest, bytes) } /// GET an immutable object after rejecting objects larger than `max_bytes`. @@ -386,7 +501,7 @@ impl GitStore { /// Pack and manifest objects are content addressed and create-only, so the /// HEAD result cannot race with a different body at the same key. The /// second length check protects against a backend that reports a bad - /// Content-Length header. + /// content length. pub async fn get_verified_limited( &self, key: &str, @@ -394,162 +509,61 @@ impl GitStore { max_bytes: u64, ) -> Result { let bytes = self.get_limited(key, max_bytes).await?; - let mut hasher = Sha256::new(); - hasher.update(&bytes); - let actual = hex::encode(hasher.finalize()); - if actual != expected_digest { - return Err(StoreError::DigestMismatch { - key: key.into(), - expected: expected_digest.into(), - actual, - }); - } - Ok(bytes) + Self::verify_digest(key, expected_digest, bytes) } /// GET an object after rejecting bodies larger than `max_bytes`. pub async fn get_limited(&self, key: &str, max_bytes: u64) -> Result { - let (head, status) = self.bucket.head_object(key).await.map_err(|e| match e { - S3Error::HttpFailWithBody(404, _) => StoreError::NotFound(key.into()), - other => StoreError::Backend(other), - })?; - if status == 404 { - return Err(StoreError::NotFound(key.into())); - } - if !(200..300).contains(&status) { - return Err(StoreError::Backend(S3Error::HttpFailWithBody( - status, - "unexpected status".into(), - ))); - } - if let Some(content_length) = head.content_length { - let size = u64::try_from(content_length).unwrap_or(u64::MAX); - if size > max_bytes { - return Err(StoreError::ObjectTooLarge { - key: key.into(), - size, - max: max_bytes, - }); - } - } - - let bytes = self.get(key).await?; - let size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - if size > max_bytes { - return Err(StoreError::ObjectTooLarge { - key: key.into(), - size, - max: max_bytes, - }); - } - Ok(bytes) + Ok(self.store.get_limited(key, max_bytes).await?) } - /// GET the pointer object, returning its ETag and bytes *from the same + /// GET the pointer object, returning its revision and bytes *from the same /// response* — atomic snapshot. /// /// Returns `Ok(None)` if the pointer does not exist (first-push case). /// /// **Why one GET, not HEAD-then-GET.** A separate HEAD followed by GET - /// can straddle a concurrent writer: the HEAD's ETag and the GET's body - /// would describe different pointer versions, and a caller that later - /// did `IfMatch(etag_from_head)` would be predicating on a version it - /// never actually read. Reading both fields from the GET response keeps - /// the snapshot consistent (A2: a single GET observes a single committed - /// object). Verified empirically in `probe::probe_get_exposes_etag`. - pub async fn get_pointer(&self, key: &str) -> Result, StoreError> { - match self.bucket.get_object(key).await { - Ok(resp) => { - let headers = resp.headers(); - let etag = headers - .get("etag") - .or_else(|| headers.get("ETag")) - .cloned() - .ok_or_else(|| { - StoreError::Backend(S3Error::HttpFailWithBody( - 500, - "GET pointer: response missing ETag".into(), - )) - })?; - Ok(Some((ETag(etag), Bytes::from(resp.to_vec())))) - } - Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), - Err(e) => Err(StoreError::Backend(e)), - } + /// can straddle a concurrent writer: the HEAD's revision and the GET's + /// body would describe different pointer versions, and a caller that + /// later predicated a CAS on the HEAD revision would be predicating on a + /// version it never actually read. Reading both fields from the GET + /// response keeps the snapshot consistent (A2: a single GET observes a + /// single committed object). + pub async fn get_pointer(&self, key: &str) -> Result, StoreError> { + Ok(self.store.get_with_revision(key).await?) } /// Write the pointer under a precondition (§Push step 7 — the CAS). /// - /// Returns `CasOutcome::LostRace` on 412 (the standard losing outcome). - /// On `CasOutcome::Won`, the returned `ETag` is read from the response - /// headers — callers use it as the `If-Match` value for the next CAS. + /// Returns [`ConditionalWrite::Conflict`] when the precondition did not + /// hold (the standard losing outcome). On + /// [`ConditionalWrite::Committed`], the returned [`Revision`] is read from + /// the write response — callers use it to predicate the next CAS. pub async fn put_pointer( &self, key: &str, body: &[u8], - precond: Precond, - ) -> Result { - let mut headers = axum::http::HeaderMap::new(); - match &precond { - Precond::IfNoneMatchStar => { - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - } - Precond::IfMatch(ETag(tag)) => { - headers.insert( - axum::http::header::IF_MATCH, - tag.parse().map_err(|_| { - StoreError::Backend(S3Error::HttpFailWithBody( - 400, - format!("invalid etag {tag}"), - )) - })?, - ); - } - } - let result = self - .bucket - .put_object_with_content_type_and_headers(key, body, "application/json", Some(headers)) - .await; - Self::classify_cas(result) + condition: WriteCondition, + ) -> Result { + Ok(self + .store + .put_conditional(key, body, "application/json", condition) + .await?) } - /// Map a rust-s3 PUT outcome to a `CasOutcome`. - /// - /// 412 → `LostRace`. 2xx → `Won(etag)` (etag read from response headers, - /// empty if missing — callers must tolerate empty etag and re-HEAD if they - /// need it strictly). Everything else bubbles as `StoreError::Backend`. - fn classify_cas( - result: Result, - ) -> Result { - match result { - Ok(resp) if (200..300).contains(&resp.status_code()) => { - let headers = resp.headers(); - let etag = headers - .get("etag") - .or_else(|| headers.get("ETag")) - .cloned() - .ok_or_else(|| { - // Fail closed: a CAS that we can't chain (because the - // backend didn't return an ETag) is not a `Won` — it's - // a non-conforming backend. The conformance probe will - // catch this; in production we'd rather refuse than - // hand the caller `ETag("")` and force-fail the next CAS. - StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "CAS succeeded but response missing ETag header \ - (backend does not satisfy ETag-token consistency)" - .into(), - )) - })?; - Ok(CasOutcome::Won(ETag(etag))) - } - Err(S3Error::HttpFailWithBody(412, _)) => Ok(CasOutcome::LostRace), - Ok(resp) => Err(StoreError::Backend(S3Error::HttpFailWithBody( - resp.status_code(), - "unexpected status".into(), - ))), - Err(e) => Err(StoreError::Backend(e)), + /// Hash `bytes` and reject anything that does not match `expected_digest`. + fn verify_digest(key: &str, expected_digest: &str, bytes: Bytes) -> Result { + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let actual = hex::encode(hasher.finalize()); + if actual != expected_digest { + return Err(StoreError::DigestMismatch { + key: key.into(), + expected: expected_digest.into(), + actual, + }); } + Ok(bytes) } /// Conformance probe — deployment gate per `docs/git-on-object-storage.md` @@ -557,24 +571,12 @@ impl GitStore { /// `StoreError::Probe(ProbeFailure)` and the caller (relay startup) MUST /// refuse to come up. /// - /// Four phases: - /// - /// 1. **`sequential`** — write a content-addressed object, read it back, - /// verify bytes. Tests A1 (content-addressed write) + A2 - /// (read-after-write). - /// 2. **`if_match_race`** — `race_width` parallel `put_pointer` calls - /// predicated on the same ETag. Exactly one must `Won`; the rest must - /// `LostRace`. Tests A3. - /// 3. **`if_none_match_race`** — `race_width` parallel create-only writes - /// against the same digest-shaped key (the same `put_immutable` path - /// `put_pack`/`put_manifest` use). Tests A1 + A3 on the create-only - /// primitive. Counts raw HTTP outcomes (exactly one 2xx, rest 412) and - /// asserts final stored bytes equal the racers' bytes. - /// 4. **`etag_consistency`** — round-trip an ETag from `get_pointer` into - /// `put_pointer(IfMatch(...))` and assert `Won`. Tests that the token - /// is opaque and stable between read and CAS. + /// The profile is chosen by the configured provider, not by the caller: the + /// axioms are the same for every backend, but the evidence that admits one + /// is provider-shaped. An S3-compatible store is admitted by a wide race on + /// ETag preconditions; Cloud Storage is admitted by a paced race on object + /// generations, where a refused (throttled) write is not a lost race. pub async fn run_conformance_probe(&self, cfg: ProbeConfig) -> Result { - use std::sync::Arc; if cfg.race_width < 2 || cfg.race_rounds == 0 { return Err(ProbeFailure { phase: "config", @@ -587,8 +589,35 @@ impl GitStore { } .into()); } + match self.store.provider() { + ProviderKind::S3 => self.run_s3_conformance_probe(cfg).await, + ProviderKind::Gcs => self.run_gcs_conformance_probe(cfg).await, + } + } + + /// The S3 profile: revision-token compare-and-swap under a wide race. + /// + /// Four phases: + /// + /// 1. **`sequential`** — write a content-addressed object, read it back, + /// verify bytes. Tests A1 (content-addressed write) + A2 + /// (read-after-write). + /// 2. **`if_match_race`** — `race_width` parallel `put_pointer` calls + /// predicated on the same revision. Exactly one must commit; the rest + /// must conflict. Tests A3. + /// 3. **`if_none_match_race`** — `race_width` parallel create-only writes + /// against the same digest-shaped key (the same `put_immutable` path + /// `put_pack`/`put_manifest` use). Tests A1 + A3 on the create-only + /// primitive. Counts raw outcomes (exactly one create, rest already + /// present) and asserts final stored bytes equal the racers' bytes. + /// 4. **`revision_consistency`** — round-trip a revision from + /// `get_pointer` into `put_pointer(Matches(...))` and assert it + /// commits. Tests that the token is opaque and stable between read and + /// CAS. + async fn run_s3_conformance_probe(&self, cfg: ProbeConfig) -> Result { + use std::sync::Arc; let nonce = uuid::Uuid::new_v4(); - let pointer_key = format!("probe/pointer-{nonce}"); + let pointer_key = format!("{}/pointer-{nonce}", cfg.key_prefix); // Accumulator for *transport-unknown* per-racer outcomes across both // race phases. See `ProbeReport::transport_drops` for the rationale. let mut transport_drops = 0usize; @@ -618,15 +647,15 @@ impl GitStore { } // -- Phase 2: if_match_race ----------------------------------------------- - // Seed the pointer with a known value, then race N IfMatch updates. + // Seed the pointer with a known value, then race N CAS updates. let seed = b"probe-pointer-seed".to_vec(); - let _ = self.bucket.delete_object(&pointer_key).await; // ignore 404 + let _ = self.store.delete(&pointer_key).await; // ignore absence let seed_outcome = self - .put_pointer(&pointer_key, &seed, Precond::IfNoneMatchStar) + .put_pointer(&pointer_key, &seed, WriteCondition::Absent) .await?; - let mut etag = match seed_outcome { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => { + let mut revision = match seed_outcome { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => { return Err(ProbeFailure { phase: "if_match_race", round: 0, @@ -642,39 +671,36 @@ impl GitStore { for i in 0..cfg.race_width { let me = Arc::clone(&arc_self); let pkey = pointer_key.clone(); - let et = etag.clone(); + let condition = WriteCondition::Matches(revision.clone()); let body = format!("round={round},racer={i},nonce={nonce}").into_bytes(); - tasks.push(async move { me.put_pointer(&pkey, &body, Precond::IfMatch(et)).await }); + tasks.push(async move { me.put_pointer(&pkey, &body, condition).await }); } let outcomes = futures_util::future::join_all(tasks).await; - // Drop-and-floor classification. A `Reqwest`/`Http`/`Io` error + // Drop-and-floor classification. An ambiguous provider outcome // means the racer never got a classified response from the // backend (couldn't open a socket, send flaked, etc.); its // outcome is *unknown*, not negative. A3 is a claim about // **observers**: dropping unknowns from the observer set // sharpens the assertion ("exactly one winner among observers") // and avoids smuggling a network-stack test into the - // conformance probe. Parse/decode errors (`Utf8`, - // `ReqwestHeaderToStr`, `SerdeXml`, ...) and `HttpFailWithBody` - // stay in the catch-all — those mean the backend *did* answer - // but not in the contract shape, which is a real conformance - // signal. + // conformance probe. Every other failure — a malformed + // response, an unexpected status — means the backend *did* + // answer but not in the contract shape, which is a real + // conformance signal and fails closed. let mut classified = 0usize; let mut winners = 0usize; - let mut new_etag: Option = None; + let mut new_revision: Option = None; for (i, outcome) in outcomes.into_iter().enumerate() { match outcome { - Ok(CasOutcome::Won(e)) => { + Ok(ConditionalWrite::Committed(committed)) => { classified += 1; winners += 1; - new_etag = Some(e); + new_revision = Some(committed); } - Ok(CasOutcome::LostRace) => { + Ok(ConditionalWrite::Conflict) => { classified += 1; } - Err(StoreError::Backend( - S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_), - )) => { + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { transport_drops += 1; tracing::warn!( phase = "if_match_race", @@ -720,17 +746,17 @@ impl GitStore { } .into()); } - etag = new_etag.expect("winner exists"); + revision = new_revision.expect("winner exists"); } // -- Phase 3: if_none_match_race ------------------------------------------ // N parallel create-only writes targeting the same digest-shaped key. - // Bypass `put_immutable`'s 412-swallow to count raw outcomes. + // Bypass `put_immutable`'s collision-swallow to count raw outcomes. for round in 0..cfg.race_rounds { let body = format!("probe-inm-race-{nonce}-{round}").into_bytes(); - let key = Self::content_key("probe/inm-race", &body); + let key = Self::content_key(&format!("{}/inm-race", cfg.key_prefix), &body); // Clean slate. - let _ = self.bucket.delete_object(&key).await; + let _ = self.store.delete(&key).await; let arc_self: Arc<&Self> = Arc::new(self); let mut tasks = Vec::with_capacity(cfg.race_width); for _ in 0..cfg.race_width { @@ -741,35 +767,23 @@ impl GitStore { } let results = futures_util::future::join_all(tasks).await; // Drop-and-floor: same classification rule as Phase 2. Drop - // `Reqwest`/`Http`/`Io` (pre-classification — socket/send - // failure); count 2xx + 412 as the classified observers. Any - // other status or any non-transport `StoreError` is a real - // conformance signal and fails closed. + // ambiguous pre-classification failures; count created + + // already-present as the classified observers. Any other + // `StoreError` is a real conformance signal and fails closed. let mut classified = 0usize; - let mut twos = 0usize; - let mut twelves = 0usize; + let mut created = 0usize; + let mut collisions = 0usize; for (i, r) in results.into_iter().enumerate() { match r { - Ok(200..=299) => { + Ok(ImmutableWrite::Created) => { classified += 1; - twos += 1; + created += 1; } - Ok(412) => { + Ok(ImmutableWrite::AlreadyPresent) => { classified += 1; - twelves += 1; - } - Ok(code) => { - return Err(ProbeFailure { - phase: "if_none_match_race", - round, - key, - reason: format!("racer {i}: unexpected status {code}"), - } - .into()) + collisions += 1; } - Err(StoreError::Backend( - S3Error::Reqwest(_) | S3Error::Http(_) | S3Error::Io(_), - )) => { + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { transport_drops += 1; tracing::warn!( phase = "if_none_match_race", @@ -802,17 +816,17 @@ impl GitStore { } .into()); } - // Create-only contract: exactly 1×2xx + (classified − 1)×412 - // *among observers*. The previous fixed `race_width − 1` would + // Create-only contract: exactly 1 create + (classified − 1) + // collisions *among observers*. A fixed `race_width − 1` would // false-positive on any transport drop; this expression honors // the drop-and-floor invariant. - if twos != 1 || twelves != classified - 1 { + if created != 1 || collisions != classified - 1 { return Err(ProbeFailure { phase: "if_none_match_race", round, key, reason: format!( - "expected 1×2xx + {}×412 among {classified} classified observers, got {twos}×2xx + {twelves}×412", + "expected 1 create + {} collisions among {classified} classified observers, got {created} creates + {collisions} collisions", classified - 1 ), } @@ -840,31 +854,31 @@ impl GitStore { } } - // -- Phase 4: etag_consistency -------------------------------------------- - // GET pointer, take its ETag, CAS-update with that ETag, expect Won. - // Proves the token round-trips opaquely between read and write. + // -- Phase 4: revision_consistency ---------------------------------------- + // GET pointer, take its revision, CAS-update with that revision, expect + // a commit. Proves the token round-trips opaquely between read and write. for round in 0..cfg.race_rounds { - let (et, _bytes) = + let (observed, _bytes) = self.get_pointer(&pointer_key) .await? .ok_or_else(|| ProbeFailure { - phase: "etag_consistency", + phase: "revision_consistency", round, key: pointer_key.clone(), reason: "pointer vanished mid-probe".into(), })?; - let body = format!("probe-etag-{round}-{nonce}").into_bytes(); + let body = format!("probe-revision-{round}-{nonce}").into_bytes(); match self - .put_pointer(&pointer_key, &body, Precond::IfMatch(et)) + .put_pointer(&pointer_key, &body, WriteCondition::Matches(observed)) .await? { - CasOutcome::Won(_) => {} - CasOutcome::LostRace => { + ConditionalWrite::Committed(_) => {} + ConditionalWrite::Conflict => { return Err(ProbeFailure { - phase: "etag_consistency", + phase: "revision_consistency", round, key: pointer_key, - reason: "GET-ETag → IfMatch chain lost race in a quiescent probe".into(), + reason: "GET-revision → CAS chain lost race in a quiescent probe".into(), } .into()) } @@ -873,152 +887,1299 @@ impl GitStore { // Cleanup pointer (immutable probe writes accumulate by design; the // bucket's retention policy handles them, not the probe). - let _ = self.bucket.delete_object(&pointer_key).await; + let _ = self.store.delete(&pointer_key).await; Ok(ProbeReport { + profile: ProviderKind::S3, race_width: cfg.race_width, race_rounds: cfg.race_rounds, transport_drops, + throttled_racers: 0, + throttled_rounds_retried: 0, + min_same_key_gap: None, + cleanup_failures: 0, }) } - /// Helper: hex SHA-256 of bytes. - fn digest_hex(bytes: &[u8]) -> String { - let mut h = Sha256::new(); - h.update(bytes); - hex::encode(h.finalize()) - } - - /// Raw create-only PUT exposed for the probe's race-counting phase, where - /// we need to *see* 412 outcomes rather than swallow them as idempotent. - /// Returns the HTTP status code on success-or-412; bubbles other errors. - async fn put_immutable_raw(&self, key: &str, bytes: &[u8]) -> Result { - let mut headers = axum::http::HeaderMap::new(); - headers.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - match self - .bucket - .put_object_with_content_type_and_headers( - key, - bytes, - "application/octet-stream", - Some(headers), - ) - .await - { - Ok(resp) => Ok(resp.status_code()), - Err(S3Error::HttpFailWithBody(412, _)) => Ok(412), - Err(e) => Err(StoreError::Backend(e)), - } - } -} + /// The Cloud Storage profile: object-generation compare-and-swap, paced to + /// the provider's published per-object write ceiling. + /// + /// Same axioms, different evidence. Cloud Storage answers a stale + /// precondition with 412 (an ordinary conflict) and an over-rate write with + /// 429 (a refusal to evaluate the precondition at all), and it publishes a + /// maximum of one write per second to a single object name. A profile that + /// ignored either fact would be measuring the rate limiter: a burst of 429s + /// would either be miscounted as lost races — turning "the backend paced us" + /// into "the backend admitted two winners' worth of losers" — or would make + /// a round pass with no race in it. So this profile races narrowly, spaces + /// same-key rounds past the ceiling, classifies throttles separately from + /// conflicts, and re-runs a round that proved nothing rather than scoring + /// it. + /// + /// Phases, in order: + /// + /// 1. **`immutable`** — create-only write of a content-addressed object, + /// read back, digest verified. A1 plus read-after-write. + /// 2. **`pointer_create`** — create the pointer under the create-only + /// precondition, which Cloud Storage spells as generation `0`. + /// 3. **`pointer_read`** — read body and generation from one response and + /// check both against what was just committed. A2. + /// 4. **`cas_replace`** — replace under the observed generation; the commit + /// must report a *different* generation. + /// 5. **`stale_cas`** — replay the superseded generation; must conflict. A + /// store that commits here is performing blind overwrites. + /// 6. **`cas_race`** — `race_width` writers on one generation, `race_rounds` + /// times: exactly one commit, every other classified racer a conflict or + /// a throttle, and the stored object equal to the winner's. + /// 7. **`generation_roundtrip`** — the winning generation predicates the + /// next successful compare-and-swap, closing the loop the push path + /// depends on. + /// + /// Probe objects are removed afterwards on both the success and the failure + /// path; a cleanup failure is reported, not fatal. + async fn run_gcs_conformance_probe(&self, cfg: ProbeConfig) -> Result { + let nonce = uuid::Uuid::new_v4(); + let mut written = Vec::new(); + let mut state = GcsProbeState::default(); -#[cfg(test)] -mod tests { - use super::*; + let outcome = self + .gcs_probe_phases(&cfg, nonce, &mut written, &mut state) + .await; + let cleanup_failures = self.remove_probe_objects(&written).await; + outcome?; - #[test] - fn idx_key_uses_pack_digest_namespace() { - let digest = "a".repeat(64); - assert_eq!( - GitStore::idx_key_for_pack_digest(&digest).unwrap(), - format!("idx/{digest}") - ); + Ok(ProbeReport { + profile: ProviderKind::Gcs, + race_width: cfg.race_width, + race_rounds: cfg.race_rounds, + transport_drops: state.transport_drops, + throttled_racers: state.throttled_racers, + throttled_rounds_retried: state.throttled_rounds_retried, + min_same_key_gap: state.min_same_key_gap, + cleanup_failures, + }) } - #[test] - fn idx_key_rejects_non_digest_input() { - for bad in ["packs/abc", "abc", "../escape", &"g".repeat(64)] { - assert!(GitStore::idx_key_for_pack_digest(bad).is_err()); + /// The Cloud Storage phases, factored out so cleanup runs on every path. + async fn gcs_probe_phases( + &self, + cfg: &ProbeConfig, + nonce: uuid::Uuid, + written: &mut Vec, + state: &mut GcsProbeState, + ) -> Result<(), StoreError> { + // -- Phase 1: immutable ------------------------------------------------ + // The nonce makes this key new, so the create-only write must report a + // create rather than a collision. + let body = format!("probe-gcs-immutable-{nonce}").into_bytes(); + let key = Self::content_key(&format!("{}/gcs-immutable", cfg.key_prefix), &body); + written.push(key.clone()); + let outcome = self + .put_immutable_raw(&key, &body) + .await + .map_err(|e| gcs_failure("immutable", 0, &key, format!("create-only write: {e}")))?; + if outcome != ImmutableWrite::Created { + return Err(gcs_failure( + "immutable", + 0, + &key, + "a freshly nonced key reported a collision, so the create-only \ + precondition is not being evaluated" + .to_string(), + ) + .into()); + } + let read = self + .get_verified(&key, &Self::digest_hex(&body)) + .await + .map_err(|e| gcs_failure("immutable", 0, &key, format!("verified read: {e}")))?; + if read[..] != body[..] { + return Err(gcs_failure( + "immutable", + 0, + &key, + "read-after-write returned different bytes".to_string(), + ) + .into()); } - } - - #[test] - fn classify_cas_412_is_lost_race() { - let r = Err(S3Error::HttpFailWithBody(412, "PreconditionFailed".into())); - assert_eq!(GitStore::classify_cas(r).unwrap(), CasOutcome::LostRace); - } - #[test] - fn classify_cas_other_4xx_bubbles() { - let r = Err(S3Error::HttpFailWithBody(403, "AccessDenied".into())); - assert!(matches!( - GitStore::classify_cas(r), - Err(StoreError::Backend(S3Error::HttpFailWithBody(403, _))) - )); - } + // -- Phase 2: pointer_create ------------------------------------------- + let pointer_key = format!("{}/gcs-pointer-{nonce}", cfg.key_prefix); + written.push(pointer_key.clone()); + let seed = format!("probe-gcs-pointer-seed-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + let created = self + .put_pointer(&pointer_key, &seed, WriteCondition::Absent) + .await + .map_err(|e| { + gcs_failure( + "pointer_create", + 0, + &pointer_key, + format!("create-only pointer write: {e}"), + ) + })?; + let mut generation = match created { + ConditionalWrite::Committed(revision) => { + gcs_generation("pointer_create", 0, &pointer_key, &revision)? + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "pointer_create", + 0, + &pointer_key, + "a freshly nonced pointer key was already taken".to_string(), + ) + .into()) + } + }; - #[test] - fn static_keys_build_store_with_configured_region() { - let store = GitStore::new( - "http://localhost:9000", - "buzz_dev", - "buzz_dev_secret", - "buzz-git", - "us-west-2", - buzz_media::config::S3AddressingStyle::Path, - ) - .expect("static creds should build a git store"); - match store.bucket.region { - Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"), - ref other => panic!("expected Custom region, got {other:?}"), + // -- Phase 3: pointer_read --------------------------------------------- + // Body and generation must describe the same committed object, or the + // generation a caller predicates its next write on names a version it + // never read. + let (revision, stored) = self + .get_pointer(&pointer_key) + .await + .map_err(|e| gcs_failure("pointer_read", 0, &pointer_key, format!("read: {e}")))? + .ok_or_else(|| { + gcs_failure( + "pointer_read", + 0, + &pointer_key, + "the pointer just committed does not exist".to_string(), + ) + })?; + let observed = gcs_generation("pointer_read", 0, &pointer_key, &revision)?; + if observed != generation { + return Err(gcs_failure( + "pointer_read", + 0, + &pointer_key, + format!( + "read reported generation {observed} for the object committed as {generation}" + ), + ) + .into()); } - } - - #[test] - fn constructor_applies_both_addressing_styles() { - for (style, expected_url, path_style) in [ - ( - buzz_media::config::S3AddressingStyle::Path, - "https://storage.example/buzz-git", - true, - ), - ( - buzz_media::config::S3AddressingStyle::Virtual, - "https://buzz-git.storage.example", - false, - ), - ] { - let store = GitStore::new( - "https://storage.example", - "buzz_dev", - "buzz_dev_secret", - "buzz-git", - "us-east-1", - style, + if stored[..] != seed[..] { + return Err(gcs_failure( + "pointer_read", + 0, + &pointer_key, + "read returned bytes other than the committed body".to_string(), ) - .expect("construct git store"); - assert_eq!(store.bucket.url(), expected_url); - assert_eq!(store.bucket.is_path_style(), path_style); + .into()); } - } - #[test] - fn partial_static_keys_are_rejected() { - for (access, secret) in [("buzz_dev", ""), ("", "buzz_dev_secret")] { - let err = match GitStore::new( - "http://localhost:9000", - access, - secret, - "buzz-git", - "us-east-1", - buzz_media::config::S3AddressingStyle::Path, - ) { - Ok(_) => { - panic!("partial static creds must not silently use the credential chain") + // -- Phase 4: cas_replace ---------------------------------------------- + let replacement = format!("probe-gcs-replace-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + let superseded = generation; + generation = match self + .put_pointer( + &pointer_key, + &replacement, + WriteCondition::Matches(Revision::GcsGeneration(generation)), + ) + .await + .map_err(|e| gcs_failure("cas_replace", 0, &pointer_key, format!("write: {e}")))? + { + ConditionalWrite::Committed(revision) => { + let committed = gcs_generation("cas_replace", 0, &pointer_key, &revision)?; + if committed == superseded { + return Err(gcs_failure( + "cas_replace", + 0, + &pointer_key, + format!( + "the replacement reported the same generation {superseded} it \ + replaced, so the token cannot distinguish versions" + ), + ) + .into()); } - Err(err) => err, - }; - assert!( - matches!(err, StoreError::Config(_)), - "expected Config error, got {err:?}" - ); - } + committed + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "cas_replace", + 0, + &pointer_key, + "a compare-and-swap on the just-read generation conflicted with no \ + competing writer" + .to_string(), + ) + .into()) + } + }; + + // -- Phase 5: stale_cas ------------------------------------------------ + let stale_body = format!("probe-gcs-stale-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + match self + .put_pointer( + &pointer_key, + &stale_body, + WriteCondition::Matches(Revision::GcsGeneration(superseded)), + ) + .await + .map_err(|e| gcs_failure("stale_cas", 0, &pointer_key, format!("write: {e}")))? + { + ConditionalWrite::Conflict => {} + ConditionalWrite::Committed(_) => { + return Err(gcs_failure( + "stale_cas", + 0, + &pointer_key, + format!( + "a write predicated on superseded generation {superseded} committed: \ + the precondition is not being enforced, so every pointer update is a \ + blind overwrite" + ), + ) + .into()) + } + } + + // -- Phase 6: cas_race ------------------------------------------------- + for round in 0..cfg.race_rounds { + let mut attempt = 0usize; + let winner = loop { + state.pace(cfg.same_key_spacing).await; + match self + .gcs_race_round(round, attempt, &pointer_key, nonce, generation, cfg, state) + .await? + { + GcsRaceOutcome::Committed { generation, body } => { + break (generation, body); + } + GcsRaceOutcome::Unproven(reason) => { + if attempt >= cfg.unproven_round_retries { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + format!( + "no round proved a race in {} attempts: {reason}", + attempt + 1 + ), + ) + .into()); + } + attempt += 1; + state.throttled_rounds_retried += 1; + tracing::warn!( + phase = "cas_race", + round, + attempt, + reason = %reason, + "conformance race round proved nothing; re-running" + ); + } + } + }; + let (committed, body) = winner; + + // The stored object must be the winner's, at the winner's + // generation: a loser's payload surviving the race is the failure + // mode the whole pointer protocol exists to exclude. + let (revision, stored) = self + .get_pointer(&pointer_key) + .await + .map_err(|e| { + gcs_failure( + "cas_race", + round, + &pointer_key, + format!("post-race read: {e}"), + ) + })? + .ok_or_else(|| { + gcs_failure( + "cas_race", + round, + &pointer_key, + "the pointer vanished during the race".to_string(), + ) + })?; + let settled = gcs_generation("cas_race", round, &pointer_key, &revision)?; + if settled != committed { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + format!( + "the winner committed generation {committed} but the object settled at \ + {settled}" + ), + ) + .into()); + } + if stored[..] != body[..] { + return Err(gcs_failure( + "cas_race", + round, + &pointer_key, + "the object holds bytes no racer reported committing".to_string(), + ) + .into()); + } + generation = committed; + } + + // -- Phase 7: generation_roundtrip ------------------------------------- + // The generation a racer won with must predicate the next write; that + // chain — commit, then compare-and-swap on the returned token — is + // exactly what the push path does between two pushes. + let final_body = format!("probe-gcs-roundtrip-{nonce}").into_bytes(); + state.pace(cfg.same_key_spacing).await; + match self + .put_pointer( + &pointer_key, + &final_body, + WriteCondition::Matches(Revision::GcsGeneration(generation)), + ) + .await + .map_err(|e| { + gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + format!("write: {e}"), + ) + })? { + ConditionalWrite::Committed(revision) => { + let committed = gcs_generation("generation_roundtrip", 0, &pointer_key, &revision)?; + if committed == generation { + return Err(gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + format!("the write reported the same generation {generation} it replaced"), + ) + .into()); + } + } + ConditionalWrite::Conflict => { + return Err(gcs_failure( + "generation_roundtrip", + 0, + &pointer_key, + "the generation a racer committed did not predicate the next write, so a \ + winner cannot chain its own pushes" + .to_string(), + ) + .into()) + } + } + + Ok(()) + } + + /// One race round: `cfg.race_width` writers on the same generation. + /// + /// Returns the winner when the round proved something, and + /// [`GcsRaceOutcome::Unproven`] when it did not. Only a semantic violation — + /// two winners, a commit with no generation, an unclassifiable backend + /// answer — fails here. + #[allow(clippy::too_many_arguments)] + async fn gcs_race_round( + &self, + round: usize, + attempt: usize, + pointer_key: &str, + nonce: uuid::Uuid, + generation: i64, + cfg: &ProbeConfig, + state: &mut GcsProbeState, + ) -> Result { + let mut tasks = Vec::with_capacity(cfg.race_width); + for racer in 0..cfg.race_width { + let body = + format!("{GCS_RACE_BODY_PREFIX}{round}:{attempt}:{racer}:{nonce}").into_bytes(); + let condition = WriteCondition::Matches(Revision::GcsGeneration(generation)); + tasks.push(async move { + let outcome = self.put_pointer(pointer_key, &body, condition).await; + (racer, body, outcome) + }); + } + + let mut winners: Vec<(i64, Vec)> = Vec::new(); + let mut conflicts = 0usize; + let mut throttled = 0usize; + let mut drops = 0usize; + for (racer, body, outcome) in futures_util::future::join_all(tasks).await { + match outcome { + Ok(ConditionalWrite::Committed(revision)) => { + let committed = gcs_generation("cas_race", round, pointer_key, &revision)?; + winners.push((committed, body)); + } + Ok(ConditionalWrite::Conflict) => conflicts += 1, + // Throttling is a refusal to evaluate the precondition, so it + // says nothing about who won. It is counted, never scored. + Err(StoreError::Backend(ObjectStoreError::Throttled { .. })) => { + throttled += 1; + state.throttled_racers += 1; + } + Err(StoreError::Backend(ref e)) if e.is_ambiguous() => { + drops += 1; + state.transport_drops += 1; + tracing::warn!( + phase = "cas_race", + round, + racer, + "transport drop (pre-classification: socket/send failure)" + ); + } + Err(e) => { + return Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!("racer {racer}: {e}"), + ) + .into()) + } + } + } + + if winners.len() > 1 { + return Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!( + "{} racers committed on one generation: the store is not linearizing \ + conditional writes", + winners.len() + ), + ) + .into()); + } + + // Classified observers. A throttled racer is one: it proves the round + // ran, even though it proves nothing about the precondition. + let classified = winners.len() + conflicts + throttled; + if let Some((committed, body)) = winners.pop() { + if classified < 2 { + return Ok(GcsRaceOutcome::Unproven(format!( + "only {classified} of {} racers were classified, so no race was witnessed \ + ({drops} transport drops)", + cfg.race_width + ))); + } + return Ok(GcsRaceOutcome::Committed { + generation: committed, + body, + }); + } + + if conflicts == 0 { + return Ok(GcsRaceOutcome::Unproven(format!( + "no racer committed and none saw the generation move ({throttled} throttled, \ + {drops} transport drops)" + ))); + } + if drops > 0 { + return Ok(GcsRaceOutcome::Unproven(format!( + "{conflicts} racers saw the generation move but the committing racer's outcome \ + was never classified ({drops} transport drops)" + ))); + } + Err(gcs_failure( + "cas_race", + round, + pointer_key, + format!( + "{conflicts} racers were told the generation had moved, but no racer committed \ + and every outcome was classified: the object changed without an acknowledged \ + writer" + ), + ) + .into()) + } + + /// Delete the probe's objects, returning how many could not be removed. + /// + /// Runs on the failure path too: a failed probe is the one that gets + /// re-run, so that is exactly when leaking keys into the deployment's own + /// bucket must not happen. + async fn remove_probe_objects(&self, keys: &[String]) -> usize { + let mut failures = 0usize; + for key in keys { + if let Err(error) = self.store.delete(key).await { + failures += 1; + tracing::warn!(%error, "conformance probe could not remove its object"); + } + } + failures + } + + /// Helper: hex SHA-256 of bytes. + fn digest_hex(bytes: &[u8]) -> String { + let mut h = Sha256::new(); + h.update(bytes); + hex::encode(h.finalize()) + } + + /// Raw create-only PUT exposed for the probe's race-counting phase, where + /// we need to *see* collision outcomes rather than swallow them as + /// idempotent. Bubbles anything that is not a create-or-collision. + async fn put_immutable_raw( + &self, + key: &str, + bytes: &[u8], + ) -> Result { + Ok(self + .store + .put_immutable(key, bytes, "application/octet-stream") + .await?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn idx_key_uses_pack_digest_namespace() { + let digest = "a".repeat(64); + assert_eq!( + GitStore::idx_key_for_pack_digest(&digest).unwrap(), + format!("idx/{digest}") + ); + } + + #[test] + fn idx_key_rejects_non_digest_input() { + for bad in ["packs/abc", "abc", "../escape", &"g".repeat(64)] { + assert!(GitStore::idx_key_for_pack_digest(bad).is_err()); + } + } + + /// The provider errors git treats as domain outcomes must surface as their + /// own variants — call sites match on `NotFound` / `ObjectTooLarge` / + /// `DigestMismatch`, not on a wrapped backend error. + #[test] + fn provider_errors_lift_into_domain_variants() { + assert!(matches!( + StoreError::from(ObjectStoreError::NotFound { + key: "packs/x".into() + }), + StoreError::NotFound(ref key) if key == "packs/x" + )); + assert!(matches!( + StoreError::from(ObjectStoreError::ObjectTooLarge { + key: "packs/x".into(), + size: 9, + max: 4, + }), + StoreError::ObjectTooLarge { + size: 9, + max: 4, + .. + } + )); + assert!(matches!( + StoreError::from(ObjectStoreError::DigestMismatch { + key: "packs/x".into(), + expected: "a".into(), + actual: "b".into(), + }), + StoreError::DigestMismatch { .. } + )); + } + + /// Everything else stays a backend error, and only pre-classification + /// failures are ambiguous — the probe's drop-and-floor rule reads exactly + /// this predicate to decide which racers leave the observer set. + #[test] + fn other_provider_errors_stay_backend_and_keep_their_classification() { + let permanent = StoreError::from(ObjectStoreError::Provider { + operation: "put_conditional", + message: "AccessDenied".into(), + }); + match permanent { + StoreError::Backend(ref e) => assert!(!e.is_ambiguous()), + other => panic!("expected Backend, got {other:?}"), + } + + let unknown = StoreError::from(ObjectStoreError::TransportAmbiguous { + operation: "put_conditional", + message: "connection reset".into(), + }); + match unknown { + StoreError::Backend(ref e) => assert!(e.is_ambiguous()), + other => panic!("expected Backend, got {other:?}"), + } + } +} + +#[cfg(test)] +mod profiles { + //! Profile behaviour against a scripted store. + //! + //! The probe consumes the object-store seam, so a test double can answer a + //! race any way a real backend could — including ways no conforming backend + //! ever would. That is the point: a conformance gate is only worth its boot + //! time if it *fails* on the answers it claims to reject, and a live bucket + //! cannot be asked to commit two writers on one generation. + + use std::collections::{HashMap, VecDeque}; + use std::sync::atomic::{AtomicI64, Ordering}; + use std::sync::Mutex; + + use async_trait::async_trait; + use buzz_object_store::{ + BulkDeleteOutcome, ByteStream, ListPage, ObjectMeta, ObjectVersionRef, ObjectVersionsPage, + }; + + use super::*; + + /// How a scripted racer answers. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum RacerOutcome { + /// Commit regardless of the precondition — the only way to stage two + /// winners on one generation. + Win, + /// Commit, but report no object generation. + WinWithoutGeneration, + /// The ordinary lost-race answer. + Conflict, + /// Refuse the write for request rate. + Throttle, + /// Never produce a classified answer. + Drop, + } + + /// An in-memory object store with real compare-and-swap semantics, plus a + /// script that can override the answers to the probe's racing writers. + struct ScriptedStore { + provider: ProviderKind, + objects: Mutex>, + next_generation: AtomicI64, + race_script: Mutex>, + /// Blind-overwrite bug: commit even when the precondition is stale. + accept_stale_precondition: bool, + /// Report a created object as committed with no generation. + zero_generation_on_create: bool, + } + + impl ScriptedStore { + fn new(provider: ProviderKind) -> Self { + Self { + provider, + objects: Mutex::new(HashMap::new()), + // No live object has generation 0; that value means "absent". + next_generation: AtomicI64::new(1), + race_script: Mutex::new(VecDeque::new()), + accept_stale_precondition: false, + zero_generation_on_create: false, + } + } + + /// Script consecutive race rounds; rounds past the script race for real. + fn scripting(self, rounds: impl IntoIterator>) -> Self { + self.race_script + .lock() + .unwrap() + .extend(rounds.into_iter().flatten()); + self + } + + fn accepting_stale_preconditions(mut self) -> Self { + self.accept_stale_precondition = true; + self + } + + fn without_create_generation(mut self) -> Self { + self.zero_generation_on_create = true; + self + } + + fn keys(&self) -> Vec { + let mut keys: Vec<_> = self.objects.lock().unwrap().keys().cloned().collect(); + keys.sort(); + keys + } + + /// Mint the revision token this provider would report. + fn revision(&self, generation: i64) -> Revision { + match self.provider { + ProviderKind::S3 => Revision::S3Etag(format!("\"{generation}\"")), + ProviderKind::Gcs => Revision::GcsGeneration(generation), + } + } + + /// Read a caller's revision back, rejecting one from another provider + /// exactly as a real provider does. + fn generation_of(&self, revision: &Revision) -> Result { + match self.provider { + ProviderKind::S3 => revision + .expect_s3_etag() + .map(|tag| tag.trim_matches('"').parse().unwrap_or(-1)), + ProviderKind::Gcs => revision.expect_gcs_generation(), + } + } + + fn commit(&self, key: &str, bytes: &[u8]) -> i64 { + let generation = self.next_generation.fetch_add(1, Ordering::SeqCst); + self.objects + .lock() + .unwrap() + .insert(key.to_string(), (generation, Bytes::copy_from_slice(bytes))); + generation + } + + fn current_generation(&self, key: &str) -> i64 { + self.objects + .lock() + .unwrap() + .get(key) + .map(|(generation, _)| *generation) + .unwrap_or(0) + } + } + + #[async_trait] + impl ObjectStore for ScriptedStore { + fn provider(&self) -> ProviderKind { + self.provider + } + + async fn put( + &self, + key: &str, + bytes: &[u8], + _content_type: &str, + ) -> Result<(), ObjectStoreError> { + self.commit(key, bytes); + Ok(()) + } + + async fn put_file( + &self, + _key: &str, + _path: &std::path::Path, + _content_type: &str, + ) -> Result<(), ObjectStoreError> { + Err(ObjectStoreError::Provider { + operation: "put_file", + message: "unused by the conformance probe".into(), + }) + } + + async fn put_immutable( + &self, + key: &str, + bytes: &[u8], + content_type: &str, + ) -> Result { + match self + .put_conditional(key, bytes, content_type, WriteCondition::Absent) + .await? + { + ConditionalWrite::Committed(_) => Ok(ImmutableWrite::Created), + ConditionalWrite::Conflict => Ok(ImmutableWrite::AlreadyPresent), + } + } + + async fn put_conditional( + &self, + key: &str, + bytes: &[u8], + _content_type: &str, + condition: WriteCondition, + ) -> Result { + let expected = match &condition { + WriteCondition::Absent => 0, + WriteCondition::Matches(revision) => self.generation_of(revision)?, + }; + + // Only the profile's racing writers are scripted; every other write + // gets real compare-and-swap semantics, so the phases around the + // race behave like a conforming store unless a test says otherwise. + if bytes.starts_with(GCS_RACE_BODY_PREFIX.as_bytes()) { + let scripted = self.race_script.lock().unwrap().pop_front(); + match scripted { + Some(RacerOutcome::Win) => { + return Ok(ConditionalWrite::Committed( + self.revision(self.commit(key, bytes)), + )) + } + Some(RacerOutcome::WinWithoutGeneration) => { + self.commit(key, bytes); + return Ok(ConditionalWrite::Committed(Revision::GcsGeneration(0))); + } + Some(RacerOutcome::Conflict) => return Ok(ConditionalWrite::Conflict), + Some(RacerOutcome::Throttle) => { + return Err(ObjectStoreError::Throttled { + operation: "put_conditional", + retry_after: None, + }) + } + Some(RacerOutcome::Drop) => { + return Err(ObjectStoreError::TransportAmbiguous { + operation: "put_conditional", + message: "connection reset by peer".into(), + }) + } + None => {} + } + } + + let current = self.current_generation(key); + let honour_stale = + self.accept_stale_precondition && matches!(condition, WriteCondition::Matches(_)); + if current != expected && !honour_stale { + return Ok(ConditionalWrite::Conflict); + } + let generation = self.commit(key, bytes); + if self.zero_generation_on_create && current == 0 { + return Ok(ConditionalWrite::Committed(Revision::GcsGeneration(0))); + } + Ok(ConditionalWrite::Committed(self.revision(generation))) + } + + async fn get(&self, key: &str) -> Result { + self.objects + .lock() + .unwrap() + .get(key) + .map(|(_, bytes)| bytes.clone()) + .ok_or_else(|| ObjectStoreError::NotFound { key: key.into() }) + } + + async fn get_range( + &self, + key: &str, + start: u64, + end: u64, + ) -> Result { + let bytes = self.get(key).await?; + Ok(bytes.slice(start as usize..=(end as usize))) + } + + async fn get_stream(&self, _key: &str) -> Result { + Err(ObjectStoreError::Provider { + operation: "get_stream", + message: "unused by the conformance probe".into(), + }) + } + + async fn get_with_revision( + &self, + key: &str, + ) -> Result, ObjectStoreError> { + let found = self + .objects + .lock() + .unwrap() + .get(key) + .map(|(generation, bytes)| (*generation, bytes.clone())); + Ok(found.map(|(generation, bytes)| (self.revision(generation), bytes))) + } + + async fn head(&self, key: &str) -> Result, ObjectStoreError> { + let found = self + .objects + .lock() + .unwrap() + .get(key) + .map(|(generation, bytes)| (*generation, bytes.len() as u64)); + Ok(found.map(|(generation, size)| ObjectMeta { + size, + revision: Some(self.revision(generation)), + })) + } + + async fn list_page( + &self, + _prefix: &str, + _continuation_token: Option, + _max_keys: usize, + ) -> Result { + Ok(ListPage::default()) + } + + async fn delete(&self, key: &str) -> Result<(), ObjectStoreError> { + self.objects.lock().unwrap().remove(key); + Ok(()) + } + + async fn delete_objects( + &self, + keys: &[String], + ) -> Result { + let mut objects = self.objects.lock().unwrap(); + let mut outcome = BulkDeleteOutcome::default(); + for key in keys { + objects.remove(key); + outcome.deleted += 1; + } + Ok(outcome) + } + + async fn list_versions_page( + &self, + _prefix: &str, + _key_marker: Option, + _version_id_marker: Option, + _max_keys: usize, + ) -> Result { + Ok(ObjectVersionsPage::default()) + } + + async fn delete_versions( + &self, + versions: &[ObjectVersionRef], + ) -> Result { + let keys = versions + .iter() + .map(|version| version.key.clone()) + .collect::>(); + self.delete_objects(&keys).await + } + + async fn ping(&self) -> Result<(), ObjectStoreError> { + Ok(()) + } + + async fn versioning_detected(&self) -> Result { + Ok(false) + } + } + + /// A Cloud Storage profile config with the sleeps taken out, so the tests + /// that are not about pacing run at memory speed. + fn unpaced(race_rounds: usize) -> ProbeConfig { + ProbeConfig { + race_width: 3, + race_rounds, + same_key_spacing: Duration::ZERO, + ..ProbeConfig::for_provider(ProviderKind::Gcs) + } + } + + fn probe_failure(error: StoreError) -> ProbeFailure { + match error { + StoreError::Probe(failure) => failure, + other => panic!("expected a probe failure, got {other:?}"), + } + } + + /// Each profile's defaults are the ones the provider can actually answer. + /// The S3 numbers are unchanged — this profile is already admitted in + /// production and a conformance gate that quietly narrows is worse than one + /// that never widened. + #[test] + fn profile_defaults_follow_the_provider() { + let s3 = ProbeConfig::for_provider(ProviderKind::S3); + assert_eq!(s3, ProbeConfig::default()); + assert_eq!(s3.race_width, 32); + assert_eq!(s3.race_rounds, 3); + assert_eq!(s3.same_key_spacing, Duration::ZERO); + + let gcs = ProbeConfig::for_provider(ProviderKind::Gcs); + assert!(gcs.race_width >= 2 && gcs.race_width < s3.race_width); + assert!(gcs.race_rounds >= 1); + assert!( + gcs.same_key_spacing > Duration::from_secs(1), + "same-key rounds must be spaced past Cloud Storage's one-write-per-second ceiling" + ); + assert!(gcs.unproven_round_retries >= 1); + } + + /// A conforming store passes, and the probe leaves nothing behind. + #[tokio::test] + async fn a_conforming_store_is_admitted_and_cleans_up() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs)); + let store = GitStore::new(backend.clone()); + + let report = store + .run_conformance_probe(unpaced(2)) + .await + .expect("a conforming store is admitted"); + + assert_eq!(report.profile, ProviderKind::Gcs); + assert_eq!(report.race_width, 3); + assert_eq!(report.race_rounds, 2); + assert_eq!(report.throttled_racers, 0); + assert_eq!(report.throttled_rounds_retried, 0); + assert_eq!(report.transport_drops, 0); + assert_eq!(report.cleanup_failures, 0); + assert!( + backend.keys().is_empty(), + "probe objects left behind: {:?}", + backend.keys() + ); + } + + /// The failure the pointer protocol exists to exclude. Two winners is never + /// a pacing artefact, a retryable round, or a degraded observation — it is + /// the store admitting a lost update. + #[tokio::test] + async fn two_committed_racers_fail_the_probe() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::Win, + RacerOutcome::Win, + RacerOutcome::Conflict, + ]])); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("two winners on one generation must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure.reason.contains("2 racers committed"), + "unexpected reason: {}", + failure.reason + ); + } + + /// A round in which every racer was refused for rate proves nothing: no + /// precondition was ever evaluated. Scoring it as "no winner" would fail a + /// conforming store for being paced, so the round is re-run instead. + #[tokio::test] + async fn a_fully_throttled_round_is_re_run_rather_than_scored() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Throttle; 3]]), + ); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a throttled round is re-run, and the re-run proves the race"); + + assert_eq!(report.throttled_rounds_retried, 1); + assert_eq!(report.throttled_racers, 3); + } + + /// Re-running is bounded. A store that only ever throttles is never + /// admitted — the probe fails rather than waiting forever or passing on no + /// evidence. + #[tokio::test] + async fn re_runs_are_bounded_and_an_unproven_race_fails_closed() { + let mut cfg = unpaced(1); + cfg.unproven_round_retries = 2; + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting( + vec![vec![RacerOutcome::Throttle; 3]; cfg.unproven_round_retries + 1], + )); + + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(cfg) + .await + .expect_err("a store that only throttles is never admitted"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure + .reason + .contains("no round proved a race in 3 attempts"), + "unexpected reason: {}", + failure.reason + ); + } + + /// One winner among a mix of conflicts and throttles is a pass: the + /// throttled racer is counted, not treated as a loser, and one classified + /// witness is enough to have seen the race. + #[tokio::test] + async fn a_throttled_racer_is_never_a_lost_race() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::Win, + RacerOutcome::Conflict, + RacerOutcome::Throttle, + ]])); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a round with one winner, one conflict and one throttle is proven"); + + assert_eq!(report.throttled_racers, 1); + assert_eq!(report.throttled_rounds_retried, 0); + } + + /// A commit the store cannot name is unusable: the caller has nothing to + /// predicate its next write on. Fatal wherever it appears. + #[tokio::test] + async fn a_commit_without_a_generation_fails_the_probe() { + let on_create = Arc::new(ScriptedStore::new(ProviderKind::Gcs).without_create_generation()); + let failure = probe_failure( + GitStore::new(on_create) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("a create with no generation must fail closed"), + ); + assert_eq!(failure.phase, "pointer_create"); + assert!( + failure.reason.contains("no object generation"), + "unexpected reason: {}", + failure.reason + ); + + let on_race = Arc::new(ScriptedStore::new(ProviderKind::Gcs).scripting([vec![ + RacerOutcome::WinWithoutGeneration, + RacerOutcome::Conflict, + RacerOutcome::Conflict, + ]])); + let failure = probe_failure( + GitStore::new(on_race) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("a race winner with no generation must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure.reason.contains("no object generation"), + "unexpected reason: {}", + failure.reason + ); + } + + /// A store that commits on a superseded generation is doing blind + /// overwrites, which silently loses pushes. The stale phase is what catches + /// it, and it must catch it before any race runs. + #[tokio::test] + async fn a_store_that_honours_a_stale_generation_fails_the_probe() { + let backend = + Arc::new(ScriptedStore::new(ProviderKind::Gcs).accepting_stale_preconditions()); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("an unenforced precondition must fail closed"), + ); + assert_eq!(failure.phase, "stale_cas"); + assert!( + failure.reason.contains("blind overwrite"), + "unexpected reason: {}", + failure.reason + ); + } + + /// Every racer was told the generation had moved, and every outcome was + /// classified — so the object changed with no writer acknowledged. That is + /// a lost update announcing itself, not an unproven round: the probe must + /// not retry its way past it. + #[tokio::test] + async fn a_round_where_every_racer_loses_fails_the_probe() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Conflict; 3]]), + ); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect_err("conflicts with no acknowledged winner must fail closed"), + ); + assert_eq!(failure.phase, "cas_race"); + assert!( + failure + .reason + .contains("the object changed without an acknowledged writer"), + "unexpected reason: {}", + failure.reason + ); + } + + /// Every racer's outcome was unknown, so the round is unproven rather than + /// a failure — the probe admits stores, not networks. + #[tokio::test] + async fn transport_drops_leave_the_observer_set_rather_than_failing() { + let backend = Arc::new( + ScriptedStore::new(ProviderKind::Gcs).scripting([vec![RacerOutcome::Drop; 3]]), + ); + let report = GitStore::new(backend) + .run_conformance_probe(unpaced(1)) + .await + .expect("a dropped round is re-run"); + + assert_eq!(report.transport_drops, 3); + assert_eq!(report.throttled_rounds_retried, 1); + } + + /// Pacing is measured, not asserted: the probe sleeps between same-key + /// mutations and reports the shortest interval it actually observed. + #[tokio::test] + async fn same_key_mutations_are_spaced_by_the_configured_interval() { + let spacing = Duration::from_millis(40); + let mut cfg = unpaced(2); + cfg.race_width = 2; + cfg.same_key_spacing = spacing; + + // create, replace, stale, two race rounds, round-trip: six same-key + // mutations, so five paced gaps. + let paced_gaps = 5; + let backend = Arc::new(ScriptedStore::new(ProviderKind::Gcs)); + let started = Instant::now(); + let report = GitStore::new(backend) + .run_conformance_probe(cfg) + .await + .expect("a conforming store is admitted"); + let elapsed = started.elapsed(); + + let observed = report + .min_same_key_gap + .expect("a paced profile reports the interval it observed"); + assert!( + observed >= spacing, + "shortest observed gap {observed:?} is under the configured {spacing:?}" + ); + assert!( + elapsed >= spacing * paced_gaps, + "the whole probe took {elapsed:?}, less than {paced_gaps} gaps of {spacing:?}" + ); + } + + /// The provider selects the profile. An S3 store still runs the ETag + /// profile, unchanged, and reports none of the Cloud Storage counters. + #[tokio::test] + async fn an_s3_store_runs_the_s3_profile() { + let backend = Arc::new(ScriptedStore::new(ProviderKind::S3)); + let report = GitStore::new(backend) + .run_conformance_probe(ProbeConfig { + race_width: 3, + race_rounds: 1, + ..ProbeConfig::default() + }) + .await + .expect("a conforming S3 store is admitted"); + + assert_eq!(report.profile, ProviderKind::S3); + assert_eq!(report.transport_drops, 0); + assert_eq!(report.throttled_racers, 0); + assert_eq!(report.min_same_key_gap, None); + } + + /// The width floor is a property of the gate, not of a profile: one writer + /// cannot witness a race whichever provider is underneath. + #[tokio::test] + async fn a_race_narrower_than_two_writers_is_rejected() { + for provider in [ProviderKind::S3, ProviderKind::Gcs] { + let backend = Arc::new(ScriptedStore::new(provider)); + let failure = probe_failure( + GitStore::new(backend) + .run_conformance_probe(ProbeConfig { + race_width: 1, + race_rounds: 1, + ..ProbeConfig::for_provider(provider) + }) + .await + .expect_err("a single writer cannot witness a race"), + ); + assert_eq!(failure.phase, "config"); + } } } #[cfg(test)] mod probe { - //! Empirical probe of rust-s3 + `fail-on-err` + MinIO surfacing of 412. + //! Empirical probe of the S3 provider's precondition surfacing against + //! live MinIO. //! //! Run manually: //! BUZZ_GIT_S3_PROBE=1 cargo test -p buzz-relay --lib \ @@ -1043,17 +2204,15 @@ mod probe { std::env::var("BUZZ_S3_SECRET_KEY").unwrap_or_else(|_| "buzz_dev_secret".into()); let bucket = std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-git".into()); let region = std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".into()); - let addressing_style = std::env::var("BUZZ_S3_ADDRESSING_STYLE") - .unwrap_or_else(|_| "path".into()) - .parse() - .expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"); - GitStore::new( + let addressing_style = + std::env::var("BUZZ_S3_ADDRESSING_STYLE").unwrap_or_else(|_| "path".into()); + crate::test_git_store( &endpoint, &access_key, &secret_key, &bucket, ®ion, - addressing_style, + &addressing_style, ) .expect("connect S3-compatible storage") } @@ -1072,24 +2231,17 @@ mod probe { } let st = store(); let key = format!("probe/cas-{}.txt", uuid::Uuid::new_v4()); - let mut hdrs = axum::http::HeaderMap::new(); - hdrs.insert(axum::http::header::IF_NONE_MATCH, "*".parse().unwrap()); - let r1 = st - .bucket - .put_object_with_content_type_and_headers( - &key, - b"first", - "text/plain", - Some(hdrs.clone()), - ) - .await; - assert!((200..300).contains(&r1.expect("first ok").status_code())); - let r2 = st - .bucket - .put_object_with_content_type_and_headers(&key, b"second", "text/plain", Some(hdrs)) - .await; - assert!(matches!(r2, Err(S3Error::HttpFailWithBody(412, _)))); - let _ = st.bucket.delete_object(&key).await; + let first = st + .put_immutable_raw(&key, b"first") + .await + .expect("first create-only write"); + assert_eq!(first, ImmutableWrite::Created); + let second = st + .put_immutable_raw(&key, b"second") + .await + .expect("second create-only write must classify, not error"); + assert_eq!(second, ImmutableWrite::AlreadyPresent); + let _ = st.store.delete(&key).await; } #[tokio::test] @@ -1118,57 +2270,64 @@ mod probe { let err = st.get_verified(&key, &bogus).await.unwrap_err(); assert!(matches!(err, StoreError::DigestMismatch { .. })); - // 4. pointer lifecycle: get_pointer (None) → put_pointer(IfNoneMatchStar) - // → get_pointer (Some) → put_pointer(IfMatch correct) → put_pointer(IfMatch wrong, LostRace). + // 4. pointer lifecycle: get_pointer (None) → put_pointer(Absent) + // → get_pointer (Some) → put_pointer(Matches correct) + // → put_pointer(Matches stale, Conflict). let pkey = format!("pointers/{}.json", uuid::Uuid::new_v4()); assert!(st.get_pointer(&pkey).await.expect("get none").is_none()); let p1 = br#"{"manifest":"d1"}"#; let r = st - .put_pointer(&pkey, p1, Precond::IfNoneMatchStar) + .put_pointer(&pkey, p1, WriteCondition::Absent) .await .expect("first cas"); - let e1 = match r { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => panic!("first INM* should win"), + let r1 = match r { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("first create-only write should commit"), }; - eprintln!("Won.etag from PUT response: {:?}", e1.0); + eprintln!("committed revision from PUT response: {r1:?}"); - // Second INM* must lose. + // Second create-only write must conflict. let r = st - .put_pointer(&pkey, b"{}", Precond::IfNoneMatchStar) + .put_pointer(&pkey, b"{}", WriteCondition::Absent) .await .expect("second cas"); - assert_eq!(r, CasOutcome::LostRace, "second INM* must lose"); + assert_eq!( + r, + ConditionalWrite::Conflict, + "second create-only write must conflict" + ); - // Chain CAS directly on the PUT-returned ETag (no HEAD round-trip). + // Chain CAS directly on the PUT-returned revision (no HEAD round-trip). // MinIO returns the ETag in the PUT response; this proves callers can - // chain `Won → IfMatch → Won` without re-reading the pointer. - assert!(!e1.0.is_empty(), "MinIO should populate PUT response ETag"); + // chain commit → CAS → commit without re-reading the pointer. let p2 = br#"{"manifest":"d2"}"#; let r = st - .put_pointer(&pkey, p2, Precond::IfMatch(e1.clone())) + .put_pointer(&pkey, p2, WriteCondition::Matches(r1.clone())) .await .expect("cas2"); - let e2 = match r { - CasOutcome::Won(e) => e, - CasOutcome::LostRace => panic!("IfMatch with fresh etag should win"), + let r2 = match r { + ConditionalWrite::Committed(revision) => revision, + ConditionalWrite::Conflict => panic!("CAS with fresh revision should commit"), }; - // Stale IfMatch (reuse the *first* etag, which has been superseded) → LostRace. + // Stale CAS (reuse the *first* revision, which has been superseded). let r = st - .put_pointer(&pkey, b"{}", Precond::IfMatch(e1)) + .put_pointer(&pkey, b"{}", WriteCondition::Matches(r1)) .await .expect("cas3"); - assert_eq!(r, CasOutcome::LostRace, "stale IfMatch must lose"); + assert_eq!(r, ConditionalWrite::Conflict, "stale CAS must conflict"); - // get_pointer's etag matches the most recent PUT-returned etag. - let (etag_now, _body) = st.get_pointer(&pkey).await.expect("get").expect("exists"); - assert_eq!(etag_now, e2, "get_pointer etag matches PUT-response etag"); + // get_pointer's revision matches the most recent PUT-returned revision. + let (revision_now, _body) = st.get_pointer(&pkey).await.expect("get").expect("exists"); + assert_eq!( + revision_now, r2, + "get_pointer revision matches PUT-response revision" + ); // Cleanup. - let _ = st.bucket.delete_object(&pkey).await; - let _ = st.bucket.delete_object(&key).await; + let _ = st.store.delete(&pkey).await; + let _ = st.store.delete(&key).await; } /// End-to-end conformance probe against MinIO. This is the same code path @@ -1183,6 +2342,7 @@ mod probe { .run_conformance_probe(ProbeConfig { race_width: 8, race_rounds: 2, + ..ProbeConfig::for_provider(ProviderKind::S3) }) .await .expect("conformance probe"); @@ -1191,24 +2351,458 @@ mod probe { assert_eq!(report.race_rounds, 2); } - /// Quick probe: confirm rust-s3's `get_object` exposes ETag on the response. + /// Quick probe: confirm a plain read exposes the object's revision. #[tokio::test] - async fn probe_get_exposes_etag() { + async fn probe_get_exposes_revision() { if !probe_enabled() { return; } let st = store(); - let key = format!("probe/etag-{}.txt", uuid::Uuid::new_v4()); - st.bucket - .put_object_with_content_type(&key, b"hi", "text/plain") + let key = format!("probe/revision-{}.txt", uuid::Uuid::new_v4()); + st.store.put(&key, b"hi", "text/plain").await.expect("put"); + let observed = st.get_pointer(&key).await.expect("get").expect("exists"); + eprintln!("revision from GET: {:?}", observed.0); + let _ = st.store.delete(&key).await; + } +} + +#[cfg(test)] +mod gcs_live { + //! The Cloud Storage profile against a real bucket. + //! + //! This is the deployment gate itself, run against the provider it was + //! written for — the scripted-store tests prove how the profile *judges* + //! answers, and only a live bucket proves which answers Cloud Storage + //! actually gives. + //! + //! Talks to a real bucket, so it is `#[ignore]`d and additionally gated on + //! `BUZZ_GCS_LIVE=1`; a bare `--ignored` run without credentials skips + //! rather than fails. + //! + //! ```bash + //! BUZZ_GCS_LIVE=1 \ + //! BUZZ_GCS_TEST_BUCKET=my-disposable-bucket \ + //! cargo test -p buzz-relay --lib api::git::store::gcs_live -- --ignored --nocapture + //! ``` + //! + //! Credentials come from Application Default Credentials, exactly as in + //! production. Every test works under its own `a3//` namespace: the + //! probe run asserts the probe's own cleanup pass emptied it, and the + //! git-facade runs delete what they wrote whether they passed or not. + + use std::panic::AssertUnwindSafe; + + use buzz_object_store::{GcsObjectStore, GcsStoreConfig}; + use futures_util::FutureExt; + + use super::*; + + /// Every key left under `prefix`. + async fn remaining(store: &Arc, prefix: &str) -> Vec { + let mut token = None; + let mut keys = Vec::new(); + loop { + let page = store + .list_page(prefix, token, 1000) + .await + .expect("list the probe namespace"); + keys.extend(page.objects.into_iter().map(|(key, _)| key)); + token = page.next_continuation_token; + if token.is_none() { + return keys; + } + } + } + + /// Connect to the test bucket, or `None` when this environment has not + /// opted in to live tests. + async fn live_backend() -> Option> { + // The relay installs this in `main` before any TLS request; a test + // binary must do the same, because both ring and aws-lc-rs are in the + // build graph and rustls will not choose between them. + static PROVIDER: std::sync::Once = std::sync::Once::new(); + PROVIDER.call_once(|| { + let _ = rustls::crypto::ring::default_provider().install_default(); + }); + + if std::env::var("BUZZ_GCS_LIVE").as_deref() != Ok("1") { + eprintln!("skipping: set BUZZ_GCS_LIVE=1 to run against a live bucket"); + return None; + } + let bucket = match std::env::var("BUZZ_GCS_TEST_BUCKET") { + Ok(bucket) if !bucket.is_empty() => bucket, + _ => panic!("BUZZ_GCS_LIVE=1 requires BUZZ_GCS_TEST_BUCKET"), + }; + + // Connecting runs the provider's admission check: a bucket with + // versioning or soft delete never reaches these tests at all. + Some(Arc::new( + GcsObjectStore::connect(&GcsStoreConfig::new(bucket)) + .await + .expect("connect to the test bucket"), + )) + } + + /// Run one git-facade test under its own namespace, then remove what it + /// wrote whether it passed or not. + /// + /// A panicking body would otherwise leak into a bucket shared with every + /// other run — and a failing test is precisely the one that gets re-run. + async fn with_git_store(test: &str, body: F) + where + F: AsyncFnOnce(&GitStore, &str), + { + let Some(backend) = live_backend().await else { + return; + }; + let prefix = format!("a4/{}/{test}", uuid::Uuid::new_v4()); + let store = GitStore::new(backend.clone()); + + let outcome = AssertUnwindSafe(body(&store, &prefix)).catch_unwind().await; + + let keys = remaining(&backend, &prefix).await; + let cleanup = if keys.is_empty() { + Ok(Default::default()) + } else { + backend.delete_objects(&keys).await + }; + if let Err(payload) = outcome { + std::panic::resume_unwind(payload); + } + let cleanup = cleanup.expect("bulk delete for cleanup"); + assert!( + cleanup.failed.is_empty(), + "cleanup left objects behind: {:?}", + cleanup.failed + ); + assert!( + remaining(&backend, &prefix).await.is_empty(), + "the namespace must be empty after cleanup" + ); + } + + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn the_cloud_storage_profile_admits_a_real_bucket() { + let Some(backend) = live_backend().await else { + return; + }; + let prefix = format!("a3/{}", uuid::Uuid::new_v4()); + let cfg = ProbeConfig::for_provider(ProviderKind::Gcs).under_key_prefix(&prefix); + let spacing = cfg.same_key_spacing; + + let report = GitStore::new(backend.clone()) + .run_conformance_probe(cfg) .await - .expect("put"); - let resp = st.bucket.get_object(&key).await.expect("get"); - let headers = resp.headers(); - eprintln!("GET headers: {headers:?}"); - let etag = headers.get("etag").or_else(|| headers.get("ETag")).cloned(); - assert!(etag.is_some(), "GET response must carry ETag header"); - eprintln!("ETag from GET: {etag:?}"); - let _ = st.bucket.delete_object(&key).await; + .expect("Cloud Storage satisfies the conditional-write axioms"); + eprintln!("✓ probe report: {report:?}"); + + assert_eq!(report.profile, ProviderKind::Gcs); + assert_eq!(report.race_width, 3); + assert_eq!(report.race_rounds, 2); + let gap = report + .min_same_key_gap + .expect("the paced profile reports the interval it observed"); + assert!( + gap >= spacing, + "same-key rounds were {gap:?} apart, inside the configured {spacing:?}" + ); + + // The bucket is shared with other runs, so an unremoved probe object + // would be invisible until it was large. + let leaked = remaining(&backend, &prefix).await; + assert_eq!(report.cleanup_failures, 0, "cleanup reported failures"); + assert!(leaked.is_empty(), "probe objects left behind: {leaked:?}"); + } + + /// The publication cycle a push performs, against a real bucket. + /// + /// Create the pointer under the create-only precondition, read body and + /// revision from one response, swap on the revision that read returned, and + /// then replay the superseded revision — which must lose rather than + /// overwrite. The last assertion is the one that matters most: an absent + /// pointer reads as `None`, distinct from every failure, because a pointer + /// read that could not be answered must never be mistaken for an empty + /// repository. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn the_pointer_cycle_publishes_exactly_one_state() { + with_git_store("pointer-cycle", async |store, prefix| { + let key = format!("{prefix}/pointers/repo"); + + assert!( + store + .get_pointer(&key) + .await + .expect("reading an absent pointer is not an error") + .is_none(), + "an unpublished repository is `None`, never an error and never empty bytes" + ); + + let ConditionalWrite::Committed(created) = store + .put_pointer(&key, b"{\"refs\":0}", WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let (observed, body) = store + .get_pointer(&key) + .await + .expect("pointer read") + .expect("the pointer exists"); + assert_eq!(body.as_ref(), b"{\"refs\":0}"); + assert_eq!( + observed, created, + "the revision a read reports must be the one the write committed" + ); + + let ConditionalWrite::Committed(swapped) = store + .put_pointer( + &key, + b"{\"refs\":1}", + WriteCondition::Matches(observed.clone()), + ) + .await + .expect("swap on the observed revision") + else { + panic!("a writer holding the current revision must commit"); + }; + assert_ne!(swapped, created, "a commit mints a new revision"); + + assert_eq!( + store + .put_pointer(&key, b"{\"refs\":99}", WriteCondition::Matches(observed)) + .await + .expect("replay the superseded revision"), + ConditionalWrite::Conflict, + "a superseded revision must lose rather than overwrite" + ); + assert_eq!( + store.get(&key).await.expect("read the published state"), + &b"{\"refs\":1}"[..], + "the losing writer must not have published anything" + ); + }) + .await; + } + + /// Content addressing end to end: what a push writes, what a hydrate reads, + /// and what happens when the bytes are not what the key says they are. + /// + /// The idx sidecar is the one object here whose key is *not* its own digest + /// — it is derived from the pack digest so a hydrate can find it without + /// changing manifest bytes — so it gets its own miss/hit assertions. The + /// corruption arm is the point of the whole discipline: bytes that do not + /// hash to their key are a detected error on read, not silent corruption + /// handed to git. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn content_addressed_objects_round_trip_and_corruption_is_detected() { + with_git_store("content-addressing", async |store, prefix| { + // Production derives these keys from the bytes, so a test cannot + // place them under its own namespace without going around the + // facade. Writing them where production would and cleaning up by + // exact key keeps the write path honest. + let pack_bytes = b"PACK\x00\x00\x00\x02 pretend pack payload".to_vec(); + let pack_key = store.put_pack(&pack_bytes).await.expect("write the pack"); + let pack_digest = pack_key + .strip_prefix("packs/") + .expect("a pack key is namespaced") + .to_string(); + assert_eq!( + pack_key, + GitStore::content_key("packs", &pack_bytes), + "the facade, not the caller, derives the key from the bytes" + ); + assert_eq!( + store + .get_verified(&pack_key, &pack_digest) + .await + .expect("verified read"), + pack_bytes, + "a hydrate reads the pack back byte-exact" + ); + + assert!( + store + .get_idx(&pack_digest, 1 << 20) + .await + .expect("an idx miss is not an error") + .is_none(), + "a missing idx is a cache miss the hydrate regenerates" + ); + let idx_bytes = b"\xfftOc\x00\x00\x00\x02 pretend idx".to_vec(); + let idx_key = store + .put_idx(&pack_digest, &idx_bytes) + .await + .expect("write the idx sidecar"); + assert_eq!(idx_key, format!("idx/{pack_digest}")); + assert_eq!( + store + .get_idx(&pack_digest, 1 << 20) + .await + .expect("idx read") + .expect("the idx exists"), + idx_bytes + ); + + let manifest_bytes = format!("{{\"packs\":[\"{pack_key}\"]}}").into_bytes(); + let manifest_key = store + .put_manifest(&manifest_bytes) + .await + .expect("write the manifest"); + let manifest_digest = manifest_key + .strip_prefix("manifests/") + .expect("a manifest key is namespaced") + .to_string(); + assert_eq!( + store + .get_verified_limited(&manifest_key, &manifest_digest, 1 << 20) + .await + .expect("bounded verified read"), + manifest_bytes + ); + assert!( + matches!( + store + .get_verified_limited(&manifest_key, &manifest_digest, 4) + .await, + Err(StoreError::ObjectTooLarge { .. }) + ), + "a bounded read rejects an oversized object before transferring it" + ); + + assert!( + matches!( + store.get(&format!("{prefix}/packs/absent")).await, + Err(StoreError::NotFound(_)) + ), + "a missing object is a not-found, never empty bytes" + ); + + // Corruption: bytes that do not hash to the key they sit under. + // Production cannot produce this (writes are create-only and the + // facade derives the key), so the test writes through the backend + // directly — which is exactly the shape of a backend that returned + // the wrong object. + let claimed = b"the bytes this key names".to_vec(); + let corrupt_key = GitStore::content_key(&format!("{prefix}/packs"), &claimed); + let claimed_digest = corrupt_key + .rsplit('/') + .next() + .expect("a content key ends in its digest") + .to_string(); + store + .store + .put(&corrupt_key, b"different bytes", "application/x-git-pack") + .await + .expect("plant mismatched bytes"); + + let error = store + .get_verified(&corrupt_key, &claimed_digest) + .await + .expect_err("a verified read must reject bytes that do not hash to their key"); + let StoreError::DigestMismatch { + key, + expected, + actual, + } = &error + else { + panic!("corruption must surface as a digest mismatch, not {error:?}"); + }; + assert_eq!(key, &corrupt_key); + assert_eq!(expected, &claimed_digest); + assert_ne!(actual, expected, "the report names the digest it computed"); + + // Only the corruption arm's key is under the test namespace; the + // rest live where production writes them, so remove them by hand. + let planted = [pack_key, idx_key, manifest_key]; + let outcome = store + .store + .delete_objects(&planted) + .await + .expect("remove the objects written outside the test namespace"); + assert!(outcome.failed.is_empty(), "{:?}", outcome.failed); + }) + .await; + } + + /// A chunked repository seed: one writer, one pointer, many sequential + /// transitions, faster than the published per-object write ceiling. + /// + /// This is the mirror's initial seed of a large repository — roughly 4.5k + /// refs at 200 refs per chunk, each chunk a pointer transition predicated + /// on the revision the previous one committed. Cloud Storage documents one + /// write per second to a single object name, so the sequence deliberately + /// exceeds it. Every transition must still commit: throttling is pacing, + /// absorbed inside the provider's bounded policy, and must never reach the + /// caller as a failed push or be mistaken for a lost race in a sequence + /// that has exactly one entrant. + #[tokio::test] + #[ignore = "requires a live GCS bucket (BUZZ_GCS_LIVE=1)"] + async fn a_chunked_seed_commits_every_sequential_transition() { + /// One more than the ~23 chunks a 4.5k-ref seed needs. + const CHUNKS: usize = 24; + + with_git_store("chunked-seed", async |store, prefix| { + let key = format!("{prefix}/pointers/seed"); + + let ConditionalWrite::Committed(mut revision) = store + .put_pointer(&key, b"{\"chunk\":0,\"refs\":0}", WriteCondition::Absent) + .await + .expect("create the pointer") + else { + panic!("creating an absent pointer must commit"); + }; + + let mut revisions = vec![revision.clone()]; + let started = Instant::now(); + for chunk in 1..=CHUNKS { + let body = format!("{{\"chunk\":{chunk},\"refs\":{}}}", chunk * 200); + let outcome = store + .put_pointer( + &key, + body.as_bytes(), + WriteCondition::Matches(revision.clone()), + ) + .await; + revision = match outcome { + Ok(ConditionalWrite::Committed(next)) => next, + Ok(ConditionalWrite::Conflict) => panic!( + "chunk {chunk} lost a race it was the only entrant in — throttling was \ + misclassified as a lost CAS" + ), + Err(StoreError::Backend(ObjectStoreError::Throttled { .. })) => panic!( + "chunk {chunk} surfaced throttling as a push failure; pacing belongs \ + inside the provider's bounded policy, never at the caller" + ), + Err(other) => panic!("chunk {chunk} failed: {other}"), + }; + revisions.push(revision.clone()); + } + let elapsed = started.elapsed(); + + let mut distinct = revisions.clone(); + distinct.dedup(); + assert_eq!( + distinct.len(), + revisions.len(), + "every transition must publish a fresh revision" + ); + assert_eq!( + store.get(&key).await.expect("read the published state"), + &format!("{{\"chunk\":{CHUNKS},\"refs\":{}}}", CHUNKS * 200).into_bytes()[..], + "the last chunk must be the published state" + ); + + eprintln!( + "chunked seed: {CHUNKS} sequential pointer transitions committed in {elapsed:?} \ + ({:.2} transitions/sec), zero failures", + CHUNKS as f64 / elapsed.as_secs_f64() + ); + }) + .await; } } diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 94fd7f8758e..41baa3d1c15 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2314,7 +2314,7 @@ mod track_c_tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index e3d05165e0d..35253918702 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -694,7 +694,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 780532ec5d0..540ac313770 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -1158,7 +1158,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index b59fd840c6d..7dea2958320 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -603,7 +603,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (mut state, _audit_shutdown) = AppState::new( config, db, @@ -1288,7 +1288,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index e035752ec3a..edd7e26477e 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -292,6 +292,13 @@ pub struct Config { /// Media storage configuration (S3/MinIO). pub media: buzz_media::MediaConfig, + /// Object-store provider backing both media blobs and Git storage. + /// + /// The relay builds exactly one client from this and shares it between the + /// two facades, so the provider is a property of the deployment rather + /// than of each call site. Selected by `BUZZ_OBJECT_STORE_PROVIDER`, which + /// defaults to the S3 settings above. + pub object_store: buzz_object_store::ObjectStoreConfig, /// Maximum concurrent media uploads handled by one relay process. pub media_max_concurrent_uploads: usize, /// Maximum concurrent media uploads accepted from one pubkey. @@ -825,28 +832,7 @@ impl Config { .and_then(|v| v.parse().ok()) .unwrap_or(9102); - let s3_addressing_style = match std::env::var("BUZZ_S3_ADDRESSING_STYLE") { - Ok(value) => value.parse().map_err(ConfigError::InvalidValue)?, - Err(std::env::VarError::NotPresent) => buzz_media::config::S3AddressingStyle::default(), - Err(std::env::VarError::NotUnicode(_)) => { - return Err(ConfigError::InvalidValue( - "BUZZ_S3_ADDRESSING_STYLE must be valid Unicode and one of 'path' or 'virtual'" - .to_string(), - )); - } - }; let media = buzz_media::MediaConfig { - s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") - .unwrap_or_else(|_| "http://localhost:9000".to_string()), - s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") - .unwrap_or_else(|_| "buzz_dev".to_string()), - s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") - .unwrap_or_else(|_| "buzz_dev_secret".to_string()), - s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), - s3_region: std::env::var("BUZZ_S3_REGION") - .or_else(|_| std::env::var("AWS_REGION")) - .unwrap_or_else(|_| "us-east-1".to_string()), - s3_addressing_style, max_image_bytes: std::env::var("BUZZ_MAX_IMAGE_BYTES") .ok() .and_then(|v| v.parse().ok()) @@ -880,6 +866,47 @@ impl Config { .map(|s| s.trim().to_lowercase()) .filter(|s| !s.is_empty()), }; + // Choose the provider before reading provider-native settings. A GCS + // deployment is unaffected by stale or malformed `BUZZ_S3_*` values, + // and vice versa. + let object_store = match buzz_object_store::ProviderSelection::from_env() + .map_err(ConfigError::InvalidValue)? + { + buzz_object_store::ProviderSelection::S3 => { + let addressing_style = match std::env::var("BUZZ_S3_ADDRESSING_STYLE") { + Ok(value) => value.parse().map_err(ConfigError::InvalidValue)?, + Err(std::env::VarError::NotPresent) => { + buzz_object_store::S3AddressingStyle::default() + } + Err(std::env::VarError::NotUnicode(_)) => { + return Err(ConfigError::InvalidValue( + "BUZZ_S3_ADDRESSING_STYLE must be valid Unicode and one of 'path' or 'virtual'" + .to_string(), + )); + } + }; + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + endpoint: std::env::var("BUZZ_S3_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:9000".to_string()), + access_key: std::env::var("BUZZ_S3_ACCESS_KEY") + .unwrap_or_else(|_| "buzz_dev".to_string()), + secret_key: std::env::var("BUZZ_S3_SECRET_KEY") + .unwrap_or_else(|_| "buzz_dev_secret".to_string()), + bucket: std::env::var("BUZZ_S3_BUCKET") + .unwrap_or_else(|_| "buzz-media".to_string()), + region: std::env::var("BUZZ_S3_REGION") + .or_else(|_| std::env::var("AWS_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + addressing_style, + }) + } + buzz_object_store::ProviderSelection::Gcs { bucket } => { + buzz_object_store::ObjectStoreConfig::Gcs(buzz_object_store::GcsStoreConfig::new( + bucket, + )) + } + }; + let media_max_concurrent_uploads: usize = std::env::var("BUZZ_MEDIA_MAX_CONCURRENT_UPLOADS") .ok() @@ -1235,6 +1262,7 @@ impl Config { allow_nip_oa_auth, klipy, media, + object_store, media_max_concurrent_uploads, media_max_concurrent_uploads_per_pubkey, media_uploads_per_minute, @@ -1371,9 +1399,14 @@ mod tests { !config.serve_git_web_gui, "serve_git_web_gui should default to false" ); - assert_eq!( - config.media.s3_addressing_style, - buzz_media::config::S3AddressingStyle::Path, + assert!( + matches!( + config.object_store, + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + addressing_style: buzz_object_store::S3AddressingStyle::Path, + .. + }) + ), "S3 addressing must default to path style for bundled MinIO compatibility" ); assert!( @@ -1752,8 +1785,7 @@ mod tests { std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "virtual"); let configured = Config::from_env() .expect("virtual style config") - .media - .s3_addressing_style; + .object_store; std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "auto"); let invalid = Config::from_env(); @@ -1764,7 +1796,13 @@ mod tests { std::env::remove_var("BUZZ_S3_ADDRESSING_STYLE"); } - assert_eq!(configured, buzz_media::config::S3AddressingStyle::Virtual); + assert!(matches!( + configured, + buzz_object_store::ObjectStoreConfig::S3(buzz_object_store::S3StoreConfig { + addressing_style: buzz_object_store::S3AddressingStyle::Virtual, + .. + }) + )); assert!(matches!( invalid, Err(ConfigError::InvalidValue(ref message)) @@ -1772,6 +1810,86 @@ mod tests { )); } + /// Restore an environment variable to whatever it was before a test. + fn restore_env(name: &str, previous: Option) { + match previous { + Some(value) => std::env::set_var(name, value), + None => std::env::remove_var(name), + } + } + + /// A deployment that has never heard of `BUZZ_OBJECT_STORE_PROVIDER` keeps + /// its S3 settings, including the endpoint and credentials that reach the + /// bundled MinIO. + #[test] + fn object_store_defaults_to_the_s3_settings() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous = std::env::var_os("BUZZ_OBJECT_STORE_PROVIDER"); + std::env::remove_var("BUZZ_OBJECT_STORE_PROVIDER"); + + let config = Config::from_env().expect("default config loads"); + + restore_env("BUZZ_OBJECT_STORE_PROVIDER", previous); + + match config.object_store { + buzz_object_store::ObjectStoreConfig::S3(s3) => { + assert_eq!(s3.endpoint, "http://localhost:9000"); + assert_eq!(s3.bucket, "buzz-media"); + assert_eq!(s3.region, "us-east-1"); + assert_eq!( + s3.addressing_style, + buzz_object_store::S3AddressingStyle::Path + ); + } + other => panic!("expected the S3 provider by default, got {other:?}"), + } + } + + /// Selecting Cloud Storage takes its bucket from + /// `BUZZ_OBJECT_STORE_BUCKET`, never from `BUZZ_S3_BUCKET`: a Cloud + /// Storage deployment sets no `BUZZ_S3_*` values, so falling back to that + /// default would address a bucket nobody configured. + #[test] + fn gcs_provider_requires_its_own_bucket() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous_provider = std::env::var_os("BUZZ_OBJECT_STORE_PROVIDER"); + let previous_bucket = std::env::var_os("BUZZ_OBJECT_STORE_BUCKET"); + let previous_s3_style = std::env::var_os("BUZZ_S3_ADDRESSING_STYLE"); + + std::env::set_var("BUZZ_OBJECT_STORE_PROVIDER", "gcs"); + // Provider-native settings from an inactive provider are not parsed. + std::env::set_var("BUZZ_S3_ADDRESSING_STYLE", "not-an-s3-style"); + std::env::remove_var("BUZZ_OBJECT_STORE_BUCKET"); + let without_bucket = Config::from_env(); + + std::env::set_var("BUZZ_OBJECT_STORE_BUCKET", "buzz-objects"); + let with_bucket = Config::from_env(); + + std::env::set_var("BUZZ_OBJECT_STORE_PROVIDER", "minio"); + let unknown_provider = Config::from_env(); + + restore_env("BUZZ_OBJECT_STORE_PROVIDER", previous_provider); + restore_env("BUZZ_OBJECT_STORE_BUCKET", previous_bucket); + restore_env("BUZZ_S3_ADDRESSING_STYLE", previous_s3_style); + + assert!(matches!( + without_bucket, + Err(ConfigError::InvalidValue(ref message)) + if message.contains("BUZZ_OBJECT_STORE_BUCKET must be set") + )); + match with_bucket.expect("gcs config loads").object_store { + buzz_object_store::ObjectStoreConfig::Gcs(gcs) => { + assert_eq!(gcs.bucket, "buzz-objects"); + } + other => panic!("expected the GCS provider, got {other:?}"), + } + assert!(matches!( + unknown_provider, + Err(ConfigError::InvalidValue(ref message)) + if message.contains("must be 's3' or 'gcs'") + )); + } + #[cfg(unix)] #[test] fn s3_addressing_style_env_rejects_non_unicode_values() { diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index ccba40f3282..afa4e97fde6 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -2031,8 +2031,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = - buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, @@ -2084,7 +2083,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (state, audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/handlers/identity_archive.rs b/crates/buzz-relay/src/handlers/identity_archive.rs index 40c54647837..b106c2fc88b 100644 --- a/crates/buzz-relay/src/handlers/identity_archive.rs +++ b/crates/buzz-relay/src/handlers/identity_archive.rs @@ -459,7 +459,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).ok()?; + let media_storage = crate::test_media_storage(&config).ok()?; let (state, _audit_shutdown) = crate::state::AppState::new( config, db, diff --git a/crates/buzz-relay/src/handlers/relay_admin.rs b/crates/buzz-relay/src/handlers/relay_admin.rs index 3782f2c516d..50dac6f2051 100644 --- a/crates/buzz-relay/src/handlers/relay_admin.rs +++ b/crates/buzz-relay/src/handlers/relay_admin.rs @@ -739,7 +739,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index d3416d673c5..1fc5034d485 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -2863,7 +2863,7 @@ const DEFAULT_HEAD: &str = "refs/heads/main"; /// Seed the manifest-pointer for a newly-announced repo with an empty manifest. /// -/// Idempotent: a `CasOutcome::LostRace` is treated as success **only if** the +/// Idempotent: a `ConditionalWrite::Conflict` is treated as success **only if** the /// existing pointer names the same empty manifest digest. Any other pre-existing /// pointer body (e.g. a non-empty manifest from a previous announce/push pair /// for the same `(owner, repo)`) surfaces as an error rather than silently @@ -2875,7 +2875,7 @@ async fn seed_manifest_pointer( repo_id: &str, ) -> anyhow::Result<()> { use crate::api::git::manifest::{pointer_key, Manifest, MANIFEST_VERSION}; - use crate::api::git::store::{CasOutcome, Precond}; + use buzz_object_store::{ConditionalWrite, WriteCondition}; use std::collections::BTreeMap; // The empty manifest. All empty manifests across all repos share canonical @@ -2906,22 +2906,22 @@ async fn seed_manifest_pointer( let pkey = pointer_key(tenant.community(), owner_hex, repo_id); let outcome = state .git_store - .put_pointer(&pkey, digest.as_bytes(), Precond::IfNoneMatchStar) + .put_pointer(&pkey, digest.as_bytes(), WriteCondition::Absent) .await .map_err(|e| anyhow::anyhow!("put_pointer: {e}"))?; match outcome { - CasOutcome::Won(_) => Ok(()), - CasOutcome::LostRace => { + ConditionalWrite::Committed(_) => Ok(()), + ConditionalWrite::Conflict => { // Pointer already exists. Idempotency check: only treat as success // if it names the same empty manifest digest. Any other value is // either a stale pointer from a prior repo lifecycle for the same // (owner, repo) or a real misconfiguration — surface, don't swallow. - let (_etag, body) = state + let (_revision, body) = state .git_store .get_pointer(&pkey) .await - .map_err(|e| anyhow::anyhow!("re-read pointer after LostRace: {e}"))? - .ok_or_else(|| anyhow::anyhow!("pointer vanished after LostRace race"))?; + .map_err(|e| anyhow::anyhow!("re-read pointer after CAS conflict: {e}"))? + .ok_or_else(|| anyhow::anyhow!("pointer vanished after CAS conflict"))?; let existing = std::str::from_utf8(&body) .map_err(|e| anyhow::anyhow!("pointer body not utf-8: {e}"))? .trim(); diff --git a/crates/buzz-relay/src/lib.rs b/crates/buzz-relay/src/lib.rs index 800433a8498..a8afbb60a33 100644 --- a/crates/buzz-relay/src/lib.rs +++ b/crates/buzz-relay/src/lib.rs @@ -55,3 +55,45 @@ pub mod workflow_sink; pub use config::Config; pub use error::{RelayError, Result}; pub use state::AppState; + +/// Build the default S3-backed media facade used by unit-test fixtures. +/// +/// Provider construction belongs here, outside media and Git domain modules. +#[cfg(test)] +pub(crate) fn test_media_storage( + config: &Config, +) -> std::result::Result { + let buzz_object_store::ObjectStoreConfig::S3(s3) = &config.object_store else { + return Err(buzz_object_store::ObjectStoreError::Config( + "synchronous test fixture requires the S3 test provider".to_string(), + )); + }; + let store = buzz_object_store::S3ObjectStore::new(s3)?; + Ok(buzz_media::MediaStorage::with_store(std::sync::Arc::new( + store, + ))) +} + +/// Build an S3-backed Git facade for MinIO/unit-test fixtures. +#[cfg(test)] +pub(crate) fn test_git_store( + endpoint: &str, + access_key: &str, + secret_key: &str, + bucket: &str, + region: &str, + addressing_style: &str, +) -> std::result::Result { + let addressing_style = addressing_style + .parse() + .map_err(buzz_object_store::ObjectStoreError::Config)?; + let store = buzz_object_store::S3ObjectStore::new(&buzz_object_store::S3StoreConfig { + endpoint: endpoint.to_string(), + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + bucket: bucket.to_string(), + region: region.to_string(), + addressing_style, + })?; + Ok(api::git::store::GitStore::new(std::sync::Arc::new(store))) +} diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index d81602e2019..25e225932af 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -449,9 +449,17 @@ async fn main() -> anyhow::Result<()> { .media .validate() .map_err(|e| anyhow::anyhow!("invalid media config: {e}"))?; - let media_storage = buzz_media::MediaStorage::new(&config.media) - .map_err(|e| anyhow::anyhow!("failed to initialize media storage: {e}"))?; - info!("Media storage connected"); + // One provider client per process, shared by the media facade and (via + // `AppState`) the Git facade. Connecting can perform provider admission + // checks — Cloud Storage refuses a bucket whose configuration would break + // deletion — so a misconfigured backend fails startup here rather than + // during the first delete. + let object_store = buzz_object_store::connect(&config.object_store) + .await + .map_err(|e| anyhow::anyhow!("failed to initialize object storage: {e}"))?; + let provider = object_store.provider(); + let media_storage = buzz_media::MediaStorage::with_store(object_store); + info!(%provider, "Object storage connected"); let (app_state, audit_shutdown) = AppState::new( config.clone(), @@ -496,29 +504,36 @@ async fn main() -> anyhow::Result<()> { info!(runtime_id = %runtime_id, "Inter-relay mesh started"); } - // Git-on-object-storage: admit the configured S3/MinIO backend against the + // Git-on-object-storage: admit the configured object store against the // linearizable conditional-write axiom (A3) before serving git traffic. // Failure is fatal: a backend that cannot satisfy pointer CAS invalidates // the manifest-pointer protocol. This is a deployment gate, not a proof. + // + // The profile follows the provider — a wide ETag race for S3, a paced + // generation race for Cloud Storage — so the defaults below are the + // profile's, and the environment only overrides them. if std::env::var("BUZZ_GIT_CONFORMANCE_PROBE") .map(|v| v != "false") .unwrap_or(true) { - let race_width = std::env::var("BUZZ_GIT_PROBE_WRITERS") + let mut cfg = buzz_relay::api::git::store::ProbeConfig::for_provider(provider); + if let Some(race_width) = std::env::var("BUZZ_GIT_PROBE_WRITERS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(32); - let race_rounds = std::env::var("BUZZ_GIT_PROBE_ROUNDS") + { + cfg.race_width = race_width; + } + if let Some(race_rounds) = std::env::var("BUZZ_GIT_PROBE_ROUNDS") .ok() .and_then(|v| v.parse().ok()) - .unwrap_or(3); - let cfg = buzz_relay::api::git::store::ProbeConfig { - race_width, - race_rounds, - }; + { + cfg.race_rounds = race_rounds; + } tracing::info!( - race_width, - race_rounds, + profile = %provider, + race_width = cfg.race_width, + race_rounds = cfg.race_rounds, + same_key_spacing_ms = cfg.same_key_spacing.as_millis() as u64, "running git object-store conformance probe (A3 gate)" ); let report = state @@ -527,9 +542,15 @@ async fn main() -> anyhow::Result<()> { .await .map_err(|e| anyhow::anyhow!("git conformance probe failed: {e}"))?; tracing::info!( + profile = %report.profile, race_width = report.race_width, race_rounds = report.race_rounds, transport_drops = report.transport_drops, + throttled_racers = report.throttled_racers, + throttled_rounds_retried = report.throttled_rounds_retried, + min_same_key_gap_ms = + report.min_same_key_gap.map(|gap| gap.as_millis() as u64), + cleanup_failures = report.cleanup_failures, "git object-store backend admitted: A3 conformance probe passed" ); } diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index dd0fde6fdcd..6d2a9fa6ac5 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -578,7 +578,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index efdb2846148..451fc3b9b00 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -836,15 +836,10 @@ impl AppState { let git_max_concurrent_ops = config.git_max_concurrent_ops; let media_max_concurrent_uploads = config.media_max_concurrent_uploads; - let git_store = crate::api::git::store::GitStore::new( - &config.media.s3_endpoint, - &config.media.s3_access_key, - &config.media.s3_secret_key, - &config.media.s3_bucket, - &config.media.s3_region, - config.media.s3_addressing_style, - ) - .expect("media storage was already constructed with this S3 config"); + // One provider per process: git and media share the client that was + // already constructed for media storage rather than opening a second + // one against the same bucket. + let git_store = crate::api::git::store::GitStore::new(media_storage.object_store()); let git_pack_cache = Arc::new( crate::api::git::pack_cache::GitPackCache::new( &config.git_pack_cache_path, @@ -1446,7 +1441,7 @@ mod tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 8ce23a2e8ea..fdb11fd6a97 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -660,7 +660,7 @@ mod integration_tests { db.clone(), buzz_workflow::WorkflowConfig::default(), )); - let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let media_storage = crate::test_media_storage(&config).expect("media storage"); let (state, _audit_shutdown) = AppState::new( config, db, diff --git a/crates/buzz-test-client/Cargo.toml b/crates/buzz-test-client/Cargo.toml index e495c163004..ddb8efeaef3 100644 --- a/crates/buzz-test-client/Cargo.toml +++ b/crates/buzz-test-client/Cargo.toml @@ -37,6 +37,7 @@ sqlx = { workspace = true } chrono = { workspace = true } s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } buzz-media = { workspace = true } +buzz-object-store = { workspace = true } buzz-sdk = { workspace = true } [[bin]] diff --git a/crates/buzz-test-client/tests/e2e_git.rs b/crates/buzz-test-client/tests/e2e_git.rs index 3c82e317649..6d0fdac17b2 100644 --- a/crates/buzz-test-client/tests/e2e_git.rs +++ b/crates/buzz-test-client/tests/e2e_git.rs @@ -21,7 +21,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; -use buzz_media::S3AddressingStyle; +use buzz_object_store::S3AddressingStyle; use nostr::{EventBuilder, Keys, Kind, Tag}; use s3::creds::Credentials; use s3::{Bucket, Region}; diff --git a/docs/git-on-object-storage.md b/docs/git-on-object-storage.md index 1d87720edc7..1bce2445184 100644 --- a/docs/git-on-object-storage.md +++ b/docs/git-on-object-storage.md @@ -5,7 +5,7 @@ ## Abstract This document specifies a protocol for hosting git repositories on an object -store (S3 and S3-compatible backends such as MinIO) with **no persistent +store (currently Google Cloud Storage, with an AWS S3 adapter) with **no persistent filesystem**, and gives a formal proof of its safety properties. Repository content is stored as create-only, content-addressed pack objects (immutable by protocol discipline — see A1, not by assuming an immutable store); the current @@ -65,6 +65,39 @@ local. Cache misses, restarts, and evictions only affect performance; object storage remains the source of truth, and per-request refs/HEAD are still materialized from the current manifest. +### Provider portability and cutover + +Object storage is a deployment choice, not a media or Git domain choice. +`crates/buzz-object-store::ObjectStore` is the only interface those domains +consume. The relay constructs one provider at composition time and shares it +between `MediaStorage` and `GitStore`; `BUZZ_OBJECT_STORE_PROVIDER` selects +`gcs` or `s3`. Provider-native vocabulary is confined to its adapter: + +- GCS generations implement revisions, conditional writes, version listing, + and exact-version deletion; Application Default Credentials provide identity. +- S3 ETags/version IDs implement the same contract; the AWS credential chain + provides identity when static interoperability credentials are absent. + +The current Mozart deployment selects GCS. Moving Buzz to AWS does not require +changes in media, Git, deletion, or key layout. It is an explicit data cutover: + +1. provision the target bucket with the deletion contract satisfied (no object + versioning or recoverable soft-delete retention), and grant only the Buzz + runtime identity; +2. run the target adapter's conformance profile before serving traffic; +3. drain writers, copy every key and byte exactly, and prove source/target key + stream digests and byte totals agree for both tenant and Git prefixes; +4. switch `BUZZ_OBJECT_STORE_PROVIDER` and its provider configuration in the + deployment layer, then prove media reads, Git hydrate/clone, CAS publication, + version enumeration, and exact deletion on the target; +5. retain the source read-only for the rollback window. A rollback after target + writes requires a reverse delta copy before switching; never point two live + writers at different buckets. + +The bucket name is configuration, never embedded in object keys or domain +records. This keeps a future GCS-to-S3 move a copy-and-composition change rather +than a schema or application rewrite. + The accepted v1 tradeoff: under concurrent same-repo pushes, every contender hydrates and runs receive-pack, and the CAS losers' subprocess work is discarded. This is wasted CPU/IO under contention, not a correctness bug — @@ -368,6 +401,51 @@ conditional writes cannot admit a backend against it. a quote mismatch between the read path and the write path silently tests the wrong thing. The probe must use the exact token format the pointer write uses. +### Provider profiles + +The axioms are the same for every backend; the *evidence* that admits one is +provider-shaped, so the probe runs the profile the configured provider needs. + +The four phases above are the **S3 profile**: a wide race (default 32 writers × +3 rounds) on revision-token preconditions, with unclassified transport failures +dropped from the observer set and a floor of two classified observers per round. + +A backend that publishes a per-object write ceiling needs a different shape. +Google Cloud Storage documents a maximum of one write per second to a single +object name and answers an over-rate write with `429` — a refusal to evaluate +the precondition at all, which is categorically not a lost race. Running the S3 +profile against it would measure the rate limiter rather than the store: a burst +of `429`s would either be scored as losers, making a correctly paced backend look +like one admitting lost updates, or would leave a round with no race in it. The +**Cloud Storage profile** therefore races narrowly (default 3 × 2), spaces +same-key rounds past the published ceiling, and adds the phases that only exist +where the revision is an object generation: + +1. create-only content-addressed write, read back, digest verified; +2. pointer creation under the create-only precondition (generation `0`); +3. body and generation read from one response, checked against the commit; +4. compare-and-swap on the observed generation, which must report a new one; +5. replay of the superseded generation, which must conflict; +6. a race on one generation, repeated, with the stored object required to equal + the winner's; +7. the winning generation predicating the next successful write. + +Three rules stop pacing from becoming leniency: + +- **Two committed racers is always fatal.** No throttle, drop, or retry budget + can explain it. +- **A commit reported without a generation is fatal**, wherever it appears. The + caller would have nothing to predicate its next write on, and dropping the + precondition turns the pointer swap into a blind overwrite. +- **A round that proves nothing is re-run, not scored.** Every racer throttled, + or too few classified to have witnessed a race, is neither pass nor fail; + the round is re-run within a bounded budget, and exhausting it fails the probe. + Conflicts with no acknowledged winner stay fatal — that is a lost update + announcing itself. + +Both profiles remain fail-closed deployment gates, and both remove their own +objects afterwards on the success and failure paths alike. + **Proof surface (explicit non-goals of the probe and the design).** The protocol depends only on conditional writes of *small single objects* (the manifest pointer). It does **not** depend on, and the probe does **not** test: @@ -413,6 +491,18 @@ knowledge, new. ## Implementation Correspondence +> **A note on vocabulary.** This specification states A1 and A3 in S3 terms — +> ETags, `If-Match`, `If-None-Match: *` — because that is the backend the +> axioms were first admitted against. The implementation states them in +> provider-neutral terms: `crates/buzz-object-store` defines an opaque +> `Revision` (the CAS token), `WriteCondition::{Absent, Matches}` (the +> precondition), and `ConditionalWrite::{Committed, Conflict}` (the outcome). +> An ETag is one inhabitant of `Revision` and exists only inside the S3 +> provider. The mapping is exact — `Absent` is `If-None-Match: *`, `Matches(e)` +> is `If-Match: e`, `Conflict` is the 412 — so every theorem below transfers +> unchanged; a backend is admitted by the §Conformance probe, not by speaking +> S3's header vocabulary. + The fence (Theorem 1) maps to a single structural obligation on the implementation, stated here as a requirement the code must meet for the proof to transfer: @@ -434,7 +524,7 @@ transfer: - **Parent observed once.** `hydrate_for_write` reads the pointer, fetches and verifies the parent manifest, materializes the workspace from it, and returns a `(HydratedRepo, ParentState)` pair where `ParentState` carries the exact - `(ETag, digest, Manifest)` triple the workspace was hydrated against. That + `(Revision, digest, Manifest)` triple the workspace was hydrated against. That same `ParentState` rides on the `PushContext` through receive-pack, and `cas_publish` predicates the CAS on `parent_state.if_match` — it never re-reads the pointer. The "build on `d_old`, publish against `d_new`" hazard is closed diff --git a/migrations/0041_function_search_paths.sql b/migrations/0041_function_search_paths.sql new file mode 100644 index 00000000000..2df99143b68 --- /dev/null +++ b/migrations/0041_function_search_paths.sql @@ -0,0 +1,36 @@ +-- PostgreSQL restore sessions deliberately clear search_path. Buzz trigger +-- functions call other Buzz functions and tables by their unqualified names, +-- so data-only restores must not inherit name resolution from the invoker. +-- Pin every application-owned public function while leaving extension-owned +-- and provider-owned functions untouched. + +SET LOCAL search_path TO public, pg_catalog; + +DO $$ +DECLARE + function_identity REGPROCEDURE; +BEGIN + FOR function_identity IN + SELECT procedure.oid::REGPROCEDURE + FROM pg_proc AS procedure + JOIN pg_namespace AS namespace + ON namespace.oid = procedure.pronamespace + WHERE namespace.nspname = 'public' + AND procedure.prokind = 'f' + AND procedure.proowner = (SELECT oid FROM pg_roles WHERE rolname = current_user) + AND NOT EXISTS ( + SELECT 1 + FROM pg_depend AS dependency + WHERE dependency.classid = 'pg_proc'::REGCLASS + AND dependency.objid = procedure.oid + AND dependency.deptype = 'e' + ) + ORDER BY procedure.oid + LOOP + EXECUTE format( + 'ALTER FUNCTION %s SET search_path TO public, pg_catalog', + function_identity + ); + END LOOP; +END +$$;