diff --git a/Cargo.lock b/Cargo.lock index d659a6a7f8..04126bb859 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -157,7 +157,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -168,7 +168,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -187,7 +187,7 @@ dependencies = [ "bon", "bzip2", "crc32fast", - "digest", + "digest 0.10.7", "liblzma", "log", "miniz_oxide", @@ -552,9 +552,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.15" +version = "1.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +checksum = "50f156acdd2cf55f5aa53ee416c4ac851cf1222694506c0b1f78c85695e9ca9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -616,9 +616,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +checksum = "5dcd93c82209ac7413532388067dce79be5a8780c1786e5fae3df22e4dee2864" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -641,9 +641,9 @@ dependencies = [ [[package]] name = "aws-sdk-glue" -version = "1.142.0" +version = "1.144.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3962675ec1f2012ae6439814e784557550fa239a4a291bd4f33d8f514d4fdb5b" +checksum = "24165072a1fd89118365d686e569b96b8376a1abfb2f312bd6755fbe3e74b9dc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -689,9 +689,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.97.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +checksum = "d69c77aafa20460c68b6b3213c84f6423b6e76dbf89accd3e1789a686ffd9489" dependencies = [ "aws-credential-types", "aws-runtime", @@ -713,9 +713,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.99.0" +version = "1.100.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +checksum = "1c7e7b09346d5ca22a2a08267555843a6a0127fb20d8964cb6ecfb8fdb190225" dependencies = [ "aws-credential-types", "aws-runtime", @@ -737,9 +737,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.101.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +checksum = "c2249b81a2e73a8027c41c378463a81ec39b8510f184f2caab87de912af0f49b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -762,9 +762,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +checksum = "68dc0b907359b120170613b5c09ccc61304eac3998ff6274b97d93ee6490115a" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -773,11 +773,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] @@ -868,9 +868,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.10.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -893,11 +893,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.11.6" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -908,6 +909,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "aws-smithy-types" version = "1.4.7" @@ -945,9 +957,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.14" +version = "1.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" +checksum = "2f4bbcaa9304ea40902d3d5f42a0428d1bd895a2b0f6999436fb279ffddc58ac" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1025,7 +1037,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1051,6 +1063,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1086,7 +1107,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1250,7 +1271,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1312,6 +1333,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1324,7 +1351,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1385,6 +1412,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1528,6 +1561,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1558,6 +1600,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.20.11" @@ -2060,7 +2111,7 @@ dependencies = [ "num-traits", "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2361,7 +2412,7 @@ dependencies = [ "rand 0.9.4", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "url", ] @@ -2436,7 +2487,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -2494,12 +2545,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2518,7 +2581,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2671,7 +2734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3150,7 +3213,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -3159,7 +3222,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", ] [[package]] @@ -3244,6 +3316,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -3302,7 +3383,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -3409,6 +3490,7 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-glue", + "aws-sdk-sts", "iceberg", "iceberg-storage-opendal", "iceberg_test_utils", @@ -3873,7 +3955,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4173,7 +4255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -4422,7 +4504,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4783,8 +4865,8 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -4918,7 +5000,7 @@ dependencies = [ "der", "pbkdf2", "scrypt", - "sha2", + "sha2 0.10.9", "spki", ] @@ -5078,7 +5160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5097,7 +5179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -5191,7 +5273,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -5228,7 +5310,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -5490,7 +5572,7 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.17", "hex", - "hmac", + "hmac 0.12.1", "home", "http 1.4.0", "jsonwebtoken", @@ -5505,7 +5587,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -5611,15 +5693,15 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -5693,7 +5775,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5868,7 +5950,7 @@ checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6094,7 +6176,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -6105,7 +6187,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -6139,7 +6232,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -6217,7 +6310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6363,7 +6456,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror 2.0.18", "tokio", @@ -6401,7 +6494,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", "syn", @@ -6421,7 +6514,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -6431,7 +6524,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5", @@ -6441,7 +6534,7 @@ dependencies = [ "rand 0.8.5", "rsa", "sha1", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -6468,7 +6561,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -6478,7 +6571,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -6656,7 +6749,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7067,9 +7160,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "typetag" @@ -7205,7 +7298,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -7552,7 +7645,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7f612c44bf..7380fef9d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,8 +55,9 @@ arrow-select = "58" arrow-string = "58" as-any = "0.3.2" async-trait = "0.1.89" -aws-config = "1.8.7" -aws-sdk-glue = { version = "1.85", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-config = "1.8.16" +aws-sdk-glue = { version = "1.144", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-sdk-sts = { version = "1.101", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-sdk-s3tables = { version = "1.28", default-features = false, features = ["default-https-client", "rt-tokio"] } backon = "1.5.1" base64 = "0.22.1" diff --git a/crates/catalog/glue/Cargo.toml b/crates/catalog/glue/Cargo.toml index d8d0927a90..cce00b15cd 100644 --- a/crates/catalog/glue/Cargo.toml +++ b/crates/catalog/glue/Cargo.toml @@ -33,10 +33,14 @@ anyhow = { workspace = true } async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-glue = { workspace = true } +aws-sdk-sts = { workspace = true } iceberg = { workspace = true } iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } serde_json = { workspace = true } tokio = { workspace = true } +[package.metadata.cargo-machete] +ignored = ["aws-sdk-sts"] + [dev-dependencies] iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index 5b3ccf3b39..87554ca0d1 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -26,8 +26,9 @@ use aws_sdk_glue::operation::create_table::CreateTableError; use aws_sdk_glue::operation::update_table::UpdateTableError; use aws_sdk_glue::types::TableInput; use iceberg::io::{ - FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, - S3_SESSION_TOKEN, StorageFactory, + FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ASSUME_ROLE_ARN, S3_ASSUME_ROLE_EXTERNAL_ID, + S3_ASSUME_ROLE_SESSION_NAME, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, + StorageFactory, }; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; @@ -43,7 +44,9 @@ use crate::utils::{ get_default_table_location, get_metadata_location, validate_namespace, }; use crate::{ - AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, with_catalog_id, + AWS_ACCESS_KEY_ID, AWS_ASSUME_ROLE_ARN, AWS_ASSUME_ROLE_EXTERNAL_ID, + AWS_ASSUME_ROLE_SESSION_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + with_catalog_id, }; /// Glue catalog URI @@ -197,6 +200,30 @@ impl GlueCatalog { { file_io_props.insert(S3_ENDPOINT.to_string(), aws_endpoint.to_string()); } + // Propagate STS assume-role properties so that the S3 FileIO also + // operates under the assumed role (using the same property keys that + // the S3 storage config already understands). + if !file_io_props.contains_key(S3_ASSUME_ROLE_ARN) + && let Some(role_arn) = file_io_props.get(AWS_ASSUME_ROLE_ARN) + { + file_io_props.insert(S3_ASSUME_ROLE_ARN.to_string(), role_arn.to_string()); + } + if !file_io_props.contains_key(S3_ASSUME_ROLE_EXTERNAL_ID) + && let Some(external_id) = file_io_props.get(AWS_ASSUME_ROLE_EXTERNAL_ID) + { + file_io_props.insert( + S3_ASSUME_ROLE_EXTERNAL_ID.to_string(), + external_id.to_string(), + ); + } + if !file_io_props.contains_key(S3_ASSUME_ROLE_SESSION_NAME) + && let Some(session_name) = file_io_props.get(AWS_ASSUME_ROLE_SESSION_NAME) + { + file_io_props.insert( + S3_ASSUME_ROLE_SESSION_NAME.to_string(), + session_name.to_string(), + ); + } let client = aws_sdk_glue::Client::new(&sdk_config); diff --git a/crates/catalog/glue/src/lib.rs b/crates/catalog/glue/src/lib.rs index 1b9efe3770..c3e0663135 100644 --- a/crates/catalog/glue/src/lib.rs +++ b/crates/catalog/glue/src/lib.rs @@ -49,5 +49,7 @@ mod schema; mod utils; pub use catalog::*; pub use utils::{ - AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + AWS_ACCESS_KEY_ID, AWS_ASSUME_ROLE_ARN, AWS_ASSUME_ROLE_EXTERNAL_ID, + AWS_ASSUME_ROLE_SESSION_NAME, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN, }; diff --git a/crates/catalog/glue/src/utils.rs b/crates/catalog/glue/src/utils.rs index 906e6fcc18..4ba961be65 100644 --- a/crates/catalog/glue/src/utils.rs +++ b/crates/catalog/glue/src/utils.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; +use aws_config::sts::AssumeRoleProvider; use aws_config::{BehaviorVersion, Region, SdkConfig}; use aws_sdk_glue::config::Credentials; use aws_sdk_glue::types::{Database, DatabaseInput, StorageDescriptor, TableInput}; @@ -36,6 +37,15 @@ pub const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; pub const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; /// Property aws session token pub const AWS_SESSION_TOKEN: &str = "aws_session_token"; +/// Property for the IAM role ARN to assume when accessing AWS services. +/// When set, the catalog will use AWS STS to assume the specified role, +/// replacing the default credential chain for both the Glue client and S3 file I/O. +pub const AWS_ASSUME_ROLE_ARN: &str = "client.assume-role.arn"; +/// Optional external ID used when assuming an IAM role (for cross-account access). +pub const AWS_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id"; +/// Optional session name used when assuming an IAM role. +/// Defaults to `"iceberg-glue-catalog"` if not specified. +pub const AWS_ASSUME_ROLE_SESSION_NAME: &str = "client.assume-role.session-name"; /// Parameter namespace description const DESCRIPTION: &str = "description"; /// Parameter namespace location uri @@ -53,6 +63,11 @@ const ICEBERG: &str = "ICEBERG"; /// Creates an aws sdk configuration based on /// provided properties and an optional endpoint URL. +/// +/// When `client.assume-role.arn` is set, the function first builds a base +/// configuration using any static credentials or profile provided, then uses +/// AWS STS to assume the specified role. The resulting temporary credentials +/// are used for all subsequent AWS API calls (both Glue and S3). pub(crate) async fn create_sdk_config( properties: &HashMap, endpoint_uri: Option<&String>, @@ -87,6 +102,39 @@ pub(crate) async fn create_sdk_config( config = config.region(region); } + // If a role ARN is provided, assume that role via STS and use the + // resulting temporary credentials for all AWS SDK calls. + if let Some(role_arn) = properties.get(AWS_ASSUME_ROLE_ARN) { + let base_config = config.load().await; + + let session_name = properties + .get(AWS_ASSUME_ROLE_SESSION_NAME) + .cloned() + .unwrap_or_else(|| "iceberg-glue-catalog".to_string()); + + let mut assume_role_builder = + AssumeRoleProvider::builder(role_arn).session_name(session_name); + + if let Some(external_id) = properties.get(AWS_ASSUME_ROLE_EXTERNAL_ID) { + assume_role_builder = assume_role_builder.external_id(external_id); + } + + let assume_role_provider = assume_role_builder + .build_from_provider( + base_config + .credentials_provider() + .expect("base credentials provider must be set for STS assume role"), + ) + .await; + + return aws_config::defaults(BehaviorVersion::latest()) + .credentials_provider(assume_role_provider) + .region(base_config.region().cloned()) + .endpoint_url(endpoint_uri.map(|s| s.as_str()).unwrap_or_default()) + .load() + .await; + } + config.load().await } @@ -267,6 +315,126 @@ mod tests { use super::*; use crate::schema::{ICEBERG_FIELD_CURRENT, ICEBERG_FIELD_ID, ICEBERG_FIELD_OPTIONAL}; + // ── STS / assume-role tests ────────────────────────────────────────────── + + /// Verify that supplying a role ARN causes `create_sdk_config` to install a + /// credentials provider (the `AssumeRoleProvider`) in the returned config. + /// We do NOT actually call STS here – the provider is wired up but lazy. + #[tokio::test] + async fn test_config_with_assume_role_arn() { + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "test-access-key".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "test-secret-key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + ( + AWS_ASSUME_ROLE_ARN.to_string(), + "arn:aws:iam::123456789012:role/MyRole".to_string(), + ), + ]); + + let sdk_config = create_sdk_config(&properties, None).await; + + // A credentials provider must be present – it wraps the AssumeRoleProvider. + assert!( + sdk_config.credentials_provider().is_some(), + "expected a credentials provider to be installed when assume-role ARN is set" + ); + assert_eq!(sdk_config.region().map(|r| r.as_ref()), Some("us-east-1")); + } + + /// When a custom session name is provided it should be used instead of the + /// default `"iceberg-glue-catalog"`. We verify config construction does not + /// panic and that the provider is still present. + #[tokio::test] + async fn test_config_assume_role_with_custom_session_name() { + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "test-access-key".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "test-secret-key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "eu-west-1".to_string()), + ( + AWS_ASSUME_ROLE_ARN.to_string(), + "arn:aws:iam::123456789012:role/MyRole".to_string(), + ), + ( + AWS_ASSUME_ROLE_SESSION_NAME.to_string(), + "my-custom-session".to_string(), + ), + ]); + + let sdk_config = create_sdk_config(&properties, None).await; + + assert!(sdk_config.credentials_provider().is_some()); + assert_eq!(sdk_config.region().map(|r| r.as_ref()), Some("eu-west-1")); + } + + /// All three optional fields (session name + external ID + endpoint) should + /// be wired without panicking. + #[tokio::test] + async fn test_config_assume_role_with_all_optional_fields() { + let endpoint = "http://localhost:4566"; + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "test-access-key".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "test-secret-key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-west-2".to_string()), + ( + AWS_ASSUME_ROLE_ARN.to_string(), + "arn:aws:iam::123456789012:role/MyRole".to_string(), + ), + ( + AWS_ASSUME_ROLE_SESSION_NAME.to_string(), + "my-session".to_string(), + ), + ( + AWS_ASSUME_ROLE_EXTERNAL_ID.to_string(), + "my-external-id".to_string(), + ), + ]); + + let sdk_config = create_sdk_config(&properties, Some(&endpoint.to_string())).await; + + assert!(sdk_config.credentials_provider().is_some()); + assert_eq!(sdk_config.region().map(|r| r.as_ref()), Some("us-west-2")); + // Endpoint should be propagated through from the final config rebuild. + assert_eq!(sdk_config.endpoint_url(), Some(endpoint)); + } + + /// Without a role ARN the existing static-credential path must still work + /// exactly as before (regression guard). + #[tokio::test] + async fn test_config_without_assume_role_unchanged() { + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "my-access-id".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "my-secret-key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + ]); + + let sdk_config = create_sdk_config(&properties, None).await; + + let region = sdk_config.region().unwrap().as_ref(); + let credentials = sdk_config + .credentials_provider() + .unwrap() + .provide_credentials() + .await + .unwrap(); + + assert_eq!(region, "us-east-1"); + assert_eq!(credentials.access_key_id(), "my-access-id"); + assert_eq!(credentials.secret_access_key(), "my-secret-key"); + } + fn create_metadata(schema: Schema) -> Result { let table_creation = TableCreation::builder() .name("my_table".to_string()) diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs index 1d40fef357..bec964f1ec 100644 --- a/crates/catalog/loader/tests/common/mod.rs +++ b/crates/catalog/loader/tests/common/mod.rs @@ -31,7 +31,8 @@ use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; use iceberg_catalog_glue::{ - AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, + AWS_ACCESS_KEY_ID, AWS_ASSUME_ROLE_ARN, AWS_ASSUME_ROLE_EXTERNAL_ID, + AWS_ASSUME_ROLE_SESSION_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, }; use iceberg_catalog_hms::{ @@ -262,6 +263,60 @@ async fn glue_catalog() -> GlueCatalog { .unwrap() } +/// Build a [`GlueCatalog`] that authenticates via STS AssumeRole. +/// +/// Requires the following environment variables to be set: +/// - `GLUE_ENDPOINT` / `MINIO_ENDPOINT` (set by the test harness) +/// - `GLUE_ASSUME_ROLE_ARN` – the IAM role ARN to assume +/// +/// Optionally: +/// - `GLUE_ASSUME_ROLE_EXTERNAL_ID` – cross-account external ID +/// - `GLUE_ASSUME_ROLE_SESSION_NAME` – STS session name (default: `"iceberg-glue-catalog"`) +/// +/// This function is intended to be called from `#[ignore]` integration tests +/// that require real AWS credentials and a live Glue endpoint. +#[allow(dead_code)] +pub async fn glue_catalog_assume_role() -> GlueCatalog { + let glue_endpoint = get_glue_endpoint(); + let minio_endpoint = get_minio_endpoint(); + + let role_arn = std::env::var("GLUE_ASSUME_ROLE_ARN") + .expect("GLUE_ASSUME_ROLE_ARN must be set for assume-role integration tests"); + + let mut props = HashMap::from([ + (GLUE_CATALOG_PROP_URI.to_string(), glue_endpoint), + ( + GLUE_CATALOG_PROP_WAREHOUSE.to_string(), + "s3a://warehouse/hive".to_string(), + ), + // Base credentials used to call STS AssumeRole. + (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "my_secret_key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + // STS assume-role configuration. + (AWS_ASSUME_ROLE_ARN.to_string(), role_arn), + // S3 / MinIO endpoint for file I/O. + (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + ]); + + if let Ok(external_id) = std::env::var("GLUE_ASSUME_ROLE_EXTERNAL_ID") { + props.insert(AWS_ASSUME_ROLE_EXTERNAL_ID.to_string(), external_id); + } + if let Ok(session_name) = std::env::var("GLUE_ASSUME_ROLE_SESSION_NAME") { + props.insert(AWS_ASSUME_ROLE_SESSION_NAME.to_string(), session_name); + } + + GlueCatalogBuilder::default() + .load("glue-assume-role", props) + .await + .unwrap() +} + async fn hms_catalog() -> HmsCatalog { let hms_endpoint = get_hms_endpoint(); let minio_endpoint = get_minio_endpoint();