diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 5c904b319d..ae3a6d1a54 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -91,6 +91,6 @@ jobs: shell: bash run: | set -e - pip install hatch==1.12.0 + pip install hatch==1.14.2 hatch run dev:pip install dist/pyiceberg_core-*.whl --force-reinstall hatch run dev:test diff --git a/Cargo.lock b/Cargo.lock index aa30a87e89..846086f7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,12 +17,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "aes" version = "0.8.4" @@ -157,42 +151,16 @@ checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "apache-avro" -version = "0.17.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" -dependencies = [ - "bigdecimal", - "bzip2 0.4.4", - "crc32fast", - "digest", - "libflate", - "log", - "num-bigint", - "quad-rand", - "rand 0.8.5", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.26.3", - "strum_macros 0.26.4", - "thiserror 1.0.69", - "typed-builder 0.19.1", - "uuid", - "xz2", - "zstd", -] - -[[package]] -name = "apache-avro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", + "bzip2", + "crc32fast", "digest", + "liblzma", "log", "miniz_oxide", "num-bigint", @@ -202,9 +170,10 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum 0.27.2", "strum_macros 0.27.2", - "thiserror 2.0.16", + "thiserror 2.0.17", "uuid", "zstd", ] @@ -229,9 +198,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "c26b57282a08ae92f727497805122fec964c6245cfa0e13f0e75452eaf3bc41f" dependencies = [ "arrow-arith", "arrow-array", @@ -250,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "cebf38ca279120ff522f4954b81a39527425b6e9f615e6b72842f4de1ffe02b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "744109142cdf8e7b02795e240e20756c2a782ac9180d4992802954a8f871c0de" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -281,9 +250,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "601bb103c4c374bcd1f62c66bcea67b42a2ee91a690486c37d4c180236f11ccc" dependencies = [ "bytes", "half", @@ -292,9 +261,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "eed61d9d73eda8df9e3014843def37af3050b5080a9acbe108f045a316d5a0be" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" +checksum = "fa95b96ce0c06b4d33ac958370db8c0d31e88e54f9d6e08b0353d18374d9f991" dependencies = [ "arrow-array", "arrow-cast", @@ -328,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "43407f2c6ba2367f64d85d4603d6fb9c4b92ed79d2ffd21021b37efa96523e12" dependencies = [ "arrow-buffer", "arrow-schema", @@ -340,23 +309,25 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "e4b0487c4d2ad121cbc42c4db204f1509f8618e589bc77e635e9c40b502e3b90" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] name = "arrow-json" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" +checksum = "26d747573390905905a2dc4c5a61a96163fe2750457f90a04ee2a88680758c79" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "c142a147dceb59d057bad82400f1693847c80dca870d008bf7b91caf902810ae" dependencies = [ "arrow-array", "arrow-buffer", @@ -389,9 +360,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "dac6620667fccdab4204689ca173bd84a15de6bb6b756c3a8764d4d7d0c2fc04" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,9 +373,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "dfa93af9ff2bb80de539e6eb2c1c8764abd0f4b73ffb0d7c82bf1f9868785e66" dependencies = [ "serde", "serde_json", @@ -412,9 +383,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "be8b2e0052cd20d36d64f32640b68a5ab54d805d24a473baee5d52017c85536c" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -426,9 +397,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "c2155e26e17f053c8975c546fc70cf19c00542f9abf43c23a88a46ef7204204f" dependencies = [ "arrow-array", "arrow-buffer", @@ -483,19 +454,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2" dependencies = [ - "bzip2 0.5.2", - "flate2", + "compression-codecs", + "compression-core", "futures-core", - "memchr", "pin-project-lite", "tokio", - "xz2", - "zstd", - "zstd-safe", ] [[package]] @@ -1105,9 +1072,9 @@ checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -1222,9 +1189,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2529c31017402be841eb45892278a6c21a000c0a17643af326c73a73f83f0fb" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" dependencies = [ "bon-macros", "rustversion", @@ -1232,9 +1199,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.7.2" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82020dadcb845a345591863adb65d74fa8dc5c18a0b6d408470e13b7adc7005" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", @@ -1360,31 +1327,11 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] @@ -1571,14 +1518,36 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.0" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "unicode-segmentation", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width 0.2.1", ] +[[package]] +name = "compression-codecs" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9b614a5787ef0c8802a55766480563cb3a93b435898c422ed2a359cf811582" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1590,15 +1559,15 @@ dependencies = [ [[package]] name = "console" -version = "0.15.11" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" dependencies = [ "encode_unicode", "libc", "once_cell", "unicode-width 0.2.1", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -1659,15 +1628,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -1870,12 +1830,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" - [[package]] name = "dashmap" version = "6.1.0" @@ -1892,16 +1846,15 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1922,6 +1875,7 @@ dependencies = [ "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", @@ -1929,7 +1883,9 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "hex", "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot", @@ -1941,15 +1897,13 @@ dependencies = [ "tokio", "url", "uuid", - "xz2", "zstd", ] [[package]] name = "datafusion-catalog" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -1973,9 +1927,8 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -1996,9 +1949,8 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85591b54815b0c9d3fbf3b0425b75e9fc49ef73e6886d6c19d622986e2551b53" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2009,6 +1961,7 @@ dependencies = [ "dirs", "env_logger", "futures", + "log", "mimalloc", "object_store", "parking_lot", @@ -2021,17 +1974,18 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", - "apache-avro 0.17.0", + "apache-avro", "arrow", "arrow-ipc", "base64 0.22.1", + "chrono", "half", "hashbrown 0.14.5", + "hex", "indexmap 2.11.0", "libc", "log", @@ -2046,9 +2000,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "futures", "log", @@ -2057,21 +2010,21 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.5.2", + "bzip2", "chrono", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", @@ -2079,6 +2032,7 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", "object_store", "parquet", @@ -2087,17 +2041,15 @@ dependencies = [ "tokio", "tokio-util", "url", - "xz2", "zstd", ] [[package]] name = "datafusion-datasource-avro" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de733d231abb0fba663ff60fd37bf7171fa8b2e46e8a99e41362001821d116e" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ - "apache-avro 0.17.0", + "apache-avro", "arrow", "async-trait", "bytes", @@ -2118,9 +2070,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2143,9 +2094,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2168,9 +2118,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2183,11 +2132,14 @@ dependencies = [ "datafusion-expr", "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", + "hex", "itertools 0.14.0", "log", "object_store", @@ -2199,17 +2151,16 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" [[package]] name = "datafusion-execution" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", + "async-trait", "dashmap", "datafusion-common", "datafusion-expr", @@ -2217,6 +2168,7 @@ dependencies = [ "log", "object_store", "parking_lot", + "parquet", "rand 0.9.2", "tempfile", "url", @@ -2224,11 +2176,11 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", + "async-trait", "chrono", "datafusion-common", "datafusion-doc", @@ -2245,9 +2197,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2258,9 +2209,8 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-buffer", @@ -2287,9 +2237,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2308,9 +2257,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2321,9 +2269,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "arrow-ord", @@ -2333,6 +2280,7 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", "itertools 0.14.0", @@ -2342,9 +2290,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2358,9 +2305,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2376,9 +2322,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2386,9 +2331,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "datafusion-expr", "quote", @@ -2397,14 +2341,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.11.0", "itertools 0.14.0", @@ -2416,9 +2360,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2432,15 +2375,30 @@ dependencies = [ "indexmap 2.11.0", "itertools 0.14.0", "log", + "parking_lot", "paste", - "petgraph 0.8.2", + "petgraph 0.8.3", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2452,9 +2410,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "datafusion-common", @@ -2464,6 +2421,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", "recursive", @@ -2471,9 +2429,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "ahash 0.8.12", "arrow", @@ -2485,6 +2442,7 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -2499,11 +2457,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-pruning" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2525,11 +2499,12 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f469b1301a5a2dec8b944eacb7b1194ed8c0f8203ce460b7451bb6fa2e89060" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", + "chrono", + "crc32fast", "datafusion-catalog", "datafusion-common", "datafusion-execution", @@ -2537,13 +2512,15 @@ dependencies = [ "datafusion-functions", "datafusion-macros", "log", + "sha1", + "url", + "xxhash-rust", ] [[package]] name = "datafusion-sql" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "bigdecimal", @@ -2558,9 +2535,8 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a05ab096d3dd4c234f4ecf3c9c08b71396ac1810d4d57d3a13049bbf8a1b30" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "arrow", "async-trait", @@ -2579,15 +2555,14 @@ dependencies = [ "sqllogictest", "sqlparser", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "datafusion-substrait" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6029c08002772fb2c38a191d21a8b03af78750f42d9b896222f6b0e16c84cf" +version = "50.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=4cc47d3418ee7dfe8f82394d356560401b950d63#4cc47d3418ee7dfe8f82394d356560401b950d63" dependencies = [ "async-recursion", "async-trait", @@ -2838,7 +2813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3596,7 +3571,7 @@ name = "iceberg" version = "0.7.0" dependencies = [ "anyhow", - "apache-avro 0.20.0", + "apache-avro", "array-init", "arrow-arith", "arrow-array", @@ -3647,7 +3622,7 @@ dependencies = [ "tera", "thrift", "tokio", - "typed-builder 0.20.1", + "typed-builder", "url", "uuid", "zstd", @@ -3734,7 +3709,7 @@ dependencies = [ "serde_json", "tokio", "tracing", - "typed-builder 0.20.1", + "typed-builder", "uuid", ] @@ -3763,7 +3738,7 @@ dependencies = [ "sqlx", "tempfile", "tokio", - "typed-builder 0.20.1", + "typed-builder", ] [[package]] @@ -4010,14 +3985,14 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.11" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +checksum = "ade6dfcba0dfb62ad59e59e7241ec8912af34fd29e0e743e3db992bd278e8b65" dependencies = [ "console", - "number_prefix", "portable-atomic", "unicode-width 0.2.1", + "unit-prefix", "web-time", ] @@ -4251,6 +4226,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.175" @@ -4258,37 +4239,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] -name = "libflate" -version = "2.1.0" +name = "libloading" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", + "cfg-if", + "windows-targets 0.53.3", ] [[package]] -name = "libflate_lz77" -version = "2.1.0" +name = "liblzma" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", + "liblzma-sys", ] [[package]] -name = "libloading" -version = "0.8.8" +name = "liblzma-sys" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" dependencies = [ - "cfg-if", - "windows-targets 0.48.5", + "cc", + "libc", + "pkg-config", ] [[package]] @@ -4429,17 +4406,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.2.0" @@ -4838,12 +4804,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "object" version = "0.36.7" @@ -4881,7 +4841,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -5015,9 +4975,9 @@ dependencies = [ [[package]] name = "parquet" -version = "55.2.0" +version = "56.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "89b56b41d1bd36aae415e42f91cae70ee75cf6cba74416b14dce3e958d5990ec" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -5040,6 +5000,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -5143,7 +5104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.16", + "thiserror 2.0.17", "ucd-trie", ] @@ -5192,9 +5153,9 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", @@ -5660,7 +5621,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.31", "socket2 0.6.0", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -5681,7 +5642,7 @@ dependencies = [ "rustls 0.23.31", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -5698,7 +5659,7 @@ dependencies = [ "once_cell", "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5839,7 +5800,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -5887,9 +5848,9 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" @@ -6069,12 +6030,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "roaring" version = "0.11.2" @@ -6186,7 +6141,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6298,9 +6253,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "16.0.0" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ "bitflags", "cfg-if", @@ -6315,7 +6270,7 @@ dependencies = [ "unicode-segmentation", "unicode-width 0.2.1", "utf8parse", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6491,27 +6446,38 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -6531,14 +6497,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -6706,7 +6673,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", ] @@ -6810,7 +6777,7 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -6862,15 +6829,15 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] [[package]] name = "sqlparser" -version = "0.55.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" dependencies = [ "log", "recursive", @@ -6929,7 +6896,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -7009,7 +6976,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "whoami", ] @@ -7046,7 +7013,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "whoami", ] @@ -7070,7 +7037,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "url", ] @@ -7169,9 +7136,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.56.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" +checksum = "de6d24c270c6c672a86c183c3a8439ba46c1936f93cf7296aa692de3b0ff0228" dependencies = [ "heck", "pbjson", @@ -7262,7 +7229,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.8", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -7304,11 +7271,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -7324,9 +7291,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", @@ -7663,33 +7630,13 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro 0.19.1", -] - [[package]] name = "typed-builder" version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro 0.20.1", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "typed-builder-macro", ] [[package]] @@ -7735,7 +7682,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.106", - "thiserror 2.0.16", + "thiserror 2.0.17", "unicode-ident", ] @@ -7857,6 +7804,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -7948,7 +7901,7 @@ dependencies = [ "pin-project", "rand 0.9.2", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tower", @@ -7979,7 +7932,7 @@ dependencies = [ "rustc-hash 2.1.1", "scopeguard", "sonic-rs", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "volo", @@ -8570,13 +8523,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] -name = "xz2" -version = "0.1.7" +name = "xxhash-rust" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yansi" diff --git a/Cargo.toml b/Cargo.toml index 999b911753..29b5ce9fda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,18 +40,20 @@ rust-version = "1.87" [workspace.dependencies] anyhow = "1.0.72" -apache-avro = { version = "0.20", features = ["zstandard"] } +# Update to match our fork's version +apache-avro = { version = "0.21", features = ["zstandard"] } array-init = "2" -arrow-arith = { version = "55.1" } -arrow-array = { version = "55.1" } -arrow-buffer = { version = "55.1" } -arrow-cast = { version = "55.1" } -arrow-ord = { version = "55.1" } -arrow-schema = { version = "55.1" } -arrow-select = { version = "55.1" } -arrow-string = { version = "55.1" } +# Align arrow version with the one used by datafusion 50.0.0 +arrow-arith = { version = "56.1.0" } +arrow-array = { version = "56.1.0" } +arrow-buffer = { version = "56.1.0" } +arrow-cast = { version = "56.1.0" } +arrow-ord = { version = "56.1.0" } +arrow-schema = { version = "56.1.0" } +arrow-select = { version = "56.1.0" } +arrow-string = { version = "56.1.0" } as-any = "0.3.2" -async-trait = "0.1.88" +async-trait = "0.1.89" aws-config = "1.8.1" aws-sdk-glue = "1.39" aws-sdk-s3tables = "1.28.0" @@ -62,9 +64,10 @@ bytes = "1.10" chrono = "0.4.41" clap = { version = "4.5.41", features = ["derive", "cargo"] } ctor = "0.2.8" -datafusion = "48.0.1" -datafusion-cli = "48.0.1" -datafusion-sqllogictest = "48.0.1" +# Use datafusion fork to make sure to reuse the same versions of other packages and to avoid build conflicts caused by lzma +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion" } +datafusion-cli = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion-cli" } +datafusion-sqllogictest = { git = "https://github.com/DataDog/datafusion", rev = "4cc47d3418ee7dfe8f82394d356560401b950d63", package = "datafusion-sqllogictest" } derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -81,7 +84,7 @@ iceberg-catalog-glue = { version = "0.7.0", path = "./crates/catalog/glue" } iceberg-catalog-s3tables = { version = "0.7.0", path = "./crates/catalog/s3tables" } iceberg-catalog-hms = { version = "0.7.0", path = "./crates/catalog/hms" } iceberg-datafusion = { version = "0.7.0", path = "./crates/integrations/datafusion" } -indicatif = "0.17" +indicatif = "0.18" itertools = "0.13" linkedbytes = "0.1.8" metainfo = "0.7.14" @@ -94,15 +97,16 @@ num-bigint = "0.4.6" once_cell = "1.20" opendal = "0.54.0" ordered-float = "4" -parquet = "55.1" +# Align parquet version with the one used by datafusion 50.0.0 +parquet = "56.1.0" pilota = "0.11.10" port_scanner = "0.1.5" pretty_assertions = "1.4" rand = "0.8.5" -regex = "1.10.5" +regex = "1.11.2" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } roaring = { version = "0.11" } -rust_decimal = "1.37.1" +rust_decimal = "1.37.2" serde = { version = "1.0.219", features = ["rc"] } serde_bytes = "0.11.17" serde_derive = "1.0.219" @@ -116,12 +120,12 @@ strum = "0.27.2" tempfile = "3.18" tera = "1" thrift = "0.17.0" -tokio = { version = "1.46.1", default-features = false } +tokio = { version = "1.47.1", default-features = false } toml = "0.8" -tracing = "0.1.40" -tracing-subscriber = "0.3.8" +tracing = "0.1.41" +tracing-subscriber = "0.3.20" typed-builder = "0.20" -url = "2.5.4" +url = "2.5.7" uuid = { version = "1.18", features = ["v7"] } volo = "0.10.6" volo-thrift = "0.10.8" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 879a1450b7..d5e4bd8e5a 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -31,9 +31,9 @@ license = "Apache-2.0" crate-type = ["cdylib"] [dependencies] -arrow = { version = "55.1", features = ["pyarrow", "chrono-tz"] } +arrow = { version = "56.1.0", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } -pyo3 = { version = "0.24.1", features = ["extension-module", "abi3-py39"] } +pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } -datafusion-ffi = { version = "48.0.1" } +datafusion-ffi = { version = "50.0.0" } tokio = { version = "1.46.1", default-features = false } diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 36b5834228..516424e0b0 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -49,7 +49,7 @@ include = [ ignore = ["F403", "F405"] [tool.hatch.envs.dev] -dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==45.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] +dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==50.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] [tool.hatch.envs.dev.scripts] build = "maturin build --out dist --sdist" diff --git a/crates/catalog/sql/Cargo.toml b/crates/catalog/sql/Cargo.toml index 33ca700bf7..8de241dc4d 100644 --- a/crates/catalog/sql/Cargo.toml +++ b/crates/catalog/sql/Cargo.toml @@ -37,7 +37,7 @@ typed-builder = { workspace = true } [dev-dependencies] iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } itertools = { workspace = true } -regex = "1.10.5" +regex = { workspace = true } sqlx = { version = "0.8.1", features = [ "tls-rustls", "runtime-tokio", diff --git a/crates/iceberg/src/arrow/reader.rs b/crates/iceberg/src/arrow/reader.rs index 05aa6a4c9f..a05803c1da 100644 --- a/crates/iceberg/src/arrow/reader.rs +++ b/crates/iceberg/src/arrow/reader.rs @@ -39,7 +39,9 @@ use parquet::arrow::arrow_reader::{ }; use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask}; -use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData}; +use parquet::file::metadata::{ + PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData, +}; use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; @@ -1390,9 +1392,9 @@ impl AsyncFileReader for ArrowFileReader { async move { let reader = ParquetMetaDataReader::new() .with_prefetch_hint(self.metadata_size_hint) - .with_column_indexes(self.preload_column_index) - .with_page_indexes(self.preload_page_index) - .with_offset_indexes(self.preload_offset_index); + .with_column_index_policy(PageIndexPolicy::from(self.preload_column_index)) + .with_page_index_policy(PageIndexPolicy::from(self.preload_page_index)) + .with_offset_index_policy(PageIndexPolicy::from(self.preload_offset_index)); let size = self.meta.size; let meta = reader.load_and_finish(self, size).await?; diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index 0ee0504faa..65e66a91fa 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -84,7 +84,10 @@ impl IcebergCommitExec { let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make count batch!".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make count batch!".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 625405c95b..8f3613235c 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -100,7 +100,10 @@ impl IcebergWriteExec { let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make result batch".to_string())) + DataFusionError::ArrowError( + Box::new(e), + Some("Failed to make result batch".to_string()), + ) }) } diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs index a6d3146e54..e8e87dd318 100644 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ b/crates/integrations/datafusion/src/table/table_provider_factory.rs @@ -241,7 +241,7 @@ mod tests { options: Default::default(), table_partition_cols: Default::default(), order_exprs: Default::default(), - constraints: Constraints::empty(), + constraints: Constraints::default(), column_defaults: Default::default(), if_not_exists: Default::default(), temporary: false,