Skip to content

Commit 06bffcd

Browse files
claudeclaude
committed
Move scalar functions out of vortex-array into vortex-functions
vortex-array keeps the scalar function machinery (the `ScalarFnVTable` trait, binding, reduction, execution, the session registry and kernel registry) and the few functions its own arrays, validity and expressions are built on: literal, get_item and pack, is_null, is_not_null, not and the binary operators, and cast, mask and fill_null, which canonical arrays apply to their own children. That module is renamed from `scalar_fn::fns` to `scalar_fn::builtin`, matching `ArrayBuiltins`. Every other scalar function moves to the new `vortex-functions` crate: - byte_length, case_when, ext_storage, list_length, list_sum, merge, replace_time_zone, select, dynamic, stat, between, like, list_contains, variant_get, is_nan and zip, with their expression constructors in `vortex_functions::expr` (and `expr::bound`) - the kernels and parent-reduce rules for vortex-array's encodings, which `vortex_functions::initialize` now registers with a session's kernel registry instead of the encodings' static rule sets - the `ArrayBuiltins`/`ExprBuiltins` methods for the moved functions, which become the `ArrayFunctions`/`ExprFunctions` extension traits `apply_reduce_parent_rule` lets a crate register a typed parent-reduce rule for an encoding it does not own. `ArrayHaystack`, `LazyBitBufferBuilder`, `DeduplicatedBuffers::{extend_from_iter, finish}`, `ListScalar::element_values`, `is_not_null_node`, `build_row_comparator` and `merge_typed_scalar_as_variant` become public for the moved kernels. The umbrella crate re-exports the moved constructors under `vortex::expr` and the crate as `vortex::functions`. The CodSpeed workflow builds `vortex-functions` in the main-library shard, so the moved benchmarks stay measured, and picks up `vortex-onpair` and `vortex-arrow`, whose benchmarks no shard built. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ps4mp7KF5c4p66aoSoXx5j Signed-off-by: Claude <noreply@anthropic.com>
1 parent d02df89 commit 06bffcd

384 files changed

Lines changed: 3992 additions & 3610 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/codspeed.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ jobs:
5656
- { shard: 3, name: "Array dictionaries", packages: "vortex-array", array-group: "dictionary", features: "--features _test-harness" }
5757
- { shard: 4, name: "Array builders & views", packages: "vortex-array", array-group: "builders", features: "--features _test-harness" }
5858
- { shard: 5, name: "Core & encodings", packages: "vortex-buffer vortex-error vortex-mask vortex-compute vortex-file vortex-alp vortex-bytebool vortex-datetime-parts" }
59-
- { shard: 6, name: "Main library & storage", packages: "vortex vortex-btrblocks vortex-compressor vortex-row" }
59+
- { shard: 6, name: "Main library, functions & storage", packages: "vortex vortex-functions vortex-arrow vortex-btrblocks vortex-compressor vortex-row" }
6060
- { shard: 7, name: "FastLanes & decimals", packages: "vortex-decimal-byte-parts vortex-fastlanes", features: "--features _test-harness" }
61-
- { shard: 8, name: "Encodings & layout", packages: "vortex-pco vortex-runend vortex-sequence vortex-sparse vortex-zigzag vortex-zstd vortex-layout" }
61+
- { shard: 8, name: "Encodings & layout", packages: "vortex-onpair vortex-pco vortex-runend vortex-sequence vortex-sparse vortex-zigzag vortex-zstd vortex-layout" }
6262
- { shard: 9, name: "Strings, tensor & spatial", packages: "vortex-fsst vortex-tensor vortex-spatial", features: "--features _test-harness" }
6363
name: "Benchmark with Codspeed (Shard #${{ matrix.shard }} - ${{ matrix.name }})"
6464
timeout-minutes: 30
@@ -92,7 +92,7 @@ jobs:
9292
groups = {
9393
"dictionary": ("dict_", "chunked_dict_", "patches_"),
9494
"selection": ("take_", "filter_", "piecewise_sequence_"),
95-
"compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool", "like", "list_sum"),
95+
"compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool"),
9696
}
9797
manifest = tomllib.loads(pathlib.Path("vortex-array/Cargo.toml").read_text())
9898
benches = []

‎Cargo.lock‎

Lines changed: 44 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ members = [
1313
"vortex-io",
1414
"vortex-cloud",
1515
"vortex-array",
16+
"vortex-functions",
1617
"vortex-expr",
1718
"vortex-arrow",
1819
"vortex-row",
@@ -324,6 +325,7 @@ vortex-expr = { version = "0.1.0", path = "./vortex-expr", default-features = fa
324325
vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
325326
vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }
326327
vortex-fsst = { version = "0.1.0", path = "./encodings/fsst", default-features = false }
328+
vortex-functions = { version = "0.1.0", path = "./vortex-functions", default-features = false }
327329
vortex-io = { version = "0.1.0", path = "./vortex-io", default-features = false }
328330
vortex-ipc = { version = "0.1.0", path = "./vortex-ipc", default-features = false }
329331
vortex-json = { version = "0.1.0", path = "./vortex-json", default-features = false }

‎encodings/alp/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ vortex-array = { workspace = true }
2424
vortex-buffer = { workspace = true }
2525
vortex-error = { workspace = true }
2626
vortex-fastlanes = { workspace = true }
27+
vortex-functions = { workspace = true }
2728
vortex-mask = { workspace = true }
2829
vortex-session = { workspace = true }
2930

‎encodings/alp/src/alp/compute/between.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@ use vortex_array::ArrayRef;
88
use vortex_array::ArrayView;
99
use vortex_array::IntoArray;
1010
use vortex_array::arrays::ConstantArray;
11-
use vortex_array::builtins::ArrayBuiltins;
1211
use vortex_array::dtype::NativeDType;
1312
use vortex_array::dtype::NativePType;
1413
use vortex_array::dtype::Nullability;
1514
use vortex_array::scalar::PValue;
1615
use vortex_array::scalar::Scalar;
17-
use vortex_array::scalar_fn::fns::between::BetweenOptions;
18-
use vortex_array::scalar_fn::fns::between::BetweenReduce;
19-
use vortex_array::scalar_fn::fns::between::StrictComparison;
2016
use vortex_error::VortexResult;
17+
use vortex_functions::between::BetweenOptions;
18+
use vortex_functions::between::BetweenReduce;
19+
use vortex_functions::between::StrictComparison;
20+
use vortex_functions::builtins::ArrayFunctions;
2121

2222
use crate::ALP;
2323
use crate::ALPFloat;
@@ -145,8 +145,8 @@ mod tests {
145145
use vortex_array::assert_arrays_eq;
146146
use vortex_array::dtype::Nullability;
147147
use vortex_array::scalar::Scalar;
148-
use vortex_array::scalar_fn::fns::between::BetweenOptions;
149-
use vortex_array::scalar_fn::fns::between::StrictComparison;
148+
use vortex_functions::between::BetweenOptions;
149+
use vortex_functions::between::StrictComparison;
150150
use vortex_session::VortexSession;
151151

152152
use crate::ALPArray;

‎encodings/alp/src/alp/compute/cast.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use vortex_array::ArrayView;
66
use vortex_array::IntoArray;
77
use vortex_array::builtins::ArrayBuiltins;
88
use vortex_array::dtype::DType;
9-
use vortex_array::scalar_fn::fns::cast::CastReduce;
9+
use vortex_array::scalar_fn::builtin::cast::CastReduce;
1010
use vortex_error::VortexResult;
1111

1212
use crate::ALPArrayExt;

‎encodings/alp/src/alp/compute/compare.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ use vortex_array::arrays::ConstantArray;
1111
use vortex_array::builtins::ArrayBuiltins;
1212
use vortex_array::dtype::NativePType;
1313
use vortex_array::scalar::Scalar;
14-
use vortex_array::scalar_fn::fns::binary::CompareKernel;
15-
use vortex_array::scalar_fn::fns::operators::CompareOperator;
16-
use vortex_array::scalar_fn::fns::operators::Operator;
14+
use vortex_array::scalar_fn::builtin::binary::CompareKernel;
15+
use vortex_array::scalar_fn::builtin::operators::CompareOperator;
16+
use vortex_array::scalar_fn::builtin::operators::Operator;
1717
use vortex_error::VortexResult;
1818
use vortex_error::vortex_bail;
1919
use vortex_error::vortex_err;
@@ -165,8 +165,8 @@ mod tests {
165165
use vortex_array::dtype::Nullability;
166166
use vortex_array::dtype::PType;
167167
use vortex_array::scalar::Scalar;
168-
use vortex_array::scalar_fn::fns::operators::CompareOperator;
169-
use vortex_array::scalar_fn::fns::operators::Operator;
168+
use vortex_array::scalar_fn::builtin::operators::CompareOperator;
169+
use vortex_array::scalar_fn::builtin::operators::Operator;
170170
use vortex_session::VortexSession;
171171

172172
use super::*;

‎encodings/alp/src/alp/compute/mask.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ use vortex_array::ArrayView;
66
use vortex_array::ExecutionCtx;
77
use vortex_array::IntoArray;
88
use vortex_array::builtins::ArrayBuiltins;
9-
use vortex_array::scalar_fn::fns::mask::MaskKernel;
10-
use vortex_array::scalar_fn::fns::mask::MaskReduce;
9+
use vortex_array::scalar_fn::builtin::mask::MaskKernel;
10+
use vortex_array::scalar_fn::builtin::mask::MaskReduce;
1111
use vortex_array::validity::Validity;
1212
use vortex_error::VortexResult;
1313

@@ -58,7 +58,7 @@ mod test {
5858
use vortex_array::arrays::PrimitiveArray;
5959
use vortex_array::compute::conformance::mask::test_mask_conformance;
6060
use vortex_array::dtype::Nullability;
61-
use vortex_array::scalar_fn::fns::mask::MaskKernel;
61+
use vortex_array::scalar_fn::builtin::mask::MaskKernel;
6262
use vortex_buffer::buffer;
6363
use vortex_session::VortexSession;
6464

‎encodings/alp/src/alp/rules.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ use vortex_array::arrays::slice::SliceExecuteAdaptor;
1111
use vortex_array::optimizer::kernels::ArrayKernelsExt;
1212
use vortex_array::optimizer::rules::ParentRuleSet;
1313
use vortex_array::scalar_fn::ScalarFnVTable;
14-
use vortex_array::scalar_fn::fns::between::BetweenReduceAdaptor;
15-
use vortex_array::scalar_fn::fns::binary::Binary;
16-
use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor;
17-
use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor;
18-
use vortex_array::scalar_fn::fns::mask::Mask;
19-
use vortex_array::scalar_fn::fns::mask::MaskExecuteAdaptor;
20-
use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor;
14+
use vortex_array::scalar_fn::builtin::binary::Binary;
15+
use vortex_array::scalar_fn::builtin::binary::CompareExecuteAdaptor;
16+
use vortex_array::scalar_fn::builtin::cast::CastReduceAdaptor;
17+
use vortex_array::scalar_fn::builtin::mask::Mask;
18+
use vortex_array::scalar_fn::builtin::mask::MaskExecuteAdaptor;
19+
use vortex_array::scalar_fn::builtin::mask::MaskReduceAdaptor;
20+
use vortex_functions::between::BetweenReduceAdaptor;
2121
use vortex_session::VortexSession;
2222

2323
use crate::ALP;

‎encodings/alp/src/alp_rd/compute/cast.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use vortex_array::ArrayView;
66
use vortex_array::IntoArray;
77
use vortex_array::builtins::ArrayBuiltins;
88
use vortex_array::dtype::DType;
9-
use vortex_array::scalar_fn::fns::cast::CastReduce;
9+
use vortex_array::scalar_fn::builtin::cast::CastReduce;
1010
use vortex_error::VortexResult;
1111

1212
use crate::ALPRDArrayExt;

0 commit comments

Comments
 (0)