Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
3126b9e
Optimize arrays only with a session; keep constructors lazy
claude Oct 25, 2026
d32f15b
Track whether an array has been optimized
claude Oct 28, 2026
bd3bc35
Check rejected casts in tests by optimizing instead of executing
claude Oct 28, 2026
ecaaae8
Recover performance lost to lazy constructors
claude Oct 28, 2026
e7e3957
Move parent-reduce rules into the session kernel registry
claude Oct 28, 2026
b3f055b
Optimize arrays as execution reaches them
claude Oct 28, 2026
dc0e3cb
Pull shared dictionary values up as a chunked reduction
claude Oct 28, 2026
d727a96
Address review: chunked dict pull-up as reduce rule, unify reduce-par…
claude Oct 29, 2026
01e619d
Update kernel registry module docs for session-registered parent rules
claude Oct 29, 2026
abb4285
Optimize with the execution context's kernel snapshot inside kernels
claude Oct 29, 2026
5d2a809
Optimize scanned arrays before the scan returns them
claude Oct 29, 2026
bd6583d
Optimize with either a session or an execution context
claude Oct 29, 2026
8f20a5c
Update the contiguous filter trace for slices pushed through filters
robert3005 Oct 29, 2026
fc55d5a
Optimize the sliced filter in slice_filter_limits_child
robert3005 Oct 29, 2026
ebb4349
Optimize a node's unoptimized children before its own rules
robert3005 Oct 29, 2026
7c7b1e1
Mark arrays decoded from disk as tree optimized
robert3005 Oct 29, 2026
3e57138
Accept lazy chunks in the dict encoder and flat writer
robert3005 Oct 29, 2026
764d817
Drop the flat writer's optimize; chunks are optimized before writing
robert3005 Oct 29, 2026
abb1826
Simplify the optimized mark and keep stats after optimize
robert3005 Oct 29, 2026
54619d8
format
robert3005 Oct 29, 2026
a40cba9
Remove stale optimize_recursive trace lines and a redundant doc link
robert3005 Oct 29, 2026
217a9c5
Optimize lazy arrays before CUDA dispatch
robert3005 Oct 30, 2026
8e3c9fc
Fix the last two CUDA test fixtures for lazy arrays
robert3005 Oct 30, 2026
1569abb
Keep the optimized check private to the optimizer and fix the decode doc
robert3005 Oct 30, 2026
b81dee5
Optimize arrays that plan v2 keeps for later use
robert3005 Oct 30, 2026
e9809aa
format
robert3005 Oct 30, 2026
9d60f2e
Optimize shared nodes once per call and optimize scan batches eagerly
robert3005 Oct 30, 2026
ba9dbb7
format
robert3005 Oct 31, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 0 additions & 9 deletions encodings/alp/src/alp/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ use vortex_session::registry::CachedId;
use crate::ALPFloat;
use crate::alp::Exponents;
use crate::alp::decompress::execute_decompress;
use crate::alp::rules::RULES;

/// A [`ALP`]-encoded Vortex array.
pub type ALPArray = Array<ALP>;
Expand Down Expand Up @@ -203,14 +202,6 @@ impl VTable for ALP {
execute_decompress(array, ctx)?.into_array(),
))
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
child_idx: usize,
) -> VortexResult<Option<ArrayRef>> {
RULES.evaluate(array, parent, child_idx)
}
}

#[array_slots(ALP)]
Expand Down
6 changes: 5 additions & 1 deletion encodings/alp/src/alp/compute/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ mod test {

use crate::alp_encode;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[rstest]
#[case(buffer![1.23f32, 4.56, 7.89, 10.11, 12.13].into_array())]
Expand Down
6 changes: 5 additions & 1 deletion encodings/alp/src/alp/compute/mask.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,11 @@ mod test {
use crate::alp::array::ALPArrayExt;
use crate::alp_encode;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[rstest]
#[case(buffer![10.5f32, 20.5, 30.5, 40.5, 50.5].into_array())]
Expand Down
6 changes: 5 additions & 1 deletion encodings/alp/src/alp/compute/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ mod test {

use crate::alp_encode;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[rstest]
#[case(buffer![1.23f32, 4.56, 7.89, 10.11, 12.13].into_array())]
Expand Down
1 change: 1 addition & 0 deletions encodings/alp/src/alp/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ pub(crate) use plugin::ALPPatchedPlugin;

pub(crate) fn initialize(session: &VortexSession) {
rules::initialize(session);
rules::register_parent_rules(session);
}

#[cfg(test)]
Expand Down
16 changes: 10 additions & 6 deletions encodings/alp/src/alp/rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ use vortex_array::arrays::dict::TakeExecuteAdaptor;
use vortex_array::arrays::filter::FilterExecuteAdaptor;
use vortex_array::arrays::slice::SliceExecuteAdaptor;
use vortex_array::optimizer::kernels::ArrayKernelsExt;
use vortex_array::optimizer::rules::ParentRuleSet;
use vortex_array::scalar_fn::ScalarFnVTable;
use vortex_array::scalar_fn::fns::between::Between;
use vortex_array::scalar_fn::fns::between::BetweenReduceAdaptor;
use vortex_array::scalar_fn::fns::binary::Binary;
use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor;
use vortex_array::scalar_fn::fns::cast::Cast;
use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor;
use vortex_array::scalar_fn::fns::mask::Mask as MaskFn;
use vortex_array::scalar_fn::fns::mask::Mask;
use vortex_array::scalar_fn::fns::mask::MaskExecuteAdaptor;
use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor;
Expand All @@ -31,8 +33,10 @@ pub(super) fn initialize(session: &VortexSession) {
kernels.register_execute_parent_kernel(Dict.id(), ALP, TakeExecuteAdaptor(ALP));
}

pub(super) const RULES: ParentRuleSet<ALP> = ParentRuleSet::new(&[
ParentRuleSet::lift(&BetweenReduceAdaptor(ALP)),
ParentRuleSet::lift(&CastReduceAdaptor(ALP)),
ParentRuleSet::lift(&MaskReduceAdaptor(ALP)),
]);
/// Registers the parent-reduce rules for [`ALP`] children in the session kernels.
pub(crate) fn register_parent_rules(session: &VortexSession) {
let kernels = session.kernels();
kernels.register_reduce_parent_rule(Between.id(), ALP, BetweenReduceAdaptor(ALP));
kernels.register_reduce_parent_rule(Cast.id(), ALP, CastReduceAdaptor(ALP));
kernels.register_reduce_parent_rule(MaskFn.id(), ALP, MaskReduceAdaptor(ALP));
}
9 changes: 0 additions & 9 deletions encodings/alp/src/alp_rd/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ use vortex_session::VortexSession;
use vortex_session::registry::CachedId;

use crate::alp_rd::check_left_parts_codes;
use crate::alp_rd::rules::RULES;
use crate::alp_rd_decode;

/// A [`ALPRD`]-encoded Vortex array.
Expand Down Expand Up @@ -335,14 +334,6 @@ impl VTable for ALPRD {

Ok(ExecutionResult::done(decoded_array.into_array()))
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
child_idx: usize,
) -> VortexResult<Option<ArrayRef>> {
RULES.evaluate(array, parent, child_idx)
}
}

#[array_slots(ALPRD)]
Expand Down
6 changes: 5 additions & 1 deletion encodings/alp/src/alp_rd/compute/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ mod tests {
use crate::RDEncoder;
use crate::RDEncoderExt;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[test]
fn test_cast_alprd_f32_to_f64() {
Expand Down
1 change: 1 addition & 0 deletions encodings/alp/src/alp_rd/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ mod slice;

pub(crate) fn initialize(session: &VortexSession) {
kernel::initialize(session);
rules::register_parent_rules(session);
}

/// Encoding of Vortex arrays with an ALP-RD [`RDEncoder`].
Expand Down
16 changes: 11 additions & 5 deletions encodings/alp/src/alp_rd/rules.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::optimizer::rules::ParentRuleSet;
use vortex_array::optimizer::kernels::ArrayKernelsExt;
use vortex_array::scalar_fn::ScalarFnVTable;
use vortex_array::scalar_fn::fns::cast::Cast;
use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor;
use vortex_array::scalar_fn::fns::mask::Mask as MaskFn;
use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor;
use vortex_session::VortexSession;

use crate::alp_rd::ALPRD;

pub(crate) static RULES: ParentRuleSet<ALPRD> = ParentRuleSet::new(&[
ParentRuleSet::lift(&CastReduceAdaptor(ALPRD)),
ParentRuleSet::lift(&MaskReduceAdaptor(ALPRD)),
]);
/// Registers the parent-reduce rules for [`ALPRD`] children in the session kernels.
pub(crate) fn register_parent_rules(session: &VortexSession) {
let kernels = session.kernels();
kernels.register_reduce_parent_rule(Cast.id(), ALPRD, CastReduceAdaptor(ALPRD));
kernels.register_reduce_parent_rule(MaskFn.id(), ALPRD, MaskReduceAdaptor(ALPRD));
}
8 changes: 0 additions & 8 deletions encodings/bytebool/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,14 +163,6 @@ impl VTable for ByteBool {
ByteBoolSlots::NAMES[idx].to_string()
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
child_idx: usize,
) -> VortexResult<Option<ArrayRef>> {
crate::rules::RULES.evaluate(array, parent, child_idx)
}

fn execute(array: Array<Self>, _ctx: &mut ExecutionCtx) -> VortexResult<ExecutionResult> {
// convert truthy values to set/unset bits
let boolean_buffer = BitBufferMut::from(array.truthy_bytes()).freeze();
Expand Down
1 change: 1 addition & 0 deletions encodings/bytebool/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,5 @@ mod slice;
pub fn initialize(session: &VortexSession) {
session.arrays().register(ByteBool);
kernel::initialize(session);
rules::register_parent_rules(session);
}
20 changes: 14 additions & 6 deletions encodings/bytebool/src/rules.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,23 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::ArrayVTable;
use vortex_array::arrays::Slice;
use vortex_array::arrays::slice::SliceReduceAdaptor;
use vortex_array::optimizer::rules::ParentRuleSet;
use vortex_array::optimizer::kernels::ArrayKernelsExt;
use vortex_array::scalar_fn::ScalarFnVTable;
use vortex_array::scalar_fn::fns::cast::Cast;
use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor;
use vortex_array::scalar_fn::fns::mask::Mask as MaskFn;
use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor;
use vortex_session::VortexSession;

use crate::ByteBool;

pub(crate) static RULES: ParentRuleSet<ByteBool> = ParentRuleSet::new(&[
ParentRuleSet::lift(&CastReduceAdaptor(ByteBool)),
ParentRuleSet::lift(&MaskReduceAdaptor(ByteBool)),
ParentRuleSet::lift(&SliceReduceAdaptor(ByteBool)),
]);
/// Registers the parent-reduce rules for [`ByteBool`] children in the session kernels.
pub(crate) fn register_parent_rules(session: &VortexSession) {
let kernels = session.kernels();
kernels.register_reduce_parent_rule(Cast.id(), ByteBool, CastReduceAdaptor(ByteBool));
kernels.register_reduce_parent_rule(MaskFn.id(), ByteBool, MaskReduceAdaptor(ByteBool));
kernels.register_reduce_parent_rule(Slice.id(), ByteBool, SliceReduceAdaptor(ByteBool));
}
9 changes: 0 additions & 9 deletions encodings/datetime-parts/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ use vortex_session::registry::CachedId;

use crate::TemporalParts;
use crate::canonical::decode_to_temporal;
use crate::compute::rules::PARENT_RULES;
use crate::split_temporal;

/// A [`DateTimeParts`]-encoded Vortex array.
Expand Down Expand Up @@ -205,14 +204,6 @@ impl VTable for DateTimeParts {
decode_to_temporal(parts, &dtype, ctx)?.into_array(),
))
}

fn reduce_parent(
array: ArrayView<'_, Self>,
parent: &ArrayRef,
child_idx: usize,
) -> VortexResult<Option<ArrayRef>> {
PARENT_RULES.evaluate(array, parent, child_idx)
}
}

#[array_slots(DateTimeParts)]
Expand Down
6 changes: 5 additions & 1 deletion encodings/datetime-parts/src/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,11 @@ mod tests {
use crate::TemporalParts;
use crate::split_temporal;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[rstest]
#[case(Validity::NonNullable)]
Expand Down
6 changes: 5 additions & 1 deletion encodings/datetime-parts/src/compute/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,11 @@ mod tests {
use crate::DateTimeParts;
use crate::DateTimePartsArray;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

fn date_time_array(validity: Validity) -> ArrayRef {
DateTimeParts::try_from_temporal(
Expand Down
6 changes: 5 additions & 1 deletion encodings/datetime-parts/src/compute/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,11 @@ mod test {
use crate::DateTimeParts;
use crate::DateTimePartsArray;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

fn dtp_array_from_timestamp<T: IntegerPType>(
value: T,
Expand Down
6 changes: 5 additions & 1 deletion encodings/datetime-parts/src/compute/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ mod test {

use crate::DateTimeParts;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

#[test]
fn test_filter_datetime_parts() {
Expand Down
6 changes: 5 additions & 1 deletion encodings/datetime-parts/src/compute/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,11 @@ mod tests {
use crate::DateTimeParts;
use crate::DateTimePartsArray;

static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
let session = VortexSession::empty();
crate::initialize(&session);
session
});

fn dtp_from_temporal(temporal: TemporalArray) -> DateTimePartsArray {
DateTimeParts::try_from_temporal(temporal, &mut SESSION.create_execution_ctx()).unwrap()
Expand Down
Loading
Loading