Skip to content

Commit 2193919

Browse files
robert3005claude
robert3005
and
claude
committed
Build arrays through non-generic constructors
The generic constructors of Constant, Primitive, Struct, Chunked and Decimal arrays called the generic Array::from_parts_unchecked and Array::try_from_parts directly. Each crate that called them compiled its own copy of the from_parts functions and of the full DynArrayData vtable for the encoding. The generic constructors now call a private non-generic function, so vortex-array compiles each vtable once. ScalarFn had the same problem because new_from_validated_slots was #[inline]; it is not inlined now. vortex-ffi release cdylib __text: 37,223,740 -> 36,613,308 bytes (-610 KB). Copies of the Constant vtable: 22 -> 1, Primitive: 17 -> 1. Signed-off-by: Robert Kruszewski <robert@spiraldb.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent a1fdb64 commit 2193919

6 files changed

Lines changed: 110 additions & 13 deletions

File tree

‎vortex-array/src/arrays/chunked/array.rs‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ impl Array<Chunked> {
215215

216216
/// Constructs a new `ChunkedArray`.
217217
pub fn try_new(chunks: impl IntoIterator<Item = ArrayRef>, dtype: DType) -> VortexResult<Self> {
218-
Array::try_from_parts(Self::parts_from_chunks::<true>(chunks, dtype)?)
218+
Self::try_from_chunked_parts(Self::parts_from_chunks::<true>(chunks, dtype)?)
219219
}
220220

221221
pub fn rechunk(
@@ -278,6 +278,26 @@ impl Array<Chunked> {
278278
pub unsafe fn new_unchecked(chunks: impl IntoIterator<Item = ArrayRef>, dtype: DType) -> Self {
279279
let parts = Self::parts_from_chunks::<false>(chunks, dtype)
280280
.vortex_expect("unchecked chunked construction cannot fail");
281+
unsafe { Self::from_chunked_parts_unchecked(parts) }
282+
}
283+
284+
/// Creates the array from parts after validation.
285+
///
286+
/// The generic constructors call this function. It is not generic, so crates that call those
287+
/// constructors do not each compile the `Chunked` array vtable.
288+
fn try_from_chunked_parts(parts: ArrayParts<Chunked>) -> VortexResult<Self> {
289+
Array::try_from_parts(parts)
290+
}
291+
292+
/// Creates the array from parts without validation.
293+
///
294+
/// The generic constructors call this function. It is not generic, so crates that call those
295+
/// constructors do not each compile the `Chunked` array vtable.
296+
///
297+
/// # Safety
298+
///
299+
/// The parts must be logically consistent.
300+
unsafe fn from_chunked_parts_unchecked(parts: ArrayParts<Chunked>) -> Self {
281301
unsafe { Array::from_parts_unchecked(parts) }
282302
}
283303
}

‎vortex-array/src/arrays/constant/array.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,14 @@ impl Array<Constant> {
4545
where
4646
S: Into<Scalar>,
4747
{
48-
let scalar = scalar.into();
48+
Self::from_scalar(scalar.into(), len)
49+
}
50+
51+
/// Creates the array from a [`Scalar`].
52+
///
53+
/// The generic constructors call this function. It is not generic, so crates that call those
54+
/// constructors do not each compile the `Constant` array vtable.
55+
fn from_scalar(scalar: Scalar, len: usize) -> Self {
4956
let dtype = scalar.dtype().clone();
5057
let data = ConstantData::new(scalar);
5158
unsafe {

‎vortex-array/src/arrays/decimal/array.rs‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,9 @@ impl Array<Decimal> {
426426
let len = buffer.len();
427427
let slots = DecimalData::make_slots(&validity, len);
428428
let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype) };
429-
unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots)) }
429+
unsafe {
430+
Self::from_decimal_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data, slots))
431+
}
430432
}
431433

432434
/// Creates a new [`DecimalArray`] from a host-native buffer with validation.
@@ -439,7 +441,27 @@ impl Array<Decimal> {
439441
let len = buffer.len();
440442
let slots = DecimalData::make_slots(&validity, len);
441443
let data = DecimalData::try_new(buffer, decimal_dtype)?;
442-
Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data, slots))
444+
Self::try_from_decimal_parts(ArrayParts::new(Decimal, dtype, len, data, slots))
445+
}
446+
447+
/// Creates the array from parts without validation.
448+
///
449+
/// The generic constructors call this function. It is not generic, so crates that call those
450+
/// constructors do not each compile the `Decimal` array vtable.
451+
///
452+
/// # Safety
453+
///
454+
/// The parts must be logically consistent.
455+
unsafe fn from_decimal_parts_unchecked(parts: ArrayParts<Decimal>) -> Self {
456+
unsafe { Array::from_parts_unchecked(parts) }
457+
}
458+
459+
/// Creates the array from parts after validation.
460+
///
461+
/// The generic constructors call this function. It is not generic, so crates that call those
462+
/// constructors do not each compile the `Decimal` array vtable.
463+
fn try_from_decimal_parts(parts: ArrayParts<Decimal>) -> VortexResult<Self> {
464+
Array::try_from_parts(parts)
443465
}
444466

445467
/// Creates a new [`DecimalArray`] from an iterator of values.

‎vortex-array/src/arrays/primitive/array/mod.rs‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,11 @@ impl Array<Primitive> {
346346
let len = 0;
347347
let data = PrimitiveData::empty::<T>(nullability);
348348
let slots = PrimitiveData::make_slots(&Validity::from(nullability), len);
349-
unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) }
349+
unsafe {
350+
Self::from_primitive_parts_unchecked(ArrayParts::new(
351+
Primitive, dtype, len, data, slots,
352+
))
353+
}
350354
}
351355

352356
/// Creates a new `PrimitiveArray`.
@@ -360,7 +364,11 @@ impl Array<Primitive> {
360364
let len = buffer.len();
361365
let slots = PrimitiveData::make_slots(&validity, len);
362366
let data = PrimitiveData::new(buffer, validity);
363-
unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) }
367+
unsafe {
368+
Self::from_primitive_parts_unchecked(ArrayParts::new(
369+
Primitive, dtype, len, data, slots,
370+
))
371+
}
364372
}
365373

366374
/// Constructs a new `PrimitiveArray`.
@@ -370,7 +378,9 @@ impl Array<Primitive> {
370378
let slots = PrimitiveData::make_slots(&validity, len);
371379
let data = PrimitiveData::try_new(buffer, validity)?;
372380
Ok(unsafe {
373-
Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots))
381+
Self::from_primitive_parts_unchecked(ArrayParts::new(
382+
Primitive, dtype, len, data, slots,
383+
))
374384
})
375385
}
376386

@@ -384,7 +394,23 @@ impl Array<Primitive> {
384394
let len = buffer.len();
385395
let slots = PrimitiveData::make_slots(&validity, len);
386396
let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) };
387-
unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data, slots)) }
397+
unsafe {
398+
Self::from_primitive_parts_unchecked(ArrayParts::new(
399+
Primitive, dtype, len, data, slots,
400+
))
401+
}
402+
}
403+
404+
/// Creates the array from parts without validation.
405+
///
406+
/// The generic constructors call this function. It is not generic, so crates that call those
407+
/// constructors do not each compile the `Primitive` array vtable.
408+
///
409+
/// # Safety
410+
///
411+
/// The parts must be logically consistent.
412+
unsafe fn from_primitive_parts_unchecked(parts: ArrayParts<Primitive>) -> Self {
413+
unsafe { Array::from_parts_unchecked(parts) }
388414
}
389415

390416
/// Create a new array from a buffer handle.

‎vortex-array/src/arrays/scalar_fn/array.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,9 @@ impl Array<ScalarFn> {
118118
/// # Safety
119119
/// The caller must ensure arity and child lengths were validated and
120120
/// "dtype" is scalar function's return dtype.
121-
#[inline]
121+
///
122+
/// This function is not inlined, so crates that construct scalar function arrays do not each
123+
/// compile the `ScalarFn` array vtable.
122124
unsafe fn new_from_validated_slots(
123125
scalar_fn: ScalarFnRef,
124126
dtype: DType,

‎vortex-array/src/arrays/struct_/array.rs‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ impl Array<Struct> {
292292
slots.push(Some(field));
293293
}
294294
let dtype = StructFields::new(names, field_dtypes);
295-
Array::try_from_parts(ArrayParts::new(
295+
Self::try_from_struct_parts(ArrayParts::new(
296296
Struct,
297297
DType::Struct(dtype, validity.nullability()),
298298
length,
@@ -315,7 +315,7 @@ impl Array<Struct> {
315315
let outer_dtype = DType::Struct(dtype, validity.nullability());
316316
let slots = make_struct_slots(fields, &validity, length);
317317
unsafe {
318-
Array::from_parts_unchecked(ArrayParts::new(
318+
Self::from_struct_parts_unchecked(ArrayParts::new(
319319
Struct,
320320
outer_dtype,
321321
length,
@@ -334,7 +334,7 @@ impl Array<Struct> {
334334
) -> VortexResult<Self> {
335335
let outer_dtype = DType::Struct(dtype, validity.nullability());
336336
let slots = make_struct_slots(fields, &validity, length);
337-
Array::try_from_parts(ArrayParts::new(
337+
Self::try_from_struct_parts(ArrayParts::new(
338338
Struct,
339339
outer_dtype,
340340
length,
@@ -343,6 +343,26 @@ impl Array<Struct> {
343343
))
344344
}
345345

346+
/// Creates the array from parts after validation.
347+
///
348+
/// The generic constructors call this function. It is not generic, so crates that call those
349+
/// constructors do not each compile the `Struct` array vtable.
350+
fn try_from_struct_parts(parts: ArrayParts<Struct>) -> VortexResult<Self> {
351+
Array::try_from_parts(parts)
352+
}
353+
354+
/// Creates the array from parts without validation.
355+
///
356+
/// The generic constructors call this function. It is not generic, so crates that call those
357+
/// constructors do not each compile the `Struct` array vtable.
358+
///
359+
/// # Safety
360+
///
361+
/// The parts must be logically consistent.
362+
unsafe fn from_struct_parts_unchecked(parts: ArrayParts<Struct>) -> Self {
363+
unsafe { Array::from_parts_unchecked(parts) }
364+
}
365+
346366
/// Construct a `StructArray` from named fields.
347367
pub fn from_fields<N: AsRef<str>>(items: &[(N, ArrayRef)]) -> VortexResult<Self> {
348368
Self::try_from_iter(items.iter().map(|(a, b)| (a, b.clone())))
@@ -460,7 +480,7 @@ impl Array<Struct> {
460480

461481
let new_dtype = struct_dtype.without_field(position).ok()?;
462482
let new_array = unsafe {
463-
Array::from_parts_unchecked(ArrayParts::new(
483+
Self::from_struct_parts_unchecked(ArrayParts::new(
464484
Struct,
465485
DType::Struct(new_dtype, self.dtype().nullability()),
466486
len,

0 commit comments

Comments
 (0)