Skip to content

Commit a6c71f5

Browse files
Jefffreyalamb
andauthored
[59_maintenance] fix: account for offsets in runarrays when using extend for MutableArrayData (#11671)
- Backport #11546 - For inclusion in #11664 Cherry-pick of 2e26ab4 --------- Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
1 parent 8c4a4bc commit a6c71f5

1 file changed

Lines changed: 55 additions & 9 deletions

File tree

  • arrow-data/src/transform

‎arrow-data/src/transform/run.rs‎

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
// under the License.
1717

1818
use super::{_MutableArrayData, ArrayData, Extend};
19-
use arrow_buffer::{ArrowNativeType, Buffer, ToByteSlice};
19+
use arrow_buffer::{ArrowNativeType, ToByteSlice};
2020
use arrow_schema::{ArrowError, DataType};
2121
use num_traits::CheckedAdd;
2222

@@ -58,8 +58,9 @@ pub fn extend_nulls(mutable: &mut _MutableArrayData, len: usize) -> Result<(), A
5858
macro_rules! extend_nulls_impl {
5959
($run_end_type:ty) => {{
6060
let last_run_end = get_last_run_end::<$run_end_type>(&mutable.child_data[0]);
61-
let new_value = last_run_end
62-
.checked_add(<$run_end_type as ArrowNativeType>::usize_as(len))
61+
let len = <$run_end_type as ArrowNativeType>::from_usize(len);
62+
let new_value = len
63+
.and_then(|len| last_run_end.checked_add(len))
6364
.ok_or_else(|| {
6465
ArrowError::InvalidArgumentError(
6566
"run end overflow when extending RunEndEncoded array: \
@@ -91,7 +92,7 @@ type ExtendArrays = (Vec<u8>, Option<(usize, usize)>);
9192

9293
/// Build run ends bytes and values range directly for batch processing
9394
fn build_extend_arrays<T: ArrowNativeType + std::ops::Add<Output = T> + CheckedAdd>(
94-
buffer: &Buffer,
95+
typed_slice: &[T],
9596
length: usize,
9697
start: usize,
9798
len: usize,
@@ -103,9 +104,6 @@ fn build_extend_arrays<T: ArrowNativeType + std::ops::Add<Output = T> + CheckedA
103104
let mut prev_end = 0;
104105
let mut current_run_end = dest_last_run_end;
105106

106-
// Convert buffer to typed slice once
107-
let typed_slice: &[T] = buffer.typed_data();
108-
109107
for i in 0..length {
110108
if i < typed_slice.len() {
111109
let run_end = typed_slice[i].to_usize().unwrap();
@@ -216,6 +214,8 @@ pub fn build_extend(array: &ArrayData) -> Extend<'_> {
216214

217215
// We need to analyze the source array's run structure
218216
let source_run_ends = &array.child_data()[0];
217+
let source_buffer_offset = source_run_ends.offset();
218+
let source_buffer_len = source_run_ends.len();
219219
let source_buffer = &source_run_ends.buffers()[0];
220220

221221
// Get the run end type from the mutable array
@@ -229,6 +229,8 @@ pub fn build_extend(array: &ArrayData) -> Extend<'_> {
229229
// Build run ends and values indices directly for batch processing
230230
macro_rules! build_and_process_impl {
231231
($run_end_type:ty) => {{
232+
let source_buffer = &source_buffer.typed_data()
233+
[source_buffer_offset..source_buffer_offset + source_buffer_len];
232234
let dest_last_run_end =
233235
get_last_run_end::<$run_end_type>(&mutable.child_data[0]);
234236
let (run_ends_bytes, values_range) = build_extend_arrays::<$run_end_type>(
@@ -541,7 +543,7 @@ mod tests {
541543
fn test_build_extend_arrays_int16() {
542544
let buffer = Buffer::from_vec(vec![3i16, 5i16, 8i16]);
543545
let (run_ends_bytes, values_range) =
544-
build_extend_arrays::<i16>(&buffer, 3, 2, 4, 0i16).unwrap();
546+
build_extend_arrays::<i16>(buffer.typed_data(), 3, 2, 4, 0i16).unwrap();
545547

546548
// Logical array: [A, A, A, B, B, C, C, C]
547549
// Requesting indices 2-6 should give us:
@@ -564,7 +566,7 @@ mod tests {
564566
fn test_build_extend_arrays_int64() {
565567
let buffer = Buffer::from_vec(vec![3i64, 5i64, 8i64]);
566568
let (run_ends_bytes, values_range) =
567-
build_extend_arrays::<i64>(&buffer, 3, 2, 4, 0i64).unwrap();
569+
build_extend_arrays::<i64>(buffer.typed_data(), 3, 2, 4, 0i64).unwrap();
568570

569571
// Same logic as above but with i64
570572
assert_eq!(run_ends_bytes.len(), 3 * std::mem::size_of::<i64>());
@@ -687,4 +689,48 @@ mod tests {
687689
"unexpected error: {err}"
688690
);
689691
}
692+
693+
#[test]
694+
fn test_extend_nulls_overflow() {
695+
let values = create_int32_array_data(vec![10]);
696+
let array = create_run_array_data_int16(vec![100], values);
697+
698+
let mut mutable = MutableArrayData::new(vec![&array], true, 10);
699+
let err = mutable.try_extend_nulls(i16::MAX as usize + 1).unwrap_err();
700+
assert_eq!(
701+
err.to_string(),
702+
"Invalid argument error: run end overflow when extending RunEndEncoded array: use a larger run-end type (e.g. Int64 instead of Int32)"
703+
);
704+
}
705+
706+
#[test]
707+
fn test_extend_accounts_for_offset() {
708+
let dt = DataType::RunEndEncoded(
709+
Arc::new(Field::new("run_ends", DataType::Int32, false)),
710+
Arc::new(Field::new("values", DataType::Float32, true)),
711+
);
712+
713+
let runs = ArrayData::builder(DataType::Int32)
714+
.len(2)
715+
.offset(2)
716+
.add_buffer(vec![-10_i32, -20, 1_i32, 4, 0, 100].into())
717+
.build()
718+
.unwrap();
719+
let values = ArrayData::builder(DataType::Float32)
720+
.len(2)
721+
.add_buffer(vec![1.5, 2.5].into())
722+
.build()
723+
.unwrap();
724+
725+
let source_data = ArrayDataBuilder::new(dt)
726+
.len(3)
727+
.child_data(vec![runs, values])
728+
.build()
729+
.unwrap();
730+
731+
let mut mutable = MutableArrayData::new(vec![&source_data], true, 10);
732+
mutable.try_extend(0, 0, 1).unwrap();
733+
let data = mutable.freeze();
734+
assert_eq!(data, source_data.slice(0, 1));
735+
}
690736
}

0 commit comments

Comments
 (0)