Drop direct arrow-select dependency (#10826)

This is still transitively picked up through arrow_cast and parquet
variant but the logic is simpler without it

Author: robert3005

Cargo.lock

@@ -24,7 +24,6 @@ arrow-buffer = { workspace = true }
 arrow-cast = { workspace = true }
 arrow-data = { workspace = true }
 arrow-schema = { workspace = true, features = ["canonical_extension_types"] }
-arrow-select = { workspace = true }
 itertools = { workspace = true }
 num-traits = { workspace = true }
 simdutf8 = { workspace = true }

vortex-arrow/Cargo.toml

@@ -11,12 +11,10 @@ use arrow_schema::FieldRef;
 use vortex_array::ArrayRef;
 use vortex_array::Canonical;
 use vortex_array::ExecutionCtx;
-use vortex_array::arrays::Chunked;
 use vortex_array::arrays::List;
 use vortex_array::arrays::ListArray;
 use vortex_array::arrays::ListView;
 use vortex_array::arrays::ListViewArray;
-use vortex_array::arrays::chunked::ChunkedArrayExt;
 use vortex_array::arrays::list::ListArraySlotsExt;
 use vortex_array::arrays::listview::ListViewArraySlotsExt;
 use vortex_array::arrays::listview::ListViewDataParts;
@@ -41,7 +39,7 @@ impl Matcher for ArrowListExportable {
     type Match<'a> = &'a ArrayRef;
 
     fn try_match(array: &ArrayRef) -> Option<Self::Match<'_>> {
-        (array.is::<List>() || array.is::<Chunked>() || array.is::<ListView>()).then_some(array)
+        (array.is::<List>() || array.is::<ListView>()).then_some(array)
     }
 }
 
@@ -66,23 +64,9 @@ pub(super) fn to_arrow_list<O: OffsetSizeTrait + NativePType>(
         return list_to_list::<O>(&list.into_owned(), elements_field, exporter, ctx);
     }
 
-    // Converting each chunk individually, then using the fast concat logic from arrow
-    if let Some(chunked) = array.as_opt::<Chunked>() {
-        let mut arrow_chunks: Vec<ArrowArrayRef> = Vec::with_capacity(chunked.nchunks());
-        for chunk in chunked.chunks() {
-            arrow_chunks.push(to_arrow_list::<O>(
-                chunk.clone(),
-                elements_field,
-                exporter,
-                ctx,
-            )?);
-        }
-
-        let refs = arrow_chunks.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
-        return Ok(arrow_select::concat::concat(&refs)?);
-    }
-
     // Otherwise the array is canonical: a ListViewArray, which we rebuild to ZCTL if needed.
+    // Chunked lists canonicalize to a zero-copy ListView over chunked elements, so they take
+    // this path without an intermediate Arrow concatenation.
     // Note: arrow_cast::cast supports ListView → List (apache/arrow-rs#8735), but it
     // unconditionally uses take. Our rebuild uses a heuristic that picks list-by-list
     // for large lists, which avoids materializing a large index buffer.
@@ -226,12 +210,14 @@ mod tests {
     use arrow_array::GenericListArray;
     use arrow_array::Int32Array;
     use arrow_array::Int64Array;
+    use arrow_buffer::OffsetBuffer;
     use arrow_schema::DataType;
     use arrow_schema::Field;
     use vortex_array::Canonical;
     use vortex_array::IntoArray;
     use vortex_array::VortexSessionExecute;
     use vortex_array::arrays::BoolArray;
+    use vortex_array::arrays::ChunkedArray;
     use vortex_array::arrays::PrimitiveArray;
     use vortex_array::arrays::SliceArray;
     use vortex_array::dtype::DType;
@@ -249,6 +235,39 @@ mod tests {
     /// A shared session for these list-executor tests, used to create execution contexts.
     static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
 
+    #[test]
+    fn chunked_list_exports() -> VortexResult<()> {
+        let mut ctx = SESSION.create_execution_ctx();
+        let first = ListArray::try_new(
+            buffer![1i32, 2, 3].into_array(),
+            buffer![0i32, 2, 3].into_array(),
+            Validity::NonNullable,
+            &mut ctx,
+        )?;
+        let second = ListArray::try_new(
+            buffer![4i32, 5].into_array(),
+            buffer![0i32, 0, 2].into_array(),
+            Validity::NonNullable,
+            &mut ctx,
+        )?;
+        let dtype = first.dtype().clone();
+        let chunked = ChunkedArray::try_new(vec![first.into_array(), second.into_array()], dtype)?;
+
+        let field = Field::new("item", DataType::Int32, false);
+        let actual = chunked
+            .into_array()
+            .execute_arrow(Some(&DataType::List(field.clone().into())), &mut ctx)?;
+
+        let expected = GenericListArray::<i32>::try_new(
+            field.into(),
+            OffsetBuffer::new(vec![0, 2, 3, 3, 5].into()),
+            Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])),
+            None,
+        )?;
+        assert_eq!(&expected as &dyn Array, actual.as_ref());
+        Ok(())
+    }
+
     #[test]
     fn test_to_arrow_list_i32() -> VortexResult<()> {
         let mut ctx = SESSION.create_execution_ctx();

vortex-arrow/src/executor/list.rs

@@ -9,6 +9,7 @@ use arrow_array::Scalar as ArrowScalar;
 use arrow_array::*;
 use arrow_buffer::NullBuffer;
 use arrow_buffer::OffsetBuffer;
+use arrow_data::transform::MutableArrayData;
 use arrow_schema::Field;
 use arrow_schema::Fields;
 use vortex_array::dtype::DType;
@@ -235,11 +236,16 @@ fn concat_scalar_arrays(
         return Ok(new_empty_array(&to_data_type_naive(dtype)?));
     }
 
-    let arrays = scalars
+    // Every scalar is a single-element array of the same Arrow type, so append them in order.
+    let data = scalars
         .iter()
-        .map(|scalar| scalar.get().0)
+        .map(|scalar| scalar.get().0.to_data())
         .collect::<Vec<_>>();
-    Ok(arrow_select::concat::concat(&arrays)?)
+    let mut builder = MutableArrayData::new(data.iter().collect(), false, data.len());
+    for index in 0..data.len() {
+        builder.try_extend(index, 0, SCALAR_ARRAY_LEN)?;
+    }
+    Ok(make_array(builder.freeze()))
 }
 
 /// Convert an [`ExtScalar`] to an Arrow [`Datum`].