feat(ffi): plumb simplify, expressions, reverse_expr, and documentation through FFI_WindowUDF

.github/workflows/dependencies.yml

@@ -39,7 +39,7 @@ jobs:
     name: Circular Dependency Check
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
@@ -58,7 +58,7 @@ jobs:
     name: Detect Unused Dependencies
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Install cargo-machete

.github/workflows/extended.yml

@@ -108,7 +108,7 @@ jobs:
     name: cargo test hash collisions (amd64)
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -130,7 +130,7 @@ jobs:
     name: "Run sqllogictests with the sqlite test suite"
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

.github/workflows/rust.yml

@@ -48,7 +48,7 @@ jobs:
     name: linux build test
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -77,7 +77,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain
@@ -102,7 +102,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain
@@ -139,7 +139,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -171,7 +171,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -237,7 +237,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain
@@ -272,7 +272,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
       volumes:
         - /usr/local:/host/usr/local
     steps:
@@ -352,7 +352,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -383,7 +383,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -405,7 +405,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -446,7 +446,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -480,7 +480,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     services:
       postgres:
         image: postgres:15
@@ -519,7 +519,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -579,7 +579,7 @@ jobs:
     name: Verify Vendored Code
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain
@@ -596,7 +596,7 @@ jobs:
     name: Check cargo fmt
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain
@@ -654,7 +654,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -680,7 +680,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
@@ -701,7 +701,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -736,7 +736,7 @@ jobs:
     needs: linux-build-lib
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
 
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -767,7 +767,7 @@ jobs:
     name: Verify MSRV (Min Supported Rust Version)
     runs-on: ubuntu-latest
     container:
-      image: amd64/rust
+      image: rust
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Setup Rust toolchain

datafusion/ffi/src/udwf/expression_args.rs

@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::ffi::FFI_ArrowSchema;
+use arrow_schema::FieldRef;
+use datafusion_common::{DataFusionError, Result};
+use datafusion_expr::function::ExpressionArgs;
+use datafusion_physical_expr::PhysicalExpr;
+use stabby::vec::Vec as SVec;
+
+use crate::arrow_wrappers::WrappedSchema;
+use crate::physical_expr::FFI_PhysicalExpr;
+use crate::util::rvec_wrapped_to_vec_fieldref;
+
+/// A stable struct for sharing [`ExpressionArgs`] across FFI boundaries.
+#[repr(C)]
+#[derive(Debug)]
+pub struct FFI_ExpressionArgs {
+    input_exprs: SVec<FFI_PhysicalExpr>,
+    input_fields: SVec<WrappedSchema>,
+}
+
+impl TryFrom<ExpressionArgs<'_>> for FFI_ExpressionArgs {
+    type Error = DataFusionError;
+
+    fn try_from(args: ExpressionArgs) -> Result<Self> {
+        let input_exprs = args
+            .input_exprs()
+            .iter()
+            .map(Arc::clone)
+            .map(FFI_PhysicalExpr::from)
+            .collect();
+
+        let input_fields = args
+            .input_fields()
+            .iter()
+            .map(|field| FFI_ArrowSchema::try_from(field.as_ref()).map(WrappedSchema))
+            .collect::<Result<Vec<_>, _>>()?
+            .into_iter()
+            .collect();
+
+        Ok(Self {
+            input_exprs,
+            input_fields,
+        })
+    }
+}
+
+pub struct ForeignExpressionArgs {
+    input_exprs: Vec<Arc<dyn PhysicalExpr>>,
+    input_fields: Vec<FieldRef>,
+}
+
+impl TryFrom<FFI_ExpressionArgs> for ForeignExpressionArgs {
+    type Error = DataFusionError;
+
+    fn try_from(value: FFI_ExpressionArgs) -> Result<Self> {
+        let input_exprs = value.input_exprs.iter().map(Into::into).collect();
+
+        let input_fields = rvec_wrapped_to_vec_fieldref(&value.input_fields)?;
+
+        Ok(Self {
+            input_exprs,
+            input_fields,
+        })
+    }
+}
+
+impl<'a> From<&'a ForeignExpressionArgs> for ExpressionArgs<'a> {
+    fn from(value: &'a ForeignExpressionArgs) -> Self {
+        ExpressionArgs::new(&value.input_exprs, &value.input_fields)
+    }
+}