bench: add predicate_eval SQL micro-benchmark suite for conjunctive filter evaluation

benchmarks/bench.sh

@@ -102,6 +102,9 @@ topk_tpch:              Benchmark of top-k (sorting with limit) queries on TPC-H
 external_aggr:          External aggregation benchmark on TPC-H dataset (SF=1)
 wide_schema:            Small-projection queries on a wide synthetic dataset (1024 cols × 256 files) — measures per-file metadata overhead
                           (runs both 'wide' and 'narrow' subgroups: narrow is an internal baseline; the wide-vs-narrow ratio is the signal)
+predicate_eval:         Implementation-agnostic conjunctive (AND) filter-evaluation micro-benchmarks (synthetic data, generated inline)
+                          (subgroups via BENCH_SUBGROUP: costsel, cost, selectivity, cardinality, width, scale, neutral, correlation, drift, nulls)
+                          (toggle a system under test with its native DATAFUSION_* env var; size data with PRED_ROWS, string width with PRED_FILL)
 
 # ClickBench Benchmarks
 clickbench_1:           ClickBench queries against a single parquet file
@@ -245,6 +248,10 @@ main() {
                 wide_schema)
                     data_wide_schema
                     ;;
+                predicate_eval)
+                    # Data is generated inline by the suite's load SQL.
+                    echo "predicate_eval: no external data to generate"
+                    ;;
                 tpcds)
                     data_tpcds
                     ;;
@@ -458,6 +465,9 @@ main() {
                 wide_schema)
                     run_wide_schema
                     ;;
+                predicate_eval)
+                    run_predicate_eval
+                    ;;
                 tpcds)
                     run_tpcds
                     ;;
@@ -778,6 +788,30 @@ run_wide_schema() {
       bash -c "$SQL_CARGO_COMMAND"
 }
 
+# Runs the predicate_eval benchmark suite. Data is generated inline by the
+# suite's load SQL, so there is no data step. The suite is implementation-
+# agnostic and sets no engine config of its own; by default it measures
+# DataFusion's built-in left-deep AND short-circuit. To evaluate a
+# predicate-ordering system under test, export its native config as a
+# DATAFUSION_* env var before invoking bench.sh -- the bench harness reads
+# SessionConfig::from_env, and that environment is inherited here, e.g.
+#   DATAFUSION_EXECUTION_ADAPTIVE_FILTER_REORDERING=true ./bench.sh run predicate_eval
+# Suite-specific knobs (string-substituted into the load SQL, not engine config):
+#   BENCH_SUBGROUP   run one subgroup (costsel, cost, selectivity, cardinality,
+#                    width, scale, neutral, correlation, drift, nulls)
+#   PRED_ROWS        synthetic row count (default 1_000_000; the scale subgroup
+#                    overrides this per query)
+#   PRED_FILL        filler chars per marker = string-column width knob
+run_predicate_eval() {
+    echo "Running predicate_eval benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-1000000})..."
+    debug_run env BENCH_NAME=predicate_eval \
+      ${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \
+      PRED_ROWS="${PRED_ROWS:-1000000}" \
+      ${PRED_FILL:+PRED_FILL="${PRED_FILL}"} \
+      ${QUERY:+BENCH_QUERY="${QUERY}"}  \
+      bash -c "$SQL_CARGO_COMMAND"
+}
+
 # Runs the tpch in memory (needs tpch parquet data)
 run_tpch_mem() {
     SCALE_FACTOR=$1

benchmarks/sql_benchmarks/README.md

@@ -42,6 +42,7 @@ in the community:
 | `tpcds`               | TPC‑DS queries                                                     |
 | `tpch`                | TPC‑H queries                                                      |
 | `wide_schema`         | Small-projection queries on a wide (1024-col, 256-file) synthetic dataset; runs `wide` + `narrow` subgroups for comparison |
+| `predicate_eval`      | Implementation-agnostic conjunctive (AND) filter-evaluation micro-benchmarks; subgroups (`BENCH_SUBGROUP`): `costsel`, `cost`, `selectivity`, `cardinality`, `width`, `scale`, `neutral`, `correlation`, `drift`, `nulls`. Toggle a system under test with its native `DATAFUSION_*` env var |
 
 # Running Benchmarks
 
@@ -94,6 +95,8 @@ Some benchmarks use custom environment variables as outlined below:
 | BENCH_SORTED                 | Used in the sort_tpch benchmark to indicate whether the lineitem table should be sorted.                                 | false         |
 | SORTED_BY                    | Used in the clickbench_sorted benchmark to indicate the column to sort by.                                               | `EventTime`   |
 | SORTED_ORDER                 | Used in the clickbench_sorted benchmark to indicate the sort order of the column.                                        | `ASC`         |
+| PRED_ROWS                    | Used in the predicate_eval benchmark to size the synthetic table (the `scale` subgroup overrides this per query).        | `1000000`     |
+| PRED_FILL                    | Used in the predicate_eval benchmark as the string-column width knob (filler chars per marker).                          | `30`          |
 
 ## How it works
 

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cardinality/q30.benchmark

@@ -0,0 +1,7 @@
+subgroup cardinality
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cardinality
+QPAD=30
+DATASET=ints
+NAME=cardinality_q30_k2

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cardinality/q31.benchmark

@@ -0,0 +1,7 @@
+subgroup cardinality
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cardinality
+QPAD=31
+DATASET=ints
+NAME=cardinality_q31_k4

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cardinality/q32.benchmark

@@ -0,0 +1,7 @@
+subgroup cardinality
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cardinality
+QPAD=32
+DATASET=ints
+NAME=cardinality_q32_k8

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cardinality/q33.benchmark

@@ -0,0 +1,7 @@
+subgroup cardinality
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cardinality
+QPAD=33
+DATASET=ints
+NAME=cardinality_q33_k16

benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q70.benchmark

@@ -0,0 +1,7 @@
+subgroup correlation
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=correlation
+QPAD=70
+DATASET=corr
+NAME=correlation_q70_independent

benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q71.benchmark

@@ -0,0 +1,7 @@
+subgroup correlation
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=correlation
+QPAD=71
+DATASET=corr
+NAME=correlation_q71_positive

benchmarks/sql_benchmarks/predicate_eval/benchmarks/correlation/q72.benchmark

@@ -0,0 +1,7 @@
+subgroup correlation
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=correlation
+QPAD=72
+DATASET=corr
+NAME=correlation_q72_anti

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cost/q10.benchmark

@@ -0,0 +1,7 @@
+subgroup cost
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cost
+QPAD=10
+DATASET=mixed
+NAME=cost_q10_expensive_first

benchmarks/sql_benchmarks/predicate_eval/benchmarks/cost/q11.benchmark

@@ -0,0 +1,7 @@
+subgroup cost
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=cost
+QPAD=11
+DATASET=mixed
+NAME=cost_q11_cheap_first

benchmarks/sql_benchmarks/predicate_eval/benchmarks/costsel/q01.benchmark

@@ -0,0 +1,7 @@
+subgroup costsel
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=costsel
+QPAD=01
+DATASET=markers
+NAME=costsel_q01_regexp_selective_last

benchmarks/sql_benchmarks/predicate_eval/benchmarks/costsel/q02.benchmark

@@ -0,0 +1,7 @@
+subgroup costsel
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=costsel
+QPAD=02
+DATASET=markers
+NAME=costsel_q02_regexp_selective_first

benchmarks/sql_benchmarks/predicate_eval/benchmarks/costsel/q03.benchmark

@@ -0,0 +1,7 @@
+subgroup costsel
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=costsel
+QPAD=03
+DATASET=mixed
+NAME=costsel_q03_cheap_unselective_then_expensive_selective

benchmarks/sql_benchmarks/predicate_eval/benchmarks/drift/q80.benchmark

@@ -0,0 +1,7 @@
+subgroup drift
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=drift
+QPAD=80
+DATASET=drift
+NAME=drift_q80_a_then_b

benchmarks/sql_benchmarks/predicate_eval/benchmarks/drift/q81.benchmark

@@ -0,0 +1,7 @@
+subgroup drift
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=drift
+QPAD=81
+DATASET=drift
+NAME=drift_q81_b_then_a

benchmarks/sql_benchmarks/predicate_eval/benchmarks/neutral/q60.benchmark

@@ -0,0 +1,7 @@
+subgroup neutral
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=neutral
+QPAD=60
+DATASET=ints
+NAME=neutral_q60_cheap_uniform

benchmarks/sql_benchmarks/predicate_eval/benchmarks/neutral/q61.benchmark

@@ -0,0 +1,7 @@
+subgroup neutral
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=neutral
+QPAD=61
+DATASET=markers
+NAME=neutral_q61_expensive_uniform

benchmarks/sql_benchmarks/predicate_eval/benchmarks/nulls/q90.benchmark

@@ -0,0 +1,7 @@
+subgroup nulls
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=nulls
+QPAD=90
+DATASET=ints
+NAME=nulls_q90_no_nulls_control

benchmarks/sql_benchmarks/predicate_eval/benchmarks/nulls/q91.benchmark

@@ -0,0 +1,7 @@
+subgroup nulls
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=nulls
+QPAD=91
+DATASET=nulls
+NAME=nulls_q91_half_null

benchmarks/sql_benchmarks/predicate_eval/benchmarks/scale/q50.benchmark

@@ -0,0 +1,8 @@
+subgroup scale
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=scale
+QPAD=50
+DATASET=mixed
+PRED_ROWS=5000
+NAME=scale_q50_5k

benchmarks/sql_benchmarks/predicate_eval/benchmarks/scale/q51.benchmark

@@ -0,0 +1,8 @@
+subgroup scale
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=scale
+QPAD=51
+DATASET=mixed
+PRED_ROWS=100000
+NAME=scale_q51_100k

benchmarks/sql_benchmarks/predicate_eval/benchmarks/scale/q52.benchmark

@@ -0,0 +1,8 @@
+subgroup scale
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=scale
+QPAD=52
+DATASET=mixed
+PRED_ROWS=5000000
+NAME=scale_q52_5m

benchmarks/sql_benchmarks/predicate_eval/benchmarks/scale/q53.benchmark

@@ -0,0 +1,8 @@
+subgroup scale
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=scale
+QPAD=53
+DATASET=mixed
+PRED_ROWS=50000000
+NAME=scale_q53_50m

benchmarks/sql_benchmarks/predicate_eval/benchmarks/selectivity/q20.benchmark

@@ -0,0 +1,7 @@
+subgroup selectivity
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=selectivity
+QPAD=20
+DATASET=ints
+NAME=selectivity_q20_unselective_first

benchmarks/sql_benchmarks/predicate_eval/benchmarks/selectivity/q21.benchmark

@@ -0,0 +1,7 @@
+subgroup selectivity
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=selectivity
+QPAD=21
+DATASET=ints
+NAME=selectivity_q21_selective_first

benchmarks/sql_benchmarks/predicate_eval/benchmarks/width/q40.benchmark

@@ -0,0 +1,8 @@
+subgroup width
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=width
+QPAD=40
+DATASET=markers
+PRED_FILL=2
+NAME=width_q40_narrow

benchmarks/sql_benchmarks/predicate_eval/benchmarks/width/q41.benchmark

@@ -0,0 +1,8 @@
+subgroup width
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=width
+QPAD=41
+DATASET=markers
+PRED_FILL=30
+NAME=width_q41_wide

benchmarks/sql_benchmarks/predicate_eval/benchmarks/width/q42.benchmark

@@ -0,0 +1,8 @@
+subgroup width
+
+template sql_benchmarks/predicate_eval/predicate_eval.benchmark.template
+SUBGROUP=width
+QPAD=42
+DATASET=markers
+PRED_FILL=170
+NAME=width_q42_xwide

benchmarks/sql_benchmarks/predicate_eval/init/cleanup.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS t;

benchmarks/sql_benchmarks/predicate_eval/load/corr.sql

@@ -0,0 +1,19 @@
+-- Correlation dataset: a base column plus derived columns that control the
+-- *conditional* selectivity of one predicate given another (its selectivity
+-- among the rows that already passed the other).
+--
+--   x       uniform [0,100)
+--   x_pos   = x         (perfectly positively correlated: `x<k AND x_pos<k`
+--                        passes ~k%, not the ~k%^2 an independence assumption
+--                        predicts)
+--   x_anti  = 99 - x    (anti-correlated: `x<k AND x_anti<k` is empty for k<=50)
+--   ind     independent control column, uniform [0,100)
+--
+-- PRED_ROWS sizes the table.
+CREATE TABLE t AS
+SELECT
+  (value * 7)  % 100        AS x,
+  (value * 7)  % 100        AS x_pos,
+  99 - ((value * 7) % 100)  AS x_anti,
+  (value * 13) % 100        AS ind
+FROM generate_series(1, ${PRED_ROWS:-1000000});

benchmarks/sql_benchmarks/predicate_eval/load/drift.sql

@@ -0,0 +1,16 @@
+-- Drift dataset: two predicates whose *relative* selectivity flips partway
+-- through the scan, so whole-table selectivity differs from per-batch
+-- selectivity. Rows are emitted in `seq` order, so batches observe the drift in
+-- order.
+--
+--   a_sel = 0  is selective (~0.1%) in the first 10% of rows, unselective
+--              (~50%) afterwards.
+--   b_sel = 0  is the mirror: unselective early, selective late.
+--
+-- PRED_ROWS sizes the table.
+CREATE TABLE t AS
+SELECT
+  value AS seq,
+  CASE WHEN value < ${PRED_ROWS:-1000000} / 10 THEN value % 1000 ELSE value % 2    END AS a_sel,
+  CASE WHEN value < ${PRED_ROWS:-1000000} / 10 THEN value % 2    ELSE value % 1000 END AS b_sel
+FROM generate_series(1, ${PRED_ROWS:-1000000});

benchmarks/sql_benchmarks/predicate_eval/load/ints.sql

@@ -0,0 +1,24 @@
+-- Sixteen independent integer columns, each uniform on [0,100). The predicate
+-- `cN < k` therefore has selectivity ~k%. All columns are equally cheap to
+-- evaluate, so only selectivity (not cost) distinguishes orderings here. The
+-- multipliers are all coprime to 100, which keeps the residues uniform and the
+-- columns mutually decorrelated. PRED_ROWS sizes the table.
+CREATE TABLE t AS
+SELECT
+  (value * 1)  % 100 AS c0,
+  (value * 3)  % 100 AS c1,
+  (value * 7)  % 100 AS c2,
+  (value * 9)  % 100 AS c3,
+  (value * 11) % 100 AS c4,
+  (value * 13) % 100 AS c5,
+  (value * 17) % 100 AS c6,
+  (value * 19) % 100 AS c7,
+  (value * 21) % 100 AS c8,
+  (value * 23) % 100 AS c9,
+  (value * 27) % 100 AS c10,
+  (value * 29) % 100 AS c11,
+  (value * 31) % 100 AS c12,
+  (value * 33) % 100 AS c13,
+  (value * 37) % 100 AS c14,
+  (value * 39) % 100 AS c15
+FROM generate_series(1, ${PRED_ROWS:-1000000});

benchmarks/sql_benchmarks/predicate_eval/load/markers.sql

@@ -0,0 +1,26 @@
+-- Wide-string dataset: five markers embedded in `PRED_FILL`-wide filler so that a
+-- non-matching `regexp_like` must scan the whole value (every string predicate
+-- is "expensive"). Selectivities are coprime so the predicates are independent:
+--
+--   'aaa'  present in ~90%  of rows  (value % 10  <> 0)
+--   'bbb'  present in ~86%  of rows  (value % 7   <> 0)
+--   'ccc'  present in ~80%  of rows  (value % 5   <> 0)
+--   'ddd'  present in ~75%  of rows  (value % 4   <> 0)
+--   'rare' present in ~0.1% of rows  (value % 1009 = 5)   <- the selective one
+--
+-- PRED_FILL sets the filler width per marker (the string-column width knob: ~6*PRED_FILL
+-- chars per row), and PRED_ROWS sizes the table.
+CREATE TABLE t AS
+SELECT
+  repeat('q', ${PRED_FILL:-30})
+    || CASE WHEN value % 10   <> 0 THEN 'aaa'  ELSE 'zzz'  END
+    || repeat('q', ${PRED_FILL:-30})
+    || CASE WHEN value % 7    <> 0 THEN 'bbb'  ELSE 'zzz'  END
+    || repeat('q', ${PRED_FILL:-30})
+    || CASE WHEN value % 5    <> 0 THEN 'ccc'  ELSE 'zzz'  END
+    || repeat('q', ${PRED_FILL:-30})
+    || CASE WHEN value % 4    <> 0 THEN 'ddd'  ELSE 'zzz'  END
+    || repeat('q', ${PRED_FILL:-30})
+    || CASE WHEN value % 1009 = 5  THEN 'rare' ELSE 'zzzz' END
+    || repeat('q', ${PRED_FILL:-30}) AS s
+FROM generate_series(1, ${PRED_ROWS:-1000000});