Skip to content

feat(parquet): Consolidate bloom filter reads with a push decoder #11975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
etseidl merged 10 commits into from
Nov 3, 2026
+885 −429
Merged

feat(parquet): Consolidate bloom filter reads with a push decoder #11975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bf5226e
refactor(parquet): consolidate Bloom filter reading and validation
etseidl Oct 28, 2026
8a73701
add SbbfPushDecoder
etseidl Nov 2, 2026
4dbecb9
clean up push API and add tests/docs
etseidl Nov 2, 2026
d9ea8af
add ascii art
etseidl Nov 3, 2026
a198a1f
Update ASCII art
etseidl Nov 3, 2026
4d4935e
Use expect_* helpers in Bloom filter push decoder tests
alamb Nov 3, 2026
55ad027
Merge remote-tracking branch 'apache/main' into consolidate_bloom_fil…
alamb Nov 3, 2026
8d0564e
consolidate tests
etseidl Nov 3, 2026
cb00818
pull push decoder into its own module
etseidl Nov 3, 2026
214bb1a
make example a doctest
etseidl Nov 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 2 additions & 54 deletions parquet/src/arrow/arrow_reader/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,7 @@ use crate::arrow::schema::{
ParquetField, parquet_to_arrow_schema_and_fields, virtual_type::is_virtual_column,
};
use crate::arrow::{FieldLevels, ProjectionMask, parquet_to_arrow_field_levels_with_virtual};
use crate::basic::{BloomFilterAlgorithm, BloomFilterCompression, BloomFilterHash};
use crate::bloom_filter::{
SBBF_HEADER_SIZE_ESTIMATE, Sbbf, chunk_read_bloom_filter_header_and_offset,
};
use crate::bloom_filter::Sbbf;
use crate::column::page::{PageIterator, PageReader};
#[cfg(feature = "encryption")]
use crate::encryption::decrypt::FileDecryptionProperties;
Expand Down Expand Up @@ -1353,56 +1350,7 @@ impl<T: ChunkReader + 'static> ParquetRecordBatchReaderBuilder<T> {
let metadata = self.metadata.row_group(row_group_idx);
let column_metadata = metadata.column(column_idx);

let offset: u64 = if let Some(offset) = column_metadata.bloom_filter_offset() {
offset
.try_into()
.map_err(|_| ParquetError::General("Bloom filter offset is invalid".to_string()))?
} else {
return Ok(None);
};

let buffer = match column_metadata.bloom_filter_length() {
Some(length) => self.input.0.get_bytes(offset, length as usize),
None => self.input.0.get_bytes(offset, SBBF_HEADER_SIZE_ESTIMATE),
}?;

let (header, bitset_offset) =
chunk_read_bloom_filter_header_and_offset(offset, buffer.clone())?;

match header.algorithm {
BloomFilterAlgorithm::BLOCK => {
// this match exists to future proof the singleton algorithm enum
}
}
match header.compression {
BloomFilterCompression::UNCOMPRESSED => {
// this match exists to future proof the singleton compression enum
}
}
match header.hash {
BloomFilterHash::XXHASH => {
// this match exists to future proof the singleton hash enum
}
}

let bitset = match column_metadata.bloom_filter_length() {
Some(_) => {
let bitset_start = bitset_offset
.checked_sub(offset)
.and_then(|start| usize::try_from(start).ok())
.ok_or_else(|| {
ParquetError::General("Bloom filter offset is invalid".to_string())
})?;
buffer.slice(bitset_start..)
}
None => {
let bitset_length: usize = header.num_bytes.try_into().map_err(|_| {
ParquetError::General("Bloom filter length is invalid".to_string())
})?;
self.input.0.get_bytes(bitset_offset, bitset_length)?
}
};
Ok(Some(Sbbf::new(&bitset)))
Sbbf::read_from_column_chunk(column_metadata, &self.input.0)
}

/// Build a [`ParquetRecordBatchReader`]
Expand Down
Loading