diff --git a/crates/kreuzberg/src/extraction/docx/parser.rs b/crates/kreuzberg/src/extraction/docx/parser.rs index e2fb8790a99..26847910855 100644 --- a/crates/kreuzberg/src/extraction/docx/parser.rs +++ b/crates/kreuzberg/src/extraction/docx/parser.rs @@ -1279,7 +1279,7 @@ impl DocxParser { b"w:fldChar" => { for attr in e.attributes().flatten() { if attr.key.as_ref() == b"w:fldCharType" { - match attr.value.as_ref() { + match attr.value.as_ref() as &[u8] { b"begin" => in_field_instruction = true, b"separate" | b"end" => in_field_instruction = false, _ => {} @@ -1413,7 +1413,7 @@ impl DocxParser { b"w:fldChar" => { for attr in e.attributes().flatten() { if attr.key.as_ref() == b"w:fldCharType" { - match attr.value.as_ref() { + match attr.value.as_ref() as &[u8] { b"begin" => in_field_instruction = true, b"separate" | b"end" => in_field_instruction = false, _ => {} diff --git a/crates/kreuzberg/src/extractors/fictionbook.rs b/crates/kreuzberg/src/extractors/fictionbook.rs index ddbb50f0177..382180312de 100644 --- a/crates/kreuzberg/src/extractors/fictionbook.rs +++ b/crates/kreuzberg/src/extractors/fictionbook.rs @@ -109,7 +109,7 @@ impl FictionBookExtractor { let decoded = String::from_utf8_lossy(t.as_ref()).to_string(); let had_trailing_space = decoded.ends_with(char::is_whitespace); let normalized = crate::utils::normalize_whitespace(&decoded); - let trimmed = normalized.as_ref(); + let trimmed: &str = normalized.as_ref(); if !trimmed.is_empty() { let starts_with_punct = trimmed.starts_with(['.', ',', ';', ':', '!', '?', ')', ']', '[']); if !text.is_empty() && !text.ends_with(' ') && !text.ends_with('\n') && !starts_with_punct { @@ -828,7 +828,7 @@ impl FictionBookExtractor { Ok(Event::Text(t)) => { let decoded = String::from_utf8_lossy(t.as_ref()).to_string(); let normalized = crate::utils::normalize_whitespace(&decoded); - let trimmed = normalized.as_ref(); + let trimmed: &str = normalized.as_ref(); if !trimmed.is_empty() { if !text.is_empty() && !text.ends_with(' ') { text.push(' '); diff --git a/crates/kreuzberg/src/extractors/pdf/mod.rs b/crates/kreuzberg/src/extractors/pdf/mod.rs index 41f9915fcd6..4d10890aeef 100644 --- a/crates/kreuzberg/src/extractors/pdf/mod.rs +++ b/crates/kreuzberg/src/extractors/pdf/mod.rs @@ -657,12 +657,53 @@ impl PdfExtractor { #[cfg(not(feature = "pdf"))] let fallback_count = 0u32; - let warning = if fallback_count > 0 { + // Drop images that are still raw-format with data too small to be + // valid pixels — these are still-compressed PDF streams that neither + // our decoder nor pdfium could decode. Returning them would give callers + // data they cannot use (see #615). + let dropped_count = pdf_images + .iter() + .filter(|img| { + if img.decoded_format != "raw" { + return false; + } + let channels: u64 = match img.color_space.as_deref() { + Some(cs) if cs.contains("CMYK") => 4, + Some(cs) if cs.contains("Gray") || cs.contains("grey") => 1, + _ => 3, + }; + let expected = (img.width as u64) * (img.height as u64) * channels; + expected > 0 && (img.data.len() as u64) < expected + }) + .count(); + if dropped_count > 0 { + pdf_images.retain(|img| { + if img.decoded_format != "raw" { + return true; + } + let channels: u64 = match img.color_space.as_deref() { + Some(cs) if cs.contains("CMYK") => 4, + Some(cs) if cs.contains("Gray") || cs.contains("grey") => 1, + _ => 3, + }; + let expected = (img.width as u64) * (img.height as u64) * channels; + expected == 0 || (img.data.len() as u64) >= expected + }); + } + + let warning = if fallback_count > 0 || dropped_count > 0 { + let mut parts = Vec::new(); + if fallback_count > 0 { + parts.push(format!("{fallback_count} image(s) re-extracted via pdfium bitmap fallback")); + } + if dropped_count > 0 { + parts.push(format!( + "{dropped_count} image(s) dropped: FlateDecode stream could not be decoded to usable pixels" + )); + } Some(crate::types::ProcessingWarning { source: std::borrow::Cow::Borrowed("image_extraction"), - message: std::borrow::Cow::Owned(format!( - "{fallback_count} image(s) re-extracted via pdfium bitmap fallback" - )), + message: std::borrow::Cow::Owned(parts.join("; ")), }) } else { None diff --git a/crates/kreuzberg/src/pdf/images.rs b/crates/kreuzberg/src/pdf/images.rs index 2cd1f00f9d7..b801f186056 100644 --- a/crates/kreuzberg/src/pdf/images.rs +++ b/crates/kreuzberg/src/pdf/images.rs @@ -133,14 +133,18 @@ fn decode_flate_to_png( palette: Option<&[u8]>, palette_base_channels: u32, ) -> std::result::Result, Box> { - use flate2::read::ZlibDecoder; + use flate2::read::{DeflateDecoder, ZlibDecoder}; use image::ImageEncoder; use std::io::Read; - // Decompress from zlib stream. - let mut decoder = ZlibDecoder::new(raw); + // Decompress from zlib stream. Many PDFs use raw deflate (no zlib header/footer), + // so fall back to DeflateDecoder if ZlibDecoder fails. let mut decompressed = Vec::new(); - decoder.read_to_end(&mut decompressed)?; + let zlib_result = ZlibDecoder::new(raw).read_to_end(&mut decompressed); + if zlib_result.is_err() { + decompressed.clear(); + DeflateDecoder::new(raw).read_to_end(&mut decompressed)?; + } let is_indexed = color_space.map(|cs| cs.contains("Indexed")).unwrap_or(false); @@ -626,6 +630,32 @@ mod tests { assert_eq!(detect_image_format(b"\x00\x01\x02\x03"), "raw"); } + #[cfg(feature = "pdf")] + #[test] + fn test_decode_flate_raw_deflate_fallback() { + // Regression test for #615: some PDFs use FlateDecode with raw deflate + // (no zlib header/footer). ZlibDecoder fails; DeflateDecoder must succeed. + use flate2::write::DeflateEncoder; + use flate2::Compression; + use std::io::Write; + + let raw_pixels: Vec = vec![ + 255, 0, 0, // red + 0, 255, 0, // green + 0, 0, 255, // blue + 255, 255, 0, // yellow + ]; + // No predictor bytes — simple 2x2 RGB + let mut enc = DeflateEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&raw_pixels).unwrap(); + let compressed = enc.finish().unwrap(); + + let filters = vec!["FlateDecode".to_string()]; + let (data, format) = decode_image_data(&compressed, &filters, Some("DeviceRGB"), 2, 2, Some(8), None, 0); + assert_eq!(format, "png", "raw-deflate FlateDecode should produce PNG, not raw"); + assert!(data.starts_with(b"\x89PNG\r\n\x1a\n"), "output should be a valid PNG"); + } + #[cfg(feature = "pdf")] #[test] fn test_decode_dct_passthrough() {